From bbd3331d59db637589c1bf2d09aacbc09ed677a9 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 24 Oct 2023 17:23:32 +0300 Subject: [PATCH 01/38] Move `BufferedStorage` from consensus module --- .../src/sync_layer/consensus/buffered.rs | 276 ++++++++++++++++ .../src/sync_layer/consensus/mod.rs | 6 + .../src/sync_layer/consensus/tests.rs | 295 ++++++++++++++++++ .../src/sync_layer/consensus/utils.rs | 48 +++ core/lib/zksync_core/src/sync_layer/mod.rs | 1 + 5 files changed, 626 insertions(+) create mode 100644 core/lib/zksync_core/src/sync_layer/consensus/buffered.rs create mode 100644 core/lib/zksync_core/src/sync_layer/consensus/mod.rs create mode 100644 core/lib/zksync_core/src/sync_layer/consensus/tests.rs create mode 100644 core/lib/zksync_core/src/sync_layer/consensus/utils.rs diff --git a/core/lib/zksync_core/src/sync_layer/consensus/buffered.rs b/core/lib/zksync_core/src/sync_layer/consensus/buffered.rs new file mode 100644 index 000000000000..35f3268f0bf7 --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/consensus/buffered.rs @@ -0,0 +1,276 @@ +//! Buffered [`BlockStore`] implementation. + +use async_trait::async_trait; + +use std::{collections::BTreeMap, ops}; + +#[cfg(test)] +use zksync_concurrency::ctx::channel; +use zksync_concurrency::{ + ctx, + sync::{self, watch, Mutex}, +}; +use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; +use zksync_consensus_storage::{BlockStore, StorageError, StorageResult, WriteBlockStore}; + +use super::utils::MissingBlockNumbers; + +/// [`BlockStore`] variation that upholds additional invariants as to how blocks are processed. +/// +/// The invariants are as follows: +/// +/// - Stored blocks always have contiguous numbers; there are no gaps. +/// - Blocks can be scheduled to be added using [`Self::schedule_next_block()`] only. New blocks do not +/// appear in the store otherwise. +#[async_trait] +pub(super) trait ContiguousBlockStore: BlockStore { + /// Schedules a block to be added to the store. Unlike [`WriteBlockStore::put_block()`], + /// there is no expectation that the block is added to the store *immediately*. It's + /// expected that it will be added to the store eventually, which will be signaled via + /// a subscriber returned from [`BlockStore::subscribe_to_block_writes()`]. + /// + /// [`BufferedStorage`] guarantees that this method will only ever be called: + /// + /// - with the next block (i.e., one immediately after [`BlockStore::head_block()`]) + /// - sequentially (i.e., multiple blocks cannot be scheduled at once) + async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()>; +} + +#[derive(Debug)] +struct BlockBuffer { + store_block_number: BlockNumber, + is_block_scheduled: bool, + blocks: BTreeMap, +} + +impl BlockBuffer { + fn new(store_block_number: BlockNumber) -> Self { + Self { + store_block_number, + is_block_scheduled: false, + blocks: BTreeMap::new(), + } + } + + fn head_block(&self) -> Option { + self.blocks.values().next_back().cloned() + } + + #[tracing::instrument(level = "trace", skip(self))] + fn set_store_block(&mut self, store_block_number: BlockNumber) { + assert_eq!( + store_block_number, + self.store_block_number.next(), + "`ContiguousBlockStore` invariant broken: unexpected new head block number" + ); + assert!( + self.is_block_scheduled, + "`ContiguousBlockStore` invariant broken: unexpected update" + ); + + self.store_block_number = store_block_number; + self.is_block_scheduled = false; + let old_len = self.blocks.len(); + self.blocks = self.blocks.split_off(&store_block_number.next()); + // ^ Removes all entries up to and including `store_block_number` + tracing::trace!("Removed {} blocks from buffer", old_len - self.blocks.len()); + } + + fn last_contiguous_block_number(&self) -> BlockNumber { + // By design, blocks in the underlying store are always contiguous. + let mut last_number = self.store_block_number; + for &number in self.blocks.keys() { + if number > last_number.next() { + return last_number; + } + last_number = number; + } + last_number + } + + fn missing_block_numbers(&self, mut range: ops::Range) -> Vec { + // Clamp the range start so we don't produce extra missing blocks. + range.start = range.start.max(self.store_block_number.next()); + if range.is_empty() { + return vec![]; // Return early to not trigger panic in `BTreeMap::range()` + } + + let keys = self.blocks.range(range.clone()).map(|(&num, _)| num); + MissingBlockNumbers::new(range, keys).collect() + } + + fn put_block(&mut self, block: FinalBlock) { + let block_number = block.block.number; + assert!(block_number > self.store_block_number); + // ^ Must be checked previously + self.blocks.insert(block_number, block); + tracing::trace!(%block_number, "Inserted block in buffer"); + } + + fn next_block_for_store(&mut self) -> Option { + if self.is_block_scheduled { + None + } else { + let next_block = self.blocks.get(&self.store_block_number.next()).cloned(); + self.is_block_scheduled = next_block.is_some(); + next_block + } + } +} + +/// Events emitted by [`BufferedStorage`]. +#[cfg(test)] +#[derive(Debug)] +pub(super) enum BufferedStorageEvent { + /// Update was received from + UpdateReceived(BlockNumber), +} + +/// [`BlockStore`] with an in-memory buffer for pending blocks. +#[derive(Debug)] +pub(super) struct BufferedStorage { + inner: T, + inner_subscriber: watch::Receiver, + block_writes_sender: watch::Sender, + buffer: Mutex, + #[cfg(test)] + events_sender: channel::UnboundedSender, +} + +impl BufferedStorage { + /// Creates a new buffered storage. The buffer is initially empty. + pub fn new(store: T) -> Self { + let inner_subscriber = store.subscribe_to_block_writes(); + let store_block_number = *inner_subscriber.borrow(); + tracing::trace!(%store_block_number, "Initialized buffer storage"); + Self { + inner: store, + inner_subscriber, + block_writes_sender: watch::channel(store_block_number).0, + buffer: Mutex::new(BlockBuffer::new(store_block_number)), + #[cfg(test)] + events_sender: channel::unbounded().0, + } + } + + #[cfg(test)] + pub(crate) fn set_events_sender( + &mut self, + sender: channel::UnboundedSender, + ) { + self.events_sender = sender; + } + + #[cfg(test)] + pub(crate) fn as_ref(&self) -> &T { + &self.inner + } + + #[cfg(test)] + pub(crate) async fn buffer_len(&self) -> usize { + self.buffer.lock().await.blocks.len() + } + + /// Listens to the updates in the underlying storage. This method must be spawned as a background task + /// which should be running as long at the [`BufferedStorage`] is in use. Otherwise, + /// `BufferedStorage` will function incorrectly. + #[tracing::instrument(level = "trace", skip_all, err)] + pub async fn listen_to_updates(&self, ctx: &ctx::Ctx) -> StorageResult<()> { + let mut subscriber = self.inner_subscriber.clone(); + loop { + let store_block_number = *sync::changed(ctx, &mut subscriber).await?; + tracing::trace!("Underlying block number updated to {store_block_number}"); + + let next_block_for_store = { + let mut buffer = sync::lock(ctx, &self.buffer).await?; + buffer.set_store_block(store_block_number); + buffer.next_block_for_store() + }; + if let Some(block) = next_block_for_store { + self.inner.schedule_next_block(ctx, &block).await?; + let block_number = block.block.number; + tracing::trace!(%block_number, "Block scheduled in underlying storage"); + } + + #[cfg(test)] + self.events_sender + .send(BufferedStorageEvent::UpdateReceived(store_block_number)); + } + } +} + +#[async_trait] +impl BlockStore for BufferedStorage { + async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { + let buffered_head_block = sync::lock(ctx, &self.buffer).await?.head_block(); + if let Some(block) = buffered_head_block { + return Ok(block); + } + self.inner.head_block(ctx).await + } + + async fn first_block(&self, ctx: &ctx::Ctx) -> StorageResult { + // First block is always situated in the underlying store + self.inner.first_block(ctx).await + } + + async fn last_contiguous_block_number(&self, ctx: &ctx::Ctx) -> StorageResult { + Ok(sync::lock(ctx, &self.buffer) + .await? + .last_contiguous_block_number()) + } + + async fn block( + &self, + ctx: &ctx::Ctx, + number: BlockNumber, + ) -> StorageResult> { + { + let buffer = sync::lock(ctx, &self.buffer).await?; + if number > buffer.store_block_number { + return Ok(buffer.blocks.get(&number).cloned()); + } + } + self.inner.block(ctx, number).await + } + + async fn missing_block_numbers( + &self, + ctx: &ctx::Ctx, + range: ops::Range, + ) -> StorageResult> { + // By design, the underlying store has no missing blocks. + Ok(sync::lock(ctx, &self.buffer) + .await? + .missing_block_numbers(range)) + } + + fn subscribe_to_block_writes(&self) -> watch::Receiver { + self.block_writes_sender.subscribe() + } +} + +#[async_trait] +impl WriteBlockStore for BufferedStorage { + async fn put_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + let next_block_for_store = { + let mut buffer = sync::lock(ctx, &self.buffer).await?; + let block_number = block.block.number; + if block_number <= buffer.store_block_number { + let err = anyhow::anyhow!( + "Cannot replace a block #{block_number} since it is already present in the underlying storage", + ); + return Err(StorageError::Database(err)); + } + buffer.put_block(block.clone()); + buffer.next_block_for_store() + }; + + if let Some(block) = next_block_for_store { + self.inner.schedule_next_block(ctx, &block).await?; + tracing::trace!(block_number = %block.block.number, "Block scheduled in underlying storage"); + } + self.block_writes_sender.send_replace(block.block.number); + Ok(()) + } +} diff --git a/core/lib/zksync_core/src/sync_layer/consensus/mod.rs b/core/lib/zksync_core/src/sync_layer/consensus/mod.rs new file mode 100644 index 000000000000..7c34d74da1a8 --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/consensus/mod.rs @@ -0,0 +1,6 @@ +//! Consensus adapter for EN synchronization logic. + +mod buffered; +#[cfg(test)] +mod tests; +mod utils; diff --git a/core/lib/zksync_core/src/sync_layer/consensus/tests.rs b/core/lib/zksync_core/src/sync_layer/consensus/tests.rs new file mode 100644 index 000000000000..efd444e175f6 --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/consensus/tests.rs @@ -0,0 +1,295 @@ +//! Tests for consensus adapters for EN synchronization logic. + +use assert_matches::assert_matches; +use async_trait::async_trait; +use rand::{rngs::StdRng, seq::SliceRandom, Rng}; +use tempfile::TempDir; +use test_casing::test_casing; + +use std::{iter, ops}; + +use zksync_concurrency::{ + ctx::{self, channel}, + scope, + sync::{self, watch}, + time, +}; +use zksync_consensus_roles::validator::{Block, BlockNumber, FinalBlock}; +use zksync_consensus_storage::{ + BlockStore, RocksdbStorage, StorageError, StorageResult, WriteBlockStore, +}; + +use super::buffered::{BufferedStorage, BufferedStorageEvent, ContiguousBlockStore}; + +async fn init_store(ctx: &ctx::Ctx, rng: &mut R) -> (FinalBlock, RocksdbStorage, TempDir) { + let genesis_block = FinalBlock { + block: Block::genesis(vec![]), + justification: rng.gen(), + }; + let temp_dir = TempDir::new().unwrap(); + let block_store = RocksdbStorage::new(ctx, &genesis_block, temp_dir.path()) + .await + .unwrap(); + (genesis_block, block_store, temp_dir) +} + +fn gen_blocks(rng: &mut impl Rng, genesis_block: FinalBlock, count: usize) -> Vec { + let blocks = iter::successors(Some(genesis_block), |parent| { + let block = Block { + parent: parent.block.hash(), + number: parent.block.number.next(), + payload: Vec::new(), + }; + Some(FinalBlock { + block, + justification: rng.gen(), + }) + }); + blocks.skip(1).take(count).collect() +} + +#[derive(Debug)] +struct MockContiguousStore { + inner: RocksdbStorage, + block_sender: channel::Sender, +} + +impl MockContiguousStore { + fn new(inner: RocksdbStorage) -> (Self, channel::Receiver) { + let (block_sender, block_receiver) = channel::bounded(1); + let this = Self { + inner, + block_sender, + }; + (this, block_receiver) + } + + async fn run_updates( + &self, + ctx: &ctx::Ctx, + mut block_receiver: channel::Receiver, + ) -> StorageResult<()> { + let rng = &mut ctx.rng(); + while let Ok(block) = block_receiver.recv(ctx).await { + let sleep_duration = time::Duration::milliseconds(rng.gen_range(0..5)); + ctx.sleep(sleep_duration).await?; + self.inner.put_block(ctx, &block).await?; + } + Ok(()) + } +} + +#[async_trait] +impl BlockStore for MockContiguousStore { + async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { + self.inner.head_block(ctx).await + } + + async fn first_block(&self, ctx: &ctx::Ctx) -> StorageResult { + self.inner.first_block(ctx).await + } + + async fn last_contiguous_block_number(&self, ctx: &ctx::Ctx) -> StorageResult { + self.inner.last_contiguous_block_number(ctx).await + } + + async fn block( + &self, + ctx: &ctx::Ctx, + number: BlockNumber, + ) -> StorageResult> { + self.inner.block(ctx, number).await + } + + async fn missing_block_numbers( + &self, + ctx: &ctx::Ctx, + range: ops::Range, + ) -> StorageResult> { + self.inner.missing_block_numbers(ctx, range).await + } + + fn subscribe_to_block_writes(&self) -> watch::Receiver { + self.inner.subscribe_to_block_writes() + } +} + +#[async_trait] +impl ContiguousBlockStore for MockContiguousStore { + async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + let head_block_number = self.head_block(ctx).await?.block.number; + assert_eq!(block.block.number, head_block_number.next()); + self.block_sender + .try_send(block.clone()) + .expect("BufferedStorage is rushing"); + Ok(()) + } +} + +#[tracing::instrument(level = "trace", skip(shuffle_blocks))] +async fn test_buffered_storage( + initial_block_count: usize, + block_count: usize, + block_interval: time::Duration, + shuffle_blocks: impl FnOnce(&mut StdRng, &mut [FinalBlock]), +) { + let ctx = &ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + let (genesis_block, block_store, _temp_dir) = init_store(ctx, rng).await; + let mut initial_blocks = gen_blocks(rng, genesis_block.clone(), initial_block_count); + for block in &initial_blocks { + block_store.put_block(ctx, block).await.unwrap(); + } + initial_blocks.insert(0, genesis_block.clone()); + + let (block_store, block_receiver) = MockContiguousStore::new(block_store); + let mut buffered_store = BufferedStorage::new(block_store); + let (events_sender, mut events_receiver) = channel::unbounded(); + buffered_store.set_events_sender(events_sender); + + // Check initial values returned by the store. + let last_initial_block = initial_blocks.last().unwrap().clone(); + assert_eq!( + buffered_store.head_block(ctx).await.unwrap(), + last_initial_block + ); + for block in &initial_blocks { + let block_result = buffered_store.block(ctx, block.block.number).await; + assert_eq!(block_result.unwrap().as_ref(), Some(block)); + } + let mut subscriber = buffered_store.subscribe_to_block_writes(); + assert_eq!( + *subscriber.borrow(), + BlockNumber(initial_block_count as u64) + ); + + let mut blocks = gen_blocks(rng, last_initial_block, block_count); + shuffle_blocks(rng, &mut blocks); + let last_block_number = BlockNumber((block_count + initial_block_count) as u64); + + scope::run!(ctx, |ctx, s| async { + s.spawn_bg(buffered_store.as_ref().run_updates(ctx, block_receiver)); + s.spawn_bg(async { + let err = buffered_store.listen_to_updates(ctx).await.unwrap_err(); + match &err { + StorageError::Canceled(_) => Ok(()), // Test has successfully finished + StorageError::Database(_) => Err(err), + } + }); + + for (idx, block) in blocks.iter().enumerate() { + buffered_store.put_block(ctx, block).await?; + let new_block_number = *sync::changed(ctx, &mut subscriber).await?; + assert_eq!(new_block_number, block.block.number); + + // Check that all written blocks are immediately accessible. + for existing_block in initial_blocks.iter().chain(&blocks[0..=idx]) { + let number = existing_block.block.number; + assert_eq!( + buffered_store.block(ctx, number).await?.as_ref(), + Some(existing_block) + ); + } + assert_eq!(buffered_store.first_block(ctx).await?, genesis_block); + + let expected_head_block = blocks[0..=idx] + .iter() + .max_by_key(|block| block.block.number) + .unwrap(); + assert_eq!(buffered_store.head_block(ctx).await?, *expected_head_block); + + let expected_last_contiguous_block = blocks[(idx + 1)..] + .iter() + .map(|block| block.block.number) + .min() + .map_or(last_block_number, BlockNumber::prev); + assert_eq!( + buffered_store.last_contiguous_block_number(ctx).await?, + expected_last_contiguous_block + ); + + ctx.sleep(block_interval).await?; + } + + let mut inner_subscriber = buffered_store.as_ref().subscribe_to_block_writes(); + while buffered_store + .as_ref() + .last_contiguous_block_number(ctx) + .await? + < last_block_number + { + sync::changed(ctx, &mut inner_subscriber).await?; + } + + // Check events emitted by the buffered storage. This also ensures that all underlying storage + // updates are processed before proceeding to the following checks. + let expected_numbers = (initial_block_count as u64 + 1)..=last_block_number.0; + for expected_number in expected_numbers.map(BlockNumber) { + assert_matches!( + events_receiver.recv(ctx).await?, + BufferedStorageEvent::UpdateReceived(number) if number == expected_number + ); + } + + assert_eq!(buffered_store.buffer_len().await, 0); + Ok(()) + }) + .await + .unwrap(); +} + +// Choose intervals so that they are both smaller and larger than the sleep duration in +// `MockContiguousStore::run_updates()`. +const BLOCK_INTERVALS: [time::Duration; 4] = [ + time::Duration::ZERO, + time::Duration::milliseconds(3), + time::Duration::milliseconds(5), + time::Duration::milliseconds(10), +]; + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_sequential_blocks(block_interval: time::Duration) { + test_buffered_storage(0, 30, block_interval, |_, _| { + // Do not perform shuffling + }) + .await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_random_blocks(block_interval: time::Duration) { + test_buffered_storage(0, 30, block_interval, |rng, blocks| blocks.shuffle(rng)).await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_slightly_shuffled_blocks(block_interval: time::Duration) { + test_buffered_storage(0, 30, block_interval, |rng, blocks| { + for chunk in blocks.chunks_mut(4) { + chunk.shuffle(rng); + } + }) + .await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_initial_blocks(block_interval: time::Duration) { + test_buffered_storage(10, 20, block_interval, |_, _| { + // Do not perform shuffling + }) + .await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_initial_blocks_and_slight_shuffling(block_interval: time::Duration) { + test_buffered_storage(10, 20, block_interval, |rng, blocks| { + for chunk in blocks.chunks_mut(5) { + chunk.shuffle(rng); + } + }) + .await; +} diff --git a/core/lib/zksync_core/src/sync_layer/consensus/utils.rs b/core/lib/zksync_core/src/sync_layer/consensus/utils.rs new file mode 100644 index 000000000000..8407821a2ec6 --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/consensus/utils.rs @@ -0,0 +1,48 @@ +use std::{iter, ops}; + +use zksync_consensus_roles::validator::BlockNumber; + +/// Iterator over missing block numbers. +pub(crate) struct MissingBlockNumbers { + range: ops::Range, + existing_numbers: iter::Peekable, +} + +impl MissingBlockNumbers +where + I: Iterator, +{ + /// Creates a new iterator based on the provided params. + pub(crate) fn new(range: ops::Range, existing_numbers: I) -> Self { + Self { + range, + existing_numbers: existing_numbers.peekable(), + } + } +} + +impl Iterator for MissingBlockNumbers +where + I: Iterator, +{ + type Item = BlockNumber; + + fn next(&mut self) -> Option { + // Loop while existing numbers match the starting numbers from the range. The check + // that the range is non-empty is redundant given how `existing_numbers` are constructed + // (they are guaranteed to be lesser than the upper range bound); we add it just to be safe. + while !self.range.is_empty() + && matches!(self.existing_numbers.peek(), Some(&num) if num == self.range.start) + { + self.range.start = self.range.start.next(); + self.existing_numbers.next(); // Advance to the next number + } + + if self.range.is_empty() { + return None; + } + let next_number = self.range.start; + self.range.start = self.range.start.next(); + Some(next_number) + } +} diff --git a/core/lib/zksync_core/src/sync_layer/mod.rs b/core/lib/zksync_core/src/sync_layer/mod.rs index e216ef4f8c55..5111ff19863d 100644 --- a/core/lib/zksync_core/src/sync_layer/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/mod.rs @@ -1,5 +1,6 @@ pub mod batch_status_updater; mod client; +mod consensus; pub mod external_io; pub mod fetcher; pub mod genesis; From 371d935aa691c0744cafe4617138bce099e08535 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 25 Oct 2023 11:02:19 +0300 Subject: [PATCH 02/38] Sketch Postgres-based `BlockStore` --- core/bin/external_node/src/main.rs | 7 +- .../src/sync_layer/consensus/mod.rs | 1 + .../src/sync_layer/consensus/storage.rs | 188 ++++++++++++++++++ .../lib/zksync_core/src/sync_layer/fetcher.rs | 144 ++++++++------ core/lib/zksync_core/src/sync_layer/tests.rs | 6 +- 5 files changed, 274 insertions(+), 72 deletions(-) create mode 100644 core/lib/zksync_core/src/sync_layer/consensus/storage.rs diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 0d954e6a91a8..274706c2b632 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -24,9 +24,8 @@ use zksync_core::{ setup_sigint_handler, state_keeper::{L1BatchExecutorBuilder, MainBatchExecutorBuilder, ZkSyncStateKeeper}, sync_layer::{ - batch_status_updater::BatchStatusUpdater, external_io::ExternalIO, - fetcher::MainNodeFetcherCursor, genesis::perform_genesis_if_needed, ActionQueue, - MainNodeClient, SyncState, + batch_status_updater::BatchStatusUpdater, external_io::ExternalIO, fetcher::FetcherCursor, + genesis::perform_genesis_if_needed, ActionQueue, MainNodeClient, SyncState, }, }; use zksync_dal::{connection::DbVariant, healthcheck::ConnectionPoolHealthCheck, ConnectionPool}; @@ -129,7 +128,7 @@ async fn init_tasks( .await .context("failed to build a connection pool for `MainNodeFetcher`")?; let mut storage = pool.access_storage_tagged("sync_layer").await?; - MainNodeFetcherCursor::new(&mut storage) + FetcherCursor::new(&mut storage) .await .context("failed to load `MainNodeFetcher` cursor from Postgres")? }; diff --git a/core/lib/zksync_core/src/sync_layer/consensus/mod.rs b/core/lib/zksync_core/src/sync_layer/consensus/mod.rs index 7c34d74da1a8..59251188fe11 100644 --- a/core/lib/zksync_core/src/sync_layer/consensus/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/consensus/mod.rs @@ -1,6 +1,7 @@ //! Consensus adapter for EN synchronization logic. mod buffered; +mod storage; #[cfg(test)] mod tests; mod utils; diff --git a/core/lib/zksync_core/src/sync_layer/consensus/storage.rs b/core/lib/zksync_core/src/sync_layer/consensus/storage.rs new file mode 100644 index 000000000000..0b66862ceddd --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/consensus/storage.rs @@ -0,0 +1,188 @@ +//! Storage implementation based on DAL. + +use anyhow::Context as _; +use async_trait::async_trait; + +use std::ops; + +use zksync_concurrency::{ + ctx, + sync::{self, watch, Mutex}, + time, +}; +use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; +use zksync_consensus_storage::{BlockStore, StorageError, StorageResult}; +use zksync_dal::{ConnectionPool, StorageProcessor}; +use zksync_types::{api::en::SyncBlock, Address, MiniblockNumber, H256}; + +use super::buffered::ContiguousBlockStore; +use crate::sync_layer::{fetcher::FetcherCursor, sync_action::ActionQueueSender}; + +fn sync_block_to_consensus_block(_block: SyncBlock) -> FinalBlock { + todo!() +} + +fn consensus_block_to_sync_block(_block: &FinalBlock) -> SyncBlock { + todo!() +} + +#[derive(Debug)] +pub(super) struct PostgresBlockStore { + pool: ConnectionPool, + actions: ActionQueueSender, + block_sender: watch::Sender, + cursor: Mutex, +} + +impl PostgresBlockStore { + pub fn new(pool: ConnectionPool, actions: ActionQueueSender, cursor: FetcherCursor) -> Self { + Self { + pool, + actions, + block_sender: watch::channel(BlockNumber(cursor.miniblock.0.into())).0, + cursor: Mutex::new(cursor), + } + } + + pub async fn listen_to_updates(&self, ctx: &ctx::Ctx) -> StorageResult<()> { + const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); + loop { + let sealed_miniblock_number = self + .sealed_miniblock_number() + .await + .map_err(StorageError::Database)?; + self.block_sender.send_if_modified(|number| { + if *number != sealed_miniblock_number { + *number = sealed_miniblock_number; + true + } else { + false + } + }); + ctx.sleep(POLL_INTERVAL).await?; + } + } + + async fn head_block(&self) -> anyhow::Result { + let mut storage = self.storage().await?; + let miniblock_number = storage + .blocks_dal() + .get_sealed_miniblock_number() + .await + .context("Failed getting sealed miniblock number")?; + // ^ The number can get stale, but it's OK for our purposes + Self::block(&mut storage, miniblock_number) + .await? + .with_context(|| format!("Miniblock #{miniblock_number} disappeared from Postgres")) + } + + async fn storage(&self) -> anyhow::Result> { + self.pool + .access_storage_tagged("sync_layer") + .await + .context("Failed to connect to Postgres") + } + + async fn block( + storage: &mut StorageProcessor<'_>, + number: MiniblockNumber, + ) -> anyhow::Result> { + let Some(block) = storage + .sync_dal() + .sync_block(number, Address::default(), true) + .await + .with_context(|| format!("Failed getting miniblock #{number} from Postgres"))? + else { + return Ok(None); + }; + Ok(Some(sync_block_to_consensus_block(block))) + } + + async fn first_block(&self) -> anyhow::Result { + let mut storage = self.storage().await?; + Self::block(&mut storage, MiniblockNumber(0)) + .await? + .context("Genesis miniblock not present in Postgres") + } + + async fn sealed_miniblock_number(&self) -> anyhow::Result { + let mut storage = self.storage().await?; + let number = storage + .blocks_dal() + .get_sealed_miniblock_number() + .await + .context("Failed getting sealed miniblock number")?; + Ok(BlockNumber(number.0.into())) + } + + async fn schedule_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> ctx::OrCanceled<()> { + let prev_miniblock_hash = H256::zero(); // FIXME + let block = consensus_block_to_sync_block(block); + let actions = sync::lock(ctx, &self.cursor) + .await? + .advance(block, prev_miniblock_hash); + tokio::select! { + () = ctx.canceled() => Err(ctx::Canceled), + () = self.actions.push_actions(actions) => Ok(()), + } + } +} + +#[async_trait] +impl BlockStore for PostgresBlockStore { + async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { + tokio::select! { + () = ctx.canceled() => Err(ctx::Canceled.into()), + result = self.head_block() => result.map_err(StorageError::Database), + } + } + + async fn first_block(&self, ctx: &ctx::Ctx) -> StorageResult { + tokio::select! { + () = ctx.canceled() => Err(ctx::Canceled.into()), + result = self.first_block() => result.map_err(StorageError::Database), + } + } + + async fn last_contiguous_block_number(&self, ctx: &ctx::Ctx) -> StorageResult { + tokio::select! { + () = ctx.canceled() => Err(ctx::Canceled.into()), + result = self.sealed_miniblock_number() => result.map_err(StorageError::Database), + } + } + + async fn block( + &self, + ctx: &ctx::Ctx, + number: BlockNumber, + ) -> StorageResult> { + let get_block = async { + let number = u32::try_from(number.0).context("block number is too large")?; + let mut storage = self.storage().await?; + Self::block(&mut storage, MiniblockNumber(number)).await + }; + tokio::select! { + () = ctx.canceled() => Err(ctx::Canceled.into()), + result = get_block => result.map_err(StorageError::Database), + } + } + + async fn missing_block_numbers( + &self, + _ctx: &ctx::Ctx, + _range: ops::Range, + ) -> StorageResult> { + Ok(vec![]) // The storage never has missing blocks by construction + } + + fn subscribe_to_block_writes(&self) -> watch::Receiver { + self.block_sender.subscribe() + } +} + +#[async_trait] +impl ContiguousBlockStore for PostgresBlockStore { + async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + self.schedule_block(ctx, block).await.map_err(Into::into) + } +} diff --git a/core/lib/zksync_core/src/sync_layer/fetcher.rs b/core/lib/zksync_core/src/sync_layer/fetcher.rs index 02d8d3b11372..ea5a84e96fdd 100644 --- a/core/lib/zksync_core/src/sync_layer/fetcher.rs +++ b/core/lib/zksync_core/src/sync_layer/fetcher.rs @@ -4,7 +4,7 @@ use tokio::sync::watch; use std::time::Duration; use zksync_dal::StorageProcessor; -use zksync_types::{L1BatchNumber, MiniblockNumber, H256}; +use zksync_types::{api::en::SyncBlock, L1BatchNumber, MiniblockNumber, H256}; use zksync_web3_decl::jsonrpsee::core::Error as RpcError; use super::{ @@ -20,13 +20,80 @@ const RETRY_DELAY_INTERVAL: Duration = Duration::from_secs(5); /// Cursor of [`MainNodeFetcher`]. #[derive(Debug)] -pub struct MainNodeFetcherCursor { +pub struct FetcherCursor { // Fields are public for testing purposes. pub(super) miniblock: MiniblockNumber, pub(super) l1_batch: L1BatchNumber, } -impl MainNodeFetcherCursor { +impl FetcherCursor { + pub(super) fn advance( + &mut self, + block: SyncBlock, + prev_miniblock_hash: H256, + ) -> Vec { + let mut new_actions = Vec::new(); + if block.l1_batch_number != self.l1_batch { + assert_eq!( + block.l1_batch_number, + self.l1_batch.next(), + "Unexpected batch number in the next received miniblock" + ); + + tracing::info!( + "New L1 batch: {}. Timestamp: {}", + block.l1_batch_number, + block.timestamp + ); + + new_actions.push(SyncAction::OpenBatch { + number: block.l1_batch_number, + timestamp: block.timestamp, + l1_gas_price: block.l1_gas_price, + l2_fair_gas_price: block.l2_fair_gas_price, + operator_address: block.operator_address, + protocol_version: block.protocol_version, + // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. + first_miniblock_info: (block.number, block.virtual_blocks.unwrap_or(0)), + prev_miniblock_hash, + }); + FETCHER_METRICS.l1_batch[&L1BatchStage::Open].set(block.l1_batch_number.0.into()); + self.l1_batch += 1; + } else { + // New batch implicitly means a new miniblock, so we only need to push the miniblock action + // if it's not a new batch. + new_actions.push(SyncAction::Miniblock { + number: block.number, + timestamp: block.timestamp, + // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. + virtual_blocks: block.virtual_blocks.unwrap_or(0), + }); + FETCHER_METRICS.miniblock.set(block.number.0.into()); + } + + let txs = block + .transactions + .expect("Transactions are always requested"); + APP_METRICS.processed_txs[&TxStage::added_to_mempool()].inc_by(txs.len() as u64); + new_actions.extend(txs.into_iter().map(SyncAction::from)); + + // Last miniblock of the batch is a "fictive" miniblock and would be replicated locally. + // We don't need to seal it explicitly, so we only put the seal miniblock command if it's not the last miniblock. + if block.last_in_batch { + new_actions.push(SyncAction::SealBatch { + // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. + virtual_blocks: block.virtual_blocks.unwrap_or(0), + }); + } else { + new_actions.push(SyncAction::SealMiniblock); + } + self.miniblock += 1; + + new_actions + } +} + +impl FetcherCursor { /// Loads the cursor pub async fn new(storage: &mut StorageProcessor<'_>) -> anyhow::Result { let last_sealed_l1_batch_header = storage @@ -87,7 +154,7 @@ impl MainNodeFetcherCursor { #[derive(Debug)] pub struct MainNodeFetcher { client: CachingMainNodeClient, - cursor: MainNodeFetcherCursor, + cursor: FetcherCursor, actions: ActionQueueSender, sync_state: SyncState, stop_receiver: watch::Receiver, @@ -174,72 +241,19 @@ impl MainNodeFetcher { .expect("Previous block must exist"); request_latency.observe(); - let mut new_actions = Vec::new(); - if block.l1_batch_number != self.cursor.l1_batch { - assert_eq!( - block.l1_batch_number, - self.cursor.l1_batch.next(), - "Unexpected batch number in the next received miniblock" - ); - - tracing::info!( - "New batch: {}. Timestamp: {}", - block.l1_batch_number, - block.timestamp - ); - - new_actions.push(SyncAction::OpenBatch { - number: block.l1_batch_number, - timestamp: block.timestamp, - l1_gas_price: block.l1_gas_price, - l2_fair_gas_price: block.l2_fair_gas_price, - operator_address: block.operator_address, - protocol_version: block.protocol_version, - // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. - first_miniblock_info: (block.number, block.virtual_blocks.unwrap_or(0)), - // Same for `prev_block.hash` as above. - prev_miniblock_hash: prev_block.hash.unwrap_or_else(H256::zero), - }); - FETCHER_METRICS.l1_batch[&L1BatchStage::Open].set(block.l1_batch_number.0.into()); - self.cursor.l1_batch += 1; - } else { - // New batch implicitly means a new miniblock, so we only need to push the miniblock action - // if it's not a new batch. - new_actions.push(SyncAction::Miniblock { - number: block.number, - timestamp: block.timestamp, - // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. - virtual_blocks: block.virtual_blocks.unwrap_or(0), - }); - FETCHER_METRICS.miniblock.set(block.number.0.into()); - } - - let txs: Vec = block - .transactions - .expect("Transactions are always requested"); - APP_METRICS.processed_txs[&TxStage::added_to_mempool()].inc_by(txs.len() as u64); - new_actions.extend(txs.into_iter().map(SyncAction::from)); - - // Last miniblock of the batch is a "fictive" miniblock and would be replicated locally. - // We don't need to seal it explicitly, so we only put the seal miniblock command if it's not the last miniblock. - if block.last_in_batch { - new_actions.push(SyncAction::SealBatch { - // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. - virtual_blocks: block.virtual_blocks.unwrap_or(0), - }); - } else { - new_actions.push(SyncAction::SealMiniblock); - } + let block_number = block.number; + let new_actions = self + .cursor + .advance(block, prev_block.hash.unwrap_or_default()); tracing::info!( - "New miniblock: {} / {}", - block.number, - self.sync_state.get_main_node_block().max(block.number) + "New miniblock: {block_number} / {}", + self.sync_state.get_main_node_block().max(block_number) ); // Forgetting only the previous one because we still need the current one in cache for the next iteration. - let prev_miniblock_number = MiniblockNumber(self.cursor.miniblock.0.saturating_sub(1)); + let prev_miniblock_number = MiniblockNumber(block_number.0.saturating_sub(1)); + // FIXME: the old implementation had block_number.0.saturating_sub(2) for some reason self.client.forget_miniblock(prev_miniblock_number); - self.cursor.miniblock += 1; self.actions.push_actions(new_actions).await; total_latency.observe(); diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index 8fe61fe16e6d..65819cbb591c 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -17,7 +17,7 @@ use zksync_types::{ }; use super::{ - fetcher::MainNodeFetcherCursor, + fetcher::FetcherCursor, sync_action::{ActionQueueSender, SyncAction}, *, }; @@ -496,7 +496,7 @@ async fn fetcher_basics() { let pool = ConnectionPool::test_pool().await; let mut storage = pool.access_storage().await.unwrap(); ensure_genesis(&mut storage).await; - let fetcher_cursor = MainNodeFetcherCursor::new(&mut storage).await.unwrap(); + let fetcher_cursor = FetcherCursor::new(&mut storage).await.unwrap(); assert_eq!(fetcher_cursor.l1_batch, L1BatchNumber(0)); assert_eq!(fetcher_cursor.miniblock, MiniblockNumber(1)); drop(storage); @@ -589,7 +589,7 @@ async fn fetcher_with_real_server() { let sync_state = SyncState::default(); let (actions_sender, mut actions) = ActionQueue::new(); let client = ::json_rpc(&format!("http://{server_addr}/")).unwrap(); - let fetcher_cursor = MainNodeFetcherCursor { + let fetcher_cursor = FetcherCursor { miniblock: MiniblockNumber(1), l1_batch: L1BatchNumber(0), }; From 259585b44bb7426046058be2121ce18fa55c7a59 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 26 Oct 2023 17:19:54 +0300 Subject: [PATCH 03/38] Sketch gossip-based block fetcher --- .../src/sync_layer/consensus/mod.rs | 7 --- .../{consensus => gossip}/buffered.rs | 5 +- .../zksync_core/src/sync_layer/gossip/mod.rs | 58 +++++++++++++++++++ .../{consensus => gossip}/storage.rs | 0 .../sync_layer/{consensus => gossip}/tests.rs | 6 +- .../sync_layer/{consensus => gossip}/utils.rs | 0 core/lib/zksync_core/src/sync_layer/mod.rs | 6 +- 7 files changed, 66 insertions(+), 16 deletions(-) delete mode 100644 core/lib/zksync_core/src/sync_layer/consensus/mod.rs rename core/lib/zksync_core/src/sync_layer/{consensus => gossip}/buffered.rs (99%) create mode 100644 core/lib/zksync_core/src/sync_layer/gossip/mod.rs rename core/lib/zksync_core/src/sync_layer/{consensus => gossip}/storage.rs (100%) rename core/lib/zksync_core/src/sync_layer/{consensus => gossip}/tests.rs (98%) rename core/lib/zksync_core/src/sync_layer/{consensus => gossip}/utils.rs (100%) diff --git a/core/lib/zksync_core/src/sync_layer/consensus/mod.rs b/core/lib/zksync_core/src/sync_layer/consensus/mod.rs deleted file mode 100644 index 59251188fe11..000000000000 --- a/core/lib/zksync_core/src/sync_layer/consensus/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Consensus adapter for EN synchronization logic. - -mod buffered; -mod storage; -#[cfg(test)] -mod tests; -mod utils; diff --git a/core/lib/zksync_core/src/sync_layer/consensus/buffered.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs similarity index 99% rename from core/lib/zksync_core/src/sync_layer/consensus/buffered.rs rename to core/lib/zksync_core/src/sync_layer/gossip/buffered.rs index 35f3268f0bf7..657bbe8f4d57 100644 --- a/core/lib/zksync_core/src/sync_layer/consensus/buffered.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs @@ -154,15 +154,14 @@ impl BufferedStorage { } #[cfg(test)] - pub(crate) fn set_events_sender( + pub(super) fn set_events_sender( &mut self, sender: channel::UnboundedSender, ) { self.events_sender = sender; } - #[cfg(test)] - pub(crate) fn as_ref(&self) -> &T { + pub(super) fn inner(&self) -> &T { &self.inner } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs new file mode 100644 index 000000000000..9f260ec62ffd --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -0,0 +1,58 @@ +//! Consensus adapter for EN synchronization logic. + +use anyhow::Context as _; + +use std::sync::Arc; + +use zksync_concurrency::{ctx, scope}; +use zksync_consensus_roles::node; +use zksync_dal::ConnectionPool; +use zksync_executor::{Executor, ExecutorConfig}; + +mod buffered; +mod storage; +#[cfg(test)] +mod tests; +mod utils; + +use self::{buffered::BufferedStorage, storage::PostgresBlockStore}; +use super::{fetcher::FetcherCursor, sync_action::ActionQueueSender}; + +/// Starts fetching L2 blocks using peer-to-peer gossip network. +pub async fn start_gossip_fetcher( + pool: ConnectionPool, + actions: ActionQueueSender, + executor_config: ExecutorConfig, + node_key: node::SecretKey, +) -> anyhow::Result<()> { + let mut storage = pool + .access_storage_tagged("sync_layer") + .await + .context("Failed acquiring Postgres connection for cursor")?; + let cursor = FetcherCursor::new(&mut storage).await?; + drop(storage); + + let store = PostgresBlockStore::new(pool, actions, cursor); + let buffered_store = Arc::new(BufferedStorage::new(store)); + let store = buffered_store.inner(); + let executor = Executor::new(executor_config, node_key, buffered_store.clone()) + .context("Node executor misconfiguration")?; + + scope::run!(&ctx::root(), |ctx, s| async { + s.spawn_bg(async { + store + .listen_to_updates(ctx) + .await + .context("`PostgresBlockStore` listener failed") + }); + s.spawn_bg(async { + buffered_store + .listen_to_updates(ctx) + .await + .context("`BufferedStore` listener failed") + }); + + executor.run(ctx).await.context("Node executor terminated") + }) + .await +} diff --git a/core/lib/zksync_core/src/sync_layer/consensus/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs similarity index 100% rename from core/lib/zksync_core/src/sync_layer/consensus/storage.rs rename to core/lib/zksync_core/src/sync_layer/gossip/storage.rs diff --git a/core/lib/zksync_core/src/sync_layer/consensus/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs similarity index 98% rename from core/lib/zksync_core/src/sync_layer/consensus/tests.rs rename to core/lib/zksync_core/src/sync_layer/gossip/tests.rs index efd444e175f6..c4a9444c1d63 100644 --- a/core/lib/zksync_core/src/sync_layer/consensus/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -169,7 +169,7 @@ async fn test_buffered_storage( let last_block_number = BlockNumber((block_count + initial_block_count) as u64); scope::run!(ctx, |ctx, s| async { - s.spawn_bg(buffered_store.as_ref().run_updates(ctx, block_receiver)); + s.spawn_bg(buffered_store.inner().run_updates(ctx, block_receiver)); s.spawn_bg(async { let err = buffered_store.listen_to_updates(ctx).await.unwrap_err(); match &err { @@ -212,9 +212,9 @@ async fn test_buffered_storage( ctx.sleep(block_interval).await?; } - let mut inner_subscriber = buffered_store.as_ref().subscribe_to_block_writes(); + let mut inner_subscriber = buffered_store.inner().subscribe_to_block_writes(); while buffered_store - .as_ref() + .inner() .last_contiguous_block_number(ctx) .await? < last_block_number diff --git a/core/lib/zksync_core/src/sync_layer/consensus/utils.rs b/core/lib/zksync_core/src/sync_layer/gossip/utils.rs similarity index 100% rename from core/lib/zksync_core/src/sync_layer/consensus/utils.rs rename to core/lib/zksync_core/src/sync_layer/gossip/utils.rs diff --git a/core/lib/zksync_core/src/sync_layer/mod.rs b/core/lib/zksync_core/src/sync_layer/mod.rs index 5111ff19863d..da521ab3b523 100644 --- a/core/lib/zksync_core/src/sync_layer/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/mod.rs @@ -1,9 +1,9 @@ pub mod batch_status_updater; mod client; -mod consensus; pub mod external_io; pub mod fetcher; pub mod genesis; +mod gossip; mod metrics; pub(crate) mod sync_action; mod sync_state; @@ -11,6 +11,6 @@ mod sync_state; mod tests; pub use self::{ - client::MainNodeClient, external_io::ExternalIO, sync_action::ActionQueue, - sync_state::SyncState, + client::MainNodeClient, external_io::ExternalIO, gossip::start_gossip_fetcher, + sync_action::ActionQueue, sync_state::SyncState, }; From cac012792c34c19a7d17bea84f447ca25dcb987f Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 30 Oct 2023 12:52:01 +0200 Subject: [PATCH 04/38] Update fetched block conversions ...and used consensus code --- .../lib/zksync_core/src/sync_layer/fetcher.rs | 159 ++++++++++-------- .../src/sync_layer/gossip/buffered.rs | 10 +- .../src/sync_layer/gossip/conversions.rs | 72 ++++++++ .../zksync_core/src/sync_layer/gossip/mod.rs | 1 + .../src/sync_layer/gossip/storage.rs | 29 ++-- .../src/sync_layer/gossip/tests.rs | 50 +++--- core/lib/zksync_core/src/sync_layer/tests.rs | 10 ++ 7 files changed, 217 insertions(+), 114 deletions(-) create mode 100644 core/lib/zksync_core/src/sync_layer/gossip/conversions.rs diff --git a/core/lib/zksync_core/src/sync_layer/fetcher.rs b/core/lib/zksync_core/src/sync_layer/fetcher.rs index ea5a84e96fdd..6e7c53b016c2 100644 --- a/core/lib/zksync_core/src/sync_layer/fetcher.rs +++ b/core/lib/zksync_core/src/sync_layer/fetcher.rs @@ -4,7 +4,9 @@ use tokio::sync::watch; use std::time::Duration; use zksync_dal::StorageProcessor; -use zksync_types::{api::en::SyncBlock, L1BatchNumber, MiniblockNumber, H256}; +use zksync_types::{ + api::en::SyncBlock, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, +}; use zksync_web3_decl::jsonrpsee::core::Error as RpcError; use super::{ @@ -18,20 +20,92 @@ use crate::metrics::{TxStage, APP_METRICS}; const DELAY_INTERVAL: Duration = Duration::from_millis(500); const RETRY_DELAY_INTERVAL: Duration = Duration::from_secs(5); +/// Common denominator for blocks fetched by an external node. +#[derive(Debug)] +pub(super) struct FetchedBlock { + pub number: MiniblockNumber, + pub l1_batch_number: L1BatchNumber, + pub protocol_version: ProtocolVersionId, + pub timestamp: u64, + pub hash: H256, + pub l1_gas_price: u64, + pub l2_fair_gas_price: u64, + pub virtual_blocks: u32, + pub operator_address: Address, + pub transactions: Vec, +} + +impl FetchedBlock { + fn from_sync_block(block: SyncBlock) -> Self { + Self { + number: block.number, + l1_batch_number: block.l1_batch_number, + protocol_version: block.protocol_version, + timestamp: block.timestamp, + hash: block.hash.unwrap_or_default(), + l1_gas_price: block.l1_gas_price, + l2_fair_gas_price: block.l2_fair_gas_price, + virtual_blocks: block.virtual_blocks.unwrap_or(0), + operator_address: block.operator_address, + transactions: block + .transactions + .expect("Transactions are always requested"), + } + } +} + /// Cursor of [`MainNodeFetcher`]. #[derive(Debug)] pub struct FetcherCursor { // Fields are public for testing purposes. pub(super) miniblock: MiniblockNumber, + pub(super) prev_miniblock_hash: H256, pub(super) l1_batch: L1BatchNumber, } impl FetcherCursor { - pub(super) fn advance( - &mut self, - block: SyncBlock, - prev_miniblock_hash: H256, - ) -> Vec { + /// Loads the cursor + pub async fn new(storage: &mut StorageProcessor<'_>) -> anyhow::Result { + let last_sealed_l1_batch_header = storage + .blocks_dal() + .get_newest_l1_batch_header() + .await + .context("Failed getting newest L1 batch header")?; + let last_miniblock_header = storage + .blocks_dal() + .get_last_sealed_miniblock_header() + .await + .context("Failed getting sealed miniblock header")? + .context("No miniblocks sealed")?; + + // It's important to know whether we have opened a new batch already or just sealed the previous one. + // Depending on it, we must either insert `OpenBatch` item into the queue, or not. + let was_new_batch_open = storage + .blocks_dal() + .pending_batch_exists() + .await + .context("Failed checking whether pending L1 batch exists")?; + + // Miniblocks are always fully processed. + let miniblock = last_miniblock_header.number + 1; + let prev_miniblock_hash = last_miniblock_header.hash; + // Decide whether the next batch should be explicitly opened or not. + let l1_batch = if was_new_batch_open { + // No `OpenBatch` action needed. + last_sealed_l1_batch_header.number + 1 + } else { + // We need to open the next batch. + last_sealed_l1_batch_header.number + }; + + Ok(Self { + miniblock, + l1_batch, + prev_miniblock_hash, + }) + } + + pub(super) fn advance(&mut self, block: FetchedBlock) -> Vec { let mut new_actions = Vec::new(); if block.l1_batch_number != self.l1_batch { assert_eq!( @@ -54,8 +128,8 @@ impl FetcherCursor { operator_address: block.operator_address, protocol_version: block.protocol_version, // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. - first_miniblock_info: (block.number, block.virtual_blocks.unwrap_or(0)), - prev_miniblock_hash, + first_miniblock_info: (block.number, block.virtual_blocks), + prev_miniblock_hash: self.prev_miniblock_hash, }); FETCHER_METRICS.l1_batch[&L1BatchStage::Open].set(block.l1_batch_number.0.into()); self.l1_batch += 1; @@ -66,71 +140,32 @@ impl FetcherCursor { number: block.number, timestamp: block.timestamp, // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. - virtual_blocks: block.virtual_blocks.unwrap_or(0), + virtual_blocks: block.virtual_blocks, }); FETCHER_METRICS.miniblock.set(block.number.0.into()); } - let txs = block - .transactions - .expect("Transactions are always requested"); - APP_METRICS.processed_txs[&TxStage::added_to_mempool()].inc_by(txs.len() as u64); - new_actions.extend(txs.into_iter().map(SyncAction::from)); + // FIXME: shaky assumption + let last_in_batch = block.transactions.is_empty(); + APP_METRICS.processed_txs[&TxStage::added_to_mempool()] + .inc_by(block.transactions.len() as u64); + new_actions.extend(block.transactions.into_iter().map(SyncAction::from)); // Last miniblock of the batch is a "fictive" miniblock and would be replicated locally. // We don't need to seal it explicitly, so we only put the seal miniblock command if it's not the last miniblock. - if block.last_in_batch { + if last_in_batch { new_actions.push(SyncAction::SealBatch { // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. - virtual_blocks: block.virtual_blocks.unwrap_or(0), + virtual_blocks: block.virtual_blocks, }); } else { new_actions.push(SyncAction::SealMiniblock); } self.miniblock += 1; + self.prev_miniblock_hash = block.hash; new_actions } -} - -impl FetcherCursor { - /// Loads the cursor - pub async fn new(storage: &mut StorageProcessor<'_>) -> anyhow::Result { - let last_sealed_l1_batch_header = storage - .blocks_dal() - .get_newest_l1_batch_header() - .await - .context("Failed getting newest L1 batch header")?; - let last_miniblock_number = storage - .blocks_dal() - .get_sealed_miniblock_number() - .await - .context("Failed getting sealed miniblock number")?; - - // It's important to know whether we have opened a new batch already or just sealed the previous one. - // Depending on it, we must either insert `OpenBatch` item into the queue, or not. - let was_new_batch_open = storage - .blocks_dal() - .pending_batch_exists() - .await - .context("Failed checking whether pending L1 batch exists")?; - - // Miniblocks are always fully processed. - let miniblock = last_miniblock_number + 1; - // Decide whether the next batch should be explicitly opened or not. - let l1_batch = if was_new_batch_open { - // No `OpenBatch` action needed. - last_sealed_l1_batch_header.number + 1 - } else { - // We need to open the next batch. - last_sealed_l1_batch_header.number - }; - - Ok(Self { - miniblock, - l1_batch, - }) - } /// Builds a fetcher from this cursor. pub fn into_fetcher( @@ -232,19 +267,11 @@ impl MainNodeFetcher { let Some(block) = self.client.fetch_l2_block(self.cursor.miniblock).await? else { return Ok(false); }; - - // This will be fetched from cache. - let prev_block = self - .client - .fetch_l2_block(self.cursor.miniblock - 1) - .await? - .expect("Previous block must exist"); request_latency.observe(); let block_number = block.number; - let new_actions = self - .cursor - .advance(block, prev_block.hash.unwrap_or_default()); + let fetched_block = FetchedBlock::from_sync_block(block); + let new_actions = self.cursor.advance(fetched_block); tracing::info!( "New miniblock: {block_number} / {}", diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs index 657bbe8f4d57..1b6e0877319d 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs @@ -100,7 +100,7 @@ impl BlockBuffer { } fn put_block(&mut self, block: FinalBlock) { - let block_number = block.block.number; + let block_number = block.header.number; assert!(block_number > self.store_block_number); // ^ Must be checked previously self.blocks.insert(block_number, block); @@ -187,7 +187,7 @@ impl BufferedStorage { }; if let Some(block) = next_block_for_store { self.inner.schedule_next_block(ctx, &block).await?; - let block_number = block.block.number; + let block_number = block.header.number; tracing::trace!(%block_number, "Block scheduled in underlying storage"); } @@ -254,7 +254,7 @@ impl WriteBlockStore for BufferedStorage { async fn put_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { let next_block_for_store = { let mut buffer = sync::lock(ctx, &self.buffer).await?; - let block_number = block.block.number; + let block_number = block.header.number; if block_number <= buffer.store_block_number { let err = anyhow::anyhow!( "Cannot replace a block #{block_number} since it is already present in the underlying storage", @@ -267,9 +267,9 @@ impl WriteBlockStore for BufferedStorage { if let Some(block) = next_block_for_store { self.inner.schedule_next_block(ctx, &block).await?; - tracing::trace!(block_number = %block.block.number, "Block scheduled in underlying storage"); + tracing::trace!(block_number = %block.header.number, "Block scheduled in underlying storage"); } - self.block_writes_sender.send_replace(block.block.number); + self.block_writes_sender.send_replace(block.header.number); Ok(()) } } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs new file mode 100644 index 000000000000..fdbc5115df6d --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -0,0 +1,72 @@ +//! Conversion logic between server and consensus types. + +use anyhow::Context as _; +use rand::{thread_rng, Rng}; +use serde::{Deserialize, Serialize}; + +use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock, Payload}; +use zksync_types::{ + api::en::SyncBlock, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, +}; + +use crate::sync_layer::fetcher::FetchedBlock; + +// FIXME: should use Protobuf +#[derive(Debug, Serialize, Deserialize)] +struct BlockPayload { + hash: H256, + l1_batch_number: L1BatchNumber, + timestamp: u64, + l1_gas_price: u64, + l2_fair_gas_price: u64, + virtual_blocks: u32, + operator_address: Address, + transactions: Vec, +} + +pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> FinalBlock { + let payload = serde_json::to_vec(&BlockPayload { + hash: block.hash.unwrap_or_default(), + l1_batch_number: block.l1_batch_number, + timestamp: block.timestamp, + l1_gas_price: block.l1_gas_price, + l2_fair_gas_price: block.l2_fair_gas_price, + virtual_blocks: block.virtual_blocks.unwrap_or(0), + operator_address: block.operator_address, + transactions: block + .transactions + .expect("Transactions are always requested"), + }); + let payload = Payload(payload.expect("Failed serializing block payload")); + FinalBlock { + header: BlockHeader { + parent: thread_rng().gen(), // FIXME + number: BlockNumber(block.number.0.into()), + payload: payload.hash(), + }, + payload, + justification: thread_rng().gen(), // FIXME + } +} + +impl FetchedBlock { + pub(super) fn from_gossip_block(block: &FinalBlock) -> anyhow::Result { + let number = u32::try_from(block.header.number.0) + .context("Integer overflow converting block number")?; + let payload: BlockPayload = serde_json::from_slice(&block.payload.0) + .context("Failed deserializing block payload")?; + + Ok(Self { + number: MiniblockNumber(number), + l1_batch_number: payload.l1_batch_number, + protocol_version: ProtocolVersionId::latest(), // FIXME + timestamp: payload.timestamp, + hash: payload.hash, + l1_gas_price: payload.l1_gas_price, + l2_fair_gas_price: payload.l2_fair_gas_price, + virtual_blocks: payload.virtual_blocks, + operator_address: payload.operator_address, + transactions: payload.transactions, + }) + } +} diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index 9f260ec62ffd..00016a17c73e 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -10,6 +10,7 @@ use zksync_dal::ConnectionPool; use zksync_executor::{Executor, ExecutorConfig}; mod buffered; +mod conversions; mod storage; #[cfg(test)] mod tests; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index 0b66862ceddd..f2d1c307177b 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -13,18 +13,13 @@ use zksync_concurrency::{ use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; use zksync_consensus_storage::{BlockStore, StorageError, StorageResult}; use zksync_dal::{ConnectionPool, StorageProcessor}; -use zksync_types::{api::en::SyncBlock, Address, MiniblockNumber, H256}; +use zksync_types::{Address, MiniblockNumber}; -use super::buffered::ContiguousBlockStore; -use crate::sync_layer::{fetcher::FetcherCursor, sync_action::ActionQueueSender}; - -fn sync_block_to_consensus_block(_block: SyncBlock) -> FinalBlock { - todo!() -} - -fn consensus_block_to_sync_block(_block: &FinalBlock) -> SyncBlock { - todo!() -} +use super::{buffered::ContiguousBlockStore, conversions::sync_block_to_consensus_block}; +use crate::sync_layer::{ + fetcher::{FetchedBlock, FetcherCursor}, + sync_action::ActionQueueSender, +}; #[derive(Debug)] pub(super) struct PostgresBlockStore { @@ -115,14 +110,12 @@ impl PostgresBlockStore { Ok(BlockNumber(number.0.into())) } - async fn schedule_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> ctx::OrCanceled<()> { - let prev_miniblock_hash = H256::zero(); // FIXME - let block = consensus_block_to_sync_block(block); - let actions = sync::lock(ctx, &self.cursor) - .await? - .advance(block, prev_miniblock_hash); + async fn schedule_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + let fetched_block = + FetchedBlock::from_gossip_block(block).map_err(StorageError::Database)?; + let actions = sync::lock(ctx, &self.cursor).await?.advance(fetched_block); tokio::select! { - () = ctx.canceled() => Err(ctx::Canceled), + () = ctx.canceled() => Err(ctx::Canceled.into()), () = self.actions.push_actions(actions) => Ok(()), } } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index c4a9444c1d63..f80be920f54f 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -3,7 +3,6 @@ use assert_matches::assert_matches; use async_trait::async_trait; use rand::{rngs::StdRng, seq::SliceRandom, Rng}; -use tempfile::TempDir; use test_casing::test_casing; use std::{iter, ops}; @@ -14,34 +13,35 @@ use zksync_concurrency::{ sync::{self, watch}, time, }; -use zksync_consensus_roles::validator::{Block, BlockNumber, FinalBlock}; +use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock, Payload}; use zksync_consensus_storage::{ - BlockStore, RocksdbStorage, StorageError, StorageResult, WriteBlockStore, + BlockStore, InMemoryStorage, StorageError, StorageResult, WriteBlockStore, }; use super::buffered::{BufferedStorage, BufferedStorageEvent, ContiguousBlockStore}; -async fn init_store(ctx: &ctx::Ctx, rng: &mut R) -> (FinalBlock, RocksdbStorage, TempDir) { +fn init_store(rng: &mut impl Rng) -> (FinalBlock, InMemoryStorage) { + let payload = Payload(vec![]); let genesis_block = FinalBlock { - block: Block::genesis(vec![]), + header: BlockHeader::genesis(payload.hash()), + payload, justification: rng.gen(), }; - let temp_dir = TempDir::new().unwrap(); - let block_store = RocksdbStorage::new(ctx, &genesis_block, temp_dir.path()) - .await - .unwrap(); - (genesis_block, block_store, temp_dir) + let block_store = InMemoryStorage::new(genesis_block.clone()); + (genesis_block, block_store) } fn gen_blocks(rng: &mut impl Rng, genesis_block: FinalBlock, count: usize) -> Vec { let blocks = iter::successors(Some(genesis_block), |parent| { - let block = Block { - parent: parent.block.hash(), - number: parent.block.number.next(), - payload: Vec::new(), + let payload = Payload(vec![]); + let header = BlockHeader { + parent: parent.header.hash(), + number: parent.header.number.next(), + payload: payload.hash(), }; Some(FinalBlock { - block, + header, + payload, justification: rng.gen(), }) }); @@ -50,12 +50,12 @@ fn gen_blocks(rng: &mut impl Rng, genesis_block: FinalBlock, count: usize) -> Ve #[derive(Debug)] struct MockContiguousStore { - inner: RocksdbStorage, + inner: InMemoryStorage, block_sender: channel::Sender, } impl MockContiguousStore { - fn new(inner: RocksdbStorage) -> (Self, channel::Receiver) { + fn new(inner: InMemoryStorage) -> (Self, channel::Receiver) { let (block_sender, block_receiver) = channel::bounded(1); let this = Self { inner, @@ -117,8 +117,8 @@ impl BlockStore for MockContiguousStore { #[async_trait] impl ContiguousBlockStore for MockContiguousStore { async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { - let head_block_number = self.head_block(ctx).await?.block.number; - assert_eq!(block.block.number, head_block_number.next()); + let head_block_number = self.head_block(ctx).await?.header.number; + assert_eq!(block.header.number, head_block_number.next()); self.block_sender .try_send(block.clone()) .expect("BufferedStorage is rushing"); @@ -136,7 +136,7 @@ async fn test_buffered_storage( let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); - let (genesis_block, block_store, _temp_dir) = init_store(ctx, rng).await; + let (genesis_block, block_store) = init_store(rng); let mut initial_blocks = gen_blocks(rng, genesis_block.clone(), initial_block_count); for block in &initial_blocks { block_store.put_block(ctx, block).await.unwrap(); @@ -155,7 +155,7 @@ async fn test_buffered_storage( last_initial_block ); for block in &initial_blocks { - let block_result = buffered_store.block(ctx, block.block.number).await; + let block_result = buffered_store.block(ctx, block.header.number).await; assert_eq!(block_result.unwrap().as_ref(), Some(block)); } let mut subscriber = buffered_store.subscribe_to_block_writes(); @@ -181,11 +181,11 @@ async fn test_buffered_storage( for (idx, block) in blocks.iter().enumerate() { buffered_store.put_block(ctx, block).await?; let new_block_number = *sync::changed(ctx, &mut subscriber).await?; - assert_eq!(new_block_number, block.block.number); + assert_eq!(new_block_number, block.header.number); // Check that all written blocks are immediately accessible. for existing_block in initial_blocks.iter().chain(&blocks[0..=idx]) { - let number = existing_block.block.number; + let number = existing_block.header.number; assert_eq!( buffered_store.block(ctx, number).await?.as_ref(), Some(existing_block) @@ -195,13 +195,13 @@ async fn test_buffered_storage( let expected_head_block = blocks[0..=idx] .iter() - .max_by_key(|block| block.block.number) + .max_by_key(|block| block.header.number) .unwrap(); assert_eq!(buffered_store.head_block(ctx).await?, *expected_head_block); let expected_last_contiguous_block = blocks[(idx + 1)..] .iter() - .map(|block| block.block.number) + .map(|block| block.header.number) .min() .map_or(last_block_number, BlockNumber::prev); assert_eq!( diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index 65819cbb591c..5ce5549859c8 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -576,6 +576,15 @@ async fn fetcher_with_real_server() { // Fill in transactions grouped in multiple miniblocks in the storage. let tx_hashes = run_state_keeper_with_multiple_miniblocks(pool.clone()).await; let mut tx_hashes = VecDeque::from(tx_hashes); + let mut connection = pool.access_storage().await.unwrap(); + let genesis_miniblock_hash = connection + .blocks_dal() + .get_miniblock_header(MiniblockNumber(0)) + .await + .unwrap() + .expect("No genesis miniblock") + .hash; + drop(connection); // Start the API server. let network_config = NetworkConfig::from_env().unwrap(); @@ -591,6 +600,7 @@ async fn fetcher_with_real_server() { let client = ::json_rpc(&format!("http://{server_addr}/")).unwrap(); let fetcher_cursor = FetcherCursor { miniblock: MiniblockNumber(1), + prev_miniblock_hash: genesis_miniblock_hash, l1_batch: L1BatchNumber(0), }; let fetcher = fetcher_cursor.into_fetcher( From e146c5d6ed1cde2eb3ee06ce64741c34819d0eb8 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 30 Oct 2023 15:23:20 +0200 Subject: [PATCH 05/38] Test Postgres-based block store implementation --- .../src/sync_layer/gossip/storage.rs | 190 ++++++++++++++++++ core/lib/zksync_core/src/sync_layer/tests.rs | 2 +- 2 files changed, 191 insertions(+), 1 deletion(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index f2d1c307177b..01bd04ea9032 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -179,3 +179,193 @@ impl ContiguousBlockStore for PostgresBlockStore { self.schedule_block(ctx, block).await.map_err(Into::into) } } + +#[cfg(test)] +mod tests { + use assert_matches::assert_matches; + + use zksync_concurrency::{scope, time}; + use zksync_types::{L1BatchNumber, L2ChainId}; + + use super::*; + use crate::{ + genesis::{ensure_genesis_state, GenesisParams}, + sync_layer::{ + sync_action::SyncAction, tests::run_state_keeper_with_multiple_miniblocks, ActionQueue, + }, + }; + + const TEST_TIMEOUT: time::Duration = time::Duration::seconds(1); + const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); + + #[tokio::test] + async fn block_store_basics_for_postgres() { + let pool = ConnectionPool::test_pool().await; + run_state_keeper_with_multiple_miniblocks(pool.clone()).await; + + let mut storage = pool.access_storage().await.unwrap(); + let cursor = FetcherCursor::new(&mut storage).await.unwrap(); + drop(storage); + let (actions_sender, _) = ActionQueue::new(); + let storage = PostgresBlockStore::new(pool.clone(), actions_sender, cursor); + + let ctx = &ctx::test_root(&ctx::RealClock); + let genesis_block = BlockStore::first_block(&storage, ctx).await.unwrap(); + assert_eq!(genesis_block.header.number, BlockNumber(0)); + let head_block = BlockStore::head_block(&storage, ctx).await.unwrap(); + assert_eq!(head_block.header.number, BlockNumber(2)); + let last_contiguous_block_number = storage.last_contiguous_block_number(ctx).await.unwrap(); + assert_eq!(last_contiguous_block_number, BlockNumber(2)); + + let block = storage + .block(ctx, BlockNumber(1)) + .await + .unwrap() + .expect("no block #1"); + assert_eq!(block.header.number, BlockNumber(1)); + let missing_block = storage.block(ctx, BlockNumber(3)).await.unwrap(); + assert!(missing_block.is_none(), "{missing_block:?}"); + } + + #[tokio::test] + async fn subscribing_to_block_updates_for_postgres() { + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.access_storage().await.unwrap(); + if storage.blocks_dal().is_genesis_needed().await.unwrap() { + ensure_genesis_state(&mut storage, L2ChainId::default(), &GenesisParams::mock()) + .await + .unwrap(); + } + let cursor = FetcherCursor::new(&mut storage).await.unwrap(); + // ^ This is logically incorrect (the storage should not be updated other than using + // `ContiguousBlockStore`), but for testing subscriptions this is fine. + drop(storage); + let (actions_sender, _) = ActionQueue::new(); + let storage = PostgresBlockStore::new(pool.clone(), actions_sender, cursor); + let mut subscriber = storage.subscribe_to_block_writes(); + + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, s| async { + s.spawn_bg(async { + match storage.listen_to_updates(ctx).await { + Ok(()) | Err(StorageError::Canceled(_)) => Ok(()), + Err(err) => Err(err.into()), + } + }); + s.spawn(async { + run_state_keeper_with_multiple_miniblocks(pool.clone()).await; + Ok(()) + }); + + loop { + let block = *sync::changed(ctx, &mut subscriber).await?; + if block == BlockNumber(2) { + // We should receive at least the last update. + break; + } + } + anyhow::Ok(()) + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn processing_new_blocks() { + let pool = ConnectionPool::test_pool().await; + run_state_keeper_with_multiple_miniblocks(pool.clone()).await; + + let mut storage = pool.access_storage().await.unwrap(); + let first_block = storage + .sync_dal() + .sync_block(MiniblockNumber(1), Address::repeat_byte(1), true) + .await + .unwrap() + .expect("no sync block #1"); + let first_block = sync_block_to_consensus_block(first_block); + let second_block = storage + .sync_dal() + .sync_block(MiniblockNumber(2), Address::repeat_byte(1), true) + .await + .unwrap() + .expect("no sync block #2"); + let second_block = sync_block_to_consensus_block(second_block); + storage + .transactions_dal() + .reset_transactions_state(MiniblockNumber(0)) + .await; + storage + .blocks_dal() + .delete_miniblocks(MiniblockNumber(0)) + .await + .unwrap(); + let cursor = FetcherCursor::new(&mut storage).await.unwrap(); + drop(storage); + + let (actions_sender, mut actions) = ActionQueue::new(); + let storage = PostgresBlockStore::new(pool.clone(), actions_sender, cursor); + let ctx = &ctx::test_root(&ctx::RealClock); + storage.schedule_block(ctx, &first_block).await.unwrap(); + + scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, _| async { + let mut received_actions = vec![]; + while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { + let Some(action) = actions.pop_action() else { + ctx.sleep(POLL_INTERVAL).await?; + continue; + }; + received_actions.push(action); + } + assert_matches!( + received_actions.as_slice(), + [ + SyncAction::OpenBatch { + number: L1BatchNumber(1), + timestamp: 1, + first_miniblock_info: (MiniblockNumber(1), 1), + .. + }, + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::SealMiniblock, + ] + ); + anyhow::Ok(()) + }) + .await + .unwrap(); + + storage.schedule_block(ctx, &second_block).await.unwrap(); + + scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, _| async { + let mut received_actions = vec![]; + while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { + let Some(action) = actions.pop_action() else { + ctx.sleep(POLL_INTERVAL).await?; + continue; + }; + received_actions.push(action); + } + assert_matches!( + received_actions.as_slice(), + [ + SyncAction::Miniblock { + number: MiniblockNumber(2), + timestamp: 2, + virtual_blocks: 1, + }, + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::SealMiniblock, + ] + ); + anyhow::Ok(()) + }) + .await + .unwrap(); + } +} diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index 5ce5549859c8..a4131fd8cbde 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -278,7 +278,7 @@ async fn external_io_basics() { assert_eq!(tx_receipt.transaction_index, 0.into()); } -async fn run_state_keeper_with_multiple_miniblocks(pool: ConnectionPool) -> Vec { +pub(super) async fn run_state_keeper_with_multiple_miniblocks(pool: ConnectionPool) -> Vec { let open_l1_batch = open_l1_batch(1, 1, 1); let txs = (0..5).map(|_| { let tx = create_l2_transaction(10, 100); From 8d59f5cc3010fefaa07dbc732dee07b9d46c3572 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 31 Oct 2023 15:40:24 +0200 Subject: [PATCH 06/38] Test gossip-based fetcher --- .../lib/zksync_core/src/sync_layer/fetcher.rs | 20 +- .../src/sync_layer/gossip/conversions.rs | 25 ++- .../zksync_core/src/sync_layer/gossip/mod.rs | 14 +- .../src/sync_layer/gossip/storage.rs | 98 ++-------- .../src/sync_layer/gossip/tests.rs | 174 +++++++++++++++++- core/lib/zksync_core/src/sync_layer/tests.rs | 104 ++++++----- 6 files changed, 285 insertions(+), 150 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/fetcher.rs b/core/lib/zksync_core/src/sync_layer/fetcher.rs index 6e7c53b016c2..46ac2d7e03a1 100644 --- a/core/lib/zksync_core/src/sync_layer/fetcher.rs +++ b/core/lib/zksync_core/src/sync_layer/fetcher.rs @@ -58,7 +58,7 @@ impl FetchedBlock { #[derive(Debug)] pub struct FetcherCursor { // Fields are public for testing purposes. - pub(super) miniblock: MiniblockNumber, + pub(super) next_miniblock: MiniblockNumber, pub(super) prev_miniblock_hash: H256, pub(super) l1_batch: L1BatchNumber, } @@ -87,7 +87,7 @@ impl FetcherCursor { .context("Failed checking whether pending L1 batch exists")?; // Miniblocks are always fully processed. - let miniblock = last_miniblock_header.number + 1; + let next_miniblock = last_miniblock_header.number + 1; let prev_miniblock_hash = last_miniblock_header.hash; // Decide whether the next batch should be explicitly opened or not. let l1_batch = if was_new_batch_open { @@ -99,13 +99,15 @@ impl FetcherCursor { }; Ok(Self { - miniblock, + next_miniblock, l1_batch, prev_miniblock_hash, }) } pub(super) fn advance(&mut self, block: FetchedBlock) -> Vec { + assert_eq!(block.number, self.next_miniblock); + let mut new_actions = Vec::new(); if block.l1_batch_number != self.l1_batch { assert_eq!( @@ -161,7 +163,7 @@ impl FetcherCursor { } else { new_actions.push(SyncAction::SealMiniblock); } - self.miniblock += 1; + self.next_miniblock += 1; self.prev_miniblock_hash = block.hash; new_actions @@ -199,7 +201,7 @@ impl MainNodeFetcher { pub async fn run(mut self) -> anyhow::Result<()> { tracing::info!( "Starting the fetcher routine. Initial miniblock: {}, initial l1 batch: {}", - self.cursor.miniblock, + self.cursor.next_miniblock, self.cursor.l1_batch ); // Run the main routine and reconnect upon the network errors. @@ -239,7 +241,7 @@ impl MainNodeFetcher { self.sync_state.set_main_node_block(last_main_node_block); self.client - .populate_miniblocks_cache(self.cursor.miniblock, last_main_node_block) + .populate_miniblocks_cache(self.cursor.next_miniblock, last_main_node_block) .await; let has_action_capacity = self.actions.has_action_capacity(); if has_action_capacity { @@ -264,7 +266,11 @@ impl MainNodeFetcher { async fn fetch_next_miniblock(&mut self) -> anyhow::Result { let total_latency = FETCHER_METRICS.fetch_next_miniblock.start(); let request_latency = FETCHER_METRICS.requests[&FetchStage::SyncL2Block].start(); - let Some(block) = self.client.fetch_l2_block(self.cursor.miniblock).await? else { + let Some(block) = self + .client + .fetch_l2_block(self.cursor.next_miniblock) + .await? + else { return Ok(false); }; request_latency.observe(); diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index fdbc5115df6d..e8643d4b03b4 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -4,7 +4,10 @@ use anyhow::Context as _; use rand::{thread_rng, Rng}; use serde::{Deserialize, Serialize}; -use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock, Payload}; +use zksync_consensus_roles::validator::{ + BlockHeader, BlockNumber, CommitQC, FinalBlock, Payload, ReplicaCommit, ViewNumber, + CURRENT_VERSION, +}; use zksync_types::{ api::en::SyncBlock, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, }; @@ -38,14 +41,22 @@ pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> FinalBlock { .expect("Transactions are always requested"), }); let payload = Payload(payload.expect("Failed serializing block payload")); + let header = BlockHeader { + parent: thread_rng().gen(), // FIXME + number: BlockNumber(block.number.0.into()), + payload: payload.hash(), + }; FinalBlock { - header: BlockHeader { - parent: thread_rng().gen(), // FIXME - number: BlockNumber(block.number.0.into()), - payload: payload.hash(), - }, + header, payload, - justification: thread_rng().gen(), // FIXME + justification: CommitQC { + message: ReplicaCommit { + protocol_version: CURRENT_VERSION, + view: ViewNumber(header.number.0), + proposal: header, + }, + ..thread_rng().gen() // FIXME + }, } } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index 00016a17c73e..63bd40c9ec5f 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -26,6 +26,18 @@ pub async fn start_gossip_fetcher( executor_config: ExecutorConfig, node_key: node::SecretKey, ) -> anyhow::Result<()> { + start_gossip_fetcher_inner(&ctx::root(), pool, actions, executor_config, node_key).await +} + +async fn start_gossip_fetcher_inner( + ctx: &ctx::Ctx, + pool: ConnectionPool, + actions: ActionQueueSender, + mut executor_config: ExecutorConfig, + node_key: node::SecretKey, +) -> anyhow::Result<()> { + executor_config.skip_qc_validation = true; + let mut storage = pool .access_storage_tagged("sync_layer") .await @@ -39,7 +51,7 @@ pub async fn start_gossip_fetcher( let executor = Executor::new(executor_config, node_key, buffered_store.clone()) .context("Node executor misconfiguration")?; - scope::run!(&ctx::root(), |ctx, s| async { + scope::run!(ctx, |ctx, s| async { s.spawn_bg(async { store .listen_to_updates(ctx) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index 01bd04ea9032..40734261a9fd 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -31,10 +31,11 @@ pub(super) struct PostgresBlockStore { impl PostgresBlockStore { pub fn new(pool: ConnectionPool, actions: ActionQueueSender, cursor: FetcherCursor) -> Self { + let current_block_number = cursor.next_miniblock.0.saturating_sub(1).into(); Self { pool, actions, - block_sender: watch::channel(BlockNumber(cursor.miniblock.0.into())).0, + block_sender: watch::channel(BlockNumber(current_block_number)).0, cursor: Mutex::new(cursor), } } @@ -182,22 +183,22 @@ impl ContiguousBlockStore for PostgresBlockStore { #[cfg(test)] mod tests { - use assert_matches::assert_matches; - - use zksync_concurrency::{scope, time}; - use zksync_types::{L1BatchNumber, L2ChainId}; + use zksync_concurrency::scope; + use zksync_types::L2ChainId; use super::*; use crate::{ genesis::{ensure_genesis_state, GenesisParams}, sync_layer::{ - sync_action::SyncAction, tests::run_state_keeper_with_multiple_miniblocks, ActionQueue, + gossip::tests::{ + assert_first_block_actions, assert_second_block_actions, load_final_block, + TEST_TIMEOUT, + }, + tests::run_state_keeper_with_multiple_miniblocks, + ActionQueue, }, }; - const TEST_TIMEOUT: time::Duration = time::Duration::seconds(1); - const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); - #[tokio::test] async fn block_store_basics_for_postgres() { let pool = ConnectionPool::test_pool().await; @@ -276,20 +277,8 @@ mod tests { run_state_keeper_with_multiple_miniblocks(pool.clone()).await; let mut storage = pool.access_storage().await.unwrap(); - let first_block = storage - .sync_dal() - .sync_block(MiniblockNumber(1), Address::repeat_byte(1), true) - .await - .unwrap() - .expect("no sync block #1"); - let first_block = sync_block_to_consensus_block(first_block); - let second_block = storage - .sync_dal() - .sync_block(MiniblockNumber(2), Address::repeat_byte(1), true) - .await - .unwrap() - .expect("no sync block #2"); - let second_block = sync_block_to_consensus_block(second_block); + let first_block = load_final_block(&mut storage, 1).await; + let second_block = load_final_block(&mut storage, 2).await; storage .transactions_dal() .reset_transactions_state(MiniblockNumber(0)) @@ -306,66 +295,11 @@ mod tests { let storage = PostgresBlockStore::new(pool.clone(), actions_sender, cursor); let ctx = &ctx::test_root(&ctx::RealClock); storage.schedule_block(ctx, &first_block).await.unwrap(); - - scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, _| async { - let mut received_actions = vec![]; - while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { - let Some(action) = actions.pop_action() else { - ctx.sleep(POLL_INTERVAL).await?; - continue; - }; - received_actions.push(action); - } - assert_matches!( - received_actions.as_slice(), - [ - SyncAction::OpenBatch { - number: L1BatchNumber(1), - timestamp: 1, - first_miniblock_info: (MiniblockNumber(1), 1), - .. - }, - SyncAction::Tx(_), - SyncAction::Tx(_), - SyncAction::Tx(_), - SyncAction::Tx(_), - SyncAction::Tx(_), - SyncAction::SealMiniblock, - ] - ); - anyhow::Ok(()) - }) - .await - .unwrap(); + assert_first_block_actions(ctx, &mut actions).await.unwrap(); storage.schedule_block(ctx, &second_block).await.unwrap(); - - scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, _| async { - let mut received_actions = vec![]; - while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { - let Some(action) = actions.pop_action() else { - ctx.sleep(POLL_INTERVAL).await?; - continue; - }; - received_actions.push(action); - } - assert_matches!( - received_actions.as_slice(), - [ - SyncAction::Miniblock { - number: MiniblockNumber(2), - timestamp: 2, - virtual_blocks: 1, - }, - SyncAction::Tx(_), - SyncAction::Tx(_), - SyncAction::Tx(_), - SyncAction::SealMiniblock, - ] - ); - anyhow::Ok(()) - }) - .await - .unwrap(); + assert_second_block_actions(ctx, &mut actions) + .await + .unwrap(); } } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index f80be920f54f..88db5ccf9270 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use rand::{rngs::StdRng, seq::SliceRandom, Rng}; use test_casing::test_casing; -use std::{iter, ops}; +use std::{future::Future, iter, ops}; use zksync_concurrency::{ ctx::{self, channel}, @@ -17,8 +17,21 @@ use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock, Pa use zksync_consensus_storage::{ BlockStore, InMemoryStorage, StorageError, StorageResult, WriteBlockStore, }; +use zksync_dal::{ConnectionPool, StorageProcessor}; +use zksync_executor::testonly::FullValidatorConfig; +use zksync_types::{Address, L1BatchNumber, MiniblockNumber}; -use super::buffered::{BufferedStorage, BufferedStorageEvent, ContiguousBlockStore}; +use super::{ + buffered::{BufferedStorage, BufferedStorageEvent, ContiguousBlockStore}, + *, +}; +use crate::sync_layer::tests::StateKeeperHandles; +use crate::sync_layer::{ + sync_action::SyncAction, tests::run_state_keeper_with_multiple_miniblocks, ActionQueue, +}; + +pub(super) const TEST_TIMEOUT: time::Duration = time::Duration::seconds(1); +pub(super) const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); fn init_store(rng: &mut impl Rng) -> (FinalBlock, InMemoryStorage) { let payload = Payload(vec![]); @@ -48,6 +61,20 @@ fn gen_blocks(rng: &mut impl Rng, genesis_block: FinalBlock, count: usize) -> Ve blocks.skip(1).take(count).collect() } +/// Loads a block from the storage and converts it to a `FinalBlock`. +pub(super) async fn load_final_block( + storage: &mut StorageProcessor<'_>, + number: u32, +) -> FinalBlock { + let sync_block = storage + .sync_dal() + .sync_block(MiniblockNumber(number), Address::repeat_byte(1), true) + .await + .unwrap() + .unwrap_or_else(|| panic!("no sync block #{number}")); + conversions::sync_block_to_consensus_block(sync_block) +} + #[derive(Debug)] struct MockContiguousStore { inner: InMemoryStorage, @@ -293,3 +320,146 @@ async fn buffered_storage_with_initial_blocks_and_slight_shuffling(block_interva }) .await; } + +pub(super) async fn assert_first_block_actions( + ctx: &ctx::Ctx, + actions: &mut ActionQueue, +) -> ctx::OrCanceled> { + let mut received_actions = vec![]; + while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { + let Some(action) = actions.pop_action() else { + ctx.sleep(POLL_INTERVAL).await?; + continue; + }; + received_actions.push(action); + } + assert_matches!( + received_actions.as_slice(), + [ + SyncAction::OpenBatch { + number: L1BatchNumber(1), + timestamp: 1, + first_miniblock_info: (MiniblockNumber(1), 1), + .. + }, + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::SealMiniblock, + ] + ); + Ok(received_actions) +} + +pub(super) async fn assert_second_block_actions( + ctx: &ctx::Ctx, + actions: &mut ActionQueue, +) -> ctx::OrCanceled> { + let mut received_actions = vec![]; + while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { + let Some(action) = actions.pop_action() else { + ctx.sleep(POLL_INTERVAL).await?; + continue; + }; + received_actions.push(action); + } + assert_matches!( + received_actions.as_slice(), + [ + SyncAction::Miniblock { + number: MiniblockNumber(2), + timestamp: 2, + virtual_blocks: 1, + }, + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::Tx(_), + SyncAction::SealMiniblock, + ] + ); + Ok(received_actions) +} + +async fn wrap_bg_task(task: impl Future>) -> anyhow::Result<()> { + match task.await { + Ok(()) => Ok(()), + Err(err) if err.root_cause().is::() => Ok(()), + Err(err) => Err(err), + } +} + +#[tokio::test] +async fn syncing_via_gossip_fetcher() { + zksync_concurrency::testonly::abort_on_panic(); + let pool = ConnectionPool::test_pool().await; + let tx_hashes = run_state_keeper_with_multiple_miniblocks(pool.clone()).await; + + let mut storage = pool.access_storage().await.unwrap(); + let genesis_block = load_final_block(&mut storage, 0).await; + let first_block = load_final_block(&mut storage, 1).await; + let second_block = load_final_block(&mut storage, 2).await; + storage + .transactions_dal() + .reset_transactions_state(MiniblockNumber(0)) + .await; + storage + .blocks_dal() + .delete_miniblocks(MiniblockNumber(0)) + .await + .unwrap(); + drop(storage); + + let ctx = &ctx::test_root(&ctx::AffineClock::new(20.0)); + let rng = &mut ctx.rng(); + let mut validator = + FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()); + let external_node = validator.connect_external_node(rng); + + let validator_storage = Arc::new(InMemoryStorage::new(genesis_block)); + validator_storage + .put_block(ctx, &first_block) + .await + .unwrap(); + validator_storage + .put_block(ctx, &second_block) + .await + .unwrap(); + let validator = + Executor::new(validator.node_config, validator.node_key, validator_storage).unwrap(); + // ^ We intentionally do not run consensus on the validator node, since it'll produce blocks + // with payloads that cannot be parsed by the external node. + + let (actions_sender, mut actions) = ActionQueue::new(); + let state_keeper = StateKeeperHandles::new(pool.clone(), &[&tx_hashes]).await; + scope::run!(ctx, |ctx, s| async { + s.spawn_bg(wrap_bg_task(validator.run(ctx))); + s.spawn_bg(wrap_bg_task(start_gossip_fetcher_inner( + ctx, + pool, + actions_sender, + external_node.node_config, + external_node.node_key, + ))); + + let received_actions = assert_first_block_actions(ctx, &mut actions).await?; + // Manually replicate actions to the state keeper. + state_keeper + .actions_sender + .push_actions(received_actions) + .await; + + let received_actions = assert_second_block_actions(ctx, &mut actions).await?; + state_keeper + .actions_sender + .push_actions(received_actions) + .await; + state_keeper + .wait(|state| state.get_local_block() == MiniblockNumber(2)) + .await; + Ok(()) + }) + .await + .unwrap(); +} diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index a4131fd8cbde..6960d01caa4f 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -145,15 +145,55 @@ fn open_l1_batch(number: u32, timestamp: u64, first_miniblock_number: u32) -> Sy } #[derive(Debug)] -struct StateKeeperHandles { - actions_sender: ActionQueueSender, - stop_sender: watch::Sender, - sync_state: SyncState, - task: JoinHandle>, +pub(super) struct StateKeeperHandles { + pub actions_sender: ActionQueueSender, + pub stop_sender: watch::Sender, + pub sync_state: SyncState, + pub task: JoinHandle>, } impl StateKeeperHandles { - async fn wait(self, mut condition: impl FnMut(&SyncState) -> bool) { + /// `tx_hashes` are grouped by the L1 batch. + pub async fn new(pool: ConnectionPool, tx_hashes: &[&[H256]]) -> Self { + assert!(!tx_hashes.is_empty()); + assert!(tx_hashes.iter().all(|tx_hashes| !tx_hashes.is_empty())); + + ensure_genesis(&mut pool.access_storage().await.unwrap()).await; + + let (actions_sender, actions) = ActionQueue::new(); + let sync_state = SyncState::new(); + let io = ExternalIO::new( + pool, + actions, + sync_state.clone(), + Box::::default(), + Address::repeat_byte(1), + u32::MAX, + L2ChainId::default(), + ) + .await; + + let (stop_sender, stop_receiver) = watch::channel(false); + let mut batch_executor_base = TestBatchExecutorBuilder::default(); + for &tx_hashes_in_l1_batch in tx_hashes { + batch_executor_base.push_successful_transactions(tx_hashes_in_l1_batch); + } + + let state_keeper = ZkSyncStateKeeper::without_sealer( + stop_receiver, + Box::new(io), + Box::new(batch_executor_base), + ); + Self { + actions_sender, + stop_sender, + sync_state, + task: tokio::spawn(state_keeper.run()), + } + } + + /// Waits for the given condition. + pub async fn wait(self, mut condition: impl FnMut(&SyncState) -> bool) { let started_at = Instant::now(); loop { assert!( @@ -186,45 +226,6 @@ async fn ensure_genesis(storage: &mut StorageProcessor<'_>) { } } -/// `tx_hashes` are grouped by the L1 batch. -async fn run_state_keeper(pool: ConnectionPool, tx_hashes: &[&[H256]]) -> StateKeeperHandles { - assert!(!tx_hashes.is_empty()); - assert!(tx_hashes.iter().all(|tx_hashes| !tx_hashes.is_empty())); - - ensure_genesis(&mut pool.access_storage().await.unwrap()).await; - - let (actions_sender, actions) = ActionQueue::new(); - let sync_state = SyncState::new(); - let io = ExternalIO::new( - pool, - actions, - sync_state.clone(), - Box::::default(), - Address::repeat_byte(1), - u32::MAX, - L2ChainId::default(), - ) - .await; - - let (stop_sender, stop_receiver) = watch::channel(false); - let mut batch_executor_base = TestBatchExecutorBuilder::default(); - for &tx_hashes_in_l1_batch in tx_hashes { - batch_executor_base.push_successful_transactions(tx_hashes_in_l1_batch); - } - - let state_keeper = ZkSyncStateKeeper::without_sealer( - stop_receiver, - Box::new(io), - Box::new(batch_executor_base), - ); - StateKeeperHandles { - actions_sender, - stop_sender, - sync_state, - task: tokio::spawn(state_keeper.run()), - } -} - fn extract_tx_hashes<'a>(actions: impl IntoIterator) -> Vec { actions .into_iter() @@ -247,7 +248,7 @@ async fn external_io_basics() { let tx = SyncAction::Tx(Box::new(tx.into())); let actions = vec![open_l1_batch, tx, SyncAction::SealMiniblock]; - let state_keeper = run_state_keeper(pool.clone(), &[&extract_tx_hashes(&actions)]).await; + let state_keeper = StateKeeperHandles::new(pool.clone(), &[&extract_tx_hashes(&actions)]).await; state_keeper.actions_sender.push_actions(actions).await; // Wait until the miniblock is sealed. state_keeper @@ -308,7 +309,7 @@ pub(super) async fn run_state_keeper_with_multiple_miniblocks(pool: ConnectionPo .iter() .chain(&second_miniblock_actions), ); - let state_keeper = run_state_keeper(pool, &[&tx_hashes]).await; + let state_keeper = StateKeeperHandles::new(pool, &[&tx_hashes]).await; state_keeper .actions_sender .push_actions(first_miniblock_actions) @@ -365,7 +366,7 @@ async fn test_external_io_recovery(pool: ConnectionPool, mut tx_hashes: Vec::json_rpc(&format!("http://{server_addr}/")).unwrap(); let fetcher_cursor = FetcherCursor { - miniblock: MiniblockNumber(1), + next_miniblock: MiniblockNumber(1), prev_miniblock_hash: genesis_miniblock_hash, l1_batch: L1BatchNumber(0), }; From 6e3127f94a440b5ff116e5bf757c918fa5c45ccc Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 1 Nov 2023 12:41:30 +0200 Subject: [PATCH 07/38] Refactor and add more tests for gossip fetcher --- .../gossip/{buffered.rs => buffered/mod.rs} | 10 +- .../src/sync_layer/gossip/buffered/tests.rs | 295 +++++++++++ .../zksync_core/src/sync_layer/gossip/mod.rs | 2 +- .../src/sync_layer/gossip/storage.rs | 3 +- .../src/sync_layer/gossip/tests.rs | 489 ++++++------------ core/lib/zksync_core/src/sync_layer/tests.rs | 76 ++- 6 files changed, 498 insertions(+), 377 deletions(-) rename core/lib/zksync_core/src/sync_layer/gossip/{buffered.rs => buffered/mod.rs} (98%) create mode 100644 core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs similarity index 98% rename from core/lib/zksync_core/src/sync_layer/gossip/buffered.rs rename to core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index 1b6e0877319d..21acde98c423 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -13,6 +13,9 @@ use zksync_concurrency::{ use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; use zksync_consensus_storage::{BlockStore, StorageError, StorageResult, WriteBlockStore}; +#[cfg(test)] +mod tests; + use super::utils::MissingBlockNumbers; /// [`BlockStore`] variation that upholds additional invariants as to how blocks are processed. @@ -154,10 +157,7 @@ impl BufferedStorage { } #[cfg(test)] - pub(super) fn set_events_sender( - &mut self, - sender: channel::UnboundedSender, - ) { + fn set_events_sender(&mut self, sender: channel::UnboundedSender) { self.events_sender = sender; } @@ -166,7 +166,7 @@ impl BufferedStorage { } #[cfg(test)] - pub(crate) async fn buffer_len(&self) -> usize { + async fn buffer_len(&self) -> usize { self.buffer.lock().await.blocks.len() } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs new file mode 100644 index 000000000000..9b974c49a8ae --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs @@ -0,0 +1,295 @@ +//! Tests for buffered storage. + +use assert_matches::assert_matches; +use async_trait::async_trait; +use rand::{rngs::StdRng, seq::SliceRandom, Rng}; +use test_casing::test_casing; + +use std::{iter, ops}; + +use zksync_concurrency::{ + ctx::{self, channel}, + scope, + sync::{self, watch}, + time, +}; +use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock, Payload}; +use zksync_consensus_storage::{ + BlockStore, InMemoryStorage, StorageError, StorageResult, WriteBlockStore, +}; + +use super::*; + +fn init_store(rng: &mut impl Rng) -> (FinalBlock, InMemoryStorage) { + let payload = Payload(vec![]); + let genesis_block = FinalBlock { + header: BlockHeader::genesis(payload.hash()), + payload, + justification: rng.gen(), + }; + let block_store = InMemoryStorage::new(genesis_block.clone()); + (genesis_block, block_store) +} + +fn gen_blocks(rng: &mut impl Rng, genesis_block: FinalBlock, count: usize) -> Vec { + let blocks = iter::successors(Some(genesis_block), |parent| { + let payload = Payload(vec![]); + let header = BlockHeader { + parent: parent.header.hash(), + number: parent.header.number.next(), + payload: payload.hash(), + }; + Some(FinalBlock { + header, + payload, + justification: rng.gen(), + }) + }); + blocks.skip(1).take(count).collect() +} + +#[derive(Debug)] +struct MockContiguousStore { + inner: InMemoryStorage, + block_sender: channel::Sender, +} + +impl MockContiguousStore { + fn new(inner: InMemoryStorage) -> (Self, channel::Receiver) { + let (block_sender, block_receiver) = channel::bounded(1); + let this = Self { + inner, + block_sender, + }; + (this, block_receiver) + } + + async fn run_updates( + &self, + ctx: &ctx::Ctx, + mut block_receiver: channel::Receiver, + ) -> StorageResult<()> { + let rng = &mut ctx.rng(); + while let Ok(block) = block_receiver.recv(ctx).await { + let sleep_duration = time::Duration::milliseconds(rng.gen_range(0..5)); + ctx.sleep(sleep_duration).await?; + self.inner.put_block(ctx, &block).await?; + } + Ok(()) + } +} + +#[async_trait] +impl BlockStore for MockContiguousStore { + async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { + self.inner.head_block(ctx).await + } + + async fn first_block(&self, ctx: &ctx::Ctx) -> StorageResult { + self.inner.first_block(ctx).await + } + + async fn last_contiguous_block_number(&self, ctx: &ctx::Ctx) -> StorageResult { + self.inner.last_contiguous_block_number(ctx).await + } + + async fn block( + &self, + ctx: &ctx::Ctx, + number: BlockNumber, + ) -> StorageResult> { + self.inner.block(ctx, number).await + } + + async fn missing_block_numbers( + &self, + ctx: &ctx::Ctx, + range: ops::Range, + ) -> StorageResult> { + self.inner.missing_block_numbers(ctx, range).await + } + + fn subscribe_to_block_writes(&self) -> watch::Receiver { + self.inner.subscribe_to_block_writes() + } +} + +#[async_trait] +impl ContiguousBlockStore for MockContiguousStore { + async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + let head_block_number = self.head_block(ctx).await?.header.number; + assert_eq!(block.header.number, head_block_number.next()); + self.block_sender + .try_send(block.clone()) + .expect("BufferedStorage is rushing"); + Ok(()) + } +} + +#[tracing::instrument(level = "trace", skip(shuffle_blocks))] +async fn test_buffered_storage( + initial_block_count: usize, + block_count: usize, + block_interval: time::Duration, + shuffle_blocks: impl FnOnce(&mut StdRng, &mut [FinalBlock]), +) { + let ctx = &ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + let (genesis_block, block_store) = init_store(rng); + let mut initial_blocks = gen_blocks(rng, genesis_block.clone(), initial_block_count); + for block in &initial_blocks { + block_store.put_block(ctx, block).await.unwrap(); + } + initial_blocks.insert(0, genesis_block.clone()); + + let (block_store, block_receiver) = MockContiguousStore::new(block_store); + let mut buffered_store = BufferedStorage::new(block_store); + let (events_sender, mut events_receiver) = channel::unbounded(); + buffered_store.set_events_sender(events_sender); + + // Check initial values returned by the store. + let last_initial_block = initial_blocks.last().unwrap().clone(); + assert_eq!( + buffered_store.head_block(ctx).await.unwrap(), + last_initial_block + ); + for block in &initial_blocks { + let block_result = buffered_store.block(ctx, block.header.number).await; + assert_eq!(block_result.unwrap().as_ref(), Some(block)); + } + let mut subscriber = buffered_store.subscribe_to_block_writes(); + assert_eq!( + *subscriber.borrow(), + BlockNumber(initial_block_count as u64) + ); + + let mut blocks = gen_blocks(rng, last_initial_block, block_count); + shuffle_blocks(rng, &mut blocks); + let last_block_number = BlockNumber((block_count + initial_block_count) as u64); + + scope::run!(ctx, |ctx, s| async { + s.spawn_bg(buffered_store.inner().run_updates(ctx, block_receiver)); + s.spawn_bg(async { + let err = buffered_store.listen_to_updates(ctx).await.unwrap_err(); + match &err { + StorageError::Canceled(_) => Ok(()), // Test has successfully finished + StorageError::Database(_) => Err(err), + } + }); + + for (idx, block) in blocks.iter().enumerate() { + buffered_store.put_block(ctx, block).await?; + let new_block_number = *sync::changed(ctx, &mut subscriber).await?; + assert_eq!(new_block_number, block.header.number); + + // Check that all written blocks are immediately accessible. + for existing_block in initial_blocks.iter().chain(&blocks[0..=idx]) { + let number = existing_block.header.number; + assert_eq!( + buffered_store.block(ctx, number).await?.as_ref(), + Some(existing_block) + ); + } + assert_eq!(buffered_store.first_block(ctx).await?, genesis_block); + + let expected_head_block = blocks[0..=idx] + .iter() + .max_by_key(|block| block.header.number) + .unwrap(); + assert_eq!(buffered_store.head_block(ctx).await?, *expected_head_block); + + let expected_last_contiguous_block = blocks[(idx + 1)..] + .iter() + .map(|block| block.header.number) + .min() + .map_or(last_block_number, BlockNumber::prev); + assert_eq!( + buffered_store.last_contiguous_block_number(ctx).await?, + expected_last_contiguous_block + ); + + ctx.sleep(block_interval).await?; + } + + let mut inner_subscriber = buffered_store.inner().subscribe_to_block_writes(); + while buffered_store + .inner() + .last_contiguous_block_number(ctx) + .await? + < last_block_number + { + sync::changed(ctx, &mut inner_subscriber).await?; + } + + // Check events emitted by the buffered storage. This also ensures that all underlying storage + // updates are processed before proceeding to the following checks. + let expected_numbers = (initial_block_count as u64 + 1)..=last_block_number.0; + for expected_number in expected_numbers.map(BlockNumber) { + assert_matches!( + events_receiver.recv(ctx).await?, + BufferedStorageEvent::UpdateReceived(number) if number == expected_number + ); + } + + assert_eq!(buffered_store.buffer_len().await, 0); + Ok(()) + }) + .await + .unwrap(); +} + +// Choose intervals so that they are both smaller and larger than the sleep duration in +// `MockContiguousStore::run_updates()`. +const BLOCK_INTERVALS: [time::Duration; 4] = [ + time::Duration::ZERO, + time::Duration::milliseconds(3), + time::Duration::milliseconds(5), + time::Duration::milliseconds(10), +]; + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_sequential_blocks(block_interval: time::Duration) { + test_buffered_storage(0, 30, block_interval, |_, _| { + // Do not perform shuffling + }) + .await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_random_blocks(block_interval: time::Duration) { + test_buffered_storage(0, 30, block_interval, |rng, blocks| blocks.shuffle(rng)).await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_slightly_shuffled_blocks(block_interval: time::Duration) { + test_buffered_storage(0, 30, block_interval, |rng, blocks| { + for chunk in blocks.chunks_mut(4) { + chunk.shuffle(rng); + } + }) + .await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_initial_blocks(block_interval: time::Duration) { + test_buffered_storage(10, 20, block_interval, |_, _| { + // Do not perform shuffling + }) + .await; +} + +#[test_casing(4, BLOCK_INTERVALS)] +#[tokio::test] +async fn buffered_storage_with_initial_blocks_and_slight_shuffling(block_interval: time::Duration) { + test_buffered_storage(10, 20, block_interval, |rng, blocks| { + for chunk in blocks.chunks_mut(5) { + chunk.shuffle(rng); + } + }) + .await; +} diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index 63bd40c9ec5f..d8282eacf7e7 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -5,9 +5,9 @@ use anyhow::Context as _; use std::sync::Arc; use zksync_concurrency::{ctx, scope}; +use zksync_consensus_executor::{Executor, ExecutorConfig}; use zksync_consensus_roles::node; use zksync_dal::ConnectionPool; -use zksync_executor::{Executor, ExecutorConfig}; mod buffered; mod conversions; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index 40734261a9fd..a91427c5c15b 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -192,13 +192,14 @@ mod tests { sync_layer::{ gossip::tests::{ assert_first_block_actions, assert_second_block_actions, load_final_block, - TEST_TIMEOUT, }, tests::run_state_keeper_with_multiple_miniblocks, ActionQueue, }, }; + const TEST_TIMEOUT: time::Duration = time::Duration::seconds(1); + #[tokio::test] async fn block_store_basics_for_postgres() { let pool = ConnectionPool::test_pool().await; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 88db5ccf9270..d5eafa421411 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -1,65 +1,29 @@ //! Tests for consensus adapters for EN synchronization logic. use assert_matches::assert_matches; -use async_trait::async_trait; -use rand::{rngs::StdRng, seq::SliceRandom, Rng}; -use test_casing::test_casing; +use test_casing::{test_casing, Product}; -use std::{future::Future, iter, ops}; +use std::future::Future; -use zksync_concurrency::{ - ctx::{self, channel}, - scope, - sync::{self, watch}, - time, -}; -use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock, Payload}; -use zksync_consensus_storage::{ - BlockStore, InMemoryStorage, StorageError, StorageResult, WriteBlockStore, -}; +use zksync_concurrency::{ctx, scope, time}; +use zksync_consensus_executor::testonly::FullValidatorConfig; +use zksync_consensus_roles::validator::FinalBlock; +use zksync_consensus_storage::{InMemoryStorage, WriteBlockStore}; use zksync_dal::{ConnectionPool, StorageProcessor}; -use zksync_executor::testonly::FullValidatorConfig; use zksync_types::{Address, L1BatchNumber, MiniblockNumber}; -use super::{ - buffered::{BufferedStorage, BufferedStorageEvent, ContiguousBlockStore}, - *, -}; -use crate::sync_layer::tests::StateKeeperHandles; +use super::*; use crate::sync_layer::{ - sync_action::SyncAction, tests::run_state_keeper_with_multiple_miniblocks, ActionQueue, + sync_action::SyncAction, + tests::{ + mock_l1_batch_hash_computation, run_state_keeper_with_multiple_l1_batches, + run_state_keeper_with_multiple_miniblocks, StateKeeperHandles, + }, + ActionQueue, }; -pub(super) const TEST_TIMEOUT: time::Duration = time::Duration::seconds(1); -pub(super) const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); - -fn init_store(rng: &mut impl Rng) -> (FinalBlock, InMemoryStorage) { - let payload = Payload(vec![]); - let genesis_block = FinalBlock { - header: BlockHeader::genesis(payload.hash()), - payload, - justification: rng.gen(), - }; - let block_store = InMemoryStorage::new(genesis_block.clone()); - (genesis_block, block_store) -} - -fn gen_blocks(rng: &mut impl Rng, genesis_block: FinalBlock, count: usize) -> Vec { - let blocks = iter::successors(Some(genesis_block), |parent| { - let payload = Payload(vec![]); - let header = BlockHeader { - parent: parent.header.hash(), - number: parent.header.number.next(), - payload: payload.hash(), - }; - Some(FinalBlock { - header, - payload, - justification: rng.gen(), - }) - }); - blocks.skip(1).take(count).collect() -} +const CLOCK_SPEEDUP: i64 = 20; +const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50 * CLOCK_SPEEDUP); /// Loads a block from the storage and converts it to a `FinalBlock`. pub(super) async fn load_final_block( @@ -75,252 +39,6 @@ pub(super) async fn load_final_block( conversions::sync_block_to_consensus_block(sync_block) } -#[derive(Debug)] -struct MockContiguousStore { - inner: InMemoryStorage, - block_sender: channel::Sender, -} - -impl MockContiguousStore { - fn new(inner: InMemoryStorage) -> (Self, channel::Receiver) { - let (block_sender, block_receiver) = channel::bounded(1); - let this = Self { - inner, - block_sender, - }; - (this, block_receiver) - } - - async fn run_updates( - &self, - ctx: &ctx::Ctx, - mut block_receiver: channel::Receiver, - ) -> StorageResult<()> { - let rng = &mut ctx.rng(); - while let Ok(block) = block_receiver.recv(ctx).await { - let sleep_duration = time::Duration::milliseconds(rng.gen_range(0..5)); - ctx.sleep(sleep_duration).await?; - self.inner.put_block(ctx, &block).await?; - } - Ok(()) - } -} - -#[async_trait] -impl BlockStore for MockContiguousStore { - async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { - self.inner.head_block(ctx).await - } - - async fn first_block(&self, ctx: &ctx::Ctx) -> StorageResult { - self.inner.first_block(ctx).await - } - - async fn last_contiguous_block_number(&self, ctx: &ctx::Ctx) -> StorageResult { - self.inner.last_contiguous_block_number(ctx).await - } - - async fn block( - &self, - ctx: &ctx::Ctx, - number: BlockNumber, - ) -> StorageResult> { - self.inner.block(ctx, number).await - } - - async fn missing_block_numbers( - &self, - ctx: &ctx::Ctx, - range: ops::Range, - ) -> StorageResult> { - self.inner.missing_block_numbers(ctx, range).await - } - - fn subscribe_to_block_writes(&self) -> watch::Receiver { - self.inner.subscribe_to_block_writes() - } -} - -#[async_trait] -impl ContiguousBlockStore for MockContiguousStore { - async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { - let head_block_number = self.head_block(ctx).await?.header.number; - assert_eq!(block.header.number, head_block_number.next()); - self.block_sender - .try_send(block.clone()) - .expect("BufferedStorage is rushing"); - Ok(()) - } -} - -#[tracing::instrument(level = "trace", skip(shuffle_blocks))] -async fn test_buffered_storage( - initial_block_count: usize, - block_count: usize, - block_interval: time::Duration, - shuffle_blocks: impl FnOnce(&mut StdRng, &mut [FinalBlock]), -) { - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - let (genesis_block, block_store) = init_store(rng); - let mut initial_blocks = gen_blocks(rng, genesis_block.clone(), initial_block_count); - for block in &initial_blocks { - block_store.put_block(ctx, block).await.unwrap(); - } - initial_blocks.insert(0, genesis_block.clone()); - - let (block_store, block_receiver) = MockContiguousStore::new(block_store); - let mut buffered_store = BufferedStorage::new(block_store); - let (events_sender, mut events_receiver) = channel::unbounded(); - buffered_store.set_events_sender(events_sender); - - // Check initial values returned by the store. - let last_initial_block = initial_blocks.last().unwrap().clone(); - assert_eq!( - buffered_store.head_block(ctx).await.unwrap(), - last_initial_block - ); - for block in &initial_blocks { - let block_result = buffered_store.block(ctx, block.header.number).await; - assert_eq!(block_result.unwrap().as_ref(), Some(block)); - } - let mut subscriber = buffered_store.subscribe_to_block_writes(); - assert_eq!( - *subscriber.borrow(), - BlockNumber(initial_block_count as u64) - ); - - let mut blocks = gen_blocks(rng, last_initial_block, block_count); - shuffle_blocks(rng, &mut blocks); - let last_block_number = BlockNumber((block_count + initial_block_count) as u64); - - scope::run!(ctx, |ctx, s| async { - s.spawn_bg(buffered_store.inner().run_updates(ctx, block_receiver)); - s.spawn_bg(async { - let err = buffered_store.listen_to_updates(ctx).await.unwrap_err(); - match &err { - StorageError::Canceled(_) => Ok(()), // Test has successfully finished - StorageError::Database(_) => Err(err), - } - }); - - for (idx, block) in blocks.iter().enumerate() { - buffered_store.put_block(ctx, block).await?; - let new_block_number = *sync::changed(ctx, &mut subscriber).await?; - assert_eq!(new_block_number, block.header.number); - - // Check that all written blocks are immediately accessible. - for existing_block in initial_blocks.iter().chain(&blocks[0..=idx]) { - let number = existing_block.header.number; - assert_eq!( - buffered_store.block(ctx, number).await?.as_ref(), - Some(existing_block) - ); - } - assert_eq!(buffered_store.first_block(ctx).await?, genesis_block); - - let expected_head_block = blocks[0..=idx] - .iter() - .max_by_key(|block| block.header.number) - .unwrap(); - assert_eq!(buffered_store.head_block(ctx).await?, *expected_head_block); - - let expected_last_contiguous_block = blocks[(idx + 1)..] - .iter() - .map(|block| block.header.number) - .min() - .map_or(last_block_number, BlockNumber::prev); - assert_eq!( - buffered_store.last_contiguous_block_number(ctx).await?, - expected_last_contiguous_block - ); - - ctx.sleep(block_interval).await?; - } - - let mut inner_subscriber = buffered_store.inner().subscribe_to_block_writes(); - while buffered_store - .inner() - .last_contiguous_block_number(ctx) - .await? - < last_block_number - { - sync::changed(ctx, &mut inner_subscriber).await?; - } - - // Check events emitted by the buffered storage. This also ensures that all underlying storage - // updates are processed before proceeding to the following checks. - let expected_numbers = (initial_block_count as u64 + 1)..=last_block_number.0; - for expected_number in expected_numbers.map(BlockNumber) { - assert_matches!( - events_receiver.recv(ctx).await?, - BufferedStorageEvent::UpdateReceived(number) if number == expected_number - ); - } - - assert_eq!(buffered_store.buffer_len().await, 0); - Ok(()) - }) - .await - .unwrap(); -} - -// Choose intervals so that they are both smaller and larger than the sleep duration in -// `MockContiguousStore::run_updates()`. -const BLOCK_INTERVALS: [time::Duration; 4] = [ - time::Duration::ZERO, - time::Duration::milliseconds(3), - time::Duration::milliseconds(5), - time::Duration::milliseconds(10), -]; - -#[test_casing(4, BLOCK_INTERVALS)] -#[tokio::test] -async fn buffered_storage_with_sequential_blocks(block_interval: time::Duration) { - test_buffered_storage(0, 30, block_interval, |_, _| { - // Do not perform shuffling - }) - .await; -} - -#[test_casing(4, BLOCK_INTERVALS)] -#[tokio::test] -async fn buffered_storage_with_random_blocks(block_interval: time::Duration) { - test_buffered_storage(0, 30, block_interval, |rng, blocks| blocks.shuffle(rng)).await; -} - -#[test_casing(4, BLOCK_INTERVALS)] -#[tokio::test] -async fn buffered_storage_with_slightly_shuffled_blocks(block_interval: time::Duration) { - test_buffered_storage(0, 30, block_interval, |rng, blocks| { - for chunk in blocks.chunks_mut(4) { - chunk.shuffle(rng); - } - }) - .await; -} - -#[test_casing(4, BLOCK_INTERVALS)] -#[tokio::test] -async fn buffered_storage_with_initial_blocks(block_interval: time::Duration) { - test_buffered_storage(10, 20, block_interval, |_, _| { - // Do not perform shuffling - }) - .await; -} - -#[test_casing(4, BLOCK_INTERVALS)] -#[tokio::test] -async fn buffered_storage_with_initial_blocks_and_slight_shuffling(block_interval: time::Duration) { - test_buffered_storage(10, 20, block_interval, |rng, blocks| { - for chunk in blocks.chunks_mut(5) { - chunk.shuffle(rng); - } - }) - .await; -} - pub(super) async fn assert_first_block_actions( ctx: &ctx::Ctx, actions: &mut ActionQueue, @@ -382,6 +100,7 @@ pub(super) async fn assert_second_block_actions( Ok(received_actions) } +/// Wraps a background task so that it returns `Ok(())` if it's canceled. async fn wrap_bg_task(task: impl Future>) -> anyhow::Result<()> { match task.await { Ok(()) => Ok(()), @@ -390,16 +109,106 @@ async fn wrap_bg_task(task: impl Future>) -> anyhow: } } +#[test_casing(4, Product(([false, true], [false, true])))] #[tokio::test] -async fn syncing_via_gossip_fetcher() { +async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: bool) { zksync_concurrency::testonly::abort_on_panic(); let pool = ConnectionPool::test_pool().await; let tx_hashes = run_state_keeper_with_multiple_miniblocks(pool.clone()).await; - let mut storage = pool.access_storage().await.unwrap(); + let storage = pool.access_storage().await.unwrap(); + let (genesis_block, blocks) = get_blocks_and_reset_storage(storage).await; + let [first_block, second_block] = blocks.as_slice() else { + unreachable!("Unexpected blocks in storage: {blocks:?}"); + }; + tracing::trace!("Node storage reset"); + + let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); + let rng = &mut ctx.rng(); + let mut validator = + FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()); + let external_node = validator.connect_external_node(rng); + + let validator_storage = Arc::new(InMemoryStorage::new(genesis_block)); + if !delay_first_block { + validator_storage.put_block(ctx, first_block).await.unwrap(); + if !delay_second_block { + validator_storage + .put_block(ctx, second_block) + .await + .unwrap(); + } + } + let validator = Executor::new( + validator.node_config, + validator.node_key, + validator_storage.clone(), + ) + .unwrap(); + // ^ We intentionally do not run consensus on the validator node, since it'll produce blocks + // with payloads that cannot be parsed by the external node. + + let (actions_sender, mut actions) = ActionQueue::new(); + let (keeper_actions_sender, keeper_actions) = ActionQueue::new(); + let state_keeper = StateKeeperHandles::new(pool.clone(), keeper_actions, &[&tx_hashes]).await; + scope::run!(ctx, |ctx, s| async { + s.spawn_bg(wrap_bg_task(validator.run(ctx))); + s.spawn_bg(wrap_bg_task(start_gossip_fetcher_inner( + ctx, + pool, + actions_sender, + external_node.node_config, + external_node.node_key, + ))); + + if delay_first_block { + ctx.sleep(POLL_INTERVAL).await?; + validator_storage.put_block(ctx, first_block).await.unwrap(); + if !delay_second_block { + validator_storage + .put_block(ctx, second_block) + .await + .unwrap(); + } + } + + let received_actions = assert_first_block_actions(ctx, &mut actions).await?; + // Manually replicate actions to the state keeper. + keeper_actions_sender.push_actions(received_actions).await; + + if delay_second_block { + validator_storage + .put_block(ctx, second_block) + .await + .unwrap(); + } + + let received_actions = assert_second_block_actions(ctx, &mut actions).await?; + keeper_actions_sender.push_actions(received_actions).await; + state_keeper + .wait(|state| state.get_local_block() == MiniblockNumber(2)) + .await; + Ok(()) + }) + .await + .unwrap(); +} + +async fn get_blocks_and_reset_storage( + mut storage: StorageProcessor<'_>, +) -> (FinalBlock, Vec) { let genesis_block = load_final_block(&mut storage, 0).await; - let first_block = load_final_block(&mut storage, 1).await; - let second_block = load_final_block(&mut storage, 2).await; + let sealed_miniblock_number = storage + .blocks_dal() + .get_sealed_miniblock_number() + .await + .unwrap(); + + let mut blocks = Vec::with_capacity(sealed_miniblock_number.0 as usize); + for number in 1..=sealed_miniblock_number.0 { + blocks.push(load_final_block(&mut storage, number).await); + } + storage .transactions_dal() .reset_transactions_state(MiniblockNumber(0)) @@ -409,32 +218,64 @@ async fn syncing_via_gossip_fetcher() { .delete_miniblocks(MiniblockNumber(0)) .await .unwrap(); - drop(storage); + storage + .blocks_dal() + .delete_l1_batches(L1BatchNumber(0)) + .await + .unwrap(); + (genesis_block, blocks) +} + +#[test_casing(4, [3, 2, 1, 0])] +#[tokio::test] +async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count: usize) { + assert!(initial_block_count <= 3); + zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::AffineClock::new(20.0)); + let pool = ConnectionPool::test_pool().await; + let tx_hashes = run_state_keeper_with_multiple_l1_batches(pool.clone()).await; + let tx_hashes: Vec<_> = tx_hashes.iter().map(Vec::as_slice).collect(); + + let storage = pool.access_storage().await.unwrap(); + let (genesis_block, blocks) = get_blocks_and_reset_storage(storage).await; + assert_eq!(blocks.len(), 3); // 2 real + 1 fictive blocks + tracing::trace!("Node storage reset"); + let (initial_blocks, delayed_blocks) = blocks.split_at(initial_block_count); + + let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()); let external_node = validator.connect_external_node(rng); let validator_storage = Arc::new(InMemoryStorage::new(genesis_block)); - validator_storage - .put_block(ctx, &first_block) - .await - .unwrap(); - validator_storage - .put_block(ctx, &second_block) - .await - .unwrap(); - let validator = - Executor::new(validator.node_config, validator.node_key, validator_storage).unwrap(); - // ^ We intentionally do not run consensus on the validator node, since it'll produce blocks - // with payloads that cannot be parsed by the external node. + for block in initial_blocks { + validator_storage.put_block(ctx, block).await.unwrap(); + } + let validator = Executor::new( + validator.node_config, + validator.node_key, + validator_storage.clone(), + ) + .unwrap(); - let (actions_sender, mut actions) = ActionQueue::new(); - let state_keeper = StateKeeperHandles::new(pool.clone(), &[&tx_hashes]).await; + let (actions_sender, actions) = ActionQueue::new(); + let state_keeper = StateKeeperHandles::new(pool.clone(), actions, &tx_hashes).await; scope::run!(ctx, |ctx, s| async { s.spawn_bg(wrap_bg_task(validator.run(ctx))); + s.spawn_bg(async { + for block in delayed_blocks { + ctx.sleep(POLL_INTERVAL).await?; + validator_storage.put_block(ctx, block).await?; + } + Ok(()) + }); + + let cloned_pool = pool.clone(); + s.spawn_bg(async { + mock_l1_batch_hash_computation(cloned_pool, 1).await; + Ok(()) + }); s.spawn_bg(wrap_bg_task(start_gossip_fetcher_inner( ctx, pool, @@ -443,20 +284,8 @@ async fn syncing_via_gossip_fetcher() { external_node.node_key, ))); - let received_actions = assert_first_block_actions(ctx, &mut actions).await?; - // Manually replicate actions to the state keeper. state_keeper - .actions_sender - .push_actions(received_actions) - .await; - - let received_actions = assert_second_block_actions(ctx, &mut actions).await?; - state_keeper - .actions_sender - .push_actions(received_actions) - .await; - state_keeper - .wait(|state| state.get_local_block() == MiniblockNumber(2)) + .wait(|state| state.get_local_block() == MiniblockNumber(3)) .await; Ok(()) }) diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index 6960d01caa4f..8ee3dfe9d919 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -16,11 +16,7 @@ use zksync_types::{ api, Address, L1BatchNumber, L2ChainId, MiniblockNumber, ProtocolVersionId, Transaction, H256, }; -use super::{ - fetcher::FetcherCursor, - sync_action::{ActionQueueSender, SyncAction}, - *, -}; +use super::{fetcher::FetcherCursor, sync_action::SyncAction, *}; use crate::{ api_server::web3::tests::spawn_http_server, genesis::{ensure_genesis_state, GenesisParams}, @@ -146,7 +142,6 @@ fn open_l1_batch(number: u32, timestamp: u64, first_miniblock_number: u32) -> Sy #[derive(Debug)] pub(super) struct StateKeeperHandles { - pub actions_sender: ActionQueueSender, pub stop_sender: watch::Sender, pub sync_state: SyncState, pub task: JoinHandle>, @@ -154,13 +149,12 @@ pub(super) struct StateKeeperHandles { impl StateKeeperHandles { /// `tx_hashes` are grouped by the L1 batch. - pub async fn new(pool: ConnectionPool, tx_hashes: &[&[H256]]) -> Self { + pub async fn new(pool: ConnectionPool, actions: ActionQueue, tx_hashes: &[&[H256]]) -> Self { assert!(!tx_hashes.is_empty()); assert!(tx_hashes.iter().all(|tx_hashes| !tx_hashes.is_empty())); ensure_genesis(&mut pool.access_storage().await.unwrap()).await; - let (actions_sender, actions) = ActionQueue::new(); let sync_state = SyncState::new(); let io = ExternalIO::new( pool, @@ -185,7 +179,6 @@ impl StateKeeperHandles { Box::new(batch_executor_base), ); Self { - actions_sender, stop_sender, sync_state, task: tokio::spawn(state_keeper.run()), @@ -248,8 +241,10 @@ async fn external_io_basics() { let tx = SyncAction::Tx(Box::new(tx.into())); let actions = vec![open_l1_batch, tx, SyncAction::SealMiniblock]; - let state_keeper = StateKeeperHandles::new(pool.clone(), &[&extract_tx_hashes(&actions)]).await; - state_keeper.actions_sender.push_actions(actions).await; + let (actions_sender, action_queue) = ActionQueue::new(); + let state_keeper = + StateKeeperHandles::new(pool.clone(), action_queue, &[&extract_tx_hashes(&actions)]).await; + actions_sender.push_actions(actions).await; // Wait until the miniblock is sealed. state_keeper .wait(|state| state.get_local_block() == MiniblockNumber(1)) @@ -309,15 +304,10 @@ pub(super) async fn run_state_keeper_with_multiple_miniblocks(pool: ConnectionPo .iter() .chain(&second_miniblock_actions), ); - let state_keeper = StateKeeperHandles::new(pool, &[&tx_hashes]).await; - state_keeper - .actions_sender - .push_actions(first_miniblock_actions) - .await; - state_keeper - .actions_sender - .push_actions(second_miniblock_actions) - .await; + let (actions_sender, action_queue) = ActionQueue::new(); + let state_keeper = StateKeeperHandles::new(pool, action_queue, &[&tx_hashes]).await; + actions_sender.push_actions(first_miniblock_actions).await; + actions_sender.push_actions(second_miniblock_actions).await; // Wait until both miniblocks are sealed. state_keeper .wait(|state| state.get_local_block() == MiniblockNumber(2)) @@ -366,7 +356,8 @@ async fn test_external_io_recovery(pool: ConnectionPool, mut tx_hashes: Vec Vec> { let l1_batch = open_l1_batch(1, 1, 1); let first_tx = create_l2_transaction(10, 100); let first_tx_hash = first_tx.hash(); @@ -442,20 +434,16 @@ async fn external_io_with_multiple_l1_batches() { let second_tx = SyncAction::Tx(Box::new(second_tx.into())); let second_l1_batch_actions = vec![l1_batch, second_tx, SyncAction::SealMiniblock]; - let state_keeper = - StateKeeperHandles::new(pool.clone(), &[&[first_tx_hash], &[second_tx_hash]]).await; - state_keeper - .actions_sender - .push_actions(first_l1_batch_actions) - .await; - state_keeper - .actions_sender - .push_actions(fictive_miniblock_actions) - .await; - state_keeper - .actions_sender - .push_actions(second_l1_batch_actions) - .await; + let (actions_sender, action_queue) = ActionQueue::new(); + let state_keeper = StateKeeperHandles::new( + pool.clone(), + action_queue, + &[&[first_tx_hash], &[second_tx_hash]], + ) + .await; + actions_sender.push_actions(first_l1_batch_actions).await; + actions_sender.push_actions(fictive_miniblock_actions).await; + actions_sender.push_actions(second_l1_batch_actions).await; let hash_task = tokio::spawn(mock_l1_batch_hash_computation(pool.clone(), 1)); // Wait until the miniblocks are sealed. @@ -464,6 +452,14 @@ async fn external_io_with_multiple_l1_batches() { .await; hash_task.await.unwrap(); + vec![vec![first_tx_hash], vec![second_tx_hash]] +} + +#[tokio::test] +async fn external_io_with_multiple_l1_batches() { + let pool = ConnectionPool::test_pool().await; + run_state_keeper_with_multiple_l1_batches(pool.clone()).await; + let mut storage = pool.access_storage().await.unwrap(); let l1_batch_header = storage .blocks_dal() From 2ba0f408c1abdfb00aab8876041c404a68d20ba6 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 1 Nov 2023 13:51:50 +0200 Subject: [PATCH 08/38] Use git revision of consensus deps --- Cargo.lock | 871 ++++++++++++++++++++++++++++++-- core/lib/zksync_core/Cargo.toml | 11 +- 2 files changed, 824 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 150123dc6145..1b63e9316476 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,6 +225,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aead" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" +dependencies = [ + "generic-array 0.14.7", +] + [[package]] name = "aes" version = "0.6.0" @@ -233,7 +242,19 @@ checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" dependencies = [ "aes-soft", "aesni", - "cipher", + "cipher 0.2.5", +] + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if 1.0.0", + "cipher 0.3.0", + "cpufeatures", + "opaque-debug 0.3.0", ] [[package]] @@ -244,8 +265,22 @@ checksum = "7729c3cde54d67063be556aeac75a81330d802f0259500ca40cb52967f975763" dependencies = [ "aes-soft", "aesni", - "cipher", - "ctr", + "cipher 0.2.5", + "ctr 0.6.0", +] + +[[package]] +name = "aes-gcm" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" +dependencies = [ + "aead", + "aes 0.7.5", + "cipher 0.3.0", + "ctr 0.8.0", + "ghash", + "subtle", ] [[package]] @@ -254,7 +289,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" dependencies = [ - "cipher", + "cipher 0.2.5", "opaque-debug 0.3.0", ] @@ -264,7 +299,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" dependencies = [ - "cipher", + "cipher 0.2.5", "opaque-debug 0.3.0", ] @@ -507,7 +542,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -518,7 +553,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -749,13 +784,13 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease", + "prettyplease 0.2.6", "proc-macro2 1.0.66", "quote 1.0.33", "regex", "rustc-hash", "shlex", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -779,6 +814,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + [[package]] name = "bitvec" version = "0.20.4" @@ -911,7 +955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57a0e8073e8baa88212fb5823574c02ebccb395136ba9a164ab89379ec6072f0" dependencies = [ "block-padding 0.2.1", - "cipher", + "cipher 0.2.5", ] [[package]] @@ -944,6 +988,18 @@ dependencies = [ "zksync_types", ] +[[package]] +name = "blst" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c94087b935a822949d3291a9989ad2b2051ea141eda0fd4e478a75f6aa3e604b" +dependencies = [ + "cc", + "glob", + "threadpool", + "zeroize", +] + [[package]] name = "boojum" version = "0.1.0" @@ -1139,6 +1195,31 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chacha20" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c80e5460aa66fe3b91d40bcbdab953a597b60053e34d684ac6903f863b680a6" +dependencies = [ + "cfg-if 1.0.0", + "cipher 0.3.0", + "cpufeatures", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18446b09be63d457bbec447509e85f662f32952b035ce892290396bc0b0cff5" +dependencies = [ + "aead", + "chacha20", + "cipher 0.3.0", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.31" @@ -1190,6 +1271,15 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array 0.14.7", +] + [[package]] name = "circuit_definitions" version = "0.1.0" @@ -1245,7 +1335,7 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", - "indexmap", + "indexmap 1.9.3", "textwrap 0.16.0", ] @@ -1282,7 +1372,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -1331,7 +1421,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff61280aed771c3070e7dcc9e050c66f1eb1e3b96431ba66f9f74641d02fc41d" dependencies = [ - "indexmap", + "indexmap 1.9.3", ] [[package]] @@ -1350,6 +1440,44 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrency" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "once_cell", + "pin-project", + "rand 0.8.5", + "sha2 0.10.8", + "thiserror", + "time", + "tokio", + "tracing", + "tracing-subscriber", + "vise", +] + +[[package]] +name = "consensus" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "concurrency", + "crypto", + "network", + "once_cell", + "rand 0.8.5", + "roles", + "schema", + "storage", + "thiserror", + "tracing", + "utils", + "vise", +] + [[package]] name = "console" version = "0.15.7" @@ -1674,6 +1802,20 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "blst", + "ed25519-dalek", + "hex", + "rand 0.8.5", + "sha2 0.10.8", + "thiserror", +] + [[package]] name = "crypto-bigint" version = "0.3.2" @@ -1765,7 +1907,16 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" dependencies = [ - "cipher", + "cipher 0.2.5", +] + +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher 0.3.0", ] [[package]] @@ -1778,6 +1929,34 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "platforms", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.32", +] + [[package]] name = "darling" version = "0.13.4" @@ -1857,6 +2036,26 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +dependencies = [ + "const-oid 0.9.2", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "derivative" version = "2.2.0" @@ -1951,7 +2150,31 @@ dependencies = [ "der 0.6.1", "elliptic-curve", "rfc6979", - "signature", + "signature 1.6.4", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8 0.10.2", + "signature 2.1.0", +] + +[[package]] +name = "ed25519-dalek" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.8", + "zeroize", ] [[package]] @@ -2029,6 +2252,12 @@ dependencies = [ "serde", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -2136,6 +2365,26 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "executor" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "concurrency", + "consensus", + "crypto", + "network", + "rand 0.8.5", + "roles", + "schema", + "storage", + "sync_blocks", + "tracing", + "utils", + "vise", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -2189,6 +2438,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "fiat-crypto" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" + [[package]] name = "findshlibs" version = "0.10.2" @@ -2231,6 +2486,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.26" @@ -2285,7 +2546,7 @@ dependencies = [ "byteorder", "digest 0.9.0", "hex", - "indexmap", + "indexmap 1.9.3", "itertools", "lazy_static", "num-bigint 0.4.3", @@ -2317,7 +2578,7 @@ dependencies = [ "derivative", "digest 0.9.0", "hex", - "indexmap", + "indexmap 1.9.3", "itertools", "lazy_static", "num-bigint 0.4.3", @@ -2441,7 +2702,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -2526,6 +2787,16 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "ghash" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99" +dependencies = [ + "opaque-debug 0.3.0", + "polyval", +] + [[package]] name = "gimli" version = "0.27.3" @@ -2709,7 +2980,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util 0.7.8", @@ -2760,6 +3031,12 @@ dependencies = [ "ahash 0.8.3", ] +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + [[package]] name = "hashlink" version = "0.7.0" @@ -2948,9 +3225,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes 1.4.0", "futures-channel", @@ -3043,6 +3320,20 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + [[package]] name = "impl-codec" version = "0.5.1" @@ -3109,6 +3400,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +dependencies = [ + "equivalent", + "hashbrown 0.14.2", +] + [[package]] name = "insta" version = "1.29.0" @@ -3640,7 +3941,7 @@ checksum = "bc28438cad73dcc90ff3466fc329a9252b1b8ba668eb0d5668ba97088cf4eef0" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -3828,7 +4129,7 @@ checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ "base64 0.21.2", "hyper", - "indexmap", + "indexmap 1.9.3", "ipnet", "metrics", "metrics-util", @@ -3846,7 +4147,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -3974,6 +4275,12 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "multivm" version = "0.1.0" @@ -4027,6 +4334,28 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "network" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "async-trait", + "concurrency", + "crypto", + "im", + "once_cell", + "pin-project", + "rand 0.8.5", + "roles", + "schema", + "snow", + "thiserror", + "tracing", + "utils", + "vise", +] + [[package]] name = "nix" version = "0.26.2" @@ -4281,7 +4610,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -4340,7 +4669,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -4361,6 +4690,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "os_info" version = "3.7.0" @@ -4425,7 +4763,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b92ea9ddac0d6e1db7c49991e7d397d34a9fd814b4c93cda53788e8eef94e35" dependencies = [ - "aes", + "aes 0.6.0", "aes-ctr", "block-modes", "digest 0.9.0", @@ -4659,7 +4997,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -4673,6 +5011,16 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.0.2", +] + [[package]] name = "pin-project" version = "1.1.0" @@ -4690,7 +5038,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -4737,12 +5085,28 @@ dependencies = [ "spki 0.6.0", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.8", + "spki 0.7.2", +] + [[package]] name = "pkg-config" version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "platforms" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" + [[package]] name = "plotters" version = "0.3.4" @@ -4771,18 +5135,57 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "poly1305" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede" +dependencies = [ + "cpufeatures", + "opaque-debug 0.3.0", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "opaque-debug 0.3.0", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2 1.0.66", + "syn 1.0.109", +] + [[package]] name = "prettyplease" version = "0.2.6" @@ -4790,7 +5193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" dependencies = [ "proc-macro2 1.0.66", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -4906,7 +5309,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -4921,6 +5324,146 @@ dependencies = [ "vise-exporter", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes 1.4.0", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes 1.4.0", + "heck 0.4.1", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease 0.1.25", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "prost-reflect" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b823de344848e011658ac981009100818b322421676740546f8b52ed5249428" +dependencies = [ + "base64 0.21.2", + "once_cell", + "prost", + "prost-reflect-derive", + "prost-types", + "serde", + "serde-value", +] + +[[package]] +name = "prost-reflect-build" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d959f576df574d088e0409c45ef11897f64727158c554ce65edc2d345f8afcf" +dependencies = [ + "prost-build", + "prost-reflect", +] + +[[package]] +name = "prost-reflect-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7718375aa8f966df66e583b608a305a45bc87eeb1ffd5db87fae673bea17a7e4" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.32", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" + [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -4964,6 +5507,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + [[package]] name = "quote" version = "0.6.13" @@ -5184,6 +5736,15 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "raw-cpuid" version = "10.7.0" @@ -5461,6 +6022,23 @@ dependencies = [ "zksync_storage", ] +[[package]] +name = "roles" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "bit-vec", + "concurrency", + "crypto", + "hex", + "rand 0.8.5", + "schema", + "serde", + "tracing", + "utils", +] + [[package]] name = "rsa" version = "0.6.1" @@ -5593,7 +6171,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "399f290ffc409596022fce5ea5d4138184be4784f2b28c62c59f0d8389059a15" dependencies = [ - "cipher", + "cipher 0.2.5", ] [[package]] @@ -5614,6 +6192,29 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "schema" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "bit-vec", + "concurrency", + "once_cell", + "prettyplease 0.2.6", + "prost", + "prost-build", + "prost-reflect", + "prost-reflect-build", + "protoc-bin-vendored", + "quick-protobuf", + "rand 0.8.5", + "serde", + "serde_json", + "syn 2.0.32", + "tokio", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -5845,22 +6446,32 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.175" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_derive" -version = "1.0.175" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -6066,6 +6677,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" + [[package]] name = "similar" version = "2.2.1" @@ -6084,6 +6701,16 @@ dependencies = [ "time", ] +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + [[package]] name = "skeptic" version = "0.13.7" @@ -6135,6 +6762,22 @@ dependencies = [ "rescue_poseidon 0.4.1 (git+https://github.com/matter-labs/rescue-poseidon.git?branch=poseidon2)", ] +[[package]] +name = "snow" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9d1425eb528a21de2755c75af4c9b5d57f50a0d4c3b7f1828a4cd03f8ba155" +dependencies = [ + "aes-gcm", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "chacha20poly1305", + "curve25519-dalek", + "rand_core 0.6.4", + "rustc_version", + "sha2 0.10.8", + "subtle", +] + [[package]] name = "socket2" version = "0.4.9" @@ -6187,6 +6830,16 @@ dependencies = [ "der 0.6.1", ] +[[package]] +name = "spki" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +dependencies = [ + "base64ct", + "der 0.7.8", +] + [[package]] name = "splitmut" version = "0.2.1" @@ -6241,7 +6894,7 @@ dependencies = [ "hex", "hkdf", "hmac 0.12.1", - "indexmap", + "indexmap 1.9.3", "ipnetwork", "itoa", "libc", @@ -6313,6 +6966,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "storage" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "async-trait", + "concurrency", + "rand 0.8.5", + "roles", + "schema", + "thiserror", + "tracing", +] + [[package]] name = "storage_logs_dedup_migration" version = "0.1.0" @@ -6421,15 +7089,30 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.27" +version = "2.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", "unicode-ident", ] +[[package]] +name = "sync_blocks" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "anyhow", + "concurrency", + "network", + "roles", + "storage", + "thiserror", + "tracing", + "utils", +] + [[package]] name = "sync_vm" version = "1.3.3" @@ -6522,6 +7205,24 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "test-casing" +version = "0.1.1" +source = "git+https://github.com/slowli/test-casing.git?rev=cb033e145c0de24d71532f3f6f219c0234ff4c44#cb033e145c0de24d71532f3f6f219c0234ff4c44" +dependencies = [ + "test-casing-macro", +] + +[[package]] +name = "test-casing-macro" +version = "0.1.1" +source = "git+https://github.com/slowli/test-casing.git?rev=cb033e145c0de24d71532f3f6f219c0234ff4c44#cb033e145c0de24d71532f3f6f219c0234ff4c44" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.32", +] + [[package]] name = "test-log" version = "0.2.12" @@ -6565,7 +7266,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -6578,6 +7279,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "tikv-jemalloc-sys" version = "0.5.4+5.3.0-patched" @@ -6600,11 +7310,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ + "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -6612,15 +7324,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -6695,7 +7407,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -6780,7 +7492,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5376256e44f2443f8896ac012507c19a012df0fe8758b55246ae51a2279db51f" dependencies = [ "combine", - "indexmap", + "indexmap 1.9.3", "itertools", ] @@ -6790,7 +7502,7 @@ version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ - "indexmap", + "indexmap 1.9.3", "toml_datetime", "winnow", ] @@ -6804,7 +7516,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand 0.8.5", @@ -6879,7 +7591,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -7040,6 +7752,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "universal-hash" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +dependencies = [ + "generic-array 0.14.7", + "subtle", +] + [[package]] name = "unroll" version = "0.1.5" @@ -7093,6 +7815,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "utils" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +dependencies = [ + "concurrency", + "thiserror", +] + [[package]] name = "uuid" version = "1.3.4" @@ -7170,7 +7901,7 @@ source = "git+https://github.com/matter-labs/vise.git?rev=dd05139b76ab0843443ab3 dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", ] [[package]] @@ -7268,7 +7999,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -7302,7 +8033,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", - "syn 2.0.27", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7395,6 +8126,17 @@ dependencies = [ "rustls-webpki 0.101.6", ] +[[package]] +name = "which" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +dependencies = [ + "either", + "libc", + "once_cell", +] + [[package]] name = "whoami" version = "1.4.0" @@ -7661,6 +8403,20 @@ name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.32", +] [[package]] name = "zk_evm" @@ -7983,7 +8739,9 @@ dependencies = [ "bigdecimal", "bitflags 1.3.2", "chrono", + "concurrency", "ctrlc", + "executor", "futures 0.3.28", "governor", "hex", @@ -8001,9 +8759,12 @@ dependencies = [ "prometheus_exporter", "rand 0.8.5", "reqwest", + "roles", "serde", "serde_json", + "storage", "tempfile", + "test-casing", "thiserror", "tokio", "tower", diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index 4d84757b6fa1..f995870f955c 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zksync_core" version = "0.1.0" -edition = "2018" +edition = "2021" authors = ["The Matter Labs Team "] homepage = "https://zksync.io/" repository = "https://github.com/matter-labs/zksync-era" @@ -39,6 +39,11 @@ zksync_health_check = { path = "../health_check" } vlog = { path = "../vlog" } multivm = { path = "../multivm" } +# Consensus dependenices +zksync_concurrency = { package = "concurrency", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } +zksync_consensus_roles = { package = "roles", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } +zksync_consensus_storage = { package = "storage", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } +zksync_consensus_executor = { package = "executor", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -84,8 +89,8 @@ actix-web = "4.0.0-beta.8" tracing = "0.1.26" [dev-dependencies] - -assert_matches = "1.5" zksync_test_account = { path = "../test_account" } +assert_matches = "1.5" tempfile = "3.0.2" +test-casing = { version = "0.1.1", git = "https://github.com/slowli/test-casing.git", rev = "cb033e145c0de24d71532f3f6f219c0234ff4c44" } From b033e9410f337e48bd9bb5317e822a845e047d3e Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 1 Nov 2023 14:15:58 +0200 Subject: [PATCH 09/38] Brush up `start_gossip_fetcher()` entrypoint --- .../src/sync_layer/gossip/buffered/mod.rs | 14 +++--- .../src/sync_layer/gossip/buffered/tests.rs | 2 +- .../zksync_core/src/sync_layer/gossip/mod.rs | 46 +++++++++++++++---- .../src/sync_layer/gossip/storage.rs | 14 +++--- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index 21acde98c423..0c2bc7a89a08 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -32,7 +32,7 @@ pub(super) trait ContiguousBlockStore: BlockStore { /// expected that it will be added to the store eventually, which will be signaled via /// a subscriber returned from [`BlockStore::subscribe_to_block_writes()`]. /// - /// [`BufferedStorage`] guarantees that this method will only ever be called: + /// [`Buffered`] guarantees that this method will only ever be called: /// /// - with the next block (i.e., one immediately after [`BlockStore::head_block()`]) /// - sequentially (i.e., multiple blocks cannot be scheduled at once) @@ -121,7 +121,7 @@ impl BlockBuffer { } } -/// Events emitted by [`BufferedStorage`]. +/// Events emitted by [`Buffered`]. #[cfg(test)] #[derive(Debug)] pub(super) enum BufferedStorageEvent { @@ -131,7 +131,7 @@ pub(super) enum BufferedStorageEvent { /// [`BlockStore`] with an in-memory buffer for pending blocks. #[derive(Debug)] -pub(super) struct BufferedStorage { +pub(super) struct Buffered { inner: T, inner_subscriber: watch::Receiver, block_writes_sender: watch::Sender, @@ -140,7 +140,7 @@ pub(super) struct BufferedStorage { events_sender: channel::UnboundedSender, } -impl BufferedStorage { +impl Buffered { /// Creates a new buffered storage. The buffer is initially empty. pub fn new(store: T) -> Self { let inner_subscriber = store.subscribe_to_block_writes(); @@ -171,7 +171,7 @@ impl BufferedStorage { } /// Listens to the updates in the underlying storage. This method must be spawned as a background task - /// which should be running as long at the [`BufferedStorage`] is in use. Otherwise, + /// which should be running as long at the [`Buffered`] is in use. Otherwise, /// `BufferedStorage` will function incorrectly. #[tracing::instrument(level = "trace", skip_all, err)] pub async fn listen_to_updates(&self, ctx: &ctx::Ctx) -> StorageResult<()> { @@ -199,7 +199,7 @@ impl BufferedStorage { } #[async_trait] -impl BlockStore for BufferedStorage { +impl BlockStore for Buffered { async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { let buffered_head_block = sync::lock(ctx, &self.buffer).await?.head_block(); if let Some(block) = buffered_head_block { @@ -250,7 +250,7 @@ impl BlockStore for BufferedStorage { } #[async_trait] -impl WriteBlockStore for BufferedStorage { +impl WriteBlockStore for Buffered { async fn put_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { let next_block_for_store = { let mut buffer = sync::lock(ctx, &self.buffer).await?; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs index 9b974c49a8ae..b0a69c38188a 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs @@ -144,7 +144,7 @@ async fn test_buffered_storage( initial_blocks.insert(0, genesis_block.clone()); let (block_store, block_receiver) = MockContiguousStore::new(block_store); - let mut buffered_store = BufferedStorage::new(block_store); + let mut buffered_store = Buffered::new(block_store); let (events_sender, mut events_receiver) = channel::unbounded(); buffered_store.set_events_sender(events_sender); diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index d8282eacf7e7..3367914e52f9 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -1,6 +1,7 @@ //! Consensus adapter for EN synchronization logic. use anyhow::Context as _; +use tokio::sync::watch; use std::sync::Arc; @@ -16,7 +17,7 @@ mod storage; mod tests; mod utils; -use self::{buffered::BufferedStorage, storage::PostgresBlockStore}; +use self::{buffered::Buffered, storage::PostgresBlockStorage}; use super::{fetcher::FetcherCursor, sync_action::ActionQueueSender}; /// Starts fetching L2 blocks using peer-to-peer gossip network. @@ -25,8 +26,31 @@ pub async fn start_gossip_fetcher( actions: ActionQueueSender, executor_config: ExecutorConfig, node_key: node::SecretKey, + mut stop_receiver: watch::Receiver, ) -> anyhow::Result<()> { - start_gossip_fetcher_inner(&ctx::root(), pool, actions, executor_config, node_key).await + let result = scope::run!(&ctx::root(), |ctx, s| { + s.spawn_bg(async { + if stop_receiver.changed().await.is_err() { + tracing::warn!( + "Stop signal sender for gossip fetcher was dropped without sending a signal" + ); + } + s.cancel(); + tracing::info!("Stop signal received, gossip fetcher is shutting down"); + Ok(()) + }); + start_gossip_fetcher_inner(ctx, pool, actions, executor_config, node_key) + }) + .await; + + result.or_else(|err| { + if err.root_cause().is::() { + tracing::info!("Gossip fetcher is shut down"); + Ok(()) + } else { + Err(err) + } + }) } async fn start_gossip_fetcher_inner( @@ -37,6 +61,10 @@ async fn start_gossip_fetcher_inner( node_key: node::SecretKey, ) -> anyhow::Result<()> { executor_config.skip_qc_validation = true; + tracing::info!( + "Starting gossip fetcher with {executor_config:?} and node key {:?}", + node_key.public() + ); let mut storage = pool .access_storage_tagged("sync_layer") @@ -45,10 +73,10 @@ async fn start_gossip_fetcher_inner( let cursor = FetcherCursor::new(&mut storage).await?; drop(storage); - let store = PostgresBlockStore::new(pool, actions, cursor); - let buffered_store = Arc::new(BufferedStorage::new(store)); - let store = buffered_store.inner(); - let executor = Executor::new(executor_config, node_key, buffered_store.clone()) + let store = PostgresBlockStorage::new(pool, actions, cursor); + let buffered = Arc::new(Buffered::new(store)); + let store = buffered.inner(); + let executor = Executor::new(executor_config, node_key, buffered.clone()) .context("Node executor misconfiguration")?; scope::run!(ctx, |ctx, s| async { @@ -56,13 +84,13 @@ async fn start_gossip_fetcher_inner( store .listen_to_updates(ctx) .await - .context("`PostgresBlockStore` listener failed") + .context("`PostgresBlockStorage` listener failed") }); s.spawn_bg(async { - buffered_store + buffered .listen_to_updates(ctx) .await - .context("`BufferedStore` listener failed") + .context("`Buffered` storage listener failed") }); executor.run(ctx).await.context("Node executor terminated") diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index a91427c5c15b..d0103f93a357 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -22,14 +22,14 @@ use crate::sync_layer::{ }; #[derive(Debug)] -pub(super) struct PostgresBlockStore { +pub(super) struct PostgresBlockStorage { pool: ConnectionPool, actions: ActionQueueSender, block_sender: watch::Sender, cursor: Mutex, } -impl PostgresBlockStore { +impl PostgresBlockStorage { pub fn new(pool: ConnectionPool, actions: ActionQueueSender, cursor: FetcherCursor) -> Self { let current_block_number = cursor.next_miniblock.0.saturating_sub(1).into(); Self { @@ -123,7 +123,7 @@ impl PostgresBlockStore { } #[async_trait] -impl BlockStore for PostgresBlockStore { +impl BlockStore for PostgresBlockStorage { async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { tokio::select! { () = ctx.canceled() => Err(ctx::Canceled.into()), @@ -175,7 +175,7 @@ impl BlockStore for PostgresBlockStore { } #[async_trait] -impl ContiguousBlockStore for PostgresBlockStore { +impl ContiguousBlockStore for PostgresBlockStorage { async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { self.schedule_block(ctx, block).await.map_err(Into::into) } @@ -209,7 +209,7 @@ mod tests { let cursor = FetcherCursor::new(&mut storage).await.unwrap(); drop(storage); let (actions_sender, _) = ActionQueue::new(); - let storage = PostgresBlockStore::new(pool.clone(), actions_sender, cursor); + let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); let ctx = &ctx::test_root(&ctx::RealClock); let genesis_block = BlockStore::first_block(&storage, ctx).await.unwrap(); @@ -243,7 +243,7 @@ mod tests { // `ContiguousBlockStore`), but for testing subscriptions this is fine. drop(storage); let (actions_sender, _) = ActionQueue::new(); - let storage = PostgresBlockStore::new(pool.clone(), actions_sender, cursor); + let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); let mut subscriber = storage.subscribe_to_block_writes(); let ctx = &ctx::test_root(&ctx::RealClock); @@ -293,7 +293,7 @@ mod tests { drop(storage); let (actions_sender, mut actions) = ActionQueue::new(); - let storage = PostgresBlockStore::new(pool.clone(), actions_sender, cursor); + let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); let ctx = &ctx::test_root(&ctx::RealClock); storage.schedule_block(ctx, &first_block).await.unwrap(); assert_first_block_actions(ctx, &mut actions).await.unwrap(); From 5dff11c0792a41b9a7f48343b0a48c0b926d61af Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 1 Nov 2023 16:37:43 +0200 Subject: [PATCH 10/38] Simplify `ActionQueue` management in tests --- .../src/sync_layer/gossip/storage.rs | 9 +++--- .../src/sync_layer/gossip/tests.rs | 30 +++++-------------- .../zksync_core/src/sync_layer/sync_action.rs | 15 ++++++++-- core/lib/zksync_core/src/sync_layer/tests.rs | 24 +++++---------- 4 files changed, 33 insertions(+), 45 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index d0103f93a357..9db8a7162722 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -198,7 +198,7 @@ mod tests { }, }; - const TEST_TIMEOUT: time::Duration = time::Duration::seconds(1); + const TEST_TIMEOUT: time::Duration = time::Duration::seconds(10); #[tokio::test] async fn block_store_basics_for_postgres() { @@ -295,12 +295,11 @@ mod tests { let (actions_sender, mut actions) = ActionQueue::new(); let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); let ctx = &ctx::test_root(&ctx::RealClock); + let ctx = &ctx.with_timeout(TEST_TIMEOUT); storage.schedule_block(ctx, &first_block).await.unwrap(); - assert_first_block_actions(ctx, &mut actions).await.unwrap(); + assert_first_block_actions(&mut actions).await; storage.schedule_block(ctx, &second_block).await.unwrap(); - assert_second_block_actions(ctx, &mut actions) - .await - .unwrap(); + assert_second_block_actions(&mut actions).await; } } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index d5eafa421411..3709bae4f5b8 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -39,17 +39,10 @@ pub(super) async fn load_final_block( conversions::sync_block_to_consensus_block(sync_block) } -pub(super) async fn assert_first_block_actions( - ctx: &ctx::Ctx, - actions: &mut ActionQueue, -) -> ctx::OrCanceled> { +pub(super) async fn assert_first_block_actions(actions: &mut ActionQueue) -> Vec { let mut received_actions = vec![]; while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { - let Some(action) = actions.pop_action() else { - ctx.sleep(POLL_INTERVAL).await?; - continue; - }; - received_actions.push(action); + received_actions.push(actions.recv_action().await); } assert_matches!( received_actions.as_slice(), @@ -68,20 +61,13 @@ pub(super) async fn assert_first_block_actions( SyncAction::SealMiniblock, ] ); - Ok(received_actions) + received_actions } -pub(super) async fn assert_second_block_actions( - ctx: &ctx::Ctx, - actions: &mut ActionQueue, -) -> ctx::OrCanceled> { +pub(super) async fn assert_second_block_actions(actions: &mut ActionQueue) -> Vec { let mut received_actions = vec![]; while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { - let Some(action) = actions.pop_action() else { - ctx.sleep(POLL_INTERVAL).await?; - continue; - }; - received_actions.push(action); + received_actions.push(actions.recv_action().await); } assert_matches!( received_actions.as_slice(), @@ -97,7 +83,7 @@ pub(super) async fn assert_second_block_actions( SyncAction::SealMiniblock, ] ); - Ok(received_actions) + received_actions } /// Wraps a background task so that it returns `Ok(())` if it's canceled. @@ -172,7 +158,7 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: } } - let received_actions = assert_first_block_actions(ctx, &mut actions).await?; + let received_actions = assert_first_block_actions(&mut actions).await; // Manually replicate actions to the state keeper. keeper_actions_sender.push_actions(received_actions).await; @@ -183,7 +169,7 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: .unwrap(); } - let received_actions = assert_second_block_actions(ctx, &mut actions).await?; + let received_actions = assert_second_block_actions(&mut actions).await; keeper_actions_sender.push_actions(received_actions).await; state_keeper .wait(|state| state.get_local_block() == MiniblockNumber(2)) diff --git a/core/lib/zksync_core/src/sync_layer/sync_action.rs b/core/lib/zksync_core/src/sync_layer/sync_action.rs index 977d03dd5329..4077cdc60bde 100644 --- a/core/lib/zksync_core/src/sync_layer/sync_action.rs +++ b/core/lib/zksync_core/src/sync_layer/sync_action.rs @@ -89,7 +89,7 @@ impl ActionQueue { } /// Removes the first action from the queue. - pub(crate) fn pop_action(&mut self) -> Option { + pub(super) fn pop_action(&mut self) -> Option { if let Some(peeked) = self.peeked.take() { QUEUE_METRICS.action_queue_size.dec_by(1); return Some(peeked); @@ -101,8 +101,19 @@ impl ActionQueue { action } + #[cfg(test)] + pub(super) async fn recv_action(&mut self) -> SyncAction { + if let Some(peeked) = self.peeked.take() { + return peeked; + } + self.receiver + .recv() + .await + .expect("actions sender was dropped prematurely") + } + /// Returns the first action from the queue without removing it. - pub(crate) fn peek_action(&mut self) -> Option { + pub(super) fn peek_action(&mut self) -> Option { if let Some(action) = &self.peeked { return Some(action.clone()); } diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index 8ee3dfe9d919..eccf29ec3699 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -526,15 +526,11 @@ async fn fetcher_basics() { let mut current_miniblock_number = MiniblockNumber(0); let mut tx_count_in_miniblock = 0; let started_at = Instant::now(); + let deadline = started_at + TEST_TIMEOUT; loop { - assert!( - started_at.elapsed() <= TEST_TIMEOUT, - "Timed out waiting for fetcher" - ); - let Some(action) = actions.pop_action() else { - tokio::time::sleep(POLL_INTERVAL).await; - continue; - }; + let action = tokio::time::timeout_at(deadline.into(), actions.recv_action()) + .await + .unwrap(); match action { SyncAction::OpenBatch { number, .. } => { current_l1_batch_number += 1; @@ -614,15 +610,11 @@ async fn fetcher_with_real_server() { let mut tx_count_in_miniblock = 0; let miniblock_number_to_tx_count = HashMap::from([(1, 5), (2, 3)]); let started_at = Instant::now(); + let deadline = started_at + TEST_TIMEOUT; loop { - assert!( - started_at.elapsed() <= TEST_TIMEOUT, - "Timed out waiting for fetcher actions" - ); - let Some(action) = actions.pop_action() else { - tokio::time::sleep(POLL_INTERVAL).await; - continue; - }; + let action = tokio::time::timeout_at(deadline.into(), actions.recv_action()) + .await + .unwrap(); match action { SyncAction::OpenBatch { number, From 0e8c6512c27d621891170b73d72e51719a16cec4 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 2 Nov 2023 10:31:32 +0200 Subject: [PATCH 11/38] Use published version of `test-casing` --- Cargo.lock | 10 ++++++---- core/lib/zksync_core/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b63e9316476..bcbda290c855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7207,16 +7207,18 @@ dependencies = [ [[package]] name = "test-casing" -version = "0.1.1" -source = "git+https://github.com/slowli/test-casing.git?rev=cb033e145c0de24d71532f3f6f219c0234ff4c44#cb033e145c0de24d71532f3f6f219c0234ff4c44" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2378d657757969a2cec9ec4eb616be8f01be98c21c6467991f91cb182e4653b" dependencies = [ "test-casing-macro", ] [[package]] name = "test-casing-macro" -version = "0.1.1" -source = "git+https://github.com/slowli/test-casing.git?rev=cb033e145c0de24d71532f3f6f219c0234ff4c44#cb033e145c0de24d71532f3f6f219c0234ff4c44" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfbe7811249c4c914b06141b8ac0f2cee2733fb883d05eb19668a45fc60c3d5" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index f995870f955c..19cb4af7638b 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -93,4 +93,4 @@ zksync_test_account = { path = "../test_account" } assert_matches = "1.5" tempfile = "3.0.2" -test-casing = { version = "0.1.1", git = "https://github.com/slowli/test-casing.git", rev = "cb033e145c0de24d71532f3f6f219c0234ff4c44" } +test-casing = "0.1.2" From 0033abbbe3f8d6331018d8f6b752ad12e834b145 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 2 Nov 2023 11:17:36 +0200 Subject: [PATCH 12/38] Add metrics for gossip fetcher --- .../src/sync_layer/gossip/buffered/mod.rs | 49 ++++++++++++++----- .../src/sync_layer/gossip/metrics.rs | 29 +++++++++++ .../zksync_core/src/sync_layer/gossip/mod.rs | 1 + .../src/sync_layer/gossip/storage.rs | 2 +- 4 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 core/lib/zksync_core/src/sync_layer/gossip/metrics.rs diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index 0c2bc7a89a08..ca63ebef98b1 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; -use std::{collections::BTreeMap, ops}; +use std::{collections::BTreeMap, ops, time::Instant}; #[cfg(test)] use zksync_concurrency::ctx::channel; @@ -16,7 +16,10 @@ use zksync_consensus_storage::{BlockStore, StorageError, StorageResult, WriteBlo #[cfg(test)] mod tests; -use super::utils::MissingBlockNumbers; +use super::{ + metrics::{BlockResponseKind, METRICS}, + utils::MissingBlockNumbers, +}; /// [`BlockStore`] variation that upholds additional invariants as to how blocks are processed. /// @@ -76,7 +79,8 @@ impl BlockBuffer { let old_len = self.blocks.len(); self.blocks = self.blocks.split_off(&store_block_number.next()); // ^ Removes all entries up to and including `store_block_number` - tracing::trace!("Removed {} blocks from buffer", old_len - self.blocks.len()); + tracing::debug!("Removed {} blocks from buffer", old_len - self.blocks.len()); + METRICS.buffer_size.set(self.blocks.len()); } fn last_contiguous_block_number(&self) -> BlockNumber { @@ -107,7 +111,8 @@ impl BlockBuffer { assert!(block_number > self.store_block_number); // ^ Must be checked previously self.blocks.insert(block_number, block); - tracing::trace!(%block_number, "Inserted block in buffer"); + tracing::debug!(%block_number, "Inserted block in buffer"); + METRICS.buffer_size.set(self.blocks.len()); } fn next_block_for_store(&mut self) -> Option { @@ -121,11 +126,11 @@ impl BlockBuffer { } } -/// Events emitted by [`Buffered`]. +/// Events emitted by [`Buffered`] storage. #[cfg(test)] #[derive(Debug)] pub(super) enum BufferedStorageEvent { - /// Update was received from + /// Update was received from the underlying storage. UpdateReceived(BlockNumber), } @@ -145,7 +150,10 @@ impl Buffered { pub fn new(store: T) -> Self { let inner_subscriber = store.subscribe_to_block_writes(); let store_block_number = *inner_subscriber.borrow(); - tracing::trace!(%store_block_number, "Initialized buffer storage"); + tracing::debug!( + store_block_number = store_block_number.0, + "Initialized buffer storage" + ); Self { inner: store, inner_subscriber, @@ -178,7 +186,10 @@ impl Buffered { let mut subscriber = self.inner_subscriber.clone(); loop { let store_block_number = *sync::changed(ctx, &mut subscriber).await?; - tracing::trace!("Underlying block number updated to {store_block_number}"); + tracing::debug!( + store_block_number = store_block_number.0, + "Underlying block number updated" + ); let next_block_for_store = { let mut buffer = sync::lock(ctx, &self.buffer).await?; @@ -188,7 +199,10 @@ impl Buffered { if let Some(block) = next_block_for_store { self.inner.schedule_next_block(ctx, &block).await?; let block_number = block.header.number; - tracing::trace!(%block_number, "Block scheduled in underlying storage"); + tracing::debug!( + block_number = block_number.0, + "Block scheduled in underlying storage" + ); } #[cfg(test)] @@ -224,13 +238,19 @@ impl BlockStore for Buffered { ctx: &ctx::Ctx, number: BlockNumber, ) -> StorageResult> { + let started_at = Instant::now(); { let buffer = sync::lock(ctx, &self.buffer).await?; if number > buffer.store_block_number { - return Ok(buffer.blocks.get(&number).cloned()); + let block = buffer.blocks.get(&number).cloned(); + METRICS.get_block_latency[&BlockResponseKind::InMemory] + .observe(started_at.elapsed()); + return Ok(block); } } - self.inner.block(ctx, number).await + let block = self.inner.block(ctx, number).await?; + METRICS.get_block_latency[&BlockResponseKind::Persisted].observe(started_at.elapsed()); + Ok(block) } async fn missing_block_numbers( @@ -252,6 +272,7 @@ impl BlockStore for Buffered { #[async_trait] impl WriteBlockStore for Buffered { async fn put_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + let buffer_block_latency = METRICS.buffer_block_latency.start(); let next_block_for_store = { let mut buffer = sync::lock(ctx, &self.buffer).await?; let block_number = block.header.number; @@ -267,9 +288,13 @@ impl WriteBlockStore for Buffered { if let Some(block) = next_block_for_store { self.inner.schedule_next_block(ctx, &block).await?; - tracing::trace!(block_number = %block.header.number, "Block scheduled in underlying storage"); + tracing::debug!( + block_number = block.header.number.0, + "Block scheduled in underlying storage" + ); } self.block_writes_sender.send_replace(block.header.number); + buffer_block_latency.observe(); Ok(()) } } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/metrics.rs b/core/lib/zksync_core/src/sync_layer/gossip/metrics.rs new file mode 100644 index 000000000000..f67c150b99c0 --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/gossip/metrics.rs @@ -0,0 +1,29 @@ +//! Metrics for gossip-powered syncing. + +use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; + +use std::time::Duration; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)] +#[metrics(label = "kind", rename_all = "snake_case")] +pub(super) enum BlockResponseKind { + Persisted, + InMemory, +} + +#[derive(Debug, Metrics)] +#[metrics(prefix = "external_node_gossip_fetcher")] +pub(super) struct GossipFetcherMetrics { + /// Number of currently buffered unexecuted blocks. + pub buffer_size: Gauge, + /// Latency of a `get_block` call. + #[metrics(unit = Unit::Seconds, buckets = Buckets::LATENCIES)] + pub get_block_latency: Family>, + /// Latency of putting a block into the buffered storage. This may include the time to queue + /// block actions, but does not include block execution. + #[metrics(unit = Unit::Seconds, buckets = Buckets::LATENCIES)] + pub buffer_block_latency: Histogram, +} + +#[vise::register] +pub(super) static METRICS: vise::Global = vise::Global::new(); diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index 3367914e52f9..cdabd4f5df63 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -12,6 +12,7 @@ use zksync_dal::ConnectionPool; mod buffered; mod conversions; +mod metrics; mod storage; #[cfg(test)] mod tests; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index 9db8a7162722..d1936e4d9665 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -177,7 +177,7 @@ impl BlockStore for PostgresBlockStorage { #[async_trait] impl ContiguousBlockStore for PostgresBlockStorage { async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { - self.schedule_block(ctx, block).await.map_err(Into::into) + self.schedule_block(ctx, block).await } } From 3e0bc8141ac18afafb01c8219b3c0fb9a58c7d23 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 2 Nov 2023 14:30:59 +0200 Subject: [PATCH 13/38] Update revision for consensus deps --- Cargo.lock | 38 +++++++++---------- core/lib/zksync_core/Cargo.toml | 8 ++-- .../src/sync_layer/gossip/tests.rs | 8 ++-- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcbda290c855..1e31b4430862 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1443,7 +1443,7 @@ dependencies = [ [[package]] name = "concurrency" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "once_cell", @@ -1461,7 +1461,7 @@ dependencies = [ [[package]] name = "consensus" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "concurrency", @@ -1805,7 +1805,7 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "blst", @@ -1948,9 +1948,9 @@ dependencies = [ [[package]] name = "curve25519-dalek-derive" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", @@ -2368,7 +2368,7 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "executor" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "concurrency", @@ -2440,9 +2440,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" +checksum = "a481586acf778f1b1455424c343f71124b048ffa5f4fc3f8f6ae9dc432dcb3c7" [[package]] name = "findshlibs" @@ -3402,9 +3402,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", @@ -4337,7 +4337,7 @@ dependencies = [ [[package]] name = "network" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "async-trait", @@ -5018,7 +5018,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.2", + "indexmap 2.1.0", ] [[package]] @@ -5103,9 +5103,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platforms" -version = "3.1.2" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" +checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" [[package]] name = "plotters" @@ -6025,7 +6025,7 @@ dependencies = [ [[package]] name = "roles" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "bit-vec", @@ -6195,7 +6195,7 @@ dependencies = [ [[package]] name = "schema" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "bit-vec", @@ -6969,7 +6969,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "storage" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "async-trait", @@ -7101,7 +7101,7 @@ dependencies = [ [[package]] name = "sync_blocks" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "anyhow", "concurrency", @@ -7820,7 +7820,7 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=470e75d9318c2961ecfaf1d6e9223eeb5326a189#470e75d9318c2961ecfaf1d6e9223eeb5326a189" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" dependencies = [ "concurrency", "thiserror", diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index 19cb4af7638b..45d01081ed0d 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -40,10 +40,10 @@ vlog = { path = "../vlog" } multivm = { path = "../multivm" } # Consensus dependenices -zksync_concurrency = { package = "concurrency", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } -zksync_consensus_roles = { package = "roles", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } -zksync_consensus_storage = { package = "storage", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } -zksync_consensus_executor = { package = "executor", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "470e75d9318c2961ecfaf1d6e9223eeb5326a189" } +zksync_concurrency = { package = "concurrency", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } +zksync_consensus_roles = { package = "roles", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } +zksync_consensus_storage = { package = "storage", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } +zksync_consensus_executor = { package = "executor", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 3709bae4f5b8..91f8d03203a0 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -112,8 +112,8 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); let mut validator = - FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()); - let external_node = validator.connect_external_node(rng); + FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()).await; + let external_node = validator.connect_external_node(rng).await; let validator_storage = Arc::new(InMemoryStorage::new(genesis_block)); if !delay_first_block { @@ -231,8 +231,8 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); let mut validator = - FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()); - let external_node = validator.connect_external_node(rng); + FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()).await; + let external_node = validator.connect_external_node(rng).await; let validator_storage = Arc::new(InMemoryStorage::new(genesis_block)); for block in initial_blocks { From 7d89e00e38d5b452b519f5679bd8878b89f66e56 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 2 Nov 2023 17:07:08 +0200 Subject: [PATCH 14/38] Add consensus-specific fields for miniblocks --- ...d_consensus_fields_for_miniblocks.down.sql | 4 + ...add_consensus_fields_for_miniblocks.up.sql | 4 + core/lib/dal/sqlx-data.json | 196 ++++++++++-------- core/lib/dal/src/models/storage_sync.rs | 8 +- core/lib/dal/src/sync_dal.rs | 36 ++-- core/lib/types/src/api/en.rs | 7 +- core/lib/zksync_core/src/sync_layer/tests.rs | 2 + 7 files changed, 145 insertions(+), 112 deletions(-) create mode 100644 core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.down.sql create mode 100644 core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.up.sql diff --git a/core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.down.sql b/core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.down.sql new file mode 100644 index 000000000000..3ca723d4d713 --- /dev/null +++ b/core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.down.sql @@ -0,0 +1,4 @@ +ALTER TABLE miniblocks + DROP COLUMN IF EXISTS commit_qc; +ALTER TABLE miniblocks + DROP COLUMN IF EXISTS prev_consensus_block_hash; diff --git a/core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.up.sql b/core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.up.sql new file mode 100644 index 000000000000..da821ddc98b9 --- /dev/null +++ b/core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.up.sql @@ -0,0 +1,4 @@ +ALTER TABLE miniblocks + ADD COLUMN commit_qc BYTEA NULL; +ALTER TABLE miniblocks + ADD COLUMN prev_consensus_block_hash BYTEA NULL; diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index 7fc1026a3d5a..7abf8169bd19 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -3854,98 +3854,6 @@ }, "query": "UPDATE eth_txs SET has_failed = TRUE WHERE id = $1" }, - "5190fad25f0c476380af4013761d42ae97dbd55f87e38ceec33f8e148c5cbb14": { - "describe": { - "columns": [ - { - "name": "number", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "l1_batch_number!", - "ordinal": 1, - "type_info": "Int8" - }, - { - "name": "last_batch_miniblock?", - "ordinal": 2, - "type_info": "Int8" - }, - { - "name": "timestamp", - "ordinal": 3, - "type_info": "Int8" - }, - { - "name": "root_hash?", - "ordinal": 4, - "type_info": "Bytea" - }, - { - "name": "l1_gas_price", - "ordinal": 5, - "type_info": "Int8" - }, - { - "name": "l2_fair_gas_price", - "ordinal": 6, - "type_info": "Int8" - }, - { - "name": "bootloader_code_hash", - "ordinal": 7, - "type_info": "Bytea" - }, - { - "name": "default_aa_code_hash", - "ordinal": 8, - "type_info": "Bytea" - }, - { - "name": "virtual_blocks", - "ordinal": 9, - "type_info": "Int8" - }, - { - "name": "hash", - "ordinal": 10, - "type_info": "Bytea" - }, - { - "name": "protocol_version!", - "ordinal": 11, - "type_info": "Int4" - }, - { - "name": "fee_account_address?", - "ordinal": 12, - "type_info": "Bytea" - } - ], - "nullable": [ - false, - null, - null, - false, - false, - false, - false, - true, - true, - false, - false, - true, - false - ], - "parameters": { - "Left": [ - "Int8" - ] - } - }, - "query": "\n SELECT miniblocks.number,\n COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\",\n (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\",\n miniblocks.timestamp,\n miniblocks.hash as \"root_hash?\",\n miniblocks.l1_gas_price,\n miniblocks.l2_fair_gas_price,\n miniblocks.bootloader_code_hash,\n miniblocks.default_aa_code_hash,\n miniblocks.virtual_blocks,\n miniblocks.hash,\n miniblocks.protocol_version as \"protocol_version!\",\n l1_batches.fee_account_address as \"fee_account_address?\"\n FROM miniblocks\n LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number\n WHERE miniblocks.number = $1\n " - }, "51cb712685991ffd600dce59f5ed8b5a1bfce8feed46ebd02471c43802e6e65a": { "describe": { "columns": [ @@ -9283,6 +9191,110 @@ }, "query": "UPDATE proof_compression_jobs_fri SET status = $1, attempts = attempts + 1, updated_at = now(), processing_started_at = now(), picked_by = $3 WHERE l1_batch_number = ( SELECT l1_batch_number FROM proof_compression_jobs_fri WHERE status = $2 ORDER BY l1_batch_number ASC LIMIT 1 FOR UPDATE SKIP LOCKED ) RETURNING proof_compression_jobs_fri.l1_batch_number" }, + "bc8e2479b25c3f173888f0dfdace9be8fac8547d91982d3b8fa5ee0f9f86a707": { + "describe": { + "columns": [ + { + "name": "number", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "l1_batch_number!", + "ordinal": 1, + "type_info": "Int8" + }, + { + "name": "last_batch_miniblock?", + "ordinal": 2, + "type_info": "Int8" + }, + { + "name": "timestamp", + "ordinal": 3, + "type_info": "Int8" + }, + { + "name": "root_hash?", + "ordinal": 4, + "type_info": "Bytea" + }, + { + "name": "l1_gas_price", + "ordinal": 5, + "type_info": "Int8" + }, + { + "name": "l2_fair_gas_price", + "ordinal": 6, + "type_info": "Int8" + }, + { + "name": "bootloader_code_hash", + "ordinal": 7, + "type_info": "Bytea" + }, + { + "name": "default_aa_code_hash", + "ordinal": 8, + "type_info": "Bytea" + }, + { + "name": "virtual_blocks", + "ordinal": 9, + "type_info": "Int8" + }, + { + "name": "hash", + "ordinal": 10, + "type_info": "Bytea" + }, + { + "name": "commit_qc", + "ordinal": 11, + "type_info": "Bytea" + }, + { + "name": "prev_consensus_block_hash", + "ordinal": 12, + "type_info": "Bytea" + }, + { + "name": "protocol_version!", + "ordinal": 13, + "type_info": "Int4" + }, + { + "name": "fee_account_address?", + "ordinal": 14, + "type_info": "Bytea" + } + ], + "nullable": [ + false, + null, + null, + false, + false, + false, + false, + true, + true, + false, + false, + true, + true, + true, + false + ], + "parameters": { + "Left": [ + "Int8" + ] + } + }, + "query": "SELECT miniblocks.number, COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\", (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\", miniblocks.timestamp, miniblocks.hash as \"root_hash?\", miniblocks.l1_gas_price, miniblocks.l2_fair_gas_price, miniblocks.bootloader_code_hash, miniblocks.default_aa_code_hash, miniblocks.virtual_blocks, miniblocks.hash, miniblocks.commit_qc, miniblocks.prev_consensus_block_hash,\n miniblocks.protocol_version as \"protocol_version!\", l1_batches.fee_account_address as \"fee_account_address?\" FROM miniblocks LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number WHERE miniblocks.number = $1" + }, "be824de76050461afe29dfd229e524bdf113eab3ca24208782c200531db1c940": { "describe": { "columns": [ diff --git a/core/lib/dal/src/models/storage_sync.rs b/core/lib/dal/src/models/storage_sync.rs index 052fadcf60a3..64acedf0ab82 100644 --- a/core/lib/dal/src/models/storage_sync.rs +++ b/core/lib/dal/src/models/storage_sync.rs @@ -3,7 +3,7 @@ use std::convert::TryInto; use zksync_contracts::BaseSystemContractsHashes; use zksync_types::api::en::SyncBlock; use zksync_types::Transaction; -use zksync_types::{Address, L1BatchNumber, MiniblockNumber, H256}; +use zksync_types::{Address, Bytes, L1BatchNumber, MiniblockNumber, H256}; #[derive(Debug, Clone, sqlx::FromRow)] pub struct StorageSyncBlock { @@ -22,6 +22,8 @@ pub struct StorageSyncBlock { pub protocol_version: i32, pub virtual_blocks: i64, pub hash: Vec, + pub commit_qc: Option>, + pub prev_consensus_block_hash: Option>, } impl StorageSyncBlock { @@ -60,6 +62,10 @@ impl StorageSyncBlock { virtual_blocks: Some(self.virtual_blocks as u32), hash: Some(H256::from_slice(&self.hash)), protocol_version: (self.protocol_version as u16).try_into().unwrap(), + prev_consensus_block_hash: self + .prev_consensus_block_hash + .map(|hash| H256::from_slice(&hash)), + commit_qc_bytes: self.commit_qc.map(Bytes::from), } } } diff --git a/core/lib/dal/src/sync_dal.rs b/core/lib/dal/src/sync_dal.rs index ab905dd5cb22..ac4dd1829052 100644 --- a/core/lib/dal/src/sync_dal.rs +++ b/core/lib/dal/src/sync_dal.rs @@ -23,24 +23,24 @@ impl SyncDal<'_, '_> { let latency = MethodLatency::new("sync_dal_sync_block"); let storage_block_details = sqlx::query_as!( StorageSyncBlock, - r#" - SELECT miniblocks.number, - COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as "l1_batch_number!", - (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as "last_batch_miniblock?", - miniblocks.timestamp, - miniblocks.hash as "root_hash?", - miniblocks.l1_gas_price, - miniblocks.l2_fair_gas_price, - miniblocks.bootloader_code_hash, - miniblocks.default_aa_code_hash, - miniblocks.virtual_blocks, - miniblocks.hash, - miniblocks.protocol_version as "protocol_version!", - l1_batches.fee_account_address as "fee_account_address?" - FROM miniblocks - LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number - WHERE miniblocks.number = $1 - "#, + "SELECT miniblocks.number, \ + COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\", \ + (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\", \ + miniblocks.timestamp, \ + miniblocks.hash as \"root_hash?\", \ + miniblocks.l1_gas_price, \ + miniblocks.l2_fair_gas_price, \ + miniblocks.bootloader_code_hash, \ + miniblocks.default_aa_code_hash, \ + miniblocks.virtual_blocks, \ + miniblocks.hash, \ + miniblocks.commit_qc, \ + miniblocks.prev_consensus_block_hash, + miniblocks.protocol_version as \"protocol_version!\", \ + l1_batches.fee_account_address as \"fee_account_address?\" \ + FROM miniblocks \ + LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number \ + WHERE miniblocks.number = $1", block_number.0 as i64 ) .instrument("sync_dal_sync_block.block") diff --git a/core/lib/types/src/api/en.rs b/core/lib/types/src/api/en.rs index 18a83f9f821b..213f3baf7589 100644 --- a/core/lib/types/src/api/en.rs +++ b/core/lib/types/src/api/en.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use zk_evm::ethereum_types::Address; -use zksync_basic_types::{L1BatchNumber, MiniblockNumber, H256}; +use zksync_basic_types::{Bytes, L1BatchNumber, MiniblockNumber, H256}; use zksync_contracts::BaseSystemContractsHashes; use crate::ProtocolVersionId; @@ -44,4 +44,9 @@ pub struct SyncBlock { pub hash: Option, /// Version of the protocol used for this block. pub protocol_version: ProtocolVersionId, + + /// Hash of the previous consensus block. + pub prev_consensus_block_hash: Option, + /// Protobuf serialization of quorum certificate for the block. + pub commit_qc_bytes: Option, } diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index eccf29ec3699..dd6f35448ce9 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -69,6 +69,8 @@ impl MockMainNodeClient { virtual_blocks: Some(!is_fictive as u32), hash: Some(H256::repeat_byte(1)), protocol_version: ProtocolVersionId::latest(), + prev_consensus_block_hash: None, + commit_qc_bytes: None, } }); From 2ee6142f3bb4b538475ef5452579496e841fb6ca Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 2 Nov 2023 17:21:45 +0200 Subject: [PATCH 15/38] Use new fields in block conversions --- Cargo.lock | 1 + core/lib/zksync_core/Cargo.toml | 1 + .../src/sync_layer/gossip/conversions.rs | 31 +++++++++---------- .../src/sync_layer/gossip/storage.rs | 3 +- .../src/sync_layer/gossip/tests.rs | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e31b4430862..34abb1fa5e01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8762,6 +8762,7 @@ dependencies = [ "rand 0.8.5", "reqwest", "roles", + "schema", "serde", "serde_json", "storage", diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index 45d01081ed0d..e5c4f648bffd 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -41,6 +41,7 @@ vlog = { path = "../vlog" } multivm = { path = "../multivm" } # Consensus dependenices zksync_concurrency = { package = "concurrency", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } +zksync_consensus_schema = { package = "schema", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } zksync_consensus_roles = { package = "roles", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } zksync_consensus_storage = { package = "storage", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } zksync_consensus_executor = { package = "executor", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index e8643d4b03b4..29be9bf3b491 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -1,12 +1,10 @@ //! Conversion logic between server and consensus types. use anyhow::Context as _; -use rand::{thread_rng, Rng}; use serde::{Deserialize, Serialize}; use zksync_consensus_roles::validator::{ - BlockHeader, BlockNumber, CommitQC, FinalBlock, Payload, ReplicaCommit, ViewNumber, - CURRENT_VERSION, + BlockHeader, BlockHeaderHash, BlockNumber, CommitQC, FinalBlock, Payload, }; use zksync_types::{ api::en::SyncBlock, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, @@ -27,7 +25,7 @@ struct BlockPayload { transactions: Vec, } -pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> FinalBlock { +pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> anyhow::Result { let payload = serde_json::to_vec(&BlockPayload { hash: block.hash.unwrap_or_default(), l1_batch_number: block.l1_batch_number, @@ -38,26 +36,25 @@ pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> FinalBlock { operator_address: block.operator_address, transactions: block .transactions - .expect("Transactions are always requested"), + .context("Transactions are always requested")?, }); - let payload = Payload(payload.expect("Failed serializing block payload")); + let payload = Payload(payload.context("Failed serializing block payload")?); + let prev_block_hash = block + .prev_consensus_block_hash + .context("Missing previous block hash")?; let header = BlockHeader { - parent: thread_rng().gen(), // FIXME + parent: BlockHeaderHash::from_bytes(prev_block_hash.0), number: BlockNumber(block.number.0.into()), payload: payload.hash(), }; - FinalBlock { + let justification = block.commit_qc_bytes.context("Missing commit QC")?; + let justification: CommitQC = zksync_consensus_schema::decode(&justification.0) + .context("Failed deserializing commit QC from Protobuf")?; + Ok(FinalBlock { header, payload, - justification: CommitQC { - message: ReplicaCommit { - protocol_version: CURRENT_VERSION, - view: ViewNumber(header.number.0), - proposal: header, - }, - ..thread_rng().gen() // FIXME - }, - } + justification, + }) } impl FetchedBlock { diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index d1936e4d9665..350aca48941b 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -91,7 +91,8 @@ impl PostgresBlockStorage { else { return Ok(None); }; - Ok(Some(sync_block_to_consensus_block(block))) + let block = sync_block_to_consensus_block(block)?; + Ok(Some(block)) } async fn first_block(&self) -> anyhow::Result { diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 91f8d03203a0..f14f54c6f5c9 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -36,7 +36,7 @@ pub(super) async fn load_final_block( .await .unwrap() .unwrap_or_else(|| panic!("no sync block #{number}")); - conversions::sync_block_to_consensus_block(sync_block) + conversions::sync_block_to_consensus_block(sync_block).unwrap() } pub(super) async fn assert_first_block_actions(actions: &mut ActionQueue) -> Vec { From e46f1e4fdabf01a7232e278b9e16ff7ddd91cb9b Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 3 Nov 2023 10:03:14 +0200 Subject: [PATCH 16/38] Sketch consensus fields storage in seal logic --- core/lib/dal/sqlx-data.json | 14 ++++++++++ core/lib/dal/src/blocks_dal.rs | 26 ++++++++++++++++- core/lib/dal/src/models/storage_sync.rs | 20 ++++++++----- core/lib/types/src/api/en.rs | 12 ++++---- core/lib/types/src/block.rs | 12 +++++++- .../src/state_keeper/io/mempool.rs | 2 ++ .../src/state_keeper/io/seal_logic.rs | 11 +++++++- .../src/state_keeper/io/tests/mod.rs | 6 ++++ .../src/state_keeper/updates/mod.rs | 10 +++++-- .../zksync_core/src/sync_layer/external_io.rs | 28 ++++++++++--------- .../lib/zksync_core/src/sync_layer/fetcher.rs | 8 ++++-- .../src/sync_layer/gossip/conversions.rs | 13 +++++---- .../src/sync_layer/gossip/tests.rs | 8 +++--- .../zksync_core/src/sync_layer/sync_action.rs | 20 +++++++++---- core/lib/zksync_core/src/sync_layer/tests.rs | 26 +++++++++-------- 15 files changed, 154 insertions(+), 62 deletions(-) diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index 7abf8169bd19..a9da03a5de9e 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -8641,6 +8641,20 @@ }, "query": "SELECT * FROM eth_txs_history WHERE eth_tx_id = $1 ORDER BY created_at DESC LIMIT 1" }, + "ad48c26546edaa5f872e5698eb0b0d3ced291db178dd6c065fe6a39def4ea751": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8", + "Bytea", + "Bytea" + ] + } + }, + "query": "UPDATE miniblocks SET prev_consensus_block_hash = $2, commit_qc = $3 WHERE number = $1" + }, "ad4f74aa6f131df0243f4fa500ade1b98aa335bd71ed417b02361e2c697e60f8": { "describe": { "columns": [], diff --git a/core/lib/dal/src/blocks_dal.rs b/core/lib/dal/src/blocks_dal.rs index bec95eb4df9b..ba7f53acd3ac 100644 --- a/core/lib/dal/src/blocks_dal.rs +++ b/core/lib/dal/src/blocks_dal.rs @@ -10,7 +10,7 @@ use sqlx::Row; use zksync_types::{ aggregated_operations::AggregatedActionType, - block::{BlockGasCount, L1BatchHeader, MiniblockHeader}, + block::{BlockGasCount, ConsensusBlockFields, L1BatchHeader, MiniblockHeader}, commitment::{L1BatchMetadata, L1BatchWithMetadata}, Address, L1BatchNumber, LogQuery, MiniblockNumber, ProtocolVersionId, H256, MAX_GAS_PER_PUBDATA_BYTE, U256, @@ -466,6 +466,30 @@ impl BlocksDal<'_, '_> { Ok(()) } + /// Sets consensus-related fields for the specified miniblock. + pub async fn set_miniblock_consensus_fields( + &mut self, + miniblock_number: MiniblockNumber, + consensus: &ConsensusBlockFields, + ) -> anyhow::Result<()> { + let result = sqlx::query!( + "UPDATE miniblocks \ + SET prev_consensus_block_hash = $2, commit_qc = $3 \ + WHERE number = $1", + miniblock_number.0 as i64, + consensus.prev_block_hash.as_bytes(), + &consensus.commit_qc_bytes.0 + ) + .execute(self.storage.conn()) + .await?; + + anyhow::ensure!( + result.rows_affected() == 1, + "Miniblock #{miniblock_number} is not present in Postgres" + ); + Ok(()) + } + pub async fn update_hashes( &mut self, number_and_hashes: &[(MiniblockNumber, H256)], diff --git a/core/lib/dal/src/models/storage_sync.rs b/core/lib/dal/src/models/storage_sync.rs index 64acedf0ab82..af6273885e03 100644 --- a/core/lib/dal/src/models/storage_sync.rs +++ b/core/lib/dal/src/models/storage_sync.rs @@ -2,8 +2,9 @@ use std::convert::TryInto; use zksync_contracts::BaseSystemContractsHashes; use zksync_types::api::en::SyncBlock; -use zksync_types::Transaction; -use zksync_types::{Address, Bytes, L1BatchNumber, MiniblockNumber, H256}; +use zksync_types::{ + block::ConsensusBlockFields, Address, Bytes, L1BatchNumber, MiniblockNumber, Transaction, H256, +}; #[derive(Debug, Clone, sqlx::FromRow)] pub struct StorageSyncBlock { @@ -32,12 +33,15 @@ impl StorageSyncBlock { current_operator_address: Address, transactions: Option>, ) -> SyncBlock { + let number = self.number; + let commit_qc = self.commit_qc; + SyncBlock { number: MiniblockNumber(self.number as u32), l1_batch_number: L1BatchNumber(self.l1_batch_number as u32), last_in_batch: self .last_batch_miniblock - .map(|n| n == self.number) + .map(|n| n == number) .unwrap_or(false), timestamp: self.timestamp as u64, root_hash: self.root_hash.as_deref().map(H256::from_slice), @@ -62,10 +66,12 @@ impl StorageSyncBlock { virtual_blocks: Some(self.virtual_blocks as u32), hash: Some(H256::from_slice(&self.hash)), protocol_version: (self.protocol_version as u16).try_into().unwrap(), - prev_consensus_block_hash: self - .prev_consensus_block_hash - .map(|hash| H256::from_slice(&hash)), - commit_qc_bytes: self.commit_qc.map(Bytes::from), + consensus: self.prev_consensus_block_hash.and_then(|hash| { + Some(ConsensusBlockFields { + prev_block_hash: H256::from_slice(&hash), + commit_qc_bytes: Bytes(commit_qc?), + }) + }), } } } diff --git a/core/lib/types/src/api/en.rs b/core/lib/types/src/api/en.rs index 213f3baf7589..aa3d2955e2e0 100644 --- a/core/lib/types/src/api/en.rs +++ b/core/lib/types/src/api/en.rs @@ -2,10 +2,10 @@ use serde::{Deserialize, Serialize}; use zk_evm::ethereum_types::Address; -use zksync_basic_types::{Bytes, L1BatchNumber, MiniblockNumber, H256}; +use zksync_basic_types::{L1BatchNumber, MiniblockNumber, H256}; use zksync_contracts::BaseSystemContractsHashes; -use crate::ProtocolVersionId; +use crate::{block::ConsensusBlockFields, ProtocolVersionId}; /// Representation of the L2 block, as needed for the EN synchronization. /// This structure has several fields that describe *L1 batch* rather than @@ -44,9 +44,7 @@ pub struct SyncBlock { pub hash: Option, /// Version of the protocol used for this block. pub protocol_version: ProtocolVersionId, - - /// Hash of the previous consensus block. - pub prev_consensus_block_hash: Option, - /// Protobuf serialization of quorum certificate for the block. - pub commit_qc_bytes: Option, + /// Consensus-related information about the block. Not present if consensus is not enabled + /// for the environment. + pub consensus: Option, } diff --git a/core/lib/types/src/block.rs b/core/lib/types/src/block.rs index c41fdd93ed98..3d10ab77c3cc 100644 --- a/core/lib/types/src/block.rs +++ b/core/lib/types/src/block.rs @@ -3,7 +3,7 @@ use zksync_system_constants::SYSTEM_BLOCK_INFO_BLOCK_NUMBER_MULTIPLIER; use std::{fmt, ops}; -use zksync_basic_types::{H2048, H256, U256}; +use zksync_basic_types::{Bytes, H2048, H256, U256}; use zksync_contracts::BaseSystemContractsHashes; use crate::{ @@ -84,6 +84,16 @@ pub struct MiniblockHeader { pub virtual_blocks: u32, } +/// Consensus-related L2 block (= miniblock) fields. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConsensusBlockFields { + /// Hash of the previous consensus block. + pub prev_block_hash: H256, + /// Protobuf serialization of quorum certificate for the block. + pub commit_qc_bytes: Bytes, +} + /// Data needed to execute a miniblock in the VM. #[derive(Debug)] pub struct MiniblockExecutionData { diff --git a/core/lib/zksync_core/src/state_keeper/io/mempool.rs b/core/lib/zksync_core/src/state_keeper/io/mempool.rs index fced87479283..cb5078cef144 100644 --- a/core/lib/zksync_core/src/state_keeper/io/mempool.rs +++ b/core/lib/zksync_core/src/state_keeper/io/mempool.rs @@ -274,6 +274,7 @@ where self.current_l1_batch_number, self.current_miniblock_number, self.l2_erc20_bridge_addr, + None, ); self.miniblock_sealer_handle.submit(command).await; self.current_miniblock_number += 1; @@ -326,6 +327,7 @@ where l1_batch_env, finished_batch, self.l2_erc20_bridge_addr, + None, ) .await; self.current_miniblock_number += 1; // Due to fictive miniblock being sealed. diff --git a/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs b/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs index 181d2f4df8bd..02e452313d45 100644 --- a/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs +++ b/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs @@ -15,7 +15,7 @@ use zksync_types::{ block::unpack_block_info, CURRENT_VIRTUAL_BLOCK_INFO_POSITION, SYSTEM_CONTEXT_ADDRESS, }; use zksync_types::{ - block::{L1BatchHeader, MiniblockHeader}, + block::{ConsensusBlockFields, L1BatchHeader, MiniblockHeader}, event::{extract_added_tokens, extract_long_l2_to_l1_messages}, l2_to_l1_log::L2ToL1Log, storage_writes_deduplicator::{ModifiedSlot, StorageWritesDeduplicator}, @@ -50,6 +50,7 @@ impl UpdatesManager { l1_batch_env: &L1BatchEnv, finished_batch: FinishedL1Batch, l2_erc20_bridge_addr: Address, + fictive_miniblock_consensus_fields: Option, ) { let started_at = Instant::now(); let progress = L1_BATCH_METRICS.start(L1BatchSealStage::VmFinalization); @@ -63,6 +64,7 @@ impl UpdatesManager { l1_batch_env.number, current_miniblock_number, l2_erc20_bridge_addr, + fictive_miniblock_consensus_fields, ); miniblock_command.seal_inner(&mut transaction, true).await; progress.observe(None); @@ -307,6 +309,13 @@ impl MiniblockSealCommand { .insert_miniblock(&miniblock_header) .await .unwrap(); + if let Some(consensus) = &self.consensus { + transaction + .blocks_dal() + .set_miniblock_consensus_fields(miniblock_number, consensus) + .await + .unwrap(); + } progress.observe(None); let progress = diff --git a/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs b/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs index e70964c4957f..6d09e48b62e9 100644 --- a/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs +++ b/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs @@ -245,6 +245,7 @@ async fn processing_storage_logs_when_sealing_miniblock() { base_system_contracts_hashes: BaseSystemContractsHashes::default(), protocol_version: Some(ProtocolVersionId::latest()), l2_erc20_bridge_addr: Address::default(), + consensus: None, }; let mut conn = connection_pool .access_storage_tagged("state_keeper") @@ -321,6 +322,7 @@ async fn processing_events_when_sealing_miniblock() { base_system_contracts_hashes: BaseSystemContractsHashes::default(), protocol_version: Some(ProtocolVersionId::latest()), l2_erc20_bridge_addr: Address::default(), + consensus: None, }; let mut conn = pool.access_storage_tagged("state_keeper").await.unwrap(); conn.protocol_versions_dal() @@ -434,6 +436,7 @@ async fn miniblock_sealer_handle_blocking() { L1BatchNumber(1), MiniblockNumber(1), Address::default(), + None, ); sealer_handle.submit(seal_command).await; @@ -442,6 +445,7 @@ async fn miniblock_sealer_handle_blocking() { L1BatchNumber(1), MiniblockNumber(2), Address::default(), + None, ); { let submit_future = sealer_handle.submit(seal_command); @@ -470,6 +474,7 @@ async fn miniblock_sealer_handle_blocking() { L1BatchNumber(2), MiniblockNumber(3), Address::default(), + None, ); sealer_handle.submit(seal_command).await; let command = sealer.commands_receiver.recv().await.unwrap(); @@ -489,6 +494,7 @@ async fn miniblock_sealer_handle_parallel_processing() { L1BatchNumber(1), MiniblockNumber(i), Address::default(), + None, ); sealer_handle.submit(seal_command).await; } diff --git a/core/lib/zksync_core/src/state_keeper/updates/mod.rs b/core/lib/zksync_core/src/state_keeper/updates/mod.rs index dc72893e7034..3b36c5ba8c70 100644 --- a/core/lib/zksync_core/src/state_keeper/updates/mod.rs +++ b/core/lib/zksync_core/src/state_keeper/updates/mod.rs @@ -3,9 +3,10 @@ use multivm::interface::{L1BatchEnv, VmExecutionResultAndLogs}; use zksync_contracts::BaseSystemContractsHashes; use zksync_types::vm_trace::Call; use zksync_types::{ - block::BlockGasCount, storage_writes_deduplicator::StorageWritesDeduplicator, - tx::tx_execution_info::ExecutionMetrics, Address, L1BatchNumber, MiniblockNumber, - ProtocolVersionId, Transaction, + block::{BlockGasCount, ConsensusBlockFields}, + storage_writes_deduplicator::StorageWritesDeduplicator, + tx::tx_execution_info::ExecutionMetrics, + Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, Transaction, }; use zksync_utils::bytecode::CompressedBytecodeInfo; @@ -81,6 +82,7 @@ impl UpdatesManager { l1_batch_number: L1BatchNumber, miniblock_number: MiniblockNumber, l2_erc20_bridge_addr: Address, + consensus: Option, ) -> MiniblockSealCommand { MiniblockSealCommand { l1_batch_number, @@ -93,6 +95,7 @@ impl UpdatesManager { base_system_contracts_hashes: self.base_system_contract_hashes, protocol_version: Some(self.protocol_version), l2_erc20_bridge_addr, + consensus, } } @@ -172,6 +175,7 @@ pub(crate) struct MiniblockSealCommand { pub base_system_contracts_hashes: BaseSystemContractsHashes, pub protocol_version: Option, pub l2_erc20_bridge_addr: Address, + pub consensus: Option, } #[cfg(test)] diff --git a/core/lib/zksync_core/src/sync_layer/external_io.rs b/core/lib/zksync_core/src/sync_layer/external_io.rs index c451587ff025..ac765f9e51ce 100644 --- a/core/lib/zksync_core/src/sync_layer/external_io.rs +++ b/core/lib/zksync_core/src/sync_layer/external_io.rs @@ -273,7 +273,10 @@ impl IoSealCriteria for ExternalIO { } fn should_seal_miniblock(&mut self, _manager: &UpdatesManager) -> bool { - matches!(self.actions.peek_action(), Some(SyncAction::SealMiniblock)) + matches!( + self.actions.peek_action(), + Some(SyncAction::SealMiniblock(_)) + ) } } @@ -426,7 +429,7 @@ impl StateKeeperIO for ExternalIO { virtual_blocks, }); } - Some(SyncAction::SealBatch { virtual_blocks }) => { + Some(SyncAction::SealBatch { virtual_blocks, .. }) => { // We've reached the next batch, so this situation would be handled by the batch sealer. // No need to pop the action from the queue. // It also doesn't matter which timestamp we return, since there will be no more miniblocks in this @@ -492,12 +495,9 @@ impl StateKeeperIO for ExternalIO { } async fn seal_miniblock(&mut self, updates_manager: &UpdatesManager) { - match self.actions.pop_action() { - Some(SyncAction::SealMiniblock) => {} - other => panic!( - "State keeper requested to seal miniblock, but the next action is {:?}", - other - ), + let action = self.actions.pop_action(); + let Some(SyncAction::SealMiniblock(consensus)) = action else { + panic!("State keeper requested to seal miniblock, but the next action is {action:?}"); }; let mut storage = self.pool.access_storage_tagged("sync_layer").await.unwrap(); @@ -537,6 +537,7 @@ impl StateKeeperIO for ExternalIO { self.current_l1_batch_number, self.current_miniblock_number, self.l2_erc20_bridge_addr, + consensus, ); command.seal(&mut transaction).await; transaction.commit().await.unwrap(); @@ -555,11 +556,11 @@ impl StateKeeperIO for ExternalIO { l1_batch_env: &L1BatchEnv, finished_batch: FinishedL1Batch, ) -> anyhow::Result<()> { - match self.actions.pop_action() { - Some(SyncAction::SealBatch { .. }) => {} - other => anyhow::bail!( - "State keeper requested to seal the batch, but the next action is {other:?}" - ), + let action = self.actions.pop_action(); + let Some(SyncAction::SealBatch { consensus, .. }) = action else { + anyhow::bail!( + "State keeper requested to seal the batch, but the next action is {action:?}" + ); }; let mut storage = self.pool.access_storage_tagged("sync_layer").await.unwrap(); @@ -570,6 +571,7 @@ impl StateKeeperIO for ExternalIO { l1_batch_env, finished_batch, self.l2_erc20_bridge_addr, + consensus, ) .await; diff --git a/core/lib/zksync_core/src/sync_layer/fetcher.rs b/core/lib/zksync_core/src/sync_layer/fetcher.rs index 46ac2d7e03a1..f22f06ee2609 100644 --- a/core/lib/zksync_core/src/sync_layer/fetcher.rs +++ b/core/lib/zksync_core/src/sync_layer/fetcher.rs @@ -5,7 +5,8 @@ use std::time::Duration; use zksync_dal::StorageProcessor; use zksync_types::{ - api::en::SyncBlock, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, + api::en::SyncBlock, block::ConsensusBlockFields, Address, L1BatchNumber, MiniblockNumber, + ProtocolVersionId, H256, }; use zksync_web3_decl::jsonrpsee::core::Error as RpcError; @@ -33,6 +34,7 @@ pub(super) struct FetchedBlock { pub virtual_blocks: u32, pub operator_address: Address, pub transactions: Vec, + pub consensus: Option, } impl FetchedBlock { @@ -50,6 +52,7 @@ impl FetchedBlock { transactions: block .transactions .expect("Transactions are always requested"), + consensus: block.consensus, } } } @@ -159,9 +162,10 @@ impl FetcherCursor { new_actions.push(SyncAction::SealBatch { // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. virtual_blocks: block.virtual_blocks, + consensus: block.consensus, }); } else { - new_actions.push(SyncAction::SealMiniblock); + new_actions.push(SyncAction::SealMiniblock(block.consensus)); } self.next_miniblock += 1; self.prev_miniblock_hash = block.hash; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index 29be9bf3b491..58ac779fb5c0 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use zksync_consensus_roles::validator::{ BlockHeader, BlockHeaderHash, BlockNumber, CommitQC, FinalBlock, Payload, }; +use zksync_types::block::ConsensusBlockFields; use zksync_types::{ api::en::SyncBlock, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, }; @@ -39,16 +40,14 @@ pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> anyhow::Result< .context("Transactions are always requested")?, }); let payload = Payload(payload.context("Failed serializing block payload")?); - let prev_block_hash = block - .prev_consensus_block_hash - .context("Missing previous block hash")?; + let consensus = block.consensus.context("Missing consensus fields")?; + let prev_block_hash = consensus.prev_block_hash; let header = BlockHeader { parent: BlockHeaderHash::from_bytes(prev_block_hash.0), number: BlockNumber(block.number.0.into()), payload: payload.hash(), }; - let justification = block.commit_qc_bytes.context("Missing commit QC")?; - let justification: CommitQC = zksync_consensus_schema::decode(&justification.0) + let justification: CommitQC = zksync_consensus_schema::decode(&consensus.commit_qc_bytes.0) .context("Failed deserializing commit QC from Protobuf")?; Ok(FinalBlock { header, @@ -75,6 +74,10 @@ impl FetchedBlock { virtual_blocks: payload.virtual_blocks, operator_address: payload.operator_address, transactions: payload.transactions, + consensus: Some(ConsensusBlockFields { + prev_block_hash: H256(*block.header.parent.as_bytes()), + commit_qc_bytes: zksync_consensus_schema::canonical(&block.justification).into(), + }), }) } } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index f14f54c6f5c9..63a44633b4fc 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -41,7 +41,7 @@ pub(super) async fn load_final_block( pub(super) async fn assert_first_block_actions(actions: &mut ActionQueue) -> Vec { let mut received_actions = vec![]; - while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { + while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock(_))) { received_actions.push(actions.recv_action().await); } assert_matches!( @@ -58,7 +58,7 @@ pub(super) async fn assert_first_block_actions(actions: &mut ActionQueue) -> Vec SyncAction::Tx(_), SyncAction::Tx(_), SyncAction::Tx(_), - SyncAction::SealMiniblock, + SyncAction::SealMiniblock(_), ] ); received_actions @@ -66,7 +66,7 @@ pub(super) async fn assert_first_block_actions(actions: &mut ActionQueue) -> Vec pub(super) async fn assert_second_block_actions(actions: &mut ActionQueue) -> Vec { let mut received_actions = vec![]; - while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock)) { + while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock(_))) { received_actions.push(actions.recv_action().await); } assert_matches!( @@ -80,7 +80,7 @@ pub(super) async fn assert_second_block_actions(actions: &mut ActionQueue) -> Ve SyncAction::Tx(_), SyncAction::Tx(_), SyncAction::Tx(_), - SyncAction::SealMiniblock, + SyncAction::SealMiniblock(_), ] ); received_actions diff --git a/core/lib/zksync_core/src/sync_layer/sync_action.rs b/core/lib/zksync_core/src/sync_layer/sync_action.rs index 4077cdc60bde..b4f56999d4fd 100644 --- a/core/lib/zksync_core/src/sync_layer/sync_action.rs +++ b/core/lib/zksync_core/src/sync_layer/sync_action.rs @@ -1,6 +1,9 @@ use tokio::sync::mpsc; -use zksync_types::{Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, Transaction, H256}; +use zksync_types::{ + block::ConsensusBlockFields, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, + Transaction, H256, +}; use super::metrics::QUEUE_METRICS; @@ -52,7 +55,7 @@ impl ActionQueueSender { return Err(format!("Unexpected Tx: {:?}", actions)); } } - SyncAction::SealMiniblock | SyncAction::SealBatch { .. } => { + SyncAction::SealMiniblock(_) | SyncAction::SealBatch { .. } => { if !opened || miniblock_sealed { return Err(format!("Unexpected SealMiniblock/SealBatch: {:?}", actions)); } @@ -146,11 +149,13 @@ pub(crate) enum SyncAction { /// that they are sealed, but at the same time the next miniblock may not exist yet. /// By having a dedicated action for that we prevent a situation where the miniblock is kept open on the EN until /// the next one is sealed on the main node. - SealMiniblock, + SealMiniblock(Option), /// Similarly to `SealMiniblock` we must be able to seal the batch even if there is no next miniblock yet. SealBatch { - // Virtual blocks count for the fictive miniblock. + /// Virtual blocks count for the fictive miniblock. virtual_blocks: u32, + /// Consensus-related fields for the fictive miniblock. + consensus: Option, }, } @@ -204,11 +209,14 @@ mod tests { } fn seal_miniblock() -> SyncAction { - SyncAction::SealMiniblock + SyncAction::SealMiniblock(None) } fn seal_batch() -> SyncAction { - SyncAction::SealBatch { virtual_blocks: 1 } + SyncAction::SealBatch { + virtual_blocks: 1, + consensus: None, + } } #[test] diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index dd6f35448ce9..e24d958e2ce6 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -69,8 +69,7 @@ impl MockMainNodeClient { virtual_blocks: Some(!is_fictive as u32), hash: Some(H256::repeat_byte(1)), protocol_version: ProtocolVersionId::latest(), - prev_consensus_block_hash: None, - commit_qc_bytes: None, + consensus: None, } }); @@ -241,7 +240,7 @@ async fn external_io_basics() { let tx = create_l2_transaction(10, 100); let tx_hash = tx.hash(); let tx = SyncAction::Tx(Box::new(tx.into())); - let actions = vec![open_l1_batch, tx, SyncAction::SealMiniblock]; + let actions = vec![open_l1_batch, tx, SyncAction::SealMiniblock(None)]; let (actions_sender, action_queue) = ActionQueue::new(); let state_keeper = @@ -284,7 +283,7 @@ pub(super) async fn run_state_keeper_with_multiple_miniblocks(pool: ConnectionPo }); let first_miniblock_actions: Vec<_> = iter::once(open_l1_batch) .chain(txs) - .chain([SyncAction::SealMiniblock]) + .chain([SyncAction::SealMiniblock(None)]) .collect(); let open_miniblock = SyncAction::Miniblock { @@ -298,7 +297,7 @@ pub(super) async fn run_state_keeper_with_multiple_miniblocks(pool: ConnectionPo }); let second_miniblock_actions: Vec<_> = iter::once(open_miniblock) .chain(more_txs) - .chain([SyncAction::SealMiniblock]) + .chain([SyncAction::SealMiniblock(None)]) .collect(); let tx_hashes = extract_tx_hashes( @@ -372,7 +371,7 @@ async fn test_external_io_recovery(pool: ConnectionPool, mut tx_hashes: Vec { + SyncAction::SealBatch { virtual_blocks, .. } => { assert_eq!(virtual_blocks, 0); assert_eq!(tx_count_in_miniblock, 0); if current_miniblock_number == MiniblockNumber(5) { @@ -556,7 +558,7 @@ async fn fetcher_basics() { assert_eq!(tx.hash(), tx_hashes.pop_front().unwrap()); tx_count_in_miniblock += 1; } - SyncAction::SealMiniblock => { + SyncAction::SealMiniblock(_) => { assert_eq!(tx_count_in_miniblock, 1); } } @@ -638,7 +640,7 @@ async fn fetcher_with_real_server() { assert_eq!(tx.hash(), tx_hashes.pop_front().unwrap()); tx_count_in_miniblock += 1; } - SyncAction::SealMiniblock => { + SyncAction::SealMiniblock(_) => { assert_eq!( tx_count_in_miniblock, miniblock_number_to_tx_count[¤t_miniblock_number] From 9371f97519fa319f50831e8c9d394e587a709a37 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 3 Nov 2023 11:19:37 +0200 Subject: [PATCH 17/38] Update gossip fetcher tests --- .../src/sync_layer/gossip/conversions.rs | 15 ++-- .../zksync_core/src/sync_layer/gossip/mod.rs | 3 +- .../src/sync_layer/gossip/storage.rs | 7 +- .../src/sync_layer/gossip/tests.rs | 89 +++++++++++++++---- 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index 58ac779fb5c0..0cf6dad85965 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -26,7 +26,7 @@ struct BlockPayload { transactions: Vec, } -pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> anyhow::Result { +pub(super) fn sync_block_to_payload(block: SyncBlock) -> Payload { let payload = serde_json::to_vec(&BlockPayload { hash: block.hash.unwrap_or_default(), l1_batch_number: block.l1_batch_number, @@ -37,14 +37,19 @@ pub(super) fn sync_block_to_consensus_block(block: SyncBlock) -> anyhow::Result< operator_address: block.operator_address, transactions: block .transactions - .context("Transactions are always requested")?, + .expect("Transactions are always requested"), }); - let payload = Payload(payload.context("Failed serializing block payload")?); - let consensus = block.consensus.context("Missing consensus fields")?; + Payload(payload.expect("Failed serializing block payload")) +} + +pub(super) fn sync_block_to_consensus_block(mut block: SyncBlock) -> anyhow::Result { + let number = BlockNumber(block.number.0.into()); + let consensus = block.consensus.take().context("Missing consensus fields")?; let prev_block_hash = consensus.prev_block_hash; + let payload = sync_block_to_payload(block); let header = BlockHeader { parent: BlockHeaderHash::from_bytes(prev_block_hash.0), - number: BlockNumber(block.number.0.into()), + number, payload: payload.hash(), }; let justification: CommitQC = zksync_consensus_schema::decode(&consensus.commit_qc_bytes.0) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index cdabd4f5df63..e40cc34efef8 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -58,10 +58,9 @@ async fn start_gossip_fetcher_inner( ctx: &ctx::Ctx, pool: ConnectionPool, actions: ActionQueueSender, - mut executor_config: ExecutorConfig, + executor_config: ExecutorConfig, node_key: node::SecretKey, ) -> anyhow::Result<()> { - executor_config.skip_qc_validation = true; tracing::info!( "Starting gossip fetcher with {executor_config:?} and node key {:?}", node_key.public() diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index 350aca48941b..fc2afa08f32f 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -184,6 +184,8 @@ impl ContiguousBlockStore for PostgresBlockStorage { #[cfg(test)] mod tests { + use rand::{thread_rng, Rng}; + use zksync_concurrency::scope; use zksync_types::L2ChainId; @@ -192,7 +194,8 @@ mod tests { genesis::{ensure_genesis_state, GenesisParams}, sync_layer::{ gossip::tests::{ - assert_first_block_actions, assert_second_block_actions, load_final_block, + add_consensus_fields, assert_first_block_actions, assert_second_block_actions, + load_final_block, }, tests::run_state_keeper_with_multiple_miniblocks, ActionQueue, @@ -207,6 +210,7 @@ mod tests { run_state_keeper_with_multiple_miniblocks(pool.clone()).await; let mut storage = pool.access_storage().await.unwrap(); + add_consensus_fields(&mut storage, &thread_rng().gen(), 3).await; let cursor = FetcherCursor::new(&mut storage).await.unwrap(); drop(storage); let (actions_sender, _) = ActionQueue::new(); @@ -279,6 +283,7 @@ mod tests { run_state_keeper_with_multiple_miniblocks(pool.clone()).await; let mut storage = pool.access_storage().await.unwrap(); + add_consensus_fields(&mut storage, &thread_rng().gen(), 3).await; let first_block = load_final_block(&mut storage, 1).await; let second_block = load_final_block(&mut storage, 2).await; storage diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 63a44633b4fc..0e3a1d46bbb5 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -7,10 +7,10 @@ use std::future::Future; use zksync_concurrency::{ctx, scope, time}; use zksync_consensus_executor::testonly::FullValidatorConfig; -use zksync_consensus_roles::validator::FinalBlock; +use zksync_consensus_roles::validator::{self, FinalBlock}; use zksync_consensus_storage::{InMemoryStorage, WriteBlockStore}; use zksync_dal::{ConnectionPool, StorageProcessor}; -use zksync_types::{Address, L1BatchNumber, MiniblockNumber}; +use zksync_types::{block::ConsensusBlockFields, Address, L1BatchNumber, MiniblockNumber, H256}; use super::*; use crate::sync_layer::{ @@ -39,6 +39,53 @@ pub(super) async fn load_final_block( conversions::sync_block_to_consensus_block(sync_block).unwrap() } +pub async fn block_payload(storage: &mut StorageProcessor<'_>, number: u32) -> validator::Payload { + let sync_block = storage + .sync_dal() + .sync_block(MiniblockNumber(number), Address::repeat_byte(1), true) + .await + .unwrap() + .unwrap_or_else(|| panic!("no sync block #{number}")); + conversions::sync_block_to_payload(sync_block) +} + +/// Adds consensus information for the specified `count` of miniblocks, starting from the genesis. +pub(super) async fn add_consensus_fields( + storage: &mut StorageProcessor<'_>, + validator_key: &validator::SecretKey, + count: u32, +) { + let mut prev_block_hash = validator::BlockHeaderHash::from_bytes([0; 32]); + let validator_set = validator::ValidatorSet::new([validator_key.public()]).unwrap(); + for number in 0..count { + let payload = block_payload(storage, number).await; + let block_header = validator::BlockHeader { + parent: prev_block_hash, + number: validator::BlockNumber(number.into()), + payload: payload.hash(), + }; + let replica_commit = validator::ReplicaCommit { + protocol_version: validator::CURRENT_VERSION, + view: validator::ViewNumber(number.into()), + proposal: block_header, + }; + let replica_commit = validator_key.sign_msg(replica_commit); + let commit_qc = validator::CommitQC::from(&[replica_commit], &validator_set) + .expect("Failed creating QC"); + + let consensus = ConsensusBlockFields { + prev_block_hash: H256(*prev_block_hash.as_bytes()), + commit_qc_bytes: zksync_consensus_schema::canonical(&commit_qc).into(), + }; + storage + .blocks_dal() + .set_miniblock_consensus_fields(MiniblockNumber(number), &consensus) + .await + .unwrap(); + prev_block_hash = block_header.hash(); + } +} + pub(super) async fn assert_first_block_actions(actions: &mut ActionQueue) -> Vec { let mut received_actions = vec![]; while !matches!(received_actions.last(), Some(SyncAction::SealMiniblock(_))) { @@ -102,19 +149,20 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: let pool = ConnectionPool::test_pool().await; let tx_hashes = run_state_keeper_with_multiple_miniblocks(pool.clone()).await; - let storage = pool.access_storage().await.unwrap(); - let (genesis_block, blocks) = get_blocks_and_reset_storage(storage).await; + let mut storage = pool.access_storage().await.unwrap(); + let genesis_block_payload = block_payload(&mut storage, 0).await; + let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); + let rng = &mut ctx.rng(); + let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload).await; + let external_node = validator.connect_external_node(rng).await; + + let (genesis_block, blocks) = + get_blocks_and_reset_storage(storage, &validator.validator_key).await; let [first_block, second_block] = blocks.as_slice() else { unreachable!("Unexpected blocks in storage: {blocks:?}"); }; tracing::trace!("Node storage reset"); - let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); - let rng = &mut ctx.rng(); - let mut validator = - FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()).await; - let external_node = validator.connect_external_node(rng).await; - let validator_storage = Arc::new(InMemoryStorage::new(genesis_block)); if !delay_first_block { validator_storage.put_block(ctx, first_block).await.unwrap(); @@ -182,13 +230,15 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: async fn get_blocks_and_reset_storage( mut storage: StorageProcessor<'_>, + validator_key: &validator::SecretKey, ) -> (FinalBlock, Vec) { - let genesis_block = load_final_block(&mut storage, 0).await; let sealed_miniblock_number = storage .blocks_dal() .get_sealed_miniblock_number() .await .unwrap(); + add_consensus_fields(&mut storage, validator_key, sealed_miniblock_number.0 + 1).await; + let genesis_block = load_final_block(&mut storage, 0).await; let mut blocks = Vec::with_capacity(sealed_miniblock_number.0 as usize); for number in 1..=sealed_miniblock_number.0 { @@ -222,18 +272,19 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count let tx_hashes = run_state_keeper_with_multiple_l1_batches(pool.clone()).await; let tx_hashes: Vec<_> = tx_hashes.iter().map(Vec::as_slice).collect(); - let storage = pool.access_storage().await.unwrap(); - let (genesis_block, blocks) = get_blocks_and_reset_storage(storage).await; - assert_eq!(blocks.len(), 3); // 2 real + 1 fictive blocks - tracing::trace!("Node storage reset"); - let (initial_blocks, delayed_blocks) = blocks.split_at(initial_block_count); - + let mut storage = pool.access_storage().await.unwrap(); + let genesis_block_payload = block_payload(&mut storage, 0).await; let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); - let mut validator = - FullValidatorConfig::for_single_validator(rng, genesis_block.payload.clone()).await; + let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload).await; let external_node = validator.connect_external_node(rng).await; + let (genesis_block, blocks) = + get_blocks_and_reset_storage(storage, &validator.validator_key).await; + assert_eq!(blocks.len(), 3); // 2 real + 1 fictive blocks + tracing::trace!("Node storage reset"); + let (initial_blocks, delayed_blocks) = blocks.split_at(initial_block_count); + let validator_storage = Arc::new(InMemoryStorage::new(genesis_block)); for block in initial_blocks { validator_storage.put_block(ctx, block).await.unwrap(); From 2574d353223a0c68a3ed031738bfaaf65fce3874 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 3 Nov 2023 11:35:08 +0200 Subject: [PATCH 18/38] Test consensus fields persistence --- .../src/sync_layer/gossip/tests.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 0e3a1d46bbb5..7e7ffd7c6e60 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -154,6 +154,7 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload).await; + let validator_set = validator.node_config.validators.clone(); let external_node = validator.connect_external_node(rng).await; let (genesis_block, blocks) = @@ -189,7 +190,7 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: s.spawn_bg(wrap_bg_task(validator.run(ctx))); s.spawn_bg(wrap_bg_task(start_gossip_fetcher_inner( ctx, - pool, + pool.clone(), actions_sender, external_node.node_config, external_node.node_key, @@ -226,6 +227,13 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: }) .await .unwrap(); + + // Check that received blocks have consensus fields persisted. + let mut storage = pool.access_storage().await.unwrap(); + for number in [1, 2] { + let block = load_final_block(&mut storage, number).await; + block.justification.verify(&validator_set, 1).unwrap(); + } } async fn get_blocks_and_reset_storage( @@ -277,6 +285,7 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload).await; + let validator_set = validator.node_config.validators.clone(); let external_node = validator.connect_external_node(rng).await; let (genesis_block, blocks) = @@ -315,7 +324,7 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count }); s.spawn_bg(wrap_bg_task(start_gossip_fetcher_inner( ctx, - pool, + pool.clone(), actions_sender, external_node.node_config, external_node.node_key, @@ -328,4 +337,11 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count }) .await .unwrap(); + + // Check that received blocks have consensus fields persisted. + let mut storage = pool.access_storage().await.unwrap(); + for number in [1, 2, 3] { + let block = load_final_block(&mut storage, number).await; + block.justification.verify(&validator_set, 1).unwrap(); + } } From 4481106a4707c0e61c99ff86c2fa2ed0b969dc25 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 3 Nov 2023 14:04:45 +0200 Subject: [PATCH 19/38] Update revision of consensus deps --- Cargo.lock | 20 +++++++++---------- core/lib/zksync_core/Cargo.toml | 10 +++++----- .../src/sync_layer/gossip/tests.rs | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 34abb1fa5e01..e6b37f8e8d2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1443,7 +1443,7 @@ dependencies = [ [[package]] name = "concurrency" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "once_cell", @@ -1461,7 +1461,7 @@ dependencies = [ [[package]] name = "consensus" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "concurrency", @@ -1805,7 +1805,7 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "blst", @@ -2368,7 +2368,7 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "executor" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "concurrency", @@ -4337,7 +4337,7 @@ dependencies = [ [[package]] name = "network" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "async-trait", @@ -6025,7 +6025,7 @@ dependencies = [ [[package]] name = "roles" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "bit-vec", @@ -6195,7 +6195,7 @@ dependencies = [ [[package]] name = "schema" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "bit-vec", @@ -6969,7 +6969,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "storage" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "async-trait", @@ -7101,7 +7101,7 @@ dependencies = [ [[package]] name = "sync_blocks" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "anyhow", "concurrency", @@ -7820,7 +7820,7 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3be8165f0d4490ab2ab85d9d869644478e55bed5#3be8165f0d4490ab2ab85d9d869644478e55bed5" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" dependencies = [ "concurrency", "thiserror", diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index e5c4f648bffd..879abf78e04f 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -40,11 +40,11 @@ vlog = { path = "../vlog" } multivm = { path = "../multivm" } # Consensus dependenices -zksync_concurrency = { package = "concurrency", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } -zksync_consensus_schema = { package = "schema", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } -zksync_consensus_roles = { package = "roles", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } -zksync_consensus_storage = { package = "storage", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } -zksync_consensus_executor = { package = "executor", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "3be8165f0d4490ab2ab85d9d869644478e55bed5" } +zksync_concurrency = { package = "concurrency", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } +zksync_consensus_schema = { package = "schema", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } +zksync_consensus_roles = { package = "roles", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } +zksync_consensus_storage = { package = "storage", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } +zksync_consensus_executor = { package = "executor", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 7e7ffd7c6e60..491a7faa8510 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -153,9 +153,9 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: let genesis_block_payload = block_payload(&mut storage, 0).await; let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); - let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload).await; + let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload); let validator_set = validator.node_config.validators.clone(); - let external_node = validator.connect_external_node(rng).await; + let external_node = validator.connect_external_node(rng); let (genesis_block, blocks) = get_blocks_and_reset_storage(storage, &validator.validator_key).await; @@ -284,9 +284,9 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count let genesis_block_payload = block_payload(&mut storage, 0).await; let ctx = &ctx::test_root(&ctx::AffineClock::new(CLOCK_SPEEDUP as f64)); let rng = &mut ctx.rng(); - let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload).await; + let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload); let validator_set = validator.node_config.validators.clone(); - let external_node = validator.connect_external_node(rng).await; + let external_node = validator.connect_external_node(rng); let (genesis_block, blocks) = get_blocks_and_reset_storage(storage, &validator.validator_key).await; From 97e4a9ad032d55471dffdff2c3f6ead5559e0a81 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 3 Nov 2023 14:16:38 +0200 Subject: [PATCH 20/38] Resolve FIXMEs in fetcher logic --- core/lib/zksync_core/src/sync_layer/fetcher.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/fetcher.rs b/core/lib/zksync_core/src/sync_layer/fetcher.rs index f22f06ee2609..c329b54d80ba 100644 --- a/core/lib/zksync_core/src/sync_layer/fetcher.rs +++ b/core/lib/zksync_core/src/sync_layer/fetcher.rs @@ -67,7 +67,7 @@ pub struct FetcherCursor { } impl FetcherCursor { - /// Loads the cursor + /// Loads the cursor from Postgres. pub async fn new(storage: &mut StorageProcessor<'_>) -> anyhow::Result { let last_sealed_l1_batch_header = storage .blocks_dal() @@ -150,7 +150,8 @@ impl FetcherCursor { FETCHER_METRICS.miniblock.set(block.number.0.into()); } - // FIXME: shaky assumption + // This detection is somewhat shaky. For now, the only empty miniblocks are fictive ones (i.e., + // the last miniblock in an L1 batch); all other miniblocks must contain at least 1 transaction. let last_in_batch = block.transactions.is_empty(); APP_METRICS.processed_txs[&TxStage::added_to_mempool()] .inc_by(block.transactions.len() as u64); @@ -289,7 +290,6 @@ impl MainNodeFetcher { ); // Forgetting only the previous one because we still need the current one in cache for the next iteration. let prev_miniblock_number = MiniblockNumber(block_number.0.saturating_sub(1)); - // FIXME: the old implementation had block_number.0.saturating_sub(2) for some reason self.client.forget_miniblock(prev_miniblock_number); self.actions.push_actions(new_actions).await; From ab79a154c53c7e54d068801763f8f950993b61d4 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 3 Nov 2023 14:58:38 +0200 Subject: [PATCH 21/38] Fix newline ending in new SQL query --- core/lib/dal/sqlx-data.json | 208 +++++++++++++++++------------------ core/lib/dal/src/sync_dal.rs | 2 +- 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index a9da03a5de9e..a3e7a0be1ff5 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -1344,6 +1344,110 @@ }, "query": "\n UPDATE node_aggregation_witness_jobs_fri\n SET status='queued'\n WHERE (l1_batch_number, circuit_id, depth) IN\n (SELECT prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth\n FROM prover_jobs_fri\n JOIN node_aggregation_witness_jobs_fri nawj ON\n prover_jobs_fri.l1_batch_number = nawj.l1_batch_number\n AND prover_jobs_fri.circuit_id = nawj.circuit_id\n AND prover_jobs_fri.depth = nawj.depth\n WHERE nawj.status = 'waiting_for_proofs'\n AND prover_jobs_fri.status = 'successful'\n AND prover_jobs_fri.aggregation_round = 1\n AND prover_jobs_fri.depth = 0\n GROUP BY prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth, nawj.number_of_dependent_jobs\n HAVING COUNT(*) = nawj.number_of_dependent_jobs)\n RETURNING l1_batch_number, circuit_id, depth;\n " }, + "1dc3019f127fd7aa760489457b6eba8dbbde21e03927e4ca71ebb6ab859ac8d1": { + "describe": { + "columns": [ + { + "name": "number", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "l1_batch_number!", + "ordinal": 1, + "type_info": "Int8" + }, + { + "name": "last_batch_miniblock?", + "ordinal": 2, + "type_info": "Int8" + }, + { + "name": "timestamp", + "ordinal": 3, + "type_info": "Int8" + }, + { + "name": "root_hash?", + "ordinal": 4, + "type_info": "Bytea" + }, + { + "name": "l1_gas_price", + "ordinal": 5, + "type_info": "Int8" + }, + { + "name": "l2_fair_gas_price", + "ordinal": 6, + "type_info": "Int8" + }, + { + "name": "bootloader_code_hash", + "ordinal": 7, + "type_info": "Bytea" + }, + { + "name": "default_aa_code_hash", + "ordinal": 8, + "type_info": "Bytea" + }, + { + "name": "virtual_blocks", + "ordinal": 9, + "type_info": "Int8" + }, + { + "name": "hash", + "ordinal": 10, + "type_info": "Bytea" + }, + { + "name": "commit_qc", + "ordinal": 11, + "type_info": "Bytea" + }, + { + "name": "prev_consensus_block_hash", + "ordinal": 12, + "type_info": "Bytea" + }, + { + "name": "protocol_version!", + "ordinal": 13, + "type_info": "Int4" + }, + { + "name": "fee_account_address?", + "ordinal": 14, + "type_info": "Bytea" + } + ], + "nullable": [ + false, + null, + null, + false, + false, + false, + false, + true, + true, + false, + false, + true, + true, + true, + false + ], + "parameters": { + "Left": [ + "Int8" + ] + } + }, + "query": "SELECT miniblocks.number, COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\", (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\", miniblocks.timestamp, miniblocks.hash as \"root_hash?\", miniblocks.l1_gas_price, miniblocks.l2_fair_gas_price, miniblocks.bootloader_code_hash, miniblocks.default_aa_code_hash, miniblocks.virtual_blocks, miniblocks.hash, miniblocks.commit_qc, miniblocks.prev_consensus_block_hash, miniblocks.protocol_version as \"protocol_version!\", l1_batches.fee_account_address as \"fee_account_address?\" FROM miniblocks LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number WHERE miniblocks.number = $1" + }, "1ed353a16e8d0abaf426e5c235b20a79c727c08bc23fb1708a833a6930131691": { "describe": { "columns": [], @@ -9205,110 +9309,6 @@ }, "query": "UPDATE proof_compression_jobs_fri SET status = $1, attempts = attempts + 1, updated_at = now(), processing_started_at = now(), picked_by = $3 WHERE l1_batch_number = ( SELECT l1_batch_number FROM proof_compression_jobs_fri WHERE status = $2 ORDER BY l1_batch_number ASC LIMIT 1 FOR UPDATE SKIP LOCKED ) RETURNING proof_compression_jobs_fri.l1_batch_number" }, - "bc8e2479b25c3f173888f0dfdace9be8fac8547d91982d3b8fa5ee0f9f86a707": { - "describe": { - "columns": [ - { - "name": "number", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "l1_batch_number!", - "ordinal": 1, - "type_info": "Int8" - }, - { - "name": "last_batch_miniblock?", - "ordinal": 2, - "type_info": "Int8" - }, - { - "name": "timestamp", - "ordinal": 3, - "type_info": "Int8" - }, - { - "name": "root_hash?", - "ordinal": 4, - "type_info": "Bytea" - }, - { - "name": "l1_gas_price", - "ordinal": 5, - "type_info": "Int8" - }, - { - "name": "l2_fair_gas_price", - "ordinal": 6, - "type_info": "Int8" - }, - { - "name": "bootloader_code_hash", - "ordinal": 7, - "type_info": "Bytea" - }, - { - "name": "default_aa_code_hash", - "ordinal": 8, - "type_info": "Bytea" - }, - { - "name": "virtual_blocks", - "ordinal": 9, - "type_info": "Int8" - }, - { - "name": "hash", - "ordinal": 10, - "type_info": "Bytea" - }, - { - "name": "commit_qc", - "ordinal": 11, - "type_info": "Bytea" - }, - { - "name": "prev_consensus_block_hash", - "ordinal": 12, - "type_info": "Bytea" - }, - { - "name": "protocol_version!", - "ordinal": 13, - "type_info": "Int4" - }, - { - "name": "fee_account_address?", - "ordinal": 14, - "type_info": "Bytea" - } - ], - "nullable": [ - false, - null, - null, - false, - false, - false, - false, - true, - true, - false, - false, - true, - true, - true, - false - ], - "parameters": { - "Left": [ - "Int8" - ] - } - }, - "query": "SELECT miniblocks.number, COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\", (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\", miniblocks.timestamp, miniblocks.hash as \"root_hash?\", miniblocks.l1_gas_price, miniblocks.l2_fair_gas_price, miniblocks.bootloader_code_hash, miniblocks.default_aa_code_hash, miniblocks.virtual_blocks, miniblocks.hash, miniblocks.commit_qc, miniblocks.prev_consensus_block_hash,\n miniblocks.protocol_version as \"protocol_version!\", l1_batches.fee_account_address as \"fee_account_address?\" FROM miniblocks LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number WHERE miniblocks.number = $1" - }, "be824de76050461afe29dfd229e524bdf113eab3ca24208782c200531db1c940": { "describe": { "columns": [ diff --git a/core/lib/dal/src/sync_dal.rs b/core/lib/dal/src/sync_dal.rs index ac4dd1829052..111b7297bcf7 100644 --- a/core/lib/dal/src/sync_dal.rs +++ b/core/lib/dal/src/sync_dal.rs @@ -35,7 +35,7 @@ impl SyncDal<'_, '_> { miniblocks.virtual_blocks, \ miniblocks.hash, \ miniblocks.commit_qc, \ - miniblocks.prev_consensus_block_hash, + miniblocks.prev_consensus_block_hash, \ miniblocks.protocol_version as \"protocol_version!\", \ l1_batches.fee_account_address as \"fee_account_address?\" \ FROM miniblocks \ From 691171bf834a42b8c3de8fd47432051f050d76da Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 6 Nov 2023 10:18:54 +0200 Subject: [PATCH 22/38] Sketch more advanced "last block in batch" logic --- .../lib/zksync_core/src/sync_layer/fetcher.rs | 7 +- .../src/sync_layer/gossip/buffered/mod.rs | 2 +- .../src/sync_layer/gossip/conversions.rs | 3 + .../src/sync_layer/gossip/storage.rs | 76 +++++++++++++++---- 4 files changed, 67 insertions(+), 21 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/fetcher.rs b/core/lib/zksync_core/src/sync_layer/fetcher.rs index c329b54d80ba..4aabd163f21e 100644 --- a/core/lib/zksync_core/src/sync_layer/fetcher.rs +++ b/core/lib/zksync_core/src/sync_layer/fetcher.rs @@ -26,6 +26,7 @@ const RETRY_DELAY_INTERVAL: Duration = Duration::from_secs(5); pub(super) struct FetchedBlock { pub number: MiniblockNumber, pub l1_batch_number: L1BatchNumber, + pub last_in_batch: bool, pub protocol_version: ProtocolVersionId, pub timestamp: u64, pub hash: H256, @@ -42,6 +43,7 @@ impl FetchedBlock { Self { number: block.number, l1_batch_number: block.l1_batch_number, + last_in_batch: block.last_in_batch, protocol_version: block.protocol_version, timestamp: block.timestamp, hash: block.hash.unwrap_or_default(), @@ -150,16 +152,13 @@ impl FetcherCursor { FETCHER_METRICS.miniblock.set(block.number.0.into()); } - // This detection is somewhat shaky. For now, the only empty miniblocks are fictive ones (i.e., - // the last miniblock in an L1 batch); all other miniblocks must contain at least 1 transaction. - let last_in_batch = block.transactions.is_empty(); APP_METRICS.processed_txs[&TxStage::added_to_mempool()] .inc_by(block.transactions.len() as u64); new_actions.extend(block.transactions.into_iter().map(SyncAction::from)); // Last miniblock of the batch is a "fictive" miniblock and would be replicated locally. // We don't need to seal it explicitly, so we only put the seal miniblock command if it's not the last miniblock. - if last_in_batch { + if block.last_in_batch { new_actions.push(SyncAction::SealBatch { // `block.virtual_blocks` can be `None` only for old VM versions where it's not used, so it's fine to provide any number. virtual_blocks: block.virtual_blocks, diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index ca63ebef98b1..00d4d6e4dc69 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -45,7 +45,7 @@ pub(super) trait ContiguousBlockStore: BlockStore { #[derive(Debug)] struct BlockBuffer { store_block_number: BlockNumber, - is_block_scheduled: bool, + is_block_scheduled: bool, // FIXME: remove in favor of "last / next scheduled block" blocks: BTreeMap, } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index 0cf6dad85965..69c8e6eecf86 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -62,6 +62,8 @@ pub(super) fn sync_block_to_consensus_block(mut block: SyncBlock) -> anyhow::Res } impl FetchedBlock { + /// **Important.** `last_in_batch` is always set to `false` by this method; it must be properly + /// set by the outside logic (currently implemented by `CursorWithCachedBlock`). pub(super) fn from_gossip_block(block: &FinalBlock) -> anyhow::Result { let number = u32::try_from(block.header.number.0) .context("Integer overflow converting block number")?; @@ -71,6 +73,7 @@ impl FetchedBlock { Ok(Self { number: MiniblockNumber(number), l1_batch_number: payload.l1_batch_number, + last_in_batch: false, protocol_version: ProtocolVersionId::latest(), // FIXME timestamp: payload.timestamp, hash: payload.hash, diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index fc2afa08f32f..4477ce964530 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -18,15 +18,52 @@ use zksync_types::{Address, MiniblockNumber}; use super::{buffered::ContiguousBlockStore, conversions::sync_block_to_consensus_block}; use crate::sync_layer::{ fetcher::{FetchedBlock, FetcherCursor}, - sync_action::ActionQueueSender, + sync_action::{ActionQueueSender, SyncAction}, }; +#[derive(Debug)] +struct CursorWithCachedBlock { + inner: FetcherCursor, + maybe_last_block_in_batch: Option, +} + +impl From for CursorWithCachedBlock { + fn from(inner: FetcherCursor) -> Self { + Self { + inner, + maybe_last_block_in_batch: None, + } + } +} + +impl CursorWithCachedBlock { + fn advance(&mut self, block: FetchedBlock) -> Vec> { + let mut actions = Vec::with_capacity(2); + if let Some(mut prev_block) = self.maybe_last_block_in_batch.take() { + prev_block.last_in_batch = prev_block.l1_batch_number != block.l1_batch_number; + actions.push(self.inner.advance(prev_block)); + } + + // We take advantage of the fact that the last block in a batch is a *fictive* block that + // does not contain transactions. Thus, any block with transactions cannot be last in an L1 batch. + let can_be_last_in_batch = block.transactions.is_empty(); + if can_be_last_in_batch { + self.maybe_last_block_in_batch = Some(block); + // We cannot convert the block into actions yet, since we don't know whether it seals an L1 batch. + } else { + actions.push(self.inner.advance(block)); + } + dbg!(&self.inner); + dbg!(actions) + } +} + #[derive(Debug)] pub(super) struct PostgresBlockStorage { pool: ConnectionPool, actions: ActionQueueSender, block_sender: watch::Sender, - cursor: Mutex, + cursor: Mutex, } impl PostgresBlockStorage { @@ -36,7 +73,7 @@ impl PostgresBlockStorage { pool, actions, block_sender: watch::channel(BlockNumber(current_block_number)).0, - cursor: Mutex::new(cursor), + cursor: Mutex::new(cursor.into()), } } @@ -111,16 +148,6 @@ impl PostgresBlockStorage { .context("Failed getting sealed miniblock number")?; Ok(BlockNumber(number.0.into())) } - - async fn schedule_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { - let fetched_block = - FetchedBlock::from_gossip_block(block).map_err(StorageError::Database)?; - let actions = sync::lock(ctx, &self.cursor).await?.advance(fetched_block); - tokio::select! { - () = ctx.canceled() => Err(ctx::Canceled.into()), - () = self.actions.push_actions(actions) => Ok(()), - } - } } #[async_trait] @@ -178,7 +205,18 @@ impl BlockStore for PostgresBlockStorage { #[async_trait] impl ContiguousBlockStore for PostgresBlockStorage { async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { - self.schedule_block(ctx, block).await + let fetched_block = + FetchedBlock::from_gossip_block(block).map_err(StorageError::Database)?; + let actions = sync::lock(ctx, &self.cursor).await?.advance(fetched_block); + let push_all_actions = async { + for actions_chunk in actions { + self.actions.push_actions(actions_chunk).await; + } + }; + tokio::select! { + () = ctx.canceled() => Err(ctx::Canceled.into()), + () = push_all_actions => Ok(()), + } } } @@ -302,10 +340,16 @@ mod tests { let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); let ctx = &ctx::test_root(&ctx::RealClock); let ctx = &ctx.with_timeout(TEST_TIMEOUT); - storage.schedule_block(ctx, &first_block).await.unwrap(); + storage + .schedule_next_block(ctx, &first_block) + .await + .unwrap(); assert_first_block_actions(&mut actions).await; - storage.schedule_block(ctx, &second_block).await.unwrap(); + storage + .schedule_next_block(ctx, &second_block) + .await + .unwrap(); assert_second_block_actions(&mut actions).await; } } From 88c26f811557bc88a2bdd819c91c22474dd6b053 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 6 Nov 2023 11:22:16 +0200 Subject: [PATCH 23/38] Fix background tasks for `Buffered` storage --- .../src/sync_layer/gossip/buffered/mod.rs | 102 +++++++++--------- .../src/sync_layer/gossip/buffered/tests.rs | 22 ++-- .../zksync_core/src/sync_layer/gossip/mod.rs | 8 +- .../src/sync_layer/gossip/storage.rs | 7 +- 4 files changed, 71 insertions(+), 68 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index 00d4d6e4dc69..a28f71ec423f 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -7,7 +7,7 @@ use std::{collections::BTreeMap, ops, time::Instant}; #[cfg(test)] use zksync_concurrency::ctx::channel; use zksync_concurrency::{ - ctx, + ctx, scope, sync::{self, watch, Mutex}, }; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; @@ -45,7 +45,6 @@ pub(super) trait ContiguousBlockStore: BlockStore { #[derive(Debug)] struct BlockBuffer { store_block_number: BlockNumber, - is_block_scheduled: bool, // FIXME: remove in favor of "last / next scheduled block" blocks: BTreeMap, } @@ -53,7 +52,6 @@ impl BlockBuffer { fn new(store_block_number: BlockNumber) -> Self { Self { store_block_number, - is_block_scheduled: false, blocks: BTreeMap::new(), } } @@ -64,18 +62,12 @@ impl BlockBuffer { #[tracing::instrument(level = "trace", skip(self))] fn set_store_block(&mut self, store_block_number: BlockNumber) { - assert_eq!( - store_block_number, - self.store_block_number.next(), - "`ContiguousBlockStore` invariant broken: unexpected new head block number" - ); assert!( - self.is_block_scheduled, - "`ContiguousBlockStore` invariant broken: unexpected update" + store_block_number > self.store_block_number, + "`ContiguousBlockStore` invariant broken: unexpected new head block number" ); self.store_block_number = store_block_number; - self.is_block_scheduled = false; let old_len = self.blocks.len(); self.blocks = self.blocks.split_off(&store_block_number.next()); // ^ Removes all entries up to and including `store_block_number` @@ -114,16 +106,6 @@ impl BlockBuffer { tracing::debug!(%block_number, "Inserted block in buffer"); METRICS.buffer_size.set(self.blocks.len()); } - - fn next_block_for_store(&mut self) -> Option { - if self.is_block_scheduled { - None - } else { - let next_block = self.blocks.get(&self.store_block_number.next()).cloned(); - self.is_block_scheduled = next_block.is_some(); - next_block - } - } } /// Events emitted by [`Buffered`] storage. @@ -178,11 +160,9 @@ impl Buffered { self.buffer.lock().await.blocks.len() } - /// Listens to the updates in the underlying storage. This method must be spawned as a background task - /// which should be running as long at the [`Buffered`] is in use. Otherwise, - /// `BufferedStorage` will function incorrectly. + /// Listens to the updates in the underlying storage. #[tracing::instrument(level = "trace", skip_all, err)] - pub async fn listen_to_updates(&self, ctx: &ctx::Ctx) -> StorageResult<()> { + async fn listen_to_updates(&self, ctx: &ctx::Ctx) -> StorageResult<()> { let mut subscriber = self.inner_subscriber.clone(); loop { let store_block_number = *sync::changed(ctx, &mut subscriber).await?; @@ -191,25 +171,58 @@ impl Buffered { "Underlying block number updated" ); - let next_block_for_store = { - let mut buffer = sync::lock(ctx, &self.buffer).await?; - buffer.set_store_block(store_block_number); - buffer.next_block_for_store() - }; - if let Some(block) = next_block_for_store { - self.inner.schedule_next_block(ctx, &block).await?; - let block_number = block.header.number; - tracing::debug!( - block_number = block_number.0, - "Block scheduled in underlying storage" - ); - } - + sync::lock(ctx, &self.buffer) + .await? + .set_store_block(store_block_number); #[cfg(test)] self.events_sender .send(BufferedStorageEvent::UpdateReceived(store_block_number)); } } + + /// Schedules blocks in the underlying store as they are pushed to this store. + #[tracing::instrument(level = "trace", skip_all, err)] + async fn schedule_blocks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { + let mut blocks_subscriber = self.block_writes_sender.subscribe(); + let mut next_scheduled_block_number = sync::lock(ctx, &self.buffer) + .await? + .store_block_number + .next(); + loop { + while let Some(block) = self + .buffered_block(ctx, next_scheduled_block_number) + .await? + { + self.inner.schedule_next_block(ctx, &block).await?; + next_scheduled_block_number = next_scheduled_block_number.next(); + } + // Wait until some more blocks are pushed into the buffer. + let number = *sync::changed(ctx, &mut blocks_subscriber).await?; + tracing::debug!(block_number = number.0, "Received new block"); + } + } + + async fn buffered_block( + &self, + ctx: &ctx::Ctx, + number: BlockNumber, + ) -> ctx::OrCanceled> { + Ok(sync::lock(ctx, &self.buffer) + .await? + .blocks + .get(&number) + .cloned()) + } + + /// Schedules background tasks for this store. This method **must** be spawned as a background task + /// which should be running as long at the [`Buffered`] is in use; otherwise, it will function incorrectly. + pub async fn run_background_tasks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { + scope::run!(ctx, |ctx, s| { + s.spawn(self.listen_to_updates(ctx)); + self.schedule_blocks(ctx) + }) + .await + } } #[async_trait] @@ -273,7 +286,7 @@ impl BlockStore for Buffered { impl WriteBlockStore for Buffered { async fn put_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { let buffer_block_latency = METRICS.buffer_block_latency.start(); - let next_block_for_store = { + { let mut buffer = sync::lock(ctx, &self.buffer).await?; let block_number = block.header.number; if block_number <= buffer.store_block_number { @@ -283,15 +296,6 @@ impl WriteBlockStore for Buffered { return Err(StorageError::Database(err)); } buffer.put_block(block.clone()); - buffer.next_block_for_store() - }; - - if let Some(block) = next_block_for_store { - self.inner.schedule_next_block(ctx, &block).await?; - tracing::debug!( - block_number = block.header.number.0, - "Block scheduled in underlying storage" - ); } self.block_writes_sender.send_replace(block.header.number); buffer_block_latency.observe(); diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs index b0a69c38188a..99f83b09f34e 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs @@ -51,12 +51,12 @@ fn gen_blocks(rng: &mut impl Rng, genesis_block: FinalBlock, count: usize) -> Ve #[derive(Debug)] struct MockContiguousStore { inner: InMemoryStorage, - block_sender: channel::Sender, + block_sender: channel::UnboundedSender, } impl MockContiguousStore { - fn new(inner: InMemoryStorage) -> (Self, channel::Receiver) { - let (block_sender, block_receiver) = channel::bounded(1); + fn new(inner: InMemoryStorage) -> (Self, channel::UnboundedReceiver) { + let (block_sender, block_receiver) = channel::unbounded(); let this = Self { inner, block_sender, @@ -67,10 +67,13 @@ impl MockContiguousStore { async fn run_updates( &self, ctx: &ctx::Ctx, - mut block_receiver: channel::Receiver, + mut block_receiver: channel::UnboundedReceiver, ) -> StorageResult<()> { let rng = &mut ctx.rng(); while let Ok(block) = block_receiver.recv(ctx).await { + let head_block_number = self.head_block(ctx).await?.header.number; + assert_eq!(block.header.number, head_block_number.next()); + let sleep_duration = time::Duration::milliseconds(rng.gen_range(0..5)); ctx.sleep(sleep_duration).await?; self.inner.put_block(ctx, &block).await?; @@ -116,12 +119,9 @@ impl BlockStore for MockContiguousStore { #[async_trait] impl ContiguousBlockStore for MockContiguousStore { - async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { - let head_block_number = self.head_block(ctx).await?.header.number; - assert_eq!(block.header.number, head_block_number.next()); - self.block_sender - .try_send(block.clone()) - .expect("BufferedStorage is rushing"); + async fn schedule_next_block(&self, _ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + tracing::trace!(block_number = block.header.number.0, "Scheduled next block"); + self.block_sender.send(block.clone()); Ok(()) } } @@ -171,7 +171,7 @@ async fn test_buffered_storage( scope::run!(ctx, |ctx, s| async { s.spawn_bg(buffered_store.inner().run_updates(ctx, block_receiver)); s.spawn_bg(async { - let err = buffered_store.listen_to_updates(ctx).await.unwrap_err(); + let err = buffered_store.run_background_tasks(ctx).await.unwrap_err(); match &err { StorageError::Canceled(_) => Ok(()), // Test has successfully finished StorageError::Database(_) => Err(err), diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index e40cc34efef8..f81dea312683 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -82,15 +82,15 @@ async fn start_gossip_fetcher_inner( scope::run!(ctx, |ctx, s| async { s.spawn_bg(async { store - .listen_to_updates(ctx) + .run_background_tasks(ctx) .await - .context("`PostgresBlockStorage` listener failed") + .context("`PostgresBlockStorage` background tasks failed") }); s.spawn_bg(async { buffered - .listen_to_updates(ctx) + .run_background_tasks(ctx) .await - .context("`Buffered` storage listener failed") + .context("`Buffered` storage background tasks failed") }); executor.run(ctx).await.context("Node executor terminated") diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index 4477ce964530..a622ed37b37f 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -53,8 +53,7 @@ impl CursorWithCachedBlock { } else { actions.push(self.inner.advance(block)); } - dbg!(&self.inner); - dbg!(actions) + actions } } @@ -77,7 +76,7 @@ impl PostgresBlockStorage { } } - pub async fn listen_to_updates(&self, ctx: &ctx::Ctx) -> StorageResult<()> { + pub async fn run_background_tasks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); loop { let sealed_miniblock_number = self @@ -292,7 +291,7 @@ mod tests { let ctx = &ctx::test_root(&ctx::RealClock); scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, s| async { s.spawn_bg(async { - match storage.listen_to_updates(ctx).await { + match storage.run_background_tasks(ctx).await { Ok(()) | Err(StorageError::Canceled(_)) => Ok(()), Err(err) => Err(err.into()), } From 5995eeca0a66074452e6760bc93b72b902d36013 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 6 Nov 2023 12:20:08 +0200 Subject: [PATCH 24/38] Do not propagate cancellations in background tasks --- .../src/sync_layer/gossip/buffered/mod.rs | 48 ++++++++++++------- .../src/sync_layer/gossip/buffered/tests.rs | 12 +---- .../src/sync_layer/gossip/storage.rs | 13 ++--- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index a28f71ec423f..1a379b69435a 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -161,19 +161,25 @@ impl Buffered { } /// Listens to the updates in the underlying storage. - #[tracing::instrument(level = "trace", skip_all, err)] - async fn listen_to_updates(&self, ctx: &ctx::Ctx) -> StorageResult<()> { + #[tracing::instrument(level = "trace", skip_all)] + async fn listen_to_updates(&self, ctx: &ctx::Ctx) { let mut subscriber = self.inner_subscriber.clone(); loop { - let store_block_number = *sync::changed(ctx, &mut subscriber).await?; + let store_block_number = { + let Ok(number) = sync::changed(ctx, &mut subscriber).await else { + return; // Do not propagate cancellation errors + }; + *number + }; tracing::debug!( store_block_number = store_block_number.0, "Underlying block number updated" ); - sync::lock(ctx, &self.buffer) - .await? - .set_store_block(store_block_number); + let Ok(mut buffer) = sync::lock(ctx, &self.buffer).await else { + return; // Do not propagate cancellation errors + }; + buffer.set_store_block(store_block_number); #[cfg(test)] self.events_sender .send(BufferedStorageEvent::UpdateReceived(store_block_number)); @@ -184,20 +190,27 @@ impl Buffered { #[tracing::instrument(level = "trace", skip_all, err)] async fn schedule_blocks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { let mut blocks_subscriber = self.block_writes_sender.subscribe(); - let mut next_scheduled_block_number = sync::lock(ctx, &self.buffer) - .await? - .store_block_number - .next(); + + let mut next_scheduled_block_number = { + let Ok(buffer) = sync::lock(ctx, &self.buffer).await else { + return Ok(()); // Do not propagate cancellation errors + }; + buffer.store_block_number.next() + }; loop { - while let Some(block) = self - .buffered_block(ctx, next_scheduled_block_number) - .await? - { + loop { + let block = match self.buffered_block(ctx, next_scheduled_block_number).await { + Err(ctx::Canceled) => return Ok(()), // Do not propagate cancellation errors + Ok(None) => break, + Ok(Some(block)) => block, + }; self.inner.schedule_next_block(ctx, &block).await?; next_scheduled_block_number = next_scheduled_block_number.next(); } // Wait until some more blocks are pushed into the buffer. - let number = *sync::changed(ctx, &mut blocks_subscriber).await?; + let Ok(number) = sync::changed(ctx, &mut blocks_subscriber).await else { + return Ok(()); // Do not propagate cancellation errors + }; tracing::debug!(block_number = number.0, "Received new block"); } } @@ -218,7 +231,10 @@ impl Buffered { /// which should be running as long at the [`Buffered`] is in use; otherwise, it will function incorrectly. pub async fn run_background_tasks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { scope::run!(ctx, |ctx, s| { - s.spawn(self.listen_to_updates(ctx)); + s.spawn(async { + self.listen_to_updates(ctx).await; + Ok(()) + }); self.schedule_blocks(ctx) }) .await diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs index 99f83b09f34e..de5ef8a88cb0 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/tests.rs @@ -14,9 +14,7 @@ use zksync_concurrency::{ time, }; use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock, Payload}; -use zksync_consensus_storage::{ - BlockStore, InMemoryStorage, StorageError, StorageResult, WriteBlockStore, -}; +use zksync_consensus_storage::{BlockStore, InMemoryStorage, StorageResult, WriteBlockStore}; use super::*; @@ -170,13 +168,7 @@ async fn test_buffered_storage( scope::run!(ctx, |ctx, s| async { s.spawn_bg(buffered_store.inner().run_updates(ctx, block_receiver)); - s.spawn_bg(async { - let err = buffered_store.run_background_tasks(ctx).await.unwrap_err(); - match &err { - StorageError::Canceled(_) => Ok(()), // Test has successfully finished - StorageError::Database(_) => Err(err), - } - }); + s.spawn_bg(buffered_store.run_background_tasks(ctx)); for (idx, block) in blocks.iter().enumerate() { buffered_store.put_block(ctx, block).await?; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs index a622ed37b37f..01b4f7ad80a1 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage.rs @@ -91,7 +91,9 @@ impl PostgresBlockStorage { false } }); - ctx.sleep(POLL_INTERVAL).await?; + if ctx.sleep(POLL_INTERVAL).await.is_err() { + return Ok(()); // Do not propagate cancellation errors + } } } @@ -290,12 +292,7 @@ mod tests { let ctx = &ctx::test_root(&ctx::RealClock); scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, s| async { - s.spawn_bg(async { - match storage.run_background_tasks(ctx).await { - Ok(()) | Err(StorageError::Canceled(_)) => Ok(()), - Err(err) => Err(err.into()), - } - }); + s.spawn_bg(storage.run_background_tasks(ctx)); s.spawn(async { run_state_keeper_with_multiple_miniblocks(pool.clone()).await; Ok(()) @@ -308,7 +305,7 @@ mod tests { break; } } - anyhow::Ok(()) + Ok(()) }) .await .unwrap(); From b9fb194cebb28a0ea66650b943f273220a62edd6 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 7 Nov 2023 10:20:45 +0200 Subject: [PATCH 25/38] Update cancellation logic in line with consensus repo --- .../zksync_core/src/sync_layer/gossip/mod.rs | 40 ++++++++----------- .../src/sync_layer/gossip/tests.rs | 23 +++-------- core/lib/zksync_core/src/sync_layer/mod.rs | 2 +- 3 files changed, 24 insertions(+), 41 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs index f81dea312683..630ded953453 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/mod.rs @@ -22,39 +22,33 @@ use self::{buffered::Buffered, storage::PostgresBlockStorage}; use super::{fetcher::FetcherCursor, sync_action::ActionQueueSender}; /// Starts fetching L2 blocks using peer-to-peer gossip network. -pub async fn start_gossip_fetcher( +pub async fn run_gossip_fetcher( pool: ConnectionPool, actions: ActionQueueSender, executor_config: ExecutorConfig, node_key: node::SecretKey, mut stop_receiver: watch::Receiver, ) -> anyhow::Result<()> { - let result = scope::run!(&ctx::root(), |ctx, s| { - s.spawn_bg(async { - if stop_receiver.changed().await.is_err() { - tracing::warn!( - "Stop signal sender for gossip fetcher was dropped without sending a signal" - ); - } - s.cancel(); - tracing::info!("Stop signal received, gossip fetcher is shutting down"); - Ok(()) - }); - start_gossip_fetcher_inner(ctx, pool, actions, executor_config, node_key) - }) - .await; - - result.or_else(|err| { - if err.root_cause().is::() { - tracing::info!("Gossip fetcher is shut down"); - Ok(()) - } else { - Err(err) + scope::run!(&ctx::root(), |ctx, s| async { + s.spawn_bg(run_gossip_fetcher_inner( + ctx, + pool, + actions, + executor_config, + node_key, + )); + if stop_receiver.changed().await.is_err() { + tracing::warn!( + "Stop signal sender for gossip fetcher was dropped without sending a signal" + ); } + tracing::info!("Stop signal received, gossip fetcher is shutting down"); + Ok(()) }) + .await } -async fn start_gossip_fetcher_inner( +async fn run_gossip_fetcher_inner( ctx: &ctx::Ctx, pool: ConnectionPool, actions: ActionQueueSender, diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 491a7faa8510..7ab2670f26a9 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -3,8 +3,6 @@ use assert_matches::assert_matches; use test_casing::{test_casing, Product}; -use std::future::Future; - use zksync_concurrency::{ctx, scope, time}; use zksync_consensus_executor::testonly::FullValidatorConfig; use zksync_consensus_roles::validator::{self, FinalBlock}; @@ -133,15 +131,6 @@ pub(super) async fn assert_second_block_actions(actions: &mut ActionQueue) -> Ve received_actions } -/// Wraps a background task so that it returns `Ok(())` if it's canceled. -async fn wrap_bg_task(task: impl Future>) -> anyhow::Result<()> { - match task.await { - Ok(()) => Ok(()), - Err(err) if err.root_cause().is::() => Ok(()), - Err(err) => Err(err), - } -} - #[test_casing(4, Product(([false, true], [false, true])))] #[tokio::test] async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: bool) { @@ -187,14 +176,14 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: let (keeper_actions_sender, keeper_actions) = ActionQueue::new(); let state_keeper = StateKeeperHandles::new(pool.clone(), keeper_actions, &[&tx_hashes]).await; scope::run!(ctx, |ctx, s| async { - s.spawn_bg(wrap_bg_task(validator.run(ctx))); - s.spawn_bg(wrap_bg_task(start_gossip_fetcher_inner( + s.spawn_bg(validator.run(ctx)); + s.spawn_bg(run_gossip_fetcher_inner( ctx, pool.clone(), actions_sender, external_node.node_config, external_node.node_key, - ))); + )); if delay_first_block { ctx.sleep(POLL_INTERVAL).await?; @@ -308,7 +297,7 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count let (actions_sender, actions) = ActionQueue::new(); let state_keeper = StateKeeperHandles::new(pool.clone(), actions, &tx_hashes).await; scope::run!(ctx, |ctx, s| async { - s.spawn_bg(wrap_bg_task(validator.run(ctx))); + s.spawn_bg(validator.run(ctx)); s.spawn_bg(async { for block in delayed_blocks { ctx.sleep(POLL_INTERVAL).await?; @@ -322,13 +311,13 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count mock_l1_batch_hash_computation(cloned_pool, 1).await; Ok(()) }); - s.spawn_bg(wrap_bg_task(start_gossip_fetcher_inner( + s.spawn_bg(run_gossip_fetcher_inner( ctx, pool.clone(), actions_sender, external_node.node_config, external_node.node_key, - ))); + )); state_keeper .wait(|state| state.get_local_block() == MiniblockNumber(3)) diff --git a/core/lib/zksync_core/src/sync_layer/mod.rs b/core/lib/zksync_core/src/sync_layer/mod.rs index da521ab3b523..df059947e3e3 100644 --- a/core/lib/zksync_core/src/sync_layer/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/mod.rs @@ -11,6 +11,6 @@ mod sync_state; mod tests; pub use self::{ - client::MainNodeClient, external_io::ExternalIO, gossip::start_gossip_fetcher, + client::MainNodeClient, external_io::ExternalIO, gossip::run_gossip_fetcher, sync_action::ActionQueue, sync_state::SyncState, }; From 530bcdf82d5223d6435a831edae1fa7f834e2f86 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 7 Nov 2023 10:29:57 +0200 Subject: [PATCH 26/38] Fix issues in `storage` module --- .../src/sync_layer/gossip/buffered/mod.rs | 2 +- .../gossip/{storage.rs => storage/mod.rs} | 140 ++---------------- .../src/sync_layer/gossip/storage/tests.rs | 127 ++++++++++++++++ 3 files changed, 138 insertions(+), 131 deletions(-) rename core/lib/zksync_core/src/sync_layer/gossip/{storage.rs => storage/mod.rs} (60%) create mode 100644 core/lib/zksync_core/src/sync_layer/gossip/storage/tests.rs diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index 1a379b69435a..2a313034844f 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -227,7 +227,7 @@ impl Buffered { .cloned()) } - /// Schedules background tasks for this store. This method **must** be spawned as a background task + /// Runs background tasks for this store. This method **must** be spawned as a background task /// which should be running as long at the [`Buffered`] is in use; otherwise, it will function incorrectly. pub async fn run_background_tasks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { scope::run!(ctx, |ctx, s| { diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs similarity index 60% rename from core/lib/zksync_core/src/sync_layer/gossip/storage.rs rename to core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs index 01b4f7ad80a1..42289e994d39 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs @@ -15,6 +15,9 @@ use zksync_consensus_storage::{BlockStore, StorageError, StorageResult}; use zksync_dal::{ConnectionPool, StorageProcessor}; use zksync_types::{Address, MiniblockNumber}; +#[cfg(test)] +mod tests; + use super::{buffered::ContiguousBlockStore, conversions::sync_block_to_consensus_block}; use crate::sync_layer::{ fetcher::{FetchedBlock, FetcherCursor}, @@ -57,6 +60,8 @@ impl CursorWithCachedBlock { } } +/// Postgres-based [`BlockStore`] implementation. New blocks are scheduled to be written via +/// [`ContiguousBlockStore`] trait, which internally uses an [`ActionQueueSender`]. #[derive(Debug)] pub(super) struct PostgresBlockStorage { pool: ConnectionPool, @@ -66,6 +71,7 @@ pub(super) struct PostgresBlockStorage { } impl PostgresBlockStorage { + /// Creates a new storage handle. `pool` should have multiple connections to work efficiently. pub fn new(pool: ConnectionPool, actions: ActionQueueSender, cursor: FetcherCursor) -> Self { let current_block_number = cursor.next_miniblock.0.saturating_sub(1).into(); Self { @@ -76,6 +82,9 @@ impl PostgresBlockStorage { } } + /// Runs background tasks for this store. This method **must** be spawned as a background task + /// which should be running as long at the [`PostgresBlockStorage`] is in use; otherwise, + /// it will function incorrectly. pub async fn run_background_tasks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); loop { @@ -91,7 +100,7 @@ impl PostgresBlockStorage { false } }); - if ctx.sleep(POLL_INTERVAL).await.is_err() { + if let Err(ctx::Canceled) = ctx.sleep(POLL_INTERVAL).await { return Ok(()); // Do not propagate cancellation errors } } @@ -220,132 +229,3 @@ impl ContiguousBlockStore for PostgresBlockStorage { } } } - -#[cfg(test)] -mod tests { - use rand::{thread_rng, Rng}; - - use zksync_concurrency::scope; - use zksync_types::L2ChainId; - - use super::*; - use crate::{ - genesis::{ensure_genesis_state, GenesisParams}, - sync_layer::{ - gossip::tests::{ - add_consensus_fields, assert_first_block_actions, assert_second_block_actions, - load_final_block, - }, - tests::run_state_keeper_with_multiple_miniblocks, - ActionQueue, - }, - }; - - const TEST_TIMEOUT: time::Duration = time::Duration::seconds(10); - - #[tokio::test] - async fn block_store_basics_for_postgres() { - let pool = ConnectionPool::test_pool().await; - run_state_keeper_with_multiple_miniblocks(pool.clone()).await; - - let mut storage = pool.access_storage().await.unwrap(); - add_consensus_fields(&mut storage, &thread_rng().gen(), 3).await; - let cursor = FetcherCursor::new(&mut storage).await.unwrap(); - drop(storage); - let (actions_sender, _) = ActionQueue::new(); - let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); - - let ctx = &ctx::test_root(&ctx::RealClock); - let genesis_block = BlockStore::first_block(&storage, ctx).await.unwrap(); - assert_eq!(genesis_block.header.number, BlockNumber(0)); - let head_block = BlockStore::head_block(&storage, ctx).await.unwrap(); - assert_eq!(head_block.header.number, BlockNumber(2)); - let last_contiguous_block_number = storage.last_contiguous_block_number(ctx).await.unwrap(); - assert_eq!(last_contiguous_block_number, BlockNumber(2)); - - let block = storage - .block(ctx, BlockNumber(1)) - .await - .unwrap() - .expect("no block #1"); - assert_eq!(block.header.number, BlockNumber(1)); - let missing_block = storage.block(ctx, BlockNumber(3)).await.unwrap(); - assert!(missing_block.is_none(), "{missing_block:?}"); - } - - #[tokio::test] - async fn subscribing_to_block_updates_for_postgres() { - let pool = ConnectionPool::test_pool().await; - let mut storage = pool.access_storage().await.unwrap(); - if storage.blocks_dal().is_genesis_needed().await.unwrap() { - ensure_genesis_state(&mut storage, L2ChainId::default(), &GenesisParams::mock()) - .await - .unwrap(); - } - let cursor = FetcherCursor::new(&mut storage).await.unwrap(); - // ^ This is logically incorrect (the storage should not be updated other than using - // `ContiguousBlockStore`), but for testing subscriptions this is fine. - drop(storage); - let (actions_sender, _) = ActionQueue::new(); - let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); - let mut subscriber = storage.subscribe_to_block_writes(); - - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, s| async { - s.spawn_bg(storage.run_background_tasks(ctx)); - s.spawn(async { - run_state_keeper_with_multiple_miniblocks(pool.clone()).await; - Ok(()) - }); - - loop { - let block = *sync::changed(ctx, &mut subscriber).await?; - if block == BlockNumber(2) { - // We should receive at least the last update. - break; - } - } - Ok(()) - }) - .await - .unwrap(); - } - - #[tokio::test] - async fn processing_new_blocks() { - let pool = ConnectionPool::test_pool().await; - run_state_keeper_with_multiple_miniblocks(pool.clone()).await; - - let mut storage = pool.access_storage().await.unwrap(); - add_consensus_fields(&mut storage, &thread_rng().gen(), 3).await; - let first_block = load_final_block(&mut storage, 1).await; - let second_block = load_final_block(&mut storage, 2).await; - storage - .transactions_dal() - .reset_transactions_state(MiniblockNumber(0)) - .await; - storage - .blocks_dal() - .delete_miniblocks(MiniblockNumber(0)) - .await - .unwrap(); - let cursor = FetcherCursor::new(&mut storage).await.unwrap(); - drop(storage); - - let (actions_sender, mut actions) = ActionQueue::new(); - let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); - let ctx = &ctx::test_root(&ctx::RealClock); - let ctx = &ctx.with_timeout(TEST_TIMEOUT); - storage - .schedule_next_block(ctx, &first_block) - .await - .unwrap(); - assert_first_block_actions(&mut actions).await; - - storage - .schedule_next_block(ctx, &second_block) - .await - .unwrap(); - assert_second_block_actions(&mut actions).await; - } -} diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage/tests.rs new file mode 100644 index 000000000000..437c51883308 --- /dev/null +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage/tests.rs @@ -0,0 +1,127 @@ +//! Tests for Postgres storage implementation. + +use rand::{thread_rng, Rng}; + +use zksync_concurrency::scope; +use zksync_types::L2ChainId; + +use super::*; +use crate::{ + genesis::{ensure_genesis_state, GenesisParams}, + sync_layer::{ + gossip::tests::{ + add_consensus_fields, assert_first_block_actions, assert_second_block_actions, + load_final_block, + }, + tests::run_state_keeper_with_multiple_miniblocks, + ActionQueue, + }, +}; + +const TEST_TIMEOUT: time::Duration = time::Duration::seconds(10); + +#[tokio::test] +async fn block_store_basics_for_postgres() { + let pool = ConnectionPool::test_pool().await; + run_state_keeper_with_multiple_miniblocks(pool.clone()).await; + + let mut storage = pool.access_storage().await.unwrap(); + add_consensus_fields(&mut storage, &thread_rng().gen(), 3).await; + let cursor = FetcherCursor::new(&mut storage).await.unwrap(); + drop(storage); + let (actions_sender, _) = ActionQueue::new(); + let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); + + let ctx = &ctx::test_root(&ctx::RealClock); + let genesis_block = BlockStore::first_block(&storage, ctx).await.unwrap(); + assert_eq!(genesis_block.header.number, BlockNumber(0)); + let head_block = BlockStore::head_block(&storage, ctx).await.unwrap(); + assert_eq!(head_block.header.number, BlockNumber(2)); + let last_contiguous_block_number = storage.last_contiguous_block_number(ctx).await.unwrap(); + assert_eq!(last_contiguous_block_number, BlockNumber(2)); + + let block = storage + .block(ctx, BlockNumber(1)) + .await + .unwrap() + .expect("no block #1"); + assert_eq!(block.header.number, BlockNumber(1)); + let missing_block = storage.block(ctx, BlockNumber(3)).await.unwrap(); + assert!(missing_block.is_none(), "{missing_block:?}"); +} + +#[tokio::test] +async fn subscribing_to_block_updates_for_postgres() { + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.access_storage().await.unwrap(); + if storage.blocks_dal().is_genesis_needed().await.unwrap() { + ensure_genesis_state(&mut storage, L2ChainId::default(), &GenesisParams::mock()) + .await + .unwrap(); + } + let cursor = FetcherCursor::new(&mut storage).await.unwrap(); + // ^ This is logically incorrect (the storage should not be updated other than using + // `ContiguousBlockStore`), but for testing subscriptions this is fine. + drop(storage); + let (actions_sender, _) = ActionQueue::new(); + let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); + let mut subscriber = storage.subscribe_to_block_writes(); + + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(&ctx.with_timeout(TEST_TIMEOUT), |ctx, s| async { + s.spawn_bg(storage.run_background_tasks(ctx)); + s.spawn(async { + run_state_keeper_with_multiple_miniblocks(pool.clone()).await; + Ok(()) + }); + + loop { + let block = *sync::changed(ctx, &mut subscriber).await?; + if block == BlockNumber(2) { + // We should receive at least the last update. + break; + } + } + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn processing_new_blocks() { + let pool = ConnectionPool::test_pool().await; + run_state_keeper_with_multiple_miniblocks(pool.clone()).await; + + let mut storage = pool.access_storage().await.unwrap(); + add_consensus_fields(&mut storage, &thread_rng().gen(), 3).await; + let first_block = load_final_block(&mut storage, 1).await; + let second_block = load_final_block(&mut storage, 2).await; + storage + .transactions_dal() + .reset_transactions_state(MiniblockNumber(0)) + .await; + storage + .blocks_dal() + .delete_miniblocks(MiniblockNumber(0)) + .await + .unwrap(); + let cursor = FetcherCursor::new(&mut storage).await.unwrap(); + drop(storage); + + let (actions_sender, mut actions) = ActionQueue::new(); + let storage = PostgresBlockStorage::new(pool.clone(), actions_sender, cursor); + let ctx = &ctx::test_root(&ctx::RealClock); + let ctx = &ctx.with_timeout(TEST_TIMEOUT); + storage + .schedule_next_block(ctx, &first_block) + .await + .unwrap(); + assert_first_block_actions(&mut actions).await; + + storage + .schedule_next_block(ctx, &second_block) + .await + .unwrap(); + assert_second_block_actions(&mut actions).await; +} From b618aaf3d59b6781456395a1f9074cb6636c14c5 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 7 Nov 2023 11:59:06 +0200 Subject: [PATCH 27/38] Create `CommitQCBytes` wrapper --- core/lib/dal/src/blocks_dal.rs | 2 +- core/lib/dal/src/models/storage_sync.rs | 5 ++-- core/lib/types/src/block.rs | 23 +++++++++++++++++-- .../src/sync_layer/gossip/conversions.rs | 14 +++++++---- .../src/sync_layer/gossip/tests.rs | 7 ++++-- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/core/lib/dal/src/blocks_dal.rs b/core/lib/dal/src/blocks_dal.rs index ba7f53acd3ac..604545f2c265 100644 --- a/core/lib/dal/src/blocks_dal.rs +++ b/core/lib/dal/src/blocks_dal.rs @@ -478,7 +478,7 @@ impl BlocksDal<'_, '_> { WHERE number = $1", miniblock_number.0 as i64, consensus.prev_block_hash.as_bytes(), - &consensus.commit_qc_bytes.0 + consensus.commit_qc_bytes.as_ref() ) .execute(self.storage.conn()) .await?; diff --git a/core/lib/dal/src/models/storage_sync.rs b/core/lib/dal/src/models/storage_sync.rs index af6273885e03..790abf95961e 100644 --- a/core/lib/dal/src/models/storage_sync.rs +++ b/core/lib/dal/src/models/storage_sync.rs @@ -3,7 +3,8 @@ use std::convert::TryInto; use zksync_contracts::BaseSystemContractsHashes; use zksync_types::api::en::SyncBlock; use zksync_types::{ - block::ConsensusBlockFields, Address, Bytes, L1BatchNumber, MiniblockNumber, Transaction, H256, + block::{CommitQCBytes, ConsensusBlockFields}, + Address, L1BatchNumber, MiniblockNumber, Transaction, H256, }; #[derive(Debug, Clone, sqlx::FromRow)] @@ -69,7 +70,7 @@ impl StorageSyncBlock { consensus: self.prev_consensus_block_hash.and_then(|hash| { Some(ConsensusBlockFields { prev_block_hash: H256::from_slice(&hash), - commit_qc_bytes: Bytes(commit_qc?), + commit_qc_bytes: CommitQCBytes::new(commit_qc?), }) }), } diff --git a/core/lib/types/src/block.rs b/core/lib/types/src/block.rs index 3d10ab77c3cc..88c15151c403 100644 --- a/core/lib/types/src/block.rs +++ b/core/lib/types/src/block.rs @@ -84,14 +84,33 @@ pub struct MiniblockHeader { pub virtual_blocks: u32, } +/// Protobuf serialization of a quorum certificate for an L2 block (= miniblock). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct CommitQCBytes(pub Bytes); + +impl CommitQCBytes { + /// It is caller's responsibility to ensure that `bytes` is actually a valid Protobuf message + /// for a quorum certificate. + pub fn new(bytes: Vec) -> Self { + Self(Bytes(bytes)) + } +} + +impl AsRef<[u8]> for CommitQCBytes { + fn as_ref(&self) -> &[u8] { + &(self.0).0 + } +} + /// Consensus-related L2 block (= miniblock) fields. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ConsensusBlockFields { /// Hash of the previous consensus block. pub prev_block_hash: H256, - /// Protobuf serialization of quorum certificate for the block. - pub commit_qc_bytes: Bytes, + /// Protobuf serialization of a quorum certificate for the block. + pub commit_qc_bytes: CommitQCBytes, } /// Data needed to execute a miniblock in the VM. diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index 69c8e6eecf86..e260f3b3e569 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -6,9 +6,10 @@ use serde::{Deserialize, Serialize}; use zksync_consensus_roles::validator::{ BlockHeader, BlockHeaderHash, BlockNumber, CommitQC, FinalBlock, Payload, }; -use zksync_types::block::ConsensusBlockFields; use zksync_types::{ - api::en::SyncBlock, Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, + api::en::SyncBlock, + block::{CommitQCBytes, ConsensusBlockFields}, + Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, }; use crate::sync_layer::fetcher::FetchedBlock; @@ -52,8 +53,9 @@ pub(super) fn sync_block_to_consensus_block(mut block: SyncBlock) -> anyhow::Res number, payload: payload.hash(), }; - let justification: CommitQC = zksync_consensus_schema::decode(&consensus.commit_qc_bytes.0) - .context("Failed deserializing commit QC from Protobuf")?; + let justification: CommitQC = + zksync_consensus_schema::decode(consensus.commit_qc_bytes.as_ref()) + .context("Failed deserializing commit QC from Protobuf")?; Ok(FinalBlock { header, payload, @@ -84,7 +86,9 @@ impl FetchedBlock { transactions: payload.transactions, consensus: Some(ConsensusBlockFields { prev_block_hash: H256(*block.header.parent.as_bytes()), - commit_qc_bytes: zksync_consensus_schema::canonical(&block.justification).into(), + commit_qc_bytes: CommitQCBytes::new(zksync_consensus_schema::canonical( + &block.justification, + )), }), }) } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 7ab2670f26a9..4b3fe1072295 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -8,7 +8,10 @@ use zksync_consensus_executor::testonly::FullValidatorConfig; use zksync_consensus_roles::validator::{self, FinalBlock}; use zksync_consensus_storage::{InMemoryStorage, WriteBlockStore}; use zksync_dal::{ConnectionPool, StorageProcessor}; -use zksync_types::{block::ConsensusBlockFields, Address, L1BatchNumber, MiniblockNumber, H256}; +use zksync_types::{ + block::{CommitQCBytes, ConsensusBlockFields}, + Address, L1BatchNumber, MiniblockNumber, H256, +}; use super::*; use crate::sync_layer::{ @@ -73,7 +76,7 @@ pub(super) async fn add_consensus_fields( let consensus = ConsensusBlockFields { prev_block_hash: H256(*prev_block_hash.as_bytes()), - commit_qc_bytes: zksync_consensus_schema::canonical(&commit_qc).into(), + commit_qc_bytes: CommitQCBytes::new(zksync_consensus_schema::canonical(&commit_qc)), }; storage .blocks_dal() From a7de013b67f19d15f1d4e784bcdde1899f68cc39 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 7 Nov 2023 12:01:28 +0200 Subject: [PATCH 28/38] Rework `FetchedBlock::from_gossip_block()` --- .../lib/zksync_core/src/sync_layer/gossip/conversions.rs | 9 +++++---- .../lib/zksync_core/src/sync_layer/gossip/storage/mod.rs | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index e260f3b3e569..b3ab3d100047 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -64,9 +64,10 @@ pub(super) fn sync_block_to_consensus_block(mut block: SyncBlock) -> anyhow::Res } impl FetchedBlock { - /// **Important.** `last_in_batch` is always set to `false` by this method; it must be properly - /// set by the outside logic (currently implemented by `CursorWithCachedBlock`). - pub(super) fn from_gossip_block(block: &FinalBlock) -> anyhow::Result { + pub(super) fn from_gossip_block( + block: &FinalBlock, + last_in_batch: bool, + ) -> anyhow::Result { let number = u32::try_from(block.header.number.0) .context("Integer overflow converting block number")?; let payload: BlockPayload = serde_json::from_slice(&block.payload.0) @@ -75,7 +76,7 @@ impl FetchedBlock { Ok(Self { number: MiniblockNumber(number), l1_batch_number: payload.l1_batch_number, - last_in_batch: false, + last_in_batch, protocol_version: ProtocolVersionId::latest(), // FIXME timestamp: payload.timestamp, hash: payload.hash, diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs index 42289e994d39..c79800a95360 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs @@ -215,8 +215,9 @@ impl BlockStore for PostgresBlockStorage { #[async_trait] impl ContiguousBlockStore for PostgresBlockStorage { async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()> { + // last_in_batch` is always set to `false` by this call; it is properly set by `CursorWithCachedBlock`. let fetched_block = - FetchedBlock::from_gossip_block(block).map_err(StorageError::Database)?; + FetchedBlock::from_gossip_block(block, false).map_err(StorageError::Database)?; let actions = sync::lock(ctx, &self.cursor).await?.advance(fetched_block); let push_all_actions = async { for actions_chunk in actions { From c12ff08b60e4ae83f87b8efaf96dd2177d2180eb Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 13 Nov 2023 11:59:02 +0200 Subject: [PATCH 29/38] Update revision of consensus crates --- Cargo.lock | 573 ++++++++++-------- core/lib/zksync_core/Cargo.toml | 10 +- .../src/sync_layer/gossip/conversions.rs | 7 +- .../src/sync_layer/gossip/tests.rs | 2 +- 4 files changed, 316 insertions(+), 276 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8aa2f0f08f65..2d74f4b59bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease 0.2.15", + "prettyplease", "proc-macro2 1.0.69", "quote 1.0.33", "regex", @@ -1423,44 +1423,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "concurrency" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "once_cell", - "pin-project", - "rand 0.8.5", - "sha2 0.10.8", - "thiserror", - "time", - "tokio", - "tracing", - "tracing-subscriber", - "vise", -] - -[[package]] -name = "consensus" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "concurrency", - "crypto", - "network", - "once_cell", - "rand 0.8.5", - "roles", - "schema", - "storage", - "thiserror", - "tracing", - "utils", - "vise", -] - [[package]] name = "console" version = "0.15.7" @@ -1785,20 +1747,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -[[package]] -name = "crypto" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "blst", - "ed25519-dalek", - "hex", - "rand 0.8.5", - "sha2 0.10.8", - "thiserror", -] - [[package]] name = "crypto-bigint" version = "0.3.2" @@ -2350,26 +2298,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "executor" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "concurrency", - "consensus", - "crypto", - "network", - "rand 0.8.5", - "roles", - "schema", - "storage", - "sync_blocks", - "tracing", - "utils", - "vise", -] - [[package]] name = "fake-simd" version = "0.1.2" @@ -3993,6 +3921,38 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2 1.0.69", + "quote 1.0.33", + "regex-syntax 0.6.29", + "syn 2.0.38", +] + +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen", +] + [[package]] name = "mach" version = "0.3.2" @@ -4142,6 +4102,29 @@ dependencies = [ "sketches-ddsketch", ] +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.38", +] + [[package]] name = "mime" version = "0.3.17" @@ -4303,28 +4286,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "network" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "async-trait", - "concurrency", - "crypto", - "im", - "once_cell", - "pin-project", - "rand 0.8.5", - "roles", - "schema", - "snow", - "thiserror", - "tracing", - "utils", - "vise", -] - [[package]] name = "nix" version = "0.27.1" @@ -4713,6 +4674,18 @@ dependencies = [ "serde", ] +[[package]] +name = "pairing_ce" +version = "0.28.5" +source = "git+https://github.com/matter-labs/pairing.git?rev=f55393f#f55393fd366596eac792d78525d26e9c4d6ed1ca" +dependencies = [ + "byteorder", + "cfg-if 1.0.0", + "ff_ce", + "rand 0.4.6", + "serde", +] + [[package]] name = "pairing_ce" version = "0.28.5" @@ -5145,16 +5118,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2 1.0.69", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.15" @@ -5295,9 +5258,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" dependencies = [ "bytes 1.5.0", "prost-derive", @@ -5305,134 +5268,91 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes 1.5.0", "heck 0.4.1", "itertools", - "lazy_static", "log", "multimap", + "once_cell", "petgraph", - "prettyplease 0.1.25", + "prettyplease", "prost", "prost-types", "regex", - "syn 1.0.109", + "syn 2.0.38", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", "itertools", "proc-macro2 1.0.69", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] name = "prost-reflect" -version = "0.11.5" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b823de344848e011658ac981009100818b322421676740546f8b52ed5249428" +checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" dependencies = [ "base64 0.21.5", + "logos", + "miette", "once_cell", "prost", - "prost-reflect-derive", "prost-types", "serde", "serde-value", ] [[package]] -name = "prost-reflect-build" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d959f576df574d088e0409c45ef11897f64727158c554ce65edc2d345f8afcf" -dependencies = [ - "prost-build", - "prost-reflect", -] - -[[package]] -name = "prost-reflect-derive" -version = "0.11.0" +name = "prost-types" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7718375aa8f966df66e583b608a305a45bc87eeb1ffd5db87fae673bea17a7e4" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" dependencies = [ - "proc-macro2 1.0.69", - "quote 1.0.33", - "syn 2.0.38", + "prost", ] [[package]] -name = "prost-types" -version = "0.11.9" +name = "protox" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "00bb76c5f6221de491fe2c8f39b106330bbd9762c6511119c07940e10eb9ff11" dependencies = [ + "bytes 1.5.0", + "miette", "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", ] [[package]] -name = "protoc-bin-vendored" -version = "3.0.0" +name = "protox-parse" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d" +checksum = "7b4581f441c58863525a3e6bec7b8de98188cf75239a56c725a3e7288450a33f" dependencies = [ - "protoc-bin-vendored-linux-aarch_64", - "protoc-bin-vendored-linux-ppcle_64", - "protoc-bin-vendored-linux-x86_32", - "protoc-bin-vendored-linux-x86_64", - "protoc-bin-vendored-macos-x86_64", - "protoc-bin-vendored-win32", + "logos", + "miette", + "prost-types", + "thiserror", ] -[[package]] -name = "protoc-bin-vendored-linux-aarch_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435" - -[[package]] -name = "protoc-bin-vendored-linux-ppcle_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516" - -[[package]] -name = "protoc-bin-vendored-linux-x86_32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0" - -[[package]] -name = "protoc-bin-vendored-linux-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924" - -[[package]] -name = "protoc-bin-vendored-macos-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537" - -[[package]] -name = "protoc-bin-vendored-win32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" - [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -6000,23 +5920,6 @@ dependencies = [ "zksync_storage", ] -[[package]] -name = "roles" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "bit-vec", - "concurrency", - "crypto", - "hex", - "rand 0.8.5", - "schema", - "serde", - "tracing", - "utils", -] - [[package]] name = "rsa" version = "0.6.1" @@ -6159,29 +6062,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "schema" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "bit-vec", - "concurrency", - "once_cell", - "prettyplease 0.2.15", - "prost", - "prost-build", - "prost-reflect", - "prost-reflect-build", - "protoc-bin-vendored", - "quick-protobuf", - "rand 0.8.5", - "serde", - "serde_json", - "syn 2.0.38", - "tokio", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -6941,21 +6821,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "storage" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "async-trait", - "concurrency", - "rand 0.8.5", - "roles", - "schema", - "thiserror", - "tracing", -] - [[package]] name = "storage_logs_dedup_migration" version = "0.1.0" @@ -7074,21 +6939,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_blocks" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "anyhow", - "concurrency", - "network", - "roles", - "storage", - "thiserror", - "tracing", - "utils", -] - [[package]] name = "sync_vm" version = "1.3.3" @@ -7809,15 +7659,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "utils" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=a0905d4896a12d60296e24130dee6a0c0b730c32#a0905d4896a12d60296e24130dee6a0c0b730c32" -dependencies = [ - "concurrency", - "thiserror", -] - [[package]] name = "uuid" version = "1.5.0" @@ -8651,6 +8492,24 @@ dependencies = [ "zksync_utils", ] +[[package]] +name = "zksync_concurrency" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "once_cell", + "pin-project", + "rand 0.8.5", + "sha3 0.10.8", + "thiserror", + "time", + "tokio", + "tracing", + "tracing-subscriber", + "vise", +] + [[package]] name = "zksync_config" version = "0.1.0" @@ -8662,6 +8521,146 @@ dependencies = [ "zksync_contracts", ] +[[package]] +name = "zksync_consensus_bft" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "once_cell", + "rand 0.8.5", + "thiserror", + "tracing", + "vise", + "zksync_concurrency", + "zksync_consensus_crypto", + "zksync_consensus_network", + "zksync_consensus_roles", + "zksync_consensus_storage", + "zksync_consensus_utils", + "zksync_protobuf", +] + +[[package]] +name = "zksync_consensus_crypto" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "blst", + "ed25519-dalek", + "ff_ce", + "hex", + "pairing_ce 0.28.5 (git+https://github.com/matter-labs/pairing.git?rev=f55393f)", + "rand 0.4.6", + "rand 0.8.5", + "sha3 0.10.8", + "thiserror", + "tracing", +] + +[[package]] +name = "zksync_consensus_executor" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "prost", + "rand 0.8.5", + "tracing", + "vise", + "zksync_concurrency", + "zksync_consensus_bft", + "zksync_consensus_crypto", + "zksync_consensus_network", + "zksync_consensus_roles", + "zksync_consensus_storage", + "zksync_consensus_sync_blocks", + "zksync_consensus_utils", + "zksync_protobuf", +] + +[[package]] +name = "zksync_consensus_network" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "async-trait", + "im", + "once_cell", + "pin-project", + "prost", + "rand 0.8.5", + "snow", + "thiserror", + "tracing", + "vise", + "zksync_concurrency", + "zksync_consensus_crypto", + "zksync_consensus_roles", + "zksync_consensus_utils", + "zksync_protobuf", +] + +[[package]] +name = "zksync_consensus_roles" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "bit-vec", + "hex", + "prost", + "rand 0.8.5", + "serde", + "tracing", + "zksync_concurrency", + "zksync_consensus_crypto", + "zksync_consensus_utils", + "zksync_protobuf", +] + +[[package]] +name = "zksync_consensus_storage" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "async-trait", + "prost", + "rand 0.8.5", + "thiserror", + "tracing", + "zksync_concurrency", + "zksync_consensus_roles", + "zksync_protobuf", +] + +[[package]] +name = "zksync_consensus_sync_blocks" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "thiserror", + "tracing", + "zksync_concurrency", + "zksync_consensus_network", + "zksync_consensus_roles", + "zksync_consensus_storage", + "zksync_consensus_utils", +] + +[[package]] +name = "zksync_consensus_utils" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "thiserror", + "zksync_concurrency", +] + [[package]] name = "zksync_contract_verifier" version = "0.1.0" @@ -8720,9 +8719,7 @@ dependencies = [ "bigdecimal", "bitflags 1.3.2", "chrono", - "concurrency", "ctrlc", - "executor", "futures 0.3.28", "governor", "hex", @@ -8740,11 +8737,8 @@ dependencies = [ "prometheus_exporter", "rand 0.8.5", "reqwest", - "roles", - "schema", "serde", "serde_json", - "storage", "tempfile", "test-casing", "thiserror", @@ -8756,7 +8750,11 @@ dependencies = [ "vlog", "zksync_circuit_breaker", "zksync_commitment_utils", + "zksync_concurrency", "zksync_config", + "zksync_consensus_executor", + "zksync_consensus_roles", + "zksync_consensus_storage", "zksync_contracts", "zksync_dal", "zksync_eth_client", @@ -8766,6 +8764,7 @@ dependencies = [ "zksync_merkle_tree", "zksync_mini_merkle_tree", "zksync_object_store", + "zksync_protobuf", "zksync_prover_utils", "zksync_queued_job_processor", "zksync_state", @@ -8976,6 +8975,48 @@ dependencies = [ "zksync_types", ] +[[package]] +name = "zksync_protobuf" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "bit-vec", + "once_cell", + "prost", + "prost-reflect", + "quick-protobuf", + "rand 0.8.5", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "zksync_concurrency", + "zksync_protobuf_build", +] + +[[package]] +name = "zksync_protobuf_build" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4#fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" +dependencies = [ + "anyhow", + "heck 0.4.1", + "once_cell", + "prettyplease", + "proc-macro2 1.0.69", + "prost", + "prost-build", + "prost-reflect", + "prost-types", + "protox", + "protox-parse", + "quote 1.0.33", + "rand 0.8.5", + "syn 2.0.38", +] + [[package]] name = "zksync_prover_utils" version = "0.1.0" diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index 879abf78e04f..7b10afbdfc95 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -40,11 +40,11 @@ vlog = { path = "../vlog" } multivm = { path = "../multivm" } # Consensus dependenices -zksync_concurrency = { package = "concurrency", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } -zksync_consensus_schema = { package = "schema", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } -zksync_consensus_roles = { package = "roles", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } -zksync_consensus_storage = { package = "storage", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } -zksync_consensus_executor = { package = "executor", version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "a0905d4896a12d60296e24130dee6a0c0b730c32" } +zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" } +zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" } +zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" } +zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" } +zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "fefb04ac6d7c58428184d93f5a53e9e85b8f4cc4" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index b3ab3d100047..a7dc22757ca1 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -53,9 +53,8 @@ pub(super) fn sync_block_to_consensus_block(mut block: SyncBlock) -> anyhow::Res number, payload: payload.hash(), }; - let justification: CommitQC = - zksync_consensus_schema::decode(consensus.commit_qc_bytes.as_ref()) - .context("Failed deserializing commit QC from Protobuf")?; + let justification: CommitQC = zksync_protobuf::decode(consensus.commit_qc_bytes.as_ref()) + .context("Failed deserializing commit QC from Protobuf")?; Ok(FinalBlock { header, payload, @@ -87,7 +86,7 @@ impl FetchedBlock { transactions: payload.transactions, consensus: Some(ConsensusBlockFields { prev_block_hash: H256(*block.header.parent.as_bytes()), - commit_qc_bytes: CommitQCBytes::new(zksync_consensus_schema::canonical( + commit_qc_bytes: CommitQCBytes::new(zksync_protobuf::canonical( &block.justification, )), }), diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 4b3fe1072295..b0a5488aa734 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -76,7 +76,7 @@ pub(super) async fn add_consensus_fields( let consensus = ConsensusBlockFields { prev_block_hash: H256(*prev_block_hash.as_bytes()), - commit_qc_bytes: CommitQCBytes::new(zksync_consensus_schema::canonical(&commit_qc)), + commit_qc_bytes: CommitQCBytes::new(zksync_protobuf::canonical(&commit_qc)), }; storage .blocks_dal() From 0678d5613cd69327a30d6d6903a87e62ebbee139 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 13 Nov 2023 12:19:46 +0200 Subject: [PATCH 30/38] Use `Ctx::wait()` instead of `tokio::select!` --- .../src/sync_layer/gossip/storage/mod.rs | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs index c79800a95360..c54d35881084 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs @@ -163,24 +163,21 @@ impl PostgresBlockStorage { #[async_trait] impl BlockStore for PostgresBlockStorage { async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { - tokio::select! { - () = ctx.canceled() => Err(ctx::Canceled.into()), - result = self.head_block() => result.map_err(StorageError::Database), - } + ctx.wait(self.head_block()) + .await? + .map_err(StorageError::Database) } async fn first_block(&self, ctx: &ctx::Ctx) -> StorageResult { - tokio::select! { - () = ctx.canceled() => Err(ctx::Canceled.into()), - result = self.first_block() => result.map_err(StorageError::Database), - } + ctx.wait(self.first_block()) + .await? + .map_err(StorageError::Database) } async fn last_contiguous_block_number(&self, ctx: &ctx::Ctx) -> StorageResult { - tokio::select! { - () = ctx.canceled() => Err(ctx::Canceled.into()), - result = self.sealed_miniblock_number() => result.map_err(StorageError::Database), - } + ctx.wait(self.sealed_miniblock_number()) + .await? + .map_err(StorageError::Database) } async fn block( @@ -188,15 +185,13 @@ impl BlockStore for PostgresBlockStorage { ctx: &ctx::Ctx, number: BlockNumber, ) -> StorageResult> { - let get_block = async { + ctx.wait(async { let number = u32::try_from(number.0).context("block number is too large")?; let mut storage = self.storage().await?; Self::block(&mut storage, MiniblockNumber(number)).await - }; - tokio::select! { - () = ctx.canceled() => Err(ctx::Canceled.into()), - result = get_block => result.map_err(StorageError::Database), - } + }) + .await? + .map_err(StorageError::Database) } async fn missing_block_numbers( @@ -219,14 +214,12 @@ impl ContiguousBlockStore for PostgresBlockStorage { let fetched_block = FetchedBlock::from_gossip_block(block, false).map_err(StorageError::Database)?; let actions = sync::lock(ctx, &self.cursor).await?.advance(fetched_block); - let push_all_actions = async { + ctx.wait(async { for actions_chunk in actions { self.actions.push_actions(actions_chunk).await; } - }; - tokio::select! { - () = ctx.canceled() => Err(ctx::Canceled.into()), - () = push_all_actions => Ok(()), - } + }) + .await?; + Ok(()) } } From f03e79ac5e60e4fa8e657e5c5784826c1142405b Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 17 Nov 2023 11:41:22 +0200 Subject: [PATCH 31/38] Update `sync_layer` logic --- core/lib/zksync_core/src/consensus/mod.rs | 8 +-- core/lib/zksync_core/src/consensus/payload.rs | 39 ++++++++----- core/lib/zksync_core/src/lib.rs | 1 + .../src/sync_layer/gossip/conversions.rs | 58 ++++--------------- .../src/sync_layer/gossip/tests.rs | 28 ++++----- 5 files changed, 50 insertions(+), 84 deletions(-) diff --git a/core/lib/zksync_core/src/consensus/mod.rs b/core/lib/zksync_core/src/consensus/mod.rs index 08a02e1dd2a4..a229666e76c8 100644 --- a/core/lib/zksync_core/src/consensus/mod.rs +++ b/core/lib/zksync_core/src/consensus/mod.rs @@ -1,10 +1,6 @@ -use anyhow::Context as _; -use zksync_concurrency::{ctx, time}; -use zksync_consensus_roles::validator; -use zksync_types::block::ConsensusBlockFields; -use zksync_types::{Address, MiniblockNumber}; +//! Consensus-related functionality. mod payload; mod proto; -pub(crate) use payload::Payload; +pub(crate) use self::payload::Payload; diff --git a/core/lib/zksync_core/src/consensus/payload.rs b/core/lib/zksync_core/src/consensus/payload.rs index 818d63d74146..8d53fdf21f31 100644 --- a/core/lib/zksync_core/src/consensus/payload.rs +++ b/core/lib/zksync_core/src/consensus/payload.rs @@ -1,9 +1,12 @@ use anyhow::Context as _; + use zksync_consensus_roles::validator; use zksync_protobuf::{required, ProtoFmt}; use zksync_types::api::en::SyncBlock; use zksync_types::{Address, L1BatchNumber, Transaction, H256}; +/// L2 block (= miniblock) payload. +#[derive(Debug)] pub(crate) struct Payload { pub hash: H256, pub l1_batch_number: L1BatchNumber, @@ -17,28 +20,31 @@ pub(crate) struct Payload { impl ProtoFmt for Payload { type Proto = super::proto::Payload; - fn read(r: &Self::Proto) -> anyhow::Result { - let mut transactions = vec![]; - for (i, t) in r.transactions.iter().enumerate() { + + fn read(message: &Self::Proto) -> anyhow::Result { + let mut transactions = Vec::with_capacity(message.transactions.len()); + for (i, tx) in message.transactions.iter().enumerate() { transactions.push( - required(&t.json) - .and_then(|s| Ok(serde_json::from_str(&*s)?)) + required(&tx.json) + .and_then(|json_str| Ok(serde_json::from_str(json_str)?)) .with_context(|| format!("transaction[{i}]"))?, ); } + Ok(Self { - hash: required(&r.hash) - .and_then(|h| Ok(<[u8; 32]>::try_from(h.as_slice())?.into())) + hash: required(&message.hash) + .and_then(|bytes| Ok(<[u8; 32]>::try_from(bytes.as_slice())?.into())) .context("hash")?, l1_batch_number: L1BatchNumber( - *required(&r.l1_batch_number).context("l1_batch_number")?, + *required(&message.l1_batch_number).context("l1_batch_number")?, ), - timestamp: *required(&r.timestamp).context("timestamp")?, - l1_gas_price: *required(&r.l1_gas_price).context("l1_gas_price")?, - l2_fair_gas_price: *required(&r.l2_fair_gas_price).context("l2_fair_gas_price")?, - virtual_blocks: *required(&r.virtual_blocks).context("virtual_blocks")?, - operator_address: required(&r.operator_address) - .and_then(|a| Ok(<[u8; 20]>::try_from(a.as_slice())?.into())) + timestamp: *required(&message.timestamp).context("timestamp")?, + l1_gas_price: *required(&message.l1_gas_price).context("l1_gas_price")?, + l2_fair_gas_price: *required(&message.l2_fair_gas_price) + .context("l2_fair_gas_price")?, + virtual_blocks: *required(&message.virtual_blocks).context("virtual_blocks")?, + operator_address: required(&message.operator_address) + .and_then(|bytes| Ok(<[u8; 20]>::try_from(bytes.as_slice())?.into())) .context("operator_address")?, transactions, }) @@ -67,6 +73,7 @@ impl ProtoFmt for Payload { impl TryFrom for Payload { type Error = anyhow::Error; + fn try_from(block: SyncBlock) -> anyhow::Result { Ok(Self { hash: block.hash.unwrap_or_default(), @@ -82,8 +89,8 @@ impl TryFrom for Payload { } impl Payload { - pub fn decode(p: &validator::Payload) -> anyhow::Result { - zksync_protobuf::decode(&p.0) + pub fn decode(payload: &validator::Payload) -> anyhow::Result { + zksync_protobuf::decode(&payload.0) } pub fn encode(&self) -> validator::Payload { diff --git a/core/lib/zksync_core/src/lib.rs b/core/lib/zksync_core/src/lib.rs index be7e75dbc1e9..db2787a9e3e8 100644 --- a/core/lib/zksync_core/src/lib.rs +++ b/core/lib/zksync_core/src/lib.rs @@ -45,6 +45,7 @@ use zksync_verification_key_server::get_cached_commitments; pub mod api_server; pub mod basic_witness_input_producer; pub mod block_reverter; +mod consensus; pub mod consistency_checker; pub mod data_fetchers; pub mod eth_sender; diff --git a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs index a7dc22757ca1..8face4e69426 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/conversions.rs @@ -1,64 +1,28 @@ //! Conversion logic between server and consensus types. use anyhow::Context as _; -use serde::{Deserialize, Serialize}; -use zksync_consensus_roles::validator::{ - BlockHeader, BlockHeaderHash, BlockNumber, CommitQC, FinalBlock, Payload, -}; +use zksync_consensus_roles::validator::{BlockHeader, BlockNumber, FinalBlock}; use zksync_types::{ - api::en::SyncBlock, - block::{CommitQCBytes, ConsensusBlockFields}, - Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, H256, + api::en::SyncBlock, block::ConsensusBlockFields, MiniblockNumber, ProtocolVersionId, }; -use crate::sync_layer::fetcher::FetchedBlock; - -// FIXME: should use Protobuf -#[derive(Debug, Serialize, Deserialize)] -struct BlockPayload { - hash: H256, - l1_batch_number: L1BatchNumber, - timestamp: u64, - l1_gas_price: u64, - l2_fair_gas_price: u64, - virtual_blocks: u32, - operator_address: Address, - transactions: Vec, -} - -pub(super) fn sync_block_to_payload(block: SyncBlock) -> Payload { - let payload = serde_json::to_vec(&BlockPayload { - hash: block.hash.unwrap_or_default(), - l1_batch_number: block.l1_batch_number, - timestamp: block.timestamp, - l1_gas_price: block.l1_gas_price, - l2_fair_gas_price: block.l2_fair_gas_price, - virtual_blocks: block.virtual_blocks.unwrap_or(0), - operator_address: block.operator_address, - transactions: block - .transactions - .expect("Transactions are always requested"), - }); - Payload(payload.expect("Failed serializing block payload")) -} +use crate::{consensus, sync_layer::fetcher::FetchedBlock}; pub(super) fn sync_block_to_consensus_block(mut block: SyncBlock) -> anyhow::Result { let number = BlockNumber(block.number.0.into()); let consensus = block.consensus.take().context("Missing consensus fields")?; - let prev_block_hash = consensus.prev_block_hash; - let payload = sync_block_to_payload(block); + let payload: consensus::Payload = block.try_into().context("Missing `SyncBlock` data")?; + let payload = payload.encode(); let header = BlockHeader { - parent: BlockHeaderHash::from_bytes(prev_block_hash.0), + parent: consensus.parent, number, payload: payload.hash(), }; - let justification: CommitQC = zksync_protobuf::decode(consensus.commit_qc_bytes.as_ref()) - .context("Failed deserializing commit QC from Protobuf")?; Ok(FinalBlock { header, payload, - justification, + justification: consensus.justification, }) } @@ -69,7 +33,7 @@ impl FetchedBlock { ) -> anyhow::Result { let number = u32::try_from(block.header.number.0) .context("Integer overflow converting block number")?; - let payload: BlockPayload = serde_json::from_slice(&block.payload.0) + let payload = consensus::Payload::decode(&block.payload) .context("Failed deserializing block payload")?; Ok(Self { @@ -85,10 +49,8 @@ impl FetchedBlock { operator_address: payload.operator_address, transactions: payload.transactions, consensus: Some(ConsensusBlockFields { - prev_block_hash: H256(*block.header.parent.as_bytes()), - commit_qc_bytes: CommitQCBytes::new(zksync_protobuf::canonical( - &block.justification, - )), + parent: block.header.parent, + justification: block.justification.clone(), }), }) } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index b0a5488aa734..88a73b3cb088 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -8,19 +8,19 @@ use zksync_consensus_executor::testonly::FullValidatorConfig; use zksync_consensus_roles::validator::{self, FinalBlock}; use zksync_consensus_storage::{InMemoryStorage, WriteBlockStore}; use zksync_dal::{ConnectionPool, StorageProcessor}; -use zksync_types::{ - block::{CommitQCBytes, ConsensusBlockFields}, - Address, L1BatchNumber, MiniblockNumber, H256, -}; +use zksync_types::{block::ConsensusBlockFields, Address, L1BatchNumber, MiniblockNumber}; use super::*; -use crate::sync_layer::{ - sync_action::SyncAction, - tests::{ - mock_l1_batch_hash_computation, run_state_keeper_with_multiple_l1_batches, - run_state_keeper_with_multiple_miniblocks, StateKeeperHandles, +use crate::{ + consensus, + sync_layer::{ + sync_action::SyncAction, + tests::{ + mock_l1_batch_hash_computation, run_state_keeper_with_multiple_l1_batches, + run_state_keeper_with_multiple_miniblocks, StateKeeperHandles, + }, + ActionQueue, }, - ActionQueue, }; const CLOCK_SPEEDUP: i64 = 20; @@ -47,7 +47,7 @@ pub async fn block_payload(storage: &mut StorageProcessor<'_>, number: u32) -> v .await .unwrap() .unwrap_or_else(|| panic!("no sync block #{number}")); - conversions::sync_block_to_payload(sync_block) + consensus::Payload::try_from(sync_block).unwrap().encode() } /// Adds consensus information for the specified `count` of miniblocks, starting from the genesis. @@ -71,12 +71,12 @@ pub(super) async fn add_consensus_fields( proposal: block_header, }; let replica_commit = validator_key.sign_msg(replica_commit); - let commit_qc = validator::CommitQC::from(&[replica_commit], &validator_set) + let justification = validator::CommitQC::from(&[replica_commit], &validator_set) .expect("Failed creating QC"); let consensus = ConsensusBlockFields { - prev_block_hash: H256(*prev_block_hash.as_bytes()), - commit_qc_bytes: CommitQCBytes::new(zksync_protobuf::canonical(&commit_qc)), + parent: prev_block_hash, + justification, }; storage .blocks_dal() From 986ba28f74213d8a8fbe195f1111a023784d498f Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 17 Nov 2023 12:03:34 +0200 Subject: [PATCH 32/38] Remove unused `sqlx` query --- core/lib/dal/sqlx-data.json | 118 ------------------------------------ 1 file changed, 118 deletions(-) diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index 2b83317f9ccd..e031fe7d6715 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -1803,110 +1803,6 @@ }, "query": "\n UPDATE node_aggregation_witness_jobs_fri\n SET status='queued'\n WHERE (l1_batch_number, circuit_id, depth) IN\n (SELECT prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth\n FROM prover_jobs_fri\n JOIN node_aggregation_witness_jobs_fri nawj ON\n prover_jobs_fri.l1_batch_number = nawj.l1_batch_number\n AND prover_jobs_fri.circuit_id = nawj.circuit_id\n AND prover_jobs_fri.depth = nawj.depth\n WHERE nawj.status = 'waiting_for_proofs'\n AND prover_jobs_fri.status = 'successful'\n AND prover_jobs_fri.aggregation_round = 1\n AND prover_jobs_fri.depth = 0\n GROUP BY prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth, nawj.number_of_dependent_jobs\n HAVING COUNT(*) = nawj.number_of_dependent_jobs)\n RETURNING l1_batch_number, circuit_id, depth;\n " }, - "1dc3019f127fd7aa760489457b6eba8dbbde21e03927e4ca71ebb6ab859ac8d1": { - "describe": { - "columns": [ - { - "name": "number", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "l1_batch_number!", - "ordinal": 1, - "type_info": "Int8" - }, - { - "name": "last_batch_miniblock?", - "ordinal": 2, - "type_info": "Int8" - }, - { - "name": "timestamp", - "ordinal": 3, - "type_info": "Int8" - }, - { - "name": "root_hash?", - "ordinal": 4, - "type_info": "Bytea" - }, - { - "name": "l1_gas_price", - "ordinal": 5, - "type_info": "Int8" - }, - { - "name": "l2_fair_gas_price", - "ordinal": 6, - "type_info": "Int8" - }, - { - "name": "bootloader_code_hash", - "ordinal": 7, - "type_info": "Bytea" - }, - { - "name": "default_aa_code_hash", - "ordinal": 8, - "type_info": "Bytea" - }, - { - "name": "virtual_blocks", - "ordinal": 9, - "type_info": "Int8" - }, - { - "name": "hash", - "ordinal": 10, - "type_info": "Bytea" - }, - { - "name": "commit_qc", - "ordinal": 11, - "type_info": "Bytea" - }, - { - "name": "prev_consensus_block_hash", - "ordinal": 12, - "type_info": "Bytea" - }, - { - "name": "protocol_version!", - "ordinal": 13, - "type_info": "Int4" - }, - { - "name": "fee_account_address?", - "ordinal": 14, - "type_info": "Bytea" - } - ], - "nullable": [ - false, - null, - null, - false, - false, - false, - false, - true, - true, - false, - false, - true, - true, - true, - false - ], - "parameters": { - "Left": [ - "Int8" - ] - } - }, - "query": "SELECT miniblocks.number, COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\", (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\", miniblocks.timestamp, miniblocks.hash as \"root_hash?\", miniblocks.l1_gas_price, miniblocks.l2_fair_gas_price, miniblocks.bootloader_code_hash, miniblocks.default_aa_code_hash, miniblocks.virtual_blocks, miniblocks.hash, miniblocks.commit_qc, miniblocks.prev_consensus_block_hash, miniblocks.protocol_version as \"protocol_version!\", l1_batches.fee_account_address as \"fee_account_address?\" FROM miniblocks LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number WHERE miniblocks.number = $1" - }, "1ed353a16e8d0abaf426e5c235b20a79c727c08bc23fb1708a833a6930131691": { "describe": { "columns": [], @@ -9243,20 +9139,6 @@ }, "query": "SELECT * FROM eth_txs_history WHERE eth_tx_id = $1 ORDER BY created_at DESC LIMIT 1" }, - "ad48c26546edaa5f872e5698eb0b0d3ced291db178dd6c065fe6a39def4ea751": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Bytea", - "Bytea" - ] - } - }, - "query": "UPDATE miniblocks SET prev_consensus_block_hash = $2, commit_qc = $3 WHERE number = $1" - }, "ad495160a947cf1bd7343819e723d18c9332bc95cfc2014ed8d04907eff3896e": { "describe": { "columns": [ From 4b0c920825730f46e84c7b39e1443c38a0b44c83 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 17 Nov 2023 12:35:33 +0200 Subject: [PATCH 33/38] Remove consensus fields from sealing logic --- .../src/state_keeper/io/mempool.rs | 2 -- .../src/state_keeper/io/seal_logic.rs | 11 +-------- .../src/state_keeper/io/tests/mod.rs | 6 ----- .../src/state_keeper/updates/mod.rs | 10 +++----- .../zksync_core/src/sync_layer/external_io.rs | 23 ++++++++++++++++--- 5 files changed, 24 insertions(+), 28 deletions(-) diff --git a/core/lib/zksync_core/src/state_keeper/io/mempool.rs b/core/lib/zksync_core/src/state_keeper/io/mempool.rs index db1ac81ab502..f10ad87580c0 100644 --- a/core/lib/zksync_core/src/state_keeper/io/mempool.rs +++ b/core/lib/zksync_core/src/state_keeper/io/mempool.rs @@ -274,7 +274,6 @@ where self.current_l1_batch_number, self.current_miniblock_number, self.l2_erc20_bridge_addr, - None, ); self.miniblock_sealer_handle.submit(command).await; self.current_miniblock_number += 1; @@ -324,7 +323,6 @@ where l1_batch_env, finished_batch, self.l2_erc20_bridge_addr, - None, ) .await; self.current_miniblock_number += 1; // Due to fictive miniblock being sealed. diff --git a/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs b/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs index 90a0a3faff19..ca2dc6419098 100644 --- a/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs +++ b/core/lib/zksync_core/src/state_keeper/io/seal_logic.rs @@ -16,7 +16,7 @@ use zksync_types::{ CURRENT_VIRTUAL_BLOCK_INFO_POSITION, SYSTEM_CONTEXT_ADDRESS, }; use zksync_types::{ - block::{ConsensusBlockFields, L1BatchHeader, MiniblockHeader}, + block::{L1BatchHeader, MiniblockHeader}, event::{extract_added_tokens, extract_long_l2_to_l1_messages}, storage_writes_deduplicator::{ModifiedSlot, StorageWritesDeduplicator}, tx::{ @@ -50,7 +50,6 @@ impl UpdatesManager { l1_batch_env: &L1BatchEnv, finished_batch: FinishedL1Batch, l2_erc20_bridge_addr: Address, - fictive_miniblock_consensus_fields: Option, ) { let started_at = Instant::now(); let progress = L1_BATCH_METRICS.start(L1BatchSealStage::VmFinalization); @@ -64,7 +63,6 @@ impl UpdatesManager { l1_batch_env.number, current_miniblock_number, l2_erc20_bridge_addr, - fictive_miniblock_consensus_fields, ); miniblock_command.seal_inner(&mut transaction, true).await; progress.observe(None); @@ -313,13 +311,6 @@ impl MiniblockSealCommand { .insert_miniblock(&miniblock_header) .await .unwrap(); - if let Some(consensus) = &self.consensus { - transaction - .blocks_dal() - .set_miniblock_consensus_fields(miniblock_number, consensus) - .await - .unwrap(); - } progress.observe(None); let progress = diff --git a/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs b/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs index c550d4f0024f..0c13a7a614b4 100644 --- a/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs +++ b/core/lib/zksync_core/src/state_keeper/io/tests/mod.rs @@ -245,7 +245,6 @@ async fn processing_storage_logs_when_sealing_miniblock() { base_system_contracts_hashes: BaseSystemContractsHashes::default(), protocol_version: Some(ProtocolVersionId::latest()), l2_erc20_bridge_addr: Address::default(), - consensus: None, }; let mut conn = connection_pool .access_storage_tagged("state_keeper") @@ -322,7 +321,6 @@ async fn processing_events_when_sealing_miniblock() { base_system_contracts_hashes: BaseSystemContractsHashes::default(), protocol_version: Some(ProtocolVersionId::latest()), l2_erc20_bridge_addr: Address::default(), - consensus: None, }; let mut conn = pool.access_storage_tagged("state_keeper").await.unwrap(); conn.protocol_versions_dal() @@ -436,7 +434,6 @@ async fn miniblock_sealer_handle_blocking() { L1BatchNumber(1), MiniblockNumber(1), Address::default(), - None, ); sealer_handle.submit(seal_command).await; @@ -445,7 +442,6 @@ async fn miniblock_sealer_handle_blocking() { L1BatchNumber(1), MiniblockNumber(2), Address::default(), - None, ); { let submit_future = sealer_handle.submit(seal_command); @@ -474,7 +470,6 @@ async fn miniblock_sealer_handle_blocking() { L1BatchNumber(2), MiniblockNumber(3), Address::default(), - None, ); sealer_handle.submit(seal_command).await; let command = sealer.commands_receiver.recv().await.unwrap(); @@ -494,7 +489,6 @@ async fn miniblock_sealer_handle_parallel_processing() { L1BatchNumber(1), MiniblockNumber(i), Address::default(), - None, ); sealer_handle.submit(seal_command).await; } diff --git a/core/lib/zksync_core/src/state_keeper/updates/mod.rs b/core/lib/zksync_core/src/state_keeper/updates/mod.rs index 3b36c5ba8c70..dc72893e7034 100644 --- a/core/lib/zksync_core/src/state_keeper/updates/mod.rs +++ b/core/lib/zksync_core/src/state_keeper/updates/mod.rs @@ -3,10 +3,9 @@ use multivm::interface::{L1BatchEnv, VmExecutionResultAndLogs}; use zksync_contracts::BaseSystemContractsHashes; use zksync_types::vm_trace::Call; use zksync_types::{ - block::{BlockGasCount, ConsensusBlockFields}, - storage_writes_deduplicator::StorageWritesDeduplicator, - tx::tx_execution_info::ExecutionMetrics, - Address, L1BatchNumber, MiniblockNumber, ProtocolVersionId, Transaction, + block::BlockGasCount, storage_writes_deduplicator::StorageWritesDeduplicator, + tx::tx_execution_info::ExecutionMetrics, Address, L1BatchNumber, MiniblockNumber, + ProtocolVersionId, Transaction, }; use zksync_utils::bytecode::CompressedBytecodeInfo; @@ -82,7 +81,6 @@ impl UpdatesManager { l1_batch_number: L1BatchNumber, miniblock_number: MiniblockNumber, l2_erc20_bridge_addr: Address, - consensus: Option, ) -> MiniblockSealCommand { MiniblockSealCommand { l1_batch_number, @@ -95,7 +93,6 @@ impl UpdatesManager { base_system_contracts_hashes: self.base_system_contract_hashes, protocol_version: Some(self.protocol_version), l2_erc20_bridge_addr, - consensus, } } @@ -175,7 +172,6 @@ pub(crate) struct MiniblockSealCommand { pub base_system_contracts_hashes: BaseSystemContractsHashes, pub protocol_version: Option, pub l2_erc20_bridge_addr: Address, - pub consensus: Option, } #[cfg(test)] diff --git a/core/lib/zksync_core/src/sync_layer/external_io.rs b/core/lib/zksync_core/src/sync_layer/external_io.rs index b4b83d325677..6b2e0b54150b 100644 --- a/core/lib/zksync_core/src/sync_layer/external_io.rs +++ b/core/lib/zksync_core/src/sync_layer/external_io.rs @@ -537,9 +537,18 @@ impl StateKeeperIO for ExternalIO { self.current_l1_batch_number, self.current_miniblock_number, self.l2_erc20_bridge_addr, - consensus, ); command.seal(&mut transaction).await; + + // We want to add miniblock consensus fields atomically with the miniblock data so that we + // don't need to deal with corner cases (e.g., a miniblock w/o consensus fields). + if let Some(consensus) = &consensus { + transaction + .blocks_dal() + .set_miniblock_consensus_fields(self.current_miniblock_number, consensus) + .await + .unwrap(); + } transaction.commit().await.unwrap(); self.sync_state @@ -564,16 +573,24 @@ impl StateKeeperIO for ExternalIO { }; let mut storage = self.pool.access_storage_tagged("sync_layer").await.unwrap(); + let mut transaction = storage.start_transaction().await.unwrap(); updates_manager .seal_l1_batch( - &mut storage, + &mut transaction, self.current_miniblock_number, l1_batch_env, finished_batch, self.l2_erc20_bridge_addr, - consensus, ) .await; + if let Some(consensus) = &consensus { + transaction + .blocks_dal() + .set_miniblock_consensus_fields(self.current_miniblock_number, consensus) + .await + .unwrap(); + } + transaction.commit().await.unwrap(); tracing::info!("Batch {} is sealed", self.current_l1_batch_number); From 195d9216ab7763299c8595c242a9468e7b5aa253 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 17 Nov 2023 12:35:49 +0200 Subject: [PATCH 34/38] Update prover workspace lockfile --- prover/Cargo.lock | 469 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 466 insertions(+), 3 deletions(-) diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 38ea58ac4366..56d75c9ea73a 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -324,6 +324,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bellman_ce" version = "0.3.2" @@ -617,6 +623,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" +[[package]] +name = "blst" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c94087b935a822949d3291a9989ad2b2051ea141eda0fd4e478a75f6aa3e604b" +dependencies = [ + "cc", + "glob", + "threadpool", + "zeroize", +] + [[package]] name = "boojum" version = "0.1.0" @@ -1350,6 +1368,34 @@ dependencies = [ "serde_json", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "platforms", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + [[package]] name = "darling" version = "0.13.4" @@ -1429,6 +1475,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +dependencies = [ + "const-oid 0.9.5", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.9" @@ -1524,7 +1580,32 @@ dependencies = [ "der 0.6.1", "elliptic-curve", "rfc6979", - "signature", + "signature 1.6.4", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8 0.10.2", + "signature 2.2.0", +] + +[[package]] +name = "ed25519-dalek" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.8", + "subtle", + "zeroize", ] [[package]] @@ -1764,6 +1845,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "fiat-crypto" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" + [[package]] name = "findshlibs" version = "0.10.2" @@ -1812,6 +1899,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fnv" version = "1.0.7" @@ -2840,6 +2933,38 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2 1.0.69", + "quote 1.0.33", + "regex-syntax 0.6.29", + "syn 2.0.39", +] + +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen", +] + [[package]] name = "mach2" version = "0.4.1" @@ -2959,6 +3084,29 @@ dependencies = [ "sketches-ddsketch", ] +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + [[package]] name = "mime" version = "0.3.17" @@ -3016,6 +3164,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "multivm" version = "0.1.0" @@ -3395,6 +3549,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "os_info" version = "3.7.0" @@ -3435,6 +3598,18 @@ dependencies = [ "serde", ] +[[package]] +name = "pairing_ce" +version = "0.28.5" +source = "git+https://github.com/matter-labs/pairing.git?rev=f55393f#f55393fd366596eac792d78525d26e9c4d6ed1ca" +dependencies = [ + "byteorder", + "cfg-if 1.0.0", + "ff_ce", + "rand 0.4.6", + "serde", +] + [[package]] name = "pairing_ce" version = "0.28.5" @@ -3684,6 +3859,16 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.1.0", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -3748,12 +3933,28 @@ dependencies = [ "spki 0.6.0", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.8", + "spki 0.7.2", +] + [[package]] name = "pkg-config" version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "platforms" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" + [[package]] name = "plotters" version = "0.3.5" @@ -3949,6 +4150,103 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5a410fc7882af66deb8d01d01737353cf3ad6204c408177ba494291a626312" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa3d084c8704911bfefb2771be2f9b6c5c0da7343a71e0021ee3c665cada738" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools 0.11.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.39", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065717a5dfaca4a83d2fe57db3487b311365200000551d7a364e715dbf4346bc" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + +[[package]] +name = "prost-reflect" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" +dependencies = [ + "base64 0.21.5", + "logos", + "miette", + "once_cell", + "prost", + "prost-types", + "serde", + "serde-value", +] + +[[package]] +name = "prost-types" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8339f32236f590281e2f6368276441394fcd1b2133b549cc895d0ae80f2f9a52" +dependencies = [ + "prost", +] + +[[package]] +name = "protox" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bb76c5f6221de491fe2c8f39b106330bbd9762c6511119c07940e10eb9ff11" +dependencies = [ + "bytes", + "miette", + "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", +] + +[[package]] +name = "protox-parse" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4581f441c58863525a3e6bec7b8de98188cf75239a56c725a3e7288450a33f" +dependencies = [ + "logos", + "miette", + "prost-types", + "thiserror", +] + [[package]] name = "prover-service" version = "0.1.0" @@ -4004,6 +4302,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + [[package]] name = "quote" version = "0.6.13" @@ -4881,6 +5188,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.192" @@ -5096,6 +5413,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "simple_asn1" version = "0.6.2" @@ -5209,6 +5535,16 @@ dependencies = [ "der 0.6.1", ] +[[package]] +name = "spki" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +dependencies = [ + "base64ct", + "der 0.7.8", +] + [[package]] name = "splitmut" version = "0.2.1" @@ -5571,6 +5907,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" version = "0.3.30" @@ -6448,6 +6793,20 @@ name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] [[package]] name = "zk_evm" @@ -6697,15 +7056,77 @@ dependencies = [ "zksync_verification_key_generator_and_server", ] +[[package]] +name = "zksync_concurrency" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +dependencies = [ + "anyhow", + "once_cell", + "pin-project", + "rand 0.8.5", + "sha3 0.10.8", + "thiserror", + "time", + "tokio", + "tracing", + "tracing-subscriber", + "vise", +] + [[package]] name = "zksync_config" version = "0.1.0" dependencies = [ "anyhow", - "envy", "serde", "zksync_basic_types", - "zksync_contracts", +] + +[[package]] +name = "zksync_consensus_crypto" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +dependencies = [ + "anyhow", + "blst", + "ed25519-dalek", + "ff_ce", + "hex", + "pairing_ce 0.28.5 (git+https://github.com/matter-labs/pairing.git?rev=f55393f)", + "rand 0.4.6", + "rand 0.8.5", + "sha3 0.10.8", + "thiserror", + "tracing", +] + +[[package]] +name = "zksync_consensus_roles" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +dependencies = [ + "anyhow", + "bit-vec", + "hex", + "prost", + "rand 0.8.5", + "serde", + "tracing", + "zksync_concurrency", + "zksync_consensus_crypto", + "zksync_consensus_utils", + "zksync_protobuf", + "zksync_protobuf_build", +] + +[[package]] +name = "zksync_consensus_utils" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +dependencies = [ + "thiserror", + "zksync_concurrency", ] [[package]] @@ -6877,6 +7298,43 @@ dependencies = [ "zksync_utils", ] +[[package]] +name = "zksync_protobuf" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +dependencies = [ + "anyhow", + "bit-vec", + "once_cell", + "prost", + "prost-reflect", + "quick-protobuf", + "rand 0.8.5", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "zksync_concurrency", + "zksync_protobuf_build", +] + +[[package]] +name = "zksync_protobuf_build" +version = "0.1.0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +dependencies = [ + "anyhow", + "heck 0.4.1", + "prettyplease", + "proc-macro2 1.0.69", + "prost-build", + "prost-reflect", + "protox", + "quote 1.0.33", + "syn 2.0.39", +] + [[package]] name = "zksync_prover" version = "0.1.0" @@ -7073,6 +7531,7 @@ dependencies = [ name = "zksync_types" version = "0.1.0" dependencies = [ + "anyhow", "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", "codegen 0.1.0", @@ -7082,6 +7541,7 @@ dependencies = [ "num_enum", "once_cell", "parity-crypto", + "prost", "rlp", "serde", "serde_json", @@ -7092,8 +7552,11 @@ dependencies = [ "zk_evm 1.4.0", "zkevm_test_harness 1.3.3", "zksync_basic_types", + "zksync_consensus_roles", "zksync_contracts", "zksync_mini_merkle_tree", + "zksync_protobuf", + "zksync_protobuf_build", "zksync_system_constants", "zksync_utils", ] From 783556bb117077e2fd0cd4f7b7fa92bb1fdfdaf6 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 17 Nov 2023 13:03:40 +0200 Subject: [PATCH 35/38] Update consensus deps revision --- Cargo.lock | 25 ++++++++----------- core/lib/types/Cargo.toml | 6 ++--- core/lib/zksync_core/Cargo.toml | 12 ++++----- .../src/sync_layer/gossip/tests.rs | 4 +-- prover/Cargo.lock | 15 +++++------ 5 files changed, 28 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb120cdf5a95..56eb6fb9f485 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8514,7 +8514,7 @@ dependencies = [ [[package]] name = "zksync_concurrency" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "once_cell", @@ -8541,7 +8541,7 @@ dependencies = [ [[package]] name = "zksync_consensus_bft" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "once_cell", @@ -8561,7 +8561,7 @@ dependencies = [ [[package]] name = "zksync_consensus_crypto" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "blst", @@ -8579,7 +8579,7 @@ dependencies = [ [[package]] name = "zksync_consensus_executor" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "prost", @@ -8601,7 +8601,7 @@ dependencies = [ [[package]] name = "zksync_consensus_network" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "async-trait", @@ -8625,7 +8625,7 @@ dependencies = [ [[package]] name = "zksync_consensus_roles" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "bit-vec", @@ -8644,7 +8644,7 @@ dependencies = [ [[package]] name = "zksync_consensus_storage" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "async-trait", @@ -8661,7 +8661,7 @@ dependencies = [ [[package]] name = "zksync_consensus_sync_blocks" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "thiserror", @@ -8676,7 +8676,7 @@ dependencies = [ [[package]] name = "zksync_consensus_utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "thiserror", "zksync_concurrency", @@ -9001,7 +9001,7 @@ dependencies = [ [[package]] name = "zksync_protobuf" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "bit-vec", @@ -9012,9 +9012,6 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "tokio", - "tracing", - "tracing-subscriber", "zksync_concurrency", "zksync_protobuf_build", ] @@ -9022,7 +9019,7 @@ dependencies = [ [[package]] name = "zksync_protobuf_build" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "heck 0.4.1", diff --git a/core/lib/types/Cargo.toml b/core/lib/types/Cargo.toml index 117cbdcec8dc..6bf130bc70c0 100644 --- a/core/lib/types/Cargo.toml +++ b/core/lib/types/Cargo.toml @@ -23,8 +23,8 @@ codegen = { git = "https://github.com/matter-labs/solidity_plonk_verifier.git", zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.3.3" } zk_evm_1_4_0 = { git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.4.0", package = "zk_evm" } zk_evm = { git = "https://github.com/matter-labs/era-zk_evm.git", tag = "v1.3.3-rc2" } -zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } -zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } +zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } +zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } anyhow = "1.0.75" chrono = { version = "0.4", features = ["serde"] } @@ -55,4 +55,4 @@ tokio = { version = "1", features = ["rt", "macros"] } serde_with = { version = "1", features = ["hex"] } [build-dependencies] -zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } +zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index 6c8e43763fd1..2bccff98ae9e 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -40,11 +40,11 @@ vlog = { path = "../vlog" } multivm = { path = "../multivm" } # Consensus dependenices -zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } -zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } -zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } -zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } -zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } +zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } +zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } +zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } +zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } +zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } prost = "0.12.1" serde = { version = "1.0", features = ["derive"] } @@ -98,4 +98,4 @@ tempfile = "3.0.2" test-casing = "0.1.2" [build-dependencies] -zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" } +zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ed71b2e817c980a2daffef6a01885219e1dc6fa0" } diff --git a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs index 88a73b3cb088..ca3ce29f4d37 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/tests.rs @@ -147,7 +147,7 @@ async fn syncing_via_gossip_fetcher(delay_first_block: bool, delay_second_block: let rng = &mut ctx.rng(); let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload); let validator_set = validator.node_config.validators.clone(); - let external_node = validator.connect_external_node(rng); + let external_node = validator.connect_full_node(rng); let (genesis_block, blocks) = get_blocks_and_reset_storage(storage, &validator.validator_key).await; @@ -278,7 +278,7 @@ async fn syncing_via_gossip_fetcher_with_multiple_l1_batches(initial_block_count let rng = &mut ctx.rng(); let mut validator = FullValidatorConfig::for_single_validator(rng, genesis_block_payload); let validator_set = validator.node_config.validators.clone(); - let external_node = validator.connect_external_node(rng); + let external_node = validator.connect_full_node(rng); let (genesis_block, blocks) = get_blocks_and_reset_storage(storage, &validator.validator_key).await; diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 56d75c9ea73a..d27b787084f1 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -7059,7 +7059,7 @@ dependencies = [ [[package]] name = "zksync_concurrency" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "once_cell", @@ -7086,7 +7086,7 @@ dependencies = [ [[package]] name = "zksync_consensus_crypto" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "blst", @@ -7104,7 +7104,7 @@ dependencies = [ [[package]] name = "zksync_consensus_roles" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "bit-vec", @@ -7123,7 +7123,7 @@ dependencies = [ [[package]] name = "zksync_consensus_utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "thiserror", "zksync_concurrency", @@ -7301,7 +7301,7 @@ dependencies = [ [[package]] name = "zksync_protobuf" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "bit-vec", @@ -7312,9 +7312,6 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "tokio", - "tracing", - "tracing-subscriber", "zksync_concurrency", "zksync_protobuf_build", ] @@ -7322,7 +7319,7 @@ dependencies = [ [[package]] name = "zksync_protobuf_build" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=bdf9ed0af965cc7fa32d6c46a35ea065779ede8b#bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" dependencies = [ "anyhow", "heck 0.4.1", From ee180fc8bf81889f788ca6ec9ac4c13eff443f40 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 20 Nov 2023 11:10:03 +0200 Subject: [PATCH 36/38] Propagate context awareness further in `PostgresBlockStorage` --- .../src/sync_layer/gossip/storage/mod.rs | 99 +++++++++---------- 1 file changed, 46 insertions(+), 53 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs index c54d35881084..0c42661b9d42 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs @@ -88,10 +88,11 @@ impl PostgresBlockStorage { pub async fn run_background_tasks(&self, ctx: &ctx::Ctx) -> StorageResult<()> { const POLL_INTERVAL: time::Duration = time::Duration::milliseconds(50); loop { - let sealed_miniblock_number = self - .sealed_miniblock_number() - .await - .map_err(StorageError::Database)?; + let sealed_miniblock_number = match self.sealed_miniblock_number(ctx).await { + Ok(number) => number, + Err(err @ StorageError::Database(_)) => return Err(err), + Err(StorageError::Canceled(_)) => return Ok(()), // Do not propagate cancellation errors + }; self.block_sender.send_if_modified(|number| { if *number != sealed_miniblock_number { *number = sealed_miniblock_number; @@ -106,56 +107,42 @@ impl PostgresBlockStorage { } } - async fn head_block(&self) -> anyhow::Result { - let mut storage = self.storage().await?; - let miniblock_number = storage - .blocks_dal() - .get_sealed_miniblock_number() - .await - .context("Failed getting sealed miniblock number")?; - // ^ The number can get stale, but it's OK for our purposes - Self::block(&mut storage, miniblock_number) + async fn storage(&self, ctx: &ctx::Ctx) -> StorageResult> { + ctx.wait(self.pool.access_storage_tagged("sync_layer")) .await? - .with_context(|| format!("Miniblock #{miniblock_number} disappeared from Postgres")) - } - - async fn storage(&self) -> anyhow::Result> { - self.pool - .access_storage_tagged("sync_layer") - .await .context("Failed to connect to Postgres") + .map_err(StorageError::Database) } async fn block( + ctx: &ctx::Ctx, storage: &mut StorageProcessor<'_>, number: MiniblockNumber, - ) -> anyhow::Result> { - let Some(block) = storage - .sync_dal() - .sync_block(number, Address::default(), true) - .await - .with_context(|| format!("Failed getting miniblock #{number} from Postgres"))? + ) -> StorageResult> { + let operator_address = Address::default(); // FIXME: where to get this address from? + let Some(block) = ctx + .wait( + storage + .sync_dal() + .sync_block(number, operator_address, true), + ) + .await? + .with_context(|| format!("Failed getting miniblock #{number} from Postgres")) + .map_err(StorageError::Database)? else { return Ok(None); }; - let block = sync_block_to_consensus_block(block)?; + let block = sync_block_to_consensus_block(block).map_err(StorageError::Database)?; Ok(Some(block)) } - async fn first_block(&self) -> anyhow::Result { - let mut storage = self.storage().await?; - Self::block(&mut storage, MiniblockNumber(0)) + async fn sealed_miniblock_number(&self, ctx: &ctx::Ctx) -> StorageResult { + let mut storage = self.storage(ctx).await?; + let number = ctx + .wait(storage.blocks_dal().get_sealed_miniblock_number()) .await? - .context("Genesis miniblock not present in Postgres") - } - - async fn sealed_miniblock_number(&self) -> anyhow::Result { - let mut storage = self.storage().await?; - let number = storage - .blocks_dal() - .get_sealed_miniblock_number() - .await - .context("Failed getting sealed miniblock number")?; + .context("Failed getting sealed miniblock number") + .map_err(StorageError::Database)?; Ok(BlockNumber(number.0.into())) } } @@ -163,21 +150,29 @@ impl PostgresBlockStorage { #[async_trait] impl BlockStore for PostgresBlockStorage { async fn head_block(&self, ctx: &ctx::Ctx) -> StorageResult { - ctx.wait(self.head_block()) + let mut storage = self.storage(ctx).await?; + let miniblock_number = ctx + .wait(storage.blocks_dal().get_sealed_miniblock_number()) .await? - .map_err(StorageError::Database) + .context("Failed getting sealed miniblock number") + .map_err(StorageError::Database)?; + // ^ The number can get stale, but it's OK for our purposes + Ok(Self::block(ctx, &mut storage, miniblock_number) + .await? + .with_context(|| format!("Miniblock #{miniblock_number} disappeared from Postgres")) + .map_err(StorageError::Database)?) } async fn first_block(&self, ctx: &ctx::Ctx) -> StorageResult { - ctx.wait(self.first_block()) + let mut storage = self.storage(ctx).await?; + Self::block(ctx, &mut storage, MiniblockNumber(0)) .await? + .context("Genesis miniblock not present in Postgres") .map_err(StorageError::Database) } async fn last_contiguous_block_number(&self, ctx: &ctx::Ctx) -> StorageResult { - ctx.wait(self.sealed_miniblock_number()) - .await? - .map_err(StorageError::Database) + self.sealed_miniblock_number(ctx).await } async fn block( @@ -185,13 +180,11 @@ impl BlockStore for PostgresBlockStorage { ctx: &ctx::Ctx, number: BlockNumber, ) -> StorageResult> { - ctx.wait(async { - let number = u32::try_from(number.0).context("block number is too large")?; - let mut storage = self.storage().await?; - Self::block(&mut storage, MiniblockNumber(number)).await - }) - .await? - .map_err(StorageError::Database) + let number = u32::try_from(number.0) + .context("block number is too large") + .map_err(StorageError::Database)?; + let mut storage = self.storage(ctx).await?; + Self::block(ctx, &mut storage, MiniblockNumber(number)).await } async fn missing_block_numbers( From c2c8d18d594ceb1840598792227bdbb4c8f454b6 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 20 Nov 2023 11:14:08 +0200 Subject: [PATCH 37/38] Remove `ctx.wait()` in `schedule_next_block()` --- .../zksync_core/src/sync_layer/gossip/storage/mod.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs index 0c42661b9d42..8359ade67cd6 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs @@ -207,12 +207,11 @@ impl ContiguousBlockStore for PostgresBlockStorage { let fetched_block = FetchedBlock::from_gossip_block(block, false).map_err(StorageError::Database)?; let actions = sync::lock(ctx, &self.cursor).await?.advance(fetched_block); - ctx.wait(async { - for actions_chunk in actions { - self.actions.push_actions(actions_chunk).await; - } - }) - .await?; + for actions_chunk in actions { + // We don't wrap this in `ctx.wait()` because `PostgresBlockStorage` will get broken + // if it gets reused after context cancellation. + self.actions.push_actions(actions_chunk).await; + } Ok(()) } } From 78d49449a777fa502fdf72e77b70979af34e1672 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 22 Nov 2023 10:50:42 +0200 Subject: [PATCH 38/38] Document data flow --- .../src/sync_layer/gossip/buffered/mod.rs | 20 +++++++++++++++++++ .../src/sync_layer/gossip/storage/mod.rs | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs index 2a313034844f..41ca50e1cf2f 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/buffered/mod.rs @@ -42,6 +42,11 @@ pub(super) trait ContiguousBlockStore: BlockStore { async fn schedule_next_block(&self, ctx: &ctx::Ctx, block: &FinalBlock) -> StorageResult<()>; } +/// In-memory buffer or [`FinalBlock`]s received from peers, but not executed and persisted locally yet. +/// +/// Unlike with executed / persisted blocks, there may be gaps between blocks in the buffer. +/// These blocks are shared with peers using the gossip network, but are not persisted and lost +/// on the node restart. #[derive(Debug)] struct BlockBuffer { store_block_number: BlockNumber, @@ -117,6 +122,21 @@ pub(super) enum BufferedStorageEvent { } /// [`BlockStore`] with an in-memory buffer for pending blocks. +/// +/// # Data flow +/// +/// The store is plugged into the `SyncBlocks` actor, so that it can receive new blocks +/// from peers over the gossip network and to share blocks with peers. Received blocks are stored +/// in a [`BlockBuffer`]. The `SyncBlocks` actor doesn't guarantee that blocks are received in order, +/// so we have a background task that waits for successive blocks and feeds them to +/// the underlying storage ([`ContiguousBlockStore`]). The underlying storage executes and persists +/// blocks using the state keeper; see [`PostgresBlockStorage`](super::PostgresBlockStorage) for more details. +/// This logic is largely shared with the old syncing logic using JSON-RPC; the only differing part +/// is producing block data. +/// +/// Once a block is processed and persisted by the state keeper, it can be removed from the [`BlockBuffer`]; +/// we do this in another background task. Removing blocks from the buffer ensures that it doesn't +/// grow infinitely; it also allows to track syncing progress via metrics. #[derive(Debug)] pub(super) struct Buffered { inner: T, diff --git a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs index 8359ade67cd6..d4e95c9e2d46 100644 --- a/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs +++ b/core/lib/zksync_core/src/sync_layer/gossip/storage/mod.rs @@ -61,7 +61,9 @@ impl CursorWithCachedBlock { } /// Postgres-based [`BlockStore`] implementation. New blocks are scheduled to be written via -/// [`ContiguousBlockStore`] trait, which internally uses an [`ActionQueueSender`]. +/// [`ContiguousBlockStore`] trait, which internally uses an [`ActionQueueSender`] to queue +/// block data (miniblock and L1 batch parameters, transactions) for the state keeper. Block data processing +/// is shared with JSON-RPC-based syncing. #[derive(Debug)] pub(super) struct PostgresBlockStorage { pool: ConnectionPool,