From d375cbf46b4456abf84a4bcbadf3a44c36cb4548 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 31 Jul 2024 17:49:10 +0200 Subject: [PATCH 01/31] simplified setup --- .../network/src/gossip/attestation_status.rs | 46 ++-- node/actors/network/src/gossip/batch_votes.rs | 248 +++++------------- node/actors/network/src/gossip/mod.rs | 12 +- .../libs/roles/src/attester/messages/batch.rs | 45 ++-- 4 files changed, 110 insertions(+), 241 deletions(-) diff --git a/node/actors/network/src/gossip/attestation_status.rs b/node/actors/network/src/gossip/attestation_status.rs index 5509bf81..4d26037f 100644 --- a/node/actors/network/src/gossip/attestation_status.rs +++ b/node/actors/network/src/gossip/attestation_status.rs @@ -1,26 +1,27 @@ use std::fmt; use zksync_concurrency::sync; -use zksync_consensus_roles::attester; +use zksync_consensus_roles::{validator,attester}; use crate::watch::Watch; /// Coordinate the attestation by showing the status as seen by the main node. #[derive(Debug, Clone)] pub struct AttestationStatus { - /// Next batch number where voting is expected. - /// - /// Its value is `None` until the background process polling the main node - /// can establish a value to start from. - pub next_batch_to_attest: Option, + pub batch_to_attest: attester::Batch, + /// Committee for that batch. + /// NOTE: the committee is not supposed to change often, + /// so you might want to use `Arc` instead + /// to avoid extra copying. + pub committee: attester::Committee, } /// The subscription over the attestation status which voters can monitor for change. -pub type AttestationStatusReceiver = sync::watch::Receiver; +pub type AttestationStatusReceiver = sync::watch::Receiver>; /// A [Watch] over an [AttestationStatus] which we can use to notify components about /// changes in the batch number the main node expects attesters to vote on. -pub struct AttestationStatusWatch(Watch); +pub struct AttestationStatusWatch(Watch>); impl fmt::Debug for AttestationStatusWatch { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { @@ -29,29 +30,26 @@ impl fmt::Debug for AttestationStatusWatch { } } -impl Default for AttestationStatusWatch { - fn default() -> Self { - Self(Watch::new(AttestationStatus { - next_batch_to_attest: None, - })) - } -} - impl AttestationStatusWatch { + /// Constructs AttestationStatusWatch. + pub fn new(s :AttestationStatus) -> Self { Self(Watch::new(s)) } + /// Subscribes to AttestationStatus updates. pub fn subscribe(&self) -> AttestationStatusReceiver { self.0.subscribe() } /// Set the next batch number to attest on and notify subscribers it changed. - pub async fn update(&self, next_batch_to_attest: attester::BatchNumber) { - let this = self.0.lock().await; - this.send_if_modified(|status| { - if status.next_batch_to_attest == Some(next_batch_to_attest) { - return false; + /// Returns an error if the update it not valid. + pub async fn update(&self, new: AttestationStatus) -> anyhow::Result<()> { + sync::try_send_modify(&self.0.lock().await, |s| { + anyhow::ensure!(s.genesis == new.genesis, "tried to change genesis"); + anyhow::ensure!(s.batch_to_attest.number <= new.batch_to_attest.number, "tried to decrease batch number"); + if s.batch_to_attest.number == new.batch_to_attest.number { + anyhow::ensure!(s == new, "tried to change attestation status for teh batch"); } - status.next_batch_to_attest = Some(next_batch_to_attest); - true - }); + *s = Arc::new(new); + Ok(()) + }).await } } diff --git a/node/actors/network/src/gossip/batch_votes.rs b/node/actors/network/src/gossip/batch_votes.rs index 1ea6b8cb..d3b80eec 100644 --- a/node/actors/network/src/gossip/batch_votes.rs +++ b/node/actors/network/src/gossip/batch_votes.rs @@ -1,6 +1,7 @@ //! Global state distributed by active attesters, observed by all the nodes in the network. use super::metrics; use crate::watch::Watch; +use crate::gossip::AttestationStatus; use std::{collections::HashSet, fmt, sync::Arc}; use zksync_concurrency::sync; use zksync_consensus_roles::attester; @@ -34,44 +35,28 @@ impl BatchUpdateStats { /// save it to the database, if not, we move on. For that, a simple protection /// mechanism is to only allow one active vote per attester, which means any /// previous vote can be removed when a new one is added. -#[derive(Clone, Default, PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq)] pub(crate) struct BatchVotes { - /// The latest vote received from each attester. We only keep the last one - /// for now, hoping that with 1 minute batches there's plenty of time for - /// the quorum to be reached, but eventually we might have to allow multiple - /// votes across different heights. - pub(crate) votes: im::HashMap>>, - - /// Total weight of votes at different heights and hashes. - /// - /// We will be looking for any hash that reaches a quorum threshold at any of the heights. - /// At that point we can remove all earlier heights, considering it final. In the future - /// we can instead keep heights until they are observed on the main node (or L1). - pub(crate) support: - im::OrdMap>, - - /// The minimum batch number for which we are still interested in votes. - /// - /// Because we only store 1 vote per attester the memory is very much bounded, - /// but this extra pruning mechanism can be used to clear votes of attesters - /// who have been removed from the committee, as well as to get rid of the - /// last quorum we found and stored, and look for the a new one in the next round. - pub(crate) min_batch_number: attester::BatchNumber, + status: Arc, + votes: im::HashMap>>, + partial_qc: attester::BatchQC, } impl BatchVotes { - /// Returns a set of votes of `self` which are newer than the entries in `b`. - pub(super) fn get_newer(&self, b: &Self) -> Vec>> { - let mut newer = vec![]; - for (k, v) in &self.votes { - if let Some(bv) = b.votes.get(k) { - if v.msg <= bv.msg { - continue; - } - } - newer.push(v.clone()); + fn new(status: Arc) -> Self { + Self { status, votes: [].into(), partial_qcs: [].into() } + } + + /// Returns a set of votes of `self` which are newer than the entries in `old`. + pub(super) fn get_newer(&self, old: &Self) -> Vec>> { + match self.status.batch_to_attest.number.cmp(old.status.batch_to_attest.number) { + Ordering::Less => vec![], + Ordering::Greater => self.votes.values().cloned().collect(), + Ordering::Equal => self.votes.iter() + .filter(|(k,_)|!old.contains(k)) + .map(|(_,v)|v.clone()) + .collect() } - newer } /// Updates the discovery map with entries from `data`. @@ -82,142 +67,67 @@ impl BatchVotes { /// Returns statistics about new entries added. pub(super) fn update( &mut self, - attesters: &attester::Committee, - genesis: &attester::GenesisHash, - data: &[Arc>], + votes: &[Arc>], ) -> anyhow::Result { let mut stats = BatchUpdateStats::default(); - let mut done = HashSet::new(); - for d in data { + for vote in votes { // Disallow multiple entries for the same key: // it is important because a malicious attester may spam us with // new versions and verifying signatures is expensive. if done.contains(&d.key) { anyhow::bail!("duplicate entry for {:?}", d.key); } - done.insert(d.key.clone()); - - // Disallow votes from different genesis. It might indicate a reorg, - // in which case either this node or the remote peer has to be restarted. - anyhow::ensure!( - d.msg.genesis == *genesis, - "vote for batch with different genesis hash: {:?}", - d.msg.genesis - ); - - if d.msg.number < self.min_batch_number { - continue; - } - - let Some(weight) = attesters.weight(&d.key) else { - // We just skip the entries we are not interested in. - // For now the set of attesters is static, so we could treat this as an error, - // however we eventually want the attester set to be dynamic. - continue; - }; - - // If we already have a newer vote for this key, we can ignore this one. - if let Some(x) = self.votes.get(&d.key) { - if d.msg <= x.msg { - continue; - } - } - - // Check the signature before insertion. - d.verify()?; - - self.add(d.clone(), weight); - stats.added(d.msg.number, weight); + done.insert(vote.key.clone()); + self.add(vote, &mut stats)?; } - Ok(stats) } - /// Check if we have achieved quorum for any of the batch hashes. - /// - /// Returns the first quorum it finds, after which we expect that the state of the main node or L1 - /// will indicate that attestation on the next height can happen, which will either naturally move - /// the QC, or we can do so by increasing the `min_batch_number`. - /// - /// While we only store 1 vote per attester we'll only ever have at most one quorum anyway. - pub(super) fn find_quorum( - &self, - attesters: &attester::Committee, - genesis: &attester::GenesisHash, - ) -> Option { - let threshold = attesters.threshold(); - self.support - .iter() - .flat_map(|(number, candidates)| { - candidates - .iter() - .filter(|(_, weight)| **weight >= threshold) - .map(|(hash, _)| { - let sigs = self - .votes - .values() - .filter(|vote| vote.msg.hash == *hash) - .map(|vote| (vote.key.clone(), vote.sig.clone())) - .fold(attester::MultiSig::default(), |mut sigs, (key, sig)| { - sigs.add(key, sig); - sigs - }); - attester::BatchQC { - message: attester::Batch { - number: *number, - hash: *hash, - // This was checked during insertion; we could look up the first in `votes` - genesis: *genesis, - }, - signatures: sigs, - } - }) - }) - .next() + /// Check if we have achieved quorum for the current batch number. + pub(super) fn find_quorum(&self) -> Option { + match self.partial_qc.verify() { + Ok(()) => Some(self.partial_qc.clone()), + Err(_) => None + } } - /// Set the minimum batch number for which we admit votes. - /// /// Discards data about earlier heights. - pub(super) fn set_min_batch_number(&mut self, min_batch_number: attester::BatchNumber) { - self.min_batch_number = min_batch_number; - self.votes.retain(|_, v| v.msg.number >= min_batch_number); - if let Some(prev) = min_batch_number.prev() { - self.support = self.support.split(&prev).1; - } + pub(super) fn set_status(&mut self, status: Arc) { + self.votes.clear(); + self.partial_qc = attester::BatchQC::new(status.batch_to_attest.clone()); + self.status = status; } - /// Add an already validated vote from an attester into the register. - fn add(&mut self, vote: Arc>, weight: attester::Weight) { - self.remove(&vote.key, weight); - - let batch = self.support.entry(vote.msg.number).or_default(); - let support = batch.entry(vote.msg.hash).or_default(); - *support = support.saturating_add(weight); - - self.votes.insert(vote.key.clone(), vote); - } + /// Verifies and adds a vote. + fn add(&mut self, vote: Arc>, stats: &mut BatchUpdateStats) -> anyhow::Result<()> { + // Genesis has to match + anyhow::ensure!( + vote.msg.genesis == self.status.genesis, + "vote for batch with different genesis hash: {:?}", + vote.msg.genesis + ); + + // Skip the signatures for the irrelevant batch. + if vote.message != self.status.batch_to_attest { + return Ok(()); + } - /// Remove any existing vote. - /// - /// This is for DoS protection, until we have better control over the acceptable vote range. - fn remove(&mut self, key: &attester::PublicKey, weight: attester::Weight) { - let Some(vote) = self.votes.remove(key) else { - return; + // We just skip the entries we are not interested in. + let Some(weight) = self.status.committee.weight(&vote.key) else { + return Ok(()); }; - let batch = self.support.entry(vote.msg.number).or_default(); - let support = batch.entry(vote.msg.hash).or_default(); - *support = support.saturating_sub(weight); + // If we already have a newer vote for this key, we can ignore this one. + if self.votes.contains(&vote.key) { return Ok(()) } - if *support == 0u64 { - batch.remove(&vote.msg.hash); - } - - if batch.is_empty() { - self.support.remove(&vote.msg.number); - } + // Check the signature before insertion. + vote.verify().context("verify()")?; + + // Insert the vote. + stats.added(vote.msg.number, weight); + self.partial_qc.signatures.add(vote.key, vote.sig); + Ok(()) } } @@ -225,13 +135,11 @@ impl BatchVotes { /// which supports subscribing to BatchVotes updates. pub(crate) struct BatchVotesWatch(Watch); -impl Default for BatchVotesWatch { - fn default() -> Self { - Self(Watch::new(BatchVotes::default())) +impl BatchVotesWatch { + pub(crate) fn new(status: AttestationStatus) -> Self { + Self(Watch::new(BatchVotes::new(status)) } -} -impl BatchVotesWatch { /// Subscribes to BatchVotes updates. pub(crate) fn subscribe(&self) -> sync::watch::Receiver { self.0.subscribe() @@ -244,19 +152,17 @@ impl BatchVotesWatch { /// invalid entry should be banned. pub(crate) async fn update( &self, - attesters: &attester::Committee, - genesis: &attester::GenesisHash, data: &[Arc>], ) -> anyhow::Result<()> { let this = self.0.lock().await; let mut votes = this.borrow().clone(); - let stats = votes.update(attesters, genesis, data)?; + let stats = votes.update(data)?; if let Some(last_added) = stats.last_added { this.send_replace(votes); #[allow(clippy::float_arithmetic)] - let weight_added = stats.weight_added as f64 / attesters.total_weight() as f64; + let weight_added = stats.weight_added as f64 / votes.status.committee.total_weight() as f64; metrics::BATCH_VOTES_METRICS .last_added_vote_batch_number @@ -270,22 +176,14 @@ impl BatchVotesWatch { .weight_added .inc_by(weight_added); } - - metrics::BATCH_VOTES_METRICS - .committee_size - .set(attesters.len()); - Ok(()) } /// Set the minimum batch number on the votes and discard old data. - pub(crate) async fn set_min_batch_number(&self, min_batch_number: attester::BatchNumber) { + pub(crate) async fn set_status(&self, status: Arc) { + metrics::BATCH_VOTES_METRICS.min_batch_number.set(status.next_batch_to_attest.0); let this = self.0.lock().await; - this.send_modify(|votes| votes.set_min_batch_number(min_batch_number)); - - metrics::BATCH_VOTES_METRICS - .min_batch_number - .set(min_batch_number.0); + this.send_modify(|votes| votes.set_status(status)); } } @@ -301,24 +199,16 @@ impl fmt::Debug for BatchVotesPublisher { impl BatchVotesPublisher { /// Sign an L1 batch and push it into the batch, which should cause it to be gossiped by the network. - pub async fn publish( - &self, - attesters: &attester::Committee, - genesis: &attester::GenesisHash, - attester: &attester::SecretKey, - batch: attester::Batch, - ) -> anyhow::Result<()> { - if !attesters.contains(&attester.public()) { + pub async fn publish(&self, attester: &attester::SecretKey) -> anyhow::Result<()> { + if !self.committee.contains(&attester.public()) { return Ok(()); } - let attestation = attester.sign_msg(batch); + let attestation = attester.sign_msg(self.batch_to_attest); metrics::BATCH_VOTES_METRICS .last_signed_batch_number .set(attestation.msg.number.0); - self.0 - .update(attesters, genesis, &[Arc::new(attestation)]) - .await + self.0.update(&[Arc::new(attestation)]).await } } diff --git a/node/actors/network/src/gossip/mod.rs b/node/actors/network/src/gossip/mod.rs index 5da55cce..a6e35228 100644 --- a/node/actors/network/src/gossip/mod.rs +++ b/node/actors/network/src/gossip/mod.rs @@ -12,7 +12,7 @@ //! Static connections constitute a rigid "backbone" of the gossip network, which is insensitive to //! eclipse attack. Dynamic connections are supposed to improve the properties of the gossip //! network graph (minimize its diameter, increase connectedness). -pub use self::attestation_status::{AttestationStatusReceiver, AttestationStatusWatch}; +pub use self::attestation_status::{AttestationStatus,AttestationStatusReceiver, AttestationStatusWatch}; pub use self::batch_votes::BatchVotesPublisher; use self::batch_votes::BatchVotesWatch; use crate::{gossip::ValidatorAddrsWatch, io, pool::PoolWatch, Config, MeteredStreamStats}; @@ -169,18 +169,14 @@ impl Network { // Subscribe starts as seen but we don't want to miss the first item. recv_status.mark_changed(); + let next = attester::BatchNumber(0); loop { // Wait until the status indicates that we're ready to sign the next batch. - let Some(next_batch_number) = sync::changed(ctx, &mut recv_status) - .await? - .next_batch_to_attest - else { - continue; - }; + let status = sync::wait_for(ctx, &mut recv_status, |s| s.next_batch_to_attest >= next).await?.clone(); // Get rid of all previous votes. We don't expect this to go backwards without regenesis, which will involve a restart. self.batch_votes - .set_min_batch_number(next_batch_number) + .set_min_batch_number(status.next_batch_number) .await; // Now wait until we find the next quorum, whatever it is: diff --git a/node/libs/roles/src/attester/messages/batch.rs b/node/libs/roles/src/attester/messages/batch.rs index e9554314..26146364 100644 --- a/node/libs/roles/src/attester/messages/batch.rs +++ b/node/libs/roles/src/attester/messages/batch.rs @@ -125,11 +125,10 @@ pub enum BatchQCVerifyError { want: u64, }, /// Bad signer set. - #[error("signers set doesn't match genesis")] + #[error("signers not in committee")] BadSignersSet, - /// No attester committee in genesis. - #[error("No attester committee in genesis")] - AttestersNotInGenesis, + #[error("genesis mismatch")] + GenesisMismatch, } /// Error returned by `BatchQC::add()` if the signature is invalid. @@ -160,46 +159,32 @@ impl BatchQC { /// Add a attester's signature. /// Signature is assumed to be already verified. - pub fn add(&mut self, msg: &Signed, genesis: &Genesis) -> anyhow::Result<()> { - use BatchQCAddError as Error; - - let committee = genesis - .attesters - .as_ref() - .context("no attester committee in genesis")?; - - ensure!(self.message == msg.msg, Error::InconsistentMessages); - ensure!(!self.signatures.contains(&msg.key), Error::Exists); - ensure!( - committee.contains(&msg.key), - Error::SignerNotInCommittee { - signer: Box::new(msg.key.clone()), - } - ); - + pub fn add(&mut self, msg: &Signed, committee: &attester::Committee) -> anyhow::Result<()> { + anyhow::ensure!(self.message == msg.msg, "inconsistent messages"); + anyhow::ensure!(self.signatures.contains(&msg.key), "signature already present"); + anyhow::ensure!(committee.contains(&msg.key), "not in committee"); self.signatures.add(msg.key.clone(), msg.sig.clone()); - Ok(()) } /// Verifies the signature of the BatchQC. - pub fn verify(&self, genesis: &Genesis) -> Result<(), BatchQCVerifyError> { + pub fn verify(&self, genesis: &GenesisHash, committee: &Committee) -> Result<(), BatchQCVerifyError> { use BatchQCVerifyError as Error; - let attesters = genesis - .attesters - .as_ref() - .ok_or(Error::AttestersNotInGenesis)?; + + if self.message.genesis!=genesis { + return Err(Error::GenesisMismatch); + } // Verify that all signers are attesters. for pk in self.signatures.keys() { - if !attesters.contains(pk) { + if !committee.contains(pk) { return Err(Error::BadSignersSet); } } // Verify that the signer's weight is sufficient. - let weight = attesters.weight_of_keys(self.signatures.keys()); - let threshold = attesters.threshold(); + let weight = committee.weight_of_keys(self.signatures.keys()); + let threshold = committee.threshold(); if weight < threshold { return Err(Error::NotEnoughSigners { got: weight, From eb1ab15ab12149fc89bd2127531f68522d0f382b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 1 Aug 2024 12:55:30 +0200 Subject: [PATCH 02/31] before removing persist_batch_qc --- .../network/src/gossip/attestation_status.rs | 4 +-- node/actors/network/src/gossip/batch_votes.rs | 7 +++--- node/actors/network/src/gossip/mod.rs | 8 +++--- .../libs/roles/src/attester/messages/batch.rs | 8 +++--- node/libs/roles/src/attester/tests.rs | 25 ++++++++++--------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/node/actors/network/src/gossip/attestation_status.rs b/node/actors/network/src/gossip/attestation_status.rs index 4d26037f..c02027fb 100644 --- a/node/actors/network/src/gossip/attestation_status.rs +++ b/node/actors/network/src/gossip/attestation_status.rs @@ -1,7 +1,7 @@ use std::fmt; - +use std::sync::Arc; use zksync_concurrency::sync; -use zksync_consensus_roles::{validator,attester}; +use zksync_consensus_roles::{attester}; use crate::watch::Watch; diff --git a/node/actors/network/src/gossip/batch_votes.rs b/node/actors/network/src/gossip/batch_votes.rs index d3b80eec..748ac6c7 100644 --- a/node/actors/network/src/gossip/batch_votes.rs +++ b/node/actors/network/src/gossip/batch_votes.rs @@ -2,6 +2,7 @@ use super::metrics; use crate::watch::Watch; use crate::gossip::AttestationStatus; +use std::cmp::Ordering; use std::{collections::HashSet, fmt, sync::Arc}; use zksync_concurrency::sync; use zksync_consensus_roles::attester; @@ -75,8 +76,8 @@ impl BatchVotes { // Disallow multiple entries for the same key: // it is important because a malicious attester may spam us with // new versions and verifying signatures is expensive. - if done.contains(&d.key) { - anyhow::bail!("duplicate entry for {:?}", d.key); + if done.contains(&vote.key) { + anyhow::bail!("duplicate entry for {:?}", vote.key); } done.insert(vote.key.clone()); self.add(vote, &mut stats)?; @@ -137,7 +138,7 @@ pub(crate) struct BatchVotesWatch(Watch); impl BatchVotesWatch { pub(crate) fn new(status: AttestationStatus) -> Self { - Self(Watch::new(BatchVotes::new(status)) + Self(Watch::new(BatchVotes::new(status))) } /// Subscribes to BatchVotes updates. diff --git a/node/actors/network/src/gossip/mod.rs b/node/actors/network/src/gossip/mod.rs index a6e35228..820dc2b8 100644 --- a/node/actors/network/src/gossip/mod.rs +++ b/node/actors/network/src/gossip/mod.rs @@ -20,7 +20,7 @@ use fetch::RequestItem; use std::sync::{atomic::AtomicUsize, Arc}; pub(crate) use validator_addrs::*; use zksync_concurrency::{ctx, ctx::channel, error::Wrap as _, scope, sync}; -use zksync_consensus_roles::{node, validator}; +use zksync_consensus_roles::{node, validator, attester}; use zksync_consensus_storage::{BatchStore, BlockStore}; mod attestation_status; @@ -172,12 +172,10 @@ impl Network { let next = attester::BatchNumber(0); loop { // Wait until the status indicates that we're ready to sign the next batch. - let status = sync::wait_for(ctx, &mut recv_status, |s| s.next_batch_to_attest >= next).await?.clone(); + let status = sync::wait_for(ctx, &mut recv_status, |s| s.batch_to_attest.number >= next).await?.clone(); // Get rid of all previous votes. We don't expect this to go backwards without regenesis, which will involve a restart. - self.batch_votes - .set_min_batch_number(status.next_batch_number) - .await; + self.batch_votes.set_status(status).await; // Now wait until we find the next quorum, whatever it is: // * on the main node, if attesters are honest, they will vote on the next batch number and the main node will not see gaps diff --git a/node/libs/roles/src/attester/messages/batch.rs b/node/libs/roles/src/attester/messages/batch.rs index 26146364..ad6e8f55 100644 --- a/node/libs/roles/src/attester/messages/batch.rs +++ b/node/libs/roles/src/attester/messages/batch.rs @@ -1,9 +1,8 @@ use super::{GenesisHash, Signed}; use crate::{ attester, - validator::{Genesis, Payload}, + validator::{Payload}, }; -use anyhow::{ensure, Context as _}; use zksync_consensus_crypto::{keccak256::Keccak256, ByteFmt, Text, TextFmt}; use zksync_consensus_utils::enum_util::Variant; @@ -127,6 +126,7 @@ pub enum BatchQCVerifyError { /// Bad signer set. #[error("signers not in committee")] BadSignersSet, + /// Genesis mismatch. #[error("genesis mismatch")] GenesisMismatch, } @@ -167,8 +167,8 @@ impl BatchQC { Ok(()) } - /// Verifies the signature of the BatchQC. - pub fn verify(&self, genesis: &GenesisHash, committee: &Committee) -> Result<(), BatchQCVerifyError> { + /// Verifies the the BatchQC. + pub fn verify(&self, genesis: GenesisHash, committee: &attester::Committee) -> Result<(), BatchQCVerifyError> { use BatchQCVerifyError as Error; if self.message.genesis!=genesis { diff --git a/node/libs/roles/src/attester/tests.rs b/node/libs/roles/src/attester/tests.rs index 57c577a8..17eacb6a 100644 --- a/node/libs/roles/src/attester/tests.rs +++ b/node/libs/roles/src/attester/tests.rs @@ -167,23 +167,23 @@ fn test_batch_qc() { for i in 0..setup1.attester_keys.len() + 1 { let mut qc = BatchQC::new(rng.gen()); for key in &setup1.attester_keys[0..i] { - qc.add(&key.sign_msg(qc.message.clone()), &setup1.genesis) + qc.add(&key.sign_msg(qc.message.clone()), &attesters) .unwrap(); } let expected_weight: u64 = attesters.iter().take(i).map(|w| w.weight).sum(); if expected_weight >= attesters.threshold() { - qc.verify(&setup1.genesis).expect("failed to verify QC"); + qc.verify(setup1.genesis.hash(), &attesters).expect("failed to verify QC"); } else { assert_matches!( - qc.verify(&setup1.genesis), + qc.verify(setup1.genesis.hash(), &attesters), Err(Error::NotEnoughSigners { .. }) ); } // Mismatching attesters sets. - assert!(qc.verify(&setup2.genesis).is_err()); - assert!(qc.verify(&genesis3).is_err()); + assert!(qc.verify(setup1.genesis.hash(), setup2.genesis.attesters.as_ref().unwrap()).is_err()); + assert!(qc.verify(setup1.genesis.hash(), genesis3.attesters.as_ref().unwrap()).is_err()); } } @@ -196,19 +196,20 @@ fn test_attester_committee_weights() { let setup = Setup::new_with_weights(rng, vec![1000, 600, 800, 6000, 900, 700]); // Expected sum of the attesters weights let sums = [1000, 1600, 2400, 8400, 9300, 10000]; + let attesters = setup + .genesis + .attesters + .as_ref() + .unwrap(); let msg: Batch = rng.gen(); let mut qc = BatchQC::new(msg.clone()); for (n, weight) in sums.iter().enumerate() { let key = &setup.attester_keys[n]; - qc.add(&key.sign_msg(msg.clone()), &setup.genesis).unwrap(); + qc.add(&key.sign_msg(msg.clone()), &attesters).unwrap(); assert_eq!( - setup - .genesis - .attesters - .as_ref() - .unwrap() - .weight_of_keys(qc.signatures.keys()), + + attesters.weight_of_keys(qc.signatures.keys()), *weight ); } From 13a4ca890878ae093487324ba277b0369d3913c9 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 1 Aug 2024 14:14:26 +0200 Subject: [PATCH 03/31] looks nice --- .../network/src/gossip/attestation_status.rs | 94 +++++++++++++++---- node/actors/network/src/gossip/batch_votes.rs | 64 +------------ node/actors/network/src/gossip/mod.rs | 5 - 3 files changed, 77 insertions(+), 86 deletions(-) diff --git a/node/actors/network/src/gossip/attestation_status.rs b/node/actors/network/src/gossip/attestation_status.rs index c02027fb..a694795e 100644 --- a/node/actors/network/src/gossip/attestation_status.rs +++ b/node/actors/network/src/gossip/attestation_status.rs @@ -13,43 +13,97 @@ pub struct AttestationStatus { /// NOTE: the committee is not supposed to change often, /// so you might want to use `Arc` instead /// to avoid extra copying. - pub committee: attester::Committee, + pub committee: attester::Committee, } -/// The subscription over the attestation status which voters can monitor for change. -pub type AttestationStatusReceiver = sync::watch::Receiver>; - /// A [Watch] over an [AttestationStatus] which we can use to notify components about /// changes in the batch number the main node expects attesters to vote on. -pub struct AttestationStatusWatch(Watch>); +struct Inner { + status: Arc, + votes: im::HashMap>>, + weight: attester::Weight, +} -impl fmt::Debug for AttestationStatusWatch { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.debug_struct("AttestationStatusWatch") - .finish_non_exhaustive() +impl Inner { + /// Verifies and adds a vote. + fn insert_vote(&mut self, vote: Arc>) -> anyhow::Result<()> { + anyhow::ensure(vote.msg.genesis == self.status.batch_to_attest.genesis, "Genesis mismatch"); + if vote.msg != self.status.batch_to_attest { return Ok(()) } + let Some(weight) = self.status.committee.contains(&vote.key) else { return Ok(()) }; + if self.votes.contains(&vote.key) { return Ok(()) } + vote.verify().context("verify")?; + self.votes.insert(vote.key.clone(),vote); + self.weight += weight; + Ok(()) + } + + pub fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { + let mut done = HashSet::new(); + for vote in votes { + // Disallow multiple entries for the same key: + // it is important because a malicious attester may spam us with + // new versions and verifying signatures is expensive. + if done.contains(&vote.key) { + anyhow::bail!("duplicate entry for {:?}", vote.key); + } + done.insert(&vote.key); + self.insert_vote(vote); + } + Ok(()) } } +struct AttestationStatusWatch(watch::Sender); + impl AttestationStatusWatch { /// Constructs AttestationStatusWatch. - pub fn new(s :AttestationStatus) -> Self { Self(Watch::new(s)) } + pub fn new(s: AttestationStatus) -> Self { + Self(Watch::new(Inner { + status: s.into(), + votes: [].into(), + weight: 0, + })) + } + + pub fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { + let mut res; + self.0.send_if_modified(|this| { + let before = this.weight; + res = this.insert_votes(votes); + this.weight > before + }); + res + } + + pub async fn wait_for_qc(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled { + let sub = &mut self.0.subscribe(); + let t = sub.borrow().status.committee.threshold(); + let this = sync::wait_for(ctx, sub, |this| this.weight >= t).await?; + let mut sigs = attester::MultiSig::default(); + for vote in this.votes.values() { + sigs.add(vote.key.clone(),vote.sig.clone()); + } + Ok(attester::BatchQC { + message: this.status.batch_to_attest.clone(), + signatures: sigs, + }) + } /// Subscribes to AttestationStatus updates. pub fn subscribe(&self) -> AttestationStatusReceiver { self.0.subscribe() - } + } /// Set the next batch number to attest on and notify subscribers it changed. /// Returns an error if the update it not valid. - pub async fn update(&self, new: AttestationStatus) -> anyhow::Result<()> { - sync::try_send_modify(&self.0.lock().await, |s| { - anyhow::ensure!(s.genesis == new.genesis, "tried to change genesis"); - anyhow::ensure!(s.batch_to_attest.number <= new.batch_to_attest.number, "tried to decrease batch number"); - if s.batch_to_attest.number == new.batch_to_attest.number { - anyhow::ensure!(s == new, "tried to change attestation status for teh batch"); - } - *s = Arc::new(new); + pub fn update(&self, s: AttestationStatus) -> anyhow::Result<()> { + sync::try_send_modify(&self.0, |this| { + anyhow::ensure!(this.status.batch_to_attest.genesis == s.batch_to_attest.genesis, "tried to change genesis"); + anyhow::ensure!(this.status.batch_to_attest.message.number < s.batch_to_attest.message.number, "tried to decrease batch number"); + this.status = s; + this.votes.clear(); + this.weight = 0; Ok(()) - }).await + }) } } diff --git a/node/actors/network/src/gossip/batch_votes.rs b/node/actors/network/src/gossip/batch_votes.rs index 748ac6c7..fc2b1c36 100644 --- a/node/actors/network/src/gossip/batch_votes.rs +++ b/node/actors/network/src/gossip/batch_votes.rs @@ -38,9 +38,7 @@ impl BatchUpdateStats { /// previous vote can be removed when a new one is added. #[derive(Clone, PartialEq, Eq)] pub(crate) struct BatchVotes { - status: Arc, - votes: im::HashMap>>, - partial_qc: attester::BatchQC, + status: Arc, } impl BatchVotes { @@ -66,70 +64,21 @@ impl BatchVotes { /// (all entries verified so far are added). /// /// Returns statistics about new entries added. - pub(super) fn update( + pub(super) async fn update( &mut self, votes: &[Arc>], ) -> anyhow::Result { let mut stats = BatchUpdateStats::default(); - let mut done = HashSet::new(); - for vote in votes { - // Disallow multiple entries for the same key: - // it is important because a malicious attester may spam us with - // new versions and verifying signatures is expensive. - if done.contains(&vote.key) { - anyhow::bail!("duplicate entry for {:?}", vote.key); - } - done.insert(vote.key.clone()); - self.add(vote, &mut stats)?; - } + votes. Ok(stats) } - /// Check if we have achieved quorum for the current batch number. - pub(super) fn find_quorum(&self) -> Option { - match self.partial_qc.verify() { - Ok(()) => Some(self.partial_qc.clone()), - Err(_) => None - } - } - /// Discards data about earlier heights. pub(super) fn set_status(&mut self, status: Arc) { self.votes.clear(); self.partial_qc = attester::BatchQC::new(status.batch_to_attest.clone()); self.status = status; } - - /// Verifies and adds a vote. - fn add(&mut self, vote: Arc>, stats: &mut BatchUpdateStats) -> anyhow::Result<()> { - // Genesis has to match - anyhow::ensure!( - vote.msg.genesis == self.status.genesis, - "vote for batch with different genesis hash: {:?}", - vote.msg.genesis - ); - - // Skip the signatures for the irrelevant batch. - if vote.message != self.status.batch_to_attest { - return Ok(()); - } - - // We just skip the entries we are not interested in. - let Some(weight) = self.status.committee.weight(&vote.key) else { - return Ok(()); - }; - - // If we already have a newer vote for this key, we can ignore this one. - if self.votes.contains(&vote.key) { return Ok(()) } - - // Check the signature before insertion. - vote.verify().context("verify()")?; - - // Insert the vote. - stats.added(vote.msg.number, weight); - self.partial_qc.signatures.add(vote.key, vote.sig); - Ok(()) - } } /// Watch wrapper of BatchVotes, @@ -179,13 +128,6 @@ impl BatchVotesWatch { } Ok(()) } - - /// Set the minimum batch number on the votes and discard old data. - pub(crate) async fn set_status(&self, status: Arc) { - metrics::BATCH_VOTES_METRICS.min_batch_number.set(status.next_batch_to_attest.0); - let this = self.0.lock().await; - this.send_modify(|votes| votes.set_status(status)); - } } /// Wrapper around [BatchVotesWatch] to publish votes over batches signed by an attester key. diff --git a/node/actors/network/src/gossip/mod.rs b/node/actors/network/src/gossip/mod.rs index 820dc2b8..5c01fc52 100644 --- a/node/actors/network/src/gossip/mod.rs +++ b/node/actors/network/src/gossip/mod.rs @@ -187,11 +187,6 @@ impl Network { votes.find_quorum(attesters, &genesis) }) .await?; - - self.batch_store - .persist_batch_qc(ctx, qc) - .await - .wrap("persist_batch_qc")?; } } } From dfda9a905737fae9570d105df0321b8269cf3afc Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 1 Aug 2024 17:11:09 +0200 Subject: [PATCH 04/31] wip --- .../network/src/gossip/attestation_status.rs | 93 ++++++----- node/actors/network/src/gossip/batch_votes.rs | 145 ------------------ 2 files changed, 55 insertions(+), 183 deletions(-) diff --git a/node/actors/network/src/gossip/attestation_status.rs b/node/actors/network/src/gossip/attestation_status.rs index a694795e..b92c1d4b 100644 --- a/node/actors/network/src/gossip/attestation_status.rs +++ b/node/actors/network/src/gossip/attestation_status.rs @@ -2,12 +2,13 @@ use std::fmt; use std::sync::Arc; use zksync_concurrency::sync; use zksync_consensus_roles::{attester}; +use std::collections::HashSet; use crate::watch::Watch; -/// Coordinate the attestation by showing the status as seen by the main node. +/// Coordinate the attestation by showing the config as seen by the main node. #[derive(Debug, Clone)] -pub struct AttestationStatus { +pub struct AttestationConfig { pub batch_to_attest: attester::Batch, /// Committee for that batch. /// NOTE: the committee is not supposed to change often, @@ -19,17 +20,29 @@ pub struct AttestationStatus { /// A [Watch] over an [AttestationStatus] which we can use to notify components about /// changes in the batch number the main node expects attesters to vote on. struct Inner { - status: Arc, + config: Arc, votes: im::HashMap>>, weight: attester::Weight, } impl Inner { + /// Returns a set of votes of `self` which are newer than the entries in `old`. + fn new_votes(&self, old: &Self) -> Vec>> { + match self.config.batch_to_attest.number.cmp(old.config.batch_to_attest.number) { + Ordering::Less => vec![], + Ordering::Greater => self.votes.values().cloned().collect(), + Ordering::Equal => self.votes.iter() + .filter(|(k,_)|!old.contains(k)) + .map(|(_,v)|v.clone()) + .collect() + } + } + /// Verifies and adds a vote. fn insert_vote(&mut self, vote: Arc>) -> anyhow::Result<()> { - anyhow::ensure(vote.msg.genesis == self.status.batch_to_attest.genesis, "Genesis mismatch"); - if vote.msg != self.status.batch_to_attest { return Ok(()) } - let Some(weight) = self.status.committee.contains(&vote.key) else { return Ok(()) }; + anyhow::ensure(vote.msg.genesis == self.config.batch_to_attest.genesis, "Genesis mismatch"); + if vote.msg != self.config.batch_to_attest { return Ok(()) } + let Some(weight) = self.config.committee.contains(&vote.key) else { return Ok(()) }; if self.votes.contains(&vote.key) { return Ok(()) } vote.verify().context("verify")?; self.votes.insert(vote.key.clone(),vote); @@ -46,63 +59,67 @@ impl Inner { if done.contains(&vote.key) { anyhow::bail!("duplicate entry for {:?}", vote.key); } - done.insert(&vote.key); + done.insert(vote.key.clone()); self.insert_vote(vote); } Ok(()) } } -struct AttestationStatusWatch(watch::Sender); +pub struct AttestationState { + key: Option, + inner: Watch, +} -impl AttestationStatusWatch { +impl AttestationState { /// Constructs AttestationStatusWatch. - pub fn new(s: AttestationStatus) -> Self { - Self(Watch::new(Inner { - status: s.into(), - votes: [].into(), - weight: 0, - })) + pub fn new(s: AttestationConfig, key: Option) -> Self { + Self { + key, + inner: Watch::new(Inner { + config: s.into(), + votes: [].into(), + weight: 0, + }), + } } - pub fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { - let mut res; - self.0.send_if_modified(|this| { - let before = this.weight; - res = this.insert_votes(votes); - this.weight > before - }); + pub async fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { + let this = self.inner.lock().await; + let mut inner = this.borrow().clone(); + let before = inner.weight; + let res = inner.insert_votes(votes); + if inner.weight > before { + this.send_replace(inner); + } res } - pub async fn wait_for_qc(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled { - let sub = &mut self.0.subscribe(); - let t = sub.borrow().status.committee.threshold(); + pub async fn wait_for_qc(&self, ctx: &ctx::Ctx) -> ctx::OrCanceled { + let sub = &mut self.inner.subscribe(); + let t = sub.borrow().config.committee.threshold(); let this = sync::wait_for(ctx, sub, |this| this.weight >= t).await?; let mut sigs = attester::MultiSig::default(); for vote in this.votes.values() { sigs.add(vote.key.clone(),vote.sig.clone()); } Ok(attester::BatchQC { - message: this.status.batch_to_attest.clone(), + message: this.config.batch_to_attest.clone(), signatures: sigs, }) } - /// Subscribes to AttestationStatus updates. - pub fn subscribe(&self) -> AttestationStatusReceiver { - self.0.subscribe() - } - - /// Set the next batch number to attest on and notify subscribers it changed. - /// Returns an error if the update it not valid. - pub fn update(&self, s: AttestationStatus) -> anyhow::Result<()> { - sync::try_send_modify(&self.0, |this| { - anyhow::ensure!(this.status.batch_to_attest.genesis == s.batch_to_attest.genesis, "tried to change genesis"); - anyhow::ensure!(this.status.batch_to_attest.message.number < s.batch_to_attest.message.number, "tried to decrease batch number"); - this.status = s; + pub async fn advance(&self, s: AttestationConfig) -> anyhow::Result<()> { + let vote = self.key.as_ref().map(|key| key.sign_msg(s.batch_to_attest.clone().insert())); + sync::try_send_modify(&self.inner.lock().await, |this| { + anyhow::ensure!(this.config.batch_to_attest.genesis == s.batch_to_attest.genesis, "tried to change genesis"); + anyhow::ensure!(this.config.batch_to_attest.message.number < s.batch_to_attest.message.number, "tried to decrease batch number"); + this.config = Arc::new(s); this.votes.clear(); this.weight = 0; + if let Some(vote) = vote { + this.insert_vote(Arc::new(vote)).unwrap(); + } Ok(()) }) } diff --git a/node/actors/network/src/gossip/batch_votes.rs b/node/actors/network/src/gossip/batch_votes.rs index fc2b1c36..9ce6fad1 100644 --- a/node/actors/network/src/gossip/batch_votes.rs +++ b/node/actors/network/src/gossip/batch_votes.rs @@ -7,151 +7,6 @@ use std::{collections::HashSet, fmt, sync::Arc}; use zksync_concurrency::sync; use zksync_consensus_roles::attester; -#[derive(Debug, Default)] -pub(super) struct BatchUpdateStats { - num_added: usize, - weight_added: u64, - last_added: Option, -} - -impl BatchUpdateStats { - fn added(&mut self, number: attester::BatchNumber, weight: u64) { - self.num_added += 1; - self.weight_added += weight; - self.last_added = Some(number); - } -} - -/// Represents the current state of node's knowledge about the attester votes. -/// -/// Eventually this data structure will have to track voting potentially happening -/// simultaneously on multiple heights, if we decrease the batch interval to be -/// several seconds, instead of a minute. By that point, the replicas should be -/// following the main node (or L1) to know what is the highest finalized batch, -/// which will act as a floor to the batch number we have to track here. It will -/// also help to protect ourselves from DoS attacks by malicious attesters casting -/// votes far into the future. -/// -/// For now, however, we just want a best effort where if we find a quorum, we -/// save it to the database, if not, we move on. For that, a simple protection -/// mechanism is to only allow one active vote per attester, which means any -/// previous vote can be removed when a new one is added. -#[derive(Clone, PartialEq, Eq)] -pub(crate) struct BatchVotes { - status: Arc, -} - -impl BatchVotes { - fn new(status: Arc) -> Self { - Self { status, votes: [].into(), partial_qcs: [].into() } - } - - /// Returns a set of votes of `self` which are newer than the entries in `old`. - pub(super) fn get_newer(&self, old: &Self) -> Vec>> { - match self.status.batch_to_attest.number.cmp(old.status.batch_to_attest.number) { - Ordering::Less => vec![], - Ordering::Greater => self.votes.values().cloned().collect(), - Ordering::Equal => self.votes.iter() - .filter(|(k,_)|!old.contains(k)) - .map(|(_,v)|v.clone()) - .collect() - } - } - - /// Updates the discovery map with entries from `data`. - /// It exits as soon as an invalid entry is found. - /// `self` might get modified even if an error is returned - /// (all entries verified so far are added). - /// - /// Returns statistics about new entries added. - pub(super) async fn update( - &mut self, - votes: &[Arc>], - ) -> anyhow::Result { - let mut stats = BatchUpdateStats::default(); - votes. - Ok(stats) - } - - /// Discards data about earlier heights. - pub(super) fn set_status(&mut self, status: Arc) { - self.votes.clear(); - self.partial_qc = attester::BatchQC::new(status.batch_to_attest.clone()); - self.status = status; - } -} - /// Watch wrapper of BatchVotes, /// which supports subscribing to BatchVotes updates. pub(crate) struct BatchVotesWatch(Watch); - -impl BatchVotesWatch { - pub(crate) fn new(status: AttestationStatus) -> Self { - Self(Watch::new(BatchVotes::new(status))) - } - - /// Subscribes to BatchVotes updates. - pub(crate) fn subscribe(&self) -> sync::watch::Receiver { - self.0.subscribe() - } - - /// Inserts data to BatchVotes. - /// Subscribers are notified iff at least 1 new entry has - /// been inserted. Returns an error iff an invalid - /// entry in `data` has been found. The provider of the - /// invalid entry should be banned. - pub(crate) async fn update( - &self, - data: &[Arc>], - ) -> anyhow::Result<()> { - let this = self.0.lock().await; - let mut votes = this.borrow().clone(); - let stats = votes.update(data)?; - - if let Some(last_added) = stats.last_added { - this.send_replace(votes); - - #[allow(clippy::float_arithmetic)] - let weight_added = stats.weight_added as f64 / votes.status.committee.total_weight() as f64; - - metrics::BATCH_VOTES_METRICS - .last_added_vote_batch_number - .set(last_added.0); - - metrics::BATCH_VOTES_METRICS - .votes_added - .inc_by(stats.num_added as u64); - - metrics::BATCH_VOTES_METRICS - .weight_added - .inc_by(weight_added); - } - Ok(()) - } -} - -/// Wrapper around [BatchVotesWatch] to publish votes over batches signed by an attester key. -pub struct BatchVotesPublisher(pub(crate) Arc); - -impl fmt::Debug for BatchVotesPublisher { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.debug_struct("BatchVotesPublisher") - .finish_non_exhaustive() - } -} - -impl BatchVotesPublisher { - /// Sign an L1 batch and push it into the batch, which should cause it to be gossiped by the network. - pub async fn publish(&self, attester: &attester::SecretKey) -> anyhow::Result<()> { - if !self.committee.contains(&attester.public()) { - return Ok(()); - } - let attestation = attester.sign_msg(self.batch_to_attest); - - metrics::BATCH_VOTES_METRICS - .last_signed_batch_number - .set(attestation.msg.number.0); - - self.0.update(&[Arc::new(attestation)]).await - } -} From 031e863ddc0aff8f9fdea25bf30ffb8b295581a4 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 2 Aug 2024 10:41:02 +0200 Subject: [PATCH 05/31] somewhat compiles --- node/actors/network/src/gossip/attestation.rs | 155 ++++++++++++++++++ .../network/src/gossip/attestation_status.rs | 126 -------------- node/actors/network/src/gossip/batch_votes.rs | 12 -- node/actors/network/src/gossip/mod.rs | 55 +------ node/actors/network/src/gossip/runner.rs | 27 +-- node/actors/network/src/lib.rs | 14 +- node/actors/network/src/testonly.rs | 20 +-- node/libs/storage/src/batch_store/mod.rs | 96 ----------- node/libs/storage/src/testonly/in_memory.rs | 62 +------ node/libs/storage/src/testonly/mod.rs | 2 +- 10 files changed, 176 insertions(+), 393 deletions(-) create mode 100644 node/actors/network/src/gossip/attestation.rs delete mode 100644 node/actors/network/src/gossip/attestation_status.rs delete mode 100644 node/actors/network/src/gossip/batch_votes.rs diff --git a/node/actors/network/src/gossip/attestation.rs b/node/actors/network/src/gossip/attestation.rs new file mode 100644 index 00000000..74320d35 --- /dev/null +++ b/node/actors/network/src/gossip/attestation.rs @@ -0,0 +1,155 @@ +use std::sync::Arc; +use std::cmp::Ordering; +use zksync_concurrency::{ctx,sync}; +use zksync_consensus_roles::attester; +use std::collections::HashSet; +use anyhow::Context as _; +use crate::watch::Watch; + +/// Coordinate the attestation by showing the config as seen by the main node. +#[derive(Debug, Clone)] +pub struct Config { + pub batch_to_attest: attester::Batch, + /// Committee for that batch. + /// NOTE: the committee is not supposed to change often, + /// so you might want to use `Arc` instead + /// to avoid extra copying. + pub committee: attester::Committee, +} + +/// A [Watch] over an [AttestationStatus] which we can use to notify components about +/// changes in the batch number the main node expects attesters to vote on. +#[derive(Clone)] +pub(crate) struct State { + config: Arc, + votes: im::HashMap>>, + weight: attester::Weight, +} + +impl State { + /// Returns a set of votes of `self` which are newer than the entries in `old`. + pub fn new_votes(&self, old: &Option) -> Vec>> { + let Some(old) = old.as_ref() else { return self.votes.values().cloned().collect() }; + match self.config.batch_to_attest.number.cmp(&old.config.batch_to_attest.number) { + Ordering::Less => vec![], + Ordering::Greater => self.votes.values().cloned().collect(), + Ordering::Equal => self.votes.iter() + .filter(|(k,_)|!old.votes.contains_key(k)) + .map(|(_,v)|v.clone()) + .collect() + } + } + + /// Verifies and adds a vote. + /// Noop if vote is not signed by a committee memver or already inserted. + /// Returns an error if genesis doesn't match or the signature is invalid. + fn insert_vote(&mut self, vote: Arc>) -> anyhow::Result<()> { + anyhow::ensure!(vote.msg.genesis == self.config.batch_to_attest.genesis, "Genesis mismatch"); + if vote.msg != self.config.batch_to_attest { return Ok(()) } + let Some(weight) = self.config.committee.weight(&vote.key) else { return Ok(()) }; + if self.votes.contains_key(&vote.key) { return Ok(()) } + // Verify signature only after checking all the other preconditions. + vote.verify().context("verify")?; + self.votes.insert(vote.key.clone(),vote); + self.weight += weight; + Ok(()) + } + + pub fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { + let mut done = HashSet::new(); + for vote in votes { + // Disallow multiple entries for the same key: + // it is important because a malicious attester may spam us with + // new versions and verifying signatures is expensive. + if done.contains(&vote.key) { + anyhow::bail!("duplicate entry for {:?}", vote.key); + } + done.insert(vote.key.clone()); + self.insert_vote(vote); + } + Ok(()) + } +} + +pub(crate) struct StateReceiver { + prev: Option, + recv: sync::watch::Receiver>, +} + +impl StateReceiver { + pub async fn wait_for_new_votes(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled>>> { + loop { + let Some(new) = (*sync::changed(ctx, &mut self.recv).await?).clone() else { continue }; + let new_votes = new.new_votes(&self.prev); + self.prev = Some(new); + if !new_votes.is_empty() { + return Ok(new_votes) + } + } + } +} + +pub struct StateWatch { + key: Option, + state: Watch>, +} + +impl StateWatch { + /// Constructs AttestationStatusWatch. + pub fn new(key: Option) -> Self { + Self { key, state: Watch::new(None) } + } + + pub(crate) fn subscribe(&self) -> StateReceiver { + let mut recv = self.state.subscribe(); + recv.mark_changed(); + StateReceiver { prev: None, recv } + } + + pub async fn insert_votes(&self, votes: impl Iterator>>) -> anyhow::Result<()> { + let locked = self.state.lock().await; + let Some(mut state) = locked.borrow().clone() else { return Ok(()) }; + let before = state.weight; + let res = state.insert_votes(votes); + if state.weight > before { + locked.send_replace(Some(state)); + } + res + } + + pub async fn wait_for_qc(&self, ctx: &ctx::Ctx) -> ctx::OrCanceled { + sync::wait_for_some(ctx, &mut self.state.subscribe(), |state| { + let state = state.as_ref()?; + if state.weight < state.config.committee.threshold() { + return None; + } + let mut sigs = attester::MultiSig::default(); + for vote in state.votes.values() { + sigs.add(vote.key.clone(),vote.sig.clone()); + } + Some(attester::BatchQC { + message: state.config.batch_to_attest.clone(), + signatures: sigs, + }) + }).await + } + + pub async fn set_config(&self, config: Config) -> anyhow::Result<()> { + let locked = self.state.lock().await; + let old = locked.borrow().clone(); + if let Some(old) = old.as_ref() { + anyhow::ensure!(old.config.batch_to_attest.genesis == config.batch_to_attest.genesis, "tried to change genesis"); + anyhow::ensure!(old.config.batch_to_attest.number < config.batch_to_attest.number, "tried to decrease batch number"); + } + let mut new = State { config: Arc::new(config), votes: im::HashMap::new(), weight: 0 }; + if let Some(key) = self.key.as_ref() { + if new.config.committee.contains(&key.public()) { + let vote = key.sign_msg(new.config.batch_to_attest.clone()); + // This is our own vote, so it always should be valid. + new.insert_vote(Arc::new(vote)).unwrap(); + } + } + locked.send_replace(Some(new)); + Ok(()) + } +} diff --git a/node/actors/network/src/gossip/attestation_status.rs b/node/actors/network/src/gossip/attestation_status.rs deleted file mode 100644 index b92c1d4b..00000000 --- a/node/actors/network/src/gossip/attestation_status.rs +++ /dev/null @@ -1,126 +0,0 @@ -use std::fmt; -use std::sync::Arc; -use zksync_concurrency::sync; -use zksync_consensus_roles::{attester}; -use std::collections::HashSet; - -use crate::watch::Watch; - -/// Coordinate the attestation by showing the config as seen by the main node. -#[derive(Debug, Clone)] -pub struct AttestationConfig { - pub batch_to_attest: attester::Batch, - /// Committee for that batch. - /// NOTE: the committee is not supposed to change often, - /// so you might want to use `Arc` instead - /// to avoid extra copying. - pub committee: attester::Committee, -} - -/// A [Watch] over an [AttestationStatus] which we can use to notify components about -/// changes in the batch number the main node expects attesters to vote on. -struct Inner { - config: Arc, - votes: im::HashMap>>, - weight: attester::Weight, -} - -impl Inner { - /// Returns a set of votes of `self` which are newer than the entries in `old`. - fn new_votes(&self, old: &Self) -> Vec>> { - match self.config.batch_to_attest.number.cmp(old.config.batch_to_attest.number) { - Ordering::Less => vec![], - Ordering::Greater => self.votes.values().cloned().collect(), - Ordering::Equal => self.votes.iter() - .filter(|(k,_)|!old.contains(k)) - .map(|(_,v)|v.clone()) - .collect() - } - } - - /// Verifies and adds a vote. - fn insert_vote(&mut self, vote: Arc>) -> anyhow::Result<()> { - anyhow::ensure(vote.msg.genesis == self.config.batch_to_attest.genesis, "Genesis mismatch"); - if vote.msg != self.config.batch_to_attest { return Ok(()) } - let Some(weight) = self.config.committee.contains(&vote.key) else { return Ok(()) }; - if self.votes.contains(&vote.key) { return Ok(()) } - vote.verify().context("verify")?; - self.votes.insert(vote.key.clone(),vote); - self.weight += weight; - Ok(()) - } - - pub fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { - let mut done = HashSet::new(); - for vote in votes { - // Disallow multiple entries for the same key: - // it is important because a malicious attester may spam us with - // new versions and verifying signatures is expensive. - if done.contains(&vote.key) { - anyhow::bail!("duplicate entry for {:?}", vote.key); - } - done.insert(vote.key.clone()); - self.insert_vote(vote); - } - Ok(()) - } -} - -pub struct AttestationState { - key: Option, - inner: Watch, -} - -impl AttestationState { - /// Constructs AttestationStatusWatch. - pub fn new(s: AttestationConfig, key: Option) -> Self { - Self { - key, - inner: Watch::new(Inner { - config: s.into(), - votes: [].into(), - weight: 0, - }), - } - } - - pub async fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { - let this = self.inner.lock().await; - let mut inner = this.borrow().clone(); - let before = inner.weight; - let res = inner.insert_votes(votes); - if inner.weight > before { - this.send_replace(inner); - } - res - } - - pub async fn wait_for_qc(&self, ctx: &ctx::Ctx) -> ctx::OrCanceled { - let sub = &mut self.inner.subscribe(); - let t = sub.borrow().config.committee.threshold(); - let this = sync::wait_for(ctx, sub, |this| this.weight >= t).await?; - let mut sigs = attester::MultiSig::default(); - for vote in this.votes.values() { - sigs.add(vote.key.clone(),vote.sig.clone()); - } - Ok(attester::BatchQC { - message: this.config.batch_to_attest.clone(), - signatures: sigs, - }) - } - - pub async fn advance(&self, s: AttestationConfig) -> anyhow::Result<()> { - let vote = self.key.as_ref().map(|key| key.sign_msg(s.batch_to_attest.clone().insert())); - sync::try_send_modify(&self.inner.lock().await, |this| { - anyhow::ensure!(this.config.batch_to_attest.genesis == s.batch_to_attest.genesis, "tried to change genesis"); - anyhow::ensure!(this.config.batch_to_attest.message.number < s.batch_to_attest.message.number, "tried to decrease batch number"); - this.config = Arc::new(s); - this.votes.clear(); - this.weight = 0; - if let Some(vote) = vote { - this.insert_vote(Arc::new(vote)).unwrap(); - } - Ok(()) - }) - } -} diff --git a/node/actors/network/src/gossip/batch_votes.rs b/node/actors/network/src/gossip/batch_votes.rs deleted file mode 100644 index 9ce6fad1..00000000 --- a/node/actors/network/src/gossip/batch_votes.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Global state distributed by active attesters, observed by all the nodes in the network. -use super::metrics; -use crate::watch::Watch; -use crate::gossip::AttestationStatus; -use std::cmp::Ordering; -use std::{collections::HashSet, fmt, sync::Arc}; -use zksync_concurrency::sync; -use zksync_consensus_roles::attester; - -/// Watch wrapper of BatchVotes, -/// which supports subscribing to BatchVotes updates. -pub(crate) struct BatchVotesWatch(Watch); diff --git a/node/actors/network/src/gossip/mod.rs b/node/actors/network/src/gossip/mod.rs index 5c01fc52..773d9e13 100644 --- a/node/actors/network/src/gossip/mod.rs +++ b/node/actors/network/src/gossip/mod.rs @@ -12,19 +12,15 @@ //! Static connections constitute a rigid "backbone" of the gossip network, which is insensitive to //! eclipse attack. Dynamic connections are supposed to improve the properties of the gossip //! network graph (minimize its diameter, increase connectedness). -pub use self::attestation_status::{AttestationStatus,AttestationStatusReceiver, AttestationStatusWatch}; -pub use self::batch_votes::BatchVotesPublisher; -use self::batch_votes::BatchVotesWatch; use crate::{gossip::ValidatorAddrsWatch, io, pool::PoolWatch, Config, MeteredStreamStats}; use fetch::RequestItem; use std::sync::{atomic::AtomicUsize, Arc}; pub(crate) use validator_addrs::*; -use zksync_concurrency::{ctx, ctx::channel, error::Wrap as _, scope, sync}; -use zksync_consensus_roles::{node, validator, attester}; +use zksync_concurrency::{ctx, ctx::channel, scope, sync}; +use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BatchStore, BlockStore}; -mod attestation_status; -mod batch_votes; +pub mod attestation; mod fetch; mod handshake; pub mod loadtest; @@ -46,8 +42,6 @@ pub(crate) struct Network { pub(crate) outbound: PoolWatch>, /// Current state of knowledge about validators' endpoints. pub(crate) validator_addrs: ValidatorAddrsWatch, - /// Current state of knowledge about batch votes. - pub(crate) batch_votes: Arc, /// Block store to serve `get_block` requests from. pub(crate) block_store: Arc, /// Batch store to serve `get_batch` requests from. @@ -61,7 +55,7 @@ pub(crate) struct Network { /// TESTONLY: how many time push_validator_addrs rpc was called by the peers. pub(crate) push_validator_addrs_calls: AtomicUsize, /// Shared watch over the current attestation status as indicated by the main node. - pub(crate) attestation_status: Arc, + pub(crate) attestation_state: Arc, } impl Network { @@ -71,7 +65,7 @@ impl Network { block_store: Arc, batch_store: Arc, sender: channel::UnboundedSender, - attestation_status: Arc, + attestation_state: Arc, ) -> Arc { Arc::new(Self { sender, @@ -81,13 +75,12 @@ impl Network { ), outbound: PoolWatch::new(cfg.gossip.static_outbound.keys().cloned().collect(), 0), validator_addrs: ValidatorAddrsWatch::default(), - batch_votes: Arc::new(BatchVotesWatch::default()), cfg, fetch_queue: fetch::Queue::default(), block_store, batch_store, push_validator_addrs_calls: 0.into(), - attestation_status, + attestation_state, }) } @@ -153,40 +146,4 @@ impl Network { }) .await; } - - /// Task that reacts to new votes being added and looks for an L1 batch QC. - /// It persists the certificate once the quorum threshold is passed. - pub(crate) async fn run_batch_qc_finder(&self, ctx: &ctx::Ctx) -> ctx::Result<()> { - let Some(attesters) = self.genesis().attesters.as_ref() else { - tracing::info!("no attesters in genesis, not looking for batch QCs"); - return Ok(()); - }; - let genesis = self.genesis().hash(); - - let mut recv_votes = self.batch_votes.subscribe(); - let mut recv_status = self.attestation_status.subscribe(); - - // Subscribe starts as seen but we don't want to miss the first item. - recv_status.mark_changed(); - - let next = attester::BatchNumber(0); - loop { - // Wait until the status indicates that we're ready to sign the next batch. - let status = sync::wait_for(ctx, &mut recv_status, |s| s.batch_to_attest.number >= next).await?.clone(); - - // Get rid of all previous votes. We don't expect this to go backwards without regenesis, which will involve a restart. - self.batch_votes.set_status(status).await; - - // Now wait until we find the next quorum, whatever it is: - // * on the main node, if attesters are honest, they will vote on the next batch number and the main node will not see gaps - // * on external nodes the votes might be affected by changes in the value returned by the API, and there might be gaps - // What is important, though, is that the batch number does not move backwards while we look for a quorum, because attesters - // (re)casting earlier votes will go ignored by those fixed on a higher min_batch_number, and gossip will only be attempted once. - // The possibility of this will be fixed by deterministally picking a start batch number based on fork indicated by genesis. - let qc = sync::wait_for_some(ctx, &mut recv_votes, |votes| { - votes.find_quorum(attesters, &genesis) - }) - .await?; - } - } } diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index 2596ccd7..0ee21bde 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -1,4 +1,4 @@ -use super::{batch_votes::BatchVotes, handshake, Network, ValidatorAddrs}; +use super::{handshake, Network, ValidatorAddrs}; use crate::{noise, preface, rpc}; use anyhow::Context as _; use async_trait::async_trait; @@ -64,15 +64,7 @@ impl rpc::Handler for &PushServer<'_> { } async fn handle(&self, _ctx: &ctx::Ctx, req: rpc::push_batch_votes::Req) -> anyhow::Result<()> { - self.net - .batch_votes - .update( - self.net.genesis().attesters.as_ref().context("attesters")?, - &self.net.genesis().hash(), - &req.0, - ) - .await?; - Ok(()) + self.net.attestation_state.insert_votes(req.0.into_iter()).await } } @@ -199,20 +191,11 @@ impl Network { // Push L1 batch votes updates to peer. s.spawn::<()>(async { let push_batch_votes_client = push_batch_votes_client; - // Snapshot of the batches when we last pushed to the peer. - let mut old = BatchVotes::default(); // Subscribe to what we know about the state of the whole network. - let mut sub = self.batch_votes.subscribe(); - sub.mark_changed(); + let mut recv = self.attestation_state.subscribe(); loop { - let new = sync::changed(ctx, &mut sub).await?.clone(); - // Get the *new* votes, which haven't been pushed before. - let diff = new.get_newer(&old); - if diff.is_empty() { - continue; - } - old = new; - let req = rpc::push_batch_votes::Req(diff); + let new = recv.wait_for_new_votes(ctx).await?; + let req = rpc::push_batch_votes::Req(new); push_batch_votes_client.call(ctx, &req, kB).await?; } }); diff --git a/node/actors/network/src/lib.rs b/node/actors/network/src/lib.rs index 1d5f3cbd..ae581958 100644 --- a/node/actors/network/src/lib.rs +++ b/node/actors/network/src/lib.rs @@ -1,6 +1,6 @@ //! Network actor maintaining a pool of outbound and inbound connections to other nodes. use anyhow::Context as _; -use gossip::{AttestationStatusWatch, BatchVotesPublisher}; +use gossip::attestation; use std::sync::Arc; use tracing::Instrument as _; use zksync_concurrency::{ @@ -57,10 +57,10 @@ impl Network { block_store: Arc, batch_store: Arc, pipe: ActorPipe, - attestation_status: Arc, + attestation_state: Arc, ) -> (Arc, Runner) { let gossip = - gossip::Network::new(cfg, block_store, batch_store, pipe.send, attestation_status); + gossip::Network::new(cfg, block_store, batch_store, pipe.send, attestation_state); let consensus = consensus::Network::new(gossip.clone()); let net = Arc::new(Self { gossip, consensus }); ( @@ -77,11 +77,6 @@ impl Network { metrics::NetworkGauges::register(Arc::downgrade(self)); } - /// Create a batch vote publisher to push attestations to gossip. - pub fn batch_vote_publisher(&self) -> BatchVotesPublisher { - BatchVotesPublisher(self.gossip.batch_votes.clone()) - } - /// Handles a dispatcher message. async fn handle_message( &self, @@ -133,9 +128,6 @@ impl Runner { Ok(()) }); - // Update QC batches in the background. - s.spawn(self.net.gossip.run_batch_qc_finder(ctx)); - // Fetch missing batches in the background. s.spawn(async { self.net.gossip.run_batch_fetcher(ctx).await; diff --git a/node/actors/network/src/testonly.rs b/node/actors/network/src/testonly.rs index d8d8c8ed..8e3fc97d 100644 --- a/node/actors/network/src/testonly.rs +++ b/node/actors/network/src/testonly.rs @@ -1,7 +1,7 @@ //! Testonly utilities. #![allow(dead_code)] use crate::{ - gossip::AttestationStatusWatch, + gossip::attestation, io::{ConsensusInputMessage, Target}, Config, GossipConfig, Network, RpcConfig, Runner, }; @@ -15,7 +15,7 @@ use std::{ }; use zksync_concurrency::{ ctx::{self, channel}, - io, limiter, net, scope, sync, time, + io, limiter, net, scope, sync, }; use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BatchStore, BlockStore}; @@ -161,7 +161,6 @@ pub fn new_fullnode(rng: &mut impl Rng, peer: &Config) -> Config { /// Runner for Instance. pub struct InstanceRunner { net_runner: Runner, - attestation_status: Arc, batch_store: Arc, terminate: channel::Receiver<()>, } @@ -171,16 +170,6 @@ impl InstanceRunner { pub async fn run(mut self, ctx: &ctx::Ctx) -> anyhow::Result<()> { scope::run!(ctx, |ctx, s| async { s.spawn_bg(self.net_runner.run(ctx)); - s.spawn_bg(async { - loop { - if let Ok(Some(n)) = self.batch_store.next_batch_to_attest(ctx).await { - self.attestation_status.update(n).await; - } - if ctx.sleep(time::Duration::seconds(1)).await.is_err() { - return Ok(()); - } - } - }); let _ = self.terminate.recv(ctx).await; Ok(()) }) @@ -199,7 +188,7 @@ impl Instance { ) -> (Self, InstanceRunner) { // Semantically we'd want this to be created at the same level as the stores, // but doing so would introduce a lot of extra cruft in setting up tests. - let attestation_status = Arc::new(AttestationStatusWatch::default()); + let attestation_state = Arc::new(attestation::StateWatch::new(None)); let (actor_pipe, dispatcher_pipe) = pipe::new(); let (net, net_runner) = Network::new( @@ -207,7 +196,7 @@ impl Instance { block_store, batch_store.clone(), actor_pipe, - attestation_status.clone(), + attestation_state, ); let (terminate_send, terminate_recv) = channel::bounded(1); ( @@ -218,7 +207,6 @@ impl Instance { }, InstanceRunner { net_runner, - attestation_status, batch_store, terminate: terminate_recv, }, diff --git a/node/libs/storage/src/batch_store/mod.rs b/node/libs/storage/src/batch_store/mod.rs index 889014f1..aae532dc 100644 --- a/node/libs/storage/src/batch_store/mod.rs +++ b/node/libs/storage/src/batch_store/mod.rs @@ -53,21 +53,6 @@ pub trait PersistentBatchStore: 'static + fmt::Debug + Send + Sync { /// Range of batches persisted in storage. fn persisted(&self) -> sync::watch::Receiver; - /// Get the next L1 batch for which attesters are expected to produce a quorum certificate. - /// - /// An external node might never have a complete history of L1 batch QCs. Once the L1 batch is included on L1, - /// the external nodes might use the [attester::SyncBatch] route to obtain them, in which case they will not - /// have a QC and no reason to get them either. The main node, however, will want to have a QC for all batches. - async fn next_batch_to_attest( - &self, - ctx: &ctx::Ctx, - ) -> ctx::Result>; - - /// Get the L1 batch QC from storage with the highest number. - /// - /// Returns `None` if we don't have a QC for any of the batches yet. - async fn last_batch_qc(&self, ctx: &ctx::Ctx) -> ctx::Result>; - /// Returns the [attester::SyncBatch] with the given number, which is used by peers /// to catch up with L1 batches that they might have missed if they went offline. async fn get_batch( @@ -76,25 +61,6 @@ pub trait PersistentBatchStore: 'static + fmt::Debug + Send + Sync { number: attester::BatchNumber, ) -> ctx::Result>; - /// Returns the [attester::Batch] with the given number, which is the `message` that - /// appears in [attester::BatchQC], and represents the content that needs to be signed - /// by the attesters. - async fn get_batch_to_sign( - &self, - ctx: &ctx::Ctx, - number: attester::BatchNumber, - ) -> ctx::Result>; - - /// Returns the QC of the batch with the given number. - async fn get_batch_qc( - &self, - ctx: &ctx::Ctx, - number: attester::BatchNumber, - ) -> ctx::Result>; - - /// Store the given batch QC in the storage persistently. - async fn store_qc(&self, ctx: &ctx::Ctx, qc: attester::BatchQC) -> ctx::Result<()>; - /// Queue the batch to be persisted in storage. /// `queue_next_batch()` may return BEFORE the batch is actually persisted, /// but if the call succeeded the batch is expected to be persisted eventually. @@ -275,48 +241,6 @@ impl BatchStore { Ok(batch) } - /// Retrieve the next batch number that doesn't have a QC yet and will need to be signed. - pub async fn next_batch_to_attest( - &self, - ctx: &ctx::Ctx, - ) -> ctx::Result> { - let t = metrics::PERSISTENT_BATCH_STORE - .next_batch_to_attest_latency - .start(); - - let batch = self - .persistent - .next_batch_to_attest(ctx) - .await - .wrap("persistent.next_batch_to_attest()")?; - - t.observe(); - Ok(batch) - } - - /// Retrieve a batch to be signed. - /// - /// This might be `None` even if the L1 batch already exists, because the commitment - /// in it is populated asynchronously. - pub async fn batch_to_sign( - &self, - ctx: &ctx::Ctx, - number: attester::BatchNumber, - ) -> ctx::Result> { - let t = metrics::PERSISTENT_BATCH_STORE - .batch_to_sign_latency - .start(); - - let batch = self - .persistent - .get_batch_to_sign(ctx, number) - .await - .wrap("persistent.get_batch_to_sign()")?; - - t.observe(); - Ok(batch) - } - /// Append batch to a queue to be persisted eventually. /// Since persisting a batch may take a significant amount of time, /// BatchStore contains a queue of batches waiting to be persisted. @@ -346,26 +270,6 @@ impl BatchStore { Ok(()) } - /// Wait until the database has a batch, then attach the corresponding QC. - pub async fn persist_batch_qc(&self, ctx: &ctx::Ctx, qc: attester::BatchQC) -> ctx::Result<()> { - let t = metrics::BATCH_STORE.persist_batch_qc.start(); - // The `store_qc` implementation in `zksync-era` retries the insertion of the QC if the payload - // isn't yet available, but to be safe we can wait for the availability signal here as well. - sync::wait_for(ctx, &mut self.persistent.persisted(), |persisted| { - qc.message.number < persisted.next() - }) - .await?; - // Now it's definitely safe to store it. - metrics::BATCH_STORE - .last_persisted_batch_qc - .set(qc.message.number.0); - - self.persistent.store_qc(ctx, qc).await?; - - t.observe(); - Ok(()) - } - /// Waits until the given batch is queued (in memory, or persisted). /// Note that it doesn't mean that the batch is actually available, as old batches might get pruned. pub async fn wait_until_queued( diff --git a/node/libs/storage/src/testonly/in_memory.rs b/node/libs/storage/src/testonly/in_memory.rs index 79a21207..93139789 100644 --- a/node/libs/storage/src/testonly/in_memory.rs +++ b/node/libs/storage/src/testonly/in_memory.rs @@ -5,7 +5,7 @@ use crate::{ }; use anyhow::Context as _; use std::{ - collections::{HashMap, VecDeque}, + collections::{VecDeque}, sync::{Arc, Mutex}, }; use zksync_concurrency::{ctx, sync}; @@ -20,10 +20,8 @@ struct BlockStoreInner { #[derive(Debug)] struct BatchStoreInner { - genesis: validator::Genesis, persisted: sync::watch::Sender, batches: Mutex>, - certs: Mutex>, } /// In-memory block store. @@ -70,12 +68,10 @@ impl BlockStore { impl BatchStore { /// New In-memory `BatchStore`. - pub fn new(genesis: validator::Genesis, first: attester::BatchNumber) -> Self { + pub fn new(first: attester::BatchNumber) -> Self { Self(Arc::new(BatchStoreInner { - genesis, persisted: sync::watch::channel(BatchStoreState { first, last: None }).0, batches: Mutex::default(), - certs: Mutex::default(), })) } } @@ -133,60 +129,6 @@ impl PersistentBatchStore for BatchStore { self.0.persisted.subscribe() } - async fn last_batch_qc(&self, _ctx: &ctx::Ctx) -> ctx::Result> { - let certs = self.0.certs.lock().unwrap(); - let last_batch_number = certs.keys().max().unwrap(); - Ok(certs.get(last_batch_number).cloned()) - } - - async fn next_batch_to_attest( - &self, - _ctx: &ctx::Ctx, - ) -> ctx::Result> { - let batches = self.0.batches.lock().unwrap(); - let certs = self.0.certs.lock().unwrap(); - - Ok(batches - .iter() - .map(|b| b.number) - .find(|n| !certs.contains_key(n))) - } - - async fn get_batch_to_sign( - &self, - ctx: &ctx::Ctx, - number: attester::BatchNumber, - ) -> ctx::Result> { - // Here we just produce some deterministic mock hash. The real hash is available in the database. - // and contains a commitment to the data submitted to L1. It is *not* over SyncBatch. - let Some(batch) = self.get_batch(ctx, number).await? else { - return Ok(None); - }; - - let bz = zksync_protobuf::canonical(&batch); - let hash = zksync_consensus_crypto::keccak256::Keccak256::new(&bz); - - Ok(Some(attester::Batch { - number, - hash: attester::BatchHash(hash), - genesis: self.0.genesis.hash(), - })) - } - - async fn get_batch_qc( - &self, - _ctx: &ctx::Ctx, - number: attester::BatchNumber, - ) -> ctx::Result> { - let certs = self.0.certs.lock().unwrap(); - Ok(certs.get(&number).cloned()) - } - - async fn store_qc(&self, _ctx: &ctx::Ctx, qc: attester::BatchQC) -> ctx::Result<()> { - self.0.certs.lock().unwrap().insert(qc.message.number, qc); - Ok(()) - } - async fn get_batch( &self, _ctx: &ctx::Ctx, diff --git a/node/libs/storage/src/testonly/mod.rs b/node/libs/storage/src/testonly/mod.rs index a6329ba8..38c4a4ba 100644 --- a/node/libs/storage/src/testonly/mod.rs +++ b/node/libs/storage/src/testonly/mod.rs @@ -89,7 +89,7 @@ impl TestMemoryStorage { first: validator::BlockNumber, ) -> Self { let im_blocks = in_memory::BlockStore::new(genesis.clone(), first); - let im_batches = in_memory::BatchStore::new(genesis.clone(), attester::BatchNumber(0)); + let im_batches = in_memory::BatchStore::new(attester::BatchNumber(0)); Self::new_with_im(ctx, im_blocks, im_batches).await } From b558ddcd02141f150fb3f26e24a9bff8e9534047 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 2 Aug 2024 12:19:23 +0200 Subject: [PATCH 06/31] compiles --- node/actors/executor/src/attestation.rs | 185 ++---------------- node/actors/executor/src/io.rs | 6 +- node/actors/executor/src/lib.rs | 42 +--- node/actors/network/src/config.rs | 5 +- node/actors/network/src/gossip/attestation.rs | 32 ++- node/actors/network/src/testonly.rs | 2 - node/tools/src/config.rs | 47 +---- 7 files changed, 64 insertions(+), 255 deletions(-) diff --git a/node/actors/executor/src/attestation.rs b/node/actors/executor/src/attestation.rs index 621fa1c3..68092f6c 100644 --- a/node/actors/executor/src/attestation.rs +++ b/node/actors/executor/src/attestation.rs @@ -1,181 +1,49 @@ //! Module to publish attestations over batches. - -use crate::Attester; -use anyhow::Context; use std::sync::Arc; -use zksync_concurrency::{ctx, sync, time}; -use zksync_consensus_network::gossip::{ - AttestationStatusReceiver, AttestationStatusWatch, BatchVotesPublisher, -}; -use zksync_consensus_roles::attester; -use zksync_consensus_storage::{BatchStore, BlockStore}; - -/// Polls the database for new batches to be signed and publishes them to the gossip channel. -pub(super) struct AttesterRunner { - block_store: Arc, - batch_store: Arc, - attester: Attester, - publisher: BatchVotesPublisher, - status: AttestationStatusReceiver, - poll_interval: time::Duration, -} - -impl AttesterRunner { - /// Create a new instance of a runner. - pub(super) fn new( - block_store: Arc, - batch_store: Arc, - attester: Attester, - publisher: BatchVotesPublisher, - status: AttestationStatusReceiver, - poll_interval: time::Duration, - ) -> Self { - Self { - block_store, - batch_store, - attester, - publisher, - status, - poll_interval, - } - } - /// Poll the database for new L1 batches and publish our signature over the batch. - pub(super) async fn run(mut self, ctx: &ctx::Ctx) -> ctx::Result<()> { - let public_key = self.attester.key.public(); - // TODO: In the future when we have attester rotation these checks will have to be checked inside the loop. - let Some(attesters) = self.block_store.genesis().attesters.as_ref() else { - tracing::warn!("Attester key is set, but the attester committee is empty."); - return Ok(()); - }; - if !attesters.contains(&public_key) { - tracing::warn!("Attester key is set, but not part of the attester committee."); - return Ok(()); - } - - let genesis = self.block_store.genesis().hash(); - - // Subscribe starts as seen but we don't want to miss the first item. - self.status.mark_changed(); - - loop { - let Some(batch_number) = sync::changed(ctx, &mut self.status) - .await? - .next_batch_to_attest - else { - continue; - }; - - tracing::info!(%batch_number, "attestation status"); - - // We can avoid actively polling the database in `wait_for_batch_to_sign` by waiting its existence - // to be indicated in memory (which itself relies on polling). This happens once we have the commitment, - // which for nodes that get the blocks through BFT should happen after execution. Nodes which - // rely on batch sync don't participate in attestations as they need the batch on L1 first. - self.batch_store - .wait_until_persisted(ctx, batch_number) - .await?; - - // Try to get the next batch to sign; the commitment might not be available just yet. - let batch = self.wait_for_batch_to_sign(ctx, batch_number).await?; - - // The certificates might be collected out of order because of how gossip works; - // we could query the DB to see if we already have a QC, or we can just go ahead - // and publish our vote, and let others ignore it. - - tracing::info!(%batch_number, "publishing attestation"); - - // We only have to publish a vote once; future peers can pull it from the register. - self.publisher - .publish(attesters, &genesis, &self.attester.key, batch) - .await - .context("publish")?; - } - } - - /// Wait for the batch commitment to become available. - async fn wait_for_batch_to_sign( - &self, - ctx: &ctx::Ctx, - number: attester::BatchNumber, - ) -> ctx::Result { - loop { - if let Some(batch) = self - .batch_store - .batch_to_sign(ctx, number) - .await - .context("batch_to_sign")? - { - return Ok(batch); - } else { - ctx.sleep(self.poll_interval).await?; - } - } - } -} +use zksync_concurrency::{ctx, time}; +pub use zksync_consensus_network::gossip::attestation::*; +use anyhow::Context as _; /// An interface which is used by attesters and nodes collecting votes over gossip to determine /// which is the next batch they are all supposed to be voting on, according to the main node. /// /// This is a convenience interface to be used with the [AttestationStatusRunner]. #[async_trait::async_trait] -pub trait AttestationStatusClient: 'static + Send + Sync { +pub trait Client: 'static + Send + Sync { /// Get the next batch number for which the main node expects a batch QC to be formed. /// /// The API might return an error while genesis is being created, which we represent with `None` /// here and mean that we'll have to try again later. - async fn next_batch_to_attest( + async fn config( &self, ctx: &ctx::Ctx, - ) -> ctx::Result>; + ) -> ctx::Result>; } -/// Use an [AttestationStatusClient] to periodically poll the main node and update the [AttestationStatusWatch]. +/// Use an [Client] to periodically poll the main node and update the [AttestationStatusWatch]. /// /// This is provided for convenience. -pub struct AttestationStatusRunner { - status: Arc, - client: Box, - poll_interval: time::Duration, +pub struct Runner { + /// state + pub state: Arc, + /// client + pub client: Box, + /// poll interval + pub poll_interval: time::Duration, } -impl AttestationStatusRunner { - /// Create a new runner to poll the main node. - pub fn new( - status: Arc, - client: Box, - poll_interval: time::Duration, - ) -> Self { - Self { - status, - client, - poll_interval, - } - } - - /// Runner based on a [BatchStore]. - pub fn new_from_store( - status: Arc, - store: Arc, - poll_interval: time::Duration, - ) -> Self { - Self::new( - status, - Box::new(LocalAttestationStatusClient(store)), - poll_interval, - ) - } - +impl Runner { /// Run the poll loop. pub async fn run(self, ctx: &ctx::Ctx) -> anyhow::Result<()> { loop { - match self.client.next_batch_to_attest(ctx).await { - Ok(Some(batch_number)) => { - self.status.update(batch_number).await; + match self.client.config(ctx).await { + Ok(Some(config)) => { + self.state.update_config(config).await.context("update_config")?; } - Ok(None) => tracing::debug!("waiting for attestation status..."), + Ok(None) => tracing::debug!("waiting for attestation config..."), Err(error) => tracing::error!( ?error, - "failed to poll attestation status, retrying later..." + "failed to poll attestation config, retrying later..." ), } if let Err(ctx::Canceled) = ctx.sleep(self.poll_interval).await { @@ -184,16 +52,3 @@ impl AttestationStatusRunner { } } } - -/// Implement the attestation status for the main node by returning the next to vote on from the [BatchStore]. -struct LocalAttestationStatusClient(Arc); - -#[async_trait::async_trait] -impl AttestationStatusClient for LocalAttestationStatusClient { - async fn next_batch_to_attest( - &self, - ctx: &ctx::Ctx, - ) -> ctx::Result> { - self.0.next_batch_to_attest(ctx).await - } -} diff --git a/node/actors/executor/src/io.rs b/node/actors/executor/src/io.rs index 914ec4b8..6c76364b 100644 --- a/node/actors/executor/src/io.rs +++ b/node/actors/executor/src/io.rs @@ -36,8 +36,8 @@ impl Dispatcher { } /// Method to start the IO dispatcher. It is simply a loop to receive messages from the actors and then forward them. - pub(super) async fn run(mut self, ctx: &ctx::Ctx) -> anyhow::Result<()> { - scope::run!(ctx, |ctx, s| async { + pub(super) async fn run(mut self, ctx: &ctx::Ctx) { + let _ : ctx::OrCanceled<()> = scope::run!(ctx, |ctx, s| async { // Start a task to handle the messages from the consensus actor. s.spawn(async { while let Ok(msg) = self.consensus_output.recv(ctx).await { @@ -65,6 +65,6 @@ impl Dispatcher { Ok(()) }) - .await + .await; } } diff --git a/node/actors/executor/src/lib.rs b/node/actors/executor/src/lib.rs index adaf4016..e294af59 100644 --- a/node/actors/executor/src/lib.rs +++ b/node/actors/executor/src/lib.rs @@ -1,6 +1,5 @@ //! Library files for the executor. We have it separate from the binary so that we can use these files in the tools crate. use crate::io::Dispatcher; -use anyhow::Context as _; use network::http; pub use network::RpcConfig; use std::{ @@ -9,11 +8,12 @@ use std::{ }; use zksync_concurrency::{ctx, limiter, net, scope, time}; use zksync_consensus_bft as bft; -use zksync_consensus_network::{self as network, gossip::AttestationStatusWatch}; -use zksync_consensus_roles::{attester, node, validator}; +use zksync_consensus_network::{self as network}; +use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BatchStore, BlockStore, ReplicaStore}; use zksync_consensus_utils::pipe; use zksync_protobuf::kB; +use anyhow::Context as _; pub mod attestation; mod io; @@ -31,13 +31,6 @@ pub struct Validator { pub payload_manager: Box, } -/// Validator-related part of [`Executor`]. -#[derive(Debug)] -pub struct Attester { - /// Consensus network configuration. - pub key: attester::SecretKey, -} - /// Config of the node executor. #[derive(Debug)] pub struct Config { @@ -98,10 +91,8 @@ pub struct Executor { pub batch_store: Arc, /// Validator-specific node data. pub validator: Option, - /// Validator-specific node data. - pub attester: Option, /// Status showing where the main node wants attester to cast their votes. - pub attestation_status: Arc, + pub attestation_state: Arc, } impl Executor { @@ -112,7 +103,6 @@ impl Executor { public_addr: self.config.public_addr.clone(), gossip: self.config.gossip(), validator_key: self.validator.as_ref().map(|v| v.key.clone()), - attester_key: self.attester.as_ref().map(|a| a.key.clone()), ping_timeout: Some(time::Duration::seconds(10)), max_block_size: self.config.max_payload_size.saturating_add(kB), max_batch_size: self.config.max_batch_size.saturating_add(kB), @@ -137,35 +127,17 @@ impl Executor { tracing::debug!("Starting actors in separate threads."); scope::run!(ctx, |ctx, s| async { - s.spawn(async { dispatcher.run(ctx).await.context("IO Dispatcher stopped") }); + s.spawn(async { dispatcher.run(ctx).await; Ok(()) }); let (net, runner) = network::Network::new( network_config, self.block_store.clone(), self.batch_store.clone(), network_actor_pipe, - self.attestation_status.clone(), + self.attestation_state, ); net.register_metrics(); s.spawn(async { runner.run(ctx).await.context("Network stopped") }); - - if let Some(attester) = self.attester { - tracing::info!("Running the node in attester mode."); - let runner = attestation::AttesterRunner::new( - self.block_store.clone(), - self.batch_store.clone(), - attester, - net.batch_vote_publisher(), - self.attestation_status.subscribe(), - self.config.batch_poll_interval, - ); - s.spawn(async { - runner.run(ctx).await?; - Ok(()) - }); - } else { - tracing::info!("Running the node in non-attester mode."); - } - + if let Some(debug_config) = self.config.debug_page { s.spawn(async { http::DebugPageServer::new(debug_config, net) diff --git a/node/actors/network/src/config.rs b/node/actors/network/src/config.rs index 2d47cf80..ed05afed 100644 --- a/node/actors/network/src/config.rs +++ b/node/actors/network/src/config.rs @@ -1,7 +1,7 @@ //! Network actor configs. use std::collections::{HashMap, HashSet}; use zksync_concurrency::{limiter, net, time}; -use zksync_consensus_roles::{attester, node, validator}; +use zksync_consensus_roles::{node, validator}; /// How often we should retry to establish a connection to a validator. /// TODO(gprusak): once it becomes relevant, choose a more appropriate retry strategy. @@ -98,9 +98,6 @@ pub struct Config { /// Private key of the validator. /// None if the node is NOT a validator. pub validator_key: Option, - /// Private key of the attester. - /// None if the node is NOT a attester. - pub attester_key: Option, /// Maximal size of the proto-encoded `validator::FinalBlock` in bytes. pub max_block_size: usize, /// Maximal size of the proto-encoded `attester::SyncBatch` in bytes. diff --git a/node/actors/network/src/gossip/attestation.rs b/node/actors/network/src/gossip/attestation.rs index 74320d35..07870db1 100644 --- a/node/actors/network/src/gossip/attestation.rs +++ b/node/actors/network/src/gossip/attestation.rs @@ -1,3 +1,5 @@ +//! Attestation. +use std::fmt; use std::sync::Arc; use std::cmp::Ordering; use zksync_concurrency::{ctx,sync}; @@ -7,10 +9,11 @@ use anyhow::Context as _; use crate::watch::Watch; /// Coordinate the attestation by showing the config as seen by the main node. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Config { + /// Batch to attest. pub batch_to_attest: attester::Batch, - /// Committee for that batch. + /// Committee that should attest the batch. /// NOTE: the committee is not supposed to change often, /// so you might want to use `Arc` instead /// to avoid extra copying. @@ -28,7 +31,7 @@ pub(crate) struct State { impl State { /// Returns a set of votes of `self` which are newer than the entries in `old`. - pub fn new_votes(&self, old: &Option) -> Vec>> { + fn new_votes(&self, old: &Option) -> Vec>> { let Some(old) = old.as_ref() else { return self.votes.values().cloned().collect() }; match self.config.batch_to_attest.number.cmp(&old.config.batch_to_attest.number) { Ordering::Less => vec![], @@ -55,7 +58,7 @@ impl State { Ok(()) } - pub fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { + fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { let mut done = HashSet::new(); for vote in votes { // Disallow multiple entries for the same key: @@ -65,7 +68,7 @@ impl State { anyhow::bail!("duplicate entry for {:?}", vote.key); } done.insert(vote.key.clone()); - self.insert_vote(vote); + self.insert_vote(vote)?; } Ok(()) } @@ -77,7 +80,7 @@ pub(crate) struct StateReceiver { } impl StateReceiver { - pub async fn wait_for_new_votes(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled>>> { + pub(crate) async fn wait_for_new_votes(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled>>> { loop { let Some(new) = (*sync::changed(ctx, &mut self.recv).await?).clone() else { continue }; let new_votes = new.new_votes(&self.prev); @@ -89,11 +92,19 @@ impl StateReceiver { } } +/// Watch of the attestation state. pub struct StateWatch { key: Option, state: Watch>, } +impl fmt::Debug for StateWatch { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("StateWatch") + .finish_non_exhaustive() + } +} + impl StateWatch { /// Constructs AttestationStatusWatch. pub fn new(key: Option) -> Self { @@ -106,7 +117,7 @@ impl StateWatch { StateReceiver { prev: None, recv } } - pub async fn insert_votes(&self, votes: impl Iterator>>) -> anyhow::Result<()> { + pub(crate) async fn insert_votes(&self, votes: impl Iterator>>) -> anyhow::Result<()> { let locked = self.state.lock().await; let Some(mut state) = locked.borrow().clone() else { return Ok(()) }; let before = state.weight; @@ -117,6 +128,7 @@ impl StateWatch { res } + /// Waits for the certificate to be collected. pub async fn wait_for_qc(&self, ctx: &ctx::Ctx) -> ctx::OrCanceled { sync::wait_for_some(ctx, &mut self.state.subscribe(), |state| { let state = state.as_ref()?; @@ -134,10 +146,14 @@ impl StateWatch { }).await } - pub async fn set_config(&self, config: Config) -> anyhow::Result<()> { + /// Updates the attestation config. + /// Clears the votes collected for the previous config. + /// Batch number has to increase with each update. + pub async fn update_config(&self, config: Config) -> anyhow::Result<()> { let locked = self.state.lock().await; let old = locked.borrow().clone(); if let Some(old) = old.as_ref() { + if *old.config == config { return Ok(()) } anyhow::ensure!(old.config.batch_to_attest.genesis == config.batch_to_attest.genesis, "tried to change genesis"); anyhow::ensure!(old.config.batch_to_attest.number < config.batch_to_attest.number, "tried to decrease batch number"); } diff --git a/node/actors/network/src/testonly.rs b/node/actors/network/src/testonly.rs index 8e3fc97d..2d0095c2 100644 --- a/node/actors/network/src/testonly.rs +++ b/node/actors/network/src/testonly.rs @@ -104,7 +104,6 @@ where // due to timeouts. ping_timeout: None, validator_key: Some(validator_key.clone()), - attester_key: None, gossip: GossipConfig { key: rng.gen(), dynamic_inbound_limit: usize::MAX, @@ -143,7 +142,6 @@ pub fn new_fullnode(rng: &mut impl Rng, peer: &Config) -> Config { // due to timeouts. ping_timeout: None, validator_key: None, - attester_key: None, gossip: GossipConfig { key: rng.gen(), dynamic_inbound_limit: usize::MAX, diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index f659bdb3..816df838 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -10,11 +10,11 @@ use std::{ sync::Arc, }; use tokio_rustls::rustls::pki_types::{CertificateDer, PrivateKeyDer}; -use zksync_concurrency::{ctx, net, scope, time}; +use zksync_concurrency::{ctx, net, time}; use zksync_consensus_bft as bft; use zksync_consensus_crypto::{read_optional_text, read_required_text, Text, TextFmt}; -use zksync_consensus_executor::{self as executor, attestation::AttestationStatusRunner}; -use zksync_consensus_network::{gossip::AttestationStatusWatch, http}; +use zksync_consensus_executor::{self as executor, attestation}; +use zksync_consensus_network::{http}; use zksync_consensus_roles::{attester, node, validator}; use zksync_consensus_storage::testonly::{TestMemoryStorage, TestMemoryStorageRunner}; use zksync_consensus_utils::debug_page; @@ -260,22 +260,15 @@ impl Configs { pub async fn make_executor( &self, ctx: &ctx::Ctx, - ) -> ctx::Result<(executor::Executor, TestExecutorRunner)> { + ) -> ctx::Result<(executor::Executor, TestMemoryStorageRunner)> { let replica_store = store::RocksDB::open(self.app.genesis.clone(), &self.database).await?; let store = TestMemoryStorage::new(ctx, &self.app.genesis).await; // We don't have an API to poll in this setup, we can only create a local store based attestation client. - let attestation_status = Arc::new(AttestationStatusWatch::default()); - let attestation_status_runner = AttestationStatusRunner::new_from_store( - attestation_status.clone(), - store.batches.clone(), - time::Duration::seconds(1), - ); - - let runner = TestExecutorRunner { - storage_runner: store.runner, - attestation_status_runner, - }; + let attestation_state = Arc::new(attestation::StateWatch::new( + self.app.attester_key.clone() + )); + let runner = store.runner; let e = executor::Executor { config: executor::Config { @@ -313,12 +306,7 @@ impl Configs { self.app.max_payload_size, )), }), - attester: self - .app - .attester_key - .as_ref() - .map(|key| executor::Attester { key: key.clone() }), - attestation_status, + attestation_state, }; Ok((e, runner)) } @@ -345,20 +333,3 @@ fn load_private_key(path: &PathBuf) -> anyhow::Result> { // Load and return a single private key. Ok(rustls_pemfile::private_key(&mut reader).map(|key| key.expect("Private key not found"))?) } - -pub struct TestExecutorRunner { - storage_runner: TestMemoryStorageRunner, - attestation_status_runner: AttestationStatusRunner, -} - -impl TestExecutorRunner { - /// Runs the storage and the attestation status. - pub async fn run(self, ctx: &ctx::Ctx) -> anyhow::Result<()> { - scope::run!(ctx, |ctx, s| async { - s.spawn(self.storage_runner.run(ctx)); - s.spawn(self.attestation_status_runner.run(ctx)); - Ok(()) - }) - .await - } -} From 7baf2ced6502bbecd267ec41e5f1f70fe6af0063 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 2 Aug 2024 13:36:22 +0200 Subject: [PATCH 07/31] tests wip --- node/actors/executor/src/tests.rs | 10 +- .../{attestation.rs => attestation/mod.rs} | 3 + .../network/src/gossip/attestation/tests.rs | 158 ++++++++++++++ .../network/src/gossip/tests/fetch_batches.rs | 4 - node/actors/network/src/gossip/tests/mod.rs | 205 +----------------- .../network/src/gossip/tests/syncing.rs | 2 - node/libs/storage/src/tests.rs | 2 +- 7 files changed, 169 insertions(+), 215 deletions(-) rename node/actors/network/src/gossip/{attestation.rs => attestation/mod.rs} (99%) create mode 100644 node/actors/network/src/gossip/attestation/tests.rs diff --git a/node/actors/executor/src/tests.rs b/node/actors/executor/src/tests.rs index 6ce5c057..200d1e65 100644 --- a/node/actors/executor/src/tests.rs +++ b/node/actors/executor/src/tests.rs @@ -29,8 +29,8 @@ fn config(cfg: &network::Config) -> Config { /// The test executors below are not running with attesters, so we just create an [AttestationStatusWatch] /// that will never be updated. -fn never_attest() -> Arc { - Arc::new(AttestationStatusWatch::default()) +fn never_attest() -> Arc { + attestation::StateWatch::new(None).into() } fn validator( @@ -48,8 +48,7 @@ fn validator( replica_store: Box::new(replica_store), payload_manager: Box::new(bft::testonly::RandomPayload(1000)), }), - attester: None, - attestation_status: never_attest(), + attestation_state: never_attest(), } } @@ -63,8 +62,7 @@ fn fullnode( block_store, batch_store, validator: None, - attester: None, - attestation_status: never_attest(), + attestation_state: never_attest(), } } diff --git a/node/actors/network/src/gossip/attestation.rs b/node/actors/network/src/gossip/attestation/mod.rs similarity index 99% rename from node/actors/network/src/gossip/attestation.rs rename to node/actors/network/src/gossip/attestation/mod.rs index 07870db1..71b64405 100644 --- a/node/actors/network/src/gossip/attestation.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -8,6 +8,9 @@ use std::collections::HashSet; use anyhow::Context as _; use crate::watch::Watch; +#[cfg(test)] +mod tests; + /// Coordinate the attestation by showing the config as seen by the main node. #[derive(Debug, Clone, PartialEq)] pub struct Config { diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs new file mode 100644 index 00000000..6413a05e --- /dev/null +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -0,0 +1,158 @@ +use super::*; +use zksync_concurrency::testonly::abort_on_panic; +use rand::Rng as _; + +type Vote = Arc>; + +#[derive(Default, Debug, PartialEq)] +struct Votes(im::HashMap); + +impl Votes { + fn insert(&mut self, vote: Vote) { + self.0.insert(vote.key.clone(), vote); + } + + fn get(&mut self, key: &attester::SecretKey) -> Vote { + self.0.get(&key.public()).unwrap().clone() + } +} + +impl StateReceiver { + fn snapshot(&self) -> Votes { + let Some(state) = self.recv.borrow().clone() else { return Votes::default() }; + state.votes.values().cloned().into() + } +} + +impl> From for Votes { + fn from(it: I) -> Self { + Self(it.map(|v|(v.key.clone(),v)).collect()) + } +} + +#[tokio::test] +async fn test_insert_votes() { + abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + let state = StateWatch::new(None); + let keys: Vec = (0..8).map(|_| rng.gen()).collect(); + let config = Config { + batch_to_attest: rng.gen(), + committee: attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { + key: k.public(), + weight: 1250, + })).unwrap(), + }; + state.update_config(config.clone()); + let mut recv = state.subscribe(); + + let all_votes : Vec = keys.iter().map(|k| k.sign_msg(config.batch_to_attest.clone().into()).into()).collect(); + + // Initial votes. + state.insert_votes(all_votes[0..3].iter().cloned()).await.unwrap(); + assert_eq!( + Votes::from(all_votes[0..3].iter().cloned()), + recv.wait_for_new_votes(ctx).await.unwrap().into_iter().into() + ); + + // Adding votes gradually. + state.insert_votes(all_votes[3..5].iter().cloned()).await.unwrap(); + state.insert_votes(all_votes[5..7].iter().cloned()).await.unwrap(); + assert_eq!( + Votes::from(all_votes[3..7].iter().cloned()), + recv.wait_for_new_votes(ctx).await.unwrap().into_iter().into() + ); + + // Adding duplicate votes (noop). + state.insert_votes(all_votes[2..6].iter().cloned()).await.unwrap(); + assert!(!recv.recv.has_changed().unwrap()); + + // Adding votes out of committee (noop). + state.insert_votes((0..3).map(|_| { + let k : attester::SecretKey = rng.gen(); + k.sign_msg(config.batch_to_attest.clone()).into() + })).await.unwrap(); + assert!(!recv.recv.has_changed().unwrap()); + + // Adding votes for different batch (noop). + state.insert_votes((0..3).map(|_| { + let k : attester::SecretKey = rng.gen(); + k.sign_msg(attester::Batch { + genesis: config.batch_to_attest.genesis, + number: rng.gen(), + hash: rng.gen(), + }).into() + })).await.unwrap(); + assert!(!recv.recv.has_changed().unwrap()); + + // Adding incorrect votes (error). + let mut bad_vote = (*all_votes[7]).clone(); + bad_vote.sig = rng.gen(); + assert!(state.insert_votes([bad_vote.into()].into_iter()).await.is_err()); + assert!(!recv.recv.has_changed().unwrap()); +} + +/* +#[tokio::test] +fn test_batch_votes_quorum() { + abort_on_panic(); + let rng = &mut ctx::test_root(&ctx::RealClock).rng(); + + for _ in 0..10 { + let committee_size = rng.gen_range(1..20); + let keys: Vec = (0..committee_size).map(|_| rng.gen()).collect(); + let attesters = attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { + key: k.public(), + weight: rng.gen_range(1..=100), + })) + .unwrap(); + + let batch0 = rng.gen::(); + let batch1 = attester::Batch { + number: batch0.number.next(), + hash: rng.gen(), + genesis: batch0.genesis, + }; + let genesis = batch0.genesis; + let mut batches = [(batch0, 0u64), (batch1, 0u64)]; + + let mut votes = BatchVotes::default(); + for sk in &keys { + // We need 4/5+1 for quorum, so let's say ~80% vote on the second batch. + let b = usize::from(rng.gen_range(0..100) < 80); + let batch = &batches[b].0; + let vote = sk.sign_msg(batch.clone()); + votes + .update(&attesters, &genesis, &[Arc::new(vote)]) + .unwrap(); + batches[b].1 += attesters.weight(&sk.public()).unwrap(); + + // Check that as soon as we have quorum it's found. + if batches[b].1 >= attesters.threshold() { + let qc = votes + .find_quorum(&attesters, &genesis) + .expect("should find quorum"); + assert!(qc.message == *batch); + assert!(qc.signatures.keys().count() > 0); + } + } + + // Check that if there was no quoroum then we don't find any. + if !batches.iter().any(|b| b.1 >= attesters.threshold()) { + assert!(votes.find_quorum(&attesters, &genesis).is_none()); + } + + // Check that the minimum batch number prunes data. + let last_batch = batches[1].0.number; + + votes.set_min_batch_number(last_batch); + assert!(votes.votes.values().all(|v| v.msg.number >= last_batch)); + assert!(votes.support.keys().all(|n| *n >= last_batch)); + + votes.set_min_batch_number(last_batch.next()); + assert!(votes.votes.is_empty()); + assert!(votes.support.is_empty()); + } +}*/ diff --git a/node/actors/network/src/gossip/tests/fetch_batches.rs b/node/actors/network/src/gossip/tests/fetch_batches.rs index 12b8e840..eb169278 100644 --- a/node/actors/network/src/gossip/tests/fetch_batches.rs +++ b/node/actors/network/src/gossip/tests/fetch_batches.rs @@ -17,7 +17,6 @@ async fn test_simple() { cfg.rpc.push_batch_store_state_rate = limiter::Rate::INF; cfg.rpc.get_batch_rate = limiter::Rate::INF; cfg.rpc.get_batch_timeout = None; - cfg.attester_key = None; cfg.validator_key = None; scope::run!(ctx, |ctx, s| async { @@ -127,7 +126,6 @@ async fn test_concurrent_requests() { cfg.rpc.push_batch_store_state_rate = limiter::Rate::INF; cfg.rpc.get_batch_rate = limiter::Rate::INF; cfg.rpc.get_batch_timeout = None; - cfg.attester_key = None; cfg.validator_key = None; scope::run!(ctx, |ctx, s| async { @@ -204,7 +202,6 @@ async fn test_bad_responses() { cfg.rpc.get_batch_rate = limiter::Rate::INF; cfg.rpc.get_batch_timeout = None; cfg.validator_key = None; - cfg.attester_key = None; scope::run!(ctx, |ctx, s| async { let store = TestMemoryStorage::new(ctx, &setup.genesis).await; @@ -281,7 +278,6 @@ async fn test_retry() { cfg.rpc.get_batch_rate = limiter::Rate::INF; cfg.rpc.get_batch_timeout = None; cfg.validator_key = None; - cfg.attester_key = None; scope::run!(ctx, |ctx, s| async { let store = TestMemoryStorage::new(ctx, &setup.genesis).await; diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index e0f5f3c2..0f5f7c93 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -1,7 +1,6 @@ use super::ValidatorAddrs; use crate::{ gossip::{ - batch_votes::{BatchVotes, BatchVotesWatch}, handshake, validator_addrs::ValidatorAddrsWatch, }, @@ -24,7 +23,7 @@ use zksync_concurrency::{ time, }; use zksync_consensus_roles::{attester, validator}; -use zksync_consensus_storage::{testonly::TestMemoryStorage, PersistentBatchStore}; +use zksync_consensus_storage::{testonly::TestMemoryStorage}; mod fetch_batches; mod fetch_blocks; @@ -96,9 +95,6 @@ fn mk_version(rng: &mut R) -> u64 { #[derive(Default)] struct View(im::HashMap>>); -#[derive(Default)] -struct Signatures(im::HashMap>>); - fn mk_netaddr( key: &validator::SecretKey, addr: std::net::SocketAddr, @@ -137,19 +133,6 @@ fn random_netaddr( )) } -fn random_batch_vote( - rng: &mut R, - key: &attester::SecretKey, - genesis: attester::GenesisHash, -) -> Arc> { - let batch = attester::Batch { - number: attester::BatchNumber(rng.gen_range(0..1000)), - hash: rng.gen(), - genesis, - }; - Arc::new(key.sign_msg(batch.to_owned())) -} - fn update_netaddr( rng: &mut R, addr: &validator::NetAddress, @@ -165,20 +148,6 @@ fn update_netaddr( )) } -fn update_signature( - rng: &mut R, - batch: &attester::Batch, - key: &attester::SecretKey, - batch_number_diff: i64, -) -> Arc> { - let batch = attester::Batch { - number: attester::BatchNumber((batch.number.0 as i64 + batch_number_diff) as u64), - hash: rng.gen(), - genesis: batch.genesis, - }; - Arc::new(key.sign_msg(batch.to_owned())) -} - impl View { fn insert(&mut self, entry: Arc>) { self.0.insert(entry.key.clone(), entry); @@ -193,20 +162,6 @@ impl View { } } -impl Signatures { - fn insert(&mut self, entry: Arc>) { - self.0.insert(entry.key.clone(), entry); - } - - fn get(&mut self, key: &attester::SecretKey) -> Arc> { - self.0.get(&key.public()).unwrap().clone() - } - - fn as_vec(&self) -> Vec>> { - self.0.values().cloned().collect() - } -} - #[tokio::test] async fn test_validator_addrs() { abort_on_panic(); @@ -545,161 +500,7 @@ async fn rate_limiting() { } } -#[tokio::test] -async fn test_batch_votes() { - abort_on_panic(); - let rng = &mut ctx::test_root(&ctx::RealClock).rng(); - - let keys: Vec = (0..8).map(|_| rng.gen()).collect(); - let attesters = attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { - key: k.public(), - weight: 1250, - })) - .unwrap(); - let votes = BatchVotesWatch::default(); - let mut sub = votes.subscribe(); - - let genesis = rng.gen::(); - - // Initial values. - let mut want = Signatures::default(); - for k in &keys[0..6] { - want.insert(random_batch_vote(rng, k, genesis)); - } - votes - .update(&attesters, &genesis, &want.as_vec()) - .await - .unwrap(); - assert_eq!(want.0, sub.borrow_and_update().votes); - - // newer batch number, should be updated - let k0v2 = update_signature(rng, &want.get(&keys[0]).msg, &keys[0], 1); - // same batch number, should be ignored - let k1v2 = update_signature(rng, &want.get(&keys[1]).msg, &keys[1], 0); - // older batch number, should be ignored - let k4v2 = update_signature(rng, &want.get(&keys[4]).msg, &keys[4], -1); - // first entry for a key in the config, should be inserted - let k6v1 = random_batch_vote(rng, &keys[6], genesis); - // entry for a key outside of the config, should be ignored - let k8 = rng.gen(); - let k8v1 = random_batch_vote(rng, &k8, genesis); - - // Update the ones we expect to succeed - want.insert(k0v2.clone()); - want.insert(k6v1.clone()); - - // Send all of them to the votes - let update = [ - k0v2, - k1v2, - k4v2, - // no new entry for keys[5] - k6v1, - // no entry at all for keys[7] - k8v1.clone(), - ]; - votes.update(&attesters, &genesis, &update).await.unwrap(); - assert_eq!(want.0, sub.borrow_and_update().votes); - - // Invalid signature, should be rejected. - let mut k0v3 = mk_batch( - rng, - &keys[1], - attester::BatchNumber(want.get(&keys[0]).msg.number.0 + 1), - genesis, - ); - k0v3.key = keys[0].public(); - assert!(votes - .update(&attesters, &genesis, &[Arc::new(k0v3)]) - .await - .is_err()); - - // Invalid genesis, should be rejected. - let other_genesis = rng.gen(); - let k1v3 = mk_batch( - rng, - &keys[1], - attester::BatchNumber(want.get(&keys[1]).msg.number.0 + 1), - other_genesis, - ); - assert!(votes - .update(&attesters, &genesis, &[Arc::new(k1v3)]) - .await - .is_err()); - - assert_eq!(want.0, sub.borrow_and_update().votes); - - // Duplicate entry in the update, should be rejected. - assert!(votes - .update(&attesters, &genesis, &[k8v1.clone(), k8v1]) - .await - .is_err()); - assert_eq!(want.0, sub.borrow_and_update().votes); -} - -#[test] -fn test_batch_votes_quorum() { - abort_on_panic(); - let rng = &mut ctx::test_root(&ctx::RealClock).rng(); - - for _ in 0..10 { - let committee_size = rng.gen_range(1..20); - let keys: Vec = (0..committee_size).map(|_| rng.gen()).collect(); - let attesters = attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { - key: k.public(), - weight: rng.gen_range(1..=100), - })) - .unwrap(); - - let batch0 = rng.gen::(); - let batch1 = attester::Batch { - number: batch0.number.next(), - hash: rng.gen(), - genesis: batch0.genesis, - }; - let genesis = batch0.genesis; - let mut batches = [(batch0, 0u64), (batch1, 0u64)]; - - let mut votes = BatchVotes::default(); - for sk in &keys { - // We need 4/5+1 for quorum, so let's say ~80% vote on the second batch. - let b = usize::from(rng.gen_range(0..100) < 80); - let batch = &batches[b].0; - let vote = sk.sign_msg(batch.clone()); - votes - .update(&attesters, &genesis, &[Arc::new(vote)]) - .unwrap(); - batches[b].1 += attesters.weight(&sk.public()).unwrap(); - - // Check that as soon as we have quorum it's found. - if batches[b].1 >= attesters.threshold() { - let qc = votes - .find_quorum(&attesters, &genesis) - .expect("should find quorum"); - assert!(qc.message == *batch); - assert!(qc.signatures.keys().count() > 0); - } - } - - // Check that if there was no quoroum then we don't find any. - if !batches.iter().any(|b| b.1 >= attesters.threshold()) { - assert!(votes.find_quorum(&attesters, &genesis).is_none()); - } - - // Check that the minimum batch number prunes data. - let last_batch = batches[1].0.number; - - votes.set_min_batch_number(last_batch); - assert!(votes.votes.values().all(|v| v.msg.number >= last_batch)); - assert!(votes.support.keys().all(|n| *n >= last_batch)); - - votes.set_min_batch_number(last_batch.next()); - assert!(votes.votes.is_empty()); - assert!(votes.support.is_empty()); - } -} - -// +/* #[tokio::test(flavor = "multi_thread")] async fn test_batch_votes_propagation() { let _guard = set_timeout(time::Duration::seconds(10)); @@ -771,4 +572,4 @@ async fn test_batch_votes_propagation() { }) .await .unwrap(); -} +}*/ diff --git a/node/actors/network/src/gossip/tests/syncing.rs b/node/actors/network/src/gossip/tests/syncing.rs index b456350f..1bd51446 100644 --- a/node/actors/network/src/gossip/tests/syncing.rs +++ b/node/actors/network/src/gossip/tests/syncing.rs @@ -327,7 +327,6 @@ async fn coordinated_batch_syncing(node_count: usize, gossip_peers: usize) { cfg.rpc.get_batch_rate = limiter::Rate::INF; cfg.rpc.get_batch_timeout = None; cfg.validator_key = None; - cfg.attester_key = None; let store = TestMemoryStorage::new(ctx, &setup.genesis).await; s.spawn_bg(store.runner.run(ctx)); let (node, runner) = testonly::Instance::new(cfg, store.blocks, store.batches); @@ -384,7 +383,6 @@ async fn uncoordinated_batch_syncing( cfg.rpc.get_batch_rate = limiter::Rate::INF; cfg.rpc.get_batch_timeout = None; cfg.validator_key = None; - cfg.attester_key = None; let store = TestMemoryStorage::new(ctx, &setup.genesis).await; s.spawn_bg(store.runner.run(ctx)); let (node, runner) = testonly::Instance::new(cfg, store.blocks, store.batches); diff --git a/node/libs/storage/src/tests.rs b/node/libs/storage/src/tests.rs index 11fa5431..36266bba 100644 --- a/node/libs/storage/src/tests.rs +++ b/node/libs/storage/src/tests.rs @@ -32,7 +32,7 @@ async fn test_inmemory_batch_store() { let mut setup = Setup::new(rng, 3); setup.push_batches(rng, 5); - let store = &testonly::in_memory::BatchStore::new(setup.genesis.clone(), BatchNumber(0)); + let store = &testonly::in_memory::BatchStore::new(BatchNumber(0)); let mut want = vec![]; for batch in &setup.batches { store.queue_next_batch(ctx, batch.clone()).await.unwrap(); From 9e59da75a26614dc3f90bbd123e932017e3fcdd1 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 6 Aug 2024 14:57:30 +0200 Subject: [PATCH 08/31] plugged in pull votes server --- node/actors/network/src/config.rs | 8 +++- node/actors/network/src/gossip/runner.rs | 54 ++++++++++++------------ 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/node/actors/network/src/config.rs b/node/actors/network/src/config.rs index ed05afed..547b9dcf 100644 --- a/node/actors/network/src/config.rs +++ b/node/actors/network/src/config.rs @@ -26,8 +26,10 @@ pub struct RpcConfig { pub get_batch_timeout: Option, /// Max rate of sending/receiving consensus messages. pub consensus_rate: limiter::Rate, - /// Max rate of sending/receiving l1 batch votes messages. + /// Max rate of sending/receiving PushBatchVotes RPCs. pub push_batch_votes_rate: limiter::Rate, + /// Max rate of sending/reciving PullBatchVotes RPCs. + pub pull_batch_votes_rate: limiter::Rate, } impl Default for RpcConfig { @@ -63,6 +65,10 @@ impl Default for RpcConfig { burst: 2, refresh: time::Duration::milliseconds(500), }, + pull_batch_votes_rate: limiter::Rate { + burst: 2, + refresh: time::Duration::milliseconds(500), + }, } } } diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index 577badf0..9773cbc9 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -13,7 +13,6 @@ use zksync_protobuf::kB; struct PushServer<'a> { blocks: sync::watch::Sender, batches: sync::watch::Sender, - /// The network is required for the verification of messages. net: &'a Network, } @@ -156,6 +155,10 @@ impl Network { rpc::Client::::new(ctx, self.cfg.rpc.get_block_rate); let get_batch_client = rpc::Client::::new(ctx, self.cfg.rpc.get_batch_rate); + let push_batch_votes_client = rpc::Client::::new( + ctx, + self.cfg.rpc.push_batch_votes_rate, + ); scope::run!(ctx, |ctx, s| async { let mut service = rpc::Service::new() @@ -181,33 +184,32 @@ impl Network { .add_server(ctx, &*self.block_store, self.cfg.rpc.get_block_rate) .add_client(&get_batch_client) .add_server(ctx, &*self.batch_store, self.cfg.rpc.get_batch_rate) - .add_server(ctx, rpc::ping::Server, rpc::ping::RATE); - - // If there is an attester committee then - if self.genesis().attesters.as_ref().is_some() { - let push_batch_votes_client = rpc::Client::::new( + .add_server(ctx, rpc::ping::Server, rpc::ping::RATE) + .add_client(&push_batch_votes_client) + .add_server::( ctx, + &push_server, self.cfg.rpc.push_batch_votes_rate, - ); - service = service - .add_client(&push_batch_votes_client) - .add_server::( - ctx, - &push_server, - self.cfg.rpc.push_batch_votes_rate, - ); - // Push L1 batch votes updates to peer. - s.spawn::<()>(async { - let push_batch_votes_client = push_batch_votes_client; - // Subscribe to what we know about the state of the whole network. - let mut recv = self.attestation_state.subscribe(); - loop { - let new = recv.wait_for_new_votes(ctx).await?; - let req = rpc::batch_votes::Msg(new); - push_batch_votes_client.call(ctx, &req, kB).await?; - } - }); - } + ) + .add_server(ctx, &*self.attestation_state, self.cfg.rpc.pull_batch_votes_rate); + + // Push L1 batch votes updates to peer. + s.spawn::<()>(async { + let push_batch_votes_client = push_batch_votes_client; + // Subscribe to what we know about the state of the whole network. + let mut recv = self.attestation_state.subscribe(); + loop { + let new = recv.wait_for_new_votes(ctx).await?; + let req = rpc::batch_votes::Msg(new); + push_batch_votes_client.call(ctx, &req, kB).await?; + } + }); + + // Pull L1 batch votes from peers. + s.spawn(async { + // TODO + Ok(()) + }); if let Some(ping_timeout) = &self.cfg.ping_timeout { let ping_client = rpc::Client::::new(ctx, rpc::ping::RATE); From 758027268942c9f0af813c95d139b46f133b2907 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 6 Aug 2024 16:17:52 +0200 Subject: [PATCH 09/31] migrated to protobuf registry --- node/actors/network/src/gossip/testonly.rs | 6 ++-- node/actors/network/src/proto/rpc.proto | 4 +++ node/actors/network/src/rpc/batch_votes.rs | 17 +++------- node/actors/network/src/rpc/consensus.rs | 5 +-- node/actors/network/src/rpc/get_batch.rs | 5 +-- node/actors/network/src/rpc/get_block.rs | 5 +-- node/actors/network/src/rpc/mod.rs | 34 +++++++++++++------ node/actors/network/src/rpc/ping.rs | 5 +-- .../network/src/rpc/push_batch_store_state.rs | 5 +-- .../network/src/rpc/push_block_store_state.rs | 5 +-- .../network/src/rpc/push_validator_addrs.rs | 5 +-- node/actors/network/src/rpc/tests.rs | 21 +----------- 12 files changed, 56 insertions(+), 61 deletions(-) diff --git a/node/actors/network/src/gossip/testonly.rs b/node/actors/network/src/gossip/testonly.rs index d37419c0..114a838e 100644 --- a/node/actors/network/src/gossip/testonly.rs +++ b/node/actors/network/src/gossip/testonly.rs @@ -28,7 +28,7 @@ impl ConnRunner { fn mux_entry(ctx: &ctx::Ctx) -> (mux::CapabilityId, Arc) { ( - R::CAPABILITY_ID, + R::CAPABILITY.id(), mux::StreamQueue::new(ctx, R::INFLIGHT, limiter::Rate::INF), ) } @@ -89,7 +89,7 @@ impl Conn { ) -> ctx::OrCanceled> { Ok(ServerStream( self.connect - .get(&R::CAPABILITY_ID) + .get(&R::CAPABILITY.id()) .unwrap() .open(ctx) .await?, @@ -104,7 +104,7 @@ impl Conn { ) -> ctx::OrCanceled> { Ok(ClientStream( self.accept - .get(&R::CAPABILITY_ID) + .get(&R::CAPABILITY.id()) .unwrap() .open(ctx) .await?, diff --git a/node/actors/network/src/proto/rpc.proto b/node/actors/network/src/proto/rpc.proto index 7c5bc24c..04f438fa 100644 --- a/node/actors/network/src/proto/rpc.proto +++ b/node/actors/network/src/proto/rpc.proto @@ -8,6 +8,10 @@ enum Capability { PUSH_VALIDATOR_ADDRS = 1; PUSH_BLOCK_STORE_STATE = 3; GET_BLOCK = 4; + /// Deprecated, because adding `genesis_hash` to `attester::Batch` + /// was not backward compatible - old binaries couldn't verify + /// signatures on messages with `genesis_hash` and were treating it + /// as malicious behavior. PUSH_BATCH_VOTES_V1 = 5; PUSH_BATCH_VOTES_V2 = 8; PULL_BATCH_VOTES = 9; diff --git a/node/actors/network/src/rpc/batch_votes.rs b/node/actors/network/src/rpc/batch_votes.rs index f3d6e5ad..9f3a16c0 100644 --- a/node/actors/network/src/rpc/batch_votes.rs +++ b/node/actors/network/src/rpc/batch_votes.rs @@ -1,9 +1,10 @@ //! Defines RPC for passing consensus messages. -use crate::{mux, proto::gossip as proto}; +use crate::{proto::gossip as proto}; use anyhow::Context as _; use std::sync::Arc; use zksync_consensus_roles::attester; use zksync_protobuf::ProtoFmt; +use super::Capability; /// RPC pushing fresh batch votes. pub(crate) struct PushRpc; @@ -11,18 +12,8 @@ pub(crate) struct PushRpc; /// RPC requesting all batch votes from peers. pub(crate) struct PullRpc; -/// Deprecated, because adding `genesis_hash` to `attester::Batch` -/// was not backward compatible - old binaries couldn't verify -/// signatures on messages with `genesis_hash` and were treating it -/// as malicious behavior. -#[allow(dead_code)] -pub(super) const PUSH_V1: mux::CapabilityId = 5; - -/// Current version. -pub(super) const PUSH_V2: mux::CapabilityId = 8; - impl super::Rpc for PushRpc { - const CAPABILITY_ID: mux::CapabilityId = PUSH_V2; + const CAPABILITY: Capability = Capability::PushBatchVotesV2; const INFLIGHT: u32 = 1; const METHOD: &'static str = "push_batch_votes"; type Req = Msg; @@ -30,7 +21,7 @@ impl super::Rpc for PushRpc { } impl super::Rpc for PullRpc { - const CAPABILITY_ID: mux::CapabilityId = 9; + const CAPABILITY: Capability = Capability::PullBatchVotes; const INFLIGHT: u32 = 1; const METHOD: &'static str = "pull_batch_votes"; type Req = (); diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 40082c2a..0731cd91 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -1,13 +1,14 @@ //! Defines RPC for passing consensus messages. -use crate::{mux, proto::consensus as proto}; +use crate::{proto::consensus as proto}; use zksync_consensus_roles::validator; use zksync_protobuf::{read_required, ProtoFmt}; +use super::Capability; /// Consensus RPC. pub(crate) struct Rpc; impl super::Rpc for Rpc { - const CAPABILITY_ID: mux::CapabilityId = 0; + const CAPABILITY: Capability = Capability::Consensus; const INFLIGHT: u32 = 3; const METHOD: &'static str = "consensus"; type Req = Req; diff --git a/node/actors/network/src/rpc/get_batch.rs b/node/actors/network/src/rpc/get_batch.rs index 8c83e451..8f19852e 100644 --- a/node/actors/network/src/rpc/get_batch.rs +++ b/node/actors/network/src/rpc/get_batch.rs @@ -1,8 +1,9 @@ //! RPC for fetching a batch from peer. -use crate::{mux, proto::gossip as proto}; +use crate::{proto::gossip as proto}; use anyhow::Context; use zksync_consensus_roles::attester; use zksync_protobuf::{read_optional, ProtoFmt}; +use super::Capability; /// `get_batch` RPC. #[derive(Debug)] @@ -10,7 +11,7 @@ pub(crate) struct Rpc; // TODO: determine more precise `INFLIGHT` / `RATE` values as a result of load testing impl super::Rpc for Rpc { - const CAPABILITY_ID: mux::CapabilityId = 6; + const CAPABILITY: Capability = Capability::GetBatch; const INFLIGHT: u32 = 5; const METHOD: &'static str = "get_batch"; diff --git a/node/actors/network/src/rpc/get_block.rs b/node/actors/network/src/rpc/get_block.rs index 45b208d0..58486746 100644 --- a/node/actors/network/src/rpc/get_block.rs +++ b/node/actors/network/src/rpc/get_block.rs @@ -1,8 +1,9 @@ //! RPC for fetching a block from peer. -use crate::{mux, proto::gossip as proto}; +use crate::{proto::gossip as proto}; use anyhow::Context; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; use zksync_protobuf::{read_optional, ProtoFmt}; +use super::Capability; /// `get_block` RPC. #[derive(Debug)] @@ -10,7 +11,7 @@ pub(crate) struct Rpc; // TODO: determine more precise `INFLIGHT` / `RATE` values as a result of load testing impl super::Rpc for Rpc { - const CAPABILITY_ID: mux::CapabilityId = 4; + const CAPABILITY: Capability = Capability::GetBlock; const INFLIGHT: u32 = 5; const METHOD: &'static str = "get_block"; diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index ef287441..bd8a52f3 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -20,6 +20,7 @@ use crate::{frame, mux}; use anyhow::Context as _; use std::{collections::BTreeMap, sync::Arc}; use zksync_concurrency::{ctx, io, limiter, metrics::LatencyHistogramExt as _, scope}; +pub(crate) use crate::proto::rpc::Capability as Capability; pub(crate) mod consensus; pub(crate) mod get_batch; @@ -35,6 +36,12 @@ pub(crate) mod testonly; #[cfg(test)] mod tests; +impl Capability { + pub(crate) fn id(self) -> mux::CapabilityId { + self as mux::CapabilityId + } +} + /// Multiplexer configuration for the RPC services. pub(crate) const MUX_CONFIG: mux::Config = mux::Config { read_buffer_size: 160 * zksync_protobuf::kB as u64, @@ -47,16 +54,21 @@ pub(crate) const MUX_CONFIG: mux::Config = mux::Config { /// It is just a collection of associated types /// and constants. pub(crate) trait Rpc: Sync + Send + 'static { - /// CapabilityId used to identify the RPC. - /// Client and Server have to agree on CAPABILITY_ID - /// of all supported RPC, for the RPC to actually work. + /// Capability used to identify the RPC. + /// Client and Server both have to support the given + /// capability for the RPC to work. /// Each type implementing `Rpc` should use an unique - /// `CAPABILITY_ID`. - const CAPABILITY_ID: mux::CapabilityId; + /// capability. We use a protobuf enum as a + /// register of capabilitues to enforce uniqueness + /// and compatibility. + const CAPABILITY: Capability; /// Maximal number of calls executed in parallel. /// Both client and server enforce this limit. const INFLIGHT: u32; /// Name of the RPC, used in prometheus metrics. + /// TODO: we can derive it automatically from EnumValueDescriptor of the + /// CAPABILITY (or form Debug implementation of Capability, but Debug formats + /// are not stable). const METHOD: &'static str; /// Type of the request message. type Req: zksync_protobuf::ProtoFmt + Send + Sync; @@ -254,12 +266,12 @@ impl<'a> Service<'a> { if self .mux .accept - .insert(R::CAPABILITY_ID, client.queue.clone()) + .insert(R::CAPABILITY.id(), client.queue.clone()) .is_some() { panic!( - "client for capability {} already registered", - R::CAPABILITY_ID + "client for capability {:?} already registered", + R::CAPABILITY ); } self @@ -276,12 +288,12 @@ impl<'a> Service<'a> { if self .mux .connect - .insert(R::CAPABILITY_ID, queue.clone()) + .insert(R::CAPABILITY.id(), queue.clone()) .is_some() { panic!( - "server for capability {} already registered", - R::CAPABILITY_ID + "server for capability {:?} already registered", + R::CAPABILITY ); } self.servers.push(Box::new(Server { diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index a11e07ed..8ccbac1a 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -1,15 +1,16 @@ //! Defines an RPC for sending ping messages. -use crate::{mux, proto::ping as proto}; +use crate::{proto::ping as proto}; use anyhow::Context as _; use rand::Rng; use zksync_concurrency::{ctx, limiter, time}; use zksync_protobuf::{kB, required, ProtoFmt}; +use super::Capability; /// Ping RPC. pub(crate) struct Rpc; impl super::Rpc for Rpc { - const CAPABILITY_ID: mux::CapabilityId = 2; + const CAPABILITY : Capability = Capability::Ping; const INFLIGHT: u32 = 1; const METHOD: &'static str = "ping"; type Req = Req; diff --git a/node/actors/network/src/rpc/push_batch_store_state.rs b/node/actors/network/src/rpc/push_batch_store_state.rs index aa098a5a..af178453 100644 --- a/node/actors/network/src/rpc/push_batch_store_state.rs +++ b/node/actors/network/src/rpc/push_batch_store_state.rs @@ -1,16 +1,17 @@ //! RPC for fetching a batch from peer. -use crate::{mux, proto::gossip as proto}; +use crate::{proto::gossip as proto}; use anyhow::Context as _; use zksync_consensus_roles::attester; use zksync_consensus_storage::BatchStoreState; use zksync_protobuf::{required, ProtoFmt}; +use super::Capability; /// PushBatchStoreState RPC. #[derive(Debug)] pub(crate) struct Rpc; impl super::Rpc for Rpc { - const CAPABILITY_ID: mux::CapabilityId = 7; + const CAPABILITY: Capability = Capability::PushBatchStoreState; const INFLIGHT: u32 = 1; const METHOD: &'static str = "push_batch_store_state"; diff --git a/node/actors/network/src/rpc/push_block_store_state.rs b/node/actors/network/src/rpc/push_block_store_state.rs index 52c1161c..ce49ba34 100644 --- a/node/actors/network/src/rpc/push_block_store_state.rs +++ b/node/actors/network/src/rpc/push_block_store_state.rs @@ -1,16 +1,17 @@ //! RPC for notifying peer about our BlockStore state. -use crate::{mux, proto::gossip as proto}; +use crate::{proto::gossip as proto}; use anyhow::Context; use zksync_consensus_roles::validator; use zksync_consensus_storage::BlockStoreState; use zksync_protobuf::{read_optional, required, ProtoFmt}; +use super::Capability; /// PushBlockStoreState RPC. #[derive(Debug)] pub(crate) struct Rpc; impl super::Rpc for Rpc { - const CAPABILITY_ID: mux::CapabilityId = 3; + const CAPABILITY: Capability = Capability::PushBlockStoreState; const INFLIGHT: u32 = 1; const METHOD: &'static str = "push_block_store_state"; diff --git a/node/actors/network/src/rpc/push_validator_addrs.rs b/node/actors/network/src/rpc/push_validator_addrs.rs index c58803bf..7bf3ff82 100644 --- a/node/actors/network/src/rpc/push_validator_addrs.rs +++ b/node/actors/network/src/rpc/push_validator_addrs.rs @@ -1,15 +1,16 @@ //! RPC for synchronizing ValidatorAddrs data. -use crate::{mux, proto::gossip as proto}; +use crate::{proto::gossip as proto}; use anyhow::Context as _; use std::sync::Arc; use zksync_consensus_roles::validator; use zksync_protobuf::ProtoFmt; +use super::Capability; /// PushValidatorAddrs RPC. pub(crate) struct Rpc; impl super::Rpc for Rpc { - const CAPABILITY_ID: mux::CapabilityId = 1; + const CAPABILITY: Capability = Capability::PushValidatorAddrs; const INFLIGHT: u32 = 1; const METHOD: &'static str = "push_validator_addrs"; diff --git a/node/actors/network/src/rpc/tests.rs b/node/actors/network/src/rpc/tests.rs index fa0d98fb..db4ded88 100644 --- a/node/actors/network/src/rpc/tests.rs +++ b/node/actors/network/src/rpc/tests.rs @@ -2,30 +2,11 @@ use super::*; use crate::noise; use rand::Rng as _; use std::{ - collections::HashSet, sync::atomic::{AtomicU64, Ordering}, }; use zksync_concurrency::{ctx, testonly::abort_on_panic, time}; use zksync_protobuf::{kB, testonly::test_encode_random}; -/// CAPABILITY_ID should uniquely identify the RPC. -#[test] -fn test_capability_rpc_correspondence() { - let ids = [ - consensus::Rpc::CAPABILITY_ID, - push_validator_addrs::Rpc::CAPABILITY_ID, - push_block_store_state::Rpc::CAPABILITY_ID, - get_block::Rpc::CAPABILITY_ID, - ping::Rpc::CAPABILITY_ID, - batch_votes::PUSH_V1, - batch_votes::PUSH_V2, - batch_votes::PullRpc::CAPABILITY_ID, - push_batch_store_state::Rpc::CAPABILITY_ID, - get_batch::Rpc::CAPABILITY_ID, - ]; - assert_eq!(ids.len(), HashSet::from(ids).len()); -} - #[test] fn test_schema_encode_decode() { let rng = &mut ctx::test_root(&ctx::RealClock).rng(); @@ -162,7 +143,7 @@ const RATE: limiter::Rate = limiter::Rate { }; impl Rpc for ExampleRpc { - const CAPABILITY_ID: mux::CapabilityId = 0; + const CAPABILITY: Capability = Capability::Ping; const INFLIGHT: u32 = 5; const METHOD: &'static str = "example"; type Req = (); From 7ff09d02f7d4d4d783554b2a800a8914a628d5f8 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 6 Aug 2024 16:19:29 +0200 Subject: [PATCH 10/31] deprecated old push_batch_votes --- node/actors/network/src/proto/rpc.proto | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/node/actors/network/src/proto/rpc.proto b/node/actors/network/src/proto/rpc.proto index 04f438fa..925f4a13 100644 --- a/node/actors/network/src/proto/rpc.proto +++ b/node/actors/network/src/proto/rpc.proto @@ -3,16 +3,13 @@ syntax = "proto3"; package zksync.network.rpc; enum Capability { + reserved "PUSH_BATCH_VOTES"; + reserved 5; CONSENSUS = 0; PING = 2; PUSH_VALIDATOR_ADDRS = 1; PUSH_BLOCK_STORE_STATE = 3; GET_BLOCK = 4; - /// Deprecated, because adding `genesis_hash` to `attester::Batch` - /// was not backward compatible - old binaries couldn't verify - /// signatures on messages with `genesis_hash` and were treating it - /// as malicious behavior. - PUSH_BATCH_VOTES_V1 = 5; PUSH_BATCH_VOTES_V2 = 8; PULL_BATCH_VOTES = 9; PUSH_BATCH_STORE_STATE = 7; From 9876d97f55359dfdbca663f4b863fc8a428fbf93 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 09:16:40 +0200 Subject: [PATCH 11/31] more passing tests --- node/libs/roles/src/attester/messages/batch.rs | 2 +- node/libs/roles/src/attester/tests.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/node/libs/roles/src/attester/messages/batch.rs b/node/libs/roles/src/attester/messages/batch.rs index ad6e8f55..8403bc8c 100644 --- a/node/libs/roles/src/attester/messages/batch.rs +++ b/node/libs/roles/src/attester/messages/batch.rs @@ -161,7 +161,7 @@ impl BatchQC { /// Signature is assumed to be already verified. pub fn add(&mut self, msg: &Signed, committee: &attester::Committee) -> anyhow::Result<()> { anyhow::ensure!(self.message == msg.msg, "inconsistent messages"); - anyhow::ensure!(self.signatures.contains(&msg.key), "signature already present"); + anyhow::ensure!(!self.signatures.contains(&msg.key), "signature already present"); anyhow::ensure!(committee.contains(&msg.key), "not in committee"); self.signatures.add(msg.key.clone(), msg.sig.clone()); Ok(()) diff --git a/node/libs/roles/src/attester/tests.rs b/node/libs/roles/src/attester/tests.rs index 17eacb6a..370c3365 100644 --- a/node/libs/roles/src/attester/tests.rs +++ b/node/libs/roles/src/attester/tests.rs @@ -165,7 +165,11 @@ fn test_batch_qc() { // Create QCs with increasing number of attesters. for i in 0..setup1.attester_keys.len() + 1 { - let mut qc = BatchQC::new(rng.gen()); + let mut qc = BatchQC::new(Batch{ + genesis: setup1.genesis.hash(), + number: rng.gen(), + hash: rng.gen(), + }); for key in &setup1.attester_keys[0..i] { qc.add(&key.sign_msg(qc.message.clone()), &attesters) .unwrap(); @@ -204,14 +208,10 @@ fn test_attester_committee_weights() { let msg: Batch = rng.gen(); let mut qc = BatchQC::new(msg.clone()); - for (n, weight) in sums.iter().enumerate() { - let key = &setup.attester_keys[n]; + for (i, weight) in sums.iter().enumerate() { + let key = &setup.attester_keys[i]; qc.add(&key.sign_msg(msg.clone()), &attesters).unwrap(); - assert_eq!( - - attesters.weight_of_keys(qc.signatures.keys()), - *weight - ); + assert_eq!(attesters.weight_of_keys(qc.signatures.keys()), *weight); } } From f9cc6494b55f7243936cbd7672e7d7e1607ce5d1 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 09:49:43 +0200 Subject: [PATCH 12/31] wip --- .../network/src/gossip/attestation/mod.rs | 29 +-- .../network/src/gossip/attestation/tests.rs | 169 ++++++++++-------- 2 files changed, 105 insertions(+), 93 deletions(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 10e1d84b..6d6fe67c 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -75,6 +75,20 @@ impl State { } Ok(()) } + + fn qc(&self) -> Option { { + if self.weight < self.config.committee.threshold() { + return None; + } + let mut sigs = attester::MultiSig::default(); + for vote in self.votes.values() { + sigs.add(vote.key.clone(),vote.sig.clone()); + } + Some(attester::BatchQC { + message: self.config.batch_to_attest.clone(), + signatures: sigs, + }) + } } pub(crate) struct StateReceiver { @@ -138,20 +152,7 @@ impl StateWatch { /// Waits for the certificate to be collected. pub async fn wait_for_qc(&self, ctx: &ctx::Ctx) -> ctx::OrCanceled { - sync::wait_for_some(ctx, &mut self.state.subscribe(), |state| { - let state = state.as_ref()?; - if state.weight < state.config.committee.threshold() { - return None; - } - let mut sigs = attester::MultiSig::default(); - for vote in state.votes.values() { - sigs.add(vote.key.clone(),vote.sig.clone()); - } - Some(attester::BatchQC { - message: state.config.batch_to_attest.clone(), - signatures: sigs, - }) - }).await + sync::wait_for_some(ctx, &mut self.state.subscribe(), |s| s.as_ref()?.qc()).await } /// Updates the attestation config. diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index 392d7aa4..595eed69 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -25,9 +25,9 @@ impl StateReceiver { } } -impl> From for Votes { - fn from(it: I) -> Self { - Self(it.map(|v|(v.key.clone(),v)).collect()) +impl> From for Votes { + fn from(x: T) -> Self { + Self(x.into_iter().map(|v|(v.key.clone(),v)).collect()) } } @@ -38,89 +38,100 @@ async fn test_insert_votes() { let rng = &mut ctx.rng(); let state = StateWatch::new(None); - let keys: Vec = (0..8).map(|_| rng.gen()).collect(); - let config = Config { - batch_to_attest: rng.gen(), - committee: attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { - key: k.public(), - weight: 1250, - })).unwrap(), - }; - state.update_config(config.clone()).await.unwrap(); - let mut recv = state.subscribe(); - - let all_votes : Vec = keys.iter().map(|k| k.sign_msg(config.batch_to_attest.clone().into()).into()).collect(); - - // Initial votes. - state.insert_votes(all_votes[0..3].iter().cloned()).await.unwrap(); - assert_eq!( - Votes::from(all_votes[0..3].iter().cloned()), - recv.wait_for_new_votes(ctx).await.unwrap().into_iter().into() - ); - - // Adding votes gradually. - state.insert_votes(all_votes[3..5].iter().cloned()).await.unwrap(); - state.insert_votes(all_votes[5..7].iter().cloned()).await.unwrap(); - assert_eq!( - Votes::from(all_votes[3..7].iter().cloned()), - recv.wait_for_new_votes(ctx).await.unwrap().into_iter().into() - ); - - // Adding duplicate votes (noop). - state.insert_votes(all_votes[2..6].iter().cloned()).await.unwrap(); - assert!(!recv.recv.has_changed().unwrap()); - - // Adding votes out of committee (noop). - state.insert_votes((0..3).map(|_| { - let k : attester::SecretKey = rng.gen(); - k.sign_msg(config.batch_to_attest.clone()).into() - })).await.unwrap(); - assert!(!recv.recv.has_changed().unwrap()); - - // Adding votes for different batch (noop). - state.insert_votes((0..3).map(|_| { - let k : attester::SecretKey = rng.gen(); - k.sign_msg(attester::Batch { - genesis: config.batch_to_attest.genesis, - number: rng.gen(), - hash: rng.gen(), - }).into() - })).await.unwrap(); - assert!(!recv.recv.has_changed().unwrap()); - - // Adding incorrect votes (error). - let mut bad_vote = (*all_votes[7]).clone(); - bad_vote.sig = rng.gen(); - assert!(state.insert_votes([bad_vote.into()].into_iter()).await.is_err()); - assert!(!recv.recv.has_changed().unwrap()); + let genesis : attester::GenesisHash = rng.gen(); + let first : attester::BatchNumber = rng.gen(); + for i in 0..3 { + let keys: Vec = (0..8).map(|_| rng.gen()).collect(); + let config = Config { + batch_to_attest: attester::Batch { + genesis, + number: first + i, + hash: rng.gen(), + }, + committee: attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { + key: k.public(), + weight: 1250, + })).unwrap(), + }; + state.update_config(config.clone()).await.unwrap(); + assert_eq!(Votes::from([]), state.votes().into()); + let mut recv = state.subscribe(); + + let all_votes : Vec = keys.iter().map(|k| k.sign_msg(config.batch_to_attest.clone().into()).into()).collect(); + + // Initial votes. + state.insert_votes(all_votes[0..3].iter().cloned()).await.unwrap(); + assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), state.votes().into()); + assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), recv.wait_for_new_votes(ctx).await.unwrap().into()); + + // Adding votes gradually. + state.insert_votes(all_votes[3..5].iter().cloned()).await.unwrap(); + state.insert_votes(all_votes[5..7].iter().cloned()).await.unwrap(); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + assert_eq!(Votes::from(all_votes[3..7].iter().cloned()), recv.wait_for_new_votes(ctx).await.unwrap().into()); + + // Readding already inserded votes (noop). + state.insert_votes(all_votes[2..6].iter().cloned()).await.unwrap(); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + + // Adding votes out of committee (noop). + state.insert_votes((0..3).map(|_| { + let k : attester::SecretKey = rng.gen(); + k.sign_msg(config.batch_to_attest.clone()).into() + })).await.unwrap(); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + + // Adding votes for different batch (noop). + state.insert_votes((0..3).map(|_| { + let k : attester::SecretKey = rng.gen(); + k.sign_msg(attester::Batch { + genesis: config.batch_to_attest.genesis, + number: rng.gen(), + hash: rng.gen(), + }).into() + })).await.unwrap(); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + + // Adding incorrect votes (error). + let mut bad_vote = (*all_votes[7]).clone(); + bad_vote.sig = rng.gen(); + assert!(state.insert_votes([bad_vote.into()].into_iter()).await.is_err()); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + + // Add the last vote mixed with already added votes. + state.insert_votes(all_votes[5..].iter().cloned()).await.unwrap(); + assert_eq!(Votes::from(all_votes.clone()), state.votes().into()); + assert_eq!(Votes::from(all_votes[7..].iter().cloned()), recv.wait_for_new_votes(ctx).await.unwrap().into()); + } } -/* #[tokio::test] -fn test_batch_votes_quorum() { +fn test_wait_for_qc() { abort_on_panic(); - let rng = &mut ctx::test_root(&ctx::RealClock).rng(); + let ctx = &ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); - for _ in 0..10 { + let state = StateWatch::new(None); + let genesis : attester::GenesisHash = rng.gen(); + let first : attester::BatchNumber = rng.gen(); + + for i in 0..10 { let committee_size = rng.gen_range(1..20); let keys: Vec = (0..committee_size).map(|_| rng.gen()).collect(); - let attesters = attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { - key: k.public(), - weight: rng.gen_range(1..=100), - })) - .unwrap(); - - let batch0 = rng.gen::(); - let batch1 = attester::Batch { - number: batch0.number.next(), - hash: rng.gen(), - genesis: batch0.genesis, + let config = Config { + batch_to_attest: attester::Batch { + genesis, + number: first + i, + hash: rng.gen(), + }, + committee: attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { + key: k.public(), + weight: rng.gen_range(1..=100), + })).unwrap(), }; - let genesis = batch0.genesis; - let mut batches = [(batch0, 0u64), (batch1, 0u64)]; - - let mut votes = BatchVotes::default(); - for sk in &keys { + state.update_config(config.clone()).await.unwrap(); + // TODO: add votes gradually, extract qc each time and verify the qc. + for k in &keys { // We need 4/5+1 for quorum, so let's say ~80% vote on the second batch. let b = usize::from(rng.gen_range(0..100) < 80); let batch = &batches[b].0; @@ -156,4 +167,4 @@ fn test_batch_votes_quorum() { assert!(votes.votes.is_empty()); assert!(votes.support.is_empty()); } -}*/ +} From 226d6f3822adc53a221cdb54cf8d40f8bd0c9359 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 11:40:49 +0200 Subject: [PATCH 13/31] test works --- .../network/src/gossip/attestation/mod.rs | 2 +- .../network/src/gossip/attestation/tests.rs | 51 ++++++------------- 2 files changed, 16 insertions(+), 37 deletions(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 6d6fe67c..b7847b48 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -76,7 +76,7 @@ impl State { Ok(()) } - fn qc(&self) -> Option { { + fn qc(&self) -> Option { if self.weight < self.config.committee.threshold() { return None; } diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index 595eed69..994901da 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] use super::*; use zksync_concurrency::testonly::abort_on_panic; -use rand::Rng as _; +use rand::{seq::SliceRandom as _, Rng as _}; type Vote = Arc>; @@ -106,7 +106,7 @@ async fn test_insert_votes() { } #[tokio::test] -fn test_wait_for_qc() { +async fn test_wait_for_qc() { abort_on_panic(); let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); @@ -116,6 +116,7 @@ fn test_wait_for_qc() { let first : attester::BatchNumber = rng.gen(); for i in 0..10 { + tracing::info!("iteration {i}"); let committee_size = rng.gen_range(1..20); let keys: Vec = (0..committee_size).map(|_| rng.gen()).collect(); let config = Config { @@ -129,42 +130,20 @@ fn test_wait_for_qc() { weight: rng.gen_range(1..=100), })).unwrap(), }; + let mut all_votes : Vec = keys.iter().map(|k| k.sign_msg(config.batch_to_attest.clone().into()).into()).collect(); + all_votes.shuffle(rng); state.update_config(config.clone()).await.unwrap(); - // TODO: add votes gradually, extract qc each time and verify the qc. - for k in &keys { - // We need 4/5+1 for quorum, so let's say ~80% vote on the second batch. - let b = usize::from(rng.gen_range(0..100) < 80); - let batch = &batches[b].0; - let vote = sk.sign_msg(batch.clone()); - votes - .update(&attesters, &genesis, &[Arc::new(vote)]) - .unwrap(); - batches[b].1 += attesters.weight(&sk.public()).unwrap(); - - // Check that as soon as we have quorum it's found. - if batches[b].1 >= attesters.threshold() { - let qc = votes - .find_quorum(&attesters, &genesis) - .expect("should find quorum"); - assert!(qc.message == *batch); - assert!(qc.signatures.keys().count() > 0); + loop { + let end = rng.gen_range(0..=committee_size); + tracing::info!("end = {end}"); + state.insert_votes(all_votes[..end].iter().cloned()).await.unwrap(); + if config.committee.weight_of_keys(all_votes[..end].iter().map(|v|&v.key)) >= config.committee.threshold() { + let qc = state.wait_for_qc(ctx).await.unwrap(); + assert_eq!(qc.message, config.batch_to_attest); + qc.verify(genesis,&config.committee).unwrap(); + break; } + assert_eq!(None, state.state.subscribe().borrow().as_ref().unwrap().qc()); } - - // Check that if there was no quoroum then we don't find any. - if !batches.iter().any(|b| b.1 >= attesters.threshold()) { - assert!(votes.find_quorum(&attesters, &genesis).is_none()); - } - - // Check that the minimum batch number prunes data. - let last_batch = batches[1].0.number; - - votes.set_min_batch_number(last_batch); - assert!(votes.votes.values().all(|v| v.msg.number >= last_batch)); - assert!(votes.support.keys().all(|n| *n >= last_batch)); - - votes.set_min_batch_number(last_batch.next()); - assert!(votes.votes.is_empty()); - assert!(votes.support.is_empty()); } } From a384bb049f9ccaca43a524c1450937617c01ec35 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 14:55:08 +0200 Subject: [PATCH 14/31] before single RPC --- .../network/src/gossip/attestation/mod.rs | 16 +++- node/actors/network/src/gossip/tests/mod.rs | 94 ++++++++----------- 2 files changed, 53 insertions(+), 57 deletions(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index b7847b48..df6686d7 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -76,7 +76,7 @@ impl State { Ok(()) } - fn qc(&self) -> Option { + fn qc(&self) -> Option { { if self.weight < self.config.committee.threshold() { return None; } @@ -150,9 +150,17 @@ impl StateWatch { self.state.subscribe().borrow().iter().map(|s|s.votes.values().cloned()).flatten().collect() } - /// Waits for the certificate to be collected. - pub async fn wait_for_qc(&self, ctx: &ctx::Ctx) -> ctx::OrCanceled { - sync::wait_for_some(ctx, &mut self.state.subscribe(), |s| s.as_ref()?.qc()).await + /// Waits for the certificate for a batch with the given number to be collected. + /// Returns None iff attestation already skipped to collecting certificate some later batch. + pub async fn wait_for_qc(&self, ctx: &ctx::Ctx, n: attester::BatchNumber) -> ctx::OrCanceled> { + let recv = &mut self.state.subscribe(); + recv.mark_changed(); + loop { + let Some(state) = sync::changed(ctx, recv).await? else { continue }; + if state.config.batch_to_attest.number < n { continue }; + if state.config.batch_to_attest.number > n { return Ok(None); } + if let Some(qc) = state.qc() { return Ok(Some(qc)); } + } } /// Updates the attestation config. diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index e0bb2499..e402b9a7 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -1,6 +1,7 @@ use super::ValidatorAddrs; use crate::{ gossip::{ + attestation, handshake, validator_addrs::ValidatorAddrsWatch, }, @@ -22,7 +23,7 @@ use zksync_concurrency::{ testonly::{abort_on_panic, set_timeout}, time, }; -use zksync_consensus_roles::{validator}; +use zksync_consensus_roles::{attester, validator}; use zksync_consensus_storage::{testonly::TestMemoryStorage}; mod fetch_batches; @@ -487,7 +488,6 @@ async fn rate_limiting() { } } -/* #[tokio::test(flavor = "multi_thread")] async fn test_batch_votes_propagation() { let _guard = set_timeout(time::Duration::seconds(10)); @@ -496,65 +496,53 @@ async fn test_batch_votes_propagation() { let rng = &mut ctx.rng(); let mut setup = validator::testonly::Setup::new(rng, 10); - let cfgs = testonly::new_configs(rng, &setup, 1); + // Fixed attestation schedule. + let first attester::BatchNumber = rng.gen(); + let schedule = (0..10).map(|i| attestation::Config { + batch_to_attest : attester::Batch { + genesis: setup.genesis.hash(), + number: first + i, + hash: rng.gen(), + }, + committee: attester::Committee::new(setup.attester_keys.iter().filter(|_|rng.gen()).map(|k| + attester::WeightedAttester { + key: k.public(), + weight: rng.gen_range(5..10), + }, + )).unwrap(), + }).collect(); + + // Round of the schedule that nodes should collect the votes for. + let round = sync::watch::channel(0).0; + scope::run!(ctx, |ctx, s| async { - // All nodes share the same in-memory store + // All nodes share the same store - store is not used in this test anyway. let store = TestMemoryStorage::new(ctx, &setup.genesis).await; s.spawn_bg(store.runner.run(ctx)); - // Push the first batch into the store so we have something to vote on. - setup.push_blocks(rng, 2); - setup.push_batch(rng); - - let batch = setup.batches[0].clone(); - - store - .batches - .queue_batch(ctx, batch.clone(), setup.genesis.clone()) - .await - .unwrap(); - - let batch = attester::Batch { - number: batch.number, - hash: rng.gen(), - genesis: setup.genesis.hash(), - }; - // Start all nodes. - let nodes: Vec = cfgs - .iter() - .enumerate() - .map(|(i, cfg)| { - let (node, runner) = testonly::Instance::new( - cfg.clone(), - store.blocks.clone(), - store.batches.clone(), - ); - s.spawn_bg(runner.run(ctx).instrument(tracing::info_span!("node", i))); - node - }) - .collect(); - - // Cast votes on each node. It doesn't matter who signs where in this test, - // we just happen to know that we'll have as many nodes as attesters. - let attesters = setup.genesis.attesters.as_ref().unwrap(); - for (node, key) in nodes.iter().zip(setup.attester_keys.iter()) { - node.net - .batch_vote_publisher() - .publish(attesters, &setup.genesis.hash(), key, batch.clone()) - .await - .unwrap(); - } + let mut nodes = vec![]; + for (i,cfg) in cfgs.iter().enumerate() { + let (node, runner) = testonly::Instance::new( + cfg.clone(), + store.blocks.clone(), + store.batches.clone(), + ); + s.spawn_bg(runner.run(ctx).instrument(tracing::info_span!("node", i))); + // Task updating config whenever we advance to the next view. + s.spawn_bg(async { + let mut round = round.subscribe(); + round.mark_changed(); + loop { + let round = *sync::changed(ctx, &mut round).await; + let Some(cfg) = schedule.get(round) else { return Ok(()) }; + node.net.gossip.attestation_state.update_config(cfg.clone()).unwrap(); + } + }); - // Wait until one of the nodes collects a quorum over gossip and stores. - loop { - if let Some(qc) = store.im_batches.get_batch_qc(ctx, batch.number).await? { - assert_eq!(qc.message, batch); - return Ok(()); - } - ctx.sleep(time::Duration::milliseconds(100)).await?; + nodes.push(node); } }) .await From e7c95986a8b83a1466c4b5127878fa3d1f906f4c Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 15:31:42 +0200 Subject: [PATCH 15/31] introduced diff --- .../network/src/gossip/attestation/mod.rs | 56 +++++++++++++------ .../network/src/gossip/attestation/tests.rs | 27 ++------- node/actors/network/src/gossip/runner.rs | 4 +- node/actors/network/src/gossip/tests/mod.rs | 2 + 4 files changed, 48 insertions(+), 41 deletions(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index df6686d7..74ac2e19 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -32,17 +32,36 @@ pub(crate) struct State { weight: attester::Weight, } +pub(crate) struct Diff { + pub(crate) votes: Vec>>, + pub(crate) config_changed: bool, +} + +impl Diff { + fn is_empty(&self) -> bool { + self.votes.is_empty() && !self.config_changed + } +} + impl State { /// Returns a set of votes of `self` which are newer than the entries in `old`. - fn new_votes(&self, old: &Option) -> Vec>> { - let Some(old) = old.as_ref() else { return self.votes.values().cloned().collect() }; + fn diff(&self, old: &Option) -> Diff { + let Some(old) = old.as_ref() else { + return Diff { + config_changed: true, + votes: self.votes.values().cloned().collect() + }; + }; match self.config.batch_to_attest.number.cmp(&old.config.batch_to_attest.number) { - Ordering::Less => vec![], - Ordering::Greater => self.votes.values().cloned().collect(), - Ordering::Equal => self.votes.iter() - .filter(|(k,_)|!old.votes.contains_key(k)) - .map(|(_,v)|v.clone()) - .collect() + Ordering::Less => Diff { config_changed: true, votes: vec![] }, + Ordering::Greater => Diff { config_changed: true, votes: self.votes.values().cloned().collect() }, + Ordering::Equal => Diff { + config_changed: false, + votes: self.votes.iter() + .filter(|(k,_)|!old.votes.contains_key(k)) + .map(|(_,v)|v.clone()) + .collect(), + }, } } @@ -76,7 +95,7 @@ impl State { Ok(()) } - fn qc(&self) -> Option { { + fn qc(&self) -> Option { if self.weight < self.config.committee.threshold() { return None; } @@ -91,19 +110,19 @@ impl State { } } -pub(crate) struct StateReceiver { +pub(crate) struct DiffReceiver { prev: Option, recv: sync::watch::Receiver>, } -impl StateReceiver { - pub(crate) async fn wait_for_new_votes(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled>>> { +impl DiffReceiver { + pub(crate) async fn wait_for_diff(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled { loop { let Some(new) = (*sync::changed(ctx, &mut self.recv).await?).clone() else { continue }; - let new_votes = new.new_votes(&self.prev); + let diff = new.diff(&self.prev); self.prev = Some(new); - if !new_votes.is_empty() { - return Ok(new_votes) + if !diff.is_empty() { + return Ok(diff) } } } @@ -128,10 +147,10 @@ impl StateWatch { Self { key, state: Watch::new(None) } } - pub(crate) fn subscribe(&self) -> StateReceiver { + pub(crate) fn subscribe(&self) -> DiffReceiver { let mut recv = self.state.subscribe(); recv.mark_changed(); - StateReceiver { prev: None, recv } + DiffReceiver { prev: None, recv } } pub(crate) async fn insert_votes(&self, votes: impl Iterator>>) -> anyhow::Result<()> { @@ -156,7 +175,8 @@ impl StateWatch { let recv = &mut self.state.subscribe(); recv.mark_changed(); loop { - let Some(state) = sync::changed(ctx, recv).await? else { continue }; + let state = sync::changed(ctx, recv).await?; + let Some(state) = state.as_ref() else { continue }; if state.config.batch_to_attest.number < n { continue }; if state.config.batch_to_attest.number > n { return Ok(None); } if let Some(qc) = state.qc() { return Ok(Some(qc)); } diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index 994901da..bc4127d9 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -8,23 +8,6 @@ type Vote = Arc>; #[derive(Default, Debug, PartialEq)] struct Votes(im::HashMap); -impl Votes { - fn insert(&mut self, vote: Vote) { - self.0.insert(vote.key.clone(), vote); - } - - fn get(&mut self, key: &attester::SecretKey) -> Vote { - self.0.get(&key.public()).unwrap().clone() - } -} - -impl StateReceiver { - fn snapshot(&self) -> Votes { - let Some(state) = self.recv.borrow().clone() else { return Votes::default() }; - state.votes.values().cloned().into() - } -} - impl> From for Votes { fn from(x: T) -> Self { Self(x.into_iter().map(|v|(v.key.clone(),v)).collect()) @@ -62,13 +45,13 @@ async fn test_insert_votes() { // Initial votes. state.insert_votes(all_votes[0..3].iter().cloned()).await.unwrap(); assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), state.votes().into()); - assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), recv.wait_for_new_votes(ctx).await.unwrap().into()); + assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), recv.wait_for_diff(ctx).await.unwrap().votes.into()); // Adding votes gradually. state.insert_votes(all_votes[3..5].iter().cloned()).await.unwrap(); state.insert_votes(all_votes[5..7].iter().cloned()).await.unwrap(); assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); - assert_eq!(Votes::from(all_votes[3..7].iter().cloned()), recv.wait_for_new_votes(ctx).await.unwrap().into()); + assert_eq!(Votes::from(all_votes[3..7].iter().cloned()), recv.wait_for_diff(ctx).await.unwrap().votes.into()); // Readding already inserded votes (noop). state.insert_votes(all_votes[2..6].iter().cloned()).await.unwrap(); @@ -101,7 +84,7 @@ async fn test_insert_votes() { // Add the last vote mixed with already added votes. state.insert_votes(all_votes[5..].iter().cloned()).await.unwrap(); assert_eq!(Votes::from(all_votes.clone()), state.votes().into()); - assert_eq!(Votes::from(all_votes[7..].iter().cloned()), recv.wait_for_new_votes(ctx).await.unwrap().into()); + assert_eq!(Votes::from(all_votes[7..].iter().cloned()), recv.wait_for_diff(ctx).await.unwrap().votes.into()); } } @@ -137,8 +120,10 @@ async fn test_wait_for_qc() { let end = rng.gen_range(0..=committee_size); tracing::info!("end = {end}"); state.insert_votes(all_votes[..end].iter().cloned()).await.unwrap(); + // Waiting for the previous qc should immediately return None. + assert_eq!(None,state.wait_for_qc(ctx, config.batch_to_attest.number.prev().unwrap()).await.unwrap()); if config.committee.weight_of_keys(all_votes[..end].iter().map(|v|&v.key)) >= config.committee.threshold() { - let qc = state.wait_for_qc(ctx).await.unwrap(); + let qc = state.wait_for_qc(ctx, config.batch_to_attest.number).await.unwrap().unwrap(); assert_eq!(qc.message, config.batch_to_attest); qc.verify(genesis,&config.committee).unwrap(); break; diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index 9773cbc9..885bd2bd 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -199,8 +199,8 @@ impl Network { // Subscribe to what we know about the state of the whole network. let mut recv = self.attestation_state.subscribe(); loop { - let new = recv.wait_for_new_votes(ctx).await?; - let req = rpc::batch_votes::Msg(new); + let new = recv.wait_for_diff(ctx).await?; + let req = rpc::batch_votes::Msg(new.votes); push_batch_votes_client.call(ctx, &req, kB).await?; } }); diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index e402b9a7..c808bd95 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -1,3 +1,4 @@ +#![allow(dead_code,unused_imports)] use super::ValidatorAddrs; use crate::{ gossip::{ @@ -488,6 +489,7 @@ async fn rate_limiting() { } } +/* #[tokio::test(flavor = "multi_thread")] async fn test_batch_votes_propagation() { let _guard = set_timeout(time::Duration::seconds(10)); From 1f872974926332369b8c543d2edfb4556ace9fb5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 15:59:38 +0200 Subject: [PATCH 16/31] reduced rpc --- node/actors/network/src/config.rs | 6 -- node/actors/network/src/gossip/runner.rs | 48 ++++++----- node/actors/network/src/proto/gossip.proto | 8 ++ node/actors/network/src/proto/rpc.proto | 4 +- node/actors/network/src/rpc/batch_votes.rs | 53 ------------ node/actors/network/src/rpc/mod.rs | 2 +- .../network/src/rpc/push_batch_votes.rs | 81 +++++++++++++++++++ 7 files changed, 119 insertions(+), 83 deletions(-) delete mode 100644 node/actors/network/src/rpc/batch_votes.rs create mode 100644 node/actors/network/src/rpc/push_batch_votes.rs diff --git a/node/actors/network/src/config.rs b/node/actors/network/src/config.rs index 547b9dcf..03e5020b 100644 --- a/node/actors/network/src/config.rs +++ b/node/actors/network/src/config.rs @@ -28,8 +28,6 @@ pub struct RpcConfig { pub consensus_rate: limiter::Rate, /// Max rate of sending/receiving PushBatchVotes RPCs. pub push_batch_votes_rate: limiter::Rate, - /// Max rate of sending/reciving PullBatchVotes RPCs. - pub pull_batch_votes_rate: limiter::Rate, } impl Default for RpcConfig { @@ -65,10 +63,6 @@ impl Default for RpcConfig { burst: 2, refresh: time::Duration::milliseconds(500), }, - pull_batch_votes_rate: limiter::Rate { - burst: 2, - refresh: time::Duration::milliseconds(500), - }, } } } diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index 885bd2bd..e0a6576c 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -1,4 +1,4 @@ -use super::{handshake, attestation, Network, ValidatorAddrs}; +use super::{handshake, Network, ValidatorAddrs}; use crate::{noise, preface, rpc}; use anyhow::Context as _; use async_trait::async_trait; @@ -56,14 +56,21 @@ impl rpc::Handler for &PushServer<'_> { } #[async_trait::async_trait] -impl rpc::Handler for &PushServer<'_> { - /// Here we bound the buffering of incoming batch messages. +impl rpc::Handler for &PushServer<'_> { fn max_req_size(&self) -> usize { 100 * kB } - async fn handle(&self, _ctx: &ctx::Ctx, req: rpc::batch_votes::Msg) -> anyhow::Result<()> { - self.net.attestation_state.insert_votes(req.0.into_iter()).await + async fn handle(&self, _ctx: &ctx::Ctx, req: rpc::push_batch_votes::Req) -> anyhow::Result { + let want_snapshot = req.want_snapshot(); + self.net.attestation_state.insert_votes(req.votes.into_iter()).await?; + Ok(rpc::push_batch_votes::Resp { + votes: if want_snapshot { + self.net.attestation_state.votes() + } else { + vec![] + }, + }) } } @@ -127,14 +134,6 @@ impl rpc::Handler for &BatchStore { } } -#[async_trait] -impl rpc::Handler for &attestation::StateWatch { - fn max_req_size(&self) -> usize { kB } - async fn handle(&self, _ctx: &ctx::Ctx, _req: ()) -> anyhow::Result { - Ok(rpc::batch_votes::Msg(self.votes())) - } -} - impl Network { /// Manages lifecycle of a single connection. async fn run_stream(&self, ctx: &ctx::Ctx, stream: noise::Stream) -> anyhow::Result<()> { @@ -155,7 +154,7 @@ impl Network { rpc::Client::::new(ctx, self.cfg.rpc.get_block_rate); let get_batch_client = rpc::Client::::new(ctx, self.cfg.rpc.get_batch_rate); - let push_batch_votes_client = rpc::Client::::new( + let push_batch_votes_client = rpc::Client::::new( ctx, self.cfg.rpc.push_batch_votes_rate, ); @@ -186,12 +185,11 @@ impl Network { .add_server(ctx, &*self.batch_store, self.cfg.rpc.get_batch_rate) .add_server(ctx, rpc::ping::Server, rpc::ping::RATE) .add_client(&push_batch_votes_client) - .add_server::( + .add_server::( ctx, &push_server, self.cfg.rpc.push_batch_votes_rate, - ) - .add_server(ctx, &*self.attestation_state, self.cfg.rpc.pull_batch_votes_rate); + ); // Push L1 batch votes updates to peer. s.spawn::<()>(async { @@ -199,9 +197,19 @@ impl Network { // Subscribe to what we know about the state of the whole network. let mut recv = self.attestation_state.subscribe(); loop { - let new = recv.wait_for_diff(ctx).await?; - let req = rpc::batch_votes::Msg(new.votes); - push_batch_votes_client.call(ctx, &req, kB).await?; + let diff = recv.wait_for_diff(ctx).await?; + let req = rpc::push_batch_votes::Req { + want_snapshot: Some(diff.config_changed), + votes: diff.votes, + }; + // NOTE: The response should be non-empty only iff we requested a snapshot. + // Therefore, if we needed we could restrict the response size to ~1kB in + // such a case. + let resp = push_batch_votes_client.call(ctx, &req, 100 * kB).await?; + if !resp.votes.is_empty() { + anyhow::ensure!(req.want_snapshot(), "expected empty response, but votes were returned"); + self.attestation_state.insert_votes(resp.votes.into_iter()).await.context("insert_votes")?; + } } }); diff --git a/node/actors/network/src/proto/gossip.proto b/node/actors/network/src/proto/gossip.proto index 33e3d16a..90ebc1b9 100644 --- a/node/actors/network/src/proto/gossip.proto +++ b/node/actors/network/src/proto/gossip.proto @@ -19,10 +19,18 @@ message PushValidatorAddrs { } message PushBatchVotes { + // Whether peer should send back snapshot of its votes. + optional bool want_snapshot = 2; // optional // Signed roles.validator.Msg.votes repeated roles.attester.Signed votes = 1; } +message PushBatchVotesResp { + // Signed roles.validator.Msg.votes + // Empty if want_snapshot in request was not set. + repeated roles.attester.Signed votes = 1; +} + // State of the local block store. // A node is expected to store a continuous range of blocks at all times // and actively fetch newest blocks. diff --git a/node/actors/network/src/proto/rpc.proto b/node/actors/network/src/proto/rpc.proto index 925f4a13..503dbe00 100644 --- a/node/actors/network/src/proto/rpc.proto +++ b/node/actors/network/src/proto/rpc.proto @@ -3,15 +3,13 @@ syntax = "proto3"; package zksync.network.rpc; enum Capability { - reserved "PUSH_BATCH_VOTES"; reserved 5; CONSENSUS = 0; PING = 2; PUSH_VALIDATOR_ADDRS = 1; PUSH_BLOCK_STORE_STATE = 3; GET_BLOCK = 4; - PUSH_BATCH_VOTES_V2 = 8; - PULL_BATCH_VOTES = 9; + PUSH_BATCH_VOTES = 8; PUSH_BATCH_STORE_STATE = 7; GET_BATCH = 6; } diff --git a/node/actors/network/src/rpc/batch_votes.rs b/node/actors/network/src/rpc/batch_votes.rs deleted file mode 100644 index 9f3a16c0..00000000 --- a/node/actors/network/src/rpc/batch_votes.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Defines RPC for passing consensus messages. -use crate::{proto::gossip as proto}; -use anyhow::Context as _; -use std::sync::Arc; -use zksync_consensus_roles::attester; -use zksync_protobuf::ProtoFmt; -use super::Capability; - -/// RPC pushing fresh batch votes. -pub(crate) struct PushRpc; - -/// RPC requesting all batch votes from peers. -pub(crate) struct PullRpc; - -impl super::Rpc for PushRpc { - const CAPABILITY: Capability = Capability::PushBatchVotesV2; - const INFLIGHT: u32 = 1; - const METHOD: &'static str = "push_batch_votes"; - type Req = Msg; - type Resp = (); -} - -impl super::Rpc for PullRpc { - const CAPABILITY: Capability = Capability::PullBatchVotes; - const INFLIGHT: u32 = 1; - const METHOD: &'static str = "pull_batch_votes"; - type Req = (); - type Resp = Msg; -} - -/// Signed batch message that the receiving peer should process. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct Msg(pub(crate) Vec>>); - -impl ProtoFmt for Msg { - type Proto = proto::PushBatchVotes; - - fn read(r: &Self::Proto) -> anyhow::Result { - let mut votes = vec![]; - for (i, e) in r.votes.iter().enumerate() { - votes.push(Arc::new( - ProtoFmt::read(e).with_context(|| format!("votes[{i}]"))?, - )); - } - Ok(Self(votes)) - } - - fn build(&self) -> Self::Proto { - Self::Proto { - votes: self.0.iter().map(|a| ProtoFmt::build(a.as_ref())).collect(), - } - } -} diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index bd8a52f3..e390d180 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -28,7 +28,7 @@ pub(crate) mod get_block; mod metrics; pub(crate) mod ping; pub(crate) mod push_batch_store_state; -pub(crate) mod batch_votes; +pub(crate) mod push_batch_votes; pub(crate) mod push_block_store_state; pub(crate) mod push_validator_addrs; #[cfg(test)] diff --git a/node/actors/network/src/rpc/push_batch_votes.rs b/node/actors/network/src/rpc/push_batch_votes.rs new file mode 100644 index 00000000..3e705c0a --- /dev/null +++ b/node/actors/network/src/rpc/push_batch_votes.rs @@ -0,0 +1,81 @@ +//! Defines RPC for passing consensus messages. +use crate::{proto::gossip as proto}; +use anyhow::Context as _; +use std::sync::Arc; +use zksync_consensus_roles::attester; +use zksync_protobuf::ProtoFmt; +use super::Capability; + +/// RPC pushing fresh batch votes. +pub(crate) struct Rpc; + +impl super::Rpc for Rpc { + const CAPABILITY: Capability = Capability::PushBatchVotes; + const INFLIGHT: u32 = 1; + const METHOD: &'static str = "push_batch_votes"; + type Req = Req; + type Resp = Resp; +} + +/// Signed batch message that the receiving peer should process. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Req { + pub(crate) want_snapshot: Option, + pub(crate) votes: Vec>>, +} + +pub(crate) struct Resp { + // Empty if `req.want_snapshot` was not set. + pub(crate) votes: Vec>>, +} + +impl Req { + pub(crate) fn want_snapshot(&self) -> bool { + self.want_snapshot.unwrap_or(false) + } +} + +impl ProtoFmt for Req { + type Proto = proto::PushBatchVotes; + + fn read(r: &Self::Proto) -> anyhow::Result { + let mut votes = vec![]; + for (i, e) in r.votes.iter().enumerate() { + votes.push(Arc::new( + ProtoFmt::read(e).with_context(|| format!("votes[{i}]"))?, + )); + } + Ok(Self { + want_snapshot: r.want_snapshot, + votes, + }) + } + + fn build(&self) -> Self::Proto { + Self::Proto { + want_snapshot: self.want_snapshot, + votes: self.votes.iter().map(|a| ProtoFmt::build(a.as_ref())).collect(), + } + } +} + + +impl ProtoFmt for Resp { + type Proto = proto::PushBatchVotesResp; + + fn read(r: &Self::Proto) -> anyhow::Result { + let mut votes = vec![]; + for (i, e) in r.votes.iter().enumerate() { + votes.push(Arc::new( + ProtoFmt::read(e).with_context(|| format!("votes[{i}]"))?, + )); + } + Ok(Self { votes }) + } + + fn build(&self) -> Self::Proto { + Self::Proto { + votes: self.votes.iter().map(|a| ProtoFmt::build(a.as_ref())).collect(), + } + } +} From ce9527de782260eb9296ab7555030f5e76cc5fab Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 17:13:41 +0200 Subject: [PATCH 17/31] tests work --- node/actors/executor/src/attestation.rs | 12 +- node/actors/executor/src/io.rs | 2 +- node/actors/executor/src/lib.rs | 9 +- node/actors/executor/src/tests.rs | 4 +- .../network/src/gossip/attestation/mod.rs | 144 ++++++++++---- .../network/src/gossip/attestation/tests.rs | 184 +++++++++++++----- node/actors/network/src/gossip/runner.rs | 35 ++-- node/actors/network/src/gossip/tests/mod.rs | 124 +++++++----- node/actors/network/src/rpc/consensus.rs | 4 +- node/actors/network/src/rpc/get_batch.rs | 6 +- node/actors/network/src/rpc/get_block.rs | 6 +- node/actors/network/src/rpc/mod.rs | 2 +- node/actors/network/src/rpc/ping.rs | 6 +- .../network/src/rpc/push_batch_store_state.rs | 4 +- .../network/src/rpc/push_batch_votes.rs | 19 +- .../network/src/rpc/push_block_store_state.rs | 4 +- .../network/src/rpc/push_validator_addrs.rs | 4 +- node/actors/network/src/rpc/tests.rs | 4 +- node/actors/network/src/testonly.rs | 36 +++- .../libs/roles/src/attester/messages/batch.rs | 26 ++- node/libs/roles/src/attester/tests.rs | 28 +-- node/libs/storage/src/testonly/in_memory.rs | 2 +- node/tools/src/config.rs | 7 +- 23 files changed, 450 insertions(+), 222 deletions(-) diff --git a/node/actors/executor/src/attestation.rs b/node/actors/executor/src/attestation.rs index 492785e3..c1e984b8 100644 --- a/node/actors/executor/src/attestation.rs +++ b/node/actors/executor/src/attestation.rs @@ -1,8 +1,8 @@ //! Module to publish attestations over batches. +use anyhow::Context as _; use std::sync::Arc; use zksync_concurrency::{ctx, time}; pub use zksync_consensus_network::gossip::attestation::*; -use anyhow::Context as _; /// An interface which is used by attesters and nodes collecting votes over gossip to determine /// which is the next batch they are all supposed to be voting on, according to the main node. @@ -14,10 +14,7 @@ pub trait Client: 'static + Send + Sync { /// /// The API might return an error while genesis is being created, which we represent with `None` /// here and mean that we'll have to try again later. - async fn config( - &self, - ctx: &ctx::Ctx, - ) -> ctx::Result>; + async fn config(&self, ctx: &ctx::Ctx) -> ctx::Result>; } /// Use an [Client] to periodically poll the main node and update the [AttestationStatusWatch]. @@ -46,7 +43,10 @@ impl Runner { loop { match self.client.config(ctx).await { Ok(Some(config)) => { - self.state.update_config(config).await.context("update_config")?; + self.state + .update_config(config) + .await + .context("update_config")?; } Ok(None) => tracing::debug!("waiting for attestation config..."), Err(error) => tracing::error!( diff --git a/node/actors/executor/src/io.rs b/node/actors/executor/src/io.rs index 6c76364b..e6fb7dd4 100644 --- a/node/actors/executor/src/io.rs +++ b/node/actors/executor/src/io.rs @@ -37,7 +37,7 @@ impl Dispatcher { /// Method to start the IO dispatcher. It is simply a loop to receive messages from the actors and then forward them. pub(super) async fn run(mut self, ctx: &ctx::Ctx) { - let _ : ctx::OrCanceled<()> = scope::run!(ctx, |ctx, s| async { + let _: ctx::OrCanceled<()> = scope::run!(ctx, |ctx, s| async { // Start a task to handle the messages from the consensus actor. s.spawn(async { while let Ok(msg) = self.consensus_output.recv(ctx).await { diff --git a/node/actors/executor/src/lib.rs b/node/actors/executor/src/lib.rs index e294af59..def48817 100644 --- a/node/actors/executor/src/lib.rs +++ b/node/actors/executor/src/lib.rs @@ -1,5 +1,6 @@ //! Library files for the executor. We have it separate from the binary so that we can use these files in the tools crate. use crate::io::Dispatcher; +use anyhow::Context as _; use network::http; pub use network::RpcConfig; use std::{ @@ -13,7 +14,6 @@ use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BatchStore, BlockStore, ReplicaStore}; use zksync_consensus_utils::pipe; use zksync_protobuf::kB; -use anyhow::Context as _; pub mod attestation; mod io; @@ -127,7 +127,10 @@ impl Executor { tracing::debug!("Starting actors in separate threads."); scope::run!(ctx, |ctx, s| async { - s.spawn(async { dispatcher.run(ctx).await; Ok(()) }); + s.spawn(async { + dispatcher.run(ctx).await; + Ok(()) + }); let (net, runner) = network::Network::new( network_config, self.block_store.clone(), @@ -137,7 +140,7 @@ impl Executor { ); net.register_metrics(); s.spawn(async { runner.run(ctx).await.context("Network stopped") }); - + if let Some(debug_config) = self.config.debug_page { s.spawn(async { http::DebugPageServer::new(debug_config, net) diff --git a/node/actors/executor/src/tests.rs b/node/actors/executor/src/tests.rs index 1cbc5fa8..c16fea9b 100644 --- a/node/actors/executor/src/tests.rs +++ b/node/actors/executor/src/tests.rs @@ -3,10 +3,10 @@ use super::*; use rand::Rng as _; //use std::sync::{atomic::AtomicU64, Mutex}; use tracing::Instrument as _; -use zksync_concurrency::{testonly::abort_on_panic}; +use zksync_concurrency::testonly::abort_on_panic; use zksync_consensus_bft as bft; use zksync_consensus_network::testonly::{new_configs, new_fullnode}; -use zksync_consensus_roles::{ validator::{testonly::Setup, BlockNumber}}; +use zksync_consensus_roles::validator::{testonly::Setup, BlockNumber}; use zksync_consensus_storage::{ testonly::{in_memory, TestMemoryStorage}, BlockStore, diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 74ac2e19..b62fed6b 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -1,12 +1,9 @@ //! Attestation. -use std::fmt; -use std::sync::Arc; -use std::cmp::Ordering; -use zksync_concurrency::{ctx,sync}; -use zksync_consensus_roles::attester; -use std::collections::HashSet; -use anyhow::Context as _; use crate::watch::Watch; +use anyhow::Context as _; +use std::{cmp::Ordering, collections::HashSet, fmt, sync::Arc}; +use zksync_concurrency::{ctx, sync}; +use zksync_consensus_roles::attester; #[cfg(test)] mod tests; @@ -20,7 +17,7 @@ pub struct Config { /// NOTE: the committee is not supposed to change often, /// so you might want to use `Arc` instead /// to avoid extra copying. - pub committee: attester::Committee, + pub committee: attester::Committee, } /// A [Watch] over an [AttestationStatus] which we can use to notify components about @@ -49,17 +46,30 @@ impl State { let Some(old) = old.as_ref() else { return Diff { config_changed: true, - votes: self.votes.values().cloned().collect() + votes: self.votes.values().cloned().collect(), }; }; - match self.config.batch_to_attest.number.cmp(&old.config.batch_to_attest.number) { - Ordering::Less => Diff { config_changed: true, votes: vec![] }, - Ordering::Greater => Diff { config_changed: true, votes: self.votes.values().cloned().collect() }, + match self + .config + .batch_to_attest + .number + .cmp(&old.config.batch_to_attest.number) + { + Ordering::Less => Diff { + config_changed: true, + votes: vec![], + }, + Ordering::Greater => Diff { + config_changed: true, + votes: self.votes.values().cloned().collect(), + }, Ordering::Equal => Diff { config_changed: false, - votes: self.votes.iter() - .filter(|(k,_)|!old.votes.contains_key(k)) - .map(|(_,v)|v.clone()) + votes: self + .votes + .iter() + .filter(|(k, _)| !old.votes.contains_key(k)) + .map(|(_, v)| v.clone()) .collect(), }, } @@ -69,18 +79,30 @@ impl State { /// Noop if vote is not signed by a committee memver or already inserted. /// Returns an error if genesis doesn't match or the signature is invalid. fn insert_vote(&mut self, vote: Arc>) -> anyhow::Result<()> { - anyhow::ensure!(vote.msg.genesis == self.config.batch_to_attest.genesis, "Genesis mismatch"); - if vote.msg != self.config.batch_to_attest { return Ok(()) } - let Some(weight) = self.config.committee.weight(&vote.key) else { return Ok(()) }; - if self.votes.contains_key(&vote.key) { return Ok(()) } - // Verify signature only after checking all the other preconditions. + anyhow::ensure!( + vote.msg.genesis == self.config.batch_to_attest.genesis, + "Genesis mismatch" + ); + if vote.msg != self.config.batch_to_attest { + return Ok(()); + } + let Some(weight) = self.config.committee.weight(&vote.key) else { + return Ok(()); + }; + if self.votes.contains_key(&vote.key) { + return Ok(()); + } + // Verify signature only after checking all the other preconditions. vote.verify().context("verify")?; - self.votes.insert(vote.key.clone(),vote); + self.votes.insert(vote.key.clone(), vote); self.weight += weight; Ok(()) } - fn insert_votes(&mut self, votes: impl Iterator>>) -> anyhow::Result<()> { + fn insert_votes( + &mut self, + votes: impl Iterator>>, + ) -> anyhow::Result<()> { let mut done = HashSet::new(); for vote in votes { // Disallow multiple entries for the same key: @@ -101,7 +123,7 @@ impl State { } let mut sigs = attester::MultiSig::default(); for vote in self.votes.values() { - sigs.add(vote.key.clone(),vote.sig.clone()); + sigs.add(vote.key.clone(), vote.sig.clone()); } Some(attester::BatchQC { message: self.config.batch_to_attest.clone(), @@ -118,11 +140,13 @@ pub(crate) struct DiffReceiver { impl DiffReceiver { pub(crate) async fn wait_for_diff(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled { loop { - let Some(new) = (*sync::changed(ctx, &mut self.recv).await?).clone() else { continue }; + let Some(new) = (*sync::changed(ctx, &mut self.recv).await?).clone() else { + continue; + }; let diff = new.diff(&self.prev); self.prev = Some(new); if !diff.is_empty() { - return Ok(diff) + return Ok(diff); } } } @@ -136,15 +160,17 @@ pub struct StateWatch { impl fmt::Debug for StateWatch { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.debug_struct("StateWatch") - .finish_non_exhaustive() + fmt.debug_struct("StateWatch").finish_non_exhaustive() } } impl StateWatch { /// Constructs AttestationStatusWatch. pub fn new(key: Option) -> Self { - Self { key, state: Watch::new(None) } + Self { + key, + state: Watch::new(None), + } } pub(crate) fn subscribe(&self) -> DiffReceiver { @@ -153,9 +179,14 @@ impl StateWatch { DiffReceiver { prev: None, recv } } - pub(crate) async fn insert_votes(&self, votes: impl Iterator>>) -> anyhow::Result<()> { + pub(crate) async fn insert_votes( + &self, + votes: impl Iterator>>, + ) -> anyhow::Result<()> { let locked = self.state.lock().await; - let Some(mut state) = locked.borrow().clone() else { return Ok(()) }; + let Some(mut state) = locked.borrow().clone() else { + return Ok(()); + }; let before = state.weight; let res = state.insert_votes(votes); if state.weight > before { @@ -165,21 +196,38 @@ impl StateWatch { } /// All votes kept in the state. - pub(crate) fn votes(&self) -> Vec>> { - self.state.subscribe().borrow().iter().map(|s|s.votes.values().cloned()).flatten().collect() + pub(crate) fn votes(&self) -> Vec>> { + self.state + .subscribe() + .borrow() + .iter() + .flat_map(|s| s.votes.values().cloned()) + .collect() } /// Waits for the certificate for a batch with the given number to be collected. /// Returns None iff attestation already skipped to collecting certificate some later batch. - pub async fn wait_for_qc(&self, ctx: &ctx::Ctx, n: attester::BatchNumber) -> ctx::OrCanceled> { + pub async fn wait_for_qc( + &self, + ctx: &ctx::Ctx, + n: attester::BatchNumber, + ) -> ctx::OrCanceled> { let recv = &mut self.state.subscribe(); recv.mark_changed(); loop { let state = sync::changed(ctx, recv).await?; - let Some(state) = state.as_ref() else { continue }; - if state.config.batch_to_attest.number < n { continue }; - if state.config.batch_to_attest.number > n { return Ok(None); } - if let Some(qc) = state.qc() { return Ok(Some(qc)); } + let Some(state) = state.as_ref() else { + continue; + }; + if state.config.batch_to_attest.number < n { + continue; + }; + if state.config.batch_to_attest.number > n { + return Ok(None); + } + if let Some(qc) = state.qc() { + return Ok(Some(qc)); + } } } @@ -190,14 +238,26 @@ impl StateWatch { let locked = self.state.lock().await; let old = locked.borrow().clone(); if let Some(old) = old.as_ref() { - if *old.config == config { return Ok(()) } - anyhow::ensure!(old.config.batch_to_attest.genesis == config.batch_to_attest.genesis, "tried to change genesis"); - anyhow::ensure!(old.config.batch_to_attest.number < config.batch_to_attest.number, "tried to decrease batch number"); + if *old.config == config { + return Ok(()); + } + anyhow::ensure!( + old.config.batch_to_attest.genesis == config.batch_to_attest.genesis, + "tried to change genesis" + ); + anyhow::ensure!( + old.config.batch_to_attest.number < config.batch_to_attest.number, + "tried to decrease batch number" + ); } - let mut new = State { config: Arc::new(config), votes: im::HashMap::new(), weight: 0 }; + let mut new = State { + config: Arc::new(config), + votes: im::HashMap::new(), + weight: 0, + }; if let Some(key) = self.key.as_ref() { if new.config.committee.contains(&key.public()) { - let vote = key.sign_msg(new.config.batch_to_attest.clone()); + let vote = key.sign_msg(new.config.batch_to_attest.clone()); // This is our own vote, so it always should be valid. new.insert_vote(Arc::new(vote)).unwrap(); } diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index bc4127d9..e26b0119 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -1,28 +1,27 @@ -#![allow(dead_code)] use super::*; -use zksync_concurrency::testonly::abort_on_panic; use rand::{seq::SliceRandom as _, Rng as _}; +use zksync_concurrency::testonly::abort_on_panic; type Vote = Arc>; #[derive(Default, Debug, PartialEq)] struct Votes(im::HashMap); -impl> From for Votes { +impl> From for Votes { fn from(x: T) -> Self { - Self(x.into_iter().map(|v|(v.key.clone(),v)).collect()) + Self(x.into_iter().map(|v| (v.key.clone(), v)).collect()) } } #[tokio::test] async fn test_insert_votes() { abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); + let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); let state = StateWatch::new(None); - let genesis : attester::GenesisHash = rng.gen(); - let first : attester::BatchNumber = rng.gen(); + let genesis: attester::GenesisHash = rng.gen(); + let first: attester::BatchNumber = rng.gen(); for i in 0..3 { let keys: Vec = (0..8).map(|_| rng.gen()).collect(); let config = Config { @@ -34,69 +33,134 @@ async fn test_insert_votes() { committee: attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { key: k.public(), weight: 1250, - })).unwrap(), + })) + .unwrap(), }; state.update_config(config.clone()).await.unwrap(); assert_eq!(Votes::from([]), state.votes().into()); let mut recv = state.subscribe(); + let diff = recv.wait_for_diff(ctx).await.unwrap(); + assert!(diff.config_changed); + assert_eq!(Votes::default(), diff.votes.into()); - let all_votes : Vec = keys.iter().map(|k| k.sign_msg(config.batch_to_attest.clone().into()).into()).collect(); + let all_votes: Vec = keys + .iter() + .map(|k| k.sign_msg(config.batch_to_attest.clone()).into()) + .collect(); // Initial votes. - state.insert_votes(all_votes[0..3].iter().cloned()).await.unwrap(); - assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), state.votes().into()); - assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), recv.wait_for_diff(ctx).await.unwrap().votes.into()); + state + .insert_votes(all_votes[0..3].iter().cloned()) + .await + .unwrap(); + assert_eq!( + Votes::from(all_votes[0..3].iter().cloned()), + state.votes().into() + ); + let diff = recv.wait_for_diff(ctx).await.unwrap(); + assert!(!diff.config_changed); + assert_eq!( + Votes::from(all_votes[0..3].iter().cloned()), + diff.votes.into() + ); // Adding votes gradually. - state.insert_votes(all_votes[3..5].iter().cloned()).await.unwrap(); - state.insert_votes(all_votes[5..7].iter().cloned()).await.unwrap(); - assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); - assert_eq!(Votes::from(all_votes[3..7].iter().cloned()), recv.wait_for_diff(ctx).await.unwrap().votes.into()); + state + .insert_votes(all_votes[3..5].iter().cloned()) + .await + .unwrap(); + state + .insert_votes(all_votes[5..7].iter().cloned()) + .await + .unwrap(); + assert_eq!( + Votes::from(all_votes[0..7].iter().cloned()), + state.votes().into() + ); + let diff = recv.wait_for_diff(ctx).await.unwrap(); + assert!(!diff.config_changed); + assert_eq!( + Votes::from(all_votes[3..7].iter().cloned()), + diff.votes.into() + ); // Readding already inserded votes (noop). - state.insert_votes(all_votes[2..6].iter().cloned()).await.unwrap(); - assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + state + .insert_votes(all_votes[2..6].iter().cloned()) + .await + .unwrap(); + assert_eq!( + Votes::from(all_votes[0..7].iter().cloned()), + state.votes().into() + ); // Adding votes out of committee (noop). - state.insert_votes((0..3).map(|_| { - let k : attester::SecretKey = rng.gen(); - k.sign_msg(config.batch_to_attest.clone()).into() - })).await.unwrap(); - assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + state + .insert_votes((0..3).map(|_| { + let k: attester::SecretKey = rng.gen(); + k.sign_msg(config.batch_to_attest.clone()).into() + })) + .await + .unwrap(); + assert_eq!( + Votes::from(all_votes[0..7].iter().cloned()), + state.votes().into() + ); // Adding votes for different batch (noop). - state.insert_votes((0..3).map(|_| { - let k : attester::SecretKey = rng.gen(); - k.sign_msg(attester::Batch { - genesis: config.batch_to_attest.genesis, - number: rng.gen(), - hash: rng.gen(), - }).into() - })).await.unwrap(); - assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); + state + .insert_votes((0..3).map(|_| { + let k: attester::SecretKey = rng.gen(); + k.sign_msg(attester::Batch { + genesis: config.batch_to_attest.genesis, + number: rng.gen(), + hash: rng.gen(), + }) + .into() + })) + .await + .unwrap(); + assert_eq!( + Votes::from(all_votes[0..7].iter().cloned()), + state.votes().into() + ); // Adding incorrect votes (error). let mut bad_vote = (*all_votes[7]).clone(); bad_vote.sig = rng.gen(); - assert!(state.insert_votes([bad_vote.into()].into_iter()).await.is_err()); - assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), state.votes().into()); - + assert!(state + .insert_votes([bad_vote.into()].into_iter()) + .await + .is_err()); + assert_eq!( + Votes::from(all_votes[0..7].iter().cloned()), + state.votes().into() + ); + // Add the last vote mixed with already added votes. - state.insert_votes(all_votes[5..].iter().cloned()).await.unwrap(); + state + .insert_votes(all_votes[5..].iter().cloned()) + .await + .unwrap(); assert_eq!(Votes::from(all_votes.clone()), state.votes().into()); - assert_eq!(Votes::from(all_votes[7..].iter().cloned()), recv.wait_for_diff(ctx).await.unwrap().votes.into()); + let diff = recv.wait_for_diff(ctx).await.unwrap(); + assert!(!diff.config_changed); + assert_eq!( + Votes::from(all_votes[7..].iter().cloned()), + diff.votes.into() + ); } } #[tokio::test] async fn test_wait_for_qc() { abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); + let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); let state = StateWatch::new(None); - let genesis : attester::GenesisHash = rng.gen(); - let first : attester::BatchNumber = rng.gen(); + let genesis: attester::GenesisHash = rng.gen(); + let first: attester::BatchNumber = rng.gen(); for i in 0..10 { tracing::info!("iteration {i}"); @@ -111,24 +175,48 @@ async fn test_wait_for_qc() { committee: attester::Committee::new(keys.iter().map(|k| attester::WeightedAttester { key: k.public(), weight: rng.gen_range(1..=100), - })).unwrap(), + })) + .unwrap(), }; - let mut all_votes : Vec = keys.iter().map(|k| k.sign_msg(config.batch_to_attest.clone().into()).into()).collect(); + let mut all_votes: Vec = keys + .iter() + .map(|k| k.sign_msg(config.batch_to_attest.clone()).into()) + .collect(); all_votes.shuffle(rng); state.update_config(config.clone()).await.unwrap(); loop { let end = rng.gen_range(0..=committee_size); tracing::info!("end = {end}"); - state.insert_votes(all_votes[..end].iter().cloned()).await.unwrap(); + state + .insert_votes(all_votes[..end].iter().cloned()) + .await + .unwrap(); // Waiting for the previous qc should immediately return None. - assert_eq!(None,state.wait_for_qc(ctx, config.batch_to_attest.number.prev().unwrap()).await.unwrap()); - if config.committee.weight_of_keys(all_votes[..end].iter().map(|v|&v.key)) >= config.committee.threshold() { - let qc = state.wait_for_qc(ctx, config.batch_to_attest.number).await.unwrap().unwrap(); + assert_eq!( + None, + state + .wait_for_qc(ctx, config.batch_to_attest.number.prev().unwrap()) + .await + .unwrap() + ); + if config + .committee + .weight_of_keys(all_votes[..end].iter().map(|v| &v.key)) + >= config.committee.threshold() + { + let qc = state + .wait_for_qc(ctx, config.batch_to_attest.number) + .await + .unwrap() + .unwrap(); assert_eq!(qc.message, config.batch_to_attest); - qc.verify(genesis,&config.committee).unwrap(); + qc.verify(genesis, &config.committee).unwrap(); break; } - assert_eq!(None, state.state.subscribe().borrow().as_ref().unwrap().qc()); + assert_eq!( + None, + state.state.subscribe().borrow().as_ref().unwrap().qc() + ); } } } diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index e0a6576c..35dfd494 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -61,9 +61,16 @@ impl rpc::Handler for &PushServer<'_> { 100 * kB } - async fn handle(&self, _ctx: &ctx::Ctx, req: rpc::push_batch_votes::Req) -> anyhow::Result { + async fn handle( + &self, + _ctx: &ctx::Ctx, + req: rpc::push_batch_votes::Req, + ) -> anyhow::Result { let want_snapshot = req.want_snapshot(); - self.net.attestation_state.insert_votes(req.votes.into_iter()).await?; + self.net + .attestation_state + .insert_votes(req.votes.into_iter()) + .await?; Ok(rpc::push_batch_votes::Resp { votes: if want_snapshot { self.net.attestation_state.votes() @@ -154,10 +161,8 @@ impl Network { rpc::Client::::new(ctx, self.cfg.rpc.get_block_rate); let get_batch_client = rpc::Client::::new(ctx, self.cfg.rpc.get_batch_rate); - let push_batch_votes_client = rpc::Client::::new( - ctx, - self.cfg.rpc.push_batch_votes_rate, - ); + let push_batch_votes_client = + rpc::Client::::new(ctx, self.cfg.rpc.push_batch_votes_rate); scope::run!(ctx, |ctx, s| async { let mut service = rpc::Service::new() @@ -199,6 +204,8 @@ impl Network { loop { let diff = recv.wait_for_diff(ctx).await?; let req = rpc::push_batch_votes::Req { + // If the config has changed, we need to re-request all the votes + // from peer that we might have ignored earlier. want_snapshot: Some(diff.config_changed), votes: diff.votes, }; @@ -207,18 +214,18 @@ impl Network { // such a case. let resp = push_batch_votes_client.call(ctx, &req, 100 * kB).await?; if !resp.votes.is_empty() { - anyhow::ensure!(req.want_snapshot(), "expected empty response, but votes were returned"); - self.attestation_state.insert_votes(resp.votes.into_iter()).await.context("insert_votes")?; + anyhow::ensure!( + req.want_snapshot(), + "expected empty response, but votes were returned" + ); + self.attestation_state + .insert_votes(resp.votes.into_iter()) + .await + .context("insert_votes")?; } } }); - // Pull L1 batch votes from peers. - s.spawn(async { - // TODO - Ok(()) - }); - if let Some(ping_timeout) = &self.cfg.ping_timeout { let ping_client = rpc::Client::::new(ctx, rpc::ping::RATE); service = service.add_client(&ping_client); diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index c808bd95..dbb4e7e6 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -1,11 +1,6 @@ -#![allow(dead_code,unused_imports)] use super::ValidatorAddrs; use crate::{ - gossip::{ - attestation, - handshake, - validator_addrs::ValidatorAddrsWatch, - }, + gossip::{attestation, handshake, validator_addrs::ValidatorAddrsWatch}, metrics, preface, rpc, testonly, }; use anyhow::Context as _; @@ -20,12 +15,12 @@ use tracing::Instrument as _; use zksync_concurrency::{ ctx, error::Wrap as _, - net, scope, sync, + limiter, net, scope, sync, testonly::{abort_on_panic, set_timeout}, time, }; use zksync_consensus_roles::{attester, validator}; -use zksync_consensus_storage::{testonly::TestMemoryStorage}; +use zksync_consensus_storage::testonly::TestMemoryStorage; mod fetch_batches; mod fetch_blocks; @@ -489,32 +484,37 @@ async fn rate_limiting() { } } -/* #[tokio::test(flavor = "multi_thread")] async fn test_batch_votes_propagation() { - let _guard = set_timeout(time::Duration::seconds(10)); abort_on_panic(); + let _guard = set_timeout(time::Duration::seconds(10)); let ctx = &ctx::test_root(&ctx::AffineClock::new(10.)); let rng = &mut ctx.rng(); - let mut setup = validator::testonly::Setup::new(rng, 10); - let cfgs = testonly::new_configs(rng, &setup, 1); + let setup = validator::testonly::Setup::new(rng, 10); + let cfgs = testonly::new_configs(rng, &setup, 2); // Fixed attestation schedule. - let first attester::BatchNumber = rng.gen(); - let schedule = (0..10).map(|i| attestation::Config { - batch_to_attest : attester::Batch { - genesis: setup.genesis.hash(), - number: first + i, - hash: rng.gen(), - }, - committee: attester::Committee::new(setup.attester_keys.iter().filter(|_|rng.gen()).map(|k| - attester::WeightedAttester { - key: k.public(), - weight: rng.gen_range(5..10), + let first: attester::BatchNumber = rng.gen(); + let schedule: Vec<_> = (0..10) + .map(|r| attestation::Config { + batch_to_attest: attester::Batch { + genesis: setup.genesis.hash(), + number: first + r, + hash: rng.gen(), }, - )).unwrap(), - }).collect(); + committee: { + // We select a random subset here. It would be incorrect to choose an empty subset, but + // the chances of that are negligible. + let subset: Vec<_> = setup.attester_keys.iter().filter(|_| rng.gen()).collect(); + attester::Committee::new(subset.iter().map(|k| attester::WeightedAttester { + key: k.public(), + weight: rng.gen_range(5..10), + })) + .unwrap() + }, + }) + .collect(); // Round of the schedule that nodes should collect the votes for. let round = sync::watch::channel(0).0; @@ -525,28 +525,64 @@ async fn test_batch_votes_propagation() { s.spawn_bg(store.runner.run(ctx)); // Start all nodes. - let mut nodes = vec![]; - for (i,cfg) in cfgs.iter().enumerate() { - let (node, runner) = testonly::Instance::new( - cfg.clone(), - store.blocks.clone(), - store.batches.clone(), - ); + for (i, mut cfg) in cfgs.into_iter().enumerate() { + cfg.rpc.push_batch_votes_rate = limiter::Rate::INF; + cfg.validator_key = None; + let (node, runner) = testonly::Instance::new_from_config(testonly::InstanceConfig { + cfg: cfg.clone(), + block_store: store.blocks.clone(), + batch_store: store.batches.clone(), + attestation_state: attestation::StateWatch::new(Some( + setup.attester_keys[i].clone(), + )) + .into(), + }); s.spawn_bg(runner.run(ctx).instrument(tracing::info_span!("node", i))); - // Task updating config whenever we advance to the next view. - s.spawn_bg(async { - let mut round = round.subscribe(); - round.mark_changed(); - loop { - let round = *sync::changed(ctx, &mut round).await; - let Some(cfg) = schedule.get(round) else { return Ok(()) }; - node.net.gossip.attestation_state.update_config(cfg.clone()).unwrap(); + // Task going through the schedule, waiting for ANY node to collect the certificate + // to advance to the next round of the schedule. + // Test succeeds if all tasks successfully iterate through the whole schedule. + s.spawn( + async { + let node = node; + let sub = &mut round.subscribe(); + sub.mark_changed(); + loop { + let r = ctx::NoCopy(*sync::changed(ctx, sub).await?); + tracing::info!("starting round {}", *r); + let Some(cfg) = schedule.get(*r) else { + return Ok(()); + }; + let attestation_state = node.net.gossip.attestation_state.clone(); + attestation_state.update_config(cfg.clone()).await.unwrap(); + // Wait for the certificate in the background. + s.spawn_bg(async { + let r = r; + let attestation_state = attestation_state; + let Ok(Some(qc)) = attestation_state + .wait_for_qc(ctx, cfg.batch_to_attest.number) + .await + else { + return Ok(()); + }; + assert_eq!(qc.message, cfg.batch_to_attest); + qc.verify(setup.genesis.hash(), &cfg.committee).unwrap(); + tracing::info!("got cert for {}", *r); + round.send_if_modified(|round| { + if *round != *r { + return false; + } + *round = *r + 1; + true + }); + Ok(()) + }); + } } - }); - - nodes.push(node); + .instrument(tracing::info_span!("attester", i)), + ); } + Ok(()) }) .await .unwrap(); -}*/ +} diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 0731cd91..40cd6196 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -1,8 +1,8 @@ //! Defines RPC for passing consensus messages. -use crate::{proto::consensus as proto}; +use super::Capability; +use crate::proto::consensus as proto; use zksync_consensus_roles::validator; use zksync_protobuf::{read_required, ProtoFmt}; -use super::Capability; /// Consensus RPC. pub(crate) struct Rpc; diff --git a/node/actors/network/src/rpc/get_batch.rs b/node/actors/network/src/rpc/get_batch.rs index 8f19852e..18c53ca4 100644 --- a/node/actors/network/src/rpc/get_batch.rs +++ b/node/actors/network/src/rpc/get_batch.rs @@ -1,9 +1,9 @@ //! RPC for fetching a batch from peer. -use crate::{proto::gossip as proto}; +use super::Capability; +use crate::proto::gossip as proto; use anyhow::Context; use zksync_consensus_roles::attester; use zksync_protobuf::{read_optional, ProtoFmt}; -use super::Capability; /// `get_batch` RPC. #[derive(Debug)] @@ -11,7 +11,7 @@ pub(crate) struct Rpc; // TODO: determine more precise `INFLIGHT` / `RATE` values as a result of load testing impl super::Rpc for Rpc { - const CAPABILITY: Capability = Capability::GetBatch; + const CAPABILITY: Capability = Capability::GetBatch; const INFLIGHT: u32 = 5; const METHOD: &'static str = "get_batch"; diff --git a/node/actors/network/src/rpc/get_block.rs b/node/actors/network/src/rpc/get_block.rs index 58486746..84474f7a 100644 --- a/node/actors/network/src/rpc/get_block.rs +++ b/node/actors/network/src/rpc/get_block.rs @@ -1,9 +1,9 @@ //! RPC for fetching a block from peer. -use crate::{proto::gossip as proto}; +use super::Capability; +use crate::proto::gossip as proto; use anyhow::Context; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; use zksync_protobuf::{read_optional, ProtoFmt}; -use super::Capability; /// `get_block` RPC. #[derive(Debug)] @@ -11,7 +11,7 @@ pub(crate) struct Rpc; // TODO: determine more precise `INFLIGHT` / `RATE` values as a result of load testing impl super::Rpc for Rpc { - const CAPABILITY: Capability = Capability::GetBlock; + const CAPABILITY: Capability = Capability::GetBlock; const INFLIGHT: u32 = 5; const METHOD: &'static str = "get_block"; diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index e390d180..06b00f12 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -16,11 +16,11 @@ //! at the same time (max 1 client + server per CapabilityId). use self::metrics::{CallLatencyType, CallType, RPC_METRICS}; +pub(crate) use crate::proto::rpc::Capability; use crate::{frame, mux}; use anyhow::Context as _; use std::{collections::BTreeMap, sync::Arc}; use zksync_concurrency::{ctx, io, limiter, metrics::LatencyHistogramExt as _, scope}; -pub(crate) use crate::proto::rpc::Capability as Capability; pub(crate) mod consensus; pub(crate) mod get_batch; diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index 8ccbac1a..1e78ba8c 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -1,16 +1,16 @@ //! Defines an RPC for sending ping messages. -use crate::{proto::ping as proto}; +use super::Capability; +use crate::proto::ping as proto; use anyhow::Context as _; use rand::Rng; use zksync_concurrency::{ctx, limiter, time}; use zksync_protobuf::{kB, required, ProtoFmt}; -use super::Capability; /// Ping RPC. pub(crate) struct Rpc; impl super::Rpc for Rpc { - const CAPABILITY : Capability = Capability::Ping; + const CAPABILITY: Capability = Capability::Ping; const INFLIGHT: u32 = 1; const METHOD: &'static str = "ping"; type Req = Req; diff --git a/node/actors/network/src/rpc/push_batch_store_state.rs b/node/actors/network/src/rpc/push_batch_store_state.rs index af178453..4c561abe 100644 --- a/node/actors/network/src/rpc/push_batch_store_state.rs +++ b/node/actors/network/src/rpc/push_batch_store_state.rs @@ -1,10 +1,10 @@ //! RPC for fetching a batch from peer. -use crate::{proto::gossip as proto}; +use super::Capability; +use crate::proto::gossip as proto; use anyhow::Context as _; use zksync_consensus_roles::attester; use zksync_consensus_storage::BatchStoreState; use zksync_protobuf::{required, ProtoFmt}; -use super::Capability; /// PushBatchStoreState RPC. #[derive(Debug)] diff --git a/node/actors/network/src/rpc/push_batch_votes.rs b/node/actors/network/src/rpc/push_batch_votes.rs index 3e705c0a..5ffa2866 100644 --- a/node/actors/network/src/rpc/push_batch_votes.rs +++ b/node/actors/network/src/rpc/push_batch_votes.rs @@ -1,12 +1,12 @@ //! Defines RPC for passing consensus messages. -use crate::{proto::gossip as proto}; +use super::Capability; +use crate::proto::gossip as proto; use anyhow::Context as _; use std::sync::Arc; use zksync_consensus_roles::attester; use zksync_protobuf::ProtoFmt; -use super::Capability; -/// RPC pushing fresh batch votes. +/// RPC pushing fresh batch votes. pub(crate) struct Rpc; impl super::Rpc for Rpc { @@ -54,12 +54,15 @@ impl ProtoFmt for Req { fn build(&self) -> Self::Proto { Self::Proto { want_snapshot: self.want_snapshot, - votes: self.votes.iter().map(|a| ProtoFmt::build(a.as_ref())).collect(), + votes: self + .votes + .iter() + .map(|a| ProtoFmt::build(a.as_ref())) + .collect(), } } } - impl ProtoFmt for Resp { type Proto = proto::PushBatchVotesResp; @@ -75,7 +78,11 @@ impl ProtoFmt for Resp { fn build(&self) -> Self::Proto { Self::Proto { - votes: self.votes.iter().map(|a| ProtoFmt::build(a.as_ref())).collect(), + votes: self + .votes + .iter() + .map(|a| ProtoFmt::build(a.as_ref())) + .collect(), } } } diff --git a/node/actors/network/src/rpc/push_block_store_state.rs b/node/actors/network/src/rpc/push_block_store_state.rs index ce49ba34..1134185c 100644 --- a/node/actors/network/src/rpc/push_block_store_state.rs +++ b/node/actors/network/src/rpc/push_block_store_state.rs @@ -1,10 +1,10 @@ //! RPC for notifying peer about our BlockStore state. -use crate::{proto::gossip as proto}; +use super::Capability; +use crate::proto::gossip as proto; use anyhow::Context; use zksync_consensus_roles::validator; use zksync_consensus_storage::BlockStoreState; use zksync_protobuf::{read_optional, required, ProtoFmt}; -use super::Capability; /// PushBlockStoreState RPC. #[derive(Debug)] diff --git a/node/actors/network/src/rpc/push_validator_addrs.rs b/node/actors/network/src/rpc/push_validator_addrs.rs index 7bf3ff82..48a85ba4 100644 --- a/node/actors/network/src/rpc/push_validator_addrs.rs +++ b/node/actors/network/src/rpc/push_validator_addrs.rs @@ -1,10 +1,10 @@ //! RPC for synchronizing ValidatorAddrs data. -use crate::{proto::gossip as proto}; +use super::Capability; +use crate::proto::gossip as proto; use anyhow::Context as _; use std::sync::Arc; use zksync_consensus_roles::validator; use zksync_protobuf::ProtoFmt; -use super::Capability; /// PushValidatorAddrs RPC. pub(crate) struct Rpc; diff --git a/node/actors/network/src/rpc/tests.rs b/node/actors/network/src/rpc/tests.rs index db4ded88..d2523fc2 100644 --- a/node/actors/network/src/rpc/tests.rs +++ b/node/actors/network/src/rpc/tests.rs @@ -1,9 +1,7 @@ use super::*; use crate::noise; use rand::Rng as _; -use std::{ - sync::atomic::{AtomicU64, Ordering}, -}; +use std::sync::atomic::{AtomicU64, Ordering}; use zksync_concurrency::{ctx, testonly::abort_on_panic, time}; use zksync_protobuf::{kB, testonly::test_encode_random}; diff --git a/node/actors/network/src/testonly.rs b/node/actors/network/src/testonly.rs index e6e56e61..8597f252 100644 --- a/node/actors/network/src/testonly.rs +++ b/node/actors/network/src/testonly.rs @@ -177,24 +177,42 @@ impl InstanceRunner { } } +/// InstanceConfig +pub struct InstanceConfig { + /// cfg + pub cfg: Config, + /// block_store + pub block_store: Arc, + /// batch_store + pub batch_store: Arc, + /// attestation_state + pub attestation_state: Arc, +} + impl Instance { - /// Construct an instance for a given config. + /// Constructs a new instance. pub fn new( cfg: Config, block_store: Arc, batch_store: Arc, ) -> (Self, InstanceRunner) { - // Semantically we'd want this to be created at the same level as the stores, - // but doing so would introduce a lot of extra cruft in setting up tests. - let attestation_state = Arc::new(attestation::StateWatch::new(None)); + Self::new_from_config(InstanceConfig { + cfg, + block_store, + batch_store, + attestation_state: attestation::StateWatch::new(None).into(), + }) + } + /// Construct an instance for a given config. + pub fn new_from_config(cfg: InstanceConfig) -> (Self, InstanceRunner) { let (actor_pipe, dispatcher_pipe) = pipe::new(); let (net, net_runner) = Network::new( - cfg, - block_store.clone(), - batch_store.clone(), + cfg.cfg, + cfg.block_store.clone(), + cfg.batch_store.clone(), actor_pipe, - attestation_state, + cfg.attestation_state, ); let (terminate_send, terminate_recv) = channel::bounded(1); ( @@ -205,7 +223,7 @@ impl Instance { }, InstanceRunner { net_runner, - batch_store, + batch_store: cfg.batch_store.clone(), terminate: terminate_recv, }, ) diff --git a/node/libs/roles/src/attester/messages/batch.rs b/node/libs/roles/src/attester/messages/batch.rs index 8403bc8c..e850740e 100644 --- a/node/libs/roles/src/attester/messages/batch.rs +++ b/node/libs/roles/src/attester/messages/batch.rs @@ -1,8 +1,5 @@ use super::{GenesisHash, Signed}; -use crate::{ - attester, - validator::{Payload}, -}; +use crate::{attester, validator::Payload}; use zksync_consensus_crypto::{keccak256::Keccak256, ByteFmt, Text, TextFmt}; use zksync_consensus_utils::enum_util::Variant; @@ -159,19 +156,30 @@ impl BatchQC { /// Add a attester's signature. /// Signature is assumed to be already verified. - pub fn add(&mut self, msg: &Signed, committee: &attester::Committee) -> anyhow::Result<()> { + pub fn add( + &mut self, + msg: &Signed, + committee: &attester::Committee, + ) -> anyhow::Result<()> { anyhow::ensure!(self.message == msg.msg, "inconsistent messages"); - anyhow::ensure!(!self.signatures.contains(&msg.key), "signature already present"); + anyhow::ensure!( + !self.signatures.contains(&msg.key), + "signature already present" + ); anyhow::ensure!(committee.contains(&msg.key), "not in committee"); self.signatures.add(msg.key.clone(), msg.sig.clone()); Ok(()) } /// Verifies the the BatchQC. - pub fn verify(&self, genesis: GenesisHash, committee: &attester::Committee) -> Result<(), BatchQCVerifyError> { + pub fn verify( + &self, + genesis: GenesisHash, + committee: &attester::Committee, + ) -> Result<(), BatchQCVerifyError> { use BatchQCVerifyError as Error; - - if self.message.genesis!=genesis { + + if self.message.genesis != genesis { return Err(Error::GenesisMismatch); } diff --git a/node/libs/roles/src/attester/tests.rs b/node/libs/roles/src/attester/tests.rs index 370c3365..c9d85cd4 100644 --- a/node/libs/roles/src/attester/tests.rs +++ b/node/libs/roles/src/attester/tests.rs @@ -165,29 +165,37 @@ fn test_batch_qc() { // Create QCs with increasing number of attesters. for i in 0..setup1.attester_keys.len() + 1 { - let mut qc = BatchQC::new(Batch{ + let mut qc = BatchQC::new(Batch { genesis: setup1.genesis.hash(), number: rng.gen(), hash: rng.gen(), }); for key in &setup1.attester_keys[0..i] { - qc.add(&key.sign_msg(qc.message.clone()), &attesters) + qc.add(&key.sign_msg(qc.message.clone()), attesters) .unwrap(); } let expected_weight: u64 = attesters.iter().take(i).map(|w| w.weight).sum(); if expected_weight >= attesters.threshold() { - qc.verify(setup1.genesis.hash(), &attesters).expect("failed to verify QC"); + qc.verify(setup1.genesis.hash(), attesters) + .expect("failed to verify QC"); } else { assert_matches!( - qc.verify(setup1.genesis.hash(), &attesters), + qc.verify(setup1.genesis.hash(), attesters), Err(Error::NotEnoughSigners { .. }) ); } // Mismatching attesters sets. - assert!(qc.verify(setup1.genesis.hash(), setup2.genesis.attesters.as_ref().unwrap()).is_err()); - assert!(qc.verify(setup1.genesis.hash(), genesis3.attesters.as_ref().unwrap()).is_err()); + assert!(qc + .verify( + setup1.genesis.hash(), + setup2.genesis.attesters.as_ref().unwrap() + ) + .is_err()); + assert!(qc + .verify(setup1.genesis.hash(), genesis3.attesters.as_ref().unwrap()) + .is_err()); } } @@ -200,17 +208,13 @@ fn test_attester_committee_weights() { let setup = Setup::new_with_weights(rng, vec![1000, 600, 800, 6000, 900, 700]); // Expected sum of the attesters weights let sums = [1000, 1600, 2400, 8400, 9300, 10000]; - let attesters = setup - .genesis - .attesters - .as_ref() - .unwrap(); + let attesters = setup.genesis.attesters.as_ref().unwrap(); let msg: Batch = rng.gen(); let mut qc = BatchQC::new(msg.clone()); for (i, weight) in sums.iter().enumerate() { let key = &setup.attester_keys[i]; - qc.add(&key.sign_msg(msg.clone()), &attesters).unwrap(); + qc.add(&key.sign_msg(msg.clone()), attesters).unwrap(); assert_eq!(attesters.weight_of_keys(qc.signatures.keys()), *weight); } } diff --git a/node/libs/storage/src/testonly/in_memory.rs b/node/libs/storage/src/testonly/in_memory.rs index 93139789..c4a47d55 100644 --- a/node/libs/storage/src/testonly/in_memory.rs +++ b/node/libs/storage/src/testonly/in_memory.rs @@ -5,7 +5,7 @@ use crate::{ }; use anyhow::Context as _; use std::{ - collections::{VecDeque}, + collections::VecDeque, sync::{Arc, Mutex}, }; use zksync_concurrency::{ctx, sync}; diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 46ec1e97..e37c24b9 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -265,9 +265,8 @@ impl Configs { let store = TestMemoryStorage::new(ctx, &self.app.genesis).await; // We don't have an API to poll in this setup, we can only create a local store based attestation client. - let attestation_state = Arc::new(attestation::StateWatch::new( - self.app.attester_key.clone() - )); + let attestation_state = + Arc::new(attestation::StateWatch::new(self.app.attester_key.clone())); let runner = store.runner; let e = executor::Executor { @@ -306,7 +305,7 @@ impl Configs { self.app.max_payload_size, )), }), - attestation_state, + attestation_state, }; Ok((e, runner)) } From a2f38c4470e803257cf2b43ca5feba37b46a10aa Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 7 Aug 2024 17:27:46 +0200 Subject: [PATCH 18/31] cargo fmt --- node/actors/network/src/gossip/attestation/mod.rs | 11 +++++++++++ node/actors/network/src/rpc/mod.rs | 1 + node/actors/network/src/rpc/push_batch_votes.rs | 6 +++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index b62fed6b..82c00d4b 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -29,8 +29,11 @@ pub(crate) struct State { weight: attester::Weight, } +/// Diff between 2 states. pub(crate) struct Diff { + /// New votes. pub(crate) votes: Vec>>, + /// Whether the config has changed. pub(crate) config_changed: bool, } @@ -132,12 +135,14 @@ impl State { } } +/// Received of state diffs. pub(crate) struct DiffReceiver { prev: Option, recv: sync::watch::Receiver>, } impl DiffReceiver { + /// Waits for the next state diff. pub(crate) async fn wait_for_diff(&mut self, ctx: &ctx::Ctx) -> ctx::OrCanceled { loop { let Some(new) = (*sync::changed(ctx, &mut self.recv).await?).clone() else { @@ -173,12 +178,18 @@ impl StateWatch { } } + /// Subscribes to state diffs. pub(crate) fn subscribe(&self) -> DiffReceiver { let mut recv = self.state.subscribe(); recv.mark_changed(); DiffReceiver { prev: None, recv } } + /// Inserts votes to the state. + /// Irrelevant votes are silently ignored. + /// Returns an error if an invalid vote has been found. + /// It is possible that the state has been updated even if an error + /// was returned. pub(crate) async fn insert_votes( &self, votes: impl Iterator>>, diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index 06b00f12..7d53b3a5 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -37,6 +37,7 @@ pub(crate) mod testonly; mod tests; impl Capability { + /// Converts capability to `mux::CapabilityId`. pub(crate) fn id(self) -> mux::CapabilityId { self as mux::CapabilityId } diff --git a/node/actors/network/src/rpc/push_batch_votes.rs b/node/actors/network/src/rpc/push_batch_votes.rs index 5ffa2866..ad343123 100644 --- a/node/actors/network/src/rpc/push_batch_votes.rs +++ b/node/actors/network/src/rpc/push_batch_votes.rs @@ -20,16 +20,20 @@ impl super::Rpc for Rpc { /// Signed batch message that the receiving peer should process. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Req { + /// Whether the client would like to receive a snapshot of all server's votes + /// in response. pub(crate) want_snapshot: Option, + /// New votes that server might be not aware of. pub(crate) votes: Vec>>, } pub(crate) struct Resp { - // Empty if `req.want_snapshot` was not set. + /// Snapshot of all server's votes (if requested by the client). pub(crate) votes: Vec>>, } impl Req { + /// Getter for `want_snapshot`. pub(crate) fn want_snapshot(&self) -> bool { self.want_snapshot.unwrap_or(false) } From f6fb67a3845da8e7508d0c4ce6f5852e6c7f6184 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 9 Aug 2024 11:16:20 +0200 Subject: [PATCH 19/31] log instead of disconnect --- node/actors/network/src/gossip/runner.rs | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index 35dfd494..e151ebd9 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -67,10 +67,19 @@ impl rpc::Handler for &PushServer<'_> { req: rpc::push_batch_votes::Req, ) -> anyhow::Result { let want_snapshot = req.want_snapshot(); - self.net + if let Err(err) = self + .net .attestation_state .insert_votes(req.votes.into_iter()) - .await?; + .await + .context("insert_votes()") + { + // Attestation feature is still evolving, so for forward + // compatibility we just ignore any invalid data. + // Once stabilized we will drop the connection instead of + // logging the error. + tracing::warn!("{err:#}"); + } Ok(rpc::push_batch_votes::Resp { votes: if want_snapshot { self.net.attestation_state.votes() @@ -218,10 +227,18 @@ impl Network { req.want_snapshot(), "expected empty response, but votes were returned" ); - self.attestation_state + if let Err(err) = self + .attestation_state .insert_votes(resp.votes.into_iter()) .await - .context("insert_votes")?; + .context("insert_votes") + { + // Attestation feature is still evolving, so for forward + // compatibility we just ignore any invalid data. + // Once stabilized we will drop the connection instead of + // logging the error. + tracing::warn!("{err:#}"); + } } } }); From d56cb00f3b9b1eb63d88328ad82c96fc7e523365 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 9 Aug 2024 11:51:05 +0200 Subject: [PATCH 20/31] adjusted metrics --- .../network/src/gossip/attestation/metrics.rs | 19 ++++++++++ .../network/src/gossip/attestation/mod.rs | 15 ++++++++ node/actors/network/src/gossip/metrics.rs | 37 ------------------- node/actors/network/src/gossip/mod.rs | 1 - 4 files changed, 34 insertions(+), 38 deletions(-) create mode 100644 node/actors/network/src/gossip/attestation/metrics.rs delete mode 100644 node/actors/network/src/gossip/metrics.rs diff --git a/node/actors/network/src/gossip/attestation/metrics.rs b/node/actors/network/src/gossip/attestation/metrics.rs new file mode 100644 index 00000000..4e5137da --- /dev/null +++ b/node/actors/network/src/gossip/attestation/metrics.rs @@ -0,0 +1,19 @@ +/// Metrics related to the gossiping of L1 batch votes. +#[derive(Debug, vise::Metrics)] +#[metrics(prefix = "network_gossip_attestation")] +pub(crate) struct Metrics { + /// Batch to be attested. + pub(crate) batch_number: vise::Gauge, + + /// Number of members in the attester committee. + pub(crate) committee_size: vise::Gauge, + + /// Number of votes collected for the current batch. + pub(crate) votes_collected: vise::Gauge, + + /// Weight percentage (in range [0,1]) of votes collected for the current batch. + pub(crate) weight_collected: vise::Gauge, +} + +#[vise::register] +pub(super) static METRICS: vise::Global = vise::Global::new(); diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 82c00d4b..e8fd6c87 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -5,6 +5,7 @@ use std::{cmp::Ordering, collections::HashSet, fmt, sync::Arc}; use zksync_concurrency::{ctx, sync}; use zksync_consensus_roles::attester; +mod metrics; #[cfg(test)] mod tests; @@ -201,6 +202,10 @@ impl StateWatch { let before = state.weight; let res = state.insert_votes(votes); if state.weight > before { + metrics::METRICS.votes_collected.set(state.votes.len()); + metrics::METRICS + .weight_collected + .set(state.weight as f64 / state.config.committee.total_weight() as f64); locked.send_replace(Some(state)); } res @@ -273,6 +278,16 @@ impl StateWatch { new.insert_vote(Arc::new(vote)).unwrap(); } } + metrics::METRICS + .batch_number + .set(new.config.batch_to_attest.number.0); + metrics::METRICS + .committee_size + .set(new.config.committee.len()); + metrics::METRICS.votes_collected.set(new.votes.len()); + metrics::METRICS + .weight_collected + .set(new.weight as f64 / new.config.committee.total_weight() as f64); locked.send_replace(Some(new)); Ok(()) } diff --git a/node/actors/network/src/gossip/metrics.rs b/node/actors/network/src/gossip/metrics.rs deleted file mode 100644 index d9df9d0a..00000000 --- a/node/actors/network/src/gossip/metrics.rs +++ /dev/null @@ -1,37 +0,0 @@ -/// Metrics related to the gossiping of L1 batch votes. -#[derive(Debug, vise::Metrics)] -#[metrics(prefix = "network_gossip_batch_votes")] -pub(crate) struct BatchVotesMetrics { - /// Number of members in the attester committee. - pub(crate) committee_size: vise::Gauge, - - /// Number of votes added to the tally. - /// - /// Its rate of change should correlate with the attester committee size, - /// save for any new joiner casting their historic votes in a burst. - pub(crate) votes_added: vise::Counter, - - /// Weight of votes added to the tally normalized by the total committee weight. - /// - /// Its rate of change should correlate with the attester committee weight and batch production rate, - /// that is, it should go up up by 1.0 with each new batch if everyone attests. - pub(crate) weight_added: vise::Counter, - - /// The minimum batch number we still expect votes for. - /// - /// This should go up as the main node indicates the finalisation of batches, - /// or as soon as batch QCs are found and persisted. - pub(crate) min_batch_number: vise::Gauge, - - /// Batch number in the last vote added to the register. - /// - /// This should go up as L1 batches are created, save for any temporary - /// outlier from lagging attesters or ones sending votes far in the future. - pub(crate) last_added_vote_batch_number: vise::Gauge, - - /// Batch number of the last batch signed by this attester. - pub(crate) last_signed_batch_number: vise::Gauge, -} - -#[vise::register] -pub(super) static BATCH_VOTES_METRICS: vise::Global = vise::Global::new(); diff --git a/node/actors/network/src/gossip/mod.rs b/node/actors/network/src/gossip/mod.rs index b9f7f124..aab2eacd 100644 --- a/node/actors/network/src/gossip/mod.rs +++ b/node/actors/network/src/gossip/mod.rs @@ -26,7 +26,6 @@ pub mod attestation; mod fetch; mod handshake; pub mod loadtest; -mod metrics; mod runner; #[cfg(test)] mod testonly; From 3bb3e190189dfdcf32e231b06087cec733dee676 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 9 Aug 2024 11:52:52 +0200 Subject: [PATCH 21/31] clippy --- node/actors/network/src/gossip/attestation/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index e8fd6c87..a7599b64 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -203,6 +203,7 @@ impl StateWatch { let res = state.insert_votes(votes); if state.weight > before { metrics::METRICS.votes_collected.set(state.votes.len()); + #[allow(clippy::float_arithmetic)] metrics::METRICS .weight_collected .set(state.weight as f64 / state.config.committee.total_weight() as f64); @@ -285,6 +286,7 @@ impl StateWatch { .committee_size .set(new.config.committee.len()); metrics::METRICS.votes_collected.set(new.votes.len()); + #[allow(clippy::float_arithmetic)] metrics::METRICS .weight_collected .set(new.weight as f64 / new.config.committee.total_weight() as f64); From 10f1fab9b794320636dda857346671136df1fe2f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 12 Aug 2024 15:19:00 +0200 Subject: [PATCH 22/31] removed rust-toolchain --- node/actors/network/src/noise/stream.rs | 2 +- rust-toolchain | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 rust-toolchain diff --git a/node/actors/network/src/noise/stream.rs b/node/actors/network/src/noise/stream.rs index 2e9462f9..7e903a2a 100644 --- a/node/actors/network/src/noise/stream.rs +++ b/node/actors/network/src/noise/stream.rs @@ -48,7 +48,7 @@ const MAX_PAYLOAD_LEN: usize = MAX_TRANSPORT_MSG_LEN - AUTHDATA_LEN; /// ++ . /// /// Length of the frame len field. -const LENGTH_FIELD_LEN: usize = std::mem::size_of::(); +const LENGTH_FIELD_LEN: usize = size_of::(); /// Max size of the whole frame (length field + data). const MAX_FRAME_LEN: usize = MAX_TRANSPORT_MSG_LEN + LENGTH_FIELD_LEN; diff --git a/rust-toolchain b/rust-toolchain deleted file mode 100644 index 32a89c96..00000000 --- a/rust-toolchain +++ /dev/null @@ -1 +0,0 @@ -stable-2024-06-13 \ No newline at end of file From 92095f1cb36e1afba3e242d911072a9ed04f91ec Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 13 Aug 2024 12:04:12 +0200 Subject: [PATCH 23/31] applied some comments --- node/actors/executor/src/attestation.rs | 58 --------- node/actors/executor/src/lib.rs | 11 +- node/actors/executor/src/tests.rs | 101 +-------------- .../network/src/gossip/attestation/mod.rs | 116 +++++++++--------- .../network/src/gossip/attestation/tests.rs | 71 +++++------ node/actors/network/src/gossip/mod.rs | 10 +- node/actors/network/src/gossip/runner.rs | 18 ++- node/actors/network/src/gossip/tests/mod.rs | 15 +-- node/actors/network/src/lib.rs | 4 +- node/actors/network/src/proto/gossip.proto | 4 +- .../network/src/rpc/push_batch_votes.rs | 20 +-- node/actors/network/src/testonly.rs | 8 +- node/tools/src/config.rs | 6 +- 13 files changed, 146 insertions(+), 296 deletions(-) diff --git a/node/actors/executor/src/attestation.rs b/node/actors/executor/src/attestation.rs index c1e984b8..b3e53def 100644 --- a/node/actors/executor/src/attestation.rs +++ b/node/actors/executor/src/attestation.rs @@ -3,61 +3,3 @@ use anyhow::Context as _; use std::sync::Arc; use zksync_concurrency::{ctx, time}; pub use zksync_consensus_network::gossip::attestation::*; - -/// An interface which is used by attesters and nodes collecting votes over gossip to determine -/// which is the next batch they are all supposed to be voting on, according to the main node. -/// -/// This is a convenience interface to be used with the [AttestationStatusRunner]. -#[async_trait::async_trait] -pub trait Client: 'static + Send + Sync { - /// Get the next batch number for which the main node expects a batch QC to be formed. - /// - /// The API might return an error while genesis is being created, which we represent with `None` - /// here and mean that we'll have to try again later. - async fn config(&self, ctx: &ctx::Ctx) -> ctx::Result>; -} - -/// Use an [Client] to periodically poll the main node and update the [AttestationStatusWatch]. -/// -/// This is provided for convenience. -pub struct Runner { - /// state - pub state: Arc, - /// client - pub client: Box, - /// poll interval - pub poll_interval: time::Duration, -} - -impl Runner { - /// Run the poll loop. - pub async fn run(mut self, ctx: &ctx::Ctx) -> anyhow::Result<()> { - match self.poll_forever(ctx).await { - Ok(()) | Err(ctx::Error::Canceled(_)) => Ok(()), - Err(ctx::Error::Internal(err)) => Err(err), - } - } - - /// Poll the client forever in a loop or until canceled. - async fn poll_forever(&mut self, ctx: &ctx::Ctx) -> ctx::Result<()> { - loop { - match self.client.config(ctx).await { - Ok(Some(config)) => { - self.state - .update_config(config) - .await - .context("update_config")?; - } - Ok(None) => tracing::debug!("waiting for attestation config..."), - Err(error) => tracing::error!( - ?error, - "failed to poll attestation config, retrying later..." - ), - } - if let Err(ctx::Canceled) = ctx.sleep(self.poll_interval).await { - return Ok(()); - } - ctx.sleep(self.poll_interval).await?; - } - } -} diff --git a/node/actors/executor/src/lib.rs b/node/actors/executor/src/lib.rs index def48817..14219e8e 100644 --- a/node/actors/executor/src/lib.rs +++ b/node/actors/executor/src/lib.rs @@ -9,13 +9,13 @@ use std::{ }; use zksync_concurrency::{ctx, limiter, net, scope, time}; use zksync_consensus_bft as bft; -use zksync_consensus_network::{self as network}; +use zksync_consensus_network as network; use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BatchStore, BlockStore, ReplicaStore}; use zksync_consensus_utils::pipe; use zksync_protobuf::kB; +pub use network::gossip::attestation; -pub mod attestation; mod io; #[cfg(test)] mod tests; @@ -91,8 +91,9 @@ pub struct Executor { pub batch_store: Arc, /// Validator-specific node data. pub validator: Option, - /// Status showing where the main node wants attester to cast their votes. - pub attestation_state: Arc, + /// Attestation controller. Caller should actively configure the batch + /// for which the attestation votes should be collected. + pub attestation: Arc, } impl Executor { @@ -136,7 +137,7 @@ impl Executor { self.block_store.clone(), self.batch_store.clone(), network_actor_pipe, - self.attestation_state, + self.attestation, ); net.register_metrics(); s.spawn(async { runner.run(ctx).await.context("Network stopped") }); diff --git a/node/actors/executor/src/tests.rs b/node/actors/executor/src/tests.rs index c16fea9b..4e88f5ef 100644 --- a/node/actors/executor/src/tests.rs +++ b/node/actors/executor/src/tests.rs @@ -30,8 +30,8 @@ fn config(cfg: &network::Config) -> Config { /// The test executors below are not running with attesters, so we just create an [AttestationStatusWatch] /// that will never be updated. -fn never_attest() -> Arc { - attestation::StateWatch::new(None).into() +fn never_attest() -> Arc { + attestation::Controller::new(None).into() } fn validator( @@ -49,7 +49,7 @@ fn validator( replica_store: Box::new(replica_store), payload_manager: Box::new(bft::testonly::RandomPayload(1000)), }), - attestation_state: never_attest(), + attestation: never_attest(), } } @@ -63,7 +63,7 @@ fn fullnode( block_store, batch_store, validator: None, - attestation_state: never_attest(), + attestation: never_attest(), } } @@ -311,96 +311,3 @@ async fn test_validator_syncing_from_fullnode() { .await .unwrap(); } - -/* -/// Test that the AttestationStatusRunner initialises and then polls the status. -#[tokio::test] -async fn test_attestation_status_runner() { - abort_on_panic(); - let _guard = zksync_concurrency::testonly::set_timeout(time::Duration::seconds(5)); - let ctx = &ctx::test_root(&ctx::AffineClock::new(10.0)); - let rng = &mut ctx.rng(); - - let genesis: validator::Genesis = rng.gen(); - - #[derive(Clone)] - struct MockAttestationStatus { - genesis: Arc>, - batch_number: Arc, - } - - #[async_trait::async_trait] - impl attestation::Client for MockAttestationStatus { - async fn config( - &self, - ctx: &ctx::Ctx, - ) -> ctx::Result> { - let curr = self - .batch_number - .fetch_add(1u64, std::sync::atomic::Ordering::Relaxed); - if curr == 0 { - // Return None initially to see that the runner will deal with it. - Ok(None) - } else { - let genesis = *self.genesis.lock().unwrap().clone(); - // The first actual result will be 1 on the 2nd poll. - Ok(Some(attestation::Config { - batch_to_attest: attester::Batch { - genesis: genesis.hash(), - number: attester::BatchNumber(curr), - hash: ctx.rng().gen(), - }, - committee: genesis.attesters.clone().unwrap() - })) - } - } - } - - let res = scope::run!(ctx, |ctx, s| async { - let client = MockAttestationStatus { - genesis: Arc::new(Mutex::new(genesis)), - batch_number: Arc::new(AtomicU64::default()), - }; - let (status, runner) = attestation::Runner::init( - ctx, - Box::new(client.clone()), - time::Duration::milliseconds(100), - genesis, - ) - .await - .unwrap(); - - let mut recv_status = status.subscribe(); - recv_status.mark_changed(); - - // Check that the value has *not* been initialised to a non-default value. - { - let status = sync::changed(ctx, &mut recv_status).await?; - assert!(status.is_none()); - } - // Now start polling for new values. Starting in the foreground because we want it to fail in the end. - s.spawn(runner.run(ctx)); - // Check that polling sets the value. - { - let status = sync::changed(ctx, &mut recv_status).await?; - assert!(status.next_batch_to_attest.is_some()); - assert_eq!(status.next_batch_to_attest.unwrap().0, 1); - } - // Change the genesis returned by the client. It should cause the scope to fail. - { - let mut genesis = client.genesis.lock().unwrap(); - *genesis = rng.gen(); - } - Ok(()) - }) - .await; - - match res { - Ok(()) => panic!("expected to fail when the genesis changed"), - Err(e) => assert!( - e.to_string().contains("genesis changed"), - "only expect failures due to genesis change; got: {e}" - ), - } -} -*/ diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index a7599b64..733604d9 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -1,7 +1,7 @@ //! Attestation. use crate::watch::Watch; use anyhow::Context as _; -use std::{cmp::Ordering, collections::HashSet, fmt, sync::Arc}; +use std::{collections::HashSet, fmt, sync::Arc}; use zksync_concurrency::{ctx, sync}; use zksync_consensus_roles::attester; @@ -15,16 +15,12 @@ pub struct Config { /// Batch to attest. pub batch_to_attest: attester::Batch, /// Committee that should attest the batch. - /// NOTE: the committee is not supposed to change often, - /// so you might want to use `Arc` instead - /// to avoid extra copying. - pub committee: attester::Committee, + pub committee: Arc, } -/// A [Watch] over an [AttestationStatus] which we can use to notify components about -/// changes in the batch number the main node expects attesters to vote on. +// Internal attestation state: config and the set of votes collected so far. #[derive(Clone)] -pub(crate) struct State { +struct State { config: Arc, votes: im::HashMap>>, weight: attester::Weight, @@ -34,64 +30,62 @@ pub(crate) struct State { pub(crate) struct Diff { /// New votes. pub(crate) votes: Vec>>, - /// Whether the config has changed. - pub(crate) config_changed: bool, + /// New config, if changed. + pub(crate) config: Option>, } impl Diff { fn is_empty(&self) -> bool { - self.votes.is_empty() && !self.config_changed + self.votes.is_empty() && !self.config.is_none() } } impl State { - /// Returns a set of votes of `self` which are newer than the entries in `old`. + /// Returns a diff between `self` state and `old` state. + /// Diff contains votes which are present is `self`, but not in `old`. fn diff(&self, old: &Option) -> Diff { let Some(old) = old.as_ref() else { return Diff { - config_changed: true, + config: Some(self.config.clone()), votes: self.votes.values().cloned().collect(), }; }; - match self + if self .config .batch_to_attest - .number - .cmp(&old.config.batch_to_attest.number) + .number != old.config.batch_to_attest.number { - Ordering::Less => Diff { - config_changed: true, - votes: vec![], - }, - Ordering::Greater => Diff { - config_changed: true, + return Diff { + config: Some(self.config.clone()), votes: self.votes.values().cloned().collect(), - }, - Ordering::Equal => Diff { - config_changed: false, - votes: self - .votes - .iter() - .filter(|(k, _)| !old.votes.contains_key(k)) - .map(|(_, v)| v.clone()) - .collect(), - }, + }; + } + + Diff { + config: None, + votes: self + .votes + .iter() + .filter(|(k, _)| !old.votes.contains_key(k)) + .map(|(_, v)| v.clone()) + .collect(), } } /// Verifies and adds a vote. - /// Noop if vote is not signed by a committee memver or already inserted. + /// Noop if vote is not signed by a committee member or already inserted. /// Returns an error if genesis doesn't match or the signature is invalid. fn insert_vote(&mut self, vote: Arc>) -> anyhow::Result<()> { anyhow::ensure!( vote.msg.genesis == self.config.batch_to_attest.genesis, "Genesis mismatch" ); - if vote.msg != self.config.batch_to_attest { + if vote.msg.number != self.config.batch_to_attest.number { return Ok(()); } + anyhow::ensure!(vote.msg.hash == self.config.batch_to_attest.hash, "batch hash mismatch"); let Some(weight) = self.config.committee.weight(&vote.key) else { - return Ok(()); + anyhow::bail!("received vote signed by an inactive attester: {:?}",vote.key); }; if self.votes.contains_key(&vote.key) { return Ok(()); @@ -136,7 +130,7 @@ impl State { } } -/// Received of state diffs. +/// Receiver of state diffs. pub(crate) struct DiffReceiver { prev: Option, recv: sync::watch::Receiver>, @@ -158,20 +152,33 @@ impl DiffReceiver { } } -/// Watch of the attestation state. -pub struct StateWatch { +/// `Controller` manages the attestation state. +/// It maintains a set of votes matching the attestation config. +/// It allows for +/// * adding votes to the state +/// * subscribing to the vote set changes +/// * waiting for the certificate to be collected +/// +/// It also keeps an attester key used to sign the batch vote, +/// whenever it belongs the current attester committee. +/// Signing happens automatically whenever the committee is updated. +pub struct Controller { key: Option, state: Watch>, } -impl fmt::Debug for StateWatch { +impl fmt::Debug for Controller { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.debug_struct("StateWatch").finish_non_exhaustive() + fmt + .debug_struct("StateWatch") + .field("key", &self.key) + .finish_non_exhaustive() } } -impl StateWatch { +impl Controller { /// Constructs AttestationStatusWatch. + /// `key` will be used for automatically signing votes. pub fn new(key: Option) -> Self { Self { key, @@ -189,8 +196,8 @@ impl StateWatch { /// Inserts votes to the state. /// Irrelevant votes are silently ignored. /// Returns an error if an invalid vote has been found. - /// It is possible that the state has been updated even if an error - /// was returned. + /// It is possible that some votes have been added to the state + /// even if eventually an error was returned. pub(crate) async fn insert_votes( &self, votes: impl Iterator>>, @@ -212,18 +219,17 @@ impl StateWatch { res } - /// All votes kept in the state. - pub(crate) fn votes(&self) -> Vec>> { - self.state - .subscribe() - .borrow() - .iter() - .flat_map(|s| s.votes.values().cloned()) - .collect() + /// Returns votes matching the `want` batch. + pub(crate) fn votes(&self, want: &attester::Batch) -> Vec>> { + let state = self.state.subscribe(); + let state = state.borrow(); + let Some(state) = &*state else { return vec![] }; + if &state.config.batch_to_attest != want { return vec![] } + state.votes.values().cloned().collect() } /// Waits for the certificate for a batch with the given number to be collected. - /// Returns None iff attestation already skipped to collecting certificate some later batch. + /// Returns None iff attestation already skipped to collecting certificate for some later batch. pub async fn wait_for_qc( &self, ctx: &ctx::Ctx, @@ -251,11 +257,11 @@ impl StateWatch { /// Updates the attestation config. /// Clears the votes collected for the previous config. /// Batch number has to increase with each update. - pub async fn update_config(&self, config: Config) -> anyhow::Result<()> { + pub async fn update_config(&self, config: Arc) -> anyhow::Result<()> { let locked = self.state.lock().await; let old = locked.borrow().clone(); if let Some(old) = old.as_ref() { - if *old.config == config { + if *old.config == *config { return Ok(()); } anyhow::ensure!( @@ -268,7 +274,7 @@ impl StateWatch { ); } let mut new = State { - config: Arc::new(config), + config, votes: im::HashMap::new(), weight: 0, }; diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index e26b0119..affea259 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -19,12 +19,12 @@ async fn test_insert_votes() { let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); - let state = StateWatch::new(None); + let ctrl = Controller::new(None); let genesis: attester::GenesisHash = rng.gen(); let first: attester::BatchNumber = rng.gen(); for i in 0..3 { let keys: Vec = (0..8).map(|_| rng.gen()).collect(); - let config = Config { + let config = Arc::new(Config { batch_to_attest: attester::Batch { genesis, number: first + i, @@ -34,13 +34,14 @@ async fn test_insert_votes() { key: k.public(), weight: 1250, })) - .unwrap(), - }; - state.update_config(config.clone()).await.unwrap(); - assert_eq!(Votes::from([]), state.votes().into()); - let mut recv = state.subscribe(); + .unwrap().into(), + }); + let ctrl_votes = ||Votes::from(ctrl.votes(&config.batch_to_attest)); + ctrl.update_config(config.clone()).await.unwrap(); + assert_eq!(Votes::from([]), ctrl_votes()); + let mut recv = ctrl.subscribe(); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert!(diff.config_changed); + assert_eq!(diff.config.as_ref(), Some(&config)); assert_eq!(Votes::default(), diff.votes.into()); let all_votes: Vec = keys @@ -49,53 +50,53 @@ async fn test_insert_votes() { .collect(); // Initial votes. - state + ctrl .insert_votes(all_votes[0..3].iter().cloned()) .await .unwrap(); assert_eq!( Votes::from(all_votes[0..3].iter().cloned()), - state.votes().into() + ctrl_votes() ); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert!(!diff.config_changed); + assert!(diff.config.is_none()); assert_eq!( Votes::from(all_votes[0..3].iter().cloned()), diff.votes.into() ); // Adding votes gradually. - state + ctrl .insert_votes(all_votes[3..5].iter().cloned()) .await .unwrap(); - state + ctrl .insert_votes(all_votes[5..7].iter().cloned()) .await .unwrap(); assert_eq!( Votes::from(all_votes[0..7].iter().cloned()), - state.votes().into() + ctrl_votes() ); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert!(!diff.config_changed); + assert!(diff.config.is_none()); assert_eq!( Votes::from(all_votes[3..7].iter().cloned()), diff.votes.into() ); // Readding already inserded votes (noop). - state + ctrl .insert_votes(all_votes[2..6].iter().cloned()) .await .unwrap(); assert_eq!( Votes::from(all_votes[0..7].iter().cloned()), - state.votes().into() + ctrl_votes() ); // Adding votes out of committee (noop). - state + ctrl .insert_votes((0..3).map(|_| { let k: attester::SecretKey = rng.gen(); k.sign_msg(config.batch_to_attest.clone()).into() @@ -104,11 +105,11 @@ async fn test_insert_votes() { .unwrap(); assert_eq!( Votes::from(all_votes[0..7].iter().cloned()), - state.votes().into() + ctrl_votes() ); // Adding votes for different batch (noop). - state + ctrl .insert_votes((0..3).map(|_| { let k: attester::SecretKey = rng.gen(); k.sign_msg(attester::Batch { @@ -122,29 +123,29 @@ async fn test_insert_votes() { .unwrap(); assert_eq!( Votes::from(all_votes[0..7].iter().cloned()), - state.votes().into() + ctrl_votes() ); // Adding incorrect votes (error). let mut bad_vote = (*all_votes[7]).clone(); bad_vote.sig = rng.gen(); - assert!(state + assert!(ctrl .insert_votes([bad_vote.into()].into_iter()) .await .is_err()); assert_eq!( Votes::from(all_votes[0..7].iter().cloned()), - state.votes().into() + ctrl_votes() ); // Add the last vote mixed with already added votes. - state + ctrl .insert_votes(all_votes[5..].iter().cloned()) .await .unwrap(); - assert_eq!(Votes::from(all_votes.clone()), state.votes().into()); + assert_eq!(Votes::from(all_votes.clone()), ctrl_votes()); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert!(!diff.config_changed); + assert!(diff.config.is_none()); assert_eq!( Votes::from(all_votes[7..].iter().cloned()), diff.votes.into() @@ -158,7 +159,7 @@ async fn test_wait_for_qc() { let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); - let state = StateWatch::new(None); + let ctrl = Controller::new(None); let genesis: attester::GenesisHash = rng.gen(); let first: attester::BatchNumber = rng.gen(); @@ -166,7 +167,7 @@ async fn test_wait_for_qc() { tracing::info!("iteration {i}"); let committee_size = rng.gen_range(1..20); let keys: Vec = (0..committee_size).map(|_| rng.gen()).collect(); - let config = Config { + let config = Arc::new(Config { batch_to_attest: attester::Batch { genesis, number: first + i, @@ -176,25 +177,25 @@ async fn test_wait_for_qc() { key: k.public(), weight: rng.gen_range(1..=100), })) - .unwrap(), - }; + .unwrap().into(), + }); let mut all_votes: Vec = keys .iter() .map(|k| k.sign_msg(config.batch_to_attest.clone()).into()) .collect(); all_votes.shuffle(rng); - state.update_config(config.clone()).await.unwrap(); + ctrl.update_config(config.clone()).await.unwrap(); loop { let end = rng.gen_range(0..=committee_size); tracing::info!("end = {end}"); - state + ctrl .insert_votes(all_votes[..end].iter().cloned()) .await .unwrap(); // Waiting for the previous qc should immediately return None. assert_eq!( None, - state + ctrl .wait_for_qc(ctx, config.batch_to_attest.number.prev().unwrap()) .await .unwrap() @@ -204,7 +205,7 @@ async fn test_wait_for_qc() { .weight_of_keys(all_votes[..end].iter().map(|v| &v.key)) >= config.committee.threshold() { - let qc = state + let qc = ctrl .wait_for_qc(ctx, config.batch_to_attest.number) .await .unwrap() @@ -215,7 +216,7 @@ async fn test_wait_for_qc() { } assert_eq!( None, - state.state.subscribe().borrow().as_ref().unwrap().qc() + ctrl.state.subscribe().borrow().as_ref().unwrap().qc() ); } } diff --git a/node/actors/network/src/gossip/mod.rs b/node/actors/network/src/gossip/mod.rs index aab2eacd..52512425 100644 --- a/node/actors/network/src/gossip/mod.rs +++ b/node/actors/network/src/gossip/mod.rs @@ -55,8 +55,10 @@ pub(crate) struct Network { pub(crate) fetch_queue: fetch::Queue, /// TESTONLY: how many time push_validator_addrs rpc was called by the peers. pub(crate) push_validator_addrs_calls: AtomicUsize, - /// Shared watch over the current attestation status as indicated by the main node. - pub(crate) attestation_state: Arc, + /// Attestation controller, maintaining a set of batch votes. + /// Gossip network exchanges the votes with peers. + /// The batch for which the votes are collected is configured externally. + pub(crate) attestation: Arc, } impl Network { @@ -66,7 +68,7 @@ impl Network { block_store: Arc, batch_store: Arc, sender: channel::UnboundedSender, - attestation_state: Arc, + attestation: Arc, ) -> Arc { Arc::new(Self { sender, @@ -81,7 +83,7 @@ impl Network { block_store, batch_store, push_validator_addrs_calls: 0.into(), - attestation_state, + attestation, }) } diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index fb8a8b61..0d675bd3 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -66,10 +66,9 @@ impl rpc::Handler for &PushServer<'_> { _ctx: &ctx::Ctx, req: rpc::push_batch_votes::Req, ) -> anyhow::Result { - let want_snapshot = req.want_snapshot(); if let Err(err) = self .net - .attestation_state + .attestation .insert_votes(req.votes.into_iter()) .await .context("insert_votes()") @@ -81,10 +80,9 @@ impl rpc::Handler for &PushServer<'_> { tracing::warn!("{err:#}"); } Ok(rpc::push_batch_votes::Resp { - votes: if want_snapshot { - self.net.attestation_state.votes() - } else { - vec![] + votes: match req.want_votes_for.as_ref() { + Some(batch) => self.net.attestation.votes(batch), + None => vec![], }, }) } @@ -209,13 +207,13 @@ impl Network { s.spawn::<()>(async { let push_batch_votes_client = push_batch_votes_client; // Subscribe to what we know about the state of the whole network. - let mut recv = self.attestation_state.subscribe(); + let mut recv = self.attestation.subscribe(); loop { let diff = recv.wait_for_diff(ctx).await?; let req = rpc::push_batch_votes::Req { // If the config has changed, we need to re-request all the votes // from peer that we might have ignored earlier. - want_snapshot: Some(diff.config_changed), + want_votes_for: diff.config.as_ref().map(|c|c.batch_to_attest.clone()), votes: diff.votes, }; // NOTE: The response should be non-empty only iff we requested a snapshot. @@ -224,11 +222,11 @@ impl Network { let resp = push_batch_votes_client.call(ctx, &req, 100 * kB).await?; if !resp.votes.is_empty() { anyhow::ensure!( - req.want_snapshot(), + req.want_votes_for.is_some(), "expected empty response, but votes were returned" ); if let Err(err) = self - .attestation_state + .attestation .insert_votes(resp.votes.into_iter()) .await .context("insert_votes") diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index dbb4e7e6..a9738ead 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -497,7 +497,7 @@ async fn test_batch_votes_propagation() { // Fixed attestation schedule. let first: attester::BatchNumber = rng.gen(); let schedule: Vec<_> = (0..10) - .map(|r| attestation::Config { + .map(|r| Arc::new(attestation::Config { batch_to_attest: attester::Batch { genesis: setup.genesis.hash(), number: first + r, @@ -512,8 +512,9 @@ async fn test_batch_votes_propagation() { weight: rng.gen_range(5..10), })) .unwrap() + .into() }, - }) + })) .collect(); // Round of the schedule that nodes should collect the votes for. @@ -532,7 +533,7 @@ async fn test_batch_votes_propagation() { cfg: cfg.clone(), block_store: store.blocks.clone(), batch_store: store.batches.clone(), - attestation_state: attestation::StateWatch::new(Some( + attestation: attestation::Controller::new(Some( setup.attester_keys[i].clone(), )) .into(), @@ -552,13 +553,13 @@ async fn test_batch_votes_propagation() { let Some(cfg) = schedule.get(*r) else { return Ok(()); }; - let attestation_state = node.net.gossip.attestation_state.clone(); - attestation_state.update_config(cfg.clone()).await.unwrap(); + let attestation = node.net.gossip.attestation.clone(); + attestation.update_config(cfg.clone()).await.unwrap(); // Wait for the certificate in the background. s.spawn_bg(async { let r = r; - let attestation_state = attestation_state; - let Ok(Some(qc)) = attestation_state + let attestation = attestation; + let Ok(Some(qc)) = attestation .wait_for_qc(ctx, cfg.batch_to_attest.number) .await else { diff --git a/node/actors/network/src/lib.rs b/node/actors/network/src/lib.rs index 3cc45458..88bb68ec 100644 --- a/node/actors/network/src/lib.rs +++ b/node/actors/network/src/lib.rs @@ -57,10 +57,10 @@ impl Network { block_store: Arc, batch_store: Arc, pipe: ActorPipe, - attestation_state: Arc, + attestation: Arc, ) -> (Arc, Runner) { let gossip = - gossip::Network::new(cfg, block_store, batch_store, pipe.send, attestation_state); + gossip::Network::new(cfg, block_store, batch_store, pipe.send, attestation); let consensus = consensus::Network::new(gossip.clone()); let net = Arc::new(Self { gossip, consensus }); ( diff --git a/node/actors/network/src/proto/gossip.proto b/node/actors/network/src/proto/gossip.proto index 90ebc1b9..4c4d6ed2 100644 --- a/node/actors/network/src/proto/gossip.proto +++ b/node/actors/network/src/proto/gossip.proto @@ -19,8 +19,8 @@ message PushValidatorAddrs { } message PushBatchVotes { - // Whether peer should send back snapshot of its votes. - optional bool want_snapshot = 2; // optional + // Requesting the peer to respond with votes for the batch. + optional roles.attester.Batch want_votes_for = 2; // optional // Signed roles.validator.Msg.votes repeated roles.attester.Signed votes = 1; } diff --git a/node/actors/network/src/rpc/push_batch_votes.rs b/node/actors/network/src/rpc/push_batch_votes.rs index ad343123..5017edc3 100644 --- a/node/actors/network/src/rpc/push_batch_votes.rs +++ b/node/actors/network/src/rpc/push_batch_votes.rs @@ -4,7 +4,7 @@ use crate::proto::gossip as proto; use anyhow::Context as _; use std::sync::Arc; use zksync_consensus_roles::attester; -use zksync_protobuf::ProtoFmt; +use zksync_protobuf::{read_optional,ProtoFmt}; /// RPC pushing fresh batch votes. pub(crate) struct Rpc; @@ -20,25 +20,17 @@ impl super::Rpc for Rpc { /// Signed batch message that the receiving peer should process. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Req { - /// Whether the client would like to receive a snapshot of all server's votes - /// in response. - pub(crate) want_snapshot: Option, + // Requesting the peer to respond with votes for the batch. + pub(crate) want_votes_for: Option, /// New votes that server might be not aware of. pub(crate) votes: Vec>>, } pub(crate) struct Resp { - /// Snapshot of all server's votes (if requested by the client). + /// Votes requested by the peer. pub(crate) votes: Vec>>, } -impl Req { - /// Getter for `want_snapshot`. - pub(crate) fn want_snapshot(&self) -> bool { - self.want_snapshot.unwrap_or(false) - } -} - impl ProtoFmt for Req { type Proto = proto::PushBatchVotes; @@ -50,14 +42,14 @@ impl ProtoFmt for Req { )); } Ok(Self { - want_snapshot: r.want_snapshot, + want_votes_for: read_optional(&r.want_votes_for).context("want_votes_for")?, votes, }) } fn build(&self) -> Self::Proto { Self::Proto { - want_snapshot: self.want_snapshot, + want_votes_for: self.want_votes_for.as_ref().map(ProtoFmt::build), votes: self .votes .iter() diff --git a/node/actors/network/src/testonly.rs b/node/actors/network/src/testonly.rs index 8597f252..bef710b1 100644 --- a/node/actors/network/src/testonly.rs +++ b/node/actors/network/src/testonly.rs @@ -185,8 +185,8 @@ pub struct InstanceConfig { pub block_store: Arc, /// batch_store pub batch_store: Arc, - /// attestation_state - pub attestation_state: Arc, + /// Attestation controller. + pub attestation: Arc, } impl Instance { @@ -200,7 +200,7 @@ impl Instance { cfg, block_store, batch_store, - attestation_state: attestation::StateWatch::new(None).into(), + attestation: attestation::Controller::new(None).into(), }) } @@ -212,7 +212,7 @@ impl Instance { cfg.block_store.clone(), cfg.batch_store.clone(), actor_pipe, - cfg.attestation_state, + cfg.attestation, ); let (terminate_send, terminate_recv) = channel::bounded(1); ( diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index e37c24b9..e640f9af 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -265,8 +265,8 @@ impl Configs { let store = TestMemoryStorage::new(ctx, &self.app.genesis).await; // We don't have an API to poll in this setup, we can only create a local store based attestation client. - let attestation_state = - Arc::new(attestation::StateWatch::new(self.app.attester_key.clone())); + let attestation = + Arc::new(attestation::Controller::new(self.app.attester_key.clone())); let runner = store.runner; let e = executor::Executor { @@ -305,7 +305,7 @@ impl Configs { self.app.max_payload_size, )), }), - attestation_state, + attestation, }; Ok((e, runner)) } From f5a1337036177713d86974f8f43a01dee90acdce Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 13 Aug 2024 12:11:28 +0200 Subject: [PATCH 24/31] fixed test --- .../network/src/gossip/attestation/mod.rs | 2 +- .../network/src/gossip/attestation/tests.rs | 19 ++++++++++--------- node/actors/network/src/testonly.rs | 3 +++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 733604d9..5e7f6dd8 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -36,7 +36,7 @@ pub(crate) struct Diff { impl Diff { fn is_empty(&self) -> bool { - self.votes.is_empty() && !self.config.is_none() + self.votes.is_empty() && self.config.is_none() } } diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index affea259..14f18b2d 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -23,6 +23,7 @@ async fn test_insert_votes() { let genesis: attester::GenesisHash = rng.gen(); let first: attester::BatchNumber = rng.gen(); for i in 0..3 { + tracing::info!("iteration {i}"); let keys: Vec = (0..8).map(|_| rng.gen()).collect(); let config = Arc::new(Config { batch_to_attest: attester::Batch { @@ -49,7 +50,7 @@ async fn test_insert_votes() { .map(|k| k.sign_msg(config.batch_to_attest.clone()).into()) .collect(); - // Initial votes. + tracing::info!("Initial votes."); ctrl .insert_votes(all_votes[0..3].iter().cloned()) .await @@ -65,7 +66,7 @@ async fn test_insert_votes() { diff.votes.into() ); - // Adding votes gradually. + tracing::info!("Adding votes gradually."); ctrl .insert_votes(all_votes[3..5].iter().cloned()) .await @@ -85,7 +86,7 @@ async fn test_insert_votes() { diff.votes.into() ); - // Readding already inserded votes (noop). + tracing::info!("Readding already inserded votes (noop)."); ctrl .insert_votes(all_votes[2..6].iter().cloned()) .await @@ -95,20 +96,20 @@ async fn test_insert_votes() { ctrl_votes() ); - // Adding votes out of committee (noop). - ctrl + tracing::info!("Adding votes out of committee (error)."); + assert!(ctrl .insert_votes((0..3).map(|_| { let k: attester::SecretKey = rng.gen(); k.sign_msg(config.batch_to_attest.clone()).into() })) .await - .unwrap(); + .is_err()); assert_eq!( Votes::from(all_votes[0..7].iter().cloned()), ctrl_votes() ); - // Adding votes for different batch (noop). + tracing::info!("Adding votes for different batch (noop)."); ctrl .insert_votes((0..3).map(|_| { let k: attester::SecretKey = rng.gen(); @@ -126,7 +127,7 @@ async fn test_insert_votes() { ctrl_votes() ); - // Adding incorrect votes (error). + tracing::info!("Adding incorrect votes (error)."); let mut bad_vote = (*all_votes[7]).clone(); bad_vote.sig = rng.gen(); assert!(ctrl @@ -138,7 +139,7 @@ async fn test_insert_votes() { ctrl_votes() ); - // Add the last vote mixed with already added votes. + tracing::info!("Add the last vote mixed with already added votes."); ctrl .insert_votes(all_votes[5..].iter().cloned()) .await diff --git a/node/actors/network/src/testonly.rs b/node/actors/network/src/testonly.rs index bef710b1..909e7248 100644 --- a/node/actors/network/src/testonly.rs +++ b/node/actors/network/src/testonly.rs @@ -186,6 +186,9 @@ pub struct InstanceConfig { /// batch_store pub batch_store: Arc, /// Attestation controller. + /// It is not configured by default. + /// Attestation tests should configure it and consume + /// the certificates on their own (see `attestation::Controller`). pub attestation: Arc, } From b1b6cbfc8b69f98c36dacfc84e059dcae149d9bd Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 13 Aug 2024 13:30:38 +0200 Subject: [PATCH 25/31] cargo_fmt --- node/actors/executor/src/lib.rs | 3 +- .../network/src/gossip/attestation/mod.rs | 56 +++++++++--- .../network/src/gossip/attestation/tests.rs | 87 +++++++------------ node/actors/network/src/gossip/runner.rs | 2 +- node/actors/network/src/gossip/tests/mod.rs | 44 +++++----- node/actors/network/src/lib.rs | 3 +- .../network/src/rpc/push_batch_votes.rs | 6 +- .../libs/roles/src/attester/messages/batch.rs | 17 ---- node/tools/src/config.rs | 3 +- 9 files changed, 101 insertions(+), 120 deletions(-) diff --git a/node/actors/executor/src/lib.rs b/node/actors/executor/src/lib.rs index 14219e8e..2cd86e34 100644 --- a/node/actors/executor/src/lib.rs +++ b/node/actors/executor/src/lib.rs @@ -2,7 +2,7 @@ use crate::io::Dispatcher; use anyhow::Context as _; use network::http; -pub use network::RpcConfig; +pub use network::{gossip::attestation, RpcConfig}; use std::{ collections::{HashMap, HashSet}, sync::Arc, @@ -14,7 +14,6 @@ use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BatchStore, BlockStore, ReplicaStore}; use zksync_consensus_utils::pipe; use zksync_protobuf::kB; -pub use network::gossip::attestation; mod io; #[cfg(test)] diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 5e7f6dd8..e5954989 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -50,17 +50,13 @@ impl State { votes: self.votes.values().cloned().collect(), }; }; - if self - .config - .batch_to_attest - .number != old.config.batch_to_attest.number - { + if self.config.batch_to_attest.number != old.config.batch_to_attest.number { return Diff { config: Some(self.config.clone()), votes: self.votes.values().cloned().collect(), }; } - + Diff { config: None, votes: self @@ -83,9 +79,15 @@ impl State { if vote.msg.number != self.config.batch_to_attest.number { return Ok(()); } - anyhow::ensure!(vote.msg.hash == self.config.batch_to_attest.hash, "batch hash mismatch"); + anyhow::ensure!( + vote.msg.hash == self.config.batch_to_attest.hash, + "batch hash mismatch" + ); let Some(weight) = self.config.committee.weight(&vote.key) else { - anyhow::bail!("received vote signed by an inactive attester: {:?}",vote.key); + anyhow::bail!( + "received vote signed by an inactive attester: {:?}", + vote.key + ); }; if self.votes.contains_key(&vote.key) { return Ok(()); @@ -158,10 +160,32 @@ impl DiffReceiver { /// * adding votes to the state /// * subscribing to the vote set changes /// * waiting for the certificate to be collected -/// +/// /// It also keeps an attester key used to sign the batch vote, /// whenever it belongs the current attester committee. /// Signing happens automatically whenever the committee is updated. +/// +/// Expected usage: +/// ``` +/// let ctrl = attestation::Controller::new(Some(key)); +/// loop { +/// // Check the global attestation registry. +/// // Compute the next expected batch and the committee that should attest it. +/// ... +/// let config = attestation::Config { +/// batch_to_attest: ..., +/// committee: ..., +/// }; +/// ctrl.update_config(Arc::new(config.clone())).unwrap(); +/// s.spawn(async { +/// if let Some(qc) = ctrl.wait_for_qc(ctx, config.batch_to_attest.number).await?; +/// // Submit the certificate `qc` to the global registry +/// ... +/// }); +/// // Wait for the global registry to include the certificate. +/// ... +/// } +/// ``` pub struct Controller { key: Option, state: Watch>, @@ -169,15 +193,14 @@ pub struct Controller { impl fmt::Debug for Controller { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt - .debug_struct("StateWatch") + fmt.debug_struct("StateWatch") .field("key", &self.key) .finish_non_exhaustive() } } impl Controller { - /// Constructs AttestationStatusWatch. + /// Constructs Controller. /// `key` will be used for automatically signing votes. pub fn new(key: Option) -> Self { Self { @@ -220,11 +243,16 @@ impl Controller { } /// Returns votes matching the `want` batch. - pub(crate) fn votes(&self, want: &attester::Batch) -> Vec>> { + pub(crate) fn votes( + &self, + want: &attester::Batch, + ) -> Vec>> { let state = self.state.subscribe(); let state = state.borrow(); let Some(state) = &*state else { return vec![] }; - if &state.config.batch_to_attest != want { return vec![] } + if &state.config.batch_to_attest != want { + return vec![]; + } state.votes.values().cloned().collect() } diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index 14f18b2d..227dfa22 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -35,9 +35,10 @@ async fn test_insert_votes() { key: k.public(), weight: 1250, })) - .unwrap().into(), + .unwrap() + .into(), }); - let ctrl_votes = ||Votes::from(ctrl.votes(&config.batch_to_attest)); + let ctrl_votes = || Votes::from(ctrl.votes(&config.batch_to_attest)); ctrl.update_config(config.clone()).await.unwrap(); assert_eq!(Votes::from([]), ctrl_votes()); let mut recv = ctrl.subscribe(); @@ -51,14 +52,10 @@ async fn test_insert_votes() { .collect(); tracing::info!("Initial votes."); - ctrl - .insert_votes(all_votes[0..3].iter().cloned()) + ctrl.insert_votes(all_votes[0..3].iter().cloned()) .await .unwrap(); - assert_eq!( - Votes::from(all_votes[0..3].iter().cloned()), - ctrl_votes() - ); + assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), ctrl_votes()); let diff = recv.wait_for_diff(ctx).await.unwrap(); assert!(diff.config.is_none()); assert_eq!( @@ -67,18 +64,13 @@ async fn test_insert_votes() { ); tracing::info!("Adding votes gradually."); - ctrl - .insert_votes(all_votes[3..5].iter().cloned()) + ctrl.insert_votes(all_votes[3..5].iter().cloned()) .await .unwrap(); - ctrl - .insert_votes(all_votes[5..7].iter().cloned()) + ctrl.insert_votes(all_votes[5..7].iter().cloned()) .await .unwrap(); - assert_eq!( - Votes::from(all_votes[0..7].iter().cloned()), - ctrl_votes() - ); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), ctrl_votes()); let diff = recv.wait_for_diff(ctx).await.unwrap(); assert!(diff.config.is_none()); assert_eq!( @@ -87,14 +79,10 @@ async fn test_insert_votes() { ); tracing::info!("Readding already inserded votes (noop)."); - ctrl - .insert_votes(all_votes[2..6].iter().cloned()) + ctrl.insert_votes(all_votes[2..6].iter().cloned()) .await .unwrap(); - assert_eq!( - Votes::from(all_votes[0..7].iter().cloned()), - ctrl_votes() - ); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), ctrl_votes()); tracing::info!("Adding votes out of committee (error)."); assert!(ctrl @@ -104,28 +92,21 @@ async fn test_insert_votes() { })) .await .is_err()); - assert_eq!( - Votes::from(all_votes[0..7].iter().cloned()), - ctrl_votes() - ); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), ctrl_votes()); tracing::info!("Adding votes for different batch (noop)."); - ctrl - .insert_votes((0..3).map(|_| { - let k: attester::SecretKey = rng.gen(); - k.sign_msg(attester::Batch { - genesis: config.batch_to_attest.genesis, - number: rng.gen(), - hash: rng.gen(), - }) - .into() - })) - .await - .unwrap(); - assert_eq!( - Votes::from(all_votes[0..7].iter().cloned()), - ctrl_votes() - ); + ctrl.insert_votes((0..3).map(|_| { + let k: attester::SecretKey = rng.gen(); + k.sign_msg(attester::Batch { + genesis: config.batch_to_attest.genesis, + number: rng.gen(), + hash: rng.gen(), + }) + .into() + })) + .await + .unwrap(); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), ctrl_votes()); tracing::info!("Adding incorrect votes (error)."); let mut bad_vote = (*all_votes[7]).clone(); @@ -134,14 +115,10 @@ async fn test_insert_votes() { .insert_votes([bad_vote.into()].into_iter()) .await .is_err()); - assert_eq!( - Votes::from(all_votes[0..7].iter().cloned()), - ctrl_votes() - ); + assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), ctrl_votes()); tracing::info!("Add the last vote mixed with already added votes."); - ctrl - .insert_votes(all_votes[5..].iter().cloned()) + ctrl.insert_votes(all_votes[5..].iter().cloned()) .await .unwrap(); assert_eq!(Votes::from(all_votes.clone()), ctrl_votes()); @@ -178,7 +155,8 @@ async fn test_wait_for_qc() { key: k.public(), weight: rng.gen_range(1..=100), })) - .unwrap().into(), + .unwrap() + .into(), }); let mut all_votes: Vec = keys .iter() @@ -189,15 +167,13 @@ async fn test_wait_for_qc() { loop { let end = rng.gen_range(0..=committee_size); tracing::info!("end = {end}"); - ctrl - .insert_votes(all_votes[..end].iter().cloned()) + ctrl.insert_votes(all_votes[..end].iter().cloned()) .await .unwrap(); // Waiting for the previous qc should immediately return None. assert_eq!( None, - ctrl - .wait_for_qc(ctx, config.batch_to_attest.number.prev().unwrap()) + ctrl.wait_for_qc(ctx, config.batch_to_attest.number.prev().unwrap()) .await .unwrap() ); @@ -215,10 +191,7 @@ async fn test_wait_for_qc() { qc.verify(genesis, &config.committee).unwrap(); break; } - assert_eq!( - None, - ctrl.state.subscribe().borrow().as_ref().unwrap().qc() - ); + assert_eq!(None, ctrl.state.subscribe().borrow().as_ref().unwrap().qc()); } } } diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index 0d675bd3..51158734 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -213,7 +213,7 @@ impl Network { let req = rpc::push_batch_votes::Req { // If the config has changed, we need to re-request all the votes // from peer that we might have ignored earlier. - want_votes_for: diff.config.as_ref().map(|c|c.batch_to_attest.clone()), + want_votes_for: diff.config.as_ref().map(|c| c.batch_to_attest.clone()), votes: diff.votes, }; // NOTE: The response should be non-empty only iff we requested a snapshot. diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index a9738ead..e2251a60 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -497,24 +497,26 @@ async fn test_batch_votes_propagation() { // Fixed attestation schedule. let first: attester::BatchNumber = rng.gen(); let schedule: Vec<_> = (0..10) - .map(|r| Arc::new(attestation::Config { - batch_to_attest: attester::Batch { - genesis: setup.genesis.hash(), - number: first + r, - hash: rng.gen(), - }, - committee: { - // We select a random subset here. It would be incorrect to choose an empty subset, but - // the chances of that are negligible. - let subset: Vec<_> = setup.attester_keys.iter().filter(|_| rng.gen()).collect(); - attester::Committee::new(subset.iter().map(|k| attester::WeightedAttester { - key: k.public(), - weight: rng.gen_range(5..10), - })) - .unwrap() - .into() - }, - })) + .map(|r| { + Arc::new(attestation::Config { + batch_to_attest: attester::Batch { + genesis: setup.genesis.hash(), + number: first + r, + hash: rng.gen(), + }, + committee: { + // We select a random subset here. It would be incorrect to choose an empty subset, but + // the chances of that are negligible. + let subset: Vec<_> = setup.attester_keys.iter().filter(|_| rng.gen()).collect(); + attester::Committee::new(subset.iter().map(|k| attester::WeightedAttester { + key: k.public(), + weight: rng.gen_range(5..10), + })) + .unwrap() + .into() + }, + }) + }) .collect(); // Round of the schedule that nodes should collect the votes for. @@ -533,10 +535,8 @@ async fn test_batch_votes_propagation() { cfg: cfg.clone(), block_store: store.blocks.clone(), batch_store: store.batches.clone(), - attestation: attestation::Controller::new(Some( - setup.attester_keys[i].clone(), - )) - .into(), + attestation: attestation::Controller::new(Some(setup.attester_keys[i].clone())) + .into(), }); s.spawn_bg(runner.run(ctx).instrument(tracing::info_span!("node", i))); // Task going through the schedule, waiting for ANY node to collect the certificate diff --git a/node/actors/network/src/lib.rs b/node/actors/network/src/lib.rs index 88bb68ec..5578badc 100644 --- a/node/actors/network/src/lib.rs +++ b/node/actors/network/src/lib.rs @@ -59,8 +59,7 @@ impl Network { pipe: ActorPipe, attestation: Arc, ) -> (Arc, Runner) { - let gossip = - gossip::Network::new(cfg, block_store, batch_store, pipe.send, attestation); + let gossip = gossip::Network::new(cfg, block_store, batch_store, pipe.send, attestation); let consensus = consensus::Network::new(gossip.clone()); let net = Arc::new(Self { gossip, consensus }); ( diff --git a/node/actors/network/src/rpc/push_batch_votes.rs b/node/actors/network/src/rpc/push_batch_votes.rs index 5017edc3..901d120b 100644 --- a/node/actors/network/src/rpc/push_batch_votes.rs +++ b/node/actors/network/src/rpc/push_batch_votes.rs @@ -4,7 +4,7 @@ use crate::proto::gossip as proto; use anyhow::Context as _; use std::sync::Arc; use zksync_consensus_roles::attester; -use zksync_protobuf::{read_optional,ProtoFmt}; +use zksync_protobuf::{read_optional, ProtoFmt}; /// RPC pushing fresh batch votes. pub(crate) struct Rpc; @@ -20,14 +20,14 @@ impl super::Rpc for Rpc { /// Signed batch message that the receiving peer should process. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Req { - // Requesting the peer to respond with votes for the batch. + /// Requesting the peer to respond with votes for the batch. pub(crate) want_votes_for: Option, /// New votes that server might be not aware of. pub(crate) votes: Vec>>, } pub(crate) struct Resp { - /// Votes requested by the peer. + /// Votes requested by the peer. pub(crate) votes: Vec>>, } diff --git a/node/libs/roles/src/attester/messages/batch.rs b/node/libs/roles/src/attester/messages/batch.rs index e850740e..41ada768 100644 --- a/node/libs/roles/src/attester/messages/batch.rs +++ b/node/libs/roles/src/attester/messages/batch.rs @@ -128,23 +128,6 @@ pub enum BatchQCVerifyError { GenesisMismatch, } -/// Error returned by `BatchQC::add()` if the signature is invalid. -#[derive(thiserror::Error, Debug)] -pub enum BatchQCAddError { - /// Inconsistent messages. - #[error("Trying to add signature for a different message")] - InconsistentMessages, - /// Signer not present in the committee. - #[error("Signer not in committee: {signer:?}")] - SignerNotInCommittee { - /// Signer of the message. - signer: Box, - }, - /// Message already present in BatchQC. - #[error("Message already signed for BatchQC")] - Exists, -} - impl BatchQC { /// Create a new empty instance for a given `Batch` message. pub fn new(message: Batch) -> Self { diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index e640f9af..f0104610 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -265,8 +265,7 @@ impl Configs { let store = TestMemoryStorage::new(ctx, &self.app.genesis).await; // We don't have an API to poll in this setup, we can only create a local store based attestation client. - let attestation = - Arc::new(attestation::Controller::new(self.app.attester_key.clone())); + let attestation = Arc::new(attestation::Controller::new(self.app.attester_key.clone())); let runner = store.runner; let e = executor::Executor { From f627a3c6b367da98287ee9363e136975fb19f4cf Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 13 Aug 2024 13:31:05 +0200 Subject: [PATCH 26/31] unused file --- node/actors/executor/src/attestation.rs | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 node/actors/executor/src/attestation.rs diff --git a/node/actors/executor/src/attestation.rs b/node/actors/executor/src/attestation.rs deleted file mode 100644 index b3e53def..00000000 --- a/node/actors/executor/src/attestation.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Module to publish attestations over batches. -use anyhow::Context as _; -use std::sync::Arc; -use zksync_concurrency::{ctx, time}; -pub use zksync_consensus_network::gossip::attestation::*; From 356b88198b3321f9815f43d25a2be8eee2f4178b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 13 Aug 2024 13:33:07 +0200 Subject: [PATCH 27/31] applied comment --- node/actors/network/src/proto/gossip.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/actors/network/src/proto/gossip.proto b/node/actors/network/src/proto/gossip.proto index 4c4d6ed2..8f8c1327 100644 --- a/node/actors/network/src/proto/gossip.proto +++ b/node/actors/network/src/proto/gossip.proto @@ -27,7 +27,7 @@ message PushBatchVotes { message PushBatchVotesResp { // Signed roles.validator.Msg.votes - // Empty if want_snapshot in request was not set. + // Empty if want_votes_for in request was not set. repeated roles.attester.Signed votes = 1; } From 7e8af7cbd59f38b74428307376ff4676b7a4802d Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 13 Aug 2024 16:09:50 +0200 Subject: [PATCH 28/31] Update node/tools/src/config.rs Co-authored-by: Akosh Farkash --- node/tools/src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index f0104610..cb84e48d 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -264,7 +264,6 @@ impl Configs { let replica_store = store::RocksDB::open(self.app.genesis.clone(), &self.database).await?; let store = TestMemoryStorage::new(ctx, &self.app.genesis).await; - // We don't have an API to poll in this setup, we can only create a local store based attestation client. let attestation = Arc::new(attestation::Controller::new(self.app.attester_key.clone())); let runner = store.runner; From 80d12e4c528f2d7595999fd6675943d090a21009 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 14 Aug 2024 09:19:08 +0200 Subject: [PATCH 29/31] removed comment --- node/actors/network/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/node/actors/network/src/lib.rs b/node/actors/network/src/lib.rs index 5578badc..cb1fab47 100644 --- a/node/actors/network/src/lib.rs +++ b/node/actors/network/src/lib.rs @@ -165,8 +165,6 @@ impl Runner { } } - // TODO: check if we are active attester to get new L1 Batches, sign them and broadcast the signature - let accept_limiter = limiter::Limiter::new(ctx, self.net.gossip.cfg.tcp_accept_rate); loop { accept_limiter.acquire(ctx, 1).await?; From cd7bcf73a5a26896d1d6084541e3fd264bc16890 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 14 Aug 2024 15:50:16 +0200 Subject: [PATCH 30/31] applied comments, expanded example --- .../network/src/gossip/attestation/mod.rs | 94 +++++++++++++------ .../network/src/gossip/attestation/tests.rs | 15 +-- node/actors/network/src/gossip/tests/mod.rs | 4 +- node/libs/roles/src/validator/testonly.rs | 11 +-- 4 files changed, 81 insertions(+), 43 deletions(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index e5954989..0d5584b9 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -9,7 +9,8 @@ mod metrics; #[cfg(test)] mod tests; -/// Coordinate the attestation by showing the config as seen by the main node. +/// Configuration of the attestation Controller. +/// It determines what should be attested and by whom. #[derive(Debug, Clone, PartialEq)] pub struct Config { /// Batch to attest. @@ -22,8 +23,10 @@ pub struct Config { #[derive(Clone)] struct State { config: Arc, + /// Votes collected so far. votes: im::HashMap>>, - weight: attester::Weight, + // Total weight of the votes collected. + total_weight: attester::Weight, } /// Diff between 2 states. @@ -94,8 +97,9 @@ impl State { } // Verify signature only after checking all the other preconditions. vote.verify().context("verify")?; + tracing::info!("collected vote with weight {weight} from {:?}", vote.key); self.votes.insert(vote.key.clone(), vote); - self.weight += weight; + self.total_weight += weight; Ok(()) } @@ -117,8 +121,8 @@ impl State { Ok(()) } - fn qc(&self) -> Option { - if self.weight < self.config.committee.threshold() { + fn cert(&self) -> Option { + if self.total_weight < self.config.committee.threshold() { return None; } let mut sigs = attester::MultiSig::default(); @@ -167,27 +171,56 @@ impl DiffReceiver { /// /// Expected usage: /// ``` -/// let ctrl = attestation::Controller::new(Some(key)); -/// loop { -/// // Check the global attestation registry. -/// // Compute the next expected batch and the committee that should attest it. -/// ... -/// let config = attestation::Config { -/// batch_to_attest: ..., -/// committee: ..., -/// }; -/// ctrl.update_config(Arc::new(config.clone())).unwrap(); +/// let ctrl = Arc::new(attestation::Controller::new(Some(key))); +/// // Check what is the number of the next batch to be attested in a +/// // global attestation registry (i.e. L1 chain state). +/// let first : attester::BatchNumber = ... +/// scope::run!(ctx, |ctx,s| async { +/// // Loop starting attestation whenever global attestation state progresses. /// s.spawn(async { -/// if let Some(qc) = ctrl.wait_for_qc(ctx, config.batch_to_attest.number).await?; -/// // Submit the certificate `qc` to the global registry -/// ... +/// let mut next = first; +/// loop { +/// // Based on the local storage, compute the next expected batch hash +/// // and the committee that should attest it. +/// ... +/// let config = attestation::Config { +/// batch_to_attest: attester::Batch { +/// number: next, +/// ... +/// }, +/// committee: ..., +/// }; +/// ctrl.start_attestation(Arc::new(config)).unwrap(); +/// // Wait for the attestation to progress, by observing the +/// // global attestation registry. +/// next = ...; +/// } +/// }); +/// s.spawn(async { +/// // Loop waiting for a certificate to be collected and submitting +/// // it to the global registry +/// loop { +/// let mut next = first; +/// if let Some(qc) = ctrl.wait_for_cert(ctx, next).await?; +/// // Submit the certificate to the global registry. +/// ... +/// next = next.next(); +/// } /// }); -/// // Wait for the global registry to include the certificate. -/// ... +/// +/// // Make the executor establish the p2p network and +/// // collect the attestation votes. +/// executor::Executor { +/// ... +/// attestation: ctrl.clone(), +/// }.run(ctx).await; /// } /// ``` pub struct Controller { + /// Key to automatically vote for batches. + /// None, if the current node is not an attester. key: Option, + /// Internal state of the controller. state: Watch>, } @@ -229,14 +262,14 @@ impl Controller { let Some(mut state) = locked.borrow().clone() else { return Ok(()); }; - let before = state.weight; + let before = state.total_weight; let res = state.insert_votes(votes); - if state.weight > before { + if state.total_weight > before { metrics::METRICS.votes_collected.set(state.votes.len()); #[allow(clippy::float_arithmetic)] metrics::METRICS .weight_collected - .set(state.weight as f64 / state.config.committee.total_weight() as f64); + .set(state.total_weight as f64 / state.config.committee.total_weight() as f64); locked.send_replace(Some(state)); } res @@ -258,7 +291,7 @@ impl Controller { /// Waits for the certificate for a batch with the given number to be collected. /// Returns None iff attestation already skipped to collecting certificate for some later batch. - pub async fn wait_for_qc( + pub async fn wait_for_cert( &self, ctx: &ctx::Ctx, n: attester::BatchNumber, @@ -276,7 +309,7 @@ impl Controller { if state.config.batch_to_attest.number > n { return Ok(None); } - if let Some(qc) = state.qc() { + if let Some(qc) = state.cert() { return Ok(Some(qc)); } } @@ -285,7 +318,8 @@ impl Controller { /// Updates the attestation config. /// Clears the votes collected for the previous config. /// Batch number has to increase with each update. - pub async fn update_config(&self, config: Arc) -> anyhow::Result<()> { + #[tracing::instrument(name = "attestation::Controller::start_attestation", skip_all)] + pub async fn start_attestation(&self, config: Arc) -> anyhow::Result<()> { let locked = self.state.lock().await; let old = locked.borrow().clone(); if let Some(old) = old.as_ref() { @@ -301,10 +335,14 @@ impl Controller { "tried to decrease batch number" ); } + tracing::info!( + "started collecting votes for batch {:?}", + config.batch_to_attest.number + ); let mut new = State { config, votes: im::HashMap::new(), - weight: 0, + total_weight: 0, }; if let Some(key) = self.key.as_ref() { if new.config.committee.contains(&key.public()) { @@ -323,7 +361,7 @@ impl Controller { #[allow(clippy::float_arithmetic)] metrics::METRICS .weight_collected - .set(new.weight as f64 / new.config.committee.total_weight() as f64); + .set(new.total_weight as f64 / new.config.committee.total_weight() as f64); locked.send_replace(Some(new)); Ok(()) } diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index 227dfa22..ba53cc4c 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -39,7 +39,7 @@ async fn test_insert_votes() { .into(), }); let ctrl_votes = || Votes::from(ctrl.votes(&config.batch_to_attest)); - ctrl.update_config(config.clone()).await.unwrap(); + ctrl.start_attestation(config.clone()).await.unwrap(); assert_eq!(Votes::from([]), ctrl_votes()); let mut recv = ctrl.subscribe(); let diff = recv.wait_for_diff(ctx).await.unwrap(); @@ -132,7 +132,7 @@ async fn test_insert_votes() { } #[tokio::test] -async fn test_wait_for_qc() { +async fn test_wait_for_cert() { abort_on_panic(); let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); @@ -163,7 +163,7 @@ async fn test_wait_for_qc() { .map(|k| k.sign_msg(config.batch_to_attest.clone()).into()) .collect(); all_votes.shuffle(rng); - ctrl.update_config(config.clone()).await.unwrap(); + ctrl.start_attestation(config.clone()).await.unwrap(); loop { let end = rng.gen_range(0..=committee_size); tracing::info!("end = {end}"); @@ -173,7 +173,7 @@ async fn test_wait_for_qc() { // Waiting for the previous qc should immediately return None. assert_eq!( None, - ctrl.wait_for_qc(ctx, config.batch_to_attest.number.prev().unwrap()) + ctrl.wait_for_cert(ctx, config.batch_to_attest.number.prev().unwrap()) .await .unwrap() ); @@ -183,7 +183,7 @@ async fn test_wait_for_qc() { >= config.committee.threshold() { let qc = ctrl - .wait_for_qc(ctx, config.batch_to_attest.number) + .wait_for_cert(ctx, config.batch_to_attest.number) .await .unwrap() .unwrap(); @@ -191,7 +191,10 @@ async fn test_wait_for_qc() { qc.verify(genesis, &config.committee).unwrap(); break; } - assert_eq!(None, ctrl.state.subscribe().borrow().as_ref().unwrap().qc()); + assert_eq!( + None, + ctrl.state.subscribe().borrow().as_ref().unwrap().cert() + ); } } } diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index e2251a60..d1d65b2d 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -554,13 +554,13 @@ async fn test_batch_votes_propagation() { return Ok(()); }; let attestation = node.net.gossip.attestation.clone(); - attestation.update_config(cfg.clone()).await.unwrap(); + attestation.start_attestation(cfg.clone()).await.unwrap(); // Wait for the certificate in the background. s.spawn_bg(async { let r = r; let attestation = attestation; let Ok(Some(qc)) = attestation - .wait_for_qc(ctx, cfg.batch_to_attest.number) + .wait_for_cert(ctx, cfg.batch_to_attest.number) .await else { return Ok(()); diff --git a/node/libs/roles/src/validator/testonly.rs b/node/libs/roles/src/validator/testonly.rs index f4ce2b35..c9a3a314 100644 --- a/node/libs/roles/src/validator/testonly.rs +++ b/node/libs/roles/src/validator/testonly.rs @@ -6,10 +6,7 @@ use super::{ ReplicaCommit, ReplicaPrepare, SecretKey, Signature, Signed, Signers, View, ViewNumber, WeightedValidator, }; -use crate::{ - attester::{self, BatchNumber, SyncBatch}, - validator::LeaderSelectionMode, -}; +use crate::{attester, validator::LeaderSelectionMode}; use bit_vec::BitVec; use rand::{ distributions::{Distribution, Standard}, @@ -146,12 +143,12 @@ impl Setup { pub fn push_batch(&mut self, rng: &mut impl Rng) { let batch_number = match self.0.batches.last() { Some(b) => b.number.next(), - None => BatchNumber(0), + None => attester::BatchNumber(0), }; let size: usize = rng.gen_range(500..1000); let payloads = vec![Payload((0..size).map(|_| rng.gen()).collect())]; let proof = rng.gen::<[u8; 32]>().to_vec(); - let batch = SyncBatch { + let batch = attester::SyncBatch { number: batch_number, payloads, proof, @@ -205,7 +202,7 @@ pub struct SetupInner { /// Past blocks. pub blocks: Vec, /// L1 batches - pub batches: Vec, + pub batches: Vec, /// Genesis config. pub genesis: Genesis, } From eb21f6a94e4926659d8b539f94d81babdeca0637 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 14 Aug 2024 16:00:16 +0200 Subject: [PATCH 31/31] applied comments --- .../network/src/gossip/attestation/mod.rs | 72 +++++++++---------- .../network/src/gossip/attestation/tests.rs | 38 +++++----- node/actors/network/src/gossip/runner.rs | 4 +- node/actors/network/src/gossip/tests/mod.rs | 2 +- 4 files changed, 58 insertions(+), 58 deletions(-) diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 0d5584b9..6effd612 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -12,17 +12,17 @@ mod tests; /// Configuration of the attestation Controller. /// It determines what should be attested and by whom. #[derive(Debug, Clone, PartialEq)] -pub struct Config { +pub struct Info { /// Batch to attest. pub batch_to_attest: attester::Batch, /// Committee that should attest the batch. pub committee: Arc, } -// Internal attestation state: config and the set of votes collected so far. +// Internal attestation state: info and the set of votes collected so far. #[derive(Clone)] struct State { - config: Arc, + info: Arc, /// Votes collected so far. votes: im::HashMap>>, // Total weight of the votes collected. @@ -33,13 +33,13 @@ struct State { pub(crate) struct Diff { /// New votes. pub(crate) votes: Vec>>, - /// New config, if changed. - pub(crate) config: Option>, + /// New info, if changed. + pub(crate) info: Option>, } impl Diff { fn is_empty(&self) -> bool { - self.votes.is_empty() && self.config.is_none() + self.votes.is_empty() && self.info.is_none() } } @@ -49,19 +49,19 @@ impl State { fn diff(&self, old: &Option) -> Diff { let Some(old) = old.as_ref() else { return Diff { - config: Some(self.config.clone()), + info: Some(self.info.clone()), votes: self.votes.values().cloned().collect(), }; }; - if self.config.batch_to_attest.number != old.config.batch_to_attest.number { + if self.info.batch_to_attest.number != old.info.batch_to_attest.number { return Diff { - config: Some(self.config.clone()), + info: Some(self.info.clone()), votes: self.votes.values().cloned().collect(), }; } Diff { - config: None, + info: None, votes: self .votes .iter() @@ -76,17 +76,17 @@ impl State { /// Returns an error if genesis doesn't match or the signature is invalid. fn insert_vote(&mut self, vote: Arc>) -> anyhow::Result<()> { anyhow::ensure!( - vote.msg.genesis == self.config.batch_to_attest.genesis, + vote.msg.genesis == self.info.batch_to_attest.genesis, "Genesis mismatch" ); - if vote.msg.number != self.config.batch_to_attest.number { + if vote.msg.number != self.info.batch_to_attest.number { return Ok(()); } anyhow::ensure!( - vote.msg.hash == self.config.batch_to_attest.hash, + vote.msg.hash == self.info.batch_to_attest.hash, "batch hash mismatch" ); - let Some(weight) = self.config.committee.weight(&vote.key) else { + let Some(weight) = self.info.committee.weight(&vote.key) else { anyhow::bail!( "received vote signed by an inactive attester: {:?}", vote.key @@ -122,7 +122,7 @@ impl State { } fn cert(&self) -> Option { - if self.total_weight < self.config.committee.threshold() { + if self.total_weight < self.info.committee.threshold() { return None; } let mut sigs = attester::MultiSig::default(); @@ -130,7 +130,7 @@ impl State { sigs.add(vote.key.clone(), vote.sig.clone()); } Some(attester::BatchQC { - message: self.config.batch_to_attest.clone(), + message: self.info.batch_to_attest.clone(), signatures: sigs, }) } @@ -159,7 +159,7 @@ impl DiffReceiver { } /// `Controller` manages the attestation state. -/// It maintains a set of votes matching the attestation config. +/// It maintains a set of votes matching the attestation info. /// It allows for /// * adding votes to the state /// * subscribing to the vote set changes @@ -183,14 +183,14 @@ impl DiffReceiver { /// // Based on the local storage, compute the next expected batch hash /// // and the committee that should attest it. /// ... -/// let config = attestation::Config { +/// let info = attestation::Info { /// batch_to_attest: attester::Batch { /// number: next, /// ... /// }, /// committee: ..., /// }; -/// ctrl.start_attestation(Arc::new(config)).unwrap(); +/// ctrl.start_attestation(Arc::new(info)).unwrap(); /// // Wait for the attestation to progress, by observing the /// // global attestation registry. /// next = ...; @@ -269,7 +269,7 @@ impl Controller { #[allow(clippy::float_arithmetic)] metrics::METRICS .weight_collected - .set(state.total_weight as f64 / state.config.committee.total_weight() as f64); + .set(state.total_weight as f64 / state.info.committee.total_weight() as f64); locked.send_replace(Some(state)); } res @@ -283,7 +283,7 @@ impl Controller { let state = self.state.subscribe(); let state = state.borrow(); let Some(state) = &*state else { return vec![] }; - if &state.config.batch_to_attest != want { + if &state.info.batch_to_attest != want { return vec![]; } state.votes.values().cloned().collect() @@ -303,10 +303,10 @@ impl Controller { let Some(state) = state.as_ref() else { continue; }; - if state.config.batch_to_attest.number < n { + if state.info.batch_to_attest.number < n { continue; }; - if state.config.batch_to_attest.number > n { + if state.info.batch_to_attest.number > n { return Ok(None); } if let Some(qc) = state.cert() { @@ -315,53 +315,53 @@ impl Controller { } } - /// Updates the attestation config. - /// Clears the votes collected for the previous config. + /// Updates the internal configuration to start collecting votes for a new batch. + /// Clears the votes collected for the previous info. /// Batch number has to increase with each update. #[tracing::instrument(name = "attestation::Controller::start_attestation", skip_all)] - pub async fn start_attestation(&self, config: Arc) -> anyhow::Result<()> { + pub async fn start_attestation(&self, info: Arc) -> anyhow::Result<()> { let locked = self.state.lock().await; let old = locked.borrow().clone(); if let Some(old) = old.as_ref() { - if *old.config == *config { + if *old.info == *info { return Ok(()); } anyhow::ensure!( - old.config.batch_to_attest.genesis == config.batch_to_attest.genesis, + old.info.batch_to_attest.genesis == info.batch_to_attest.genesis, "tried to change genesis" ); anyhow::ensure!( - old.config.batch_to_attest.number < config.batch_to_attest.number, + old.info.batch_to_attest.number < info.batch_to_attest.number, "tried to decrease batch number" ); } tracing::info!( "started collecting votes for batch {:?}", - config.batch_to_attest.number + info.batch_to_attest.number ); let mut new = State { - config, + info, votes: im::HashMap::new(), total_weight: 0, }; if let Some(key) = self.key.as_ref() { - if new.config.committee.contains(&key.public()) { - let vote = key.sign_msg(new.config.batch_to_attest.clone()); + if new.info.committee.contains(&key.public()) { + let vote = key.sign_msg(new.info.batch_to_attest.clone()); // This is our own vote, so it always should be valid. new.insert_vote(Arc::new(vote)).unwrap(); } } metrics::METRICS .batch_number - .set(new.config.batch_to_attest.number.0); + .set(new.info.batch_to_attest.number.0); metrics::METRICS .committee_size - .set(new.config.committee.len()); + .set(new.info.committee.len()); metrics::METRICS.votes_collected.set(new.votes.len()); #[allow(clippy::float_arithmetic)] metrics::METRICS .weight_collected - .set(new.total_weight as f64 / new.config.committee.total_weight() as f64); + .set(new.total_weight as f64 / new.info.committee.total_weight() as f64); locked.send_replace(Some(new)); Ok(()) } diff --git a/node/actors/network/src/gossip/attestation/tests.rs b/node/actors/network/src/gossip/attestation/tests.rs index ba53cc4c..2a5976b4 100644 --- a/node/actors/network/src/gossip/attestation/tests.rs +++ b/node/actors/network/src/gossip/attestation/tests.rs @@ -25,7 +25,7 @@ async fn test_insert_votes() { for i in 0..3 { tracing::info!("iteration {i}"); let keys: Vec = (0..8).map(|_| rng.gen()).collect(); - let config = Arc::new(Config { + let info = Arc::new(Info { batch_to_attest: attester::Batch { genesis, number: first + i, @@ -38,17 +38,17 @@ async fn test_insert_votes() { .unwrap() .into(), }); - let ctrl_votes = || Votes::from(ctrl.votes(&config.batch_to_attest)); - ctrl.start_attestation(config.clone()).await.unwrap(); + let ctrl_votes = || Votes::from(ctrl.votes(&info.batch_to_attest)); + ctrl.start_attestation(info.clone()).await.unwrap(); assert_eq!(Votes::from([]), ctrl_votes()); let mut recv = ctrl.subscribe(); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert_eq!(diff.config.as_ref(), Some(&config)); + assert_eq!(diff.info.as_ref(), Some(&info)); assert_eq!(Votes::default(), diff.votes.into()); let all_votes: Vec = keys .iter() - .map(|k| k.sign_msg(config.batch_to_attest.clone()).into()) + .map(|k| k.sign_msg(info.batch_to_attest.clone()).into()) .collect(); tracing::info!("Initial votes."); @@ -57,7 +57,7 @@ async fn test_insert_votes() { .unwrap(); assert_eq!(Votes::from(all_votes[0..3].iter().cloned()), ctrl_votes()); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert!(diff.config.is_none()); + assert!(diff.info.is_none()); assert_eq!( Votes::from(all_votes[0..3].iter().cloned()), diff.votes.into() @@ -72,7 +72,7 @@ async fn test_insert_votes() { .unwrap(); assert_eq!(Votes::from(all_votes[0..7].iter().cloned()), ctrl_votes()); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert!(diff.config.is_none()); + assert!(diff.info.is_none()); assert_eq!( Votes::from(all_votes[3..7].iter().cloned()), diff.votes.into() @@ -88,7 +88,7 @@ async fn test_insert_votes() { assert!(ctrl .insert_votes((0..3).map(|_| { let k: attester::SecretKey = rng.gen(); - k.sign_msg(config.batch_to_attest.clone()).into() + k.sign_msg(info.batch_to_attest.clone()).into() })) .await .is_err()); @@ -98,7 +98,7 @@ async fn test_insert_votes() { ctrl.insert_votes((0..3).map(|_| { let k: attester::SecretKey = rng.gen(); k.sign_msg(attester::Batch { - genesis: config.batch_to_attest.genesis, + genesis: info.batch_to_attest.genesis, number: rng.gen(), hash: rng.gen(), }) @@ -123,7 +123,7 @@ async fn test_insert_votes() { .unwrap(); assert_eq!(Votes::from(all_votes.clone()), ctrl_votes()); let diff = recv.wait_for_diff(ctx).await.unwrap(); - assert!(diff.config.is_none()); + assert!(diff.info.is_none()); assert_eq!( Votes::from(all_votes[7..].iter().cloned()), diff.votes.into() @@ -145,7 +145,7 @@ async fn test_wait_for_cert() { tracing::info!("iteration {i}"); let committee_size = rng.gen_range(1..20); let keys: Vec = (0..committee_size).map(|_| rng.gen()).collect(); - let config = Arc::new(Config { + let info = Arc::new(Info { batch_to_attest: attester::Batch { genesis, number: first + i, @@ -160,10 +160,10 @@ async fn test_wait_for_cert() { }); let mut all_votes: Vec = keys .iter() - .map(|k| k.sign_msg(config.batch_to_attest.clone()).into()) + .map(|k| k.sign_msg(info.batch_to_attest.clone()).into()) .collect(); all_votes.shuffle(rng); - ctrl.start_attestation(config.clone()).await.unwrap(); + ctrl.start_attestation(info.clone()).await.unwrap(); loop { let end = rng.gen_range(0..=committee_size); tracing::info!("end = {end}"); @@ -173,22 +173,22 @@ async fn test_wait_for_cert() { // Waiting for the previous qc should immediately return None. assert_eq!( None, - ctrl.wait_for_cert(ctx, config.batch_to_attest.number.prev().unwrap()) + ctrl.wait_for_cert(ctx, info.batch_to_attest.number.prev().unwrap()) .await .unwrap() ); - if config + if info .committee .weight_of_keys(all_votes[..end].iter().map(|v| &v.key)) - >= config.committee.threshold() + >= info.committee.threshold() { let qc = ctrl - .wait_for_cert(ctx, config.batch_to_attest.number) + .wait_for_cert(ctx, info.batch_to_attest.number) .await .unwrap() .unwrap(); - assert_eq!(qc.message, config.batch_to_attest); - qc.verify(genesis, &config.committee).unwrap(); + assert_eq!(qc.message, info.batch_to_attest); + qc.verify(genesis, &info.committee).unwrap(); break; } assert_eq!( diff --git a/node/actors/network/src/gossip/runner.rs b/node/actors/network/src/gossip/runner.rs index 51158734..23ccf009 100644 --- a/node/actors/network/src/gossip/runner.rs +++ b/node/actors/network/src/gossip/runner.rs @@ -211,9 +211,9 @@ impl Network { loop { let diff = recv.wait_for_diff(ctx).await?; let req = rpc::push_batch_votes::Req { - // If the config has changed, we need to re-request all the votes + // If the info has changed, we need to re-request all the votes // from peer that we might have ignored earlier. - want_votes_for: diff.config.as_ref().map(|c| c.batch_to_attest.clone()), + want_votes_for: diff.info.as_ref().map(|c| c.batch_to_attest.clone()), votes: diff.votes, }; // NOTE: The response should be non-empty only iff we requested a snapshot. diff --git a/node/actors/network/src/gossip/tests/mod.rs b/node/actors/network/src/gossip/tests/mod.rs index d1d65b2d..7ab311bb 100644 --- a/node/actors/network/src/gossip/tests/mod.rs +++ b/node/actors/network/src/gossip/tests/mod.rs @@ -498,7 +498,7 @@ async fn test_batch_votes_propagation() { let first: attester::BatchNumber = rng.gen(); let schedule: Vec<_> = (0..10) .map(|r| { - Arc::new(attestation::Config { + Arc::new(attestation::Info { batch_to_attest: attester::Batch { genesis: setup.genesis.hash(), number: first + r,