diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json index 277d86a7..6f12bbf6 100644 --- a/.github/release-please/manifest.json +++ b/.github/release-please/manifest.json @@ -1,3 +1,3 @@ { - "node": "0.3.0" + "node": "0.4.0" } diff --git a/node/.config/nextest.toml b/node/.config/nextest.toml index ae557799..558ce287 100644 --- a/node/.config/nextest.toml +++ b/node/.config/nextest.toml @@ -42,5 +42,5 @@ final-status-level = "all" [[profile.ci.overrides]] # Force end-to-end tests to run sequentially and exclusively with all other tests -filter = "test(/::end_to_end::/)" +filter = "test(/::end_to_end::/)" threads-required = "num-test-threads" diff --git a/node/CHANGELOG.md b/node/CHANGELOG.md index 98d518b3..e2016412 100644 --- a/node/CHANGELOG.md +++ b/node/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.4.0](https://github.com/matter-labs/era-consensus/compare/v0.3.0...v0.4.0) (2024-10-07) + + +### Features + +* Expand metrics in HTTP debug page ([#205](https://github.com/matter-labs/era-consensus/issues/205)) ([2ef11bc](https://github.com/matter-labs/era-consensus/commit/2ef11bc0bc0ef9b332c4a4c2715c523143e844bd)) + ## [0.3.0](https://github.com/matter-labs/era-consensus/compare/v0.2.0...v0.3.0) (2024-09-26) diff --git a/node/Cargo.lock b/node/Cargo.lock index 7834f5fe..e5a547df 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -1334,6 +1334,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "human-repr" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58b778a5761513caf593693f8951c97a5b610841e754788400f32102eefdff1" + [[package]] name = "hyper" version = "0.14.30" @@ -4016,6 +4022,7 @@ dependencies = [ "build_html", "bytesize", "http-body-util", + "human-repr", "hyper 1.4.1", "hyper-util", "im", diff --git a/node/Cargo.toml b/node/Cargo.toml index a464f1aa..65a91a7b 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -16,39 +16,40 @@ members = [ resolver = "2" [workspace.package] -edition = "2021" -authors = ["The Matter Labs Team "] -homepage = "https://matter-labs.io/" +authors = ["The Matter Labs Team "] +edition = "2021" +homepage = "https://matter-labs.io/" +keywords = ["blockchain", "zksync"] +license = "MIT OR Apache-2.0" repository = "https://github.com/matter-labs/era-consensus" -license = "MIT OR Apache-2.0" -keywords = ["blockchain", "zksync"] -version = "0.3.0" +version = "0.4.0" [workspace.dependencies] # Crates from this repo. -zksync_consensus_bft = { version = "=0.3.0", path = "actors/bft" } -zksync_consensus_crypto = { version = "=0.3.0", path = "libs/crypto" } -zksync_consensus_executor = { version = "=0.3.0", path = "actors/executor" } -zksync_consensus_network = { version = "=0.3.0", path = "actors/network" } -zksync_consensus_roles = { version = "=0.3.0", path = "libs/roles" } -zksync_consensus_storage = { version = "=0.3.0", path = "libs/storage" } -zksync_consensus_tools = { version = "=0.3.0", path = "tools" } -zksync_consensus_utils = { version = "=0.3.0", path = "libs/utils" } +zksync_consensus_bft = { version = "=0.4.0", path = "actors/bft" } +zksync_consensus_crypto = { version = "=0.4.0", path = "libs/crypto" } +zksync_consensus_executor = { version = "=0.4.0", path = "actors/executor" } +zksync_consensus_network = { version = "=0.4.0", path = "actors/network" } +zksync_consensus_roles = { version = "=0.4.0", path = "libs/roles" } +zksync_consensus_storage = { version = "=0.4.0", path = "libs/storage" } +zksync_consensus_tools = { version = "=0.4.0", path = "tools" } +zksync_consensus_utils = { version = "=0.4.0", path = "libs/utils" } # Crates from this repo that might become independent in the future. -zksync_concurrency = { version = "=0.3.0", path = "libs/concurrency" } -zksync_protobuf = { version = "=0.3.0", path = "libs/protobuf" } -zksync_protobuf_build = { version = "=0.3.0", path = "libs/protobuf_build" } +zksync_concurrency = { version = "=0.4.0", path = "libs/concurrency" } +zksync_protobuf = { version = "=0.4.0", path = "libs/protobuf" } +zksync_protobuf_build = { version = "=0.4.0", path = "libs/protobuf_build" } # Crates from Matter Labs. -vise = "0.2.0" +vise = "0.2.0" vise-exporter = "0.2.0" # Crates from third-parties. -anyhow = "1" +anyhow = "1" assert_matches = "1.5.0" -async-trait = "0.1.71" -bit-vec = "0.6" +async-trait = "0.1.71" +base64 = "0.22.1" +bit-vec = "0.6" # portable feature makes blst code check in runtime if ADX instruction set is # supported at every bigint multiplication: # https://github.com/supranational/blst/commit/0dbc2ce4138e9e5d2aa941ca4cd731d8814a67a2 @@ -56,58 +57,58 @@ bit-vec = "0.6" # Apparently the cost of the check is negligible. # This is an undocumented feature, present since release 0.3.11: # https://github.com/supranational/blst/releases/tag/v0.3.11 -blst = { version = "0.3.13", features = ["portable"] } -clap = { version = "4.3.3", features = ["derive"] } -criterion = "0.5.1" -ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } -elliptic-curve = { version = "0.13" } -heck = "0.5.0" -hex = "0.4.3" -im = "15.1.0" -jsonrpsee = { version = "0.23.0", features = ["server", "http-client"] } -k256 = { version = "0.13", features = ["ecdsa"] } -k8s-openapi = { version = "0.22.0", features = ["latest"] } -kube = { version = "0.91.0", features = ["runtime", "derive"] } -num-bigint = "0.4.4" -num-traits = "0.2.18" -once_cell = "1.17.1" -pin-project = "1.1.0" -pretty_assertions = "1.4.0" -prettyplease = "0.2.6" -proc-macro2 = "1.0.66" -prost = "0.12.0" -prost-build = "0.12.0" -prost-reflect = { version = "0.12.0", features = ["serde"] } -protox = "0.5.0" -quick-protobuf = "0.8.1" -quote = "1.0.33" -rand = "0.8.0" -rocksdb = "0.21.0" -semver = "1.0.23" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.95" -serde_yaml = "0.9" -sha3 = "0.10.8" -snow = "0.9.3" -syn = { version = "2.0.17", features = ["extra-traits"] } -tempfile = "3" -test-casing = "0.1.0" -thiserror = "1.0.40" -time = "0.3.23" -tokio = { version = "1.34.0", features = ["full"] } -tokio-rustls = "0.26.0" -tower = { version = "0.4.13" } -tracing = { version = "0.1.37", features = ["attributes"] } +blst = { version = "0.3.13", features = ["portable"] } +build_html = "2.4.0" +bytesize = "1.3.0" +clap = { version = "4.3.3", features = ["derive"] } +criterion = "0.5.1" +ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } +elliptic-curve = { version = "0.13" } +heck = "0.5.0" +hex = "0.4.3" +http-body-util = "0.1" +human-repr = "1.1.0" +hyper = { version = "1", features = ["full"] } +hyper-util = { version = "0.1", features = ["full"] } +im = "15.1.0" +jsonrpsee = { version = "0.23.0", features = ["http-client", "server"] } +k256 = { version = "0.13", features = ["ecdsa"] } +k8s-openapi = { version = "0.22.0", features = ["latest"] } +kube = { version = "0.91.0", features = ["derive", "runtime"] } +num-bigint = "0.4.4" +num-traits = "0.2.18" +once_cell = "1.17.1" +pin-project = "1.1.0" +pretty_assertions = "1.4.0" +prettyplease = "0.2.6" +proc-macro2 = "1.0.66" +prost = "0.12.0" +prost-build = "0.12.0" +prost-reflect = { version = "0.12.0", features = ["serde"] } +protox = "0.5.0" +quick-protobuf = "0.8.1" +quote = "1.0.33" +rand = "0.8.0" +rocksdb = "0.21.0" +rustls-pemfile = "2" +semver = "1.0.23" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.95" +serde_yaml = "0.9" +sha3 = "0.10.8" +snow = "0.9.3" +syn = { version = "2.0.17", features = ["extra-traits"] } +tempfile = "3" +test-casing = "0.1.0" +thiserror = "1.0.40" +time = "0.3.23" +tls-listener = { version = "0.10.1", features = ["rustls"] } +tokio = { version = "1.34.0", features = ["full"] } +tokio-rustls = "0.26.0" +tower = { version = "0.4.13" } +tracing = { version = "0.1.37", features = ["attributes"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } -zeroize = { version = "1.7.0", features = ["zeroize_derive"] } -hyper = { version = "1", features = ["full"] } -http-body-util = "0.1" -hyper-util = { version = "0.1", features = ["full"] } -tls-listener = { version = "0.10.1", features = ["rustls"] } -rustls-pemfile = "2" -base64 = "0.22.1" -build_html = "2.4.0" -bytesize = "1.3.0" +zeroize = { version = "1.7.0", features = ["zeroize_derive"] } # Note that "bench" profile inherits from "release" profile and # "test" profile inherits from "dev" profile. @@ -137,45 +138,43 @@ opt-level = 3 opt-level = 3 [workspace.lints.rust] -unsafe_code = "deny" -missing_docs = "warn" -unreachable_pub = "warn" +missing_docs = "warn" +unreachable_pub = "warn" +unsafe_code = "deny" unused_qualifications = "warn" [workspace.lints.clippy] # restriction group -create_dir = "warn" -empty_structs_with_brackets = "warn" -float_arithmetic = "warn" +create_dir = "warn" +empty_structs_with_brackets = "warn" +float_arithmetic = "warn" missing_docs_in_private_items = "warn" -non_ascii_literal = "warn" -partial_pub_fields = "warn" -print_stdout = "warn" -string_to_string = "warn" -suspicious_xor_used_as_pow = "warn" -try_err = "warn" -separated_literal_suffix = "warn" +non_ascii_literal = "warn" +partial_pub_fields = "warn" +print_stdout = "warn" +separated_literal_suffix = "warn" +string_to_string = "warn" +suspicious_xor_used_as_pow = "warn" +try_err = "warn" # pedantic group -bool_to_int_with_if = "warn" -default_trait_access = "warn" -if_not_else = "warn" -manual_assert = "warn" +bool_to_int_with_if = "warn" +default_trait_access = "warn" +if_not_else = "warn" +manual_assert = "warn" manual_instant_elapsed = "warn" -manual_let_else = "warn" -manual_ok_or = "warn" -manual_string_new = "warn" -match_bool = "warn" -wildcard_imports = "warn" +manual_let_else = "warn" +manual_ok_or = "warn" +manual_string_new = "warn" +match_bool = "warn" +wildcard_imports = "warn" # cargo group wildcard_dependencies = "warn" # Produces too many false positives. -redundant_locals = "allow" +box_default = "allow" needless_pass_by_ref_mut = "allow" -box_default = "allow" -# remove once fix to https://github.com/rust-lang/rust-clippy/issues/11764 is available on CI. -map_identity = "allow" +redundant_locals = "allow" # &*x is not equivalent to x, because it affects borrowing in closures. borrow_deref_ref = "allow" diff --git a/node/actors/bft/Cargo.toml b/node/actors/bft/Cargo.toml index 911bc0bc..70d09bd3 100644 --- a/node/actors/bft/Cargo.toml +++ b/node/actors/bft/Cargo.toml @@ -1,36 +1,36 @@ [package] -name = "zksync_consensus_bft" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus bft actor" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_consensus_bft" repository.workspace = true -keywords.workspace = true -description = "ZKsync consensus bft actor" +version.workspace = true [dependencies] -zksync_concurrency.workspace = true -zksync_consensus_crypto.workspace = true +zksync_concurrency.workspace = true +zksync_consensus_crypto.workspace = true zksync_consensus_network.workspace = true -zksync_consensus_roles.workspace = true +zksync_consensus_roles.workspace = true zksync_consensus_storage.workspace = true -zksync_consensus_utils.workspace = true -zksync_protobuf.workspace = true +zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true -anyhow.workspace = true +anyhow.workspace = true async-trait.workspace = true -once_cell.workspace = true -rand.workspace = true -thiserror.workspace = true -tracing.workspace = true -vise.workspace = true +once_cell.workspace = true +rand.workspace = true +thiserror.workspace = true +tracing.workspace = true +vise.workspace = true [dev-dependencies] -tokio = { workspace = true, features = ["full","test-util"]} -assert_matches.workspace = true +assert_matches.workspace = true pretty_assertions.workspace = true -test-casing.workspace = true +test-casing.workspace = true +tokio = { workspace = true, features = ["full", "test-util"] } [lints] workspace = true diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index 128d9167..a652df00 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -1,34 +1,34 @@ [package] -name = "zksync_consensus_executor" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus executor actor" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_consensus_executor" repository.workspace = true -keywords.workspace = true -description = "ZKsync consensus executor actor" +version.workspace = true [dependencies] -zksync_concurrency.workspace = true -zksync_consensus_bft.workspace = true -zksync_consensus_crypto.workspace = true +zksync_concurrency.workspace = true +zksync_consensus_bft.workspace = true +zksync_consensus_crypto.workspace = true zksync_consensus_network.workspace = true -zksync_consensus_roles.workspace = true +zksync_consensus_roles.workspace = true zksync_consensus_storage.workspace = true -zksync_consensus_utils.workspace = true -zksync_protobuf.workspace = true +zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true -anyhow.workspace = true +anyhow.workspace = true async-trait.workspace = true -rand.workspace = true -semver.workspace = true -tracing.workspace = true -vise.workspace = true +rand.workspace = true +semver.workspace = true +tracing.workspace = true +vise.workspace = true [dev-dependencies] test-casing.workspace = true -tokio.workspace = true +tokio.workspace = true [lints] workspace = true diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index 322bf743..439ff197 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -1,48 +1,49 @@ [package] -name = "zksync_consensus_network" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus network actor" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_consensus_network" repository.workspace = true -keywords.workspace = true -description = "ZKsync consensus network actor" +version.workspace = true [dependencies] -zksync_concurrency.workspace = true -zksync_consensus_crypto.workspace = true -zksync_consensus_roles.workspace = true +zksync_concurrency.workspace = true +zksync_consensus_crypto.workspace = true +zksync_consensus_roles.workspace = true zksync_consensus_storage.workspace = true -zksync_consensus_utils.workspace = true -zksync_protobuf.workspace = true +zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true -anyhow.workspace = true -async-trait.workspace = true -im.workspace = true -once_cell.workspace = true -pin-project.workspace = true -prost.workspace = true -rand.workspace = true -snow.workspace = true -thiserror.workspace = true -tracing.workspace = true -vise.workspace = true -tokio.workspace = true -tokio-rustls.workspace = true -hyper.workspace = true +anyhow.workspace = true +async-trait.workspace = true +base64.workspace = true +build_html.workspace = true +bytesize.workspace = true http-body-util.workspace = true -hyper-util.workspace = true -tls-listener.workspace = true -base64.workspace = true -build_html.workspace = true -bytesize.workspace = true -semver.workspace = true +human-repr.workspace = true +hyper-util.workspace = true +hyper.workspace = true +im.workspace = true +once_cell.workspace = true +pin-project.workspace = true +prost.workspace = true +rand.workspace = true +semver.workspace = true +snow.workspace = true +thiserror.workspace = true +tls-listener.workspace = true +tokio-rustls.workspace = true +tokio.workspace = true +tracing.workspace = true +vise.workspace = true [dev-dependencies] -assert_matches.workspace = true +assert_matches.workspace = true pretty_assertions.workspace = true -test-casing.workspace = true +test-casing.workspace = true [build-dependencies] zksync_protobuf_build.workspace = true diff --git a/node/actors/network/src/debug_page/mod.rs b/node/actors/network/src/debug_page/mod.rs index 05472854..663f28f1 100644 --- a/node/actors/network/src/debug_page/mod.rs +++ b/node/actors/network/src/debug_page/mod.rs @@ -1,9 +1,10 @@ //! Http Server to export debug information -use crate::{MeteredStreamStats, Network}; +use crate::{gossip::Connection, MeteredStreamStats, Network}; use anyhow::Context as _; use base64::Engine; use build_html::{Html, HtmlContainer, HtmlPage, Table, TableCell, TableCellType, TableRow}; use http_body_util::Full; +use human_repr::{HumanCount, HumanDuration, HumanThroughput}; use hyper::{ body::Bytes, header::{self, HeaderValue}, @@ -13,9 +14,9 @@ use hyper::{ }; use hyper_util::rt::tokio::TokioIo; use std::{ + collections::{HashMap, HashSet}, net::SocketAddr, sync::{atomic::Ordering, Arc}, - time::{Duration, SystemTime}, }; use tls_listener::TlsListener; use tokio::net::TcpListener; @@ -27,8 +28,9 @@ use tokio_rustls::{ server::TlsStream, TlsAcceptor, }; -use zksync_concurrency::{ctx, scope}; +use zksync_concurrency::{ctx, net, scope}; use zksync_consensus_crypto::TextFmt as _; +use zksync_consensus_roles::{attester, node, validator}; const STYLE: &str = include_str!("style.css"); @@ -217,115 +219,424 @@ impl Server { fn serve(&self, _request: Request) -> Full { let mut html = HtmlPage::new() .with_title("Node debug page") - .with_style(STYLE) - .with_header(1, "Active connections"); + .with_style(STYLE); + + // Config information + html = html + .with_header(1, "Config") + .with_paragraph(format!( + "Build version: {}", + self.network + .gossip + .cfg + .build_version + .as_ref() + .map_or("N/A".to_string(), |v| v.to_string()) + )) + .with_paragraph(format!( + "Server address: {}", + self.network.gossip.cfg.server_addr + )) + .with_paragraph(format!( + "Public address: {}", + self.network.gossip.cfg.public_addr.0 + )) + .with_paragraph(format!( + "Maximum block size: {}", + self.network.gossip.cfg.max_block_size.human_count_bytes() + )) + .with_paragraph(format!( + "Maximum block queue size: {}", + self.network.gossip.cfg.max_block_queue_size + )) + .with_paragraph(format!( + "Ping timeout: {}", + self.network + .gossip + .cfg + .ping_timeout + .as_ref() + .map_or("None".to_string(), |x| x + .as_seconds_f32() + .human_duration() + .to_string()) + )) + .with_paragraph(format!( + "TCP accept rate - burst: {}, refresh: {}", + self.network.gossip.cfg.tcp_accept_rate.burst, + self.network + .gossip + .cfg + .tcp_accept_rate + .refresh + .as_seconds_f32() + .human_duration() + )) + .with_header(3, "RPC limits") + .with_paragraph(format!( + "push_validator_addrs rate - burst: {}, refresh: {}", + self.network.gossip.cfg.rpc.push_validator_addrs_rate.burst, + self.network + .gossip + .cfg + .rpc + .push_validator_addrs_rate + .refresh + .as_seconds_f32() + .human_duration() + )) + .with_paragraph(format!( + "push_block_store_state rate - burst: {}, refresh: {}", + self.network + .gossip + .cfg + .rpc + .push_block_store_state_rate + .burst, + self.network + .gossip + .cfg + .rpc + .push_block_store_state_rate + .refresh + .as_seconds_f32() + .human_duration() + )) + .with_paragraph(format!( + "push_batch_votes rate - burst: {}, refresh: {}", + self.network.gossip.cfg.rpc.push_batch_votes_rate.burst, + self.network + .gossip + .cfg + .rpc + .push_batch_votes_rate + .refresh + .as_seconds_f32() + .human_duration() + )) + .with_paragraph(format!( + "get_block rate - burst: {}, refresh: {}", + self.network.gossip.cfg.rpc.get_block_rate.burst, + self.network + .gossip + .cfg + .rpc + .get_block_rate + .refresh + .as_seconds_f32() + .human_duration() + )) + .with_paragraph(format!( + "get_block timeout: {}", + self.network + .gossip + .cfg + .rpc + .get_block_timeout + .as_ref() + .map_or("None".to_string(), |x| x + .as_seconds_f32() + .human_duration() + .to_string()) + )) + .with_paragraph(format!( + "Consensus rate - burst: {}, refresh: {}", + self.network.gossip.cfg.rpc.consensus_rate.burst, + self.network + .gossip + .cfg + .rpc + .consensus_rate + .refresh + .as_seconds_f32() + .human_duration() + )); + + // Gossip network + html = html + .with_header(1, "Gossip network") + .with_header(2, "Config") + .with_paragraph(format!( + "Node public key: {}", + self.network.gossip.cfg.gossip.key.public().encode() + )) + .with_paragraph(format!( + "Dynamic incoming connections limit: {}", + self.network.gossip.cfg.gossip.dynamic_inbound_limit + )) + .with_header(3, "Static incoming connections") + .with_paragraph(Self::static_inbound_table( + self.network.gossip.cfg.gossip.static_inbound.clone(), + )) + .with_header(3, "Static outbound connections") + .with_paragraph(Self::static_outbound_table( + self.network.gossip.cfg.gossip.static_outbound.clone(), + )) + .with_header(2, "Active connections") + .with_header(3, "Incoming connections") + .with_paragraph(Self::gossip_active_table( + self.network.gossip.inbound.current().values(), + )) + .with_header(3, "Outgoing connections") + .with_paragraph(Self::gossip_active_table( + self.network.gossip.outbound.current().values(), + )) + .with_header(2, "Validator addresses") + .with_paragraph(Self::validator_addrs_table( + self.network.gossip.validator_addrs.current().iter(), + )) + .with_header(2, "Fetch queue") + .with_paragraph(format!( + "Blocks: {:?}", + self.network.gossip.fetch_queue.current_blocks() + )); + + // Attester network + html = html + .with_header(1, "Attester network") + .with_paragraph(format!( + "Node public key: {}", + self.network + .gossip + .attestation + .key() + .map_or("None".to_string(), |k| k.public().encode()) + )); + + if let Some(state) = self + .network + .gossip + .attestation + .state() + .subscribe() + .borrow() + .clone() + { + html = html.with_paragraph(format!( + "Batch to attest - Number: {}, Hash: {}, Genesis hash: {}", + state.info().batch_to_attest.number, + state.info().batch_to_attest.hash.encode(), + state.info().batch_to_attest.genesis.encode(), + )); + + html = html + .with_header(2, "Attester committee") + .with_paragraph(Self::attester_table( + state.info().committee.iter(), + state.votes(), + )) + .with_paragraph(format!( + "Total weight: {}", + state.info().committee.total_weight() + )); + } + + // Validator network if let Some(consensus) = self.network.consensus.as_ref() { html = html - .with_header(2, "Validator network") + .with_header(1, "Validator network") + .with_paragraph(format!("Public key: {}", consensus.key.public().encode())) + .with_header(2, "Active connections") .with_header(3, "Incoming connections") - .with_paragraph( - self.connections_html( - consensus - .inbound - .current() - .iter() - .map(|(k, v)| (k.encode(), v, None)), - ), - ) + .with_paragraph(Self::validator_active_table( + consensus.inbound.current().iter(), + )) .with_header(3, "Outgoing connections") - .with_paragraph( - self.connections_html( - consensus - .outbound - .current() - .iter() - .map(|(k, v)| (k.encode(), v, None)), - ), - ); + .with_paragraph(Self::validator_active_table( + consensus.outbound.current().iter(), + )); } - html = html - .with_header(2, "Gossip network") - .with_header(3, "Incoming connections") - .with_paragraph( - self.connections_html( - self.network - .gossip - .inbound - .current() - .values() - .map(|c| (c.key.encode(), &c.stats, c.build_version.clone())), - ), - ) - .with_header(3, "Outgoing connections") - .with_paragraph( - self.connections_html( - self.network - .gossip - .outbound - .current() - .values() - .map(|c| (c.key.encode(), &c.stats, c.build_version.clone())), - ), - ); + Full::new(Bytes::from(html.to_html_string())) } - fn connections_html<'a>( - &self, + fn static_inbound_table(connections: HashSet) -> String { + let mut table = Table::new().with_header_row(vec!["Public key"]); + + for key in connections { + table.add_body_row(vec![key.encode()]); + } + + table.to_html_string() + } + + fn static_outbound_table(connections: HashMap) -> String { + let mut table = Table::new().with_header_row(vec!["Public key", "Host address"]); + + for (key, addr) in connections { + table.add_body_row(vec![key.encode(), addr.0]); + } + + table.to_html_string() + } + + fn validator_addrs_table<'a>( connections: impl Iterator< - Item = (String, &'a Arc, Option), + Item = ( + &'a validator::PublicKey, + &'a Arc>, + ), >, ) -> String { let mut table = Table::new() .with_custom_header_row( TableRow::new() .with_cell(TableCell::new(TableCellType::Header).with_raw("Public key")) - .with_cell(TableCell::new(TableCellType::Header).with_raw("Address")) - .with_cell(TableCell::new(TableCellType::Header).with_raw("Build version")) - .with_cell( - TableCell::new(TableCellType::Header) - .with_attributes([("colspan", "2")]) - .with_raw("received [B]"), - ) .with_cell( TableCell::new(TableCellType::Header) - .with_attributes([("colspan", "2")]) - .with_raw("sent [B]"), - ) - .with_cell(TableCell::new(TableCellType::Header).with_raw("Age")), + .with_attributes([("colspan", "3")]) + .with_raw("Network address"), + ), ) - .with_header_row(vec!["", "", "", "total", "avg", "total", "avg", ""]); - for (key, stats, build_version) in connections { - let age = SystemTime::now() - .duration_since(stats.established) - .ok() - .unwrap_or_else(|| Duration::new(1, 0)) - .max(Duration::new(1, 0)); // Ensure Duration is not 0 to prevent division by zero - let received = stats.received.load(Ordering::Relaxed); - let sent = stats.sent.load(Ordering::Relaxed); + .with_header_row(vec!["", "Address", "Version", "Timestamp"]); + + for (key, addr) in connections { table.add_body_row(vec![ - Self::shorten(key), + key.encode(), + addr.msg.addr.to_string(), + addr.msg.version.to_string(), + addr.msg.timestamp.to_string(), + ]) + } + + table.to_html_string() + } + + fn attester_table<'a>( + attesters: impl Iterator, + votes: &im::HashMap>>, + ) -> String { + let mut table = Table::new().with_header_row(vec!["Public key", "Weight", "Voted"]); + + for attester in attesters { + let voted = if votes.contains_key(&attester.key) { + "Yes" + } else { + "No" + } + .to_string(); + + table.add_body_row(vec![ + attester.key.encode(), + attester.weight.to_string(), + voted, + ]); + } + + table.to_html_string() + } + + fn gossip_active_table<'a>(connections: impl Iterator>) -> String { + let mut table = Table::new().with_header_row(vec![ + "Public key", + "Address", + "Build version", + "Download speed", + "Download total", + "Upload speed", + "Upload total", + "Age", + ]); + + for connection in connections { + table.add_body_row(vec![ + connection.key.encode(), + connection.stats.peer_addr.to_string(), + connection + .build_version + .as_ref() + .map_or("N/A".to_string(), |v| v.to_string()), + connection + .stats + .received_throughput() + .human_throughput_bytes() + .to_string(), + connection + .stats + .received + .load(Ordering::Relaxed) + .human_count_bytes() + .to_string(), + connection + .stats + .sent_throughput() + .human_throughput_bytes() + .to_string(), + connection + .stats + .sent + .load(Ordering::Relaxed) + .human_count_bytes() + .to_string(), + Self::human_readable_duration( + connection.stats.established.elapsed().whole_seconds() as u64, + ), + ]) + } + + table.to_html_string() + } + + fn validator_active_table<'a>( + connections: impl Iterator)>, + ) -> String { + let mut table = Table::new().with_header_row(vec![ + "Public key", + "Address", + "Download speed", + "Download total", + "Upload speed", + "Upload total", + "Age", + ]); + + for (key, stats) in connections { + table.add_body_row(vec![ + key.encode(), stats.peer_addr.to_string(), - build_version.map(|v| v.to_string()).unwrap_or_default(), - bytesize::to_string(received, false), - // TODO: this is not useful - we should display avg from the last ~1min instead. - bytesize::to_string(received / age.as_secs(), false) + "/s", - bytesize::to_string(sent, false), - bytesize::to_string(sent / age.as_secs(), false) + "/s", - // TODO: this is not a human-friendly format, use days + hours + minutes + seconds, - // or similar. - format!("{}s", age.as_secs()), + stats + .received_throughput() + .human_throughput_bytes() + .to_string(), + stats + .received + .load(Ordering::Relaxed) + .human_count_bytes() + .to_string(), + stats.sent_throughput().human_throughput_bytes().to_string(), + stats + .sent + .load(Ordering::Relaxed) + .human_count_bytes() + .to_string(), + Self::human_readable_duration(stats.established.elapsed().whole_seconds() as u64), ]) } + table.to_html_string() } - fn shorten(key: String) -> String { - key.strip_prefix("validator:public:bls12_381:") - .or(key.strip_prefix("node:public:ed25519:")) - .map_or("-".to_string(), |key| { - let len = key.len(); - format!("{}...{}", &key[..10], &key[len - 11..len]) - }) + /// Returns human readable duration. We use this function instead of `Duration::human_duration` because + /// we want to show days as well. + fn human_readable_duration(seconds: u64) -> String { + let days = seconds / 86400; + let hours = (seconds % 86400) / 3600; + let minutes = (seconds % 3600) / 60; + let seconds = seconds % 60; + + let mut components = Vec::new(); + + if days > 0 { + components.push(format!("{}d", days)); + } + if hours > 0 { + components.push(format!("{}h", hours)); + } + if minutes > 0 { + components.push(format!("{}m", minutes)); + } + + components.push(format!("{}s", seconds)); + components.join(" ") } } diff --git a/node/actors/network/src/gossip/attestation/mod.rs b/node/actors/network/src/gossip/attestation/mod.rs index 3db48085..7bc11d76 100644 --- a/node/actors/network/src/gossip/attestation/mod.rs +++ b/node/actors/network/src/gossip/attestation/mod.rs @@ -19,31 +19,35 @@ pub struct Info { pub committee: Arc, } -// Internal attestation state: info and the set of votes collected so far. +/// Internal attestation state: info and the set of votes collected so far. #[derive(Clone)] -struct State { +pub struct State { + /// Info about the batch to attest and the committee. info: Arc, /// Votes collected so far. votes: im::HashMap>>, - // Total weight of the votes collected. + /// Total weight of the votes collected. total_weight: attester::Weight, } -/// Diff between 2 states. -pub(crate) struct Diff { - /// New votes. - pub(crate) votes: Vec>>, - /// New info, if changed. - pub(crate) info: Option>, -} +impl State { + /// Returns a reference to the `info` field. + pub fn info(&self) -> &Arc { + &self.info + } -impl Diff { - fn is_empty(&self) -> bool { - self.votes.is_empty() && self.info.is_none() + /// Returns a reference to the `votes` field. + pub fn votes( + &self, + ) -> &im::HashMap>> { + &self.votes + } + + /// Returns a reference to the `total_weight` field. + pub fn total_weight(&self) -> &attester::Weight { + &self.total_weight } -} -impl State { /// Returns a diff between `self` state and `old` state. /// Diff contains votes which are present is `self`, but not in `old`. fn diff(&self, old: &Option) -> Diff { @@ -140,6 +144,20 @@ impl State { } } +/// Diff between 2 states. +pub(crate) struct Diff { + /// New votes. + pub(crate) votes: Vec>>, + /// New info, if changed. + pub(crate) info: Option>, +} + +impl Diff { + fn is_empty(&self) -> bool { + self.votes.is_empty() && self.info.is_none() + } +} + /// Receiver of state diffs. pub(crate) struct DiffReceiver { prev: Option, @@ -246,6 +264,16 @@ impl Controller { } } + /// Returns a reference to the key, if it exists. + pub fn key(&self) -> Option<&attester::SecretKey> { + self.key.as_ref() + } + + /// Returns a reference to the state. + pub(crate) fn state(&self) -> &Watch> { + &self.state + } + /// Registers metrics for this controller. pub(crate) fn register_metrics(self: &Arc) { metrics::Metrics::register(Arc::downgrade(self)); diff --git a/node/actors/network/src/gossip/fetch.rs b/node/actors/network/src/gossip/fetch.rs index 5ee58c07..f74e3e5a 100644 --- a/node/actors/network/src/gossip/fetch.rs +++ b/node/actors/network/src/gossip/fetch.rs @@ -31,6 +31,19 @@ impl Default for Queue { } impl Queue { + /// Returns the sorted list of currently requested blocks. + pub(crate) fn current_blocks(&self) -> Vec { + let mut blocks = self + .blocks + .subscribe() + .borrow() + .keys() + .map(|x| x.0) + .collect::>(); + blocks.sort(); + blocks + } + /// Requests a resource from peers and waits until it is stored. /// Note: in the current implementation concurrent calls for the same resource number are /// unsupported - second call will override the first call. diff --git a/node/actors/network/src/gossip/state.rs b/node/actors/network/src/gossip/state.rs deleted file mode 100644 index 8b137891..00000000 --- a/node/actors/network/src/gossip/state.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/node/actors/network/src/gossip/validator_addrs.rs b/node/actors/network/src/gossip/validator_addrs.rs index 51c736af..148d044a 100644 --- a/node/actors/network/src/gossip/validator_addrs.rs +++ b/node/actors/network/src/gossip/validator_addrs.rs @@ -90,6 +90,13 @@ impl ValidatorAddrsWatch { self.0.subscribe() } + /// Copy of the current ValidatorAddrs state. + pub(crate) fn current( + &self, + ) -> im::HashMap>> { + self.0.subscribe().borrow().0.clone() + } + /// Inserts a new version of the announcement signed with the given key. pub(crate) async fn announce( &self, diff --git a/node/actors/network/src/metrics.rs b/node/actors/network/src/metrics.rs index 3ecbd60e..97ca01e3 100644 --- a/node/actors/network/src/metrics.rs +++ b/node/actors/network/src/metrics.rs @@ -1,4 +1,7 @@ //! General-purpose network metrics. + +#![allow(clippy::float_arithmetic)] + use crate::Network; use anyhow::Context as _; use std::{ @@ -10,12 +13,11 @@ use std::{ Arc, Weak, }, task::{ready, Context, Poll}, - time::SystemTime, }; use vise::{ Collector, Counter, EncodeLabelSet, EncodeLabelValue, Family, Gauge, GaugeGuard, Metrics, Unit, }; -use zksync_concurrency::{ctx, io, net}; +use zksync_concurrency::{ctx, io, net, time::Instant}; /// Metered TCP stream. #[pin_project::pin_project] @@ -227,10 +229,20 @@ pub struct MeteredStreamStats { pub sent: AtomicU64, /// Total bytes received over the Stream. pub received: AtomicU64, - /// System time since the connection started. - pub established: SystemTime, - /// Ip Address and port of current connection. + /// Time when the connection started. + pub established: Instant, + /// IP Address and port of current connection. pub peer_addr: SocketAddr, + /// Total bytes sent in the current minute. + pub current_minute_sent: AtomicU64, + /// Total bytes sent in the previous minute. + pub previous_minute_sent: AtomicU64, + /// Total bytes received in the current minute. + pub current_minute_received: AtomicU64, + /// Total bytes received in the previous minute. + pub previous_minute_received: AtomicU64, + /// Minutes elapsed since the connection started, when this metrics were last updated. + pub minutes_elapsed_last: AtomicU64, } impl MeteredStreamStats { @@ -238,16 +250,81 @@ impl MeteredStreamStats { Self { sent: 0.into(), received: 0.into(), - established: SystemTime::now(), + established: Instant::now(), peer_addr, + current_minute_sent: 0.into(), + previous_minute_sent: 0.into(), + current_minute_received: 0.into(), + previous_minute_received: 0.into(), + minutes_elapsed_last: 0.into(), } } fn read(&self, amount: u64) { + self.update_minute(); self.received.fetch_add(amount, Ordering::Relaxed); + self.current_minute_received + .fetch_add(amount, Ordering::Relaxed); } fn wrote(&self, amount: u64) { + self.update_minute(); self.sent.fetch_add(amount, Ordering::Relaxed); + self.current_minute_sent + .fetch_add(amount, Ordering::Relaxed); + } + + fn update_minute(&self) { + let elapsed_minutes_now = self.established.elapsed().whole_seconds() as u64 / 60; + let elapsed_minutes_last = self.minutes_elapsed_last.load(Ordering::Relaxed); + + if elapsed_minutes_now > elapsed_minutes_last { + if elapsed_minutes_now - elapsed_minutes_last > 1 { + self.previous_minute_sent.store(0, Ordering::Relaxed); + self.previous_minute_received.store(0, Ordering::Relaxed); + } else { + self.previous_minute_sent.store( + self.current_minute_sent.load(Ordering::Relaxed), + Ordering::Relaxed, + ); + self.previous_minute_received.store( + self.current_minute_received.load(Ordering::Relaxed), + Ordering::Relaxed, + ); + } + + self.current_minute_sent.store(0, Ordering::Relaxed); + self.current_minute_received.store(0, Ordering::Relaxed); + self.minutes_elapsed_last + .store(elapsed_minutes_now, Ordering::Relaxed); + } + } + + /// Returns the upload throughput of the connection in bytes per second. + pub fn sent_throughput(&self) -> f64 { + let elapsed_minutes_now = self.established.elapsed().whole_seconds() as u64 / 60; + let elapsed_minutes_last = self.minutes_elapsed_last.load(Ordering::Relaxed); + + if elapsed_minutes_now - elapsed_minutes_last == 0 { + self.previous_minute_sent.load(Ordering::Relaxed) as f64 / 60.0 + } else if elapsed_minutes_now - elapsed_minutes_last == 1 { + self.current_minute_sent.load(Ordering::Relaxed) as f64 / 60.0 + } else { + 0.0 + } + } + + /// Returns the download throughput of the connection in bytes per second. + pub fn received_throughput(&self) -> f64 { + let elapsed_minutes_now = self.established.elapsed().whole_seconds() as u64 / 60; + let elapsed_minutes_last = self.minutes_elapsed_last.load(Ordering::Relaxed); + + if elapsed_minutes_now - elapsed_minutes_last == 0 { + self.previous_minute_received.load(Ordering::Relaxed) as f64 / 60.0 + } else if elapsed_minutes_now - elapsed_minutes_last == 1 { + self.current_minute_received.load(Ordering::Relaxed) as f64 / 60.0 + } else { + 0.0 + } } } diff --git a/node/clippy.toml b/node/clippy.toml index 5e0d86e6..6b5cca0f 100644 --- a/node/clippy.toml +++ b/node/clippy.toml @@ -1,2 +1,2 @@ enum-variant-size-threshold = 500 -missing-docs-in-crate-items = true \ No newline at end of file +missing-docs-in-crate-items = true diff --git a/node/deny.toml b/node/deny.toml index a20c3658..b5bda861 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -1,8 +1,8 @@ [graph] # We only check dependencies against these platforms. targets = [ - { triple = "x86_64-unknown-linux-musl" }, { triple = "x86_64-apple-darwin" }, + { triple = "x86_64-unknown-linux-musl" }, ] [licenses] @@ -14,9 +14,9 @@ allow = [ "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", - "OpenSSL", "ISC", "MIT", + "OpenSSL", "Unicode-DFS-2016", # Weak copyleft licenses "MPL-2.0", @@ -31,7 +31,7 @@ name = "ring" # license, for third_party/fiat, which, unlike other third_party directories, is # compiled into non-test libraries, is included below." # OpenSSL - Obviously -expression = "MIT" +expression = "MIT" license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] [bans] @@ -53,14 +53,14 @@ skip = [ { name = "base64", version = "0.21.7" }, # Old versions required by vise-exporter. - { name = "http", version = "0.2.12"}, - { name = "http-body", version = "0.4.6"}, - { name = "hyper", version = "0.14.28"}, + { name = "http", version = "0.2.12" }, + { name = "http-body", version = "0.4.6" }, + { name = "hyper", version = "0.14.28" }, # Old version required by rand. { name = "zerocopy", version = "0.6.6" }, ] [sources] +unknown-git = "deny" unknown-registry = "deny" -unknown-git = "deny" diff --git a/node/libs/concurrency/Cargo.toml b/node/libs/concurrency/Cargo.toml index a87980ac..2a79cbb3 100644 --- a/node/libs/concurrency/Cargo.toml +++ b/node/libs/concurrency/Cargo.toml @@ -1,26 +1,26 @@ [package] -name = "zksync_concurrency" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "Structured concurrency framework" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_concurrency" repository.workspace = true -keywords.workspace = true -description = "Structured concurrency framework" +version.workspace = true [dependencies] -anyhow.workspace = true -once_cell.workspace = true -pin-project.workspace = true -rand.workspace = true -sha3.workspace = true -thiserror.workspace = true -time.workspace = true -tokio.workspace = true -tracing.workspace = true +anyhow.workspace = true +once_cell.workspace = true +pin-project.workspace = true +rand.workspace = true +sha3.workspace = true +thiserror.workspace = true +time.workspace = true +tokio.workspace = true tracing-subscriber.workspace = true -vise.workspace = true +tracing.workspace = true +vise.workspace = true [lints] workspace = true diff --git a/node/libs/crypto/Cargo.toml b/node/libs/crypto/Cargo.toml index 0cb53a29..682deedf 100644 --- a/node/libs/crypto/Cargo.toml +++ b/node/libs/crypto/Cargo.toml @@ -1,28 +1,28 @@ [package] -name = "zksync_consensus_crypto" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus cryptographic utilities" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_consensus_crypto" repository.workspace = true -keywords.workspace = true -description = "ZKsync consensus cryptographic utilities" +version.workspace = true [dependencies] -anyhow.workspace = true -blst.workspace = true -ed25519-dalek.workspace = true +anyhow.workspace = true +blst.workspace = true +ed25519-dalek.workspace = true elliptic-curve.workspace = true -hex.workspace = true -k256.workspace = true -num-bigint.workspace = true -num-traits.workspace = true -rand.workspace = true -sha3.workspace = true -thiserror.workspace = true -tracing.workspace = true -zeroize.workspace = true +hex.workspace = true +k256.workspace = true +num-bigint.workspace = true +num-traits.workspace = true +rand.workspace = true +sha3.workspace = true +thiserror.workspace = true +tracing.workspace = true +zeroize.workspace = true [dev-dependencies] criterion.workspace = true @@ -31,5 +31,5 @@ criterion.workspace = true workspace = true [[bench]] -name = "bench" harness = false +name = "bench" diff --git a/node/libs/protobuf/Cargo.toml b/node/libs/protobuf/Cargo.toml index 64cd3c12..afa981fe 100644 --- a/node/libs/protobuf/Cargo.toml +++ b/node/libs/protobuf/Cargo.toml @@ -1,34 +1,34 @@ [package] -name = "zksync_protobuf" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus protobuf types and utilities" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +links = "zksync_protobuf_proto" +name = "zksync_protobuf" repository.workspace = true -keywords.workspace = true -links = "zksync_protobuf_proto" -description = "ZKsync consensus protobuf types and utilities" +version.workspace = true [dependencies] -zksync_concurrency.workspace = true +zksync_concurrency.workspace = true zksync_consensus_utils.workspace = true -anyhow.workspace = true -bit-vec.workspace = true -once_cell.workspace = true -prost.workspace = true -prost-reflect.workspace = true +anyhow.workspace = true +bit-vec.workspace = true +once_cell.workspace = true +prost-reflect.workspace = true +prost.workspace = true quick-protobuf.workspace = true -rand.workspace = true -serde.workspace = true -serde_json.workspace = true -serde_yaml.workspace = true +rand.workspace = true +serde.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true [dev-dependencies] -tokio.workspace = true -tracing.workspace = true +tokio.workspace = true tracing-subscriber.workspace = true +tracing.workspace = true [build-dependencies] zksync_protobuf_build.workspace = true diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml index 8150a8d6..889cc5e1 100644 --- a/node/libs/protobuf_build/Cargo.toml +++ b/node/libs/protobuf_build/Cargo.toml @@ -1,24 +1,24 @@ [package] -name = "zksync_protobuf_build" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus protobuf codegen" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_protobuf_build" repository.workspace = true -keywords.workspace = true -description = "ZKsync consensus protobuf codegen" +version.workspace = true [dependencies] -anyhow.workspace = true -heck.workspace = true -prettyplease.workspace = true -proc-macro2.workspace = true -prost-build.workspace = true +anyhow.workspace = true +heck.workspace = true +prettyplease.workspace = true +proc-macro2.workspace = true +prost-build.workspace = true prost-reflect.workspace = true -protox.workspace = true -quote.workspace = true -syn.workspace = true +protox.workspace = true +quote.workspace = true +syn.workspace = true [lints] workspace = true diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index 52e1656c..07be3ff1 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -1,30 +1,30 @@ [package] -name = "zksync_consensus_roles" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus node role types" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +links = "zksync_consensus_roles_proto" +name = "zksync_consensus_roles" repository.workspace = true -keywords.workspace = true -links = "zksync_consensus_roles_proto" -description = "ZKsync consensus node role types" +version.workspace = true [dependencies] -zksync_concurrency.workspace = true +zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true -zksync_consensus_utils.workspace = true -zksync_protobuf.workspace = true +zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true -anyhow.workspace = true -bit-vec.workspace = true -hex.workspace = true -prost.workspace = true -rand.workspace = true -serde.workspace = true -thiserror.workspace = true -tracing.workspace = true +anyhow.workspace = true +bit-vec.workspace = true +hex.workspace = true num-bigint.workspace = true +prost.workspace = true +rand.workspace = true +serde.workspace = true +thiserror.workspace = true +tracing.workspace = true [dev-dependencies] assert_matches.workspace = true diff --git a/node/libs/roles/src/attester/messages/msg.rs b/node/libs/roles/src/attester/messages/msg.rs index bb8d0c7e..8513a638 100644 --- a/node/libs/roles/src/attester/messages/msg.rs +++ b/node/libs/roles/src/attester/messages/msg.rs @@ -123,6 +123,11 @@ impl Committee { }) } + /// Iterates over the attesters. + pub fn iter(&self) -> impl Iterator { + self.vec.iter() + } + /// Iterates over attester keys. pub fn keys(&self) -> impl Iterator { self.vec.iter().map(|v| &v.key) diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index 5989b005..2b27fc67 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -1,33 +1,33 @@ [package] -name = "zksync_consensus_storage" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus storage" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_consensus_storage" repository.workspace = true -keywords.workspace = true -description = "ZKsync consensus storage" +version.workspace = true [dependencies] -zksync_concurrency.workspace = true -zksync_consensus_roles.workspace = true +zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true -zksync_protobuf.workspace = true +zksync_consensus_roles.workspace = true +zksync_protobuf.workspace = true -anyhow.workspace = true +anyhow.workspace = true async-trait.workspace = true -prost.workspace = true -rand.workspace = true -thiserror.workspace = true -tracing.workspace = true -vise.workspace = true +prost.workspace = true +rand.workspace = true +thiserror.workspace = true +tracing.workspace = true +vise.workspace = true [dev-dependencies] assert_matches.workspace = true -tempfile.workspace = true -test-casing.workspace = true -tokio.workspace = true +tempfile.workspace = true +test-casing.workspace = true +tokio.workspace = true [build-dependencies] zksync_protobuf_build.workspace = true diff --git a/node/libs/utils/Cargo.toml b/node/libs/utils/Cargo.toml index bd92cbd0..6c01fb62 100644 --- a/node/libs/utils/Cargo.toml +++ b/node/libs/utils/Cargo.toml @@ -1,19 +1,19 @@ [package] -name = "zksync_consensus_utils" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +description = "ZKsync consensus utilities" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_consensus_utils" repository.workspace = true -keywords.workspace = true -description = "ZKsync consensus utilities" +version.workspace = true [dependencies] +anyhow.workspace = true +rand.workspace = true +thiserror.workspace = true zksync_concurrency.workspace = true -rand.workspace = true -thiserror.workspace = true -anyhow.workspace = true [lints] workspace = true diff --git a/node/tests/Cargo.toml b/node/tests/Cargo.toml index 7f1e33ce..7cec9aae 100644 --- a/node/tests/Cargo.toml +++ b/node/tests/Cargo.toml @@ -1,24 +1,24 @@ [package] -name = "tester" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "tester" +publish = false repository.workspace = true -keywords.workspace = true -publish = false +version.workspace = true [dependencies] zksync_consensus_tools.workspace = true -serde_json.workspace = true -tokio.workspace = true -jsonrpsee.workspace = true -clap.workspace = true -tracing.workspace = true -anyhow.workspace = true +anyhow.workspace = true +clap.workspace = true +jsonrpsee.workspace = true +serde_json.workspace = true +tokio.workspace = true tracing-subscriber.workspace = true +tracing.workspace = true [lints] workspace = true diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 2134e4b1..fa904c9b 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -1,45 +1,45 @@ [package] -name = "zksync_consensus_tools" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true +authors.workspace = true +default-run = "executor" +description = "ZKsync consensus tools" +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "zksync_consensus_tools" +publish = false repository.workspace = true -keywords.workspace = true -publish = false -default-run = "executor" -description = "ZKsync consensus tools" +version.workspace = true [dependencies] -zksync_concurrency.workspace = true -zksync_consensus_bft.workspace = true -zksync_consensus_crypto.workspace = true +zksync_concurrency.workspace = true +zksync_consensus_bft.workspace = true +zksync_consensus_crypto.workspace = true zksync_consensus_executor.workspace = true -zksync_consensus_roles.workspace = true -zksync_consensus_storage.workspace = true -zksync_consensus_utils.workspace = true -zksync_consensus_network.workspace = true -zksync_protobuf.workspace = true +zksync_consensus_network.workspace = true +zksync_consensus_roles.workspace = true +zksync_consensus_storage.workspace = true +zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true -anyhow.workspace = true -async-trait.workspace = true -clap.workspace = true -prost.workspace = true -rand.workspace = true -rocksdb.workspace = true -serde.workspace = true -serde_json.workspace = true -tokio.workspace = true -tracing.workspace = true +anyhow.workspace = true +async-trait.workspace = true +clap.workspace = true +jsonrpsee.workspace = true +k8s-openapi.workspace = true +kube.workspace = true +prost.workspace = true +rand.workspace = true +rocksdb.workspace = true +rustls-pemfile.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio-rustls.workspace = true +tokio.workspace = true +tower.workspace = true tracing-subscriber.workspace = true -vise-exporter.workspace = true -jsonrpsee.workspace = true -tower.workspace = true -kube.workspace = true -k8s-openapi.workspace = true -tokio-rustls.workspace = true -rustls-pemfile.workspace = true +tracing.workspace = true +vise-exporter.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/spec/informal-spec/proposer.rs b/spec/informal-spec/proposer.rs index 12268201..49b7d9cf 100644 --- a/spec/informal-spec/proposer.rs +++ b/spec/informal-spec/proposer.rs @@ -15,13 +15,21 @@ fn on_start(replica_state: &ReplicaState) { // Get the justification for this view. If we have both a commit QC // and a timeout QC for this view (highly unlikely), we should prefer // to use the commit QC. - let justification = replica_state.get_justification(cur_view); + let justification = replica_state.create_justification(); + + assert!(justification.view() == cur_view); // Get the block number and check if this must be a reproposal. let (block_number, opt_block_hash) = justification.get_implied_block(); // Propose only if you have collected all committed blocks so far. - assert!(block_number == self.committed_blocks.last().map_or(0,|b|b.commit_qc.vote.block_number+1)); + assert!( + block_number + == self + .committed_blocks + .last() + .map_or(0, |b| b.commit_qc.vote.block_number + 1) + ); // Now we create the block. let block = if opt_block_hash.is_some() { diff --git a/spec/informal-spec/replica.rs b/spec/informal-spec/replica.rs index 901c5dc2..5c6e692b 100644 --- a/spec/informal-spec/replica.rs +++ b/spec/informal-spec/replica.rs @@ -1,12 +1,5 @@ // Replica -/// A block with a matching valid certificate. -/// invariants: hash(block) == commit_qc.vote.block_hash -struct CommittedBlock { - block: Block, - commit_qc: CommitQC, -} - struct ReplicaState { // The view this replica is currently in. view: ViewNumber, @@ -40,217 +33,231 @@ enum Phase { Timeout } -// This is called when the replica starts. At the beginning of the consensus. -// It is a loop that takes incoming messages and calls the corresponding -// method for each message. -fn on_start(self) { - // Imagine there's a timer util that just has two states (finished or not) and - // counts down from some given duration. For example, it counts down from 1s. - // If that time elapses, the timer will change state to finished. - // If it is reset before that, it starts counting down again from 1s. - let timer = Timer::new(duration); - - // Get the current view. - let mut cur_view = self.view; - - loop { - // If the view has increased before the timeout, we reset the timer. - if cur_view < self.view { - cur_view = self.view; - timer.reset(); - } - - // If the timer has finished, we send a timeout vote. - // If this is the first view, we immediately timeout. This will force the replicas - // to synchronize right at the beginning and will provide a justification for the - // proposer at view 1. - // If we have already timed out, we don't need to send another timeout vote. - if (timer.is_finished() || cur_view == 0) && self.phase != Phase::Timeout { - let vote = TimeoutVote::new(self.view, - self.high_vote, - self.high_commit_qc); - // Update our state so that we can no longer vote commit in this view. - self.phase = Phase::Timeout; - - // Send the vote to all replicas (including ourselves). - self.send(vote); - } - - // Try to get a message from the message queue and process it. We don't - // detail the message queue structure since it's boilerplate. - if let Some(message) = message_queue.pop() { - match message { - Proposal(msg) => { - self.on_proposal(msg); - } - Commit(msg) => { - self.on_commit(msg); - } - Timeout(msg) => { - self.on_timeout(msg); - } - NewView(msg) => { - self.on_new_view(msg); +impl ReplicaState { + // This is called when the replica starts. At the beginning of the consensus. + // It is a loop that takes incoming messages and calls the corresponding + // method for each message. + fn on_start(&mut self) { + // Imagine there's a timer util that just has two states (finished or not) and + // counts down from some given duration. For example, it counts down from 1s. + // If that time elapses, the timer will change state to finished. + // If it is reset before that, it starts counting down again from 1s. + let timer = Timer::new(duration); + + // Get the current view. + let mut cur_view = self.view; + + loop { + // If the view has increased before the timeout, we reset the timer. + if cur_view < self.view { + cur_view = self.view; + timer.reset(); + } + + // If the timer has finished, we send a timeout vote. + // If this is the first view, we immediately timeout. This will force the replicas + // to synchronize right at the beginning and will provide a justification for the + // proposer at view 1. + // If we have already timed out, we don't need to send another timeout vote. + if (timer.is_finished() || cur_view == 0) && self.phase != Phase::Timeout { + let vote = TimeoutVote::new(self.view, + self.high_vote, + self.high_commit_qc); + + // Update our state so that we can no longer vote commit in this view. + self.phase = Phase::Timeout; + + // Send the vote to all replicas (including ourselves). + self.send(vote); + } + + // Try to get a message from the message queue and process it. We don't + // detail the message queue structure since it's boilerplate. + if let Some(message) = message_queue.pop() { + match message { + Proposal(msg) => { + self.on_proposal(msg); + } + Commit(msg) => { + self.on_commit(msg); + } + Timeout(msg) => { + self.on_timeout(msg); + } + NewView(msg) => { + self.on_new_view(msg); + } } } } + } + + fn on_proposal(&mut self, proposal: Proposal) { + // We only allow proposals for the current view if we have not voted in + // it yet. + assert!((proposal.view() == self.view && self.phase == Prepare) || proposal.view() > self.view); + + // We ignore proposals from the wrong leader. + assert!(proposal.leader() == leader(proposal.view())); + + // Check that the proposal is valid. + assert!(proposal.verify()); + + // Get the implied block number and hash (if any). + let (block_number, opt_block_hash) = proposal.justification.get_implied_block(); + + // Vote only if you have collected all committed blocks so far. + assert!(block_number == self.committed_blocks.last().map_or(0,|b|b.commit_qc.vote.block_number+1)); + + // Check if this is a reproposal or not, and do the necessary checks. + // As a side result, get the correct block hash. + let block_hash = match opt_block_hash { + Some(hash) => { + // This is a reproposal. We let the leader repropose blocks without sending + // them in the proposal (it sends only the number + hash). That allows a + // leader to repropose a block without having it stored. + // It is an optimization that allows us to not wait for a leader that has + // the previous proposal stored (which can take 4f views), and to somewhat + // speed up reproposals by skipping block broadcast. + // This only saves time because we have a gossip network running in parallel, + // and any time a replica is able to create a finalized block (by possessing + // both the block and the commit QC) it broadcasts the finalized block (this + // was meant to propagate the block to full nodes, but of course validators + // will end up receiving it as well). + // However, this can be difficult to model and we might want to just + // ignore the gossip network in the formal model. We will still have liveness + // but in the model we'll end up waiting 4f views to get a leader that has the + // previous block before proposing a new one. This is not that bad, since + // then we can be sure that the consensus will continue even if the gossip + // network is failing for some reason. + + // For sanity reasons, we'll check that there's no block in the proposal. + // But this check is completely unnecessary (in theory at least). + assert!(proposal.block.is_none()); + + hash + } + None => { + // This is a new proposal, so we need to verify it (i.e. execute it). + assert!(proposal.block.is_some()); + let block = proposal.block.unwrap(); + // To verify the block, replica just tries to apply it to the current + // state. Current state is the result of applying all the committed blocks until now. + assert!(self.verify_block(block_number, block)); + // We cache the new proposals, waiting for them to be committed. + self.cached_proposals.insert((block_number,proposal.block.hash()),block); + block.hash() + } + }; + + // Update the state. + let vote = CommitVote::new(proposal.view(), block_number, block_hash); + + self.view = proposal.view(); + self.phase = Phase::Commit; + self.high_vote = Some(vote); + match proposal.justification { + Commit(qc) => self.process_commit_qc(Some(qc)), + Timeout(qc) => { + self.process_commit_qc(qc.high_commit_qc); + self.high_timeout_qc = max(Some(qc), self.high_timeout_qc); + } + }; + + // Send the commit vote to all replicas (including ourselves). + self.send(vote); } -} - -fn on_proposal(self, proposal: Proposal) { - // We only allow proposals for the current view if we have not voted in - // it yet. - assert!((proposal.view() == self.view && self.phase == Prepare) || proposal.view() > self.view); - - // We ignore proposals from the wrong leader. - assert!(proposal.leader() == leader(proposal.view())); - - // Check that the proposal is valid. - assert!(proposal.verify()); - // Get the implied block number and hash (if any). - let (block_number, opt_block_hash) = proposal.justification.get_implied_block(); - - // Vote only if you have collected all committed blocks so far. - assert!(block_number == self.committed_blocks.last().map_or(0,|b|b.commit_qc.vote.block_number+1)); - - // Check if this is a reproposal or not, and do the necessary checks. - // As a side result, get the correct block hash. - let block_hash = match opt_block_hash { - Some(hash) => { - // This is a reproposal. We let the leader repropose blocks without sending - // them in the proposal (it sends only the number + hash). That allows a - // leader to repropose a block without having it stored. - // It is an optimization that allows us to not wait for a leader that has - // the previous proposal stored (which can take 4f views), and to somewhat - // speed up reproposals by skipping block broadcast. - // This only saves time because we have a gossip network running in parallel, - // and any time a replica is able to create a finalized block (by possessing - // both the block and the commit QC) it broadcasts the finalized block (this - // was meant to propagate the block to full nodes, but of course validators - // will end up receiving it as well). - // However, this can be difficult to model and we might want to just - // ignore the gossip network in the formal model. We will still have liveness - // but in the model we'll end up waiting 4f views to get a leader that has the - // previous block before proposing a new one. This is not that bad, since - // then we can be sure that the consensus will continue even if the gossip - // network is failing for some reason. - - // For sanity reasons, we'll check that there's no block in the proposal. - // But this check is completely unnecessary (in theory at least). - assert!(proposal.block.is_none()); - - hash + // Processed an (already verified) commit_qc received from the network + // as part of some message. It bumps the local high_commit_qc and if + // we have the proposal corresponding to this qc, we append it to the committed_blocks. + fn process_commit_qc(&mut self, qc_opt: Option) { + if let Some(qc) = qc_opt { + self.high_commit_qc = max(Some(qc), self.high_commit_qc); + let Some(block) = self.cached_proposals.get((qc.vote.block_number,qc.vote.block_hash)) else { return }; + if self.committed_blocks.len()==qc.vote.block_number { + self.committed_blocks.push(CommittedBlock{block,commit_qc:qc}); + } } - None => { - // This is a new proposal, so we need to verify it (i.e. execute it). - assert!(proposal.block.is_some()); - let block = proposal.block.unwrap(); - // To verify the block, replica just tries to apply it to the current - // state. Current state is the result of applying all the committed blocks until now. - assert!(self.verify_block(block_number, block)); - // We cache the new proposals, waiting for them to be committed. - self.cached_proposals.insert((block_number,proposal.block.hash()),block); - block.hash() + } + + fn on_commit(&mut self, sig_vote: SignedCommitVote) { + // If the vote isn't current, just ignore it. + assert!(sig_vote.view() >= self.view) + + // Check that the signed vote is valid. + assert!(sig_vote.verify()); + + // Store the vote. We will never store duplicate (same view and sender) votes. + // If we already have this vote, we exit early. + assert!(self.store(sig_vote).is_ok()); + + // Check if we now have a commit QC for this view. + if let Some(qc) = self.get_commit_qc(sig_vote.view()) { + self.process_commit_qc(Some(qc)); + self.start_new_view(sig_vote.view() + 1); } - }; - - // Update the state. - let vote = CommitVote::new(proposal.view(), block_number, block_hash); - - self.view = proposal.view(); - self.phase = Phase::Commit; - self.high_vote = Some(vote); - match proposal.justification { - Commit(qc) => self.process_commit_qc(Some(qc)), - Timeout(qc) => { + } + + fn on_timeout(&mut self, sig_vote: SignedTimeoutVote) { + // If the vote isn't current, just ignore it. + assert!(sig_vote.view() >= self.view) + + // Check that the signed vote is valid. + assert!(sig_vote.verify()); + + // Store the vote. We will never store duplicate (same view and sender) votes. + // If we already have this vote, we exit early. + assert!(self.store(sig_vote).is_ok()); + + // Check if we now have a timeout QC for this view. + if let Some(qc) = self.get_timeout_qc(sig_vote.view()) { self.process_commit_qc(qc.high_commit_qc); self.high_timeout_qc = max(Some(qc), self.high_timeout_qc); - } - }; - - // Send the commit vote to all replicas (including ourselves). - self.send(vote); -} - -// Processed an (already verified) commit_qc received from the network -// as part of some message. It bumps the local high_commit_qc and if -// we have the proposal corresponding to this qc, we append it to the committed_blocks. -fn process_commit_qc(self, qc_opt: Option) { - if let Some(qc) = qc_opt { - self.high_commit_qc = max(Some(qc), self.high_commit_qc); - let Some(block) = self.cached_proposals.get((qc.vote.block_number,qc.vote.block_hash)) else { return }; - if self.committed_blocks.len()==qc.vote.block_number { - self.committed_blocks.push(CommittedBlock{block,commit_qc:qc}); + self.start_new_view(sig_vote.view() + 1); } } -} - -fn on_commit(self, sig_vote: SignedCommitVote) { - // If the vote isn't current, just ignore it. - assert!(sig_vote.view() >= self.view) - - // Check that the signed vote is valid. - assert!(sig_vote.verify()); - - // Store the vote. We will never store duplicate (same view and sender) votes. - // If we already have this vote, we exit early. - assert!(self.store(sig_vote).is_ok()); - - // Check if we now have a commit QC for this view. - if let Some(qc) = self.get_commit_qc(sig_vote.view()) { - self.process_commit_qc(Some(qc)); - self.start_new_view(sig_vote.view() + 1); + + fn on_new_view(&mut self, new_view: NewView) { + // If the message isn't current, just ignore it. + assert!(new_view.view() >= self.view) + + // Check that the new view is valid. + assert!(new_view.verify()); + + // Update our state. + match new_view.justification { + Commit(qc) => self.process_commit_qc(Some(qc)), + Timeout(qc) => { + self.process_commit_qc(qc.high_commit_qc); + self.high_timeout_qc = max(Some(qc), self.high_timeout_qc); + } + }; + + if new_view.view() > self.view { + self.start_new_view(new_view.view()); + } } -} - -fn on_timeout(self, sig_vote: SignedTimeoutVote) { - // If the vote isn't current, just ignore it. - assert!(sig_vote.view() >= self.view) - - // Check that the signed vote is valid. - assert!(sig_vote.verify()); - - // Store the vote. We will never store duplicate (same view and sender) votes. - // If we already have this vote, we exit early. - assert!(self.store(sig_vote).is_ok()); - - // Check if we now have a timeout QC for this view. - if let Some(qc) = self.get_timeout_qc(sig_vote.view()) { - self.process_commit_qc(qc.high_commit_qc); - self.high_timeout_qc = max(Some(qc), self.high_timeout_qc); - self.start_new_view(sig_vote.view() + 1); + + fn start_new_view(&mut self, view: ViewNumber) { + self.view = view; + self.phase = Phase::Prepare; + + // Send a new view message to the other replicas, for synchronization. + let new_view = NewView::new(self.get_justification(view)); + + self.send(new_view); } -} -fn on_new_view(self, new_view: NewView) { - // If the message isn't current, just ignore it. - assert!(new_view.view() >= self.view) + fn create_justification(&self) { + // We need some QC in order to be able to create a justification. + assert!(self.high_commit_qc.is_some() || self.high_timeout_qc.is_some()); - // Check that the new view is valid. - assert!(new_view.verify()); - - // Update our state. - match new_view.justification { - Commit(qc) => self.process_commit_qc(Some(qc)), - Timeout(qc) => { - self.process_commit_qc(qc.high_commit_qc); - self.high_timeout_qc = max(Some(qc), self.high_timeout_qc); + if self.high_commit_qc.map(|x| x.view()) >= self.high_timeout_qc.map(|x| x.view()) { + Justification::Commit(self.high_commit_qc.unwrap()) + } else { + Justification::Timeout(self.high_timeout_qc.unwrap()) } - }; - - if new_view.view() > self.view { - self.start_new_view(new_view.view()); } -} - -fn start_new_view(self, view: ViewNumber) { - self.view = view; - self.phase = Phase::Prepare; - - // Send a new view message to the other replicas, for synchronization. - let new_view = NewView::new(self.get_justification(view)); - - self.send(new_view); -} +} \ No newline at end of file diff --git a/spec/informal-spec/types.rs b/spec/informal-spec/types.rs index 7dd9885e..f5789cff 100644 --- a/spec/informal-spec/types.rs +++ b/spec/informal-spec/types.rs @@ -8,6 +8,13 @@ const QUORUM_WEIGHT = TOTAL_WEIGHT - FAULTY_WEIGHT; // The weight threshold needed to trigger a reproposal. const SUBQUORUM_WEIGHT = TOTAL_WEIGHT - 3 * FAULTY_WEIGHT; +/// A block with a matching valid certificate. +/// invariants: hash(block) == commit_qc.vote.block_hash +struct CommittedBlock { + block: Block, + commit_qc: CommitQC, +} + // Messages struct Proposal { @@ -39,6 +46,7 @@ enum Justification { // A timeout QC is just a collection of timeout votes (with at least // QUORUM_WEIGHT) for the previous view. Unlike with the Commit QC, // timeout votes don't need to be identical. + // The first proposal, for view 0, will always be a timeout. Timeout(TimeoutQC), }