diff --git a/Cargo.lock b/Cargo.lock index 7892e3a2e909..40615537255f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8646,6 +8646,7 @@ dependencies = [ "zksync_commitment_generator", "zksync_concurrency", "zksync_config", + "zksync_consensus_crypto", "zksync_consensus_roles", "zksync_consistency_checker", "zksync_contracts", diff --git a/core/bin/external_node/Cargo.toml b/core/bin/external_node/Cargo.toml index 84c0ddd16e09..63389175912f 100644 --- a/core/bin/external_node/Cargo.toml +++ b/core/bin/external_node/Cargo.toml @@ -47,6 +47,7 @@ zksync_vlog.workspace = true zksync_concurrency.workspace = true zksync_consensus_roles.workspace = true +zksync_consensus_crypto.workspace = true vise.workspace = true async-trait.workspace = true diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 9c4e9657084f..120df5f139fa 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -17,6 +17,8 @@ use zksync_config::{ }, ObjectStoreConfig, }; +use zksync_consensus_crypto::TextFmt; +use zksync_consensus_roles as roles; use zksync_core_leftovers::temp_config_store::{decode_yaml_repr, read_yaml_repr}; #[cfg(test)] use zksync_dal::{ConnectionPool, Core}; @@ -1126,6 +1128,21 @@ impl ExperimentalENConfig { } } +/// Generates all possible consensus secrets (from system entropy) +/// and prints them to stdout. +/// They should be copied over to the secrets.yaml/consensus_secrets.yaml file. +pub fn generate_consensus_secrets() { + let validator_key = roles::validator::SecretKey::generate(); + let attester_key = roles::attester::SecretKey::generate(); + let node_key = roles::node::SecretKey::generate(); + println!("# {}", validator_key.public().encode()); + println!("- validator_key: {}", validator_key.encode()); + println!("# {}", attester_key.public().encode()); + println!("- attester_key: {}", attester_key.encode()); + println!("# {}", node_key.public().encode()); + println!("- node_key: {}", node_key.encode()); +} + pub(crate) fn read_consensus_secrets() -> anyhow::Result> { let Ok(path) = env::var("EN_CONSENSUS_SECRETS_PATH") else { return Ok(None); diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 55b2133250ac..f6696d733482 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -54,7 +54,7 @@ use zksync_web3_decl::{ }; use crate::{ - config::ExternalNodeConfig, + config::{generate_consensus_secrets, ExternalNodeConfig}, init::{ensure_storage_initialized, SnapshotRecoveryConfig}, }; @@ -695,10 +695,20 @@ async fn shutdown_components( Ok(()) } +#[derive(Debug, Clone, clap::Subcommand)] +enum Command { + /// Generates consensus secret keys to use in the secrets file. + /// Prints the keys to the stdout, you need to copy the relevant keys into your secrets file. + GenerateSecrets, +} + /// External node for ZKsync Era. #[derive(Debug, Parser)] #[command(author = "Matter Labs", version)] struct Cli { + #[command(subcommand)] + command: Option, + /// Enables consensus-based syncing instead of JSON-RPC based one. This is an experimental and incomplete feature; /// do not use unless you know what you're doing. #[arg(long)] @@ -720,7 +730,14 @@ struct Cli { /// Path to the yaml with external node specific configuration. If set, it will be used instead of env vars. #[arg(long, requires = "config_path", requires = "secrets_path")] external_node_config_path: Option, - /// Path to the yaml with consensus. + /// Path to the yaml with consensus config. If set, it will be used instead of env vars. + #[arg( + long, + requires = "config_path", + requires = "secrets_path", + requires = "external_node_config_path", + requires = "enable_consensus" + )] consensus_path: Option, } @@ -778,9 +795,22 @@ async fn main() -> anyhow::Result<()> { // Initial setup. let opt = Cli::parse(); + if let Some(cmd) = &opt.command { + match cmd { + Command::GenerateSecrets => generate_consensus_secrets(), + } + return Ok(()); + } + let mut config = if let Some(config_path) = opt.config_path.clone() { let secrets_path = opt.secrets_path.clone().unwrap(); let external_node_config_path = opt.external_node_config_path.clone().unwrap(); + if opt.enable_consensus { + anyhow::ensure!( + opt.consensus_path.is_some(), + "if --config-path and --enable-consensus are specified, then --consensus-path should be used to specify the location of the consensus config" + ); + } ExternalNodeConfig::from_files( config_path, external_node_config_path, diff --git a/core/node/consensus/src/en.rs b/core/node/consensus/src/en.rs index e2e1ce480dfb..66bdc822c058 100644 --- a/core/node/consensus/src/en.rs +++ b/core/node/consensus/src/en.rs @@ -129,6 +129,10 @@ impl EN { ctx: &ctx::Ctx, actions: ActionQueueSender, ) -> anyhow::Result<()> { + tracing::warn!("\ + WARNING: this node is using ZKsync API synchronization, which will be deprecated soon. \ + Please follow this instruction to switch to p2p synchronization: \ + https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/09_decentralization.md"); let res: ctx::Result<()> = scope::run!(ctx, |ctx, s| async { // Update sync state in the background. s.spawn_bg(self.fetch_state_loop(ctx)); diff --git a/docs/guides/external-node/09_decentralization.md b/docs/guides/external-node/09_decentralization.md new file mode 100644 index 000000000000..37cd4c502ef1 --- /dev/null +++ b/docs/guides/external-node/09_decentralization.md @@ -0,0 +1,86 @@ +# Decentralization + +In the default setup the ZKsync node will fetch data from the ZKsync API endpoint maintained by Matter Labs. To reduce +the reliance on this centralized endpoint we have developed a decentralized p2p networking stack (aka gossipnet) which +will eventually be used instead of ZKsync API for synchronizing data. + +On the gossipnet, the data integrity will be protected by the BFT (byzantine fault tolerant) consensus algorithm +(currently data is signed just by the main node though). + +## Enabling gossipnet on your node + +> [!NOTE] +> +> Because the data transmitted over the gossipnet is signed by the main node (and eventually by the consensus quorum), +> the signatures need to be backfilled to the node's local storage the first time you switch from centralized (ZKsync +> API based) synchronization to the decentralized (gossipnet based) synchronization (this is a one-time thing). With the +> current implementation it may take a couple of hours and gets faster the more nodes you add to the +> `gossip_static_outbound` list (see below). We are working to remove this inconvenience. + +### Generating secrets + +Each participant node of the gossipnet has to have an identity (a public/secret key pair). When running your node for +the first time, generate the secrets by running: + +``` +cargo run -p zksync_external_node -- generate-secrets > consensus_secrets.yaml +chmod 600 consensus_secrets.yaml +``` + +> [!NOTE] +> +> NEVER reveal the secret keys used by your node. Otherwise someone can impersonate your node on the gossipnet. If you +> suspect that your secret key has been leaked, you can generate fresh keys using the same tool. +> +> If you want someone else to connect to your node, give them your PUBLIC key instead. Both public and secret keys are +> present in the `consensus_secrets.yaml` (public keys are in comments). + +### Preparing configuration file + +Copy the template of the consensus configuration file (for +[mainnet](https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/prepared_configs/mainnet_consensus_config.yaml) +or +[testnet](https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/prepared_configs/testnet_consensus_config.yaml) +). + +> [!NOTE] +> +> You need to fill in the `public_addr` field. This is the address that will (not implemented yet) be advertised over +> gossipnet to other nodes, so that they can establish connections to your node. If you don't want to expose your node +> to the public internet, you can use IP in your local network. + +Currently the config contains the following fields (refer to config +[schema](https://github.com/matter-labs/zksync-era/blob/990676c5f84afd2ff8cd337f495c82e8d1f305a4/core/lib/protobuf_config/src/proto/core/consensus.proto#L66) +for more details): + +- `server_addr` - local TCP socket address that the node should listen on for incoming connections. Note that this is an + additional TCP port that will be opened by the node. +- `public_addr` - the public address of your node that will be advertised over the gossipnet. +- `max_payload_size` - limit (in bytes) on the sized of the ZKsync ERA block received from the gossipnet. This protects + your node from getting DoS`ed by too large network messages. Use the value from the template. +- `gossip_dynamic_inbound_limit` - maximal number of unauthenticated concurrent inbound connections that can be + established to your node. This is a DDoS protection measure. +- `gossip_static_outbound` - list of trusted peers that your node should always try to connect to. The template contains + the nodes maintained by Matterlabs, but you can add more if you know any. Note that the list contains both the network + address AND the public key of the node - this prevents spoofing attacks. + +### Setting environment variables + +Uncomment (or add) the following lines in your `.env` config: + +``` +EN_CONSENSUS_CONFIG_PATH=... +EN_CONSENSUS_SECRETS_PATH=... +``` + +These variables should point to your consensus config and secrets files that we have just created. Tweak the paths to +the files if you have placed them differently. + +### Add `--enable-consensus` flag to your entry point + +For the consensus configuration to take effect you have to add `--enable-consensus` flag to the command line when +running the node: + +``` +cargo run -p zksync_external_node -- --enable-consensus +``` diff --git a/docs/guides/external-node/prepared_configs/mainnet-config.env b/docs/guides/external-node/prepared_configs/mainnet-config.env index 35278205b96f..bce812084665 100644 --- a/docs/guides/external-node/prepared_configs/mainnet-config.env +++ b/docs/guides/external-node/prepared_configs/mainnet-config.env @@ -70,6 +70,9 @@ RUST_LOG=zksync_core=debug,zksync_dal=info,zksync_eth_client=info,zksync_merkle_ RUST_BACKTRACE=full RUST_LIB_BACKTRACE=1 +# Settings related to gossip network, see `09_decentralization.md` +#EN_CONSENSUS_CONFIG_PATH=./mainnet_consensus_config.yaml +#EN_CONSENSUS_SECRETS_PATH=./consensus_secrets.yaml # ------------------------------------------------------------------------ # -------------- THE FOLLOWING VARIABLES DEPEND ON THE ENV --------------- diff --git a/docs/guides/external-node/prepared_configs/mainnet_consensus_config.yaml b/docs/guides/external-node/prepared_configs/mainnet_consensus_config.yaml new file mode 100644 index 000000000000..6d61ef3963eb --- /dev/null +++ b/docs/guides/external-node/prepared_configs/mainnet_consensus_config.yaml @@ -0,0 +1,10 @@ +server_addr: '0.0.0.0:3054' +public_addr: ':3054' +max_payload_size: 5000000 +gossip_dynamic_inbound_limit: 100 +gossip_static_outbound: + # preconfigured ENs owned by Matterlabs that you can connect to + - key: 'node:public:ed25519:68d29127ab03408bf5c838553b19c32bdb3aaaae9bf293e5e078c3a0d265822a' + addr: 'external-node-consensus-mainnet.zksync.dev:3054' + - key: 'node:public:ed25519:b521e1bb173d04bc83d46b859d1296378e94a40427a6beb9e7fdd17cbd934c11' + addr: 'external-node-moby-consensus-mainnet.zksync.dev:3054' diff --git a/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env b/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env index 98e2ee6bd510..182012e2850c 100644 --- a/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env +++ b/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env @@ -70,6 +70,9 @@ RUST_LOG=zksync_core=debug,zksync_dal=info,zksync_eth_client=info,zksync_merkle_ RUST_BACKTRACE=full RUST_LIB_BACKTRACE=1 +# Settings related to gossip network, see `09_decentralization.md` +#EN_CONSENSUS_CONFIG_PATH=./testnet_consensus_config.yaml +#EN_CONSENSUS_SECRETS_PATH=./consensus_secrets.yaml # ------------------------------------------------------------------------ # -------------- THE FOLLOWING VARIABLES DEPEND ON THE ENV --------------- diff --git a/docs/guides/external-node/prepared_configs/testnet_consensus_config.yaml b/docs/guides/external-node/prepared_configs/testnet_consensus_config.yaml new file mode 100644 index 000000000000..25461b5dfc45 --- /dev/null +++ b/docs/guides/external-node/prepared_configs/testnet_consensus_config.yaml @@ -0,0 +1,10 @@ +server_addr: '0.0.0.0:3054' +public_addr: ':3054' +max_payload_size: 5000000 +gossip_dynamic_inbound_limit: 100 +gossip_static_outbound: + # preconfigured ENs owned by Matterlabs that you can connect to + - key: 'node:public:ed25519:4a94067664e7b8d0927ab1443491dab71a1d0c63f861099e1852f2b6d0831c3e' + addr: 'external-node-consensus-sepolia.zksync.dev:3054' + - key: 'node:public:ed25519:cfbbebc74127099680584f07a051a2573e2dd7463abdd000d31aaa44a7985045' + addr: 'external-node-moby-consensus-sepolia.zksync.dev:3054'