From 95c1853dbb206c0da7769d8c96d495a6ec18758a Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 13 Nov 2023 15:19:28 +0200 Subject: [PATCH] feat: Use `package.links` trick for Protobuf compilation (#32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ Propagates Protobuf information via env variables by relying on [package.links](https://doc.rust-lang.org/cargo/reference/build-scripts.html#the-links-manifest-key). ## Why ❔ This simplifies Protobuf configuration and eliminate misconfiguration risks. --- node/Cargo.lock | 9 +- node/Cargo.toml | 2 - node/actors/executor/Cargo.toml | 2 +- node/actors/executor/build.rs | 3 +- node/actors/network/Cargo.toml | 3 +- node/actors/network/build.rs | 16 +- node/libs/protobuf/Cargo.toml | 4 +- node/libs/protobuf/build.rs | 3 + node/libs/protobuf/src/build.rs | 85 +++++++ node/libs/protobuf/src/lib.rs | 5 +- node/libs/protobuf_build/Cargo.toml | 6 - node/libs/protobuf_build/src/canonical.rs | 2 + node/libs/protobuf_build/src/lib.rs | 278 ++++++++++++++-------- node/libs/protobuf_build/src/syntax.rs | 8 +- node/libs/roles/Cargo.toml | 4 +- node/libs/roles/build.rs | 8 +- node/libs/storage/Cargo.toml | 3 +- node/libs/storage/build.rs | 8 +- 18 files changed, 304 insertions(+), 145 deletions(-) create mode 100644 node/libs/protobuf/src/build.rs diff --git a/node/Cargo.lock b/node/Cargo.lock index db87654a..07ce3a74 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -2633,6 +2633,7 @@ dependencies = [ "zksync_consensus_sync_blocks", "zksync_consensus_utils", "zksync_protobuf", + "zksync_protobuf_build", ] [[package]] @@ -2658,6 +2659,7 @@ dependencies = [ "zksync_consensus_roles", "zksync_consensus_utils", "zksync_protobuf", + "zksync_protobuf_build", ] [[package]] @@ -2675,6 +2677,7 @@ dependencies = [ "zksync_consensus_crypto", "zksync_consensus_utils", "zksync_protobuf", + "zksync_protobuf_build", ] [[package]] @@ -2695,6 +2698,7 @@ dependencies = [ "zksync_concurrency", "zksync_consensus_roles", "zksync_protobuf", + "zksync_protobuf_build", ] [[package]] @@ -2771,17 +2775,12 @@ version = "0.1.0" dependencies = [ "anyhow", "heck", - "once_cell", "prettyplease", "proc-macro2", - "prost", "prost-build", "prost-reflect", - "prost-types", "protox", - "protox-parse", "quote", - "rand 0.8.5", "syn 2.0.39", ] diff --git a/node/Cargo.toml b/node/Cargo.toml index 8ed91a3b..c998681b 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -53,9 +53,7 @@ proc-macro2 = "1.0.66" prost = "0.12.0" prost-build = "0.12.0" prost-reflect = { version = "0.12.0", features = ["serde"] } -prost-types = "0.12.0" protox = "0.5.0" -protox-parse = "0.5.0" prettyplease = "0.2.6" pretty_assertions = "1.4.0" quick-protobuf = "0.8.1" diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index 75be56b5..cdf437f1 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -27,4 +27,4 @@ vise.workspace = true tokio.workspace = true [build-dependencies] -zksync_protobuf.workspace = true +zksync_protobuf_build.workspace = true diff --git a/node/actors/executor/build.rs b/node/actors/executor/build.rs index 8c157605..816fd2e8 100644 --- a/node/actors/executor/build.rs +++ b/node/actors/executor/build.rs @@ -1,10 +1,11 @@ //! Generates rust code from protobufs. fn main() { - zksync_protobuf::build::Config { + zksync_protobuf_build::Config { input_root: "src/config/proto".into(), proto_root: "zksync/executor/config".into(), dependencies: vec![], protobuf_crate: "::zksync_protobuf".parse().unwrap(), + is_public: false, } .generate() .expect("generate(config)"); diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index 34847a9d..6c8e0541 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -31,5 +31,4 @@ test-casing.workspace = true tokio.workspace = true [build-dependencies] -zksync_consensus_roles.workspace = true -zksync_protobuf.workspace = true +zksync_protobuf_build.workspace = true diff --git a/node/actors/network/build.rs b/node/actors/network/build.rs index e8ec05eb..75a4b995 100644 --- a/node/actors/network/build.rs +++ b/node/actors/network/build.rs @@ -1,28 +1,24 @@ //! Generates rust code from protobufs. fn main() { - zksync_protobuf::build::Config { + zksync_protobuf_build::Config { input_root: "src/proto".into(), proto_root: "zksync/network".into(), dependencies: vec![ - ( - "::zksync_protobuf::proto".parse().unwrap(), - &zksync_protobuf::proto::DESCRIPTOR, - ), - ( - "::zksync_consensus_roles::proto".parse().unwrap(), - &zksync_consensus_roles::proto::DESCRIPTOR, - ), + "::zksync_protobuf::proto".parse().unwrap(), + "::zksync_consensus_roles::proto".parse().unwrap(), ], protobuf_crate: "::zksync_protobuf".parse().unwrap(), + is_public: false, } .generate() .expect("generate()"); - zksync_protobuf::build::Config { + zksync_protobuf_build::Config { input_root: "src/mux/tests/proto".into(), proto_root: "zksync/network/mux/tests".into(), dependencies: vec![], protobuf_crate: "::zksync_protobuf".parse().unwrap(), + is_public: false, } .generate() .expect("generate()"); diff --git a/node/libs/protobuf/Cargo.toml b/node/libs/protobuf/Cargo.toml index 83eba563..dcfeddfe 100644 --- a/node/libs/protobuf/Cargo.toml +++ b/node/libs/protobuf/Cargo.toml @@ -6,6 +6,8 @@ authors.workspace = true homepage.workspace = true license.workspace = true +links = "zksync_protobuf_proto" + [[bin]] name = "conformance_test" @@ -28,5 +30,3 @@ tracing-subscriber.workspace = true [build-dependencies] zksync_protobuf_build.workspace = true - -anyhow.workspace = true diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 49ba4cb6..d62d6161 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -5,6 +5,7 @@ fn main() { proto_root: "zksync".into(), dependencies: vec![], protobuf_crate: "crate".parse().expect("protobuf_crate"), + is_public: true, } .generate() .expect("generate(std)"); @@ -14,6 +15,7 @@ fn main() { proto_root: "zksync/protobuf/tests".into(), dependencies: vec![], protobuf_crate: "crate".parse().expect("protobuf_crate"), + is_public: false, } .generate() .expect("generate(test)"); @@ -23,6 +25,7 @@ fn main() { proto_root: "zksync/protobuf/conformance_test".into(), dependencies: vec![], protobuf_crate: "::zksync_protobuf".parse().expect("protobuf_crate"), + is_public: false, } .generate() .expect("generate(conformance)"); diff --git a/node/libs/protobuf/src/build.rs b/node/libs/protobuf/src/build.rs new file mode 100644 index 00000000..b1258868 --- /dev/null +++ b/node/libs/protobuf/src/build.rs @@ -0,0 +1,85 @@ +//! Build-related functionality. This should only be used by the code generated by the `protobuf_build` crate. + +pub use once_cell::sync::Lazy; +pub use prost; +use prost::Message as _; +pub use prost_reflect; +use prost_reflect::prost_types; +use std::sync::RwLock; + +/// Global descriptor pool. +static POOL: Lazy> = Lazy::new(RwLock::default); + +/// Protobuf descriptor + info about the mapping to rust code. +#[derive(Debug)] +pub struct Descriptor(()); + +impl Descriptor { + /// Constructs a descriptor and adds it to the global pool. + pub fn new(_dependencies: &[&'static Self], descriptor_bytes: &[u8]) -> Self { + let descriptor = prost_types::FileDescriptorSet::decode(descriptor_bytes).unwrap(); + let pool = &mut POOL.write().unwrap(); + + // Dependencies are already loaded to the global pool on their initialization. + // The fact that we have their refs is sufficient to prove that they are already in the global pool. + Self::load(pool, descriptor).expect("failed loading descriptor into global pool"); + Self(()) + } + + /// Loads the descriptor to the pool, if not already loaded. + fn load( + pool: &mut prost_reflect::DescriptorPool, + descriptor: prost_types::FileDescriptorSet, + ) -> anyhow::Result<()> { + let pool_has_all_files = descriptor + .file + .iter() + .all(|file| pool.get_file_by_name(file.name()).is_some()); + if pool_has_all_files { + return Ok(()); + } + pool.add_file_descriptor_set(descriptor)?; + Ok(()) + } + + /// Returns a descriptor by a fully qualified message name. + pub fn get_message_by_name(&self, name: &str) -> Option { + POOL.read().unwrap().get_message_by_name(name) + // ^ This works because this descriptor must have been loaded into the global pool + // when the instance was constructed. + } +} + +/// Expands to a descriptor declaration. +#[macro_export] +macro_rules! declare_descriptor { + ($name:ident => $descriptor_path:expr, $($rust_deps:path),*) => { + pub static $name: $crate::build::Lazy<$crate::build::Descriptor> = + $crate::build::Lazy::new(|| { + $crate::build::Descriptor::new( + &[$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*], + ::std::include_bytes!($descriptor_path), + ) + }); + } +} + +/// Implements `ReflectMessage` for a type based on a provided `Descriptor`. +#[macro_export] +macro_rules! impl_reflect_message { + ($ty:ty, $descriptor:expr, $proto_name:expr) => { + impl $crate::build::prost_reflect::ReflectMessage for $ty { + fn descriptor(&self) -> $crate::build::prost_reflect::MessageDescriptor { + static INIT: $crate::build::Lazy<$crate::build::prost_reflect::MessageDescriptor> = + $crate::build::Lazy::new(|| { + $crate::build::Descriptor::get_message_by_name($descriptor, $proto_name) + .unwrap() + }); + INIT.clone() + } + } + }; +} + +pub use declare_descriptor; +pub use impl_reflect_message; diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs index f461892a..8524d801 100644 --- a/node/libs/protobuf/src/lib.rs +++ b/node/libs/protobuf/src/lib.rs @@ -1,13 +1,14 @@ //! Code generated from protobuf schema files and //! utilities for serialization. +#[doc(hidden)] // should only be used by code generated by `protobuf_build` crate +pub mod build; pub mod proto; mod proto_fmt; mod std_conv; pub mod testonly; -pub use proto_fmt::*; -pub use zksync_protobuf_build as build; +pub use self::proto_fmt::*; #[cfg(test)] mod tests; diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml index 94e8b9bf..88fe2580 100644 --- a/node/libs/protobuf_build/Cargo.toml +++ b/node/libs/protobuf_build/Cargo.toml @@ -9,16 +9,10 @@ license.workspace = true [dependencies] anyhow.workspace = true heck.workspace = true -once_cell.workspace = true prettyplease.workspace = true proc-macro2.workspace = true -prost.workspace = true prost-build.workspace = true -prost-types.workspace = true prost-reflect.workspace = true protox.workspace = true -protox-parse.workspace = true quote.workspace = true -rand.workspace = true syn.workspace = true - diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs index fe56fc1c..426552cf 100644 --- a/node/libs/protobuf_build/src/canonical.rs +++ b/node/libs/protobuf_build/src/canonical.rs @@ -1,6 +1,8 @@ //! Checks whether messages in the given file descriptor set support canonical encoding. + use crate::syntax::extract_message_names; use anyhow::Context as _; +use prost_reflect::{self, prost_types}; use std::collections::HashSet; #[derive(Default)] diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index ff933aea..82ebc065 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -24,14 +24,17 @@ //! these are being built simultaneously from the same build script). #![allow(clippy::print_stdout)] -// Imports accessed from the generated code. + pub use self::syntax::*; use anyhow::Context as _; -pub use once_cell::sync::Lazy; -pub use prost; -use prost::Message as _; -pub use prost_reflect; -use std::{fs, path::Path, sync::Mutex}; +use prost_reflect::{prost::Message as _, prost_types}; +use protox::file::File; +use std::{ + collections::{HashMap, HashSet}, + env, fs, + path::{Path, PathBuf}, + sync::atomic::{AtomicBool, Ordering}, +}; mod canonical; mod ident; @@ -52,70 +55,71 @@ fn traverse_files( Ok(()) } -/// Protobuf descriptor + info about the mapping to rust code. +/// Manifest of a Protobuf compilation target containing information about the compilation process. #[derive(Debug)] -pub struct Descriptor { +struct Manifest { /// Root proto package that all proto files in this descriptor belong to. - /// Rust types have been generated relative to this root. - proto_root: ProtoName, - /// Raw descriptor proto. - descriptor_proto: prost_types::FileDescriptorSet, - /// Direct dependencies of this descriptor. - dependencies: Vec<&'static Descriptor>, + proto_root: ProtoPath, + /// Absolute path to the descriptor. + descriptor_path: PathBuf, + /// Tuples of `proto_root` and absolute paths to the corresponding descriptor for all dependencies + /// including transitive ones. + dependencies: Vec<(ProtoPath, PathBuf)>, } -impl Descriptor { - /// Constructs a Descriptor. - pub fn new( - proto_root: ProtoName, - dependencies: Vec<&'static Descriptor>, - descriptor_bytes: &[u8], - ) -> Self { - Descriptor { - proto_root, - dependencies, - descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes).unwrap(), - } - } +impl Manifest { + /// Loads manifest from the environment variable. + fn from_env(rust_root: &RustName) -> anyhow::Result { + let crate_name = rust_root.crate_name().context("empty `rust_root`")?; + let env_name = format!("DEP_{}_PROTO_MANIFEST", crate_name.to_uppercase()); + let manifest = env::var(env_name) + .with_context(|| format!("failed reading path to `{crate_name}` Protobuf manifest"))?; - /// Loads the descriptor to the pool, if not already loaded. - pub fn load(&self, pool: &mut prost_reflect::DescriptorPool) -> anyhow::Result<()> { - if self - .descriptor_proto - .file - .iter() - .all(|f| pool.get_file_by_name(f.name()).is_some()) - { - return Ok(()); - } - for dependency in &self.dependencies { - dependency.load(pool)?; + let mut manifest_parts = manifest.split(':'); + // ^ ':' is used as a separator since it cannot be present in paths. + let proto_root = manifest_parts.next().context("missing `proto_root`")?; + let proto_root = ProtoPath::from(proto_root); + let descriptor_path = manifest_parts + .next() + .context("missing `descriptor_path`")? + .into(); + + let mut dependencies = vec![]; + while let Some(proto_root) = manifest_parts.next() { + let proto_root = ProtoPath::from(proto_root); + let descriptor_path = manifest_parts + .next() + .context("missing `descriptor_path`")? + .into(); + dependencies.push((proto_root, descriptor_path)); } - pool.add_file_descriptor_set(self.descriptor_proto.clone())?; - Ok(()) - } - /// Loads the descriptor to the global pool and returns a copy of the global pool. - pub fn get_message_by_name(&self, name: &str) -> Option { - /// Global descriptor pool. - static POOL: Lazy> = Lazy::new(Mutex::default); - let pool = &mut POOL.lock().unwrap(); - self.load(pool).unwrap(); - pool.get_message_by_name(name) + Ok(Self { + proto_root, + descriptor_path, + dependencies, + }) } -} -/// Expands to a descriptor declaration. -#[macro_export] -macro_rules! declare_descriptor { - ($package_root:expr, $descriptor_path:expr, $($rust_deps:path),*) => { - pub static DESCRIPTOR: $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| { - $crate::Descriptor::new( - $package_root.into(), - ::std::vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*], - include_bytes!($descriptor_path), - ) - }); + /// Prints this manifest to an environment variable so that it's available to dependencies. + fn print(&self) { + use std::fmt::Write as _; + + let Self { + proto_root, + descriptor_path, + dependencies, + } = self; + let dependencies = dependencies + .iter() + .fold(String::new(), |mut acc, (root, desc_path)| { + write!(&mut acc, ":{root}:{}", desc_path.display()).unwrap(); + acc + }); + println!( + "cargo:manifest={proto_root}:{}{dependencies}", + descriptor_path.display() + ); } } @@ -125,11 +129,14 @@ pub struct Config { pub input_root: InputPath, /// Implicit prefix that should be prepended to proto paths of the proto files in the input directory. pub proto_root: ProtoPath, - /// Descriptors of the dependencies and the rust absolute paths under which they will be available from the generated code. - pub dependencies: Vec<(RustName, &'static Descriptor)>, - /// Rust absolute path under which the protobuf crate will be available from the generated - /// code. + /// Descriptors of the direct dependencies and the rust absolute paths under which they will be available from the generated code. + /// Each dependency must be a direct dependency of the built crate. OTOH, it doesn't need to be a build dependency. + pub dependencies: Vec, + /// Rust absolute path under which the protobuf crate will be available from the generated code. pub protobuf_crate: RustName, + /// Can generated Protobuf messages be included as a dependency for other crates (i.e., be mentioned + /// in `dependencies`)? Only one public target can be generated per build script. + pub is_public: bool, } impl Config { @@ -140,7 +147,7 @@ impl Config { /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated /// from a message of the given `proto_name`. - fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { + fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { let rust_name = proto_name .relative_to(&self.proto_root.to_name().context("invalid proto_root")?) .unwrap() @@ -148,31 +155,92 @@ impl Config { let proto_name = proto_name.to_string(); let this = self.this_crate(); Ok(syn::parse_quote! { - impl #this::prost_reflect::ReflectMessage for #rust_name { - fn descriptor(&self) -> #this::prost_reflect::MessageDescriptor { - static INIT: #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| { - DESCRIPTOR.get_message_by_name(#proto_name).unwrap() - }); - INIT.clone() - } - } + #this::impl_reflect_message!(#rust_name, &DESCRIPTOR, #proto_name); }) } - /// Generates rust code from the proto files according to the config. - pub fn generate(&self) -> anyhow::Result<()> { + /// Validates this configuration. + fn validate(&self) -> anyhow::Result<()> { + /// Flag set to `true` if a public compilation target was encountered in a build script. + static HAS_PUBLIC_TARGET: AtomicBool = AtomicBool::new(false); + if !self.input_root.abs()?.is_dir() { anyhow::bail!("input_root should be a directory"); } + if self.is_public { + anyhow::ensure!( + !HAS_PUBLIC_TARGET.fetch_or(true, Ordering::SeqCst), + "Only one compilation target with `is_public: true` may be specified per build script" + ); + + let crate_name = env::var("CARGO_PKG_NAME") + .context("missing $CARGO_PKG_NAME env variable")? + .replace('-', "_"); + let expected_name = format!("{crate_name}_proto"); + let links = env::var("CARGO_MANIFEST_LINKS").ok(); + anyhow::ensure!( + links.as_ref() == Some(&expected_name), + "You must specify links = \"{expected_name}\" in the [package] section \ + of the built package manifest (currently set to {links:?})" + ); + } + Ok(()) + } + + /// Generates rust code from the proto files according to the config. + pub fn generate(self) -> anyhow::Result<()> { + self.validate()?; println!("cargo:rerun-if-changed={}", self.input_root.to_str()); // Load dependencies. + let dependency_manifests = self.dependencies.iter().map(Manifest::from_env); + let dependency_manifests: Vec = + dependency_manifests.collect::>()?; + let direct_dependency_descriptor_paths: HashSet<_> = dependency_manifests + .iter() + .map(|manifest| &manifest.descriptor_path) + .collect(); + + let all_dependencies = dependency_manifests.iter().flat_map(|manifest| { + manifest + .dependencies + .iter() + .map(|(root, path)| (root, path)) + // ^ Converts a reference to a tuple to a tuple of references + .chain([(&manifest.proto_root, &manifest.descriptor_path)]) + }); + let mut pool = prost_reflect::DescriptorPool::new(); - for (root_path, descriptor) in &self.dependencies { - descriptor - .load(&mut pool) - .with_context(|| format!("failed to load dependency `{root_path}`"))?; + let mut direct_dependency_descriptors = HashMap::with_capacity(self.dependencies.len()); + let mut loaded_descriptor_paths = HashSet::new(); + let mut dependencies = vec![]; + for (proto_root, descriptor_path) in all_dependencies { + if !loaded_descriptor_paths.insert(descriptor_path) { + // Do not load the same descriptor twice. + continue; + } + dependencies.push((proto_root.clone(), descriptor_path.clone())); + + let descriptor = fs::read(descriptor_path).with_context(|| { + format!( + "failed reading descriptor for `{proto_root}` from {}", + descriptor_path.display() + ) + })?; + let descriptor = + prost_types::FileDescriptorSet::decode(&descriptor[..]).with_context(|| { + format!( + "failed decoding file descriptor set for `{proto_root}` from {}", + descriptor_path.display() + ) + })?; + + if direct_dependency_descriptor_paths.contains(descriptor_path) { + direct_dependency_descriptors.insert(descriptor_path.clone(), descriptor.clone()); + } + pool.add_file_descriptor_set(descriptor)?; } + let mut pool_raw = prost_types::FileDescriptorSet::default(); pool_raw.file.extend(pool.file_descriptor_protos().cloned()); @@ -189,15 +257,14 @@ impl Config { return Ok(()); }; - let file_raw = fs::read_to_string(path).context("fs::read()")?; + let source = fs::read_to_string(path).context("fs::read()")?; let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root) .context("ProtoPath::from_input_path()")?; - pool_raw - .file - .push(protox_parse::parse(&path.to_string(), &file_raw).map_err( - // rewrapping the error, so that source location is included in the error message. - |err| anyhow::anyhow!("{err:?}"), - )?); + let compiled = File::from_source(&path.to_string(), &source).map_err( + // rewrapping the error, so that source location is included in the error message. + |err| anyhow::anyhow!("{err:?}"), + )?; + pool_raw.file.push(compiled.into()); proto_paths.push(path); Ok(()) })?; @@ -238,6 +305,15 @@ impl Config { let output_path = output_dir.join("gen.rs"); let descriptor_path = output_dir.join("gen.binpb"); fs::write(&descriptor_path, &descriptor.encode_to_vec())?; + + if self.is_public { + let manifest = Manifest { + proto_root: self.proto_root.clone(), + descriptor_path: descriptor_path.clone(), + dependencies, + }; + manifest.print(); + } println!("PROTOBUF_DESCRIPTOR={descriptor_path:?}"); // Generate code out of compiled proto files. @@ -246,10 +322,12 @@ impl Config { let prost_path = self.this_crate().join(RustName::ident("prost")); config.prost_path(prost_path.to_string()); config.skip_protoc_run(); - for (root_path, descriptor) in &self.dependencies { - for file in &descriptor.descriptor_proto.file { + for (root_path, manifest) in self.dependencies.iter().zip(&dependency_manifests) { + let descriptor = &direct_dependency_descriptors[&manifest.descriptor_path]; + // ^ Indexing is safe by construction. + for file in &descriptor.file { let proto_rel = ProtoName::from(file.package()) - .relative_to(&descriptor.proto_root) + .relative_to(&manifest.proto_root.to_name()?) .unwrap(); let rust_path = root_path.clone().join(proto_rel.to_rust_module()?); config.extern_path(format!(".{}", file.package()), rust_path.to_string()); @@ -269,25 +347,25 @@ impl Config { .extend(code); } - // Generate the reflection code. let package_root = self.proto_root.to_name().context("invalid proto_root")?; let mut output = output.into_submodule(&package_root.to_rust_module()?); - for proto_name in extract_message_names(&descriptor) { - let impl_item = self - .reflect_impl(&proto_name) - .with_context(|| format!("reflect_impl({proto_name})"))?; - output.append_item(impl_item.into()); - } // Generate the descriptor. - let root_paths_for_deps = self.dependencies.iter().map(|(root_path, _)| root_path); + let root_paths_for_deps = self.dependencies.iter(); let this = self.this_crate(); - let package_root = package_root.to_string(); let descriptor_path = descriptor_path.display().to_string(); output.append_item(syn::parse_quote! { - #this::declare_descriptor!(#package_root, #descriptor_path, #(#root_paths_for_deps),*); + #this::declare_descriptor!(DESCRIPTOR => #descriptor_path, #(#root_paths_for_deps),*); }); + // Generate the reflection code. + for proto_name in extract_message_names(&descriptor) { + let impl_item = self + .reflect_impl(&proto_name) + .with_context(|| format!("reflect_impl({proto_name})"))?; + output.append_item(impl_item); + } + // Save output. fs::write(&output_path, output.format()).with_context(|| { format!( diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 0d9aee3b..26b1e844 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -2,6 +2,7 @@ use super::ident; use anyhow::Context as _; +use prost_reflect::prost_types; use std::{ collections::BTreeMap, fmt, @@ -46,7 +47,7 @@ impl InputPath { } /// Absolute path of the proto file used for importing other proto files. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ProtoPath(PathBuf); impl From<&str> for ProtoPath { @@ -111,6 +112,11 @@ impl RustName { Self(syn::Path::from(ident)) } + /// Returns the crate name for this name (i.e., its first segment). + pub(crate) fn crate_name(&self) -> Option { + Some(self.0.segments.first()?.ident.to_string()) + } + /// Concatenates 2 rust names. pub fn join(mut self, suffix: Self) -> Self { self.0.segments.extend(suffix.0.segments); diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index 9cb2c49d..671bb4bb 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -6,6 +6,8 @@ authors.workspace = true homepage.workspace = true license.workspace = true +links = "zksync_consensus_roles_proto" + [dependencies] zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true @@ -21,4 +23,4 @@ serde.workspace = true tracing.workspace = true [build-dependencies] -zksync_protobuf.workspace = true +zksync_protobuf_build.workspace = true diff --git a/node/libs/roles/build.rs b/node/libs/roles/build.rs index b52d64e8..0f5e7683 100644 --- a/node/libs/roles/build.rs +++ b/node/libs/roles/build.rs @@ -1,13 +1,11 @@ //! Generates rust code from protobufs. fn main() { - zksync_protobuf::build::Config { + zksync_protobuf_build::Config { input_root: "src/proto".into(), proto_root: "zksync/roles".into(), - dependencies: vec![( - "::zksync_protobuf::proto".parse().expect("dependency_path"), - &zksync_protobuf::proto::DESCRIPTOR, - )], + dependencies: vec!["::zksync_protobuf::proto".parse().expect("dependency_path")], protobuf_crate: "::zksync_protobuf".parse().expect("protobuf_crate"), + is_public: true, } .generate() .expect("generate()"); diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index 6ff85f7a..1075d92d 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -26,8 +26,7 @@ test-casing.workspace = true tokio.workspace = true [build-dependencies] -zksync_protobuf.workspace = true -zksync_consensus_roles.workspace = true +zksync_protobuf_build.workspace = true [features] default = [] diff --git a/node/libs/storage/build.rs b/node/libs/storage/build.rs index b1c9aabb..654596eb 100644 --- a/node/libs/storage/build.rs +++ b/node/libs/storage/build.rs @@ -1,13 +1,11 @@ //! Generates rust code from protobufs. fn main() { - zksync_protobuf::build::Config { + zksync_protobuf_build::Config { input_root: "src/proto".into(), proto_root: "zksync/storage".into(), - dependencies: vec![( - "::zksync_consensus_roles::proto".parse().unwrap(), - &zksync_consensus_roles::proto::DESCRIPTOR, - )], + dependencies: vec!["::zksync_consensus_roles::proto".parse().unwrap()], protobuf_crate: "::zksync_protobuf".parse().unwrap(), + is_public: false, } .generate() .expect("generate()");