Merge remote-tracking branch 'origin/main' into gprusak-split-schema

node/Cargo.toml

@@ -51,6 +51,7 @@ hex = "0.4.3"
 im = "15.1.0"
 once_cell = "1.17.1"
 pin-project = "1.1.0"
+proc-macro2 = "1.0.66"
 prost = "0.12.0"
 prost-build = "0.12.0"
 prost-reflect = { version = "0.12.0", features = ["serde"] }

node/libs/protobuf/build.rs

@@ -4,7 +4,7 @@ fn main() {
         input_root: "src/proto".into(),
         proto_root: "zksync".into(),
         dependencies: vec![],
-        protobuf_crate: "crate".into(),
+        protobuf_crate: "crate".parse().expect("protobuf_crate"),
     }
     .generate()
     .expect("generate(std)");
@@ -13,7 +13,7 @@ fn main() {
         input_root: "src/tests/proto".into(),
         proto_root: "zksync/protobuf/tests".into(),
         dependencies: vec![],
-        protobuf_crate: "crate".into(),
+        protobuf_crate: "crate".parse().expect("protobuf_crate"),
     }
     .generate()
     .expect("generate(test)");
@@ -22,7 +22,7 @@ fn main() {
         input_root: "src/bin/conformance_test/proto".into(),
         proto_root: "zksync/protobuf/conformance_test".into(),
         dependencies: vec![],
-        protobuf_crate: "::zksync_protobuf".into(),
+        protobuf_crate: "::zksync_protobuf".parse().expect("protobuf_crate"),
     }
     .generate()
     .expect("generate(conformance)");

node/libs/protobuf_build/Cargo.toml

@@ -11,6 +11,7 @@ anyhow.workspace = true
 heck.workspace = true
 once_cell.workspace = true
 prettyplease.workspace = true
+proc-macro2.workspace = true
 prost.workspace = true
 prost-build.workspace = true
 prost-types.workspace = true

node/libs/protobuf_build/src/canonical.rs

@@ -7,29 +7,30 @@ use std::collections::HashSet;
 struct Check(HashSet<String>);
 
 impl Check {
-    /// Checks if messages of type `m` support canonical encoding.
-    fn check_message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> {
-        if self.0.contains(m.full_name()) {
+    /// Checks if messages of type `message` support canonical encoding.
+    fn check_message(&mut self, message: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> {
+        if self.0.contains(message.full_name()) {
             return Ok(());
         }
-        self.0.insert(m.full_name().to_string());
-        for f in m.fields() {
-            self.check_field(&f).with_context(|| f.name().to_string())?;
+        self.0.insert(message.full_name().to_string());
+        for field in message.fields() {
+            self.check_field(&field)
+                .with_context(|| field.name().to_string())?;
         }
         Ok(())
     }
 
-    /// Checks if field `f` supports canonical encoding.
-    fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> {
-        if f.is_map() {
+    /// Checks if field `field` supports canonical encoding.
+    fn check_field(&mut self, field: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> {
+        if field.is_map() {
             anyhow::bail!("maps unsupported");
         }
-        if !f.is_list() && !f.supports_presence() {
+        if !field.is_list() && !field.supports_presence() {
             anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional");
         }
-        if let prost_reflect::Kind::Message(msg) = &f.kind() {
-            self.check_message(msg)
-                .with_context(|| msg.name().to_string())?;
+        if let prost_reflect::Kind::Message(message) = &field.kind() {
+            self.check_message(message)
+                .with_context(|| message.name().to_string())?;
         }
         Ok(())
     }
@@ -41,18 +42,18 @@ pub(crate) fn check(
     descriptor: &prost_types::FileDescriptorSet,
     pool: &prost_reflect::DescriptorPool,
 ) -> anyhow::Result<()> {
-    for f in &descriptor.file {
-        if f.syntax() != "proto3" {
-            anyhow::bail!("{}: only proto3 syntax is supported", f.name());
+    for file in &descriptor.file {
+        if file.syntax() != "proto3" {
+            anyhow::bail!("{}: only proto3 syntax is supported", file.name());
         }
     }
-    let mut c = Check::default();
+    let mut check = Check::default();
     for msg_name in extract_message_names(descriptor) {
-        let msg_name = msg_name.to_string();
-        let msg = pool
-            .get_message_by_name(&msg_name)
+        let message_name = msg_name.to_string();
+        let message = pool
+            .get_message_by_name(&message_name)
             .with_context(|| format!("{msg_name} not found in pool"))?;
-        c.check_message(&msg).with_context(|| msg_name)?;
+        check.check_message(&message).context(message_name)?;
     }
     Ok(())
 }

node/libs/protobuf_build/src/lib.rs

@@ -22,6 +22,7 @@
 //! different crates you need to specify them as dependencies in the Config.dependencies.
 //! It is not possible to depend on a different proto bundle within the same crate (because
 //! these are being built simultaneously from the same build script).
+
 #![allow(clippy::print_stdout)]
 // Imports accessed from the generated code.
 pub use self::syntax::*;
@@ -39,19 +40,20 @@ mod syntax;
 /// Traverses all the files in a directory recursively.
 fn traverse_files(
     path: &Path,
-    f: &mut impl FnMut(&Path) -> anyhow::Result<()>,
+    action: &mut impl FnMut(&Path) -> anyhow::Result<()>,
 ) -> anyhow::Result<()> {
     if !path.is_dir() {
-        f(path).with_context(|| path.display().to_string())?;
+        action(path).with_context(|| path.display().to_string())?;
         return Ok(());
     }
     for entry in fs::read_dir(path)? {
-        traverse_files(&entry?.path(), f)?;
+        traverse_files(&entry?.path(), action)?;
     }
     Ok(())
 }
 
 /// Protobuf descriptor + info about the mapping to rust code.
+#[derive(Debug)]
 pub struct Descriptor {
     /// Root proto package that all proto files in this descriptor belong to.
     /// Rust types have been generated relative to this root.
@@ -86,8 +88,8 @@ impl Descriptor {
         {
             return Ok(());
         }
-        for d in &self.dependencies {
-            d.load(pool)?;
+        for dependency in &self.dependencies {
+            dependency.load(pool)?;
         }
         pool.add_file_descriptor_set(self.descriptor_proto.clone())?;
         Ok(())
@@ -107,11 +109,11 @@ impl Descriptor {
 #[macro_export]
 macro_rules! declare_descriptor {
     ($package_root:expr, $descriptor_path:expr, $($rust_deps:path),*) => {
-        pub static DESCRIPTOR : $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| {
+        pub static DESCRIPTOR: $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| {
             $crate::Descriptor::new(
                 $package_root.into(),
-                vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*],
-                &include_bytes!($descriptor_path)[..],
+                ::std::vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR })*],
+                include_bytes!($descriptor_path),
             )
         });
     }
@@ -133,30 +135,28 @@ pub struct Config {
 impl Config {
     /// Location of the protobuf_build crate, visible from the generated code.
     fn this_crate(&self) -> RustName {
-        self.protobuf_crate.clone().join("build")
+        self.protobuf_crate.clone().join(RustName::ident("build"))
     }
 
     /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated
     /// from a message of the given `proto_name`.
-    fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result<String> {
+    fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result<syn::ItemImpl> {
         let rust_name = proto_name
             .relative_to(&self.proto_root.to_name().context("invalid proto_root")?)
             .unwrap()
-            .to_rust_type()
-            .to_string();
-        let rust_name: syn::Path = syn::parse_str(&rust_name).context("rust_name")?;
+            .to_rust_type()?;
         let proto_name = proto_name.to_string();
-        let this: syn::Path = syn::parse_str(&self.this_crate().to_string()).context("this")?;
-        Ok(quote::quote! {
+        let this = self.this_crate();
+        Ok(syn::parse_quote! {
             impl #this::prost_reflect::ReflectMessage for #rust_name {
                 fn descriptor(&self) -> #this::prost_reflect::MessageDescriptor {
-                    static INIT : #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| {
+                    static INIT: #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| {
                         DESCRIPTOR.get_message_by_name(#proto_name).unwrap()
                     });
                     INIT.clone()
                 }
             }
-        }.to_string())
+        })
     }
 
     /// Generates rust code from the proto files according to the config.
@@ -168,9 +168,10 @@ impl Config {
 
         // Load dependencies.
         let mut pool = prost_reflect::DescriptorPool::new();
-        for d in &self.dependencies {
-            d.1.load(&mut pool)
-                .with_context(|| format!("failed to load dependency {}", d.0))?;
+        for (root_path, descriptor) in &self.dependencies {
+            descriptor
+                .load(&mut pool)
+                .with_context(|| format!("failed to load dependency `{root_path}`"))?;
         }
         let mut pool_raw = prost_types::FileDescriptorSet::default();
         pool_raw.file.extend(pool.file_descriptor_protos().cloned());
@@ -207,24 +208,23 @@ impl Config {
         );
         compiler.include_source_info(true);
         compiler
-            .open_files(proto_paths.iter().map(|p| p.to_path()))
+            .open_files(proto_paths)
             // rewrapping the error, so that source location is included in the error message.
             .map_err(|err| anyhow::anyhow!("{err:?}"))?;
         let descriptor = compiler.file_descriptor_set();
         // Unwrap is ok, because we add a descriptor from a successful compilation.
         pool.add_file_descriptor_set(descriptor.clone()).unwrap();
 
         // Check that the compiled proto files belong to the declared proto package.
-        for f in &descriptor.file {
-            let got = ProtoName::from(f.package());
+        for file in &descriptor.file {
+            let got = ProtoName::from(file.package());
             // Unwrap is ok, because descriptor file here has never an empty name.
-            let want_prefix = ProtoPath::from(f.name()).parent().unwrap().to_name()?;
-            if !got.starts_with(&want_prefix) {
-                anyhow::bail!(
-                    "{got} ({:?}) does not belong to package {want_prefix}",
-                    f.name(),
-                );
-            }
+            let want_prefix = ProtoPath::from(file.name()).parent().unwrap().to_name()?;
+            anyhow::ensure!(
+                got.starts_with(&want_prefix),
+                "{got} ({:?}) does not belong to package {want_prefix}",
+                file.name()
+            );
         }
 
         // Check that the compiled proto messages support canonical encoding.
@@ -243,55 +243,58 @@ impl Config {
         // Generate code out of compiled proto files.
         let mut output = RustModule::default();
         let mut config = prost_build::Config::new();
-        config.prost_path(self.this_crate().join("prost").to_string());
+        let prost_path = self.this_crate().join(RustName::ident("prost"));
+        config.prost_path(prost_path.to_string());
         config.skip_protoc_run();
-        for d in &self.dependencies {
-            for f in &d.1.descriptor_proto.file {
-                let proto_rel = ProtoName::from(f.package())
-                    .relative_to(&d.1.proto_root)
+        for (root_path, descriptor) in &self.dependencies {
+            for file in &descriptor.descriptor_proto.file {
+                let proto_rel = ProtoName::from(file.package())
+                    .relative_to(&descriptor.proto_root)
                     .unwrap();
-                let rust_abs = d.0.clone().join(proto_rel.to_rust_module());
-                config.extern_path(format!(".{}", f.package()), rust_abs.to_string());
+                let rust_path = root_path.clone().join(proto_rel.to_rust_module()?);
+                config.extern_path(format!(".{}", file.package()), rust_path.to_string());
             }
         }
-        let m = prost_build::Module::from_parts([""]);
-        for f in &descriptor.file {
+        let module = prost_build::Module::from_parts([""]);
+        for file in &descriptor.file {
             let code = config
-                .generate(vec![(m.clone(), f.clone())])
+                .generate(vec![(module.clone(), file.clone())])
                 .context("generation failed")?;
+            let code = &code[&module];
+            let code = syn::parse_str(code).with_context(|| {
+                format!("prost_build generated invalid code for {}", file.name())
+            })?;
             output
-                .sub(&ProtoName::from(f.package()).to_rust_module())
-                .append(&code[&m]);
+                .submodule(&ProtoName::from(file.package()).to_rust_module()?)
+                .extend(code);
         }
 
         // Generate the reflection code.
         let package_root = self.proto_root.to_name().context("invalid proto_root")?;
-        let output = output.sub(&package_root.to_rust_module());
+        let mut output = output.into_submodule(&package_root.to_rust_module()?);
         for proto_name in extract_message_names(&descriptor) {
-            output.append(
-                &self
-                    .reflect_impl(&proto_name)
-                    .with_context(|| format!("reflect_impl({proto_name})"))?,
-            );
+            let impl_item = self
+                .reflect_impl(&proto_name)
+                .with_context(|| format!("reflect_impl({proto_name})"))?;
+            output.append_item(impl_item.into());
         }
 
         // Generate the descriptor.
-        let rust_deps = self
-            .dependencies
-            .iter()
-            .map(|d| syn::parse_str::<syn::Path>(&d.0.to_string()).unwrap());
-        let this: syn::Path = syn::parse_str(&self.this_crate().to_string())?;
+        let root_paths_for_deps = self.dependencies.iter().map(|(root_path, _)| root_path);
+        let this = self.this_crate();
         let package_root = package_root.to_string();
         let descriptor_path = descriptor_path.display().to_string();
-        output.append(
-            &quote::quote! {
-                #this::declare_descriptor!(#package_root,#descriptor_path,#(#rust_deps),*);
-            }
-            .to_string(),
-        );
+        output.append_item(syn::parse_quote! {
+            #this::declare_descriptor!(#package_root, #descriptor_path, #(#root_paths_for_deps),*);
+        });
 
         // Save output.
-        fs::write(output_path, output.format().context("output.format()")?)?;
+        fs::write(&output_path, output.format()).with_context(|| {
+            format!(
+                "failed writing generated code to `{}`",
+                output_path.display()
+            )
+        })?;
         Ok(())
     }
 }