From 25ab1b0e2aa2058d9c46d57b8388f8cf04c59ddd Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 30 Oct 2023 11:04:21 +0100 Subject: [PATCH 01/40] snapshot --- node/Cargo.lock | 40 ++++++ node/Cargo.toml | 2 + node/libs/protobuf/build/Cargo.toml | 27 +++++ node/libs/protobuf/build/src/lib.rs | 181 ++++++++++++++++++++++++++++ node/libs/schema/Cargo.toml | 7 +- node/libs/schema/build.rs | 152 +---------------------- node/libs/schema/src/lib.rs | 7 -- 7 files changed, 253 insertions(+), 163 deletions(-) create mode 100644 node/libs/protobuf/build/Cargo.toml create mode 100644 node/libs/protobuf/build/src/lib.rs diff --git a/node/Cargo.lock b/node/Cargo.lock index f81a3c58..57b633c9 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -1469,6 +1469,28 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf_build" +version = "0.1.0" +dependencies = [ + "anyhow", + "bit-vec", + "concurrency", + "once_cell", + "prettyplease 0.2.12", + "prost", + "prost-build", + "prost-reflect", + "prost-reflect-build", + "protoc-bin-vendored", + "quick-protobuf", + "rand", + "serde", + "serde_json", + "syn 2.0.29", + "tokio", +] + [[package]] name = "protoc-bin-vendored" version = "3.0.0" @@ -1698,6 +1720,24 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "schema" version = "0.1.0" +dependencies = [ + "anyhow", + "bit-vec", + "concurrency", + "once_cell", + "prost", + "prost-reflect", + "protobuf_build", + "quick-protobuf", + "rand", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "schema2" +version = "0.1.0" dependencies = [ "anyhow", "bit-vec", diff --git a/node/Cargo.toml b/node/Cargo.toml index 4ef28062..fe388e92 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "libs/concurrency", + "libs/protobuf/build", "actors/consensus", "libs/crypto", "actors/executor", @@ -8,6 +9,7 @@ members = [ "actors/sync_blocks", "libs/roles", "libs/schema", + "libs/schema2", "libs/storage", "tools", "libs/utils", diff --git a/node/libs/protobuf/build/Cargo.toml b/node/libs/protobuf/build/Cargo.toml new file mode 100644 index 00000000..656be38f --- /dev/null +++ b/node/libs/protobuf/build/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "protobuf_build" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +bit-vec.workspace = true +serde.workspace = true +once_cell.workspace = true +quick-protobuf.workspace = true +prost.workspace = true +prost-reflect.workspace = true +rand.workspace = true +serde_json.workspace = true +tokio.workspace = true + +concurrency = { path = "../../concurrency" } + +syn.workspace = true +protoc-bin-vendored.workspace = true +prost-build.workspace = true +prost-reflect-build.workspace = true +prettyplease.workspace = true diff --git a/node/libs/protobuf/build/src/lib.rs b/node/libs/protobuf/build/src/lib.rs new file mode 100644 index 00000000..2c33c18c --- /dev/null +++ b/node/libs/protobuf/build/src/lib.rs @@ -0,0 +1,181 @@ +//! Generates rust code from the capnp schema files in the `capnp/` directory. +use anyhow::Context as _; +use std::{collections::BTreeMap, env, fs, path::{PathBuf,Path}}; +use std::process::Command; + +/// Traversed all the files in a directory recursively. +fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> { + if !path.is_dir() { + f(&path); + return Ok(()); + } + for entry in fs::read_dir(path)? { + traverse_files(&entry?.path(), f)?; + } + Ok(()) +} + +/// A rust module representation. +/// It is used to collect the generated protobuf code. +#[derive(Default)] +struct Module { + /// Nested modules which transitively contain the generated code. + nested: BTreeMap, + /// Nested modules directly contains the generated code. + include: BTreeMap, +} + +impl Module { + /// Inserts a nested generated protobuf module. + /// `name` is a sequence of module names. + fn insert(&mut self, name: &[String], file: PathBuf) { + println!(" -- {name:?}"); + match name.len() { + 0 => panic!("empty module path"), + 1 => assert!( + self.include.insert(name[0].clone(), file).is_none(), + "duplicate module" + ), + _ => self + .nested + .entry(name[0].clone()) + .or_default() + .insert(&name[1..], file), + } + } + + /// Generates rust code of the module. + fn generate(&self) -> String { + let mut entries = vec![]; + entries.extend( + self.nested + .iter() + .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), + ); + entries.extend( + self.include + .iter() + .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), + ); + entries.join("\n") + } +} + +/// Checks if field `f` supports canonical encoding. +fn check_canonical_field(f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + if f.is_map() { + anyhow::bail!("maps unsupported"); + } + if !f.is_list() && !f.supports_presence() { + anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); + } + Ok(()) +} + +/// Checks if messages of type `m` support canonical encoding. +fn check_canonical_message(m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { + for m in m.child_messages() { + check_canonical_message(&m).with_context(|| m.name().to_string())?; + } + for f in m.fields() { + check_canonical_field(&f).with_context(|| f.name().to_string())?; + } + Ok(()) +} + +/// Checks if message types in file `f` support canonical encoding. +fn check_canonical_file(f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { + if f.syntax() != prost_reflect::Syntax::Proto3 { + anyhow::bail!("only proto3 syntax is supported"); + } + for m in f.messages() { + check_canonical_message(&m).with_context(|| m.name().to_string())?; + } + Ok(()) +} + +/// Checks if message types in descriptor pool `d` support canonical encoding. +fn check_canonical_pool(d: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { + for f in d.files() { + check_canonical_file(&f).with_context(|| f.name().to_string())?; + } + Ok(()) +} + +pub fn compile(proto_include: &Path, module_name: &str) -> anyhow::Result<()> { + println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); + let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) + .canonicalize()? + .join("proto"); + let _ = fs::remove_dir_all(&proto_output); + fs::create_dir_all(&proto_output).unwrap(); + + // Find all proto files. + let mut proto_inputs : Vec = vec![]; + traverse_files(proto_include, &mut |path| { + let Some(ext) = path.extension() else { return }; + let Some(ext) = ext.to_str() else { return }; + if ext != "proto" { + return; + }; + proto_inputs.push(path.into()); + })?; + + // Compile input files into descriptor. + let descriptor_path = proto_output.join("descriptor.binpb"); + let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); + cmd.arg("-o").arg(&descriptor_path); + cmd.arg("-I").arg(&proto_include); + /*.arg("--descriptor_set_in").arg( + format!("{}:{}", + proto_include.join("b/b.desc").to_str().unwrap(), + proto_include.join("c/c.desc").to_str().unwrap(), + ) + )*/ + + for input in &proto_inputs { + cmd.arg(&input); + } + + let out = cmd.output().context("protoc execution failed")?; + + if !out.status.success() { + anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); + } + + // Generate protobuf code from schema (with reflection). + env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); + let mut config = prost_build::Config::new(); + config.skip_protoc_run(); + config.out_dir(&proto_output); + prost_reflect_build::Builder::new() + .file_descriptor_set_path(&descriptor_path) + .file_descriptor_set_bytes(format!("{module_name}::DESCRIPTOR_POOL")) + .compile_protos_with_config(config, &proto_inputs, &[&proto_include]) + .unwrap(); + let descriptor = fs::read(&descriptor_path)?; + let pool = prost_reflect::DescriptorPool::decode(descriptor.as_ref()).unwrap(); + + // Check that messages are compatible with `proto_fmt::canonical`. + check_canonical_pool(&pool)?; + + // Generate mod file collecting all proto-generated code. + let mut m = Module::default(); + for entry in fs::read_dir(&proto_output).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().into_string().unwrap(); + let Some(name) = name.strip_suffix(".rs") else { + continue; + }; + let name: Vec<_> = name.split('.').map(String::from).collect(); + println!("name = {name:?}"); + m.insert(&name, entry.path()); + } + let mut file = m.generate(); + + file += &format!("const DESCRIPTOR_POOL: &'static [u8] = include_bytes!({descriptor_path:?});"); + + let file = syn::parse_str(&file).unwrap(); + fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; + Ok(()) +} diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index ea9c6941..4d1639f7 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -25,9 +25,4 @@ concurrency = { path = "../concurrency" } [build-dependencies] anyhow.workspace = true -syn.workspace = true -protoc-bin-vendored.workspace = true -prost-build.workspace = true -prost-reflect.workspace = true -prost-reflect-build.workspace = true -prettyplease.workspace = true +protobuf_build = { path = "../protobuf/build" } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 748a0a7e..22fc6091 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,159 +1,11 @@ //! Generates rust code from the capnp schema files in the `capnp/` directory. -use anyhow::Context as _; -use std::{collections::BTreeMap, env, fs, path::PathBuf}; - -/// Traversed all the files in a directory recursively. -fn traverse_files(path: PathBuf, f: &mut dyn FnMut(PathBuf)) -> std::io::Result<()> { - if !path.is_dir() { - f(path); - return Ok(()); - } - for entry in fs::read_dir(path)? { - traverse_files(entry?.path(), f)?; - } - Ok(()) -} - -/// A rust module representation. -/// It is used to collect the generated protobuf code. -#[derive(Default)] -struct Module { - /// Nested modules which transitively contain the generated code. - nested: BTreeMap, - /// Nested modules directly contains the generated code. - include: BTreeMap, -} - -impl Module { - /// Inserts a nested generated protobuf module. - /// `name` is a sequence of module names. - fn insert(&mut self, name: &[String], file: PathBuf) { - println!(" -- {name:?}"); - match name.len() { - 0 => panic!("empty module path"), - 1 => assert!( - self.include.insert(name[0].clone(), file).is_none(), - "duplicate module" - ), - _ => self - .nested - .entry(name[0].clone()) - .or_default() - .insert(&name[1..], file), - } - } - - /// Generates rust code of the module. - fn generate(&self) -> String { - let mut entries = vec![]; - entries.extend( - self.nested - .iter() - .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), - ); - entries.extend( - self.include - .iter() - .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), - ); - entries.join("\n") - } -} - -/// Checks if field `f` supports canonical encoding. -fn check_canonical_field(f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { - if f.is_map() { - anyhow::bail!("maps unsupported"); - } - if !f.is_list() && !f.supports_presence() { - anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); - } - Ok(()) -} - -/// Checks if messages of type `m` support canonical encoding. -fn check_canonical_message(m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { - for m in m.child_messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; - } - for f in m.fields() { - check_canonical_field(&f).with_context(|| f.name().to_string())?; - } - Ok(()) -} - -/// Checks if message types in file `f` support canonical encoding. -fn check_canonical_file(f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { - if f.syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - for m in f.messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; - } - Ok(()) -} - -/// Checks if message types in descriptor pool `d` support canonical encoding. -fn check_canonical_pool(d: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { - for f in d.files() { - check_canonical_file(&f).with_context(|| f.name().to_string())?; - } - Ok(()) -} +use std::{env, path::PathBuf}; fn main() -> anyhow::Result<()> { // Prepare input and output root dirs. let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) - .canonicalize()? - .join("proto"); - println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); - let _ = fs::remove_dir_all(&proto_output); - fs::create_dir_all(&proto_output).unwrap(); - - // Find all proto files. - let mut proto_inputs = vec![]; - traverse_files(proto_include.clone(), &mut |path| { - let Some(ext) = path.extension() else { return }; - let Some(ext) = ext.to_str() else { return }; - if ext != "proto" { - return; - }; - proto_inputs.push(path); - })?; - - // Generate protobuf code from schema (with reflection). - env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); - let mut config = prost_build::Config::new(); - let descriptor_path = proto_output.join("descriptor.bin"); - config.out_dir(&proto_output); - prost_reflect_build::Builder::new() - .file_descriptor_set_path(&descriptor_path) - .descriptor_pool("crate::proto::DESCRIPTOR_POOL") - .compile_protos_with_config(config, &proto_inputs, &[&proto_include]) - .unwrap(); - let descriptor = fs::read(descriptor_path)?; - let pool = prost_reflect::DescriptorPool::decode(descriptor.as_ref()).unwrap(); - - // Check that messages are compatible with `proto_fmt::canonical`. - check_canonical_pool(&pool)?; - - // Generate mod file collecting all proto-generated code. - let mut m = Module::default(); - for entry in fs::read_dir(&proto_output).unwrap() { - let entry = entry.unwrap(); - let name = entry.file_name().into_string().unwrap(); - let Some(name) = name.strip_suffix(".rs") else { - continue; - }; - let name: Vec<_> = name.split('.').map(String::from).collect(); - println!("name = {name:?}"); - m.insert(&name, entry.path()); - } - let file = m.generate(); - let file = syn::parse_str(&file).unwrap(); - fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; + protobuf_build::compile(&proto_include,"crate::proto")?; Ok(()) } diff --git a/node/libs/schema/src/lib.rs b/node/libs/schema/src/lib.rs index eeb2ba7e..b36a9bb7 100644 --- a/node/libs/schema/src/lib.rs +++ b/node/libs/schema/src/lib.rs @@ -13,11 +13,4 @@ mod tests; #[allow(warnings)] pub mod proto { include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); - static DESCRIPTOR_POOL: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| { - prost_reflect::DescriptorPool::decode( - include_bytes!(concat!(env!("OUT_DIR"), "/proto/descriptor.bin")).as_ref(), - ) - .unwrap() - }); } From 1a9752c87e091f814c7afcb76ed96484da821c5f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 30 Oct 2023 11:21:42 +0100 Subject: [PATCH 02/40] extracted protobuf utils --- node/Cargo.lock | 17 + node/Cargo.toml | 1 + node/libs/protobuf/utils/Cargo.toml | 27 ++ node/libs/protobuf/utils/build.rs | 11 + .../proto/conformance/conformance.proto | 0 .../conformance/test_messages_proto3.proto | 0 .../utils}/proto/std.proto | 0 .../utils}/proto/testonly.proto | 0 .../utils}/src/bin/conformance_test.rs | 12 +- .../src/bin/conformance_test_failure_list.txt | 0 node/libs/protobuf/utils/src/lib.rs | 16 + node/libs/protobuf/utils/src/proto_fmt.rs | 329 ++++++++++++++++++ node/libs/protobuf/utils/src/std_conv.rs | 103 ++++++ node/libs/protobuf/utils/src/testonly.rs | 89 +++++ .../{schema => protobuf/utils}/src/tests.rs | 0 node/libs/schema/Cargo.toml | 3 - 16 files changed, 599 insertions(+), 9 deletions(-) create mode 100644 node/libs/protobuf/utils/Cargo.toml create mode 100644 node/libs/protobuf/utils/build.rs rename node/libs/{schema => protobuf/utils}/proto/conformance/conformance.proto (100%) rename node/libs/{schema => protobuf/utils}/proto/conformance/test_messages_proto3.proto (100%) rename node/libs/{schema => protobuf/utils}/proto/std.proto (100%) rename node/libs/{schema => protobuf/utils}/proto/testonly.proto (100%) rename node/libs/{schema => protobuf/utils}/src/bin/conformance_test.rs (87%) rename node/libs/{schema => protobuf/utils}/src/bin/conformance_test_failure_list.txt (100%) create mode 100644 node/libs/protobuf/utils/src/lib.rs create mode 100644 node/libs/protobuf/utils/src/proto_fmt.rs create mode 100644 node/libs/protobuf/utils/src/std_conv.rs create mode 100644 node/libs/protobuf/utils/src/testonly.rs rename node/libs/{schema => protobuf/utils}/src/tests.rs (100%) diff --git a/node/Cargo.lock b/node/Cargo.lock index 57b633c9..09934438 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -1491,6 +1491,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "protobuf_utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "bit-vec", + "concurrency", + "prost", + "prost-reflect", + "protobuf_build", + "quick-protobuf", + "rand", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "protoc-bin-vendored" version = "3.0.0" diff --git a/node/Cargo.toml b/node/Cargo.toml index fe388e92..8ae2ba7e 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -2,6 +2,7 @@ members = [ "libs/concurrency", "libs/protobuf/build", + "libs/protobuf/utils", "actors/consensus", "libs/crypto", "actors/executor", diff --git a/node/libs/protobuf/utils/Cargo.toml b/node/libs/protobuf/utils/Cargo.toml new file mode 100644 index 00000000..1bc3e13b --- /dev/null +++ b/node/libs/protobuf/utils/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "protobuf_utils" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[[bin]] +name = "conformance_test" + +[dependencies] +anyhow.workspace = true +bit-vec.workspace = true +serde.workspace = true +quick-protobuf.workspace = true +prost.workspace = true +prost-reflect.workspace = true +rand.workspace = true +serde_json.workspace = true +tokio.workspace = true + +concurrency = { path = "../../concurrency" } + +[build-dependencies] +anyhow.workspace = true +protobuf_build = { path = "../build" } diff --git a/node/libs/protobuf/utils/build.rs b/node/libs/protobuf/utils/build.rs new file mode 100644 index 00000000..22fc6091 --- /dev/null +++ b/node/libs/protobuf/utils/build.rs @@ -0,0 +1,11 @@ +//! Generates rust code from the capnp schema files in the `capnp/` directory. +use std::{env, path::PathBuf}; + +fn main() -> anyhow::Result<()> { + // Prepare input and output root dirs. + let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) + .canonicalize()? + .join("proto"); + protobuf_build::compile(&proto_include,"crate::proto")?; + Ok(()) +} diff --git a/node/libs/schema/proto/conformance/conformance.proto b/node/libs/protobuf/utils/proto/conformance/conformance.proto similarity index 100% rename from node/libs/schema/proto/conformance/conformance.proto rename to node/libs/protobuf/utils/proto/conformance/conformance.proto diff --git a/node/libs/schema/proto/conformance/test_messages_proto3.proto b/node/libs/protobuf/utils/proto/conformance/test_messages_proto3.proto similarity index 100% rename from node/libs/schema/proto/conformance/test_messages_proto3.proto rename to node/libs/protobuf/utils/proto/conformance/test_messages_proto3.proto diff --git a/node/libs/schema/proto/std.proto b/node/libs/protobuf/utils/proto/std.proto similarity index 100% rename from node/libs/schema/proto/std.proto rename to node/libs/protobuf/utils/proto/std.proto diff --git a/node/libs/schema/proto/testonly.proto b/node/libs/protobuf/utils/proto/testonly.proto similarity index 100% rename from node/libs/schema/proto/testonly.proto rename to node/libs/protobuf/utils/proto/testonly.proto diff --git a/node/libs/schema/src/bin/conformance_test.rs b/node/libs/protobuf/utils/src/bin/conformance_test.rs similarity index 87% rename from node/libs/schema/src/bin/conformance_test.rs rename to node/libs/protobuf/utils/src/bin/conformance_test.rs index e6de4eaa..db2f2abb 100644 --- a/node/libs/schema/src/bin/conformance_test.rs +++ b/node/libs/protobuf/utils/src/bin/conformance_test.rs @@ -9,7 +9,7 @@ use anyhow::Context as _; use concurrency::{ctx, io}; use prost::Message as _; use prost_reflect::ReflectMessage; -use schema::proto::conformance as proto; +use protobuf_utils::proto::conformance as proto; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -36,10 +36,10 @@ async fn main() -> anyhow::Result<()> { // Decode. let payload = req.payload.context("missing payload")?; - use schema::proto::protobuf_test_messages::proto3::TestAllTypesProto3 as T; + use protobuf_utils::proto::protobuf_test_messages::proto3::TestAllTypesProto3 as T; let p = match payload { proto::conformance_request::Payload::JsonPayload(payload) => { - match schema::decode_json_proto(&payload) { + match protobuf_utils::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } @@ -50,7 +50,7 @@ async fn main() -> anyhow::Result<()> { return Ok(R::ParseError("parsing failed".to_string())); }; // Then check if there are any unknown fields in the original payload. - if schema::canonical_raw(&payload[..], &p.descriptor()).is_err() { + if protobuf_utils::canonical_raw(&payload[..], &p.descriptor()).is_err() { return Ok(R::Skipped("unsupported fields".to_string())); } p @@ -64,11 +64,11 @@ async fn main() -> anyhow::Result<()> { .context("missing output format")?; match proto::WireFormat::from_i32(format).context("unknown format")? { proto::WireFormat::Json => { - anyhow::Ok(R::JsonPayload(schema::encode_json_proto(&p))) + anyhow::Ok(R::JsonPayload(protobuf_utils::encode_json_proto(&p))) } proto::WireFormat::Protobuf => { // Reencode the parsed proto. - anyhow::Ok(R::ProtobufPayload(schema::canonical_raw( + anyhow::Ok(R::ProtobufPayload(protobuf_utils::canonical_raw( &p.encode_to_vec(), &p.descriptor(), )?)) diff --git a/node/libs/schema/src/bin/conformance_test_failure_list.txt b/node/libs/protobuf/utils/src/bin/conformance_test_failure_list.txt similarity index 100% rename from node/libs/schema/src/bin/conformance_test_failure_list.txt rename to node/libs/protobuf/utils/src/bin/conformance_test_failure_list.txt diff --git a/node/libs/protobuf/utils/src/lib.rs b/node/libs/protobuf/utils/src/lib.rs new file mode 100644 index 00000000..b36a9bb7 --- /dev/null +++ b/node/libs/protobuf/utils/src/lib.rs @@ -0,0 +1,16 @@ +//! Code generated from protobuf schema files and +//! utilities for serialization. + +mod proto_fmt; +mod std_conv; +pub mod testonly; + +pub use proto_fmt::*; + +#[cfg(test)] +mod tests; + +#[allow(warnings)] +pub mod proto { + include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); +} diff --git a/node/libs/protobuf/utils/src/proto_fmt.rs b/node/libs/protobuf/utils/src/proto_fmt.rs new file mode 100644 index 00000000..541b0f0b --- /dev/null +++ b/node/libs/protobuf/utils/src/proto_fmt.rs @@ -0,0 +1,329 @@ +//! Canonical encoding for protobufs with known schema. +//! +//! Protobuf encoding spec: https://protobuf.dev/programming-guides/encoding/ +//! By default the encoding is not canonical (multiple encodings may represent the same message). +//! Here we extend this spec to support a canonical encoding. +//! +//! Canonical encoding spec: +//! * fields are encoded in ascending order of tags. +//! * varints are encoded using minimal amount of bytes. +//! * map fields are not supported +//! * groups are not supported +//! * only proto3 syntax is supported +//! * hence assigning a field a default value (proto2) is not supported +//! * all fields should have explicit presence (explicit optional or repeated) +//! * hence a default value for primitive types (proto3) is not supported. +//! * a field of primitive type (I32, I64, Varint) is encoded as at most 1 TLV, depending on +//! the number of values: +//! * 0 values => field is not encoded at all +//! * 1 value => field is encoded as a single I32/I64/Varint TLV. +//! * >1 values => field is encoded as a single TLV using packed encoding (https://protobuf.dev/programming-guides/encoding/#packed) +//! * protobuf spec allows for packed encoding only for repeated fields. We therefore don't +//! support specifying multiple values for a singular field (which is pathological case +//! anyway). +//! * protobuf spec also doesn't allow for encoding repeated enum fields (which are encoded as +//! Varint), however it mentions that it might allow for it in the future versions. +//! TODO(gprusak): decide what we should do with those: prost implementation decodes correctly +//! packed enums, I don't know about other protobuf implementations (rust and non-rust). We may either: +//! * allow packed encoding for enums (despite the incompatibility with protobuf spec) +//! * disallow packed encoding for enums specifically (although it complicates the canonical spec) +//! * drop packed encoding altogether (at the cost of larger messages) +//! +//! We need canonical encoding to be able to compute hashes and verify signatures on +//! messages. Note that with this spec it is not possible to encode canonically a message with unknown +//! fields. It won't be a problem though if we decide on one of the following: +//! * require all communication to just use canonical encoding: then as long as we carry around the +//! unknown fields untouched, they will stay canonical. +//! * hash/verify only messages that don't contain unknown fields: then we can just canonicalize +//! whatever we understand and reject messages with unknown fields. +//! * drop the idea of canonical encoding altogether and pass around the received encoded message +//! for hashing/verifying (i.e. keep the raw bytes together with the parsed message). +use anyhow::Context as _; +use prost::Message as _; +use prost_reflect::ReflectMessage; +use std::collections::BTreeMap; + +/// Kilobyte. +#[allow(non_upper_case_globals)] +pub const kB: usize = 1 << 10; +/// Megabyte. +pub const MB: usize = 1 << 20; + +/// Protobuf wire type. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Wire { + /// VARINT. + Varint, + /// I64 + I64, + /// LEN + Len, + /// I32 + I32, +} + +/// Raw write type values, as defined in +/// https://protobuf.dev/programming-guides/encoding/#structure +/// VARINT +const VARINT: u32 = 0; +/// I64 +const I64: u32 = 1; +/// LEN +const LEN: u32 = 2; +/// I32 +const I32: u32 = 5; + +impl Wire { + /// Extracts a wire type from a tag. + pub(crate) const fn from_tag(tag: u32) -> Option { + match tag & 7 { + VARINT => Some(Self::Varint), + I64 => Some(Self::I64), + LEN => Some(Self::Len), + I32 => Some(Self::I32), + _ => None, + } + } + + /// Converts wire type to the raw wite type value. + pub(crate) const fn raw(self) -> u32 { + match self { + Self::Varint => VARINT, + Self::I64 => I64, + Self::Len => LEN, + Self::I32 => I32, + } + } +} + +impl From for Wire { + fn from(kind: prost_reflect::Kind) -> Self { + use prost_reflect::Kind; + match kind { + Kind::Int32 + | Kind::Int64 + | Kind::Uint32 + | Kind::Uint64 + | Kind::Sint32 + | Kind::Sint64 + | Kind::Bool + | Kind::Enum(_) => Self::Varint, + Kind::Fixed64 | Kind::Sfixed64 | Kind::Double => Self::I64, + Kind::Fixed32 | Kind::Sfixed32 | Kind::Float => Self::I32, + Kind::String | Kind::Bytes | Kind::Message(_) => Self::Len, + } + } +} + +/// Wrapper of the quick_protobuf reader. +// TODO(gprusak): reimplement the reader and writer instead to make it more efficient. +pub(super) struct Reader<'a>(quick_protobuf::BytesReader, &'a [u8]); + +impl<'a> Reader<'a> { + /// Constructs a reader of a slice. + fn new(bytes: &'a [u8]) -> Self { + Self(quick_protobuf::BytesReader::from_bytes(bytes), bytes) + } + + /// Reads a value of the given wire type. + fn read(&mut self, wire: Wire) -> anyhow::Result> { + let mut v = vec![]; + let mut w = quick_protobuf::Writer::new(&mut v); + match wire { + Wire::Varint => w.write_varint(self.0.read_varint64(self.1)?).unwrap(), + Wire::I64 => w.write_fixed64(self.0.read_fixed64(self.1)?).unwrap(), + Wire::Len => return Ok(self.0.read_bytes(self.1)?.into()), + Wire::I32 => w.write_fixed32(self.0.read_fixed32(self.1)?).unwrap(), + } + Ok(v) + } + + /// Reads a (possibly packed) field based on the expected field wire type and the actual + /// wire type from the tag (which has been already read). + /// The values of the field are appended to `out`. + fn read_field( + &mut self, + out: &mut Vec>, + field_wire: Wire, + got_wire: Wire, + ) -> anyhow::Result<()> { + if got_wire == field_wire { + out.push(self.read(field_wire)?); + return Ok(()); + } + if got_wire != Wire::Len { + anyhow::bail!("unexpected wire type"); + } + let mut r = Self::new(self.0.read_bytes(self.1)?); + while !r.0.is_eof() { + out.push(r.read(field_wire)?); + } + Ok(()) + } +} + +/// Parses a message to a `field num -> values` map. +pub(super) fn read_fields( + buf: &[u8], + desc: &prost_reflect::MessageDescriptor, +) -> anyhow::Result>>> { + if desc.parent_file().syntax() != prost_reflect::Syntax::Proto3 { + anyhow::bail!("only proto3 syntax is supported"); + } + let mut r = Reader::new(buf); + let mut fields = BTreeMap::new(); + // Collect the field values from the reader. + while !r.0.is_eof() { + let tag = r.0.next_tag(r.1)?; + let wire = Wire::from_tag(tag).context("invalid wire type")?; + let field = desc.get_field(tag >> 3).context("unknown field")?; + if field.is_map() { + anyhow::bail!("maps unsupported"); + } + if !field.is_list() && !field.supports_presence() { + anyhow::bail!( + "{}::{} : fields with implicit presence are not supported", + field.parent_message().name(), + field.name() + ); + } + r.read_field( + fields.entry(field.number()).or_default(), + field.kind().into(), + wire, + )?; + } + Ok(fields) +} + +/// Converts an encoded protobuf message to its canonical form, given the descriptor of the message +/// type. Retuns an error if: +/// * an unknown field is detected +/// * the message type doesn't support canonical encoding (implicit presence, map fields) +pub fn canonical_raw( + buf: &[u8], + desc: &prost_reflect::MessageDescriptor, +) -> anyhow::Result> { + let mut v = vec![]; + let mut w = quick_protobuf::Writer::new(&mut v); + // Append fields in ascending tags order. + for (num, mut values) in read_fields(buf, desc)? { + let fd = desc.get_field(num).unwrap(); + if values.len() > 1 && !fd.is_list() { + anyhow::bail!("non-repeated field with multiple values"); + } + if let prost_reflect::Kind::Message(desc) = &fd.kind() { + for v in &mut values { + *v = canonical_raw(v, desc)?; + } + } + let wire = Wire::from(fd.kind()); + match wire { + Wire::Varint | Wire::I64 | Wire::I32 => { + if values.len() > 1 { + w.write_tag(num << 3 | LEN).unwrap(); + w.write_bytes(&values.into_iter().flatten().collect::>()) + .unwrap(); + } else { + w.write_tag(num << 3 | wire.raw()).unwrap(); + // inefficient workaround of the fact that quick_protobuf::Writer + // doesn't support just appending a sequence of bytes. + for b in &values[0] { + w.write_u8(*b).unwrap(); + } + } + } + Wire::Len => { + for v in &values { + w.write_tag(num << 3 | LEN).unwrap(); + w.write_bytes(v).unwrap(); + } + } + } + } + Ok(v) +} + +/// Encodes a proto message of known schema (no unknown fields) into a canonical form. +pub fn canonical(x: &T) -> Vec { + let msg = x.build(); + canonical_raw(&msg.encode_to_vec(), &msg.descriptor()).unwrap() +} + +/// Encodes a proto message. +/// Currently it outputs a canonical encoding, but `decode` accepts +/// non-canonical encoding as well. +/// It would simplify invariants if we required all messages to be canonical, +/// but we will see whether it is feasible once we have an RPC framework +/// in place. +pub fn encode(x: &T) -> Vec { + canonical(x) +} + +/// Decodes a proto message. +pub fn decode(bytes: &[u8]) -> anyhow::Result { + T::read(&::Proto::decode(bytes)?) +} + +/// Encodes a generated proto message to json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn encode_json_proto(x: &T) -> String { + let mut s = serde_json::Serializer::pretty(vec![]); + let opts = prost_reflect::SerializeOptions::new(); + x.transcode_to_dynamic() + .serialize_with_options(&mut s, &opts) + .unwrap(); + String::from_utf8(s.into_inner()).unwrap() +} + +/// Encodes a proto message to json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn encode_json(x: &T) -> String { + encode_json_proto(&x.build()) +} + +/// Decodes a generated proto message from json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn decode_json_proto(json: &str) -> anyhow::Result { + let mut d = serde_json::de::Deserializer::from_str(json); + let mut p = T::default(); + let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), &mut d)?; + d.end()?; + p.merge(msg.encode_to_vec().as_slice()).unwrap(); + Ok(p) +} + +/// Decodes a proto message from json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn decode_json(json: &str) -> anyhow::Result { + T::read(&decode_json_proto(json)?) +} + +/// Trait defining a proto representation for a type. +pub trait ProtoFmt: Sized { + /// Proto message type representing Self. + type Proto: ReflectMessage + Default; + /// Converts Proto to Self. + fn read(r: &Self::Proto) -> anyhow::Result; + /// Converts Self to Proto. + fn build(&self) -> Self::Proto; + /// Maximal allowed message size (in bytes). + fn max_size() -> usize { + usize::MAX + } +} + +/// Parses a required proto field. +pub fn read_required(field: &Option) -> anyhow::Result { + ProtoFmt::read(field.as_ref().context("missing field")?) +} + +/// Parses an optional proto field. +pub fn read_optional(field: &Option) -> anyhow::Result> { + field.as_ref().map(ProtoFmt::read).transpose() +} + +/// Extracts a required field. +pub fn required(field: &Option) -> anyhow::Result<&T> { + field.as_ref().context("missing") +} diff --git a/node/libs/protobuf/utils/src/std_conv.rs b/node/libs/protobuf/utils/src/std_conv.rs new file mode 100644 index 00000000..2f2fa89b --- /dev/null +++ b/node/libs/protobuf/utils/src/std_conv.rs @@ -0,0 +1,103 @@ +//! Proto conversion for messages in std package. +use crate::{proto::std as proto, required, ProtoFmt}; +use anyhow::Context as _; +use concurrency::time; +use std::net; + +impl ProtoFmt for () { + type Proto = proto::Void; + fn read(_r: &Self::Proto) -> anyhow::Result { + Ok(()) + } + fn build(&self) -> Self::Proto { + Self::Proto {} + } +} + +impl ProtoFmt for std::net::SocketAddr { + type Proto = proto::SocketAddr; + + fn read(r: &Self::Proto) -> anyhow::Result { + let ip = required(&r.ip).context("ip")?; + let ip = match ip.len() { + 4 => net::IpAddr::from(<[u8; 4]>::try_from(&ip[..]).unwrap()), + 16 => net::IpAddr::from(<[u8; 16]>::try_from(&ip[..]).unwrap()), + _ => anyhow::bail!("invalid ip length"), + }; + let port = *required(&r.port).context("port")?; + let port = u16::try_from(port).context("port")?; + Ok(Self::new(ip, port)) + } + + fn build(&self) -> Self::Proto { + Self::Proto { + ip: Some(match self.ip() { + net::IpAddr::V4(ip) => ip.octets().to_vec(), + net::IpAddr::V6(ip) => ip.octets().to_vec(), + }), + port: Some(self.port() as u32), + } + } +} + +impl ProtoFmt for time::Utc { + type Proto = proto::Timestamp; + + fn read(r: &Self::Proto) -> anyhow::Result { + let seconds = *required(&r.seconds).context("seconds")?; + let nanos = *required(&r.nanos).context("nanos")?; + Ok(time::UNIX_EPOCH + time::Duration::new(seconds, nanos)) + } + + fn build(&self) -> Self::Proto { + let d = (*self - time::UNIX_EPOCH).build(); + Self::Proto { + seconds: d.seconds, + nanos: d.nanos, + } + } +} + +impl ProtoFmt for time::Duration { + type Proto = proto::Duration; + + fn read(r: &Self::Proto) -> anyhow::Result { + let seconds = *required(&r.seconds).context("seconds")?; + let nanos = *required(&r.nanos).context("nanos")?; + Ok(Self::new(seconds, nanos)) + } + + fn build(&self) -> Self::Proto { + let mut seconds = self.whole_seconds(); + let mut nanos = self.subsec_nanoseconds(); + if nanos < 0 { + seconds -= 1; + nanos += 1_000_000_000; + } + Self::Proto { + seconds: Some(seconds), + nanos: Some(nanos), + } + } +} + +impl ProtoFmt for bit_vec::BitVec { + type Proto = proto::BitVector; + + fn read(r: &Self::Proto) -> anyhow::Result { + let size = *required(&r.size).context("size")? as usize; + let mut this = Self::from_bytes(required(&r.bytes).context("bytes_")?); + if this.len() < size { + anyhow::bail!("'vector' has less than 'size' bits"); + } + this.truncate(size); + Ok(this) + } + + fn build(&self) -> Self::Proto { + Self::Proto { + size: Some(self.len() as u64), + bytes: Some(self.to_bytes()), + } + } +} diff --git a/node/libs/protobuf/utils/src/testonly.rs b/node/libs/protobuf/utils/src/testonly.rs new file mode 100644 index 00000000..9cf15ae0 --- /dev/null +++ b/node/libs/protobuf/utils/src/testonly.rs @@ -0,0 +1,89 @@ +//! Testonly utilities. +use super::{canonical, canonical_raw, decode, encode, read_fields, ProtoFmt, Wire}; +use prost::Message as _; +use prost_reflect::ReflectMessage as _; +use rand::{ + distributions::{Distribution, Standard}, + Rng, +}; + +/// Test encoding and canonical encoding properties. +#[track_caller] +pub fn test_encode(rng: &mut R, x: &T) { + let x_encode = encode(x); + let x_canonical = canonical(x); + let x_shuffled = encode_shuffled(rng, x); + + let desc = &::Proto::default().descriptor(); + for e in [&x_encode, &x_canonical, &x_shuffled] { + // e is a valid encoding. + assert_eq!(x, &decode::(e).unwrap()); + // canonical encoding is consistent. + assert_eq!(x_canonical, canonical_raw(e, desc).unwrap()); + } +} + +/// Syntax sugar for `test_encode`, +/// because `test_encode(rng,&rng::gen())` doesn't compile. +#[track_caller] +pub fn test_encode_random(rng: &mut R) +where + Standard: Distribution, +{ + let msg = rng.gen::(); + test_encode(rng, &msg); +} + +/// shuffles recursively the order of fields in a protobuf encoding. +fn encode_shuffled_raw( + rng: &mut R, + buf: &[u8], + desc: &prost_reflect::MessageDescriptor, +) -> anyhow::Result> { + // Vector of field values, values of each field are in reverse order. + // This way we can pop the next value of the field in O(1). + let mut fields: Vec<_> = read_fields(buf, desc)?.into_iter().collect(); + for f in &mut fields { + f.1.reverse(); + } + + // Append fields in random order. + let mut v = vec![]; + let mut w = quick_protobuf::Writer::new(&mut v); + while !fields.is_empty() { + // Select a random field. + let i = rng.gen_range(0..fields.len()); + // Select the next value of this field. + // Note that the values are stored in reverse order. + let Some(mut v) = fields[i].1.pop() else { + let last = fields.len() - 1; + fields.swap(i, last); + fields.pop(); + continue; + }; + let num = fields[i].0; + let fd = desc.get_field(num).unwrap(); + if let prost_reflect::Kind::Message(desc) = &fd.kind() { + v = encode_shuffled_raw(rng, &v, desc)?; + } + let wire = Wire::from(fd.kind()); + w.write_tag(num << 3 | wire.raw()).unwrap(); + match wire { + Wire::Varint | Wire::I64 | Wire::I32 => { + // inefficient workaround of the fact that quick_protobuf::Writer + // doesn't support just appending a sequence of bytes. + for b in &v { + w.write_u8(*b).unwrap(); + } + } + Wire::Len => w.write_bytes(&v).unwrap(), + } + } + Ok(v) +} + +/// Encodes a proto message of known schema in a random encoding order of fields. +pub(crate) fn encode_shuffled(rng: &mut R, x: &T) -> Vec { + let msg = x.build(); + encode_shuffled_raw(rng, &msg.encode_to_vec(), &msg.descriptor()).unwrap() +} diff --git a/node/libs/schema/src/tests.rs b/node/libs/protobuf/utils/src/tests.rs similarity index 100% rename from node/libs/schema/src/tests.rs rename to node/libs/protobuf/utils/src/tests.rs diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index 4d1639f7..19b73f08 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -6,9 +6,6 @@ authors.workspace = true homepage.workspace = true license.workspace = true -[[bin]] -name = "conformance_test" - [dependencies] anyhow.workspace = true bit-vec.workspace = true From e34905805736c02e613a93be8c16d1bfa7b791de Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 30 Oct 2023 12:17:34 +0100 Subject: [PATCH 03/40] snapshot --- node/Cargo.lock | 1 + node/libs/protobuf/build/src/lib.rs | 49 ++++++++++++++++++----------- node/libs/protobuf/utils/build.rs | 2 +- node/libs/schema/Cargo.toml | 2 ++ node/libs/schema/build.rs | 2 +- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index 09934438..a80ef4fa 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -1745,6 +1745,7 @@ dependencies = [ "prost", "prost-reflect", "protobuf_build", + "protobuf_utils", "quick-protobuf", "rand", "serde", diff --git a/node/libs/protobuf/build/src/lib.rs b/node/libs/protobuf/build/src/lib.rs index 2c33c18c..4cc22394 100644 --- a/node/libs/protobuf/build/src/lib.rs +++ b/node/libs/protobuf/build/src/lib.rs @@ -102,7 +102,7 @@ fn check_canonical_pool(d: &prost_reflect::DescriptorPool) -> anyhow::Result<()> Ok(()) } -pub fn compile(proto_include: &Path, module_name: &str) -> anyhow::Result<()> { +pub fn compile(proto_include: &Path, module_name: &str, deps: &[&'static [u8]]) -> anyhow::Result<()> { println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) .canonicalize()? @@ -126,13 +126,19 @@ pub fn compile(proto_include: &Path, module_name: &str) -> anyhow::Result<()> { let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); cmd.arg("-o").arg(&descriptor_path); cmd.arg("-I").arg(&proto_include); - /*.arg("--descriptor_set_in").arg( - format!("{}:{}", - proto_include.join("b/b.desc").to_str().unwrap(), - proto_include.join("c/c.desc").to_str().unwrap(), - ) - )*/ - + + let mut pool = prost_reflect::DescriptorPool::new(); + if deps.len() > 0 { + let mut deps_list = vec![]; + for (i,d) in deps.iter().enumerate() { + pool.decode_file_descriptor_set(*d).unwrap(); // TODO: make it transitive. + let name = proto_output.join(format!("dep{i}.binpb")); + fs::write(&name,d)?; + deps_list.push(name.to_str().unwrap().to_string()); + } + cmd.arg("--descriptor_set_in").arg(deps_list.join(":")); + } + for input in &proto_inputs { cmd.arg(&input); } @@ -142,19 +148,26 @@ pub fn compile(proto_include: &Path, module_name: &str) -> anyhow::Result<()> { if !out.status.success() { anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); } - + // Generate protobuf code from schema (with reflection). - env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); + let descriptor = fs::read(&descriptor_path)?; + pool.decode_file_descriptor_set(&descriptor[..])?; + let file_descriptor_set_bytes = format!("{module_name}::DESCRIPTOR_POOL"); + let pool_attribute = format!(r#"#[prost_reflect(file_descriptor_set_bytes = "{}")]"#,file_descriptor_set_bytes); + + let empty : &[&Path] = &[]; let mut config = prost_build::Config::new(); + for message in pool.all_messages() { + let full_name = message.full_name(); + config + .type_attribute(full_name, "#[derive(::prost_reflect::ReflectMessage)]") + .type_attribute(full_name, &format!(r#"#[prost_reflect(message_name = "{}")]"#, full_name)) + .type_attribute(full_name, &pool_attribute); + } + config.file_descriptor_set_path(&descriptor_path); config.skip_protoc_run(); config.out_dir(&proto_output); - prost_reflect_build::Builder::new() - .file_descriptor_set_path(&descriptor_path) - .file_descriptor_set_bytes(format!("{module_name}::DESCRIPTOR_POOL")) - .compile_protos_with_config(config, &proto_inputs, &[&proto_include]) - .unwrap(); - let descriptor = fs::read(&descriptor_path)?; - let pool = prost_reflect::DescriptorPool::decode(descriptor.as_ref()).unwrap(); + config.compile_protos(empty,empty).unwrap(); // Check that messages are compatible with `proto_fmt::canonical`. check_canonical_pool(&pool)?; @@ -173,7 +186,7 @@ pub fn compile(proto_include: &Path, module_name: &str) -> anyhow::Result<()> { } let mut file = m.generate(); - file += &format!("const DESCRIPTOR_POOL: &'static [u8] = include_bytes!({descriptor_path:?});"); + file += &format!("pub const DESCRIPTOR_POOL: &'static [u8] = include_bytes!({descriptor_path:?});"); let file = syn::parse_str(&file).unwrap(); fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; diff --git a/node/libs/protobuf/utils/build.rs b/node/libs/protobuf/utils/build.rs index 22fc6091..bf9a738e 100644 --- a/node/libs/protobuf/utils/build.rs +++ b/node/libs/protobuf/utils/build.rs @@ -6,6 +6,6 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf_build::compile(&proto_include,"crate::proto")?; + protobuf_build::compile(&proto_include,"crate::proto",&[])?; Ok(()) } diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index 19b73f08..4211f989 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -18,8 +18,10 @@ rand.workspace = true serde_json.workspace = true tokio.workspace = true +protobuf_utils = { path = "../protobuf/utils" } concurrency = { path = "../concurrency" } [build-dependencies] anyhow.workspace = true protobuf_build = { path = "../protobuf/build" } +protobuf_utils = { path = "../protobuf/utils" } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 22fc6091..1c5e6db1 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -6,6 +6,6 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf_build::compile(&proto_include,"crate::proto")?; + protobuf_build::compile(&proto_include,"crate::proto",&[&protobuf_utils::proto::DESCRIPTOR_POOL])?; Ok(()) } From 9260b2d437c606ea16b1ef2eec639197cd34ebda Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 30 Oct 2023 12:48:27 +0100 Subject: [PATCH 04/40] reflection is broken --- node/Cargo.lock | 6 + node/Cargo.toml | 2 + node/actors/consensus/Cargo.toml | 1 + node/actors/consensus/src/inner.rs | 2 +- node/actors/consensus/src/metrics.rs | 2 +- node/actors/executor/Cargo.toml | 1 + .../src/lib/configurator/config_paths.rs | 2 +- .../src/lib/configurator/node_config.rs | 3 +- node/actors/network/Cargo.toml | 1 + .../network/src/consensus/handshake/mod.rs | 3 +- node/actors/network/src/frame.rs | 16 +- .../network/src/gossip/handshake/mod.rs | 3 +- node/actors/network/src/mux/handshake.rs | 7 +- node/actors/network/src/preface.rs | 7 +- node/actors/network/src/rpc/consensus.rs | 7 +- node/actors/network/src/rpc/metrics.rs | 2 +- node/actors/network/src/rpc/mod.rs | 10 +- node/actors/network/src/rpc/ping.rs | 7 +- node/actors/network/src/rpc/sync_blocks.rs | 13 +- .../network/src/rpc/sync_validator_addrs.rs | 7 +- node/libs/protobuf/build/src/lib.rs | 8 +- node/libs/roles/Cargo.toml | 1 + node/libs/roles/src/node/conv.rs | 2 +- node/libs/roles/src/node/messages.rs | 2 +- node/libs/roles/src/validator/conv.rs | 8 +- .../roles/src/validator/messages/block.rs | 6 +- node/libs/roles/src/validator/messages/msg.rs | 2 +- node/libs/schema/build.rs | 4 +- node/libs/schema/src/lib.rs | 12 +- node/libs/schema/src/proto_fmt.rs | 329 ------------------ node/libs/schema/src/std_conv.rs | 103 ------ node/libs/schema/src/testonly.rs | 89 ----- node/libs/storage/Cargo.toml | 1 + node/libs/storage/src/rocksdb.rs | 12 +- node/libs/storage/src/types.rs | 4 +- node/tools/Cargo.toml | 1 + node/tools/src/bin/localnet_config.rs | 2 +- 37 files changed, 91 insertions(+), 597 deletions(-) delete mode 100644 node/libs/schema/src/proto_fmt.rs delete mode 100644 node/libs/schema/src/std_conv.rs delete mode 100644 node/libs/schema/src/testonly.rs diff --git a/node/Cargo.lock b/node/Cargo.lock index a80ef4fa..980e2969 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -412,6 +412,7 @@ dependencies = [ "crypto", "network", "once_cell", + "protobuf_utils", "rand", "roles", "schema", @@ -610,6 +611,7 @@ dependencies = [ "hex", "network", "once_cell", + "protobuf_utils", "rand", "roles", "schema", @@ -1108,6 +1110,7 @@ dependencies = [ "once_cell", "pin-project", "pretty_assertions", + "protobuf_utils", "rand", "roles", "schema", @@ -1687,6 +1690,7 @@ dependencies = [ "concurrency", "crypto", "hex", + "protobuf_utils", "rand", "schema", "serde", @@ -1954,6 +1958,7 @@ dependencies = [ "assert_matches", "async-trait", "concurrency", + "protobuf_utils", "rand", "rocksdb", "roles", @@ -2161,6 +2166,7 @@ dependencies = [ "crypto", "executor", "hex", + "protobuf_utils", "rand", "roles", "schema", diff --git a/node/Cargo.toml b/node/Cargo.toml index 8ae2ba7e..fc280d3f 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -24,6 +24,8 @@ homepage = "https://matter-labs.io/" license = "MIT" [workspace.dependencies] +protobuf_utils = { path = "libs/protobuf/utils" } + anyhow = "1" assert_matches = "1.5.0" async-trait = "0.1.71" diff --git a/node/actors/consensus/Cargo.toml b/node/actors/consensus/Cargo.toml index beab70a8..82f7667c 100644 --- a/node/actors/consensus/Cargo.toml +++ b/node/actors/consensus/Cargo.toml @@ -15,6 +15,7 @@ thiserror.workspace = true tracing.workspace = true vise.workspace = true +protobuf_utils.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/consensus/src/inner.rs b/node/actors/consensus/src/inner.rs index 57acc7a5..f3de4e33 100644 --- a/node/actors/consensus/src/inner.rs +++ b/node/actors/consensus/src/inner.rs @@ -23,7 +23,7 @@ pub(crate) struct ConsensusInner { impl ConsensusInner { /// The maximum size of the payload of a block, in bytes. We will /// reject blocks with payloads larger than this. - pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * schema::kB; + pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * protobuf_utils::kB; /// Computes the validator for the given view. #[instrument(level = "trace", ret)] diff --git a/node/actors/consensus/src/metrics.rs b/node/actors/consensus/src/metrics.rs index 46d61dfb..a72ac80a 100644 --- a/node/actors/consensus/src/metrics.rs +++ b/node/actors/consensus/src/metrics.rs @@ -4,7 +4,7 @@ use std::time::Duration; use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; const PAYLOAD_SIZE_BUCKETS: Buckets = - Buckets::exponential((4 * schema::kB) as f64..=(4 * schema::MB) as f64, 4.0); + Buckets::exponential((4 * protobuf_utils::kB) as f64..=(4 * protobuf_utils::MB) as f64, 4.0); /// Label for a consensus message. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index bcd3376a..464b6316 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -20,6 +20,7 @@ tracing-subscriber.workspace = true vise.workspace = true vise-exporter.workspace = true +protobuf_utils.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/executor/src/lib/configurator/config_paths.rs b/node/actors/executor/src/lib/configurator/config_paths.rs index 90798764..60a0f0ca 100644 --- a/node/actors/executor/src/lib/configurator/config_paths.rs +++ b/node/actors/executor/src/lib/configurator/config_paths.rs @@ -46,7 +46,7 @@ impl ConfigPaths { #[instrument(level = "trace", ret)] pub(crate) fn read(self) -> anyhow::Result { let cfg = Configs { - config: schema::decode_json( + config: protobuf_utils::decode_json( &std::fs::read_to_string(&self.config).context(self.config)?, )?, validator_key: Text::new( diff --git a/node/actors/executor/src/lib/configurator/node_config.rs b/node/actors/executor/src/lib/configurator/node_config.rs index e53010b7..7a160a2a 100644 --- a/node/actors/executor/src/lib/configurator/node_config.rs +++ b/node/actors/executor/src/lib/configurator/node_config.rs @@ -2,7 +2,8 @@ use anyhow::Context as _; use crypto::{read_optional_text, read_required_text, Text, TextFmt}; use roles::{node, validator}; -use schema::{proto::executor::config as proto, read_required, required, ProtoFmt}; +use schema::proto::executor::config as proto; +use protobuf_utils::{read_required, required, ProtoFmt}; use std::{ collections::{HashMap, HashSet}, net, diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index 7d24287a..b9dc27c7 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -18,6 +18,7 @@ thiserror.workspace = true tracing.workspace = true vise.workspace = true +protobuf_utils.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/network/src/consensus/handshake/mod.rs b/node/actors/network/src/consensus/handshake/mod.rs index d0eabab9..df7d7f6b 100644 --- a/node/actors/network/src/consensus/handshake/mod.rs +++ b/node/actors/network/src/consensus/handshake/mod.rs @@ -3,7 +3,8 @@ use anyhow::Context as _; use concurrency::{ctx, time}; use crypto::{bls12_381, ByteFmt}; use roles::{node, validator}; -use schema::{proto::network::consensus as proto, read_required, ProtoFmt}; +use schema::proto::network::consensus as proto; +use protobuf_utils::{read_required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/frame.rs b/node/actors/network/src/frame.rs index d88fa8ef..bbe3583b 100644 --- a/node/actors/network/src/frame.rs +++ b/node/actors/network/src/frame.rs @@ -7,7 +7,7 @@ use concurrency::{ctx, io}; /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. /// Returns the decoded proto and the size of the received message in bytes. -pub(crate) async fn mux_recv_proto( +pub(crate) async fn mux_recv_proto( ctx: &ctx::Ctx, stream: &mut mux::ReadStream, ) -> anyhow::Result<(T, usize)> { @@ -25,19 +25,19 @@ pub(crate) async fn mux_recv_proto( if msg.len() < msg_size { anyhow::bail!("end of stream"); } - let msg = schema::decode(msg.as_slice())?; + let msg = protobuf_utils::decode(msg.as_slice())?; Ok((msg, msg_size)) } /// Sends a proto serialized to a raw frame of bytes to the stream. /// It doesn't flush the stream. /// Returns the size of the sent proto in bytes. -pub(crate) async fn mux_send_proto( +pub(crate) async fn mux_send_proto( ctx: &ctx::Ctx, stream: &mut mux::WriteStream, msg: &T, ) -> anyhow::Result { - let msg = schema::encode(msg); + let msg = protobuf_utils::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); stream .write_all(ctx, &u32::to_le_bytes(msg.len() as u32)) @@ -49,7 +49,7 @@ pub(crate) async fn mux_send_proto( /// Reads a raw frame of bytes from the stream and interprets it as proto. /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. -pub(crate) async fn recv_proto( +pub(crate) async fn recv_proto( ctx: &ctx::Ctx, stream: &mut S, ) -> anyhow::Result { @@ -61,16 +61,16 @@ pub(crate) async fn recv_proto( } let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stream, &mut msg[..]).await??; - schema::decode(&msg) + protobuf_utils::decode(&msg) } /// Sends a proto serialized to a raw frame of bytes to the stream. -pub(crate) async fn send_proto( +pub(crate) async fn send_proto( ctx: &ctx::Ctx, stream: &mut S, msg: &T, ) -> anyhow::Result<()> { - let msg = schema::encode(msg); + let msg = protobuf_utils::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); io::write_all(ctx, stream, &u32::to_le_bytes(msg.len() as u32)).await??; io::write_all(ctx, stream, &msg).await??; diff --git a/node/actors/network/src/gossip/handshake/mod.rs b/node/actors/network/src/gossip/handshake/mod.rs index c84b377c..b6e1a224 100644 --- a/node/actors/network/src/gossip/handshake/mod.rs +++ b/node/actors/network/src/gossip/handshake/mod.rs @@ -4,7 +4,8 @@ use anyhow::Context as _; use concurrency::{ctx, time}; use crypto::ByteFmt; use roles::node; -use schema::{proto::network::gossip as proto, read_required, required, ProtoFmt}; +use schema::proto::network::gossip as proto; +use protobuf_utils::{read_required, required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/mux/handshake.rs b/node/actors/network/src/mux/handshake.rs index 08450864..80d34cf9 100644 --- a/node/actors/network/src/mux/handshake.rs +++ b/node/actors/network/src/mux/handshake.rs @@ -1,6 +1,7 @@ use super::CapabilityId; use anyhow::Context as _; -use schema::{proto::network::mux as proto, required}; +use schema::proto::network::mux as proto; +use protobuf_utils::required; use std::collections::HashMap; pub(super) struct Handshake { @@ -36,11 +37,11 @@ fn build_capabilities( .collect() } -impl schema::ProtoFmt for Handshake { +impl protobuf_utils::ProtoFmt for Handshake { type Proto = proto::Handshake; fn max_size() -> usize { - schema::kB + protobuf_utils::kB } fn read(r: &Self::Proto) -> anyhow::Result { diff --git a/node/actors/network/src/preface.rs b/node/actors/network/src/preface.rs index 37542746..3ad916d6 100644 --- a/node/actors/network/src/preface.rs +++ b/node/actors/network/src/preface.rs @@ -9,7 +9,8 @@ //! and multiplex between mutliple endpoints available on the same TCP port. use crate::{frame, metrics, noise}; use concurrency::{ctx, time}; -use schema::{proto::network::preface as proto, required, ProtoFmt}; +use schema::proto::network::preface as proto; +use protobuf_utils::{required, ProtoFmt}; /// Timeout on executing the preface protocol. const TIMEOUT: time::Duration = time::Duration::seconds(5); @@ -33,7 +34,7 @@ pub(crate) enum Endpoint { impl ProtoFmt for Encryption { type Proto = proto::Encryption; fn max_size() -> usize { - 10 * schema::kB + 10 * protobuf_utils::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::encryption::T; @@ -53,7 +54,7 @@ impl ProtoFmt for Encryption { impl ProtoFmt for Endpoint { type Proto = proto::Endpoint; fn max_size() -> usize { - 10 * schema::kB + 10 * protobuf_utils::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::endpoint::T; diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 3df4e58e..06b82c21 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -2,7 +2,8 @@ use crate::mux; use concurrency::{limiter, time}; use roles::validator; -use schema::{proto::network::consensus as proto, read_required, ProtoFmt}; +use schema::proto::network::consensus as proto; +use protobuf_utils::{read_required, ProtoFmt}; /// Consensus RPC. pub(crate) struct Rpc; @@ -45,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - schema::MB + protobuf_utils::MB } } @@ -61,6 +62,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - schema::kB + protobuf_utils::kB } } diff --git a/node/actors/network/src/rpc/metrics.rs b/node/actors/network/src/rpc/metrics.rs index a8770168..2595bb85 100644 --- a/node/actors/network/src/rpc/metrics.rs +++ b/node/actors/network/src/rpc/metrics.rs @@ -86,7 +86,7 @@ pub(super) struct CallLabels { } const MESSAGE_SIZE_BUCKETS: Buckets = - Buckets::exponential(schema::kB as f64..=schema::MB as f64, 2.0); + Buckets::exponential(protobuf_utils::kB as f64..=protobuf_utils::MB as f64, 2.0); #[derive(Debug, Metrics)] #[metrics(prefix = "network_rpc")] diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index b77b359f..925b4507 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -36,10 +36,10 @@ pub(crate) mod testonly; mod tests; const MUX_CONFIG: mux::Config = mux::Config { - read_buffer_size: 160 * schema::kB as u64, - read_frame_size: 16 * schema::kB as u64, + read_buffer_size: 160 * protobuf_utils::kB as u64, + read_frame_size: 16 * protobuf_utils::kB as u64, read_frame_count: 100, - write_frame_size: 16 * schema::kB as u64, + write_frame_size: 16 * protobuf_utils::kB as u64, }; /// Trait for defining an RPC. @@ -61,9 +61,9 @@ pub(crate) trait Rpc: Sync + Send + 'static { /// Name of the RPC, used in prometheus metrics. const METHOD: &'static str; /// Type of the request message. - type Req: schema::ProtoFmt + Send + Sync; + type Req: protobuf_utils::ProtoFmt + Send + Sync; /// Type of the response message. - type Resp: schema::ProtoFmt + Send + Sync; + type Resp: protobuf_utils::ProtoFmt + Send + Sync; /// Name of the variant of the request message type. /// Useful for collecting metrics with finer than /// per-rpc type granularity. diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index 5f95def5..78a497db 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -3,7 +3,8 @@ use crate::{mux, rpc::Rpc as _}; use anyhow::Context as _; use concurrency::{ctx, limiter, time}; use rand::Rng; -use schema::{proto::network::ping as proto, required, ProtoFmt}; +use schema::proto::network::ping as proto; +use protobuf_utils::{required, ProtoFmt}; /// Ping RPC. pub(crate) struct Rpc; @@ -64,7 +65,7 @@ pub(crate) struct Resp(pub(crate) [u8; 32]); impl ProtoFmt for Req { type Proto = proto::PingReq; fn max_size() -> usize { - schema::kB + protobuf_utils::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) @@ -79,7 +80,7 @@ impl ProtoFmt for Req { impl ProtoFmt for Resp { type Proto = proto::PingResp; fn max_size() -> usize { - schema::kB + protobuf_utils::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index 84fa071b..a0d630cf 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -4,7 +4,8 @@ use crate::{io, mux}; use anyhow::Context; use concurrency::{limiter, time}; use roles::validator::{BlockNumber, FinalBlock}; -use schema::{proto::network::gossip as proto, read_required, ProtoFmt}; +use schema::proto::network::gossip as proto; +use protobuf_utils::{read_required, ProtoFmt}; /// `get_sync_state` RPC. #[derive(Debug)] @@ -47,7 +48,7 @@ impl ProtoFmt for io::SyncState { fn max_size() -> usize { // TODO: estimate maximum size more precisely - 100 * schema::kB + 100 * protobuf_utils::kB } } @@ -67,7 +68,7 @@ impl ProtoFmt for SyncStateResponse { } fn max_size() -> usize { - schema::kB + protobuf_utils::kB } } @@ -109,7 +110,7 @@ impl ProtoFmt for GetBlockRequest { } fn max_size() -> usize { - schema::kB + protobuf_utils::kB } } @@ -137,7 +138,7 @@ impl ProtoFmt for io::GetBlockError { } fn max_size() -> usize { - schema::kB + protobuf_utils::kB } } @@ -178,6 +179,6 @@ impl ProtoFmt for GetBlockResponse { } fn max_size() -> usize { - schema::MB + protobuf_utils::MB } } diff --git a/node/actors/network/src/rpc/sync_validator_addrs.rs b/node/actors/network/src/rpc/sync_validator_addrs.rs index b1441355..d328d6a2 100644 --- a/node/actors/network/src/rpc/sync_validator_addrs.rs +++ b/node/actors/network/src/rpc/sync_validator_addrs.rs @@ -4,7 +4,8 @@ use crate::mux; use anyhow::Context as _; use concurrency::{limiter, time}; use roles::validator; -use schema::{proto::network::gossip as proto, ProtoFmt}; +use schema::proto::network::gossip as proto; +use protobuf_utils::{ProtoFmt}; use std::sync::Arc; /// SyncValidatorAddrs Rpc. @@ -45,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - schema::kB + protobuf_utils::kB } } @@ -69,6 +70,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - schema::MB + protobuf_utils::MB } } diff --git a/node/libs/protobuf/build/src/lib.rs b/node/libs/protobuf/build/src/lib.rs index 4cc22394..f6738f59 100644 --- a/node/libs/protobuf/build/src/lib.rs +++ b/node/libs/protobuf/build/src/lib.rs @@ -102,7 +102,7 @@ fn check_canonical_pool(d: &prost_reflect::DescriptorPool) -> anyhow::Result<()> Ok(()) } -pub fn compile(proto_include: &Path, module_name: &str, deps: &[&'static [u8]]) -> anyhow::Result<()> { +pub fn compile(proto_include: &Path, module_name: &str, deps: &[(&str,&[u8])]) -> anyhow::Result<()> { println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) .canonicalize()? @@ -130,7 +130,7 @@ pub fn compile(proto_include: &Path, module_name: &str, deps: &[&'static [u8]]) let mut pool = prost_reflect::DescriptorPool::new(); if deps.len() > 0 { let mut deps_list = vec![]; - for (i,d) in deps.iter().enumerate() { + for (i,(_,d)) in deps.iter().enumerate() { pool.decode_file_descriptor_set(*d).unwrap(); // TODO: make it transitive. let name = proto_output.join(format!("dep{i}.binpb")); fs::write(&name,d)?; @@ -184,8 +184,8 @@ pub fn compile(proto_include: &Path, module_name: &str, deps: &[&'static [u8]]) println!("name = {name:?}"); m.insert(&name, entry.path()); } - let mut file = m.generate(); - + let mut file = deps.iter().map(|d|format!("use {}::*;",d.0)).collect::>().join("\n"); + file += &m.generate(); file += &format!("pub const DESCRIPTOR_POOL: &'static [u8] = include_bytes!({descriptor_path:?});"); let file = syn::parse_str(&file).unwrap(); diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index d63b5eab..8432c2aa 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -14,6 +14,7 @@ rand.workspace = true serde.workspace = true tracing.workspace = true +protobuf_utils.workspace = true concurrency = { path = "../concurrency" } crypto = { path = "../crypto" } schema = { path = "../schema" } diff --git a/node/libs/roles/src/node/conv.rs b/node/libs/roles/src/node/conv.rs index aad1edde..f560ba09 100644 --- a/node/libs/roles/src/node/conv.rs +++ b/node/libs/roles/src/node/conv.rs @@ -1,5 +1,5 @@ use crate::node; -use ::schema::{read_required, required, ProtoFmt}; +use protobuf_utils::{read_required, required, ProtoFmt}; use anyhow::Context as _; use crypto::ByteFmt; use utils::enum_util::Variant; diff --git a/node/libs/roles/src/node/messages.rs b/node/libs/roles/src/node/messages.rs index 233e69c1..99f68819 100644 --- a/node/libs/roles/src/node/messages.rs +++ b/node/libs/roles/src/node/messages.rs @@ -17,7 +17,7 @@ pub enum Msg { impl Msg { /// Get the hash of this message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&schema::canonical(self))) + MsgHash(sha256::Sha256::new(&protobuf_utils::canonical(self))) } } diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index 0b65e63d..5d7c8fec 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -5,7 +5,7 @@ use super::{ Signers, ViewNumber, }; use crate::node::SessionId; -use ::schema::{read_required, required, ProtoFmt}; +use protobuf_utils::{read_required, required, ProtoFmt}; use anyhow::Context as _; use crypto::ByteFmt; use schema::proto::roles::validator as proto; @@ -187,7 +187,7 @@ impl ProtoFmt for LeaderCommit { } impl ProtoFmt for Signers { - type Proto = schema::proto::std::BitVector; + type Proto = protobuf_utils::proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(ProtoFmt::read(r)?)) @@ -266,8 +266,8 @@ impl ProtoFmt for Phase { fn build(&self) -> Self::Proto { use proto::phase::T; let t = match self { - Self::Prepare => T::Prepare(schema::proto::std::Void {}), - Self::Commit => T::Commit(schema::proto::std::Void {}), + Self::Prepare => T::Prepare(protobuf_utils::proto::std::Void {}), + Self::Commit => T::Commit(protobuf_utils::proto::std::Void {}), }; Self::Proto { t: Some(t) } } diff --git a/node/libs/roles/src/validator/messages/block.rs b/node/libs/roles/src/validator/messages/block.rs index 770361d3..b3a83018 100644 --- a/node/libs/roles/src/validator/messages/block.rs +++ b/node/libs/roles/src/validator/messages/block.rs @@ -97,7 +97,7 @@ pub struct BlockHeader { impl BlockHeader { /// Returns the hash of the block. pub fn hash(&self) -> BlockHeaderHash { - BlockHeaderHash(sha256::Sha256::new(&schema::canonical(self))) + BlockHeaderHash(sha256::Sha256::new(&protobuf_utils::canonical(self))) } /// Creates a genesis block. @@ -145,10 +145,10 @@ impl FinalBlock { impl ByteFmt for FinalBlock { fn decode(bytes: &[u8]) -> anyhow::Result { - ::schema::decode(bytes) + protobuf_utils::decode(bytes) } fn encode(&self) -> Vec { - ::schema::encode(self) + protobuf_utils::encode(self) } } diff --git a/node/libs/roles/src/validator/messages/msg.rs b/node/libs/roles/src/validator/messages/msg.rs index 51068c04..e8a710cc 100644 --- a/node/libs/roles/src/validator/messages/msg.rs +++ b/node/libs/roles/src/validator/messages/msg.rs @@ -20,7 +20,7 @@ pub enum Msg { impl Msg { /// Returns the hash of the message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&schema::canonical(self))) + MsgHash(sha256::Sha256::new(&protobuf_utils::canonical(self))) } } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 1c5e6db1..1704f283 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -6,6 +6,8 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf_build::compile(&proto_include,"crate::proto",&[&protobuf_utils::proto::DESCRIPTOR_POOL])?; + protobuf_build::compile(&proto_include,"crate::proto", + &[("protobuf_utils::proto",&protobuf_utils::proto::DESCRIPTOR_POOL)] + )?; Ok(()) } diff --git a/node/libs/schema/src/lib.rs b/node/libs/schema/src/lib.rs index b36a9bb7..b669be2d 100644 --- a/node/libs/schema/src/lib.rs +++ b/node/libs/schema/src/lib.rs @@ -1,14 +1,4 @@ -//! Code generated from protobuf schema files and -//! utilities for serialization. - -mod proto_fmt; -mod std_conv; -pub mod testonly; - -pub use proto_fmt::*; - -#[cfg(test)] -mod tests; +//! Code generated from protobuf schema files. #[allow(warnings)] pub mod proto { diff --git a/node/libs/schema/src/proto_fmt.rs b/node/libs/schema/src/proto_fmt.rs deleted file mode 100644 index 541b0f0b..00000000 --- a/node/libs/schema/src/proto_fmt.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! Canonical encoding for protobufs with known schema. -//! -//! Protobuf encoding spec: https://protobuf.dev/programming-guides/encoding/ -//! By default the encoding is not canonical (multiple encodings may represent the same message). -//! Here we extend this spec to support a canonical encoding. -//! -//! Canonical encoding spec: -//! * fields are encoded in ascending order of tags. -//! * varints are encoded using minimal amount of bytes. -//! * map fields are not supported -//! * groups are not supported -//! * only proto3 syntax is supported -//! * hence assigning a field a default value (proto2) is not supported -//! * all fields should have explicit presence (explicit optional or repeated) -//! * hence a default value for primitive types (proto3) is not supported. -//! * a field of primitive type (I32, I64, Varint) is encoded as at most 1 TLV, depending on -//! the number of values: -//! * 0 values => field is not encoded at all -//! * 1 value => field is encoded as a single I32/I64/Varint TLV. -//! * >1 values => field is encoded as a single TLV using packed encoding (https://protobuf.dev/programming-guides/encoding/#packed) -//! * protobuf spec allows for packed encoding only for repeated fields. We therefore don't -//! support specifying multiple values for a singular field (which is pathological case -//! anyway). -//! * protobuf spec also doesn't allow for encoding repeated enum fields (which are encoded as -//! Varint), however it mentions that it might allow for it in the future versions. -//! TODO(gprusak): decide what we should do with those: prost implementation decodes correctly -//! packed enums, I don't know about other protobuf implementations (rust and non-rust). We may either: -//! * allow packed encoding for enums (despite the incompatibility with protobuf spec) -//! * disallow packed encoding for enums specifically (although it complicates the canonical spec) -//! * drop packed encoding altogether (at the cost of larger messages) -//! -//! We need canonical encoding to be able to compute hashes and verify signatures on -//! messages. Note that with this spec it is not possible to encode canonically a message with unknown -//! fields. It won't be a problem though if we decide on one of the following: -//! * require all communication to just use canonical encoding: then as long as we carry around the -//! unknown fields untouched, they will stay canonical. -//! * hash/verify only messages that don't contain unknown fields: then we can just canonicalize -//! whatever we understand and reject messages with unknown fields. -//! * drop the idea of canonical encoding altogether and pass around the received encoded message -//! for hashing/verifying (i.e. keep the raw bytes together with the parsed message). -use anyhow::Context as _; -use prost::Message as _; -use prost_reflect::ReflectMessage; -use std::collections::BTreeMap; - -/// Kilobyte. -#[allow(non_upper_case_globals)] -pub const kB: usize = 1 << 10; -/// Megabyte. -pub const MB: usize = 1 << 20; - -/// Protobuf wire type. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(super) enum Wire { - /// VARINT. - Varint, - /// I64 - I64, - /// LEN - Len, - /// I32 - I32, -} - -/// Raw write type values, as defined in -/// https://protobuf.dev/programming-guides/encoding/#structure -/// VARINT -const VARINT: u32 = 0; -/// I64 -const I64: u32 = 1; -/// LEN -const LEN: u32 = 2; -/// I32 -const I32: u32 = 5; - -impl Wire { - /// Extracts a wire type from a tag. - pub(crate) const fn from_tag(tag: u32) -> Option { - match tag & 7 { - VARINT => Some(Self::Varint), - I64 => Some(Self::I64), - LEN => Some(Self::Len), - I32 => Some(Self::I32), - _ => None, - } - } - - /// Converts wire type to the raw wite type value. - pub(crate) const fn raw(self) -> u32 { - match self { - Self::Varint => VARINT, - Self::I64 => I64, - Self::Len => LEN, - Self::I32 => I32, - } - } -} - -impl From for Wire { - fn from(kind: prost_reflect::Kind) -> Self { - use prost_reflect::Kind; - match kind { - Kind::Int32 - | Kind::Int64 - | Kind::Uint32 - | Kind::Uint64 - | Kind::Sint32 - | Kind::Sint64 - | Kind::Bool - | Kind::Enum(_) => Self::Varint, - Kind::Fixed64 | Kind::Sfixed64 | Kind::Double => Self::I64, - Kind::Fixed32 | Kind::Sfixed32 | Kind::Float => Self::I32, - Kind::String | Kind::Bytes | Kind::Message(_) => Self::Len, - } - } -} - -/// Wrapper of the quick_protobuf reader. -// TODO(gprusak): reimplement the reader and writer instead to make it more efficient. -pub(super) struct Reader<'a>(quick_protobuf::BytesReader, &'a [u8]); - -impl<'a> Reader<'a> { - /// Constructs a reader of a slice. - fn new(bytes: &'a [u8]) -> Self { - Self(quick_protobuf::BytesReader::from_bytes(bytes), bytes) - } - - /// Reads a value of the given wire type. - fn read(&mut self, wire: Wire) -> anyhow::Result> { - let mut v = vec![]; - let mut w = quick_protobuf::Writer::new(&mut v); - match wire { - Wire::Varint => w.write_varint(self.0.read_varint64(self.1)?).unwrap(), - Wire::I64 => w.write_fixed64(self.0.read_fixed64(self.1)?).unwrap(), - Wire::Len => return Ok(self.0.read_bytes(self.1)?.into()), - Wire::I32 => w.write_fixed32(self.0.read_fixed32(self.1)?).unwrap(), - } - Ok(v) - } - - /// Reads a (possibly packed) field based on the expected field wire type and the actual - /// wire type from the tag (which has been already read). - /// The values of the field are appended to `out`. - fn read_field( - &mut self, - out: &mut Vec>, - field_wire: Wire, - got_wire: Wire, - ) -> anyhow::Result<()> { - if got_wire == field_wire { - out.push(self.read(field_wire)?); - return Ok(()); - } - if got_wire != Wire::Len { - anyhow::bail!("unexpected wire type"); - } - let mut r = Self::new(self.0.read_bytes(self.1)?); - while !r.0.is_eof() { - out.push(r.read(field_wire)?); - } - Ok(()) - } -} - -/// Parses a message to a `field num -> values` map. -pub(super) fn read_fields( - buf: &[u8], - desc: &prost_reflect::MessageDescriptor, -) -> anyhow::Result>>> { - if desc.parent_file().syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - let mut r = Reader::new(buf); - let mut fields = BTreeMap::new(); - // Collect the field values from the reader. - while !r.0.is_eof() { - let tag = r.0.next_tag(r.1)?; - let wire = Wire::from_tag(tag).context("invalid wire type")?; - let field = desc.get_field(tag >> 3).context("unknown field")?; - if field.is_map() { - anyhow::bail!("maps unsupported"); - } - if !field.is_list() && !field.supports_presence() { - anyhow::bail!( - "{}::{} : fields with implicit presence are not supported", - field.parent_message().name(), - field.name() - ); - } - r.read_field( - fields.entry(field.number()).or_default(), - field.kind().into(), - wire, - )?; - } - Ok(fields) -} - -/// Converts an encoded protobuf message to its canonical form, given the descriptor of the message -/// type. Retuns an error if: -/// * an unknown field is detected -/// * the message type doesn't support canonical encoding (implicit presence, map fields) -pub fn canonical_raw( - buf: &[u8], - desc: &prost_reflect::MessageDescriptor, -) -> anyhow::Result> { - let mut v = vec![]; - let mut w = quick_protobuf::Writer::new(&mut v); - // Append fields in ascending tags order. - for (num, mut values) in read_fields(buf, desc)? { - let fd = desc.get_field(num).unwrap(); - if values.len() > 1 && !fd.is_list() { - anyhow::bail!("non-repeated field with multiple values"); - } - if let prost_reflect::Kind::Message(desc) = &fd.kind() { - for v in &mut values { - *v = canonical_raw(v, desc)?; - } - } - let wire = Wire::from(fd.kind()); - match wire { - Wire::Varint | Wire::I64 | Wire::I32 => { - if values.len() > 1 { - w.write_tag(num << 3 | LEN).unwrap(); - w.write_bytes(&values.into_iter().flatten().collect::>()) - .unwrap(); - } else { - w.write_tag(num << 3 | wire.raw()).unwrap(); - // inefficient workaround of the fact that quick_protobuf::Writer - // doesn't support just appending a sequence of bytes. - for b in &values[0] { - w.write_u8(*b).unwrap(); - } - } - } - Wire::Len => { - for v in &values { - w.write_tag(num << 3 | LEN).unwrap(); - w.write_bytes(v).unwrap(); - } - } - } - } - Ok(v) -} - -/// Encodes a proto message of known schema (no unknown fields) into a canonical form. -pub fn canonical(x: &T) -> Vec { - let msg = x.build(); - canonical_raw(&msg.encode_to_vec(), &msg.descriptor()).unwrap() -} - -/// Encodes a proto message. -/// Currently it outputs a canonical encoding, but `decode` accepts -/// non-canonical encoding as well. -/// It would simplify invariants if we required all messages to be canonical, -/// but we will see whether it is feasible once we have an RPC framework -/// in place. -pub fn encode(x: &T) -> Vec { - canonical(x) -} - -/// Decodes a proto message. -pub fn decode(bytes: &[u8]) -> anyhow::Result { - T::read(&::Proto::decode(bytes)?) -} - -/// Encodes a generated proto message to json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn encode_json_proto(x: &T) -> String { - let mut s = serde_json::Serializer::pretty(vec![]); - let opts = prost_reflect::SerializeOptions::new(); - x.transcode_to_dynamic() - .serialize_with_options(&mut s, &opts) - .unwrap(); - String::from_utf8(s.into_inner()).unwrap() -} - -/// Encodes a proto message to json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn encode_json(x: &T) -> String { - encode_json_proto(&x.build()) -} - -/// Decodes a generated proto message from json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn decode_json_proto(json: &str) -> anyhow::Result { - let mut d = serde_json::de::Deserializer::from_str(json); - let mut p = T::default(); - let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), &mut d)?; - d.end()?; - p.merge(msg.encode_to_vec().as_slice()).unwrap(); - Ok(p) -} - -/// Decodes a proto message from json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn decode_json(json: &str) -> anyhow::Result { - T::read(&decode_json_proto(json)?) -} - -/// Trait defining a proto representation for a type. -pub trait ProtoFmt: Sized { - /// Proto message type representing Self. - type Proto: ReflectMessage + Default; - /// Converts Proto to Self. - fn read(r: &Self::Proto) -> anyhow::Result; - /// Converts Self to Proto. - fn build(&self) -> Self::Proto; - /// Maximal allowed message size (in bytes). - fn max_size() -> usize { - usize::MAX - } -} - -/// Parses a required proto field. -pub fn read_required(field: &Option) -> anyhow::Result { - ProtoFmt::read(field.as_ref().context("missing field")?) -} - -/// Parses an optional proto field. -pub fn read_optional(field: &Option) -> anyhow::Result> { - field.as_ref().map(ProtoFmt::read).transpose() -} - -/// Extracts a required field. -pub fn required(field: &Option) -> anyhow::Result<&T> { - field.as_ref().context("missing") -} diff --git a/node/libs/schema/src/std_conv.rs b/node/libs/schema/src/std_conv.rs deleted file mode 100644 index 2f2fa89b..00000000 --- a/node/libs/schema/src/std_conv.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Proto conversion for messages in std package. -use crate::{proto::std as proto, required, ProtoFmt}; -use anyhow::Context as _; -use concurrency::time; -use std::net; - -impl ProtoFmt for () { - type Proto = proto::Void; - fn read(_r: &Self::Proto) -> anyhow::Result { - Ok(()) - } - fn build(&self) -> Self::Proto { - Self::Proto {} - } -} - -impl ProtoFmt for std::net::SocketAddr { - type Proto = proto::SocketAddr; - - fn read(r: &Self::Proto) -> anyhow::Result { - let ip = required(&r.ip).context("ip")?; - let ip = match ip.len() { - 4 => net::IpAddr::from(<[u8; 4]>::try_from(&ip[..]).unwrap()), - 16 => net::IpAddr::from(<[u8; 16]>::try_from(&ip[..]).unwrap()), - _ => anyhow::bail!("invalid ip length"), - }; - let port = *required(&r.port).context("port")?; - let port = u16::try_from(port).context("port")?; - Ok(Self::new(ip, port)) - } - - fn build(&self) -> Self::Proto { - Self::Proto { - ip: Some(match self.ip() { - net::IpAddr::V4(ip) => ip.octets().to_vec(), - net::IpAddr::V6(ip) => ip.octets().to_vec(), - }), - port: Some(self.port() as u32), - } - } -} - -impl ProtoFmt for time::Utc { - type Proto = proto::Timestamp; - - fn read(r: &Self::Proto) -> anyhow::Result { - let seconds = *required(&r.seconds).context("seconds")?; - let nanos = *required(&r.nanos).context("nanos")?; - Ok(time::UNIX_EPOCH + time::Duration::new(seconds, nanos)) - } - - fn build(&self) -> Self::Proto { - let d = (*self - time::UNIX_EPOCH).build(); - Self::Proto { - seconds: d.seconds, - nanos: d.nanos, - } - } -} - -impl ProtoFmt for time::Duration { - type Proto = proto::Duration; - - fn read(r: &Self::Proto) -> anyhow::Result { - let seconds = *required(&r.seconds).context("seconds")?; - let nanos = *required(&r.nanos).context("nanos")?; - Ok(Self::new(seconds, nanos)) - } - - fn build(&self) -> Self::Proto { - let mut seconds = self.whole_seconds(); - let mut nanos = self.subsec_nanoseconds(); - if nanos < 0 { - seconds -= 1; - nanos += 1_000_000_000; - } - Self::Proto { - seconds: Some(seconds), - nanos: Some(nanos), - } - } -} - -impl ProtoFmt for bit_vec::BitVec { - type Proto = proto::BitVector; - - fn read(r: &Self::Proto) -> anyhow::Result { - let size = *required(&r.size).context("size")? as usize; - let mut this = Self::from_bytes(required(&r.bytes).context("bytes_")?); - if this.len() < size { - anyhow::bail!("'vector' has less than 'size' bits"); - } - this.truncate(size); - Ok(this) - } - - fn build(&self) -> Self::Proto { - Self::Proto { - size: Some(self.len() as u64), - bytes: Some(self.to_bytes()), - } - } -} diff --git a/node/libs/schema/src/testonly.rs b/node/libs/schema/src/testonly.rs deleted file mode 100644 index 9cf15ae0..00000000 --- a/node/libs/schema/src/testonly.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! Testonly utilities. -use super::{canonical, canonical_raw, decode, encode, read_fields, ProtoFmt, Wire}; -use prost::Message as _; -use prost_reflect::ReflectMessage as _; -use rand::{ - distributions::{Distribution, Standard}, - Rng, -}; - -/// Test encoding and canonical encoding properties. -#[track_caller] -pub fn test_encode(rng: &mut R, x: &T) { - let x_encode = encode(x); - let x_canonical = canonical(x); - let x_shuffled = encode_shuffled(rng, x); - - let desc = &::Proto::default().descriptor(); - for e in [&x_encode, &x_canonical, &x_shuffled] { - // e is a valid encoding. - assert_eq!(x, &decode::(e).unwrap()); - // canonical encoding is consistent. - assert_eq!(x_canonical, canonical_raw(e, desc).unwrap()); - } -} - -/// Syntax sugar for `test_encode`, -/// because `test_encode(rng,&rng::gen())` doesn't compile. -#[track_caller] -pub fn test_encode_random(rng: &mut R) -where - Standard: Distribution, -{ - let msg = rng.gen::(); - test_encode(rng, &msg); -} - -/// shuffles recursively the order of fields in a protobuf encoding. -fn encode_shuffled_raw( - rng: &mut R, - buf: &[u8], - desc: &prost_reflect::MessageDescriptor, -) -> anyhow::Result> { - // Vector of field values, values of each field are in reverse order. - // This way we can pop the next value of the field in O(1). - let mut fields: Vec<_> = read_fields(buf, desc)?.into_iter().collect(); - for f in &mut fields { - f.1.reverse(); - } - - // Append fields in random order. - let mut v = vec![]; - let mut w = quick_protobuf::Writer::new(&mut v); - while !fields.is_empty() { - // Select a random field. - let i = rng.gen_range(0..fields.len()); - // Select the next value of this field. - // Note that the values are stored in reverse order. - let Some(mut v) = fields[i].1.pop() else { - let last = fields.len() - 1; - fields.swap(i, last); - fields.pop(); - continue; - }; - let num = fields[i].0; - let fd = desc.get_field(num).unwrap(); - if let prost_reflect::Kind::Message(desc) = &fd.kind() { - v = encode_shuffled_raw(rng, &v, desc)?; - } - let wire = Wire::from(fd.kind()); - w.write_tag(num << 3 | wire.raw()).unwrap(); - match wire { - Wire::Varint | Wire::I64 | Wire::I32 => { - // inefficient workaround of the fact that quick_protobuf::Writer - // doesn't support just appending a sequence of bytes. - for b in &v { - w.write_u8(*b).unwrap(); - } - } - Wire::Len => w.write_bytes(&v).unwrap(), - } - } - Ok(v) -} - -/// Encodes a proto message of known schema in a random encoding order of fields. -pub(crate) fn encode_shuffled(rng: &mut R, x: &T) -> Vec { - let msg = x.build(); - encode_shuffled_raw(rng, &msg.encode_to_vec(), &msg.descriptor()).unwrap() -} diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index 52edcaa0..1b93660a 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -14,6 +14,7 @@ rocksdb.workspace = true thiserror.workspace = true tracing.workspace = true +protobuf_utils.workspace = true concurrency = { path = "../concurrency" } roles = { path = "../roles" } schema = { path = "../schema" } diff --git a/node/libs/storage/src/rocksdb.rs b/node/libs/storage/src/rocksdb.rs index d2f1d43d..bb34c0c4 100644 --- a/node/libs/storage/src/rocksdb.rs +++ b/node/libs/storage/src/rocksdb.rs @@ -100,7 +100,7 @@ impl RocksdbStorage { .next() .context("Head block not found")? .context("RocksDB error reading head block")?; - schema::decode(&head_block).context("Failed decoding head block bytes") + protobuf_utils::decode(&head_block).context("Failed decoding head block bytes") } /// Returns a block with the least number stored in this database. @@ -114,7 +114,7 @@ impl RocksdbStorage { .next() .context("First stored block not found")? .context("RocksDB error reading first stored block")?; - schema::decode(&first_block).context("Failed decoding first stored block bytes") + protobuf_utils::decode(&first_block).context("Failed decoding first stored block bytes") } fn last_contiguous_block_number_blocking(&self) -> anyhow::Result { @@ -174,7 +174,7 @@ impl RocksdbStorage { else { return Ok(None); }; - let block = schema::decode(&raw_block) + let block = protobuf_utils::decode(&raw_block) .with_context(|| format!("Failed decoding block #{number}"))?; Ok(Some(block)) } @@ -213,7 +213,7 @@ impl RocksdbStorage { let mut write_batch = rocksdb::WriteBatch::default(); write_batch.put( DatabaseKey::Block(block_number).encode_key(), - schema::encode(finalized_block), + protobuf_utils::encode(finalized_block), ); // Commit the transaction. db.write(write_batch) @@ -232,7 +232,7 @@ impl RocksdbStorage { else { return Ok(None); }; - schema::decode(&raw_state) + protobuf_utils::decode(&raw_state) .map(Some) .context("Failed to decode replica state!") } @@ -241,7 +241,7 @@ impl RocksdbStorage { self.write() .put( DatabaseKey::ReplicaState.encode_key(), - schema::encode(replica_state), + protobuf_utils::encode(replica_state), ) .context("Failed putting ReplicaState to RocksDB") } diff --git a/node/libs/storage/src/types.rs b/node/libs/storage/src/types.rs index fd39b362..01190af1 100644 --- a/node/libs/storage/src/types.rs +++ b/node/libs/storage/src/types.rs @@ -1,10 +1,10 @@ //! Defines the schema of the database. - use anyhow::Context as _; use concurrency::ctx; use rocksdb::{Direction, IteratorMode}; use roles::validator::{self, BlockNumber}; -use schema::{proto::storage as proto, read_required, required, ProtoFmt}; +use schema::proto::storage as proto; +use protobuf_utils::{read_required, required, ProtoFmt}; use std::{iter, ops}; use thiserror::Error; diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 9628b66f..25cb9326 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -13,6 +13,7 @@ rand.workspace = true serde_json.workspace = true clap.workspace = true +protobuf_utils.workspace = true crypto = { path = "../libs/crypto" } roles = { path = "../libs/roles" } schema = { path = "../libs/schema" } diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index cc461fbf..4b58f9d6 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -108,7 +108,7 @@ fn main() -> anyhow::Result<()> { let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; - fs::write(root.join("config.json"), schema::encode_json(&node_cfg)) + fs::write(root.join("config.json"), protobuf_utils::encode_json(&node_cfg)) .context("fs::write()")?; fs::write( root.join("validator_key"), From ce390b23cbbaaa190198ca5d9f47c95c50969694 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 30 Oct 2023 15:18:30 +0100 Subject: [PATCH 05/40] works --- node/Cargo.lock | 2 + node/Cargo.toml | 1 + node/libs/protobuf/build/Cargo.toml | 1 + node/libs/protobuf/build/src/lib.rs | 126 ++++++++++++++++++---------- node/libs/protobuf/utils/Cargo.toml | 2 + node/libs/protobuf/utils/build.rs | 2 +- node/libs/schema/Cargo.toml | 1 + node/libs/schema/build.rs | 4 +- 8 files changed, 90 insertions(+), 49 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index 980e2969..6f5c03f5 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -1485,6 +1485,7 @@ dependencies = [ "prost-build", "prost-reflect", "prost-reflect-build", + "prost-types", "protoc-bin-vendored", "quick-protobuf", "rand", @@ -1501,6 +1502,7 @@ dependencies = [ "anyhow", "bit-vec", "concurrency", + "once_cell", "prost", "prost-reflect", "protobuf_build", diff --git a/node/Cargo.toml b/node/Cargo.toml index fc280d3f..b6522409 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -43,6 +43,7 @@ prost = "0.11.0" prost-build = "0.11.0" prost-reflect = { version = "0.11.0", features = ["derive","serde"] } prost-reflect-build = "0.11.0" +prost-types = "0.11.0" protoc-bin-vendored = "3.0.0" prettyplease = "0.2.6" pretty_assertions = "1.4.0" diff --git a/node/libs/protobuf/build/Cargo.toml b/node/libs/protobuf/build/Cargo.toml index 656be38f..0e33b33d 100644 --- a/node/libs/protobuf/build/Cargo.toml +++ b/node/libs/protobuf/build/Cargo.toml @@ -13,6 +13,7 @@ serde.workspace = true once_cell.workspace = true quick-protobuf.workspace = true prost.workspace = true +prost-types.workspace = true prost-reflect.workspace = true rand.workspace = true serde_json.workspace = true diff --git a/node/libs/protobuf/build/src/lib.rs b/node/libs/protobuf/build/src/lib.rs index f6738f59..a6d31225 100644 --- a/node/libs/protobuf/build/src/lib.rs +++ b/node/libs/protobuf/build/src/lib.rs @@ -2,6 +2,14 @@ use anyhow::Context as _; use std::{collections::BTreeMap, env, fs, path::{PathBuf,Path}}; use std::process::Command; +use prost::Message as _; +use std::collections::HashSet; +use once_cell::sync::Lazy; + +pub struct Descriptor { + pub crate_name: String, + pub module_path: String, +} /// Traversed all the files in a directory recursively. fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> { @@ -61,48 +69,54 @@ impl Module { } } -/// Checks if field `f` supports canonical encoding. -fn check_canonical_field(f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { - if f.is_map() { - anyhow::bail!("maps unsupported"); - } - if !f.is_list() && !f.supports_presence() { - anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); - } - Ok(()) -} +#[derive(Default)] +struct CanonicalCheckState(HashSet); -/// Checks if messages of type `m` support canonical encoding. -fn check_canonical_message(m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { - for m in m.child_messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; - } - for f in m.fields() { - check_canonical_field(&f).with_context(|| f.name().to_string())?; +impl CanonicalCheckState { + /// Checks if messages of type `m` support canonical encoding. + fn check_message(&mut self, m: &prost_reflect::MessageDescriptor, check_nested: bool) -> anyhow::Result<()> { + if self.0.contains(m.full_name()) { + return Ok(()); + } + self.0.insert(m.full_name().to_string()); + for f in m.fields() { + self.check_field(&f).with_context(|| f.name().to_string())?; + } + if check_nested { + for m in m.child_messages() { + self.check_message(&m,check_nested).with_context(||m.name().to_string())?; + } + } + Ok(()) } - Ok(()) -} -/// Checks if message types in file `f` support canonical encoding. -fn check_canonical_file(f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { - if f.syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - for m in f.messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; + /// Checks if field `f` supports canonical encoding. + fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + if f.is_map() { + anyhow::bail!("maps unsupported"); + } + if !f.is_list() && !f.supports_presence() { + anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); + } + if let prost_reflect::Kind::Message(msg) = f.kind() { + self.check_message(&msg,false).with_context(||msg.name().to_string())?; + } + Ok(()) } - Ok(()) -} -/// Checks if message types in descriptor pool `d` support canonical encoding. -fn check_canonical_pool(d: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { - for f in d.files() { - check_canonical_file(&f).with_context(|| f.name().to_string())?; + /// Checks if message types in file `f` support canonical encoding. + fn check_file(&mut self, f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { + if f.syntax() != prost_reflect::Syntax::Proto3 { + anyhow::bail!("only proto3 syntax is supported"); + } + for m in f.messages() { + self.check_message(&m,true).with_context(|| m.name().to_string())?; + } + Ok(()) } - Ok(()) } -pub fn compile(proto_include: &Path, module_name: &str, deps: &[(&str,&[u8])]) -> anyhow::Result<()> { +pub fn compile(proto_include: &Path, module_path: &str, deps: &[&Lazy]) -> anyhow::Result<()> { println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) .canonicalize()? @@ -127,17 +141,20 @@ pub fn compile(proto_include: &Path, module_name: &str, deps: &[(&str,&[u8])]) - cmd.arg("-o").arg(&descriptor_path); cmd.arg("-I").arg(&proto_include); - let mut pool = prost_reflect::DescriptorPool::new(); - if deps.len() > 0 { + /*if deps.len() > 0 { let mut deps_list = vec![]; for (i,(_,d)) in deps.iter().enumerate() { - pool.decode_file_descriptor_set(*d).unwrap(); // TODO: make it transitive. let name = proto_output.join(format!("dep{i}.binpb")); fs::write(&name,d)?; deps_list.push(name.to_str().unwrap().to_string()); } cmd.arg("--descriptor_set_in").arg(deps_list.join(":")); - } + }*/ + + let deps : Vec<_> = deps.iter().map(|d|&***d).collect(); + let deps_path = proto_output.join("deps.binpb"); + fs::write(&deps_path,prost_reflect::DescriptorPool::global().encode_to_vec())?; + cmd.arg("--descriptor_set_in").arg(&deps_path); for input in &proto_inputs { cmd.arg(&input); @@ -151,12 +168,15 @@ pub fn compile(proto_include: &Path, module_name: &str, deps: &[(&str,&[u8])]) - // Generate protobuf code from schema (with reflection). let descriptor = fs::read(&descriptor_path)?; - pool.decode_file_descriptor_set(&descriptor[..])?; - let file_descriptor_set_bytes = format!("{module_name}::DESCRIPTOR_POOL"); - let pool_attribute = format!(r#"#[prost_reflect(file_descriptor_set_bytes = "{}")]"#,file_descriptor_set_bytes); + let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); + for p in &descriptor.file { + prost_reflect::DescriptorPool::add_global_file_descriptor_proto::<&[u8]>(p.clone()).unwrap(); + } + let pool_attribute = format!(r#"#[prost_reflect(descriptor_pool = "crate::{module_path}::DESCRIPTOR_POOL")]"#); let empty : &[&Path] = &[]; let mut config = prost_build::Config::new(); + let pool = prost_reflect::DescriptorPool::global(); for message in pool.all_messages() { let full_name = message.full_name(); config @@ -170,7 +190,10 @@ pub fn compile(proto_include: &Path, module_name: &str, deps: &[(&str,&[u8])]) - config.compile_protos(empty,empty).unwrap(); // Check that messages are compatible with `proto_fmt::canonical`. - check_canonical_pool(&pool)?; + let mut check_state = CanonicalCheckState::default(); + for f in &descriptor.file { + check_state.check_file(&pool.get_file_by_name(f.name.as_ref().unwrap()).unwrap())?; + } // Generate mod file collecting all proto-generated code. let mut m = Module::default(); @@ -184,10 +207,23 @@ pub fn compile(proto_include: &Path, module_name: &str, deps: &[(&str,&[u8])]) - println!("name = {name:?}"); m.insert(&name, entry.path()); } - let mut file = deps.iter().map(|d|format!("use {}::*;",d.0)).collect::>().join("\n"); + let mut file = deps.iter().map(|d|format!("use {}::{}::*;",d.crate_name,d.module_path)).collect::>().join("\n"); file += &m.generate(); - file += &format!("pub const DESCRIPTOR_POOL: &'static [u8] = include_bytes!({descriptor_path:?});"); - + let rec = deps.iter().map(|d|format!("&*{}::{}::DESCRIPTOR;",d.crate_name,d.module_path)).collect::>().join(" "); + file += &format!("pub const DESCRIPTOR: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| {{ \ + {rec} \ + prost_reflect::DescriptorPool::decode_global_file_descriptor_set(&include_bytes!({descriptor_path:?})[..]).unwrap(); \ + protobuf_build::Descriptor {{\ + crate_name: {:?}.to_string(),\ + module_path: {module_path:?}.to_string(),\ + }}\ + }});",std::env::var("CARGO_PKG_NAME").unwrap()); + file += "pub const DESCRIPTOR_POOL: once_cell::sync::Lazy = \ + once_cell::sync::Lazy::new(|| { \ + &*DESCRIPTOR; \ + prost_reflect::DescriptorPool::global() \ + }); \ + "; let file = syn::parse_str(&file).unwrap(); fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; Ok(()) diff --git a/node/libs/protobuf/utils/Cargo.toml b/node/libs/protobuf/utils/Cargo.toml index 1bc3e13b..e929870d 100644 --- a/node/libs/protobuf/utils/Cargo.toml +++ b/node/libs/protobuf/utils/Cargo.toml @@ -19,7 +19,9 @@ prost-reflect.workspace = true rand.workspace = true serde_json.workspace = true tokio.workspace = true +once_cell.workspace = true +protobuf_build = { path = "../build" } concurrency = { path = "../../concurrency" } [build-dependencies] diff --git a/node/libs/protobuf/utils/build.rs b/node/libs/protobuf/utils/build.rs index bf9a738e..01ab9e3d 100644 --- a/node/libs/protobuf/utils/build.rs +++ b/node/libs/protobuf/utils/build.rs @@ -6,6 +6,6 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf_build::compile(&proto_include,"crate::proto",&[])?; + protobuf_build::compile(&proto_include,"proto",&[])?; Ok(()) } diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index 4211f989..e68f4c39 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -18,6 +18,7 @@ rand.workspace = true serde_json.workspace = true tokio.workspace = true +protobuf_build = { path = "../protobuf/build" } protobuf_utils = { path = "../protobuf/utils" } concurrency = { path = "../concurrency" } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 1704f283..dd92dbef 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -6,8 +6,6 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf_build::compile(&proto_include,"crate::proto", - &[("protobuf_utils::proto",&protobuf_utils::proto::DESCRIPTOR_POOL)] - )?; + protobuf_build::compile(&proto_include,"proto", &[&protobuf_utils::proto::DESCRIPTOR])?; Ok(()) } From af960b85305d0aad4822e7e26e770b2f6be7f581 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Mon, 30 Oct 2023 15:46:52 +0100 Subject: [PATCH 06/40] reorganized crates --- node/Cargo.lock | 56 +-- node/Cargo.toml | 8 +- node/actors/consensus/Cargo.toml | 2 +- node/actors/consensus/src/inner.rs | 2 +- node/actors/consensus/src/metrics.rs | 2 +- node/actors/executor/Cargo.toml | 2 +- .../src/lib/configurator/config_paths.rs | 2 +- .../src/lib/configurator/node_config.rs | 2 +- node/actors/network/Cargo.toml | 2 +- .../network/src/consensus/handshake/mod.rs | 2 +- node/actors/network/src/frame.rs | 16 +- .../network/src/gossip/handshake/mod.rs | 2 +- node/actors/network/src/mux/handshake.rs | 6 +- node/actors/network/src/preface.rs | 6 +- node/actors/network/src/rpc/consensus.rs | 6 +- node/actors/network/src/rpc/metrics.rs | 2 +- node/actors/network/src/rpc/mod.rs | 10 +- node/actors/network/src/rpc/ping.rs | 6 +- node/actors/network/src/rpc/sync_blocks.rs | 12 +- .../network/src/rpc/sync_validator_addrs.rs | 6 +- node/libs/protobuf/build/Cargo.toml | 28 -- node/libs/protobuf/build/src/lib.rs | 230 ------------ node/libs/protobuf/utils/Cargo.toml | 29 -- node/libs/protobuf/utils/build.rs | 11 - .../utils/proto/conformance/conformance.proto | 183 ---------- .../conformance/test_messages_proto3.proto | 225 ------------ node/libs/protobuf/utils/proto/std.proto | 50 --- node/libs/protobuf/utils/proto/testonly.proto | 27 -- .../utils/src/bin/conformance_test.rs | 88 ----- .../src/bin/conformance_test_failure_list.txt | 11 - node/libs/protobuf/utils/src/lib.rs | 16 - node/libs/protobuf/utils/src/proto_fmt.rs | 329 ------------------ node/libs/protobuf/utils/src/std_conv.rs | 103 ------ node/libs/protobuf/utils/src/testonly.rs | 89 ----- node/libs/protobuf/utils/src/tests.rs | 105 ------ node/libs/roles/Cargo.toml | 2 +- node/libs/roles/src/node/conv.rs | 2 +- node/libs/roles/src/node/messages.rs | 2 +- node/libs/roles/src/validator/conv.rs | 8 +- .../roles/src/validator/messages/block.rs | 6 +- node/libs/roles/src/validator/messages/msg.rs | 2 +- node/libs/schema/Cargo.toml | 7 +- node/libs/schema/build.rs | 2 +- node/libs/storage/Cargo.toml | 2 +- node/libs/storage/src/rocksdb.rs | 12 +- node/libs/storage/src/types.rs | 2 +- node/tools/Cargo.toml | 2 +- node/tools/src/bin/localnet_config.rs | 2 +- 48 files changed, 89 insertions(+), 1638 deletions(-) delete mode 100644 node/libs/protobuf/build/Cargo.toml delete mode 100644 node/libs/protobuf/build/src/lib.rs delete mode 100644 node/libs/protobuf/utils/Cargo.toml delete mode 100644 node/libs/protobuf/utils/build.rs delete mode 100644 node/libs/protobuf/utils/proto/conformance/conformance.proto delete mode 100644 node/libs/protobuf/utils/proto/conformance/test_messages_proto3.proto delete mode 100644 node/libs/protobuf/utils/proto/std.proto delete mode 100644 node/libs/protobuf/utils/proto/testonly.proto delete mode 100644 node/libs/protobuf/utils/src/bin/conformance_test.rs delete mode 100644 node/libs/protobuf/utils/src/bin/conformance_test_failure_list.txt delete mode 100644 node/libs/protobuf/utils/src/lib.rs delete mode 100644 node/libs/protobuf/utils/src/proto_fmt.rs delete mode 100644 node/libs/protobuf/utils/src/std_conv.rs delete mode 100644 node/libs/protobuf/utils/src/testonly.rs delete mode 100644 node/libs/protobuf/utils/src/tests.rs diff --git a/node/Cargo.lock b/node/Cargo.lock index 6f5c03f5..f33bb7c0 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -412,7 +412,7 @@ dependencies = [ "crypto", "network", "once_cell", - "protobuf_utils", + "protobuf", "rand", "roles", "schema", @@ -611,7 +611,7 @@ dependencies = [ "hex", "network", "once_cell", - "protobuf_utils", + "protobuf", "rand", "roles", "schema", @@ -1110,7 +1110,7 @@ dependencies = [ "once_cell", "pin-project", "pretty_assertions", - "protobuf_utils", + "protobuf", "rand", "roles", "schema", @@ -1473,43 +1473,42 @@ dependencies = [ ] [[package]] -name = "protobuf_build" +name = "protobuf" version = "0.1.0" dependencies = [ "anyhow", "bit-vec", "concurrency", "once_cell", - "prettyplease 0.2.12", "prost", - "prost-build", "prost-reflect", - "prost-reflect-build", - "prost-types", - "protoc-bin-vendored", + "protobuf_build", "quick-protobuf", "rand", "serde", "serde_json", - "syn 2.0.29", "tokio", ] [[package]] -name = "protobuf_utils" +name = "protobuf_build" version = "0.1.0" dependencies = [ "anyhow", "bit-vec", - "concurrency", "once_cell", + "prettyplease 0.2.12", "prost", + "prost-build", "prost-reflect", - "protobuf_build", + "prost-reflect-build", + "prost-types", + "protoc-bin-vendored", "quick-protobuf", "rand", "serde", "serde_json", + "syn 2.0.29", "tokio", ] @@ -1692,7 +1691,7 @@ dependencies = [ "concurrency", "crypto", "hex", - "protobuf_utils", + "protobuf", "rand", "schema", "serde", @@ -1750,8 +1749,7 @@ dependencies = [ "once_cell", "prost", "prost-reflect", - "protobuf_build", - "protobuf_utils", + "protobuf", "quick-protobuf", "rand", "serde", @@ -1759,28 +1757,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "schema2" -version = "0.1.0" -dependencies = [ - "anyhow", - "bit-vec", - "concurrency", - "once_cell", - "prettyplease 0.2.12", - "prost", - "prost-build", - "prost-reflect", - "prost-reflect-build", - "protoc-bin-vendored", - "quick-protobuf", - "rand", - "serde", - "serde_json", - "syn 2.0.29", - "tokio", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -1960,7 +1936,7 @@ dependencies = [ "assert_matches", "async-trait", "concurrency", - "protobuf_utils", + "protobuf", "rand", "rocksdb", "roles", @@ -2168,7 +2144,7 @@ dependencies = [ "crypto", "executor", "hex", - "protobuf_utils", + "protobuf", "rand", "roles", "schema", diff --git a/node/Cargo.toml b/node/Cargo.toml index b6522409..0fd32b68 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,8 +1,8 @@ [workspace] members = [ "libs/concurrency", - "libs/protobuf/build", - "libs/protobuf/utils", + "libs/protobuf_build", + "libs/protobuf", "actors/consensus", "libs/crypto", "actors/executor", @@ -10,7 +10,6 @@ members = [ "actors/sync_blocks", "libs/roles", "libs/schema", - "libs/schema2", "libs/storage", "tools", "libs/utils", @@ -24,7 +23,8 @@ homepage = "https://matter-labs.io/" license = "MIT" [workspace.dependencies] -protobuf_utils = { path = "libs/protobuf/utils" } +protobuf_build = { path = "libs/protobuf_build" } +protobuf = { path = "libs/protobuf" } anyhow = "1" assert_matches = "1.5.0" diff --git a/node/actors/consensus/Cargo.toml b/node/actors/consensus/Cargo.toml index 82f7667c..5b7fe024 100644 --- a/node/actors/consensus/Cargo.toml +++ b/node/actors/consensus/Cargo.toml @@ -15,7 +15,7 @@ thiserror.workspace = true tracing.workspace = true vise.workspace = true -protobuf_utils.workspace = true +protobuf.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/consensus/src/inner.rs b/node/actors/consensus/src/inner.rs index f3de4e33..45080e06 100644 --- a/node/actors/consensus/src/inner.rs +++ b/node/actors/consensus/src/inner.rs @@ -23,7 +23,7 @@ pub(crate) struct ConsensusInner { impl ConsensusInner { /// The maximum size of the payload of a block, in bytes. We will /// reject blocks with payloads larger than this. - pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * protobuf_utils::kB; + pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * protobuf::kB; /// Computes the validator for the given view. #[instrument(level = "trace", ret)] diff --git a/node/actors/consensus/src/metrics.rs b/node/actors/consensus/src/metrics.rs index a72ac80a..5742134b 100644 --- a/node/actors/consensus/src/metrics.rs +++ b/node/actors/consensus/src/metrics.rs @@ -4,7 +4,7 @@ use std::time::Duration; use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; const PAYLOAD_SIZE_BUCKETS: Buckets = - Buckets::exponential((4 * protobuf_utils::kB) as f64..=(4 * protobuf_utils::MB) as f64, 4.0); + Buckets::exponential((4 * protobuf::kB) as f64..=(4 * protobuf::MB) as f64, 4.0); /// Label for a consensus message. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index 464b6316..0aab5779 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -20,7 +20,7 @@ tracing-subscriber.workspace = true vise.workspace = true vise-exporter.workspace = true -protobuf_utils.workspace = true +protobuf.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/executor/src/lib/configurator/config_paths.rs b/node/actors/executor/src/lib/configurator/config_paths.rs index 60a0f0ca..1e98245e 100644 --- a/node/actors/executor/src/lib/configurator/config_paths.rs +++ b/node/actors/executor/src/lib/configurator/config_paths.rs @@ -46,7 +46,7 @@ impl ConfigPaths { #[instrument(level = "trace", ret)] pub(crate) fn read(self) -> anyhow::Result { let cfg = Configs { - config: protobuf_utils::decode_json( + config: protobuf::decode_json( &std::fs::read_to_string(&self.config).context(self.config)?, )?, validator_key: Text::new( diff --git a/node/actors/executor/src/lib/configurator/node_config.rs b/node/actors/executor/src/lib/configurator/node_config.rs index 7a160a2a..d7472a41 100644 --- a/node/actors/executor/src/lib/configurator/node_config.rs +++ b/node/actors/executor/src/lib/configurator/node_config.rs @@ -3,7 +3,7 @@ use anyhow::Context as _; use crypto::{read_optional_text, read_required_text, Text, TextFmt}; use roles::{node, validator}; use schema::proto::executor::config as proto; -use protobuf_utils::{read_required, required, ProtoFmt}; +use protobuf::{read_required, required, ProtoFmt}; use std::{ collections::{HashMap, HashSet}, net, diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index b9dc27c7..cfc45758 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -18,7 +18,7 @@ thiserror.workspace = true tracing.workspace = true vise.workspace = true -protobuf_utils.workspace = true +protobuf.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/network/src/consensus/handshake/mod.rs b/node/actors/network/src/consensus/handshake/mod.rs index df7d7f6b..7a07e5a6 100644 --- a/node/actors/network/src/consensus/handshake/mod.rs +++ b/node/actors/network/src/consensus/handshake/mod.rs @@ -4,7 +4,7 @@ use concurrency::{ctx, time}; use crypto::{bls12_381, ByteFmt}; use roles::{node, validator}; use schema::proto::network::consensus as proto; -use protobuf_utils::{read_required, ProtoFmt}; +use protobuf::{read_required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/frame.rs b/node/actors/network/src/frame.rs index bbe3583b..2a7d4464 100644 --- a/node/actors/network/src/frame.rs +++ b/node/actors/network/src/frame.rs @@ -7,7 +7,7 @@ use concurrency::{ctx, io}; /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. /// Returns the decoded proto and the size of the received message in bytes. -pub(crate) async fn mux_recv_proto( +pub(crate) async fn mux_recv_proto( ctx: &ctx::Ctx, stream: &mut mux::ReadStream, ) -> anyhow::Result<(T, usize)> { @@ -25,19 +25,19 @@ pub(crate) async fn mux_recv_proto( if msg.len() < msg_size { anyhow::bail!("end of stream"); } - let msg = protobuf_utils::decode(msg.as_slice())?; + let msg = protobuf::decode(msg.as_slice())?; Ok((msg, msg_size)) } /// Sends a proto serialized to a raw frame of bytes to the stream. /// It doesn't flush the stream. /// Returns the size of the sent proto in bytes. -pub(crate) async fn mux_send_proto( +pub(crate) async fn mux_send_proto( ctx: &ctx::Ctx, stream: &mut mux::WriteStream, msg: &T, ) -> anyhow::Result { - let msg = protobuf_utils::encode(msg); + let msg = protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); stream .write_all(ctx, &u32::to_le_bytes(msg.len() as u32)) @@ -49,7 +49,7 @@ pub(crate) async fn mux_send_proto( /// Reads a raw frame of bytes from the stream and interprets it as proto. /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. -pub(crate) async fn recv_proto( +pub(crate) async fn recv_proto( ctx: &ctx::Ctx, stream: &mut S, ) -> anyhow::Result { @@ -61,16 +61,16 @@ pub(crate) async fn recv_proto( +pub(crate) async fn send_proto( ctx: &ctx::Ctx, stream: &mut S, msg: &T, ) -> anyhow::Result<()> { - let msg = protobuf_utils::encode(msg); + let msg = protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); io::write_all(ctx, stream, &u32::to_le_bytes(msg.len() as u32)).await??; io::write_all(ctx, stream, &msg).await??; diff --git a/node/actors/network/src/gossip/handshake/mod.rs b/node/actors/network/src/gossip/handshake/mod.rs index b6e1a224..e18a6e4d 100644 --- a/node/actors/network/src/gossip/handshake/mod.rs +++ b/node/actors/network/src/gossip/handshake/mod.rs @@ -5,7 +5,7 @@ use concurrency::{ctx, time}; use crypto::ByteFmt; use roles::node; use schema::proto::network::gossip as proto; -use protobuf_utils::{read_required, required, ProtoFmt}; +use protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/mux/handshake.rs b/node/actors/network/src/mux/handshake.rs index 80d34cf9..7f971bca 100644 --- a/node/actors/network/src/mux/handshake.rs +++ b/node/actors/network/src/mux/handshake.rs @@ -1,7 +1,7 @@ use super::CapabilityId; use anyhow::Context as _; use schema::proto::network::mux as proto; -use protobuf_utils::required; +use protobuf::required; use std::collections::HashMap; pub(super) struct Handshake { @@ -37,11 +37,11 @@ fn build_capabilities( .collect() } -impl protobuf_utils::ProtoFmt for Handshake { +impl protobuf::ProtoFmt for Handshake { type Proto = proto::Handshake; fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { diff --git a/node/actors/network/src/preface.rs b/node/actors/network/src/preface.rs index 3ad916d6..88576ff6 100644 --- a/node/actors/network/src/preface.rs +++ b/node/actors/network/src/preface.rs @@ -10,7 +10,7 @@ use crate::{frame, metrics, noise}; use concurrency::{ctx, time}; use schema::proto::network::preface as proto; -use protobuf_utils::{required, ProtoFmt}; +use protobuf::{required, ProtoFmt}; /// Timeout on executing the preface protocol. const TIMEOUT: time::Duration = time::Duration::seconds(5); @@ -34,7 +34,7 @@ pub(crate) enum Endpoint { impl ProtoFmt for Encryption { type Proto = proto::Encryption; fn max_size() -> usize { - 10 * protobuf_utils::kB + 10 * protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::encryption::T; @@ -54,7 +54,7 @@ impl ProtoFmt for Encryption { impl ProtoFmt for Endpoint { type Proto = proto::Endpoint; fn max_size() -> usize { - 10 * protobuf_utils::kB + 10 * protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::endpoint::T; diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 06b82c21..14857cc8 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -3,7 +3,7 @@ use crate::mux; use concurrency::{limiter, time}; use roles::validator; use schema::proto::network::consensus as proto; -use protobuf_utils::{read_required, ProtoFmt}; +use protobuf::{read_required, ProtoFmt}; /// Consensus RPC. pub(crate) struct Rpc; @@ -46,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - protobuf_utils::MB + protobuf::MB } } @@ -62,6 +62,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } } diff --git a/node/actors/network/src/rpc/metrics.rs b/node/actors/network/src/rpc/metrics.rs index 2595bb85..1a4c2d06 100644 --- a/node/actors/network/src/rpc/metrics.rs +++ b/node/actors/network/src/rpc/metrics.rs @@ -86,7 +86,7 @@ pub(super) struct CallLabels { } const MESSAGE_SIZE_BUCKETS: Buckets = - Buckets::exponential(protobuf_utils::kB as f64..=protobuf_utils::MB as f64, 2.0); + Buckets::exponential(protobuf::kB as f64..=protobuf::MB as f64, 2.0); #[derive(Debug, Metrics)] #[metrics(prefix = "network_rpc")] diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index 925b4507..5a71ef2c 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -36,10 +36,10 @@ pub(crate) mod testonly; mod tests; const MUX_CONFIG: mux::Config = mux::Config { - read_buffer_size: 160 * protobuf_utils::kB as u64, - read_frame_size: 16 * protobuf_utils::kB as u64, + read_buffer_size: 160 * protobuf::kB as u64, + read_frame_size: 16 * protobuf::kB as u64, read_frame_count: 100, - write_frame_size: 16 * protobuf_utils::kB as u64, + write_frame_size: 16 * protobuf::kB as u64, }; /// Trait for defining an RPC. @@ -61,9 +61,9 @@ pub(crate) trait Rpc: Sync + Send + 'static { /// Name of the RPC, used in prometheus metrics. const METHOD: &'static str; /// Type of the request message. - type Req: protobuf_utils::ProtoFmt + Send + Sync; + type Req: protobuf::ProtoFmt + Send + Sync; /// Type of the response message. - type Resp: protobuf_utils::ProtoFmt + Send + Sync; + type Resp: protobuf::ProtoFmt + Send + Sync; /// Name of the variant of the request message type. /// Useful for collecting metrics with finer than /// per-rpc type granularity. diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index 78a497db..1310014f 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -4,7 +4,7 @@ use anyhow::Context as _; use concurrency::{ctx, limiter, time}; use rand::Rng; use schema::proto::network::ping as proto; -use protobuf_utils::{required, ProtoFmt}; +use protobuf::{required, ProtoFmt}; /// Ping RPC. pub(crate) struct Rpc; @@ -65,7 +65,7 @@ pub(crate) struct Resp(pub(crate) [u8; 32]); impl ProtoFmt for Req { type Proto = proto::PingReq; fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) @@ -80,7 +80,7 @@ impl ProtoFmt for Req { impl ProtoFmt for Resp { type Proto = proto::PingResp; fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index a0d630cf..0244fa4a 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -5,7 +5,7 @@ use anyhow::Context; use concurrency::{limiter, time}; use roles::validator::{BlockNumber, FinalBlock}; use schema::proto::network::gossip as proto; -use protobuf_utils::{read_required, ProtoFmt}; +use protobuf::{read_required, ProtoFmt}; /// `get_sync_state` RPC. #[derive(Debug)] @@ -48,7 +48,7 @@ impl ProtoFmt for io::SyncState { fn max_size() -> usize { // TODO: estimate maximum size more precisely - 100 * protobuf_utils::kB + 100 * protobuf::kB } } @@ -68,7 +68,7 @@ impl ProtoFmt for SyncStateResponse { } fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } } @@ -110,7 +110,7 @@ impl ProtoFmt for GetBlockRequest { } fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } } @@ -138,7 +138,7 @@ impl ProtoFmt for io::GetBlockError { } fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } } @@ -179,6 +179,6 @@ impl ProtoFmt for GetBlockResponse { } fn max_size() -> usize { - protobuf_utils::MB + protobuf::MB } } diff --git a/node/actors/network/src/rpc/sync_validator_addrs.rs b/node/actors/network/src/rpc/sync_validator_addrs.rs index d328d6a2..41329381 100644 --- a/node/actors/network/src/rpc/sync_validator_addrs.rs +++ b/node/actors/network/src/rpc/sync_validator_addrs.rs @@ -5,7 +5,7 @@ use anyhow::Context as _; use concurrency::{limiter, time}; use roles::validator; use schema::proto::network::gossip as proto; -use protobuf_utils::{ProtoFmt}; +use protobuf::{ProtoFmt}; use std::sync::Arc; /// SyncValidatorAddrs Rpc. @@ -46,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - protobuf_utils::kB + protobuf::kB } } @@ -70,6 +70,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - protobuf_utils::MB + protobuf::MB } } diff --git a/node/libs/protobuf/build/Cargo.toml b/node/libs/protobuf/build/Cargo.toml deleted file mode 100644 index 0e33b33d..00000000 --- a/node/libs/protobuf/build/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "protobuf_build" -version = "0.1.0" -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true - -[dependencies] -anyhow.workspace = true -bit-vec.workspace = true -serde.workspace = true -once_cell.workspace = true -quick-protobuf.workspace = true -prost.workspace = true -prost-types.workspace = true -prost-reflect.workspace = true -rand.workspace = true -serde_json.workspace = true -tokio.workspace = true - -concurrency = { path = "../../concurrency" } - -syn.workspace = true -protoc-bin-vendored.workspace = true -prost-build.workspace = true -prost-reflect-build.workspace = true -prettyplease.workspace = true diff --git a/node/libs/protobuf/build/src/lib.rs b/node/libs/protobuf/build/src/lib.rs deleted file mode 100644 index a6d31225..00000000 --- a/node/libs/protobuf/build/src/lib.rs +++ /dev/null @@ -1,230 +0,0 @@ -//! Generates rust code from the capnp schema files in the `capnp/` directory. -use anyhow::Context as _; -use std::{collections::BTreeMap, env, fs, path::{PathBuf,Path}}; -use std::process::Command; -use prost::Message as _; -use std::collections::HashSet; -use once_cell::sync::Lazy; - -pub struct Descriptor { - pub crate_name: String, - pub module_path: String, -} - -/// Traversed all the files in a directory recursively. -fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> { - if !path.is_dir() { - f(&path); - return Ok(()); - } - for entry in fs::read_dir(path)? { - traverse_files(&entry?.path(), f)?; - } - Ok(()) -} - -/// A rust module representation. -/// It is used to collect the generated protobuf code. -#[derive(Default)] -struct Module { - /// Nested modules which transitively contain the generated code. - nested: BTreeMap, - /// Nested modules directly contains the generated code. - include: BTreeMap, -} - -impl Module { - /// Inserts a nested generated protobuf module. - /// `name` is a sequence of module names. - fn insert(&mut self, name: &[String], file: PathBuf) { - println!(" -- {name:?}"); - match name.len() { - 0 => panic!("empty module path"), - 1 => assert!( - self.include.insert(name[0].clone(), file).is_none(), - "duplicate module" - ), - _ => self - .nested - .entry(name[0].clone()) - .or_default() - .insert(&name[1..], file), - } - } - - /// Generates rust code of the module. - fn generate(&self) -> String { - let mut entries = vec![]; - entries.extend( - self.nested - .iter() - .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), - ); - entries.extend( - self.include - .iter() - .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), - ); - entries.join("\n") - } -} - -#[derive(Default)] -struct CanonicalCheckState(HashSet); - -impl CanonicalCheckState { - /// Checks if messages of type `m` support canonical encoding. - fn check_message(&mut self, m: &prost_reflect::MessageDescriptor, check_nested: bool) -> anyhow::Result<()> { - if self.0.contains(m.full_name()) { - return Ok(()); - } - self.0.insert(m.full_name().to_string()); - for f in m.fields() { - self.check_field(&f).with_context(|| f.name().to_string())?; - } - if check_nested { - for m in m.child_messages() { - self.check_message(&m,check_nested).with_context(||m.name().to_string())?; - } - } - Ok(()) - } - - /// Checks if field `f` supports canonical encoding. - fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { - if f.is_map() { - anyhow::bail!("maps unsupported"); - } - if !f.is_list() && !f.supports_presence() { - anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); - } - if let prost_reflect::Kind::Message(msg) = f.kind() { - self.check_message(&msg,false).with_context(||msg.name().to_string())?; - } - Ok(()) - } - - /// Checks if message types in file `f` support canonical encoding. - fn check_file(&mut self, f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { - if f.syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - for m in f.messages() { - self.check_message(&m,true).with_context(|| m.name().to_string())?; - } - Ok(()) - } -} - -pub fn compile(proto_include: &Path, module_path: &str, deps: &[&Lazy]) -> anyhow::Result<()> { - println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); - let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) - .canonicalize()? - .join("proto"); - let _ = fs::remove_dir_all(&proto_output); - fs::create_dir_all(&proto_output).unwrap(); - - // Find all proto files. - let mut proto_inputs : Vec = vec![]; - traverse_files(proto_include, &mut |path| { - let Some(ext) = path.extension() else { return }; - let Some(ext) = ext.to_str() else { return }; - if ext != "proto" { - return; - }; - proto_inputs.push(path.into()); - })?; - - // Compile input files into descriptor. - let descriptor_path = proto_output.join("descriptor.binpb"); - let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); - cmd.arg("-o").arg(&descriptor_path); - cmd.arg("-I").arg(&proto_include); - - /*if deps.len() > 0 { - let mut deps_list = vec![]; - for (i,(_,d)) in deps.iter().enumerate() { - let name = proto_output.join(format!("dep{i}.binpb")); - fs::write(&name,d)?; - deps_list.push(name.to_str().unwrap().to_string()); - } - cmd.arg("--descriptor_set_in").arg(deps_list.join(":")); - }*/ - - let deps : Vec<_> = deps.iter().map(|d|&***d).collect(); - let deps_path = proto_output.join("deps.binpb"); - fs::write(&deps_path,prost_reflect::DescriptorPool::global().encode_to_vec())?; - cmd.arg("--descriptor_set_in").arg(&deps_path); - - for input in &proto_inputs { - cmd.arg(&input); - } - - let out = cmd.output().context("protoc execution failed")?; - - if !out.status.success() { - anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); - } - - // Generate protobuf code from schema (with reflection). - let descriptor = fs::read(&descriptor_path)?; - let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); - for p in &descriptor.file { - prost_reflect::DescriptorPool::add_global_file_descriptor_proto::<&[u8]>(p.clone()).unwrap(); - } - let pool_attribute = format!(r#"#[prost_reflect(descriptor_pool = "crate::{module_path}::DESCRIPTOR_POOL")]"#); - - let empty : &[&Path] = &[]; - let mut config = prost_build::Config::new(); - let pool = prost_reflect::DescriptorPool::global(); - for message in pool.all_messages() { - let full_name = message.full_name(); - config - .type_attribute(full_name, "#[derive(::prost_reflect::ReflectMessage)]") - .type_attribute(full_name, &format!(r#"#[prost_reflect(message_name = "{}")]"#, full_name)) - .type_attribute(full_name, &pool_attribute); - } - config.file_descriptor_set_path(&descriptor_path); - config.skip_protoc_run(); - config.out_dir(&proto_output); - config.compile_protos(empty,empty).unwrap(); - - // Check that messages are compatible with `proto_fmt::canonical`. - let mut check_state = CanonicalCheckState::default(); - for f in &descriptor.file { - check_state.check_file(&pool.get_file_by_name(f.name.as_ref().unwrap()).unwrap())?; - } - - // Generate mod file collecting all proto-generated code. - let mut m = Module::default(); - for entry in fs::read_dir(&proto_output).unwrap() { - let entry = entry.unwrap(); - let name = entry.file_name().into_string().unwrap(); - let Some(name) = name.strip_suffix(".rs") else { - continue; - }; - let name: Vec<_> = name.split('.').map(String::from).collect(); - println!("name = {name:?}"); - m.insert(&name, entry.path()); - } - let mut file = deps.iter().map(|d|format!("use {}::{}::*;",d.crate_name,d.module_path)).collect::>().join("\n"); - file += &m.generate(); - let rec = deps.iter().map(|d|format!("&*{}::{}::DESCRIPTOR;",d.crate_name,d.module_path)).collect::>().join(" "); - file += &format!("pub const DESCRIPTOR: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| {{ \ - {rec} \ - prost_reflect::DescriptorPool::decode_global_file_descriptor_set(&include_bytes!({descriptor_path:?})[..]).unwrap(); \ - protobuf_build::Descriptor {{\ - crate_name: {:?}.to_string(),\ - module_path: {module_path:?}.to_string(),\ - }}\ - }});",std::env::var("CARGO_PKG_NAME").unwrap()); - file += "pub const DESCRIPTOR_POOL: once_cell::sync::Lazy = \ - once_cell::sync::Lazy::new(|| { \ - &*DESCRIPTOR; \ - prost_reflect::DescriptorPool::global() \ - }); \ - "; - let file = syn::parse_str(&file).unwrap(); - fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; - Ok(()) -} diff --git a/node/libs/protobuf/utils/Cargo.toml b/node/libs/protobuf/utils/Cargo.toml deleted file mode 100644 index e929870d..00000000 --- a/node/libs/protobuf/utils/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "protobuf_utils" -version = "0.1.0" -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true - -[[bin]] -name = "conformance_test" - -[dependencies] -anyhow.workspace = true -bit-vec.workspace = true -serde.workspace = true -quick-protobuf.workspace = true -prost.workspace = true -prost-reflect.workspace = true -rand.workspace = true -serde_json.workspace = true -tokio.workspace = true -once_cell.workspace = true - -protobuf_build = { path = "../build" } -concurrency = { path = "../../concurrency" } - -[build-dependencies] -anyhow.workspace = true -protobuf_build = { path = "../build" } diff --git a/node/libs/protobuf/utils/build.rs b/node/libs/protobuf/utils/build.rs deleted file mode 100644 index 01ab9e3d..00000000 --- a/node/libs/protobuf/utils/build.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Generates rust code from the capnp schema files in the `capnp/` directory. -use std::{env, path::PathBuf}; - -fn main() -> anyhow::Result<()> { - // Prepare input and output root dirs. - let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) - .canonicalize()? - .join("proto"); - protobuf_build::compile(&proto_include,"proto",&[])?; - Ok(()) -} diff --git a/node/libs/protobuf/utils/proto/conformance/conformance.proto b/node/libs/protobuf/utils/proto/conformance/conformance.proto deleted file mode 100644 index e6403673..00000000 --- a/node/libs/protobuf/utils/proto/conformance/conformance.proto +++ /dev/null @@ -1,183 +0,0 @@ -// Modified copy of -// https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto -// -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package conformance; - -option java_package = "com.google.protobuf.conformance"; -option objc_class_prefix = "Conformance"; - -// This defines the conformance testing protocol. This protocol exists between -// the conformance test suite itself and the code being tested. For each test, -// the suite will send a ConformanceRequest message and expect a -// ConformanceResponse message. -// -// You can either run the tests in two different ways: -// -// 1. in-process (using the interface in conformance_test.h). -// -// 2. as a sub-process communicating over a pipe. Information about how to -// do this is in conformance_test_runner.cc. -// -// Pros/cons of the two approaches: -// -// - running as a sub-process is much simpler for languages other than C/C++. -// -// - running as a sub-process may be more tricky in unusual environments like -// iOS apps, where fork/stdin/stdout are not available. - -enum WireFormat { - UNSPECIFIED = 0; - PROTOBUF = 1; - JSON = 2; - JSPB = 3; // Only used inside Google. Opensource testees just skip it. - TEXT_FORMAT = 4; -} - -enum TestCategory { - UNSPECIFIED_TEST = 0; - BINARY_TEST = 1; // Test binary wire format. - JSON_TEST = 2; // Test json wire format. - // Similar to JSON_TEST. However, during parsing json, testee should ignore - // unknown fields. This feature is optional. Each implementation can decide - // whether to support it. See - // https://developers.google.com/protocol-buffers/docs/proto3#json_options - // for more detail. - JSON_IGNORE_UNKNOWN_PARSING_TEST = 3; - // Test jspb wire format. Only used inside Google. Opensource testees just - // skip it. - JSPB_TEST = 4; - // Test text format. For cpp, java and python, testees can already deal with - // this type. Testees of other languages can simply skip it. - TEXT_FORMAT_TEST = 5; -} - -// The conformance runner will request a list of failures as the first request. -// This will be known by message_type == "conformance.FailureSet", a conformance -// test should return a serialized FailureSet in protobuf_payload. -message FailureSet { - repeated string failure = 1; -} - -// Represents a single test case's input. The testee should: -// -// 1. parse this proto (which should always succeed) -// 2. parse the protobuf or JSON payload in "payload" (which may fail) -// 3. if the parse succeeded, serialize the message in the requested format. -message ConformanceRequest { - // The payload (whether protobuf of JSON) is always for a - // protobuf_test_messages.proto3.TestAllTypes proto (as defined in - // src/google/protobuf/proto3_test_messages.proto). - oneof payload { - bytes protobuf_payload = 1; - string json_payload = 2; - // Only used inside Google. Opensource testees just skip it. - string jspb_payload = 7; - string text_payload = 8; - } - - // Which format should the testee serialize its message to? - optional WireFormat requested_output_format = 3; - - // The full name for the test message to use; for the moment, either: - // protobuf_test_messages.proto3.TestAllTypesProto3 or - // protobuf_test_messages.google.protobuf.TestAllTypesProto2. - optional string message_type = 4; - - // Each test is given a specific test category. Some category may need - // specific support in testee programs. Refer to the definition of - // TestCategory for more information. - optional TestCategory test_category = 5; - - // Specify details for how to encode jspb. - optional JspbEncodingConfig jspb_encoding_options = 6; - - // This can be used in json and text format. If true, testee should print - // unknown fields instead of ignore. This feature is optional. - optional bool print_unknown_fields = 9; -} - -// Represents a single test case's output. -message ConformanceResponse { - oneof result { - // This string should be set to indicate parsing failed. The string can - // provide more information about the parse error if it is available. - // - // Setting this string does not necessarily mean the testee failed the - // test. Some of the test cases are intentionally invalid input. - string parse_error = 1; - - // If the input was successfully parsed but errors occurred when - // serializing it to the requested output format, set the error message in - // this field. - string serialize_error = 6; - - // This should be set if the test program timed out. The string should - // provide more information about what the child process was doing when it - // was killed. - string timeout_error = 9; - - // This should be set if some other error occurred. This will always - // indicate that the test failed. The string can provide more information - // about the failure. - string runtime_error = 2; - - // If the input was successfully parsed and the requested output was - // protobuf, serialize it to protobuf and set it in this field. - bytes protobuf_payload = 3; - - // If the input was successfully parsed and the requested output was JSON, - // serialize to JSON and set it in this field. - string json_payload = 4; - - // For when the testee skipped the test, likely because a certain feature - // wasn't supported, like JSON input/output. - string skipped = 5; - - // If the input was successfully parsed and the requested output was JSPB, - // serialize to JSPB and set it in this field. JSPB is only used inside - // Google. Opensource testees can just skip it. - string jspb_payload = 7; - - // If the input was successfully parsed and the requested output was - // TEXT_FORMAT, serialize to TEXT_FORMAT and set it in this field. - string text_payload = 8; - } -} - -// Encoding options for jspb format. -message JspbEncodingConfig { - // Encode the value field of Any as jspb array if true, otherwise binary. - optional bool use_jspb_array_any_format = 1; -} diff --git a/node/libs/protobuf/utils/proto/conformance/test_messages_proto3.proto b/node/libs/protobuf/utils/proto/conformance/test_messages_proto3.proto deleted file mode 100644 index d391761b..00000000 --- a/node/libs/protobuf/utils/proto/conformance/test_messages_proto3.proto +++ /dev/null @@ -1,225 +0,0 @@ -// Modified copy of -// https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/test_messages_proto3.proto -// -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Test schema for proto3 messages. This test schema is used by: -// -// - benchmarks -// - fuzz tests -// - conformance tests -// - -syntax = "proto3"; - -package protobuf_test_messages.proto3; - -option java_package = "com.google.protobuf_test_messages.proto3"; -option objc_class_prefix = "Proto3"; - -// This is the default, but we specify it here explicitly. -option optimize_for = SPEED; - -option cc_enable_arenas = true; - -// This proto includes every type of field in both singular and repeated -// forms. -// -// Also, crucially, all messages and enums in this file are eventually -// submessages of this message. So for example, a fuzz test of TestAllTypes -// could trigger bugs that occur in any message type in this file. We verify -// this stays true in a unit test. -message TestAllTypesProto3 { - message NestedMessage { - optional int32 a = 1; - optional TestAllTypesProto3 corecursive = 2; - } - - enum NestedEnum { - FOO = 0; - BAR = 1; - BAZ = 2; - NEG = -1; // Intentionally negative. - } - - enum AliasedEnum { - option allow_alias = true; - - ALIAS_FOO = 0; - ALIAS_BAR = 1; - ALIAS_BAZ = 2; - MOO = 2; - moo = 2; - bAz = 2; - } - - // Singular - optional int32 optional_int32 = 1; - optional int64 optional_int64 = 2; - optional uint32 optional_uint32 = 3; - optional uint64 optional_uint64 = 4; - optional sint32 optional_sint32 = 5; - optional sint64 optional_sint64 = 6; - optional fixed32 optional_fixed32 = 7; - optional fixed64 optional_fixed64 = 8; - optional sfixed32 optional_sfixed32 = 9; - optional sfixed64 optional_sfixed64 = 10; - optional float optional_float = 11; - optional double optional_double = 12; - optional bool optional_bool = 13; - optional string optional_string = 14; - optional bytes optional_bytes = 15; - - optional NestedMessage optional_nested_message = 18; - optional ForeignMessage optional_foreign_message = 19; - - optional NestedEnum optional_nested_enum = 21; - optional ForeignEnum optional_foreign_enum = 22; - optional AliasedEnum optional_aliased_enum = 23; - - optional string optional_string_piece = 24 [ctype = STRING_PIECE]; - optional string optional_cord = 25 [ctype = CORD]; - - optional TestAllTypesProto3 recursive_message = 27; - - // Repeated - repeated int32 repeated_int32 = 31; - repeated int64 repeated_int64 = 32; - repeated uint32 repeated_uint32 = 33; - repeated uint64 repeated_uint64 = 34; - repeated sint32 repeated_sint32 = 35; - repeated sint64 repeated_sint64 = 36; - repeated fixed32 repeated_fixed32 = 37; - repeated fixed64 repeated_fixed64 = 38; - repeated sfixed32 repeated_sfixed32 = 39; - repeated sfixed64 repeated_sfixed64 = 40; - repeated float repeated_float = 41; - repeated double repeated_double = 42; - repeated bool repeated_bool = 43; - repeated string repeated_string = 44; - repeated bytes repeated_bytes = 45; - - repeated NestedMessage repeated_nested_message = 48; - repeated ForeignMessage repeated_foreign_message = 49; - - repeated NestedEnum repeated_nested_enum = 51; - repeated ForeignEnum repeated_foreign_enum = 52; - - repeated string repeated_string_piece = 54 [ctype = STRING_PIECE]; - repeated string repeated_cord = 55 [ctype = CORD]; - - // Packed - repeated int32 packed_int32 = 75 [packed = true]; - repeated int64 packed_int64 = 76 [packed = true]; - repeated uint32 packed_uint32 = 77 [packed = true]; - repeated uint64 packed_uint64 = 78 [packed = true]; - repeated sint32 packed_sint32 = 79 [packed = true]; - repeated sint64 packed_sint64 = 80 [packed = true]; - repeated fixed32 packed_fixed32 = 81 [packed = true]; - repeated fixed64 packed_fixed64 = 82 [packed = true]; - repeated sfixed32 packed_sfixed32 = 83 [packed = true]; - repeated sfixed64 packed_sfixed64 = 84 [packed = true]; - repeated float packed_float = 85 [packed = true]; - repeated double packed_double = 86 [packed = true]; - repeated bool packed_bool = 87 [packed = true]; - repeated NestedEnum packed_nested_enum = 88 [packed = true]; - - // Unpacked - repeated int32 unpacked_int32 = 89 [packed = false]; - repeated int64 unpacked_int64 = 90 [packed = false]; - repeated uint32 unpacked_uint32 = 91 [packed = false]; - repeated uint64 unpacked_uint64 = 92 [packed = false]; - repeated sint32 unpacked_sint32 = 93 [packed = false]; - repeated sint64 unpacked_sint64 = 94 [packed = false]; - repeated fixed32 unpacked_fixed32 = 95 [packed = false]; - repeated fixed64 unpacked_fixed64 = 96 [packed = false]; - repeated sfixed32 unpacked_sfixed32 = 97 [packed = false]; - repeated sfixed64 unpacked_sfixed64 = 98 [packed = false]; - repeated float unpacked_float = 99 [packed = false]; - repeated double unpacked_double = 100 [packed = false]; - repeated bool unpacked_bool = 101 [packed = false]; - repeated NestedEnum unpacked_nested_enum = 102 [packed = false]; - - oneof oneof_field { - uint32 oneof_uint32 = 111; - NestedMessage oneof_nested_message = 112; - string oneof_string = 113; - bytes oneof_bytes = 114; - bool oneof_bool = 115; - uint64 oneof_uint64 = 116; - float oneof_float = 117; - double oneof_double = 118; - NestedEnum oneof_enum = 119; - } - - // Test field-name-to-JSON-name convention. - // (protobuf says names can be any valid C/C++ identifier.) - optional int32 fieldname1 = 401; - optional int32 field_name2 = 402; - optional int32 _field_name3 = 403; - optional int32 field__name4_ = 404; - optional int32 field0name5 = 405; - optional int32 field_0_name6 = 406; - optional int32 fieldName7 = 407; - optional int32 FieldName8 = 408; - optional int32 field_Name9 = 409; - optional int32 Field_Name10 = 410; - optional int32 FIELD_NAME11 = 411; - optional int32 FIELD_name12 = 412; - optional int32 __field_name13 = 413; - optional int32 __Field_name14 = 414; - optional int32 field__name15 = 415; - optional int32 field__Name16 = 416; - optional int32 field_name17__ = 417; - optional int32 Field_name18__ = 418; - - // Reserved for testing unknown fields - reserved 501 to 510; -} - -message ForeignMessage { - optional int32 c = 1; -} - -enum ForeignEnum { - FOREIGN_FOO = 0; - FOREIGN_BAR = 1; - FOREIGN_BAZ = 2; -} - -message NullHypothesisProto3 {} - -message EnumOnlyProto3 { - enum Bool { - kFalse = 0; - kTrue = 1; - } -} diff --git a/node/libs/protobuf/utils/proto/std.proto b/node/libs/protobuf/utils/proto/std.proto deleted file mode 100644 index f5ee5efe..00000000 --- a/node/libs/protobuf/utils/proto/std.proto +++ /dev/null @@ -1,50 +0,0 @@ -syntax = "proto3"; - -package std; - -message Void {} - -// Timestamp represented as seconds + nanoseconds since UNIX epoch. -// Equivalent of `google.protobuf.Timestamp` but supports canonical encoding. -// See `google.protobuf.Timestamp` for more detailed specification. -message Timestamp { - // Represents seconds of UTC time since Unix epoch - // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to - // 9999-12-31T23:59:59Z inclusive. - optional int64 seconds = 1; - - // Non-negative fractions of a second at nanosecond resolution. Negative - // second values with fractions must still have non-negative nanos values - // that count forward in time. Must be from 0 to 999,999,999 - // inclusive. - optional int32 nanos = 2; -} - -message Duration { - optional int64 seconds = 1; - optional int32 nanos = 2; -} - -// IP:port TCP address. -message SocketAddr { - // Ipv4 (4 bytes) or IPv6 (16 bytes) in network byte order. - optional bytes ip = 1; - // TCP port (actually uint16, however uint32 is smallest supported protobuf type). - optional uint32 port = 2; -} - -// Compressed representation of a vector of bits. -// It occupies n/8 + O(1) bytes. -// Note: this is not a highly optimized data structure: -// - since it is a proto message it requires its length to be stored. -// - it has 2 fields, which require storing their own tag -// - the `bytes_` field requires storing its length, which theoretically -// could be derived from `size` field value. -// If we eventually start caring about the size, we could encode BitVector -// directly as a field of type `bytes` with delimiter of the form "01...1". -message BitVector { - // Number of bits in the vector. - optional uint64 size = 1; - // Vector of bits encoded as bytes in big endian order. - optional bytes bytes_ = 2; -} diff --git a/node/libs/protobuf/utils/proto/testonly.proto b/node/libs/protobuf/utils/proto/testonly.proto deleted file mode 100644 index 7c5d08cd..00000000 --- a/node/libs/protobuf/utils/proto/testonly.proto +++ /dev/null @@ -1,27 +0,0 @@ -syntax = "proto3"; - -package testonly; - -message B { - // Recursive union. - oneof t { - bool u = 2; - B v = 3; - } -} - -enum E { - E0 = 0; - E1 = 1; - E2 = 2; -} - -message A { - optional bytes x = 1; - optional uint64 y = 2; - // Repeated enums are non-packable, - // But we pack them in the canonical encoding. - repeated E e = 4; - // Nested message. - optional B b = 5; -} diff --git a/node/libs/protobuf/utils/src/bin/conformance_test.rs b/node/libs/protobuf/utils/src/bin/conformance_test.rs deleted file mode 100644 index db2f2abb..00000000 --- a/node/libs/protobuf/utils/src/bin/conformance_test.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Conformance test for our canonical encoding implemented according to -//! https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto -//! Our implementation supports only a subset of proto functionality, so -//! `schema/proto/conformance/conformance.proto` and -//! `schema/proto/conformance/protobuf_test_messages.proto` contains only a -//! subset of original fields. Also we run only proto3 binary -> binary tests. -//! conformance_test_failure_list.txt contains tests which are expected to fail. -use anyhow::Context as _; -use concurrency::{ctx, io}; -use prost::Message as _; -use prost_reflect::ReflectMessage; -use protobuf_utils::proto::conformance as proto; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let ctx = &ctx::root(); - let stdin = &mut tokio::io::stdin(); - let stdout = &mut tokio::io::stdout(); - loop { - // Read the request. - let mut msg_size = [0u8; 4]; - if io::read_exact(ctx, stdin, &mut msg_size).await?.is_err() { - return Ok(()); - } - let msg_size = u32::from_le_bytes(msg_size); - let mut msg = vec![0u8; msg_size as usize]; - io::read_exact(ctx, stdin, &mut msg[..]).await??; - - use proto::conformance_response::Result as R; - let req = proto::ConformanceRequest::decode(&msg[..])?; - let res = async { - let t = req.message_type.context("missing message_type")?; - if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { - return Ok(R::Skipped("unsupported".to_string())); - } - - // Decode. - let payload = req.payload.context("missing payload")?; - use protobuf_utils::proto::protobuf_test_messages::proto3::TestAllTypesProto3 as T; - let p = match payload { - proto::conformance_request::Payload::JsonPayload(payload) => { - match protobuf_utils::decode_json_proto(&payload) { - Ok(p) => p, - Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), - } - } - proto::conformance_request::Payload::ProtobufPayload(payload) => { - // First filter out incorrect encodings. - let Ok(p) = T::decode(&payload[..]) else { - return Ok(R::ParseError("parsing failed".to_string())); - }; - // Then check if there are any unknown fields in the original payload. - if protobuf_utils::canonical_raw(&payload[..], &p.descriptor()).is_err() { - return Ok(R::Skipped("unsupported fields".to_string())); - } - p - } - _ => return Ok(R::Skipped("unsupported input format".to_string())), - }; - - // Encode. - let format = req - .requested_output_format - .context("missing output format")?; - match proto::WireFormat::from_i32(format).context("unknown format")? { - proto::WireFormat::Json => { - anyhow::Ok(R::JsonPayload(protobuf_utils::encode_json_proto(&p))) - } - proto::WireFormat::Protobuf => { - // Reencode the parsed proto. - anyhow::Ok(R::ProtobufPayload(protobuf_utils::canonical_raw( - &p.encode_to_vec(), - &p.descriptor(), - )?)) - } - _ => Ok(R::Skipped("unsupported output format".to_string())), - } - } - .await?; - let resp = proto::ConformanceResponse { result: Some(res) }; - - // Write the response. - let msg = resp.encode_to_vec(); - io::write_all(ctx, stdout, &u32::to_le_bytes(msg.len() as u32)).await??; - io::write_all(ctx, stdout, &msg).await??; - io::flush(ctx, stdout).await??; - } -} diff --git a/node/libs/protobuf/utils/src/bin/conformance_test_failure_list.txt b/node/libs/protobuf/utils/src/bin/conformance_test_failure_list.txt deleted file mode 100644 index 82ecacee..00000000 --- a/node/libs/protobuf/utils/src/bin/conformance_test_failure_list.txt +++ /dev/null @@ -1,11 +0,0 @@ -# We have added "optional" annotation to some of the fields in the test proto -# (which is required by our canonical encoding). For those fields json encoder -# cannot deduce the intended default values, which breaks the following test: -Required.Proto3.JsonInput.SkipsDefaultPrimitive.Validator - -# Json decoder slightly rounds the extreme values for f64 when parsing (it is a bug). -# This makes the following tests fail: -Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.JsonOutput -Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput -Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.JsonOutput -Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.ProtobufOutput diff --git a/node/libs/protobuf/utils/src/lib.rs b/node/libs/protobuf/utils/src/lib.rs deleted file mode 100644 index b36a9bb7..00000000 --- a/node/libs/protobuf/utils/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Code generated from protobuf schema files and -//! utilities for serialization. - -mod proto_fmt; -mod std_conv; -pub mod testonly; - -pub use proto_fmt::*; - -#[cfg(test)] -mod tests; - -#[allow(warnings)] -pub mod proto { - include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); -} diff --git a/node/libs/protobuf/utils/src/proto_fmt.rs b/node/libs/protobuf/utils/src/proto_fmt.rs deleted file mode 100644 index 541b0f0b..00000000 --- a/node/libs/protobuf/utils/src/proto_fmt.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! Canonical encoding for protobufs with known schema. -//! -//! Protobuf encoding spec: https://protobuf.dev/programming-guides/encoding/ -//! By default the encoding is not canonical (multiple encodings may represent the same message). -//! Here we extend this spec to support a canonical encoding. -//! -//! Canonical encoding spec: -//! * fields are encoded in ascending order of tags. -//! * varints are encoded using minimal amount of bytes. -//! * map fields are not supported -//! * groups are not supported -//! * only proto3 syntax is supported -//! * hence assigning a field a default value (proto2) is not supported -//! * all fields should have explicit presence (explicit optional or repeated) -//! * hence a default value for primitive types (proto3) is not supported. -//! * a field of primitive type (I32, I64, Varint) is encoded as at most 1 TLV, depending on -//! the number of values: -//! * 0 values => field is not encoded at all -//! * 1 value => field is encoded as a single I32/I64/Varint TLV. -//! * >1 values => field is encoded as a single TLV using packed encoding (https://protobuf.dev/programming-guides/encoding/#packed) -//! * protobuf spec allows for packed encoding only for repeated fields. We therefore don't -//! support specifying multiple values for a singular field (which is pathological case -//! anyway). -//! * protobuf spec also doesn't allow for encoding repeated enum fields (which are encoded as -//! Varint), however it mentions that it might allow for it in the future versions. -//! TODO(gprusak): decide what we should do with those: prost implementation decodes correctly -//! packed enums, I don't know about other protobuf implementations (rust and non-rust). We may either: -//! * allow packed encoding for enums (despite the incompatibility with protobuf spec) -//! * disallow packed encoding for enums specifically (although it complicates the canonical spec) -//! * drop packed encoding altogether (at the cost of larger messages) -//! -//! We need canonical encoding to be able to compute hashes and verify signatures on -//! messages. Note that with this spec it is not possible to encode canonically a message with unknown -//! fields. It won't be a problem though if we decide on one of the following: -//! * require all communication to just use canonical encoding: then as long as we carry around the -//! unknown fields untouched, they will stay canonical. -//! * hash/verify only messages that don't contain unknown fields: then we can just canonicalize -//! whatever we understand and reject messages with unknown fields. -//! * drop the idea of canonical encoding altogether and pass around the received encoded message -//! for hashing/verifying (i.e. keep the raw bytes together with the parsed message). -use anyhow::Context as _; -use prost::Message as _; -use prost_reflect::ReflectMessage; -use std::collections::BTreeMap; - -/// Kilobyte. -#[allow(non_upper_case_globals)] -pub const kB: usize = 1 << 10; -/// Megabyte. -pub const MB: usize = 1 << 20; - -/// Protobuf wire type. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(super) enum Wire { - /// VARINT. - Varint, - /// I64 - I64, - /// LEN - Len, - /// I32 - I32, -} - -/// Raw write type values, as defined in -/// https://protobuf.dev/programming-guides/encoding/#structure -/// VARINT -const VARINT: u32 = 0; -/// I64 -const I64: u32 = 1; -/// LEN -const LEN: u32 = 2; -/// I32 -const I32: u32 = 5; - -impl Wire { - /// Extracts a wire type from a tag. - pub(crate) const fn from_tag(tag: u32) -> Option { - match tag & 7 { - VARINT => Some(Self::Varint), - I64 => Some(Self::I64), - LEN => Some(Self::Len), - I32 => Some(Self::I32), - _ => None, - } - } - - /// Converts wire type to the raw wite type value. - pub(crate) const fn raw(self) -> u32 { - match self { - Self::Varint => VARINT, - Self::I64 => I64, - Self::Len => LEN, - Self::I32 => I32, - } - } -} - -impl From for Wire { - fn from(kind: prost_reflect::Kind) -> Self { - use prost_reflect::Kind; - match kind { - Kind::Int32 - | Kind::Int64 - | Kind::Uint32 - | Kind::Uint64 - | Kind::Sint32 - | Kind::Sint64 - | Kind::Bool - | Kind::Enum(_) => Self::Varint, - Kind::Fixed64 | Kind::Sfixed64 | Kind::Double => Self::I64, - Kind::Fixed32 | Kind::Sfixed32 | Kind::Float => Self::I32, - Kind::String | Kind::Bytes | Kind::Message(_) => Self::Len, - } - } -} - -/// Wrapper of the quick_protobuf reader. -// TODO(gprusak): reimplement the reader and writer instead to make it more efficient. -pub(super) struct Reader<'a>(quick_protobuf::BytesReader, &'a [u8]); - -impl<'a> Reader<'a> { - /// Constructs a reader of a slice. - fn new(bytes: &'a [u8]) -> Self { - Self(quick_protobuf::BytesReader::from_bytes(bytes), bytes) - } - - /// Reads a value of the given wire type. - fn read(&mut self, wire: Wire) -> anyhow::Result> { - let mut v = vec![]; - let mut w = quick_protobuf::Writer::new(&mut v); - match wire { - Wire::Varint => w.write_varint(self.0.read_varint64(self.1)?).unwrap(), - Wire::I64 => w.write_fixed64(self.0.read_fixed64(self.1)?).unwrap(), - Wire::Len => return Ok(self.0.read_bytes(self.1)?.into()), - Wire::I32 => w.write_fixed32(self.0.read_fixed32(self.1)?).unwrap(), - } - Ok(v) - } - - /// Reads a (possibly packed) field based on the expected field wire type and the actual - /// wire type from the tag (which has been already read). - /// The values of the field are appended to `out`. - fn read_field( - &mut self, - out: &mut Vec>, - field_wire: Wire, - got_wire: Wire, - ) -> anyhow::Result<()> { - if got_wire == field_wire { - out.push(self.read(field_wire)?); - return Ok(()); - } - if got_wire != Wire::Len { - anyhow::bail!("unexpected wire type"); - } - let mut r = Self::new(self.0.read_bytes(self.1)?); - while !r.0.is_eof() { - out.push(r.read(field_wire)?); - } - Ok(()) - } -} - -/// Parses a message to a `field num -> values` map. -pub(super) fn read_fields( - buf: &[u8], - desc: &prost_reflect::MessageDescriptor, -) -> anyhow::Result>>> { - if desc.parent_file().syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - let mut r = Reader::new(buf); - let mut fields = BTreeMap::new(); - // Collect the field values from the reader. - while !r.0.is_eof() { - let tag = r.0.next_tag(r.1)?; - let wire = Wire::from_tag(tag).context("invalid wire type")?; - let field = desc.get_field(tag >> 3).context("unknown field")?; - if field.is_map() { - anyhow::bail!("maps unsupported"); - } - if !field.is_list() && !field.supports_presence() { - anyhow::bail!( - "{}::{} : fields with implicit presence are not supported", - field.parent_message().name(), - field.name() - ); - } - r.read_field( - fields.entry(field.number()).or_default(), - field.kind().into(), - wire, - )?; - } - Ok(fields) -} - -/// Converts an encoded protobuf message to its canonical form, given the descriptor of the message -/// type. Retuns an error if: -/// * an unknown field is detected -/// * the message type doesn't support canonical encoding (implicit presence, map fields) -pub fn canonical_raw( - buf: &[u8], - desc: &prost_reflect::MessageDescriptor, -) -> anyhow::Result> { - let mut v = vec![]; - let mut w = quick_protobuf::Writer::new(&mut v); - // Append fields in ascending tags order. - for (num, mut values) in read_fields(buf, desc)? { - let fd = desc.get_field(num).unwrap(); - if values.len() > 1 && !fd.is_list() { - anyhow::bail!("non-repeated field with multiple values"); - } - if let prost_reflect::Kind::Message(desc) = &fd.kind() { - for v in &mut values { - *v = canonical_raw(v, desc)?; - } - } - let wire = Wire::from(fd.kind()); - match wire { - Wire::Varint | Wire::I64 | Wire::I32 => { - if values.len() > 1 { - w.write_tag(num << 3 | LEN).unwrap(); - w.write_bytes(&values.into_iter().flatten().collect::>()) - .unwrap(); - } else { - w.write_tag(num << 3 | wire.raw()).unwrap(); - // inefficient workaround of the fact that quick_protobuf::Writer - // doesn't support just appending a sequence of bytes. - for b in &values[0] { - w.write_u8(*b).unwrap(); - } - } - } - Wire::Len => { - for v in &values { - w.write_tag(num << 3 | LEN).unwrap(); - w.write_bytes(v).unwrap(); - } - } - } - } - Ok(v) -} - -/// Encodes a proto message of known schema (no unknown fields) into a canonical form. -pub fn canonical(x: &T) -> Vec { - let msg = x.build(); - canonical_raw(&msg.encode_to_vec(), &msg.descriptor()).unwrap() -} - -/// Encodes a proto message. -/// Currently it outputs a canonical encoding, but `decode` accepts -/// non-canonical encoding as well. -/// It would simplify invariants if we required all messages to be canonical, -/// but we will see whether it is feasible once we have an RPC framework -/// in place. -pub fn encode(x: &T) -> Vec { - canonical(x) -} - -/// Decodes a proto message. -pub fn decode(bytes: &[u8]) -> anyhow::Result { - T::read(&::Proto::decode(bytes)?) -} - -/// Encodes a generated proto message to json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn encode_json_proto(x: &T) -> String { - let mut s = serde_json::Serializer::pretty(vec![]); - let opts = prost_reflect::SerializeOptions::new(); - x.transcode_to_dynamic() - .serialize_with_options(&mut s, &opts) - .unwrap(); - String::from_utf8(s.into_inner()).unwrap() -} - -/// Encodes a proto message to json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn encode_json(x: &T) -> String { - encode_json_proto(&x.build()) -} - -/// Decodes a generated proto message from json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn decode_json_proto(json: &str) -> anyhow::Result { - let mut d = serde_json::de::Deserializer::from_str(json); - let mut p = T::default(); - let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), &mut d)?; - d.end()?; - p.merge(msg.encode_to_vec().as_slice()).unwrap(); - Ok(p) -} - -/// Decodes a proto message from json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn decode_json(json: &str) -> anyhow::Result { - T::read(&decode_json_proto(json)?) -} - -/// Trait defining a proto representation for a type. -pub trait ProtoFmt: Sized { - /// Proto message type representing Self. - type Proto: ReflectMessage + Default; - /// Converts Proto to Self. - fn read(r: &Self::Proto) -> anyhow::Result; - /// Converts Self to Proto. - fn build(&self) -> Self::Proto; - /// Maximal allowed message size (in bytes). - fn max_size() -> usize { - usize::MAX - } -} - -/// Parses a required proto field. -pub fn read_required(field: &Option) -> anyhow::Result { - ProtoFmt::read(field.as_ref().context("missing field")?) -} - -/// Parses an optional proto field. -pub fn read_optional(field: &Option) -> anyhow::Result> { - field.as_ref().map(ProtoFmt::read).transpose() -} - -/// Extracts a required field. -pub fn required(field: &Option) -> anyhow::Result<&T> { - field.as_ref().context("missing") -} diff --git a/node/libs/protobuf/utils/src/std_conv.rs b/node/libs/protobuf/utils/src/std_conv.rs deleted file mode 100644 index 2f2fa89b..00000000 --- a/node/libs/protobuf/utils/src/std_conv.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Proto conversion for messages in std package. -use crate::{proto::std as proto, required, ProtoFmt}; -use anyhow::Context as _; -use concurrency::time; -use std::net; - -impl ProtoFmt for () { - type Proto = proto::Void; - fn read(_r: &Self::Proto) -> anyhow::Result { - Ok(()) - } - fn build(&self) -> Self::Proto { - Self::Proto {} - } -} - -impl ProtoFmt for std::net::SocketAddr { - type Proto = proto::SocketAddr; - - fn read(r: &Self::Proto) -> anyhow::Result { - let ip = required(&r.ip).context("ip")?; - let ip = match ip.len() { - 4 => net::IpAddr::from(<[u8; 4]>::try_from(&ip[..]).unwrap()), - 16 => net::IpAddr::from(<[u8; 16]>::try_from(&ip[..]).unwrap()), - _ => anyhow::bail!("invalid ip length"), - }; - let port = *required(&r.port).context("port")?; - let port = u16::try_from(port).context("port")?; - Ok(Self::new(ip, port)) - } - - fn build(&self) -> Self::Proto { - Self::Proto { - ip: Some(match self.ip() { - net::IpAddr::V4(ip) => ip.octets().to_vec(), - net::IpAddr::V6(ip) => ip.octets().to_vec(), - }), - port: Some(self.port() as u32), - } - } -} - -impl ProtoFmt for time::Utc { - type Proto = proto::Timestamp; - - fn read(r: &Self::Proto) -> anyhow::Result { - let seconds = *required(&r.seconds).context("seconds")?; - let nanos = *required(&r.nanos).context("nanos")?; - Ok(time::UNIX_EPOCH + time::Duration::new(seconds, nanos)) - } - - fn build(&self) -> Self::Proto { - let d = (*self - time::UNIX_EPOCH).build(); - Self::Proto { - seconds: d.seconds, - nanos: d.nanos, - } - } -} - -impl ProtoFmt for time::Duration { - type Proto = proto::Duration; - - fn read(r: &Self::Proto) -> anyhow::Result { - let seconds = *required(&r.seconds).context("seconds")?; - let nanos = *required(&r.nanos).context("nanos")?; - Ok(Self::new(seconds, nanos)) - } - - fn build(&self) -> Self::Proto { - let mut seconds = self.whole_seconds(); - let mut nanos = self.subsec_nanoseconds(); - if nanos < 0 { - seconds -= 1; - nanos += 1_000_000_000; - } - Self::Proto { - seconds: Some(seconds), - nanos: Some(nanos), - } - } -} - -impl ProtoFmt for bit_vec::BitVec { - type Proto = proto::BitVector; - - fn read(r: &Self::Proto) -> anyhow::Result { - let size = *required(&r.size).context("size")? as usize; - let mut this = Self::from_bytes(required(&r.bytes).context("bytes_")?); - if this.len() < size { - anyhow::bail!("'vector' has less than 'size' bits"); - } - this.truncate(size); - Ok(this) - } - - fn build(&self) -> Self::Proto { - Self::Proto { - size: Some(self.len() as u64), - bytes: Some(self.to_bytes()), - } - } -} diff --git a/node/libs/protobuf/utils/src/testonly.rs b/node/libs/protobuf/utils/src/testonly.rs deleted file mode 100644 index 9cf15ae0..00000000 --- a/node/libs/protobuf/utils/src/testonly.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! Testonly utilities. -use super::{canonical, canonical_raw, decode, encode, read_fields, ProtoFmt, Wire}; -use prost::Message as _; -use prost_reflect::ReflectMessage as _; -use rand::{ - distributions::{Distribution, Standard}, - Rng, -}; - -/// Test encoding and canonical encoding properties. -#[track_caller] -pub fn test_encode(rng: &mut R, x: &T) { - let x_encode = encode(x); - let x_canonical = canonical(x); - let x_shuffled = encode_shuffled(rng, x); - - let desc = &::Proto::default().descriptor(); - for e in [&x_encode, &x_canonical, &x_shuffled] { - // e is a valid encoding. - assert_eq!(x, &decode::(e).unwrap()); - // canonical encoding is consistent. - assert_eq!(x_canonical, canonical_raw(e, desc).unwrap()); - } -} - -/// Syntax sugar for `test_encode`, -/// because `test_encode(rng,&rng::gen())` doesn't compile. -#[track_caller] -pub fn test_encode_random(rng: &mut R) -where - Standard: Distribution, -{ - let msg = rng.gen::(); - test_encode(rng, &msg); -} - -/// shuffles recursively the order of fields in a protobuf encoding. -fn encode_shuffled_raw( - rng: &mut R, - buf: &[u8], - desc: &prost_reflect::MessageDescriptor, -) -> anyhow::Result> { - // Vector of field values, values of each field are in reverse order. - // This way we can pop the next value of the field in O(1). - let mut fields: Vec<_> = read_fields(buf, desc)?.into_iter().collect(); - for f in &mut fields { - f.1.reverse(); - } - - // Append fields in random order. - let mut v = vec![]; - let mut w = quick_protobuf::Writer::new(&mut v); - while !fields.is_empty() { - // Select a random field. - let i = rng.gen_range(0..fields.len()); - // Select the next value of this field. - // Note that the values are stored in reverse order. - let Some(mut v) = fields[i].1.pop() else { - let last = fields.len() - 1; - fields.swap(i, last); - fields.pop(); - continue; - }; - let num = fields[i].0; - let fd = desc.get_field(num).unwrap(); - if let prost_reflect::Kind::Message(desc) = &fd.kind() { - v = encode_shuffled_raw(rng, &v, desc)?; - } - let wire = Wire::from(fd.kind()); - w.write_tag(num << 3 | wire.raw()).unwrap(); - match wire { - Wire::Varint | Wire::I64 | Wire::I32 => { - // inefficient workaround of the fact that quick_protobuf::Writer - // doesn't support just appending a sequence of bytes. - for b in &v { - w.write_u8(*b).unwrap(); - } - } - Wire::Len => w.write_bytes(&v).unwrap(), - } - } - Ok(v) -} - -/// Encodes a proto message of known schema in a random encoding order of fields. -pub(crate) fn encode_shuffled(rng: &mut R, x: &T) -> Vec { - let msg = x.build(); - encode_shuffled_raw(rng, &msg.encode_to_vec(), &msg.descriptor()).unwrap() -} diff --git a/node/libs/protobuf/utils/src/tests.rs b/node/libs/protobuf/utils/src/tests.rs deleted file mode 100644 index d4d8366e..00000000 --- a/node/libs/protobuf/utils/src/tests.rs +++ /dev/null @@ -1,105 +0,0 @@ -use super::*; -use anyhow::Context as _; -use concurrency::{ctx, time}; -use std::net; - -#[derive(Debug, PartialEq, Eq)] -enum B { - U(bool), - V(Box), -} - -impl ProtoFmt for B { - type Proto = proto::testonly::B; - fn read(r: &Self::Proto) -> anyhow::Result { - use proto::testonly::b::T; - Ok(match required(&r.t)? { - T::U(x) => Self::U(*x), - T::V(x) => Self::V(Box::new(ProtoFmt::read(x.as_ref())?)), - }) - } - fn build(&self) -> Self::Proto { - use proto::testonly::b::T; - let t = match self { - Self::U(x) => T::U(*x), - Self::V(x) => T::V(Box::new(x.build())), - }; - Self::Proto { t: Some(t) } - } -} - -#[derive(Debug, PartialEq, Eq)] -struct A { - x: Vec, - y: u64, - e: Vec, - b: B, -} - -impl ProtoFmt for A { - type Proto = proto::testonly::A; - fn read(r: &Self::Proto) -> anyhow::Result { - Ok(Self { - x: required(&r.x).context("x")?.clone(), - y: *required(&r.y).context("y")?, - e: r.e.clone(), - b: read_required(&r.b).context("b")?, - }) - } - - fn build(&self) -> Self::Proto { - Self::Proto { - x: Some(self.x.clone()), - y: Some(self.y), - e: self.e.clone(), - b: Some(self.b.build()), - } - } -} - -#[test] -fn test_encode_decode() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - let v = A { - x: vec![1, 2, 3], - y: 76, - e: vec![8, 1, 9, 4], - b: B::U(true), - }; - testonly::test_encode(rng, &v); - - // Decoding should fail if there are trailing bytes present. - let mut bytes = encode(&v); - bytes.extend([4, 4, 3]); - assert!(decode::(&bytes).is_err()); -} - -#[test] -fn test_timestamp() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - let unix = time::UNIX_EPOCH; - let zero = time::Duration::ZERO; - let nano = time::Duration::nanoseconds(1); - let sec = time::Duration::seconds(1); - for d in [zero, nano, sec, 12345678 * sec - 3 * nano] { - testonly::test_encode(rng, &(unix + d)); - testonly::test_encode(rng, &(unix - d)); - } -} - -#[test] -fn test_socket_addr() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - for addr in [ - "[::1]:432", - "74.223.12.1:63222", - "127.0.0.1:0", - "0.0.0.0:33", - ] { - let addr: net::SocketAddr = addr.parse().unwrap(); - testonly::test_encode(rng, &addr); - } -} diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index 8432c2aa..0e4b4eb8 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -14,7 +14,7 @@ rand.workspace = true serde.workspace = true tracing.workspace = true -protobuf_utils.workspace = true +protobuf.workspace = true concurrency = { path = "../concurrency" } crypto = { path = "../crypto" } schema = { path = "../schema" } diff --git a/node/libs/roles/src/node/conv.rs b/node/libs/roles/src/node/conv.rs index f560ba09..ace73f67 100644 --- a/node/libs/roles/src/node/conv.rs +++ b/node/libs/roles/src/node/conv.rs @@ -1,5 +1,5 @@ use crate::node; -use protobuf_utils::{read_required, required, ProtoFmt}; +use protobuf::{read_required, required, ProtoFmt}; use anyhow::Context as _; use crypto::ByteFmt; use utils::enum_util::Variant; diff --git a/node/libs/roles/src/node/messages.rs b/node/libs/roles/src/node/messages.rs index 99f68819..34384d54 100644 --- a/node/libs/roles/src/node/messages.rs +++ b/node/libs/roles/src/node/messages.rs @@ -17,7 +17,7 @@ pub enum Msg { impl Msg { /// Get the hash of this message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&protobuf_utils::canonical(self))) + MsgHash(sha256::Sha256::new(&protobuf::canonical(self))) } } diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index 5d7c8fec..46a4f582 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -5,7 +5,7 @@ use super::{ Signers, ViewNumber, }; use crate::node::SessionId; -use protobuf_utils::{read_required, required, ProtoFmt}; +use protobuf::{read_required, required, ProtoFmt}; use anyhow::Context as _; use crypto::ByteFmt; use schema::proto::roles::validator as proto; @@ -187,7 +187,7 @@ impl ProtoFmt for LeaderCommit { } impl ProtoFmt for Signers { - type Proto = protobuf_utils::proto::std::BitVector; + type Proto = protobuf::proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(ProtoFmt::read(r)?)) @@ -266,8 +266,8 @@ impl ProtoFmt for Phase { fn build(&self) -> Self::Proto { use proto::phase::T; let t = match self { - Self::Prepare => T::Prepare(protobuf_utils::proto::std::Void {}), - Self::Commit => T::Commit(protobuf_utils::proto::std::Void {}), + Self::Prepare => T::Prepare(protobuf::proto::std::Void {}), + Self::Commit => T::Commit(protobuf::proto::std::Void {}), }; Self::Proto { t: Some(t) } } diff --git a/node/libs/roles/src/validator/messages/block.rs b/node/libs/roles/src/validator/messages/block.rs index b3a83018..5ae0c09e 100644 --- a/node/libs/roles/src/validator/messages/block.rs +++ b/node/libs/roles/src/validator/messages/block.rs @@ -97,7 +97,7 @@ pub struct BlockHeader { impl BlockHeader { /// Returns the hash of the block. pub fn hash(&self) -> BlockHeaderHash { - BlockHeaderHash(sha256::Sha256::new(&protobuf_utils::canonical(self))) + BlockHeaderHash(sha256::Sha256::new(&protobuf::canonical(self))) } /// Creates a genesis block. @@ -145,10 +145,10 @@ impl FinalBlock { impl ByteFmt for FinalBlock { fn decode(bytes: &[u8]) -> anyhow::Result { - protobuf_utils::decode(bytes) + protobuf::decode(bytes) } fn encode(&self) -> Vec { - protobuf_utils::encode(self) + protobuf::encode(self) } } diff --git a/node/libs/roles/src/validator/messages/msg.rs b/node/libs/roles/src/validator/messages/msg.rs index e8a710cc..9012be6a 100644 --- a/node/libs/roles/src/validator/messages/msg.rs +++ b/node/libs/roles/src/validator/messages/msg.rs @@ -20,7 +20,7 @@ pub enum Msg { impl Msg { /// Returns the hash of the message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&protobuf_utils::canonical(self))) + MsgHash(sha256::Sha256::new(&protobuf::canonical(self))) } } diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index e68f4c39..b9a33e99 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -18,11 +18,10 @@ rand.workspace = true serde_json.workspace = true tokio.workspace = true -protobuf_build = { path = "../protobuf/build" } -protobuf_utils = { path = "../protobuf/utils" } +protobuf.workspace = true concurrency = { path = "../concurrency" } [build-dependencies] +protobuf.workspace = true + anyhow.workspace = true -protobuf_build = { path = "../protobuf/build" } -protobuf_utils = { path = "../protobuf/utils" } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index dd92dbef..67a88d25 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -6,6 +6,6 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf_build::compile(&proto_include,"proto", &[&protobuf_utils::proto::DESCRIPTOR])?; + protobuf::build::compile(&proto_include,"proto", &[&protobuf::proto::DESCRIPTOR])?; Ok(()) } diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index 1b93660a..8dc8096b 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -14,7 +14,7 @@ rocksdb.workspace = true thiserror.workspace = true tracing.workspace = true -protobuf_utils.workspace = true +protobuf.workspace = true concurrency = { path = "../concurrency" } roles = { path = "../roles" } schema = { path = "../schema" } diff --git a/node/libs/storage/src/rocksdb.rs b/node/libs/storage/src/rocksdb.rs index bb34c0c4..1de4f6fe 100644 --- a/node/libs/storage/src/rocksdb.rs +++ b/node/libs/storage/src/rocksdb.rs @@ -100,7 +100,7 @@ impl RocksdbStorage { .next() .context("Head block not found")? .context("RocksDB error reading head block")?; - protobuf_utils::decode(&head_block).context("Failed decoding head block bytes") + protobuf::decode(&head_block).context("Failed decoding head block bytes") } /// Returns a block with the least number stored in this database. @@ -114,7 +114,7 @@ impl RocksdbStorage { .next() .context("First stored block not found")? .context("RocksDB error reading first stored block")?; - protobuf_utils::decode(&first_block).context("Failed decoding first stored block bytes") + protobuf::decode(&first_block).context("Failed decoding first stored block bytes") } fn last_contiguous_block_number_blocking(&self) -> anyhow::Result { @@ -174,7 +174,7 @@ impl RocksdbStorage { else { return Ok(None); }; - let block = protobuf_utils::decode(&raw_block) + let block = protobuf::decode(&raw_block) .with_context(|| format!("Failed decoding block #{number}"))?; Ok(Some(block)) } @@ -213,7 +213,7 @@ impl RocksdbStorage { let mut write_batch = rocksdb::WriteBatch::default(); write_batch.put( DatabaseKey::Block(block_number).encode_key(), - protobuf_utils::encode(finalized_block), + protobuf::encode(finalized_block), ); // Commit the transaction. db.write(write_batch) @@ -232,7 +232,7 @@ impl RocksdbStorage { else { return Ok(None); }; - protobuf_utils::decode(&raw_state) + protobuf::decode(&raw_state) .map(Some) .context("Failed to decode replica state!") } @@ -241,7 +241,7 @@ impl RocksdbStorage { self.write() .put( DatabaseKey::ReplicaState.encode_key(), - protobuf_utils::encode(replica_state), + protobuf::encode(replica_state), ) .context("Failed putting ReplicaState to RocksDB") } diff --git a/node/libs/storage/src/types.rs b/node/libs/storage/src/types.rs index 01190af1..ce3d2517 100644 --- a/node/libs/storage/src/types.rs +++ b/node/libs/storage/src/types.rs @@ -4,7 +4,7 @@ use concurrency::ctx; use rocksdb::{Direction, IteratorMode}; use roles::validator::{self, BlockNumber}; use schema::proto::storage as proto; -use protobuf_utils::{read_required, required, ProtoFmt}; +use protobuf::{read_required, required, ProtoFmt}; use std::{iter, ops}; use thiserror::Error; diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 25cb9326..11bc7079 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -13,7 +13,7 @@ rand.workspace = true serde_json.workspace = true clap.workspace = true -protobuf_utils.workspace = true +protobuf.workspace = true crypto = { path = "../libs/crypto" } roles = { path = "../libs/roles" } schema = { path = "../libs/schema" } diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 4b58f9d6..6673bd3d 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -108,7 +108,7 @@ fn main() -> anyhow::Result<()> { let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; - fs::write(root.join("config.json"), protobuf_utils::encode_json(&node_cfg)) + fs::write(root.join("config.json"), protobuf::encode_json(&node_cfg)) .context("fs::write()")?; fs::write( root.join("validator_key"), From 679a1d59b668a89ec7634b04ceb953a3f5a1b76e Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 31 Oct 2023 13:50:41 +0100 Subject: [PATCH 07/40] added missing files --- node/libs/protobuf/Cargo.toml | 30 ++ node/libs/protobuf/build.rs | 11 + .../proto/conformance/conformance.proto | 183 ++++++++++ .../conformance/test_messages_proto3.proto | 225 ++++++++++++ node/libs/protobuf/proto/std.proto | 50 +++ node/libs/protobuf/proto/testonly.proto | 27 ++ .../libs/protobuf/src/bin/conformance_test.rs | 88 +++++ .../src/bin/conformance_test_failure_list.txt | 11 + node/libs/protobuf/src/lib.rs | 18 + node/libs/protobuf/src/proto_fmt.rs | 329 ++++++++++++++++++ node/libs/protobuf/src/std_conv.rs | 103 ++++++ node/libs/protobuf/src/testonly.rs | 89 +++++ node/libs/protobuf/src/tests.rs | 105 ++++++ node/libs/protobuf_build/Cargo.toml | 26 ++ node/libs/protobuf_build/src/lib.rs | 230 ++++++++++++ 15 files changed, 1525 insertions(+) create mode 100644 node/libs/protobuf/Cargo.toml create mode 100644 node/libs/protobuf/build.rs create mode 100644 node/libs/protobuf/proto/conformance/conformance.proto create mode 100644 node/libs/protobuf/proto/conformance/test_messages_proto3.proto create mode 100644 node/libs/protobuf/proto/std.proto create mode 100644 node/libs/protobuf/proto/testonly.proto create mode 100644 node/libs/protobuf/src/bin/conformance_test.rs create mode 100644 node/libs/protobuf/src/bin/conformance_test_failure_list.txt create mode 100644 node/libs/protobuf/src/lib.rs create mode 100644 node/libs/protobuf/src/proto_fmt.rs create mode 100644 node/libs/protobuf/src/std_conv.rs create mode 100644 node/libs/protobuf/src/testonly.rs create mode 100644 node/libs/protobuf/src/tests.rs create mode 100644 node/libs/protobuf_build/Cargo.toml create mode 100644 node/libs/protobuf_build/src/lib.rs diff --git a/node/libs/protobuf/Cargo.toml b/node/libs/protobuf/Cargo.toml new file mode 100644 index 00000000..71cc9f50 --- /dev/null +++ b/node/libs/protobuf/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "protobuf" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[[bin]] +name = "conformance_test" + +[dependencies] +anyhow.workspace = true +bit-vec.workspace = true +serde.workspace = true +quick-protobuf.workspace = true +prost.workspace = true +prost-reflect.workspace = true +rand.workspace = true +serde_json.workspace = true +tokio.workspace = true +once_cell.workspace = true + +protobuf_build.workspace = true +concurrency = { path = "../concurrency" } + +[build-dependencies] +protobuf_build.workspace = true + +anyhow.workspace = true diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs new file mode 100644 index 00000000..01ab9e3d --- /dev/null +++ b/node/libs/protobuf/build.rs @@ -0,0 +1,11 @@ +//! Generates rust code from the capnp schema files in the `capnp/` directory. +use std::{env, path::PathBuf}; + +fn main() -> anyhow::Result<()> { + // Prepare input and output root dirs. + let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) + .canonicalize()? + .join("proto"); + protobuf_build::compile(&proto_include,"proto",&[])?; + Ok(()) +} diff --git a/node/libs/protobuf/proto/conformance/conformance.proto b/node/libs/protobuf/proto/conformance/conformance.proto new file mode 100644 index 00000000..e6403673 --- /dev/null +++ b/node/libs/protobuf/proto/conformance/conformance.proto @@ -0,0 +1,183 @@ +// Modified copy of +// https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto +// +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package conformance; + +option java_package = "com.google.protobuf.conformance"; +option objc_class_prefix = "Conformance"; + +// This defines the conformance testing protocol. This protocol exists between +// the conformance test suite itself and the code being tested. For each test, +// the suite will send a ConformanceRequest message and expect a +// ConformanceResponse message. +// +// You can either run the tests in two different ways: +// +// 1. in-process (using the interface in conformance_test.h). +// +// 2. as a sub-process communicating over a pipe. Information about how to +// do this is in conformance_test_runner.cc. +// +// Pros/cons of the two approaches: +// +// - running as a sub-process is much simpler for languages other than C/C++. +// +// - running as a sub-process may be more tricky in unusual environments like +// iOS apps, where fork/stdin/stdout are not available. + +enum WireFormat { + UNSPECIFIED = 0; + PROTOBUF = 1; + JSON = 2; + JSPB = 3; // Only used inside Google. Opensource testees just skip it. + TEXT_FORMAT = 4; +} + +enum TestCategory { + UNSPECIFIED_TEST = 0; + BINARY_TEST = 1; // Test binary wire format. + JSON_TEST = 2; // Test json wire format. + // Similar to JSON_TEST. However, during parsing json, testee should ignore + // unknown fields. This feature is optional. Each implementation can decide + // whether to support it. See + // https://developers.google.com/protocol-buffers/docs/proto3#json_options + // for more detail. + JSON_IGNORE_UNKNOWN_PARSING_TEST = 3; + // Test jspb wire format. Only used inside Google. Opensource testees just + // skip it. + JSPB_TEST = 4; + // Test text format. For cpp, java and python, testees can already deal with + // this type. Testees of other languages can simply skip it. + TEXT_FORMAT_TEST = 5; +} + +// The conformance runner will request a list of failures as the first request. +// This will be known by message_type == "conformance.FailureSet", a conformance +// test should return a serialized FailureSet in protobuf_payload. +message FailureSet { + repeated string failure = 1; +} + +// Represents a single test case's input. The testee should: +// +// 1. parse this proto (which should always succeed) +// 2. parse the protobuf or JSON payload in "payload" (which may fail) +// 3. if the parse succeeded, serialize the message in the requested format. +message ConformanceRequest { + // The payload (whether protobuf of JSON) is always for a + // protobuf_test_messages.proto3.TestAllTypes proto (as defined in + // src/google/protobuf/proto3_test_messages.proto). + oneof payload { + bytes protobuf_payload = 1; + string json_payload = 2; + // Only used inside Google. Opensource testees just skip it. + string jspb_payload = 7; + string text_payload = 8; + } + + // Which format should the testee serialize its message to? + optional WireFormat requested_output_format = 3; + + // The full name for the test message to use; for the moment, either: + // protobuf_test_messages.proto3.TestAllTypesProto3 or + // protobuf_test_messages.google.protobuf.TestAllTypesProto2. + optional string message_type = 4; + + // Each test is given a specific test category. Some category may need + // specific support in testee programs. Refer to the definition of + // TestCategory for more information. + optional TestCategory test_category = 5; + + // Specify details for how to encode jspb. + optional JspbEncodingConfig jspb_encoding_options = 6; + + // This can be used in json and text format. If true, testee should print + // unknown fields instead of ignore. This feature is optional. + optional bool print_unknown_fields = 9; +} + +// Represents a single test case's output. +message ConformanceResponse { + oneof result { + // This string should be set to indicate parsing failed. The string can + // provide more information about the parse error if it is available. + // + // Setting this string does not necessarily mean the testee failed the + // test. Some of the test cases are intentionally invalid input. + string parse_error = 1; + + // If the input was successfully parsed but errors occurred when + // serializing it to the requested output format, set the error message in + // this field. + string serialize_error = 6; + + // This should be set if the test program timed out. The string should + // provide more information about what the child process was doing when it + // was killed. + string timeout_error = 9; + + // This should be set if some other error occurred. This will always + // indicate that the test failed. The string can provide more information + // about the failure. + string runtime_error = 2; + + // If the input was successfully parsed and the requested output was + // protobuf, serialize it to protobuf and set it in this field. + bytes protobuf_payload = 3; + + // If the input was successfully parsed and the requested output was JSON, + // serialize to JSON and set it in this field. + string json_payload = 4; + + // For when the testee skipped the test, likely because a certain feature + // wasn't supported, like JSON input/output. + string skipped = 5; + + // If the input was successfully parsed and the requested output was JSPB, + // serialize to JSPB and set it in this field. JSPB is only used inside + // Google. Opensource testees can just skip it. + string jspb_payload = 7; + + // If the input was successfully parsed and the requested output was + // TEXT_FORMAT, serialize to TEXT_FORMAT and set it in this field. + string text_payload = 8; + } +} + +// Encoding options for jspb format. +message JspbEncodingConfig { + // Encode the value field of Any as jspb array if true, otherwise binary. + optional bool use_jspb_array_any_format = 1; +} diff --git a/node/libs/protobuf/proto/conformance/test_messages_proto3.proto b/node/libs/protobuf/proto/conformance/test_messages_proto3.proto new file mode 100644 index 00000000..d391761b --- /dev/null +++ b/node/libs/protobuf/proto/conformance/test_messages_proto3.proto @@ -0,0 +1,225 @@ +// Modified copy of +// https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/test_messages_proto3.proto +// +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Test schema for proto3 messages. This test schema is used by: +// +// - benchmarks +// - fuzz tests +// - conformance tests +// + +syntax = "proto3"; + +package protobuf_test_messages.proto3; + +option java_package = "com.google.protobuf_test_messages.proto3"; +option objc_class_prefix = "Proto3"; + +// This is the default, but we specify it here explicitly. +option optimize_for = SPEED; + +option cc_enable_arenas = true; + +// This proto includes every type of field in both singular and repeated +// forms. +// +// Also, crucially, all messages and enums in this file are eventually +// submessages of this message. So for example, a fuzz test of TestAllTypes +// could trigger bugs that occur in any message type in this file. We verify +// this stays true in a unit test. +message TestAllTypesProto3 { + message NestedMessage { + optional int32 a = 1; + optional TestAllTypesProto3 corecursive = 2; + } + + enum NestedEnum { + FOO = 0; + BAR = 1; + BAZ = 2; + NEG = -1; // Intentionally negative. + } + + enum AliasedEnum { + option allow_alias = true; + + ALIAS_FOO = 0; + ALIAS_BAR = 1; + ALIAS_BAZ = 2; + MOO = 2; + moo = 2; + bAz = 2; + } + + // Singular + optional int32 optional_int32 = 1; + optional int64 optional_int64 = 2; + optional uint32 optional_uint32 = 3; + optional uint64 optional_uint64 = 4; + optional sint32 optional_sint32 = 5; + optional sint64 optional_sint64 = 6; + optional fixed32 optional_fixed32 = 7; + optional fixed64 optional_fixed64 = 8; + optional sfixed32 optional_sfixed32 = 9; + optional sfixed64 optional_sfixed64 = 10; + optional float optional_float = 11; + optional double optional_double = 12; + optional bool optional_bool = 13; + optional string optional_string = 14; + optional bytes optional_bytes = 15; + + optional NestedMessage optional_nested_message = 18; + optional ForeignMessage optional_foreign_message = 19; + + optional NestedEnum optional_nested_enum = 21; + optional ForeignEnum optional_foreign_enum = 22; + optional AliasedEnum optional_aliased_enum = 23; + + optional string optional_string_piece = 24 [ctype = STRING_PIECE]; + optional string optional_cord = 25 [ctype = CORD]; + + optional TestAllTypesProto3 recursive_message = 27; + + // Repeated + repeated int32 repeated_int32 = 31; + repeated int64 repeated_int64 = 32; + repeated uint32 repeated_uint32 = 33; + repeated uint64 repeated_uint64 = 34; + repeated sint32 repeated_sint32 = 35; + repeated sint64 repeated_sint64 = 36; + repeated fixed32 repeated_fixed32 = 37; + repeated fixed64 repeated_fixed64 = 38; + repeated sfixed32 repeated_sfixed32 = 39; + repeated sfixed64 repeated_sfixed64 = 40; + repeated float repeated_float = 41; + repeated double repeated_double = 42; + repeated bool repeated_bool = 43; + repeated string repeated_string = 44; + repeated bytes repeated_bytes = 45; + + repeated NestedMessage repeated_nested_message = 48; + repeated ForeignMessage repeated_foreign_message = 49; + + repeated NestedEnum repeated_nested_enum = 51; + repeated ForeignEnum repeated_foreign_enum = 52; + + repeated string repeated_string_piece = 54 [ctype = STRING_PIECE]; + repeated string repeated_cord = 55 [ctype = CORD]; + + // Packed + repeated int32 packed_int32 = 75 [packed = true]; + repeated int64 packed_int64 = 76 [packed = true]; + repeated uint32 packed_uint32 = 77 [packed = true]; + repeated uint64 packed_uint64 = 78 [packed = true]; + repeated sint32 packed_sint32 = 79 [packed = true]; + repeated sint64 packed_sint64 = 80 [packed = true]; + repeated fixed32 packed_fixed32 = 81 [packed = true]; + repeated fixed64 packed_fixed64 = 82 [packed = true]; + repeated sfixed32 packed_sfixed32 = 83 [packed = true]; + repeated sfixed64 packed_sfixed64 = 84 [packed = true]; + repeated float packed_float = 85 [packed = true]; + repeated double packed_double = 86 [packed = true]; + repeated bool packed_bool = 87 [packed = true]; + repeated NestedEnum packed_nested_enum = 88 [packed = true]; + + // Unpacked + repeated int32 unpacked_int32 = 89 [packed = false]; + repeated int64 unpacked_int64 = 90 [packed = false]; + repeated uint32 unpacked_uint32 = 91 [packed = false]; + repeated uint64 unpacked_uint64 = 92 [packed = false]; + repeated sint32 unpacked_sint32 = 93 [packed = false]; + repeated sint64 unpacked_sint64 = 94 [packed = false]; + repeated fixed32 unpacked_fixed32 = 95 [packed = false]; + repeated fixed64 unpacked_fixed64 = 96 [packed = false]; + repeated sfixed32 unpacked_sfixed32 = 97 [packed = false]; + repeated sfixed64 unpacked_sfixed64 = 98 [packed = false]; + repeated float unpacked_float = 99 [packed = false]; + repeated double unpacked_double = 100 [packed = false]; + repeated bool unpacked_bool = 101 [packed = false]; + repeated NestedEnum unpacked_nested_enum = 102 [packed = false]; + + oneof oneof_field { + uint32 oneof_uint32 = 111; + NestedMessage oneof_nested_message = 112; + string oneof_string = 113; + bytes oneof_bytes = 114; + bool oneof_bool = 115; + uint64 oneof_uint64 = 116; + float oneof_float = 117; + double oneof_double = 118; + NestedEnum oneof_enum = 119; + } + + // Test field-name-to-JSON-name convention. + // (protobuf says names can be any valid C/C++ identifier.) + optional int32 fieldname1 = 401; + optional int32 field_name2 = 402; + optional int32 _field_name3 = 403; + optional int32 field__name4_ = 404; + optional int32 field0name5 = 405; + optional int32 field_0_name6 = 406; + optional int32 fieldName7 = 407; + optional int32 FieldName8 = 408; + optional int32 field_Name9 = 409; + optional int32 Field_Name10 = 410; + optional int32 FIELD_NAME11 = 411; + optional int32 FIELD_name12 = 412; + optional int32 __field_name13 = 413; + optional int32 __Field_name14 = 414; + optional int32 field__name15 = 415; + optional int32 field__Name16 = 416; + optional int32 field_name17__ = 417; + optional int32 Field_name18__ = 418; + + // Reserved for testing unknown fields + reserved 501 to 510; +} + +message ForeignMessage { + optional int32 c = 1; +} + +enum ForeignEnum { + FOREIGN_FOO = 0; + FOREIGN_BAR = 1; + FOREIGN_BAZ = 2; +} + +message NullHypothesisProto3 {} + +message EnumOnlyProto3 { + enum Bool { + kFalse = 0; + kTrue = 1; + } +} diff --git a/node/libs/protobuf/proto/std.proto b/node/libs/protobuf/proto/std.proto new file mode 100644 index 00000000..f5ee5efe --- /dev/null +++ b/node/libs/protobuf/proto/std.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package std; + +message Void {} + +// Timestamp represented as seconds + nanoseconds since UNIX epoch. +// Equivalent of `google.protobuf.Timestamp` but supports canonical encoding. +// See `google.protobuf.Timestamp` for more detailed specification. +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + optional int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + optional int32 nanos = 2; +} + +message Duration { + optional int64 seconds = 1; + optional int32 nanos = 2; +} + +// IP:port TCP address. +message SocketAddr { + // Ipv4 (4 bytes) or IPv6 (16 bytes) in network byte order. + optional bytes ip = 1; + // TCP port (actually uint16, however uint32 is smallest supported protobuf type). + optional uint32 port = 2; +} + +// Compressed representation of a vector of bits. +// It occupies n/8 + O(1) bytes. +// Note: this is not a highly optimized data structure: +// - since it is a proto message it requires its length to be stored. +// - it has 2 fields, which require storing their own tag +// - the `bytes_` field requires storing its length, which theoretically +// could be derived from `size` field value. +// If we eventually start caring about the size, we could encode BitVector +// directly as a field of type `bytes` with delimiter of the form "01...1". +message BitVector { + // Number of bits in the vector. + optional uint64 size = 1; + // Vector of bits encoded as bytes in big endian order. + optional bytes bytes_ = 2; +} diff --git a/node/libs/protobuf/proto/testonly.proto b/node/libs/protobuf/proto/testonly.proto new file mode 100644 index 00000000..7c5d08cd --- /dev/null +++ b/node/libs/protobuf/proto/testonly.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package testonly; + +message B { + // Recursive union. + oneof t { + bool u = 2; + B v = 3; + } +} + +enum E { + E0 = 0; + E1 = 1; + E2 = 2; +} + +message A { + optional bytes x = 1; + optional uint64 y = 2; + // Repeated enums are non-packable, + // But we pack them in the canonical encoding. + repeated E e = 4; + // Nested message. + optional B b = 5; +} diff --git a/node/libs/protobuf/src/bin/conformance_test.rs b/node/libs/protobuf/src/bin/conformance_test.rs new file mode 100644 index 00000000..32e5510b --- /dev/null +++ b/node/libs/protobuf/src/bin/conformance_test.rs @@ -0,0 +1,88 @@ +//! Conformance test for our canonical encoding implemented according to +//! https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto +//! Our implementation supports only a subset of proto functionality, so +//! `schema/proto/conformance/conformance.proto` and +//! `schema/proto/conformance/protobuf_test_messages.proto` contains only a +//! subset of original fields. Also we run only proto3 binary -> binary tests. +//! conformance_test_failure_list.txt contains tests which are expected to fail. +use anyhow::Context as _; +use concurrency::{ctx, io}; +use prost::Message as _; +use prost_reflect::ReflectMessage; +use protobuf::proto::conformance as proto; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let ctx = &ctx::root(); + let stdin = &mut tokio::io::stdin(); + let stdout = &mut tokio::io::stdout(); + loop { + // Read the request. + let mut msg_size = [0u8; 4]; + if io::read_exact(ctx, stdin, &mut msg_size).await?.is_err() { + return Ok(()); + } + let msg_size = u32::from_le_bytes(msg_size); + let mut msg = vec![0u8; msg_size as usize]; + io::read_exact(ctx, stdin, &mut msg[..]).await??; + + use proto::conformance_response::Result as R; + let req = proto::ConformanceRequest::decode(&msg[..])?; + let res = async { + let t = req.message_type.context("missing message_type")?; + if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { + return Ok(R::Skipped("unsupported".to_string())); + } + + // Decode. + let payload = req.payload.context("missing payload")?; + use protobuf::proto::protobuf_test_messages::proto3::TestAllTypesProto3 as T; + let p = match payload { + proto::conformance_request::Payload::JsonPayload(payload) => { + match protobuf::decode_json_proto(&payload) { + Ok(p) => p, + Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), + } + } + proto::conformance_request::Payload::ProtobufPayload(payload) => { + // First filter out incorrect encodings. + let Ok(p) = T::decode(&payload[..]) else { + return Ok(R::ParseError("parsing failed".to_string())); + }; + // Then check if there are any unknown fields in the original payload. + if protobuf::canonical_raw(&payload[..], &p.descriptor()).is_err() { + return Ok(R::Skipped("unsupported fields".to_string())); + } + p + } + _ => return Ok(R::Skipped("unsupported input format".to_string())), + }; + + // Encode. + let format = req + .requested_output_format + .context("missing output format")?; + match proto::WireFormat::from_i32(format).context("unknown format")? { + proto::WireFormat::Json => { + anyhow::Ok(R::JsonPayload(protobuf::encode_json_proto(&p))) + } + proto::WireFormat::Protobuf => { + // Reencode the parsed proto. + anyhow::Ok(R::ProtobufPayload(protobuf::canonical_raw( + &p.encode_to_vec(), + &p.descriptor(), + )?)) + } + _ => Ok(R::Skipped("unsupported output format".to_string())), + } + } + .await?; + let resp = proto::ConformanceResponse { result: Some(res) }; + + // Write the response. + let msg = resp.encode_to_vec(); + io::write_all(ctx, stdout, &u32::to_le_bytes(msg.len() as u32)).await??; + io::write_all(ctx, stdout, &msg).await??; + io::flush(ctx, stdout).await??; + } +} diff --git a/node/libs/protobuf/src/bin/conformance_test_failure_list.txt b/node/libs/protobuf/src/bin/conformance_test_failure_list.txt new file mode 100644 index 00000000..82ecacee --- /dev/null +++ b/node/libs/protobuf/src/bin/conformance_test_failure_list.txt @@ -0,0 +1,11 @@ +# We have added "optional" annotation to some of the fields in the test proto +# (which is required by our canonical encoding). For those fields json encoder +# cannot deduce the intended default values, which breaks the following test: +Required.Proto3.JsonInput.SkipsDefaultPrimitive.Validator + +# Json decoder slightly rounds the extreme values for f64 when parsing (it is a bug). +# This makes the following tests fail: +Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.JsonOutput +Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput +Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.JsonOutput +Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.ProtobufOutput diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs new file mode 100644 index 00000000..4f1db3a6 --- /dev/null +++ b/node/libs/protobuf/src/lib.rs @@ -0,0 +1,18 @@ +//! Code generated from protobuf schema files and +//! utilities for serialization. + +mod proto_fmt; +mod std_conv; +pub mod testonly; + +pub use proto_fmt::*; +pub use protobuf_build as build; + +#[cfg(test)] +mod tests; + +#[allow(warnings)] +pub mod proto { + use crate as protobuf; + include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); +} diff --git a/node/libs/protobuf/src/proto_fmt.rs b/node/libs/protobuf/src/proto_fmt.rs new file mode 100644 index 00000000..541b0f0b --- /dev/null +++ b/node/libs/protobuf/src/proto_fmt.rs @@ -0,0 +1,329 @@ +//! Canonical encoding for protobufs with known schema. +//! +//! Protobuf encoding spec: https://protobuf.dev/programming-guides/encoding/ +//! By default the encoding is not canonical (multiple encodings may represent the same message). +//! Here we extend this spec to support a canonical encoding. +//! +//! Canonical encoding spec: +//! * fields are encoded in ascending order of tags. +//! * varints are encoded using minimal amount of bytes. +//! * map fields are not supported +//! * groups are not supported +//! * only proto3 syntax is supported +//! * hence assigning a field a default value (proto2) is not supported +//! * all fields should have explicit presence (explicit optional or repeated) +//! * hence a default value for primitive types (proto3) is not supported. +//! * a field of primitive type (I32, I64, Varint) is encoded as at most 1 TLV, depending on +//! the number of values: +//! * 0 values => field is not encoded at all +//! * 1 value => field is encoded as a single I32/I64/Varint TLV. +//! * >1 values => field is encoded as a single TLV using packed encoding (https://protobuf.dev/programming-guides/encoding/#packed) +//! * protobuf spec allows for packed encoding only for repeated fields. We therefore don't +//! support specifying multiple values for a singular field (which is pathological case +//! anyway). +//! * protobuf spec also doesn't allow for encoding repeated enum fields (which are encoded as +//! Varint), however it mentions that it might allow for it in the future versions. +//! TODO(gprusak): decide what we should do with those: prost implementation decodes correctly +//! packed enums, I don't know about other protobuf implementations (rust and non-rust). We may either: +//! * allow packed encoding for enums (despite the incompatibility with protobuf spec) +//! * disallow packed encoding for enums specifically (although it complicates the canonical spec) +//! * drop packed encoding altogether (at the cost of larger messages) +//! +//! We need canonical encoding to be able to compute hashes and verify signatures on +//! messages. Note that with this spec it is not possible to encode canonically a message with unknown +//! fields. It won't be a problem though if we decide on one of the following: +//! * require all communication to just use canonical encoding: then as long as we carry around the +//! unknown fields untouched, they will stay canonical. +//! * hash/verify only messages that don't contain unknown fields: then we can just canonicalize +//! whatever we understand and reject messages with unknown fields. +//! * drop the idea of canonical encoding altogether and pass around the received encoded message +//! for hashing/verifying (i.e. keep the raw bytes together with the parsed message). +use anyhow::Context as _; +use prost::Message as _; +use prost_reflect::ReflectMessage; +use std::collections::BTreeMap; + +/// Kilobyte. +#[allow(non_upper_case_globals)] +pub const kB: usize = 1 << 10; +/// Megabyte. +pub const MB: usize = 1 << 20; + +/// Protobuf wire type. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Wire { + /// VARINT. + Varint, + /// I64 + I64, + /// LEN + Len, + /// I32 + I32, +} + +/// Raw write type values, as defined in +/// https://protobuf.dev/programming-guides/encoding/#structure +/// VARINT +const VARINT: u32 = 0; +/// I64 +const I64: u32 = 1; +/// LEN +const LEN: u32 = 2; +/// I32 +const I32: u32 = 5; + +impl Wire { + /// Extracts a wire type from a tag. + pub(crate) const fn from_tag(tag: u32) -> Option { + match tag & 7 { + VARINT => Some(Self::Varint), + I64 => Some(Self::I64), + LEN => Some(Self::Len), + I32 => Some(Self::I32), + _ => None, + } + } + + /// Converts wire type to the raw wite type value. + pub(crate) const fn raw(self) -> u32 { + match self { + Self::Varint => VARINT, + Self::I64 => I64, + Self::Len => LEN, + Self::I32 => I32, + } + } +} + +impl From for Wire { + fn from(kind: prost_reflect::Kind) -> Self { + use prost_reflect::Kind; + match kind { + Kind::Int32 + | Kind::Int64 + | Kind::Uint32 + | Kind::Uint64 + | Kind::Sint32 + | Kind::Sint64 + | Kind::Bool + | Kind::Enum(_) => Self::Varint, + Kind::Fixed64 | Kind::Sfixed64 | Kind::Double => Self::I64, + Kind::Fixed32 | Kind::Sfixed32 | Kind::Float => Self::I32, + Kind::String | Kind::Bytes | Kind::Message(_) => Self::Len, + } + } +} + +/// Wrapper of the quick_protobuf reader. +// TODO(gprusak): reimplement the reader and writer instead to make it more efficient. +pub(super) struct Reader<'a>(quick_protobuf::BytesReader, &'a [u8]); + +impl<'a> Reader<'a> { + /// Constructs a reader of a slice. + fn new(bytes: &'a [u8]) -> Self { + Self(quick_protobuf::BytesReader::from_bytes(bytes), bytes) + } + + /// Reads a value of the given wire type. + fn read(&mut self, wire: Wire) -> anyhow::Result> { + let mut v = vec![]; + let mut w = quick_protobuf::Writer::new(&mut v); + match wire { + Wire::Varint => w.write_varint(self.0.read_varint64(self.1)?).unwrap(), + Wire::I64 => w.write_fixed64(self.0.read_fixed64(self.1)?).unwrap(), + Wire::Len => return Ok(self.0.read_bytes(self.1)?.into()), + Wire::I32 => w.write_fixed32(self.0.read_fixed32(self.1)?).unwrap(), + } + Ok(v) + } + + /// Reads a (possibly packed) field based on the expected field wire type and the actual + /// wire type from the tag (which has been already read). + /// The values of the field are appended to `out`. + fn read_field( + &mut self, + out: &mut Vec>, + field_wire: Wire, + got_wire: Wire, + ) -> anyhow::Result<()> { + if got_wire == field_wire { + out.push(self.read(field_wire)?); + return Ok(()); + } + if got_wire != Wire::Len { + anyhow::bail!("unexpected wire type"); + } + let mut r = Self::new(self.0.read_bytes(self.1)?); + while !r.0.is_eof() { + out.push(r.read(field_wire)?); + } + Ok(()) + } +} + +/// Parses a message to a `field num -> values` map. +pub(super) fn read_fields( + buf: &[u8], + desc: &prost_reflect::MessageDescriptor, +) -> anyhow::Result>>> { + if desc.parent_file().syntax() != prost_reflect::Syntax::Proto3 { + anyhow::bail!("only proto3 syntax is supported"); + } + let mut r = Reader::new(buf); + let mut fields = BTreeMap::new(); + // Collect the field values from the reader. + while !r.0.is_eof() { + let tag = r.0.next_tag(r.1)?; + let wire = Wire::from_tag(tag).context("invalid wire type")?; + let field = desc.get_field(tag >> 3).context("unknown field")?; + if field.is_map() { + anyhow::bail!("maps unsupported"); + } + if !field.is_list() && !field.supports_presence() { + anyhow::bail!( + "{}::{} : fields with implicit presence are not supported", + field.parent_message().name(), + field.name() + ); + } + r.read_field( + fields.entry(field.number()).or_default(), + field.kind().into(), + wire, + )?; + } + Ok(fields) +} + +/// Converts an encoded protobuf message to its canonical form, given the descriptor of the message +/// type. Retuns an error if: +/// * an unknown field is detected +/// * the message type doesn't support canonical encoding (implicit presence, map fields) +pub fn canonical_raw( + buf: &[u8], + desc: &prost_reflect::MessageDescriptor, +) -> anyhow::Result> { + let mut v = vec![]; + let mut w = quick_protobuf::Writer::new(&mut v); + // Append fields in ascending tags order. + for (num, mut values) in read_fields(buf, desc)? { + let fd = desc.get_field(num).unwrap(); + if values.len() > 1 && !fd.is_list() { + anyhow::bail!("non-repeated field with multiple values"); + } + if let prost_reflect::Kind::Message(desc) = &fd.kind() { + for v in &mut values { + *v = canonical_raw(v, desc)?; + } + } + let wire = Wire::from(fd.kind()); + match wire { + Wire::Varint | Wire::I64 | Wire::I32 => { + if values.len() > 1 { + w.write_tag(num << 3 | LEN).unwrap(); + w.write_bytes(&values.into_iter().flatten().collect::>()) + .unwrap(); + } else { + w.write_tag(num << 3 | wire.raw()).unwrap(); + // inefficient workaround of the fact that quick_protobuf::Writer + // doesn't support just appending a sequence of bytes. + for b in &values[0] { + w.write_u8(*b).unwrap(); + } + } + } + Wire::Len => { + for v in &values { + w.write_tag(num << 3 | LEN).unwrap(); + w.write_bytes(v).unwrap(); + } + } + } + } + Ok(v) +} + +/// Encodes a proto message of known schema (no unknown fields) into a canonical form. +pub fn canonical(x: &T) -> Vec { + let msg = x.build(); + canonical_raw(&msg.encode_to_vec(), &msg.descriptor()).unwrap() +} + +/// Encodes a proto message. +/// Currently it outputs a canonical encoding, but `decode` accepts +/// non-canonical encoding as well. +/// It would simplify invariants if we required all messages to be canonical, +/// but we will see whether it is feasible once we have an RPC framework +/// in place. +pub fn encode(x: &T) -> Vec { + canonical(x) +} + +/// Decodes a proto message. +pub fn decode(bytes: &[u8]) -> anyhow::Result { + T::read(&::Proto::decode(bytes)?) +} + +/// Encodes a generated proto message to json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn encode_json_proto(x: &T) -> String { + let mut s = serde_json::Serializer::pretty(vec![]); + let opts = prost_reflect::SerializeOptions::new(); + x.transcode_to_dynamic() + .serialize_with_options(&mut s, &opts) + .unwrap(); + String::from_utf8(s.into_inner()).unwrap() +} + +/// Encodes a proto message to json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn encode_json(x: &T) -> String { + encode_json_proto(&x.build()) +} + +/// Decodes a generated proto message from json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn decode_json_proto(json: &str) -> anyhow::Result { + let mut d = serde_json::de::Deserializer::from_str(json); + let mut p = T::default(); + let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), &mut d)?; + d.end()?; + p.merge(msg.encode_to_vec().as_slice()).unwrap(); + Ok(p) +} + +/// Decodes a proto message from json. +/// WARNING: this function uses reflection, so it is not very efficient. +pub fn decode_json(json: &str) -> anyhow::Result { + T::read(&decode_json_proto(json)?) +} + +/// Trait defining a proto representation for a type. +pub trait ProtoFmt: Sized { + /// Proto message type representing Self. + type Proto: ReflectMessage + Default; + /// Converts Proto to Self. + fn read(r: &Self::Proto) -> anyhow::Result; + /// Converts Self to Proto. + fn build(&self) -> Self::Proto; + /// Maximal allowed message size (in bytes). + fn max_size() -> usize { + usize::MAX + } +} + +/// Parses a required proto field. +pub fn read_required(field: &Option) -> anyhow::Result { + ProtoFmt::read(field.as_ref().context("missing field")?) +} + +/// Parses an optional proto field. +pub fn read_optional(field: &Option) -> anyhow::Result> { + field.as_ref().map(ProtoFmt::read).transpose() +} + +/// Extracts a required field. +pub fn required(field: &Option) -> anyhow::Result<&T> { + field.as_ref().context("missing") +} diff --git a/node/libs/protobuf/src/std_conv.rs b/node/libs/protobuf/src/std_conv.rs new file mode 100644 index 00000000..2f2fa89b --- /dev/null +++ b/node/libs/protobuf/src/std_conv.rs @@ -0,0 +1,103 @@ +//! Proto conversion for messages in std package. +use crate::{proto::std as proto, required, ProtoFmt}; +use anyhow::Context as _; +use concurrency::time; +use std::net; + +impl ProtoFmt for () { + type Proto = proto::Void; + fn read(_r: &Self::Proto) -> anyhow::Result { + Ok(()) + } + fn build(&self) -> Self::Proto { + Self::Proto {} + } +} + +impl ProtoFmt for std::net::SocketAddr { + type Proto = proto::SocketAddr; + + fn read(r: &Self::Proto) -> anyhow::Result { + let ip = required(&r.ip).context("ip")?; + let ip = match ip.len() { + 4 => net::IpAddr::from(<[u8; 4]>::try_from(&ip[..]).unwrap()), + 16 => net::IpAddr::from(<[u8; 16]>::try_from(&ip[..]).unwrap()), + _ => anyhow::bail!("invalid ip length"), + }; + let port = *required(&r.port).context("port")?; + let port = u16::try_from(port).context("port")?; + Ok(Self::new(ip, port)) + } + + fn build(&self) -> Self::Proto { + Self::Proto { + ip: Some(match self.ip() { + net::IpAddr::V4(ip) => ip.octets().to_vec(), + net::IpAddr::V6(ip) => ip.octets().to_vec(), + }), + port: Some(self.port() as u32), + } + } +} + +impl ProtoFmt for time::Utc { + type Proto = proto::Timestamp; + + fn read(r: &Self::Proto) -> anyhow::Result { + let seconds = *required(&r.seconds).context("seconds")?; + let nanos = *required(&r.nanos).context("nanos")?; + Ok(time::UNIX_EPOCH + time::Duration::new(seconds, nanos)) + } + + fn build(&self) -> Self::Proto { + let d = (*self - time::UNIX_EPOCH).build(); + Self::Proto { + seconds: d.seconds, + nanos: d.nanos, + } + } +} + +impl ProtoFmt for time::Duration { + type Proto = proto::Duration; + + fn read(r: &Self::Proto) -> anyhow::Result { + let seconds = *required(&r.seconds).context("seconds")?; + let nanos = *required(&r.nanos).context("nanos")?; + Ok(Self::new(seconds, nanos)) + } + + fn build(&self) -> Self::Proto { + let mut seconds = self.whole_seconds(); + let mut nanos = self.subsec_nanoseconds(); + if nanos < 0 { + seconds -= 1; + nanos += 1_000_000_000; + } + Self::Proto { + seconds: Some(seconds), + nanos: Some(nanos), + } + } +} + +impl ProtoFmt for bit_vec::BitVec { + type Proto = proto::BitVector; + + fn read(r: &Self::Proto) -> anyhow::Result { + let size = *required(&r.size).context("size")? as usize; + let mut this = Self::from_bytes(required(&r.bytes).context("bytes_")?); + if this.len() < size { + anyhow::bail!("'vector' has less than 'size' bits"); + } + this.truncate(size); + Ok(this) + } + + fn build(&self) -> Self::Proto { + Self::Proto { + size: Some(self.len() as u64), + bytes: Some(self.to_bytes()), + } + } +} diff --git a/node/libs/protobuf/src/testonly.rs b/node/libs/protobuf/src/testonly.rs new file mode 100644 index 00000000..9cf15ae0 --- /dev/null +++ b/node/libs/protobuf/src/testonly.rs @@ -0,0 +1,89 @@ +//! Testonly utilities. +use super::{canonical, canonical_raw, decode, encode, read_fields, ProtoFmt, Wire}; +use prost::Message as _; +use prost_reflect::ReflectMessage as _; +use rand::{ + distributions::{Distribution, Standard}, + Rng, +}; + +/// Test encoding and canonical encoding properties. +#[track_caller] +pub fn test_encode(rng: &mut R, x: &T) { + let x_encode = encode(x); + let x_canonical = canonical(x); + let x_shuffled = encode_shuffled(rng, x); + + let desc = &::Proto::default().descriptor(); + for e in [&x_encode, &x_canonical, &x_shuffled] { + // e is a valid encoding. + assert_eq!(x, &decode::(e).unwrap()); + // canonical encoding is consistent. + assert_eq!(x_canonical, canonical_raw(e, desc).unwrap()); + } +} + +/// Syntax sugar for `test_encode`, +/// because `test_encode(rng,&rng::gen())` doesn't compile. +#[track_caller] +pub fn test_encode_random(rng: &mut R) +where + Standard: Distribution, +{ + let msg = rng.gen::(); + test_encode(rng, &msg); +} + +/// shuffles recursively the order of fields in a protobuf encoding. +fn encode_shuffled_raw( + rng: &mut R, + buf: &[u8], + desc: &prost_reflect::MessageDescriptor, +) -> anyhow::Result> { + // Vector of field values, values of each field are in reverse order. + // This way we can pop the next value of the field in O(1). + let mut fields: Vec<_> = read_fields(buf, desc)?.into_iter().collect(); + for f in &mut fields { + f.1.reverse(); + } + + // Append fields in random order. + let mut v = vec![]; + let mut w = quick_protobuf::Writer::new(&mut v); + while !fields.is_empty() { + // Select a random field. + let i = rng.gen_range(0..fields.len()); + // Select the next value of this field. + // Note that the values are stored in reverse order. + let Some(mut v) = fields[i].1.pop() else { + let last = fields.len() - 1; + fields.swap(i, last); + fields.pop(); + continue; + }; + let num = fields[i].0; + let fd = desc.get_field(num).unwrap(); + if let prost_reflect::Kind::Message(desc) = &fd.kind() { + v = encode_shuffled_raw(rng, &v, desc)?; + } + let wire = Wire::from(fd.kind()); + w.write_tag(num << 3 | wire.raw()).unwrap(); + match wire { + Wire::Varint | Wire::I64 | Wire::I32 => { + // inefficient workaround of the fact that quick_protobuf::Writer + // doesn't support just appending a sequence of bytes. + for b in &v { + w.write_u8(*b).unwrap(); + } + } + Wire::Len => w.write_bytes(&v).unwrap(), + } + } + Ok(v) +} + +/// Encodes a proto message of known schema in a random encoding order of fields. +pub(crate) fn encode_shuffled(rng: &mut R, x: &T) -> Vec { + let msg = x.build(); + encode_shuffled_raw(rng, &msg.encode_to_vec(), &msg.descriptor()).unwrap() +} diff --git a/node/libs/protobuf/src/tests.rs b/node/libs/protobuf/src/tests.rs new file mode 100644 index 00000000..d4d8366e --- /dev/null +++ b/node/libs/protobuf/src/tests.rs @@ -0,0 +1,105 @@ +use super::*; +use anyhow::Context as _; +use concurrency::{ctx, time}; +use std::net; + +#[derive(Debug, PartialEq, Eq)] +enum B { + U(bool), + V(Box), +} + +impl ProtoFmt for B { + type Proto = proto::testonly::B; + fn read(r: &Self::Proto) -> anyhow::Result { + use proto::testonly::b::T; + Ok(match required(&r.t)? { + T::U(x) => Self::U(*x), + T::V(x) => Self::V(Box::new(ProtoFmt::read(x.as_ref())?)), + }) + } + fn build(&self) -> Self::Proto { + use proto::testonly::b::T; + let t = match self { + Self::U(x) => T::U(*x), + Self::V(x) => T::V(Box::new(x.build())), + }; + Self::Proto { t: Some(t) } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct A { + x: Vec, + y: u64, + e: Vec, + b: B, +} + +impl ProtoFmt for A { + type Proto = proto::testonly::A; + fn read(r: &Self::Proto) -> anyhow::Result { + Ok(Self { + x: required(&r.x).context("x")?.clone(), + y: *required(&r.y).context("y")?, + e: r.e.clone(), + b: read_required(&r.b).context("b")?, + }) + } + + fn build(&self) -> Self::Proto { + Self::Proto { + x: Some(self.x.clone()), + y: Some(self.y), + e: self.e.clone(), + b: Some(self.b.build()), + } + } +} + +#[test] +fn test_encode_decode() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let v = A { + x: vec![1, 2, 3], + y: 76, + e: vec![8, 1, 9, 4], + b: B::U(true), + }; + testonly::test_encode(rng, &v); + + // Decoding should fail if there are trailing bytes present. + let mut bytes = encode(&v); + bytes.extend([4, 4, 3]); + assert!(decode::(&bytes).is_err()); +} + +#[test] +fn test_timestamp() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let unix = time::UNIX_EPOCH; + let zero = time::Duration::ZERO; + let nano = time::Duration::nanoseconds(1); + let sec = time::Duration::seconds(1); + for d in [zero, nano, sec, 12345678 * sec - 3 * nano] { + testonly::test_encode(rng, &(unix + d)); + testonly::test_encode(rng, &(unix - d)); + } +} + +#[test] +fn test_socket_addr() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + for addr in [ + "[::1]:432", + "74.223.12.1:63222", + "127.0.0.1:0", + "0.0.0.0:33", + ] { + let addr: net::SocketAddr = addr.parse().unwrap(); + testonly::test_encode(rng, &addr); + } +} diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml new file mode 100644 index 00000000..e5fdfb61 --- /dev/null +++ b/node/libs/protobuf_build/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "protobuf_build" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +bit-vec.workspace = true +serde.workspace = true +once_cell.workspace = true +quick-protobuf.workspace = true +prost.workspace = true +prost-types.workspace = true +prost-reflect.workspace = true +rand.workspace = true +serde_json.workspace = true +tokio.workspace = true + +syn.workspace = true +protoc-bin-vendored.workspace = true +prost-build.workspace = true +prost-reflect-build.workspace = true +prettyplease.workspace = true diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs new file mode 100644 index 00000000..cf93d8f8 --- /dev/null +++ b/node/libs/protobuf_build/src/lib.rs @@ -0,0 +1,230 @@ +//! Generates rust code from the capnp schema files in the `capnp/` directory. +use anyhow::Context as _; +use std::{collections::BTreeMap, env, fs, path::{PathBuf,Path}}; +use std::process::Command; +use prost::Message as _; +use std::collections::HashSet; +use once_cell::sync::Lazy; + +pub struct Descriptor { + pub crate_name: String, + pub module_path: String, +} + +/// Traversed all the files in a directory recursively. +fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> { + if !path.is_dir() { + f(&path); + return Ok(()); + } + for entry in fs::read_dir(path)? { + traverse_files(&entry?.path(), f)?; + } + Ok(()) +} + +/// A rust module representation. +/// It is used to collect the generated protobuf code. +#[derive(Default)] +struct Module { + /// Nested modules which transitively contain the generated code. + nested: BTreeMap, + /// Nested modules directly contains the generated code. + include: BTreeMap, +} + +impl Module { + /// Inserts a nested generated protobuf module. + /// `name` is a sequence of module names. + fn insert(&mut self, name: &[String], file: PathBuf) { + println!(" -- {name:?}"); + match name.len() { + 0 => panic!("empty module path"), + 1 => assert!( + self.include.insert(name[0].clone(), file).is_none(), + "duplicate module" + ), + _ => self + .nested + .entry(name[0].clone()) + .or_default() + .insert(&name[1..], file), + } + } + + /// Generates rust code of the module. + fn generate(&self) -> String { + let mut entries = vec![]; + entries.extend( + self.nested + .iter() + .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), + ); + entries.extend( + self.include + .iter() + .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), + ); + entries.join("\n") + } +} + +#[derive(Default)] +struct CanonicalCheckState(HashSet); + +impl CanonicalCheckState { + /// Checks if messages of type `m` support canonical encoding. + fn check_message(&mut self, m: &prost_reflect::MessageDescriptor, check_nested: bool) -> anyhow::Result<()> { + if self.0.contains(m.full_name()) { + return Ok(()); + } + self.0.insert(m.full_name().to_string()); + for f in m.fields() { + self.check_field(&f).with_context(|| f.name().to_string())?; + } + if check_nested { + for m in m.child_messages() { + self.check_message(&m,check_nested).with_context(||m.name().to_string())?; + } + } + Ok(()) + } + + /// Checks if field `f` supports canonical encoding. + fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + if f.is_map() { + anyhow::bail!("maps unsupported"); + } + if !f.is_list() && !f.supports_presence() { + anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); + } + if let prost_reflect::Kind::Message(msg) = f.kind() { + self.check_message(&msg,false).with_context(||msg.name().to_string())?; + } + Ok(()) + } + + /// Checks if message types in file `f` support canonical encoding. + fn check_file(&mut self, f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { + if f.syntax() != prost_reflect::Syntax::Proto3 { + anyhow::bail!("only proto3 syntax is supported"); + } + for m in f.messages() { + self.check_message(&m,true).with_context(|| m.name().to_string())?; + } + Ok(()) + } +} + +pub fn compile(proto_include: &Path, module_path: &str, deps: &[&Lazy]) -> anyhow::Result<()> { + println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); + let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) + .canonicalize()? + .join("proto"); + let _ = fs::remove_dir_all(&proto_output); + fs::create_dir_all(&proto_output).unwrap(); + + // Find all proto files. + let mut proto_inputs : Vec = vec![]; + traverse_files(proto_include, &mut |path| { + let Some(ext) = path.extension() else { return }; + let Some(ext) = ext.to_str() else { return }; + if ext != "proto" { + return; + }; + proto_inputs.push(path.into()); + })?; + + // Compile input files into descriptor. + let descriptor_path = proto_output.join("descriptor.binpb"); + let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); + cmd.arg("-o").arg(&descriptor_path); + cmd.arg("-I").arg(&proto_include); + + /*if deps.len() > 0 { + let mut deps_list = vec![]; + for (i,(_,d)) in deps.iter().enumerate() { + let name = proto_output.join(format!("dep{i}.binpb")); + fs::write(&name,d)?; + deps_list.push(name.to_str().unwrap().to_string()); + } + cmd.arg("--descriptor_set_in").arg(deps_list.join(":")); + }*/ + + let deps : Vec<_> = deps.iter().map(|d|&***d).collect(); + let deps_path = proto_output.join("deps.binpb"); + fs::write(&deps_path,prost_reflect::DescriptorPool::global().encode_to_vec())?; + cmd.arg("--descriptor_set_in").arg(&deps_path); + + for input in &proto_inputs { + cmd.arg(&input); + } + + let out = cmd.output().context("protoc execution failed")?; + + if !out.status.success() { + anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); + } + + // Generate protobuf code from schema (with reflection). + let descriptor = fs::read(&descriptor_path)?; + let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); + for p in &descriptor.file { + prost_reflect::DescriptorPool::add_global_file_descriptor_proto::<&[u8]>(p.clone()).unwrap(); + } + let pool_attribute = format!(r#"#[prost_reflect(descriptor_pool = "crate::{module_path}::DESCRIPTOR_POOL")]"#); + + let empty : &[&Path] = &[]; + let mut config = prost_build::Config::new(); + let pool = prost_reflect::DescriptorPool::global(); + for message in pool.all_messages() { + let full_name = message.full_name(); + config + .type_attribute(full_name, "#[derive(::prost_reflect::ReflectMessage)]") + .type_attribute(full_name, &format!(r#"#[prost_reflect(message_name = "{}")]"#, full_name)) + .type_attribute(full_name, &pool_attribute); + } + config.file_descriptor_set_path(&descriptor_path); + config.skip_protoc_run(); + config.out_dir(&proto_output); + config.compile_protos(empty,empty).unwrap(); + + // Check that messages are compatible with `proto_fmt::canonical`. + let mut check_state = CanonicalCheckState::default(); + for f in &descriptor.file { + check_state.check_file(&pool.get_file_by_name(f.name.as_ref().unwrap()).unwrap())?; + } + + // Generate mod file collecting all proto-generated code. + let mut m = Module::default(); + for entry in fs::read_dir(&proto_output).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().into_string().unwrap(); + let Some(name) = name.strip_suffix(".rs") else { + continue; + }; + let name: Vec<_> = name.split('.').map(String::from).collect(); + println!("name = {name:?}"); + m.insert(&name, entry.path()); + } + let mut file = deps.iter().map(|d|format!("use {}::{}::*;",d.crate_name,d.module_path)).collect::>().join("\n"); + file += &m.generate(); + let rec = deps.iter().map(|d|format!("&*{}::{}::DESCRIPTOR;",d.crate_name,d.module_path)).collect::>().join(" "); + file += &format!("pub const DESCRIPTOR: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| {{ \ + {rec} \ + prost_reflect::DescriptorPool::decode_global_file_descriptor_set(&include_bytes!({descriptor_path:?})[..]).unwrap(); \ + protobuf::build::Descriptor {{\ + crate_name: {:?}.to_string(),\ + module_path: {module_path:?}.to_string(),\ + }}\ + }});",std::env::var("CARGO_PKG_NAME").unwrap()); + file += "pub const DESCRIPTOR_POOL: once_cell::sync::Lazy = \ + once_cell::sync::Lazy::new(|| { \ + &*DESCRIPTOR; \ + prost_reflect::DescriptorPool::global() \ + }); \ + "; + let file = syn::parse_str(&file).unwrap(); + fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; + Ok(()) +} From bea6f03e00c89ed06271311a935c8730ae20a282 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 1 Nov 2023 09:08:10 +0100 Subject: [PATCH 08/40] doesn't really work --- node/libs/protobuf/build.rs | 2 +- node/libs/protobuf_build/src/lib.rs | 139 +++++++++++++++++++--------- 2 files changed, 96 insertions(+), 45 deletions(-) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 01ab9e3d..6a0479c9 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -6,6 +6,6 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf_build::compile(&proto_include,"proto",&[])?; + protobuf_build::compile(&proto_include,&[])?; Ok(()) } diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index cf93d8f8..030371f4 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -1,16 +1,31 @@ -//! Generates rust code from the capnp schema files in the `capnp/` directory. +//! Generates rust code from the protobuf use anyhow::Context as _; use std::{collections::BTreeMap, env, fs, path::{PathBuf,Path}}; use std::process::Command; use prost::Message as _; use std::collections::HashSet; -use once_cell::sync::Lazy; +pub use once_cell::sync::Lazy; +use std::sync::Mutex; + +static POOL : Lazy> = Lazy::new(||Mutex::default()); + +pub fn global(_:()) -> prost_reflect::DescriptorPool { POOL.lock().unwrap().clone() } + +pub type LazyDescriptor = Lazy; pub struct Descriptor { - pub crate_name: String, pub module_path: String, } +impl Descriptor { + pub fn new(module_path: String, fds: &impl AsRef<[u8]>) -> Self { + POOL.lock().unwrap().decode_file_descriptor_set(fds.as_ref()).unwrap(); + Descriptor { module_path } + } + + pub fn load(&self) {} +} + /// Traversed all the files in a directory recursively. fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> { if !path.is_dir() { @@ -58,12 +73,12 @@ impl Module { entries.extend( self.nested .iter() - .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), + .map(|(name, m)| format!("pub mod {name} {{ use super::{{protobuf,DESCRIPTOR}}; {} }}", m.generate())), ); entries.extend( self.include .iter() - .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), + .map(|(name, path)| format!("pub mod {name} {{ use super::{{protobuf,DESCRIPTOR}}; include!({path:?}); }}",)), ); entries.join("\n") } @@ -104,7 +119,7 @@ impl CanonicalCheckState { Ok(()) } - /// Checks if message types in file `f` support canonical encoding. + /*/// Checks if message types in file `f` support canonical encoding. fn check_file(&mut self, f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { if f.syntax() != prost_reflect::Syntax::Proto3 { anyhow::bail!("only proto3 syntax is supported"); @@ -113,10 +128,53 @@ impl CanonicalCheckState { self.check_message(&m,true).with_context(|| m.name().to_string())?; } Ok(()) + }*/ +} + +fn get_messages_from_message(out: &mut Vec, m: prost_reflect::MessageDescriptor) { + for m in m.child_messages() { + get_messages_from_message(out,m); + } + out.push(m); +} + +fn get_messages_from_file(out: &mut Vec, f: prost_reflect::FileDescriptor) { + for m in f.messages() { + get_messages_from_message(out,m); + } +} + +fn get_messages(fds: prost_types::FileDescriptorSet, mut pool: prost_reflect::DescriptorPool) -> Vec { + let mut res = vec![]; + pool.add_file_descriptor_set(fds.clone()).unwrap(); + for f in fds.file { + get_messages_from_file(&mut res,pool.get_file_by_name(f.name.as_ref().unwrap()).unwrap()); + } + res +} + +/* +fn reflect_impl_msg(m: prost_reflect::MessageDescriptor) -> String { + let full_name = m.full_name(); + let mut res = format!("impl prost_reflect::ReflectMessage for {full_name} {{ \ + fn descriptor(&self) -> ::prost_reflect::MessageDescriptor {{ \ + &*DESCRIPTOR; \ + protobuf::build::global() \ + .get_message_by_name(\"{full_name}\") \ + .expect(\"descriptor for message type {full_name} not found\") \ + }}\ + }}"); + for m in m.child_messages() { + res += &reflect_impl_msg(m); } + res } -pub fn compile(proto_include: &Path, module_path: &str, deps: &[&Lazy]) -> anyhow::Result<()> { +fn reflect_impl_file(f: prost_reflect::FileDescriptor) -> String { + f.messages().map(reflect_impl_msg).collect::>().join("") +}*/ + +pub fn compile(proto_include: &Path, deps: &[&Lazy]) -> anyhow::Result<()> { println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) .canonicalize()? @@ -166,35 +224,33 @@ pub fn compile(proto_include: &Path, module_path: &str, deps: &[&Lazy(p.clone()).unwrap(); - } - let pool_attribute = format!(r#"#[prost_reflect(descriptor_pool = "crate::{module_path}::DESCRIPTOR_POOL")]"#); - + // Generate protobuf code from schema. let empty : &[&Path] = &[]; let mut config = prost_build::Config::new(); - let pool = prost_reflect::DescriptorPool::global(); - for message in pool.all_messages() { - let full_name = message.full_name(); - config - .type_attribute(full_name, "#[derive(::prost_reflect::ReflectMessage)]") - .type_attribute(full_name, &format!(r#"#[prost_reflect(message_name = "{}")]"#, full_name)) - .type_attribute(full_name, &pool_attribute); - } config.file_descriptor_set_path(&descriptor_path); config.skip_protoc_run(); config.out_dir(&proto_output); - config.compile_protos(empty,empty).unwrap(); - + // Check that messages are compatible with `proto_fmt::canonical`. + let descriptor = fs::read(&descriptor_path)?; + let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); + let new_messages = get_messages(descriptor,global(())); + let mut check_state = CanonicalCheckState::default(); - for f in &descriptor.file { - check_state.check_file(&pool.get_file_by_name(f.name.as_ref().unwrap()).unwrap())?; + for m in &new_messages { + check_state.check_message(m,false)?; } + let pool_attribute = format!("#[prost_reflect(descriptor_pool = \"protobuf::build::global(DESCRIPTOR.load())\")]"); + for m in &new_messages { + let full_name = m.full_name(); + config.type_attribute(full_name, "#[derive(::prost_reflect::ReflectMessage)]"); + config.type_attribute(full_name, &format!("#[prost_reflect(message_name = {full_name:?})]")); + config.type_attribute(full_name, &pool_attribute); + } + + config.compile_protos(empty,empty).unwrap(); + // Generate mod file collecting all proto-generated code. let mut m = Module::default(); for entry in fs::read_dir(&proto_output).unwrap() { @@ -207,23 +263,18 @@ pub fn compile(proto_include: &Path, module_path: &str, deps: &[&Lazy>().join("\n"); - file += &m.generate(); - let rec = deps.iter().map(|d|format!("&*{}::{}::DESCRIPTOR;",d.crate_name,d.module_path)).collect::>().join(" "); - file += &format!("pub const DESCRIPTOR: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| {{ \ - {rec} \ - prost_reflect::DescriptorPool::decode_global_file_descriptor_set(&include_bytes!({descriptor_path:?})[..]).unwrap(); \ - protobuf::build::Descriptor {{\ - crate_name: {:?}.to_string(),\ - module_path: {module_path:?}.to_string(),\ - }}\ - }});",std::env::var("CARGO_PKG_NAME").unwrap()); - file += "pub const DESCRIPTOR_POOL: once_cell::sync::Lazy = \ - once_cell::sync::Lazy::new(|| { \ - &*DESCRIPTOR; \ - prost_reflect::DescriptorPool::global() \ - }); \ - "; + let mut file = deps.iter().map(|d|format!("use {}::*;",d.module_path)).collect::>().join("\n"); + file += &m.generate(); + + let rec = deps.iter().map(|d|format!("&*{}::DESCRIPTOR;",d.module_path)).collect::>().join(" "); + file += &format!("\ + static DESCRIPTOR : protobuf::build::LazyDescriptor = protobuf::build::Lazy::new(|| {{\ + {rec} + protobuf::build::Descriptor::new(module_path!(), &include_bytes!({descriptor_path:?}))\ + }});\ + "); + + //fs::write(proto_output.join("mod.rs"), file)?; let file = syn::parse_str(&file).unwrap(); fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; Ok(()) From 61c9c16c843d5dabf1dd178abd4a9d157652daed Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 1 Nov 2023 10:24:59 +0100 Subject: [PATCH 09/40] works --- node/Cargo.lock | 17 +------- node/Cargo.toml | 1 + node/libs/protobuf_build/Cargo.toml | 7 +--- node/libs/protobuf_build/src/lib.rs | 60 +++++++++++++---------------- node/libs/schema/build.rs | 2 +- 5 files changed, 30 insertions(+), 57 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index f33bb7c0..336be26f 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -1442,16 +1442,6 @@ dependencies = [ "serde-value", ] -[[package]] -name = "prost-reflect-build" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d959f576df574d088e0409c45ef11897f64727158c554ce65edc2d345f8afcf" -dependencies = [ - "prost-build", - "prost-reflect", -] - [[package]] name = "prost-reflect-derive" version = "0.11.0" @@ -1495,21 +1485,16 @@ name = "protobuf_build" version = "0.1.0" dependencies = [ "anyhow", - "bit-vec", + "heck", "once_cell", "prettyplease 0.2.12", "prost", "prost-build", "prost-reflect", - "prost-reflect-build", "prost-types", "protoc-bin-vendored", - "quick-protobuf", "rand", - "serde", - "serde_json", "syn 2.0.29", - "tokio", ] [[package]] diff --git a/node/Cargo.toml b/node/Cargo.toml index 0fd32b68..a9a43caa 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -34,6 +34,7 @@ blst = "0.3.10" clap = { version = "4.3.3", features = ["derive"] } ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"] } futures = "0.3.28" +heck = "0.4.1" hex = "0.4.3" hyper = { version = "0.14.27", features = ["http1", "http2","server","tcp"] } im = "15.1.0" diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml index e5fdfb61..2ce96d7e 100644 --- a/node/libs/protobuf_build/Cargo.toml +++ b/node/libs/protobuf_build/Cargo.toml @@ -8,19 +8,14 @@ license.workspace = true [dependencies] anyhow.workspace = true -bit-vec.workspace = true -serde.workspace = true +heck.workspace = true once_cell.workspace = true -quick-protobuf.workspace = true prost.workspace = true prost-types.workspace = true prost-reflect.workspace = true rand.workspace = true -serde_json.workspace = true -tokio.workspace = true syn.workspace = true protoc-bin-vendored.workspace = true prost-build.workspace = true -prost-reflect-build.workspace = true prettyplease.workspace = true diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 030371f4..62ebbbbe 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -7,9 +7,11 @@ use std::collections::HashSet; pub use once_cell::sync::Lazy; use std::sync::Mutex; +mod ident; + static POOL : Lazy> = Lazy::new(||Mutex::default()); -pub fn global(_:()) -> prost_reflect::DescriptorPool { POOL.lock().unwrap().clone() } +pub fn global() -> prost_reflect::DescriptorPool { POOL.lock().unwrap().clone() } pub type LazyDescriptor = Lazy; @@ -18,9 +20,9 @@ pub struct Descriptor { } impl Descriptor { - pub fn new(module_path: String, fds: &impl AsRef<[u8]>) -> Self { + pub fn new(module_path: &str, fds: &impl AsRef<[u8]>) -> Self { POOL.lock().unwrap().decode_file_descriptor_set(fds.as_ref()).unwrap(); - Descriptor { module_path } + Descriptor { module_path: module_path.to_string() } } pub fn load(&self) {} @@ -73,12 +75,12 @@ impl Module { entries.extend( self.nested .iter() - .map(|(name, m)| format!("pub mod {name} {{ use super::{{protobuf,DESCRIPTOR}}; {} }}", m.generate())), + .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), ); entries.extend( self.include .iter() - .map(|(name, path)| format!("pub mod {name} {{ use super::{{protobuf,DESCRIPTOR}}; include!({path:?}); }}",)), + .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), ); entries.join("\n") } @@ -153,27 +155,23 @@ fn get_messages(fds: prost_types::FileDescriptorSet, mut pool: prost_reflect::De res } -/* -fn reflect_impl_msg(m: prost_reflect::MessageDescriptor) -> String { - let full_name = m.full_name(); - let mut res = format!("impl prost_reflect::ReflectMessage for {full_name} {{ \ + +fn reflect_impl(m: prost_reflect::MessageDescriptor) -> String { + let proto_name = m.full_name(); + let parts : Vec<&str> = proto_name.split(".").collect(); + let mut rust_name : Vec<_> = parts[0..parts.len()-1].iter().map(|p|ident::to_snake(*p)).collect(); + rust_name.push(ident::to_upper_camel(parts[parts.len()-1])); + let rust_name = rust_name.join("::"); + format!("impl prost_reflect::ReflectMessage for {rust_name} {{ \ fn descriptor(&self) -> ::prost_reflect::MessageDescriptor {{ \ &*DESCRIPTOR; \ protobuf::build::global() \ - .get_message_by_name(\"{full_name}\") \ - .expect(\"descriptor for message type {full_name} not found\") \ + .get_message_by_name(\"{proto_name}\") \ + .expect(\"descriptor for message type {proto_name} not found\") \ }}\ - }}"); - for m in m.child_messages() { - res += &reflect_impl_msg(m); - } - res + }}") } -fn reflect_impl_file(f: prost_reflect::FileDescriptor) -> String { - f.messages().map(reflect_impl_msg).collect::>().join("") -}*/ - pub fn compile(proto_include: &Path, deps: &[&Lazy]) -> anyhow::Result<()> { println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) @@ -211,7 +209,7 @@ pub fn compile(proto_include: &Path, deps: &[&Lazy]) -> anyhow::Resu let deps : Vec<_> = deps.iter().map(|d|&***d).collect(); let deps_path = proto_output.join("deps.binpb"); - fs::write(&deps_path,prost_reflect::DescriptorPool::global().encode_to_vec())?; + fs::write(&deps_path,global().encode_to_vec())?; cmd.arg("--descriptor_set_in").arg(&deps_path); for input in &proto_inputs { @@ -234,21 +232,13 @@ pub fn compile(proto_include: &Path, deps: &[&Lazy]) -> anyhow::Resu // Check that messages are compatible with `proto_fmt::canonical`. let descriptor = fs::read(&descriptor_path)?; let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); - let new_messages = get_messages(descriptor,global(())); + let new_messages = get_messages(descriptor,global()); let mut check_state = CanonicalCheckState::default(); for m in &new_messages { check_state.check_message(m,false)?; } - let pool_attribute = format!("#[prost_reflect(descriptor_pool = \"protobuf::build::global(DESCRIPTOR.load())\")]"); - for m in &new_messages { - let full_name = m.full_name(); - config.type_attribute(full_name, "#[derive(::prost_reflect::ReflectMessage)]"); - config.type_attribute(full_name, &format!("#[prost_reflect(message_name = {full_name:?})]")); - config.type_attribute(full_name, &pool_attribute); - } - config.compile_protos(empty,empty).unwrap(); // Generate mod file collecting all proto-generated code. @@ -264,17 +254,19 @@ pub fn compile(proto_include: &Path, deps: &[&Lazy]) -> anyhow::Resu m.insert(&name, entry.path()); } let mut file = deps.iter().map(|d|format!("use {}::*;",d.module_path)).collect::>().join("\n"); - file += &m.generate(); - + file += &m.generate(); + for m in &new_messages { + file += &reflect_impl(m.clone()); + } + let rec = deps.iter().map(|d|format!("&*{}::DESCRIPTOR;",d.module_path)).collect::>().join(" "); file += &format!("\ - static DESCRIPTOR : protobuf::build::LazyDescriptor = protobuf::build::Lazy::new(|| {{\ + pub static DESCRIPTOR : protobuf::build::LazyDescriptor = protobuf::build::Lazy::new(|| {{\ {rec} protobuf::build::Descriptor::new(module_path!(), &include_bytes!({descriptor_path:?}))\ }});\ "); - //fs::write(proto_output.join("mod.rs"), file)?; let file = syn::parse_str(&file).unwrap(); fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; Ok(()) diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 67a88d25..9607c0c6 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -6,6 +6,6 @@ fn main() -> anyhow::Result<()> { let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) .canonicalize()? .join("proto"); - protobuf::build::compile(&proto_include,"proto", &[&protobuf::proto::DESCRIPTOR])?; + protobuf::build::compile(&proto_include,&[&protobuf::proto::DESCRIPTOR])?; Ok(()) } From a71030f3649aeb0e81cbb8b97ccd47235ec21e3e Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 1 Nov 2023 11:57:27 +0100 Subject: [PATCH 10/40] introduced config --- node/libs/protobuf/build.rs | 20 ++- node/libs/protobuf/src/lib.rs | 1 - node/libs/protobuf_build/src/lib.rs | 260 +++++++++++++--------------- node/libs/schema/build.rs | 18 +- 4 files changed, 150 insertions(+), 149 deletions(-) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 6a0479c9..c74a2026 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -1,11 +1,17 @@ //! Generates rust code from the capnp schema files in the `capnp/` directory. -use std::{env, path::PathBuf}; +use std::{path::PathBuf, env}; +use anyhow::Context as _; fn main() -> anyhow::Result<()> { - // Prepare input and output root dirs. - let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) - .canonicalize()? - .join("proto"); - protobuf_build::compile(&proto_include,&[])?; - Ok(()) + let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; + let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; + protobuf_build::Config { + input_root: input.join("proto"), + dependencies: vec![], + + output_mod_path: output.join("proto/mod.rs"), + output_descriptor_path: output.join("proto/desc.binpb"), + + protobuf_crate: "crate".to_string(), + }.generate().context("protobuf_build::Config::generate()") } diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs index 4f1db3a6..5f4cd7e0 100644 --- a/node/libs/protobuf/src/lib.rs +++ b/node/libs/protobuf/src/lib.rs @@ -13,6 +13,5 @@ mod tests; #[allow(warnings)] pub mod proto { - use crate as protobuf; include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); } diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 62ebbbbe..8367c4c8 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -1,20 +1,21 @@ //! Generates rust code from the protobuf use anyhow::Context as _; -use std::{collections::BTreeMap, env, fs, path::{PathBuf,Path}}; +use std::{collections::BTreeMap, fs, path::{PathBuf,Path}}; use std::process::Command; use prost::Message as _; use std::collections::HashSet; -pub use once_cell::sync::Lazy; use std::sync::Mutex; +pub use prost; +pub use prost_reflect; +pub use once_cell::sync::Lazy; + mod ident; static POOL : Lazy> = Lazy::new(||Mutex::default()); pub fn global() -> prost_reflect::DescriptorPool { POOL.lock().unwrap().clone() } -pub type LazyDescriptor = Lazy; - pub struct Descriptor { pub module_path: String, } @@ -45,44 +46,28 @@ fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> #[derive(Default)] struct Module { /// Nested modules which transitively contain the generated code. - nested: BTreeMap, - /// Nested modules directly contains the generated code. - include: BTreeMap, + modules: BTreeMap, + /// Code of the module. + code: String, } impl Module { /// Inserts a nested generated protobuf module. /// `name` is a sequence of module names. - fn insert(&mut self, name: &[String], file: PathBuf) { - println!(" -- {name:?}"); - match name.len() { - 0 => panic!("empty module path"), - 1 => assert!( - self.include.insert(name[0].clone(), file).is_none(), - "duplicate module" - ), - _ => self - .nested - .entry(name[0].clone()) - .or_default() - .insert(&name[1..], file), + fn insert<'a>(&mut self, mut name: impl Iterator, code: &str) { + match name.next() { + None => self.code += code, + Some(module) => self.modules.entry(module.to_string()).or_default().insert(name,code), } } /// Generates rust code of the module. fn generate(&self) -> String { - let mut entries = vec![]; + let mut entries = vec![self.code.clone()]; entries.extend( - self.nested - .iter() - .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), + self.modules.iter().map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.generate())), ); - entries.extend( - self.include - .iter() - .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), - ); - entries.join("\n") + entries.join("") } } @@ -156,118 +141,123 @@ fn get_messages(fds: prost_types::FileDescriptorSet, mut pool: prost_reflect::De } -fn reflect_impl(m: prost_reflect::MessageDescriptor) -> String { - let proto_name = m.full_name(); - let parts : Vec<&str> = proto_name.split(".").collect(); - let mut rust_name : Vec<_> = parts[0..parts.len()-1].iter().map(|p|ident::to_snake(*p)).collect(); - rust_name.push(ident::to_upper_camel(parts[parts.len()-1])); - let rust_name = rust_name.join("::"); - format!("impl prost_reflect::ReflectMessage for {rust_name} {{ \ - fn descriptor(&self) -> ::prost_reflect::MessageDescriptor {{ \ - &*DESCRIPTOR; \ - protobuf::build::global() \ - .get_message_by_name(\"{proto_name}\") \ - .expect(\"descriptor for message type {proto_name} not found\") \ - }}\ - }}") -} +pub struct Config { + pub input_root: PathBuf, + pub dependencies: Vec<&'static Lazy>, + + pub output_mod_path: PathBuf, + pub output_descriptor_path: PathBuf, -pub fn compile(proto_include: &Path, deps: &[&Lazy]) -> anyhow::Result<()> { - println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); - let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) - .canonicalize()? - .join("proto"); - let _ = fs::remove_dir_all(&proto_output); - fs::create_dir_all(&proto_output).unwrap(); - - // Find all proto files. - let mut proto_inputs : Vec = vec![]; - traverse_files(proto_include, &mut |path| { - let Some(ext) = path.extension() else { return }; - let Some(ext) = ext.to_str() else { return }; - if ext != "proto" { - return; - }; - proto_inputs.push(path.into()); - })?; - - // Compile input files into descriptor. - let descriptor_path = proto_output.join("descriptor.binpb"); - let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); - cmd.arg("-o").arg(&descriptor_path); - cmd.arg("-I").arg(&proto_include); - - /*if deps.len() > 0 { - let mut deps_list = vec![]; - for (i,(_,d)) in deps.iter().enumerate() { - let name = proto_output.join(format!("dep{i}.binpb")); - fs::write(&name,d)?; - deps_list.push(name.to_str().unwrap().to_string()); - } - cmd.arg("--descriptor_set_in").arg(deps_list.join(":")); - }*/ + pub protobuf_crate: String, +} - let deps : Vec<_> = deps.iter().map(|d|&***d).collect(); - let deps_path = proto_output.join("deps.binpb"); - fs::write(&deps_path,global().encode_to_vec())?; - cmd.arg("--descriptor_set_in").arg(&deps_path); +impl Config { + fn import(&self, elem: &str) -> String { + format!("{}::build::{}",self.protobuf_crate,elem) + } - for input in &proto_inputs { - cmd.arg(&input); + fn reflect_impl(&self, m: prost_reflect::MessageDescriptor) -> String { + let proto_name = m.full_name(); + let parts : Vec<&str> = proto_name.split(".").collect(); + let mut rust_name : Vec<_> = parts[0..parts.len()-1].iter().map(|p|ident::to_snake(*p)).collect(); + rust_name.push(ident::to_upper_camel(parts[parts.len()-1])); + let rust_name = rust_name.join("::"); + let rust_reflect = self.import("prost_reflect"); + let rust_global = self.import("global"); + format!("impl {rust_reflect}::ReflectMessage for {rust_name} {{ \ + fn descriptor(&self) -> {rust_reflect}::MessageDescriptor {{ \ + &*DESCRIPTOR; \ + {rust_global}() \ + .get_message_by_name(\"{proto_name}\") \ + .expect(\"descriptor for message type {proto_name} not found\") \ + }}\ + }}") } - let out = cmd.output().context("protoc execution failed")?; - if !out.status.success() { - anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); - } - - // Generate protobuf code from schema. - let empty : &[&Path] = &[]; - let mut config = prost_build::Config::new(); - config.file_descriptor_set_path(&descriptor_path); - config.skip_protoc_run(); - config.out_dir(&proto_output); - - // Check that messages are compatible with `proto_fmt::canonical`. - let descriptor = fs::read(&descriptor_path)?; - let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); - let new_messages = get_messages(descriptor,global()); - - let mut check_state = CanonicalCheckState::default(); - for m in &new_messages { - check_state.check_message(m,false)?; - } + pub fn generate(&self) -> anyhow::Result<()> { + println!("cargo:rerun-if-changed={:?}", self.input_root); + + // Find all proto files. + let mut inputs : Vec = vec![]; + traverse_files(&self.input_root, &mut |path| { + let Some(ext) = path.extension() else { return }; + let Some(ext) = ext.to_str() else { return }; + if ext != "proto" { + return; + }; + inputs.push(path.into()); + })?; + + // Compile input files into descriptor. + let descriptor_path = &self.output_descriptor_path; //.canonicalize().context("output_descriptor_path.canonicalize()")?; + fs::create_dir_all(&descriptor_path.parent().unwrap()).unwrap(); + let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); + cmd.arg("-o").arg(&descriptor_path); + cmd.arg("-I").arg(&self.input_root.canonicalize()?); + + /*if deps.len() > 0 { + let mut deps_list = vec![]; + for (i,(_,d)) in deps.iter().enumerate() { + let name = proto_output.join(format!("dep{i}.binpb")); + fs::write(&name,d)?; + deps_list.push(name.to_str().unwrap().to_string()); + } + cmd.arg("--descriptor_set_in").arg(deps_list.join(":")); + }*/ - config.compile_protos(empty,empty).unwrap(); - - // Generate mod file collecting all proto-generated code. - let mut m = Module::default(); - for entry in fs::read_dir(&proto_output).unwrap() { - let entry = entry.unwrap(); - let name = entry.file_name().into_string().unwrap(); - let Some(name) = name.strip_suffix(".rs") else { - continue; - }; - let name: Vec<_> = name.split('.').map(String::from).collect(); - println!("name = {name:?}"); - m.insert(&name, entry.path()); - } - let mut file = deps.iter().map(|d|format!("use {}::*;",d.module_path)).collect::>().join("\n"); - file += &m.generate(); - for m in &new_messages { - file += &reflect_impl(m.clone()); - } + let deps : Vec<_> = self.dependencies.iter().map(|d|&***d).collect(); + let deps_path = "/tmp/deps.binpb"; + fs::write(deps_path, global().encode_to_vec())?; + cmd.arg("--descriptor_set_in").arg(deps_path); - let rec = deps.iter().map(|d|format!("&*{}::DESCRIPTOR;",d.module_path)).collect::>().join(" "); - file += &format!("\ - pub static DESCRIPTOR : protobuf::build::LazyDescriptor = protobuf::build::Lazy::new(|| {{\ - {rec} - protobuf::build::Descriptor::new(module_path!(), &include_bytes!({descriptor_path:?}))\ - }});\ - "); + for input in &inputs { cmd.arg(&input); } - let file = syn::parse_str(&file).unwrap(); - fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; - Ok(()) + let out = cmd.output().context("protoc execution failed")?; + + if !out.status.success() { + anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); + } + + // Generate protobuf code from schema. + let mut config = prost_build::Config::new(); + config.prost_path(self.import("prost")); + config.file_descriptor_set_path(&descriptor_path); + config.skip_protoc_run(); + + // Check that messages are compatible with `proto_fmt::canonical`. + let descriptor = fs::read(&descriptor_path)?; + let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); + let new_messages = get_messages(descriptor.clone(),global()); + + let mut check_state = CanonicalCheckState::default(); + for m in &new_messages { + check_state.check_message(m,false)?; + } + + let modules : Vec<_> = descriptor.file.iter().map(|d|(prost_build::Module::from_protobuf_package_name(d.package()),d.clone())).collect(); + let mut m = Module::default(); + for (name,code) in config.generate(modules).unwrap() { + m.insert(name.parts(),&code); + } + let mut file = deps.iter().map(|d|format!("use {}::*;",d.module_path)).collect::>().join("\n"); + file += &m.generate(); + for m in &new_messages { + file += &self.reflect_impl(m.clone()); + } + + let rec = deps.iter().map(|d|format!("&*{}::DESCRIPTOR;",d.module_path)).collect::>().join(" "); + let rust_lazy = self.import("Lazy"); + let rust_descriptor = self.import("Descriptor"); + file += &format!("\ + pub static DESCRIPTOR : {rust_lazy}<{rust_descriptor}> = {rust_lazy}::new(|| {{\ + {rec} + {rust_descriptor}::new(module_path!(), &include_bytes!({descriptor_path:?}))\ + }});\ + "); + + let file = syn::parse_str(&file).unwrap(); + fs::write(&self.output_mod_path, prettyplease::unparse(&file))?; + Ok(()) + } } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 9607c0c6..71092d3f 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,11 +1,17 @@ //! Generates rust code from the capnp schema files in the `capnp/` directory. use std::{env, path::PathBuf}; +use anyhow::Context as _; fn main() -> anyhow::Result<()> { - // Prepare input and output root dirs. - let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) - .canonicalize()? - .join("proto"); - protobuf::build::compile(&proto_include,&[&protobuf::proto::DESCRIPTOR])?; - Ok(()) + let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; + let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; + protobuf::build::Config { + input_root: input.join("proto"), + dependencies: vec![&protobuf::proto::DESCRIPTOR], + + output_mod_path: output.join("proto/mod.rs"), + output_descriptor_path: output.join("proto/desc.binpb"), + + protobuf_crate: "::protobuf".to_string(), + }.generate().context("protobuf_build::Config::generate()") } From 6f906e6ae5ef9489972e36fe8fd3daba282c28d0 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 1 Nov 2023 13:39:16 +0100 Subject: [PATCH 11/40] before moving protos to deeper packages --- node/libs/protobuf/build.rs | 3 +- node/libs/protobuf_build/src/lib.rs | 134 +++++++++++++++------------- node/libs/schema/build.rs | 3 +- 3 files changed, 78 insertions(+), 62 deletions(-) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index c74a2026..d9032337 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -6,7 +6,8 @@ fn main() -> anyhow::Result<()> { let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; protobuf_build::Config { - input_root: input.join("proto"), + proto_path: input.join("proto"), + proto_package: "".to_string(), dependencies: vec![], output_mod_path: output.join("proto/mod.rs"), diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 8367c4c8..10b5a2ed 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -8,27 +8,10 @@ use std::sync::Mutex; pub use prost; pub use prost_reflect; -pub use once_cell::sync::Lazy; +pub use once_cell::sync::{Lazy}; mod ident; -static POOL : Lazy> = Lazy::new(||Mutex::default()); - -pub fn global() -> prost_reflect::DescriptorPool { POOL.lock().unwrap().clone() } - -pub struct Descriptor { - pub module_path: String, -} - -impl Descriptor { - pub fn new(module_path: &str, fds: &impl AsRef<[u8]>) -> Self { - POOL.lock().unwrap().decode_file_descriptor_set(fds.as_ref()).unwrap(); - Descriptor { module_path: module_path.to_string() } - } - - pub fn load(&self) {} -} - /// Traversed all the files in a directory recursively. fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> { if !path.is_dir() { @@ -131,19 +114,54 @@ fn get_messages_from_file(out: &mut Vec, f: pr } } -fn get_messages(fds: prost_types::FileDescriptorSet, mut pool: prost_reflect::DescriptorPool) -> Vec { +fn get_messages(fds: &prost_types::FileDescriptorSet, pool: &prost_reflect::DescriptorPool) -> Vec { let mut res = vec![]; - pool.add_file_descriptor_set(fds.clone()).unwrap(); - for f in fds.file { - get_messages_from_file(&mut res,pool.get_file_by_name(f.name.as_ref().unwrap()).unwrap()); + for f in &fds.file { + get_messages_from_file(&mut res,pool.get_file_by_name(f.name()).unwrap()); } res } +pub struct Descriptor { + pub rust_module : String, + pub proto_package : String, + pub descriptor_proto : prost_types::FileDescriptorSet, + pub dependencies : Vec<&'static Descriptor>, +} + +impl Descriptor { + pub fn new(rust_module: &str, proto_package: &str, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { + Descriptor { + rust_module: rust_module.to_string(), + proto_package: proto_package.to_string(), + dependencies, + descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()).unwrap(), + } + } + + pub fn load(&self, pool: &mut prost_reflect::DescriptorPool) -> anyhow::Result<()> { + if self.descriptor_proto.file.iter().all(|f| pool.get_file_by_name(f.name()).is_some()) { + return Ok(()); + } + for d in &self.dependencies { + d.load(pool)?; + } + pool.add_file_descriptor_set(self.descriptor_proto.clone())?; + Ok(()) + } + + pub fn load_global(&self) -> prost_reflect::DescriptorPool { + static POOL : Lazy> = Lazy::new(||Mutex::default()); + let pool = &mut POOL.lock().unwrap(); + self.load(pool).unwrap(); + pool.clone() + } +} pub struct Config { - pub input_root: PathBuf, - pub dependencies: Vec<&'static Lazy>, + pub proto_path: PathBuf, + pub proto_package: String, + pub dependencies: Vec<&'static Descriptor>, pub output_mod_path: PathBuf, pub output_descriptor_path: PathBuf, @@ -163,24 +181,23 @@ impl Config { rust_name.push(ident::to_upper_camel(parts[parts.len()-1])); let rust_name = rust_name.join("::"); let rust_reflect = self.import("prost_reflect"); - let rust_global = self.import("global"); - format!("impl {rust_reflect}::ReflectMessage for {rust_name} {{ \ - fn descriptor(&self) -> {rust_reflect}::MessageDescriptor {{ \ - &*DESCRIPTOR; \ - {rust_global}() \ - .get_message_by_name(\"{proto_name}\") \ - .expect(\"descriptor for message type {proto_name} not found\") \ + let rust_lazy = self.import("Lazy"); + format!("impl {rust_reflect}::ReflectMessage for {rust_name} {{\ + fn descriptor(&self) -> {rust_reflect}::MessageDescriptor {{\ + static INIT : {rust_lazy}<{rust_reflect}::MessageDescriptor> = {rust_lazy}::new(|| {{\ + DESCRIPTOR.load_global().get_message_by_name(\"{proto_name}\").unwrap()\ + }});\ + INIT.clone()\ }}\ }}") } - pub fn generate(&self) -> anyhow::Result<()> { - println!("cargo:rerun-if-changed={:?}", self.input_root); + println!("cargo:rerun-if-changed={:?}", self.proto_path); // Find all proto files. let mut inputs : Vec = vec![]; - traverse_files(&self.input_root, &mut |path| { + traverse_files(&self.proto_path, &mut |path| { let Some(ext) = path.extension() else { return }; let Some(ext) = ext.to_str() else { return }; if ext != "proto" { @@ -194,27 +211,15 @@ impl Config { fs::create_dir_all(&descriptor_path.parent().unwrap()).unwrap(); let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); cmd.arg("-o").arg(&descriptor_path); - cmd.arg("-I").arg(&self.input_root.canonicalize()?); - - /*if deps.len() > 0 { - let mut deps_list = vec![]; - for (i,(_,d)) in deps.iter().enumerate() { - let name = proto_output.join(format!("dep{i}.binpb")); - fs::write(&name,d)?; - deps_list.push(name.to_str().unwrap().to_string()); - } - cmd.arg("--descriptor_set_in").arg(deps_list.join(":")); - }*/ - - let deps : Vec<_> = self.dependencies.iter().map(|d|&***d).collect(); - let deps_path = "/tmp/deps.binpb"; - fs::write(deps_path, global().encode_to_vec())?; - cmd.arg("--descriptor_set_in").arg(deps_path); + cmd.arg("-I").arg(&self.proto_path.canonicalize()?); + let mut pool = prost_reflect::DescriptorPool::new(); + for d in &self.dependencies { d.load(&mut pool).unwrap(); } + let pool_path = "/tmp/pool.binpb"; + fs::write(pool_path, pool.encode_to_vec())?; + cmd.arg("--descriptor_set_in").arg(pool_path); for input in &inputs { cmd.arg(&input); } - let out = cmd.output().context("protoc execution failed")?; - if !out.status.success() { anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); } @@ -224,11 +229,22 @@ impl Config { config.prost_path(self.import("prost")); config.file_descriptor_set_path(&descriptor_path); config.skip_protoc_run(); - + for d in &self.dependencies { + for f in &d.descriptor_proto.file { + config.extern_path(format!(".{}",f.package()), format!("{}::{}",&d.rust_module,f.package().split(".").map(ident::to_snake).collect::>().join("::"))); + } + } + // Check that messages are compatible with `proto_fmt::canonical`. let descriptor = fs::read(&descriptor_path)?; let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); - let new_messages = get_messages(descriptor.clone(),global()); + for f in &descriptor.file { + if !f.package().starts_with(&self.proto_package) { + anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),self.proto_package); + } + } + pool.add_file_descriptor_set(descriptor.clone()).unwrap(); + let new_messages = get_messages(&descriptor,&pool); let mut check_state = CanonicalCheckState::default(); for m in &new_messages { @@ -240,21 +256,19 @@ impl Config { for (name,code) in config.generate(modules).unwrap() { m.insert(name.parts(),&code); } - let mut file = deps.iter().map(|d|format!("use {}::*;",d.module_path)).collect::>().join("\n"); - file += &m.generate(); + let mut file = m.generate(); for m in &new_messages { file += &self.reflect_impl(m.clone()); } - let rec = deps.iter().map(|d|format!("&*{}::DESCRIPTOR;",d.module_path)).collect::>().join(" "); + let rust_deps = self.dependencies.iter().map(|d|format!("&::{}::DESCRIPTOR",d.rust_module)).collect::>().join(","); let rust_lazy = self.import("Lazy"); let rust_descriptor = self.import("Descriptor"); file += &format!("\ pub static DESCRIPTOR : {rust_lazy}<{rust_descriptor}> = {rust_lazy}::new(|| {{\ - {rec} - {rust_descriptor}::new(module_path!(), &include_bytes!({descriptor_path:?}))\ + {rust_descriptor}::new(module_path!(), {:?}, vec![{rust_deps}], &include_bytes!({descriptor_path:?}))\ }});\ - "); + ",self.proto_package); let file = syn::parse_str(&file).unwrap(); fs::write(&self.output_mod_path, prettyplease::unparse(&file))?; diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 71092d3f..2261ff6f 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -6,7 +6,8 @@ fn main() -> anyhow::Result<()> { let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; protobuf::build::Config { - input_root: input.join("proto"), + proto_path: input.join("proto"), + proto_package: "".to_string(), dependencies: vec![&protobuf::proto::DESCRIPTOR], output_mod_path: output.join("proto/mod.rs"), From f95b5ee92ea5851b195fc74c2af9c0299cc8d575 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 1 Nov 2023 15:45:03 +0100 Subject: [PATCH 12/40] snapshot --- node/libs/protobuf_build/src/ident.rs | 161 ++++++++++++++++++++++++++ node/libs/protobuf_build/src/lib.rs | 16 +-- 2 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 node/libs/protobuf_build/src/ident.rs diff --git a/node/libs/protobuf_build/src/ident.rs b/node/libs/protobuf_build/src/ident.rs new file mode 100644 index 00000000..55e8e48a --- /dev/null +++ b/node/libs/protobuf_build/src/ident.rs @@ -0,0 +1,161 @@ +//! Utility functions for working with identifiers. + +use heck::{ToSnakeCase, ToUpperCamelCase}; + +/// Converts a `camelCase` or `SCREAMING_SNAKE_CASE` identifier to a `lower_snake` case Rust field +/// identifier. +pub fn to_snake(s: &str) -> String { + let mut ident = s.to_snake_case(); + + // Use a raw identifier if the identifier matches a Rust keyword: + // https://doc.rust-lang.org/reference/keywords.html. + match ident.as_str() { + // 2015 strict keywords. + | "as" | "break" | "const" | "continue" | "else" | "enum" | "false" + | "fn" | "for" | "if" | "impl" | "in" | "let" | "loop" | "match" | "mod" | "move" | "mut" + | "pub" | "ref" | "return" | "static" | "struct" | "trait" | "true" + | "type" | "unsafe" | "use" | "where" | "while" + // 2018 strict keywords. + | "dyn" + // 2015 reserved keywords. + | "abstract" | "become" | "box" | "do" | "final" | "macro" | "override" | "priv" | "typeof" + | "unsized" | "virtual" | "yield" + // 2018 reserved keywords. + | "async" | "await" | "try" => ident.insert_str(0, "r#"), + // the following keywords are not supported as raw identifiers and are therefore suffixed with an underscore. + "self" | "super" | "extern" | "crate" => ident += "_", + _ => (), + } + ident +} + +/// Converts a `snake_case` identifier to an `UpperCamel` case Rust type identifier. +pub fn to_upper_camel(s: &str) -> String { + let mut ident = s.to_upper_camel_case(); + + // Suffix an underscore for the `Self` Rust keyword as it is not allowed as raw identifier. + if ident == "Self" { + ident += "_"; + } + ident +} + +#[cfg(test)] +mod tests { + + #![allow(clippy::cognitive_complexity)] + + use super::*; + + #[test] + fn test_to_snake() { + assert_eq!("foo_bar", &to_snake("FooBar")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("xml_http_request", &to_snake("XMLHttpRequest")); + assert_eq!("r#while", &to_snake("While")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_BUSTER")); + assert_eq!("foo_bar_baz", &to_snake("foo_bar_baz")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_buster")); + assert_eq!("fuzz", &to_snake("_FUZZ")); + assert_eq!("fuzz", &to_snake("_fuzz")); + assert_eq!("fuzz", &to_snake("_Fuzz")); + assert_eq!("fuzz", &to_snake("FUZZ_")); + assert_eq!("fuzz", &to_snake("fuzz_")); + assert_eq!("fuzz", &to_snake("Fuzz_")); + assert_eq!("fuz_z", &to_snake("FuzZ_")); + + // From test_messages_proto3.proto. + assert_eq!("fieldname1", &to_snake("fieldname1")); + assert_eq!("field_name2", &to_snake("field_name2")); + assert_eq!("field_name3", &to_snake("_field_name3")); + assert_eq!("field_name4", &to_snake("field__name4_")); + assert_eq!("field0name5", &to_snake("field0name5")); + assert_eq!("field_0_name6", &to_snake("field_0_name6")); + assert_eq!("field_name7", &to_snake("fieldName7")); + assert_eq!("field_name8", &to_snake("FieldName8")); + assert_eq!("field_name9", &to_snake("field_Name9")); + assert_eq!("field_name10", &to_snake("Field_Name10")); + + assert_eq!("field_name11", &to_snake("FIELD_NAME11")); + assert_eq!("field_name12", &to_snake("FIELD_name12")); + assert_eq!("field_name13", &to_snake("__field_name13")); + assert_eq!("field_name14", &to_snake("__Field_name14")); + assert_eq!("field_name15", &to_snake("field__name15")); + assert_eq!("field_name16", &to_snake("field__Name16")); + assert_eq!("field_name17", &to_snake("field_name17__")); + assert_eq!("field_name18", &to_snake("Field_name18__")); + } + + #[test] + fn test_to_snake_raw_keyword() { + assert_eq!("r#as", &to_snake("as")); + assert_eq!("r#break", &to_snake("break")); + assert_eq!("r#const", &to_snake("const")); + assert_eq!("r#continue", &to_snake("continue")); + assert_eq!("r#else", &to_snake("else")); + assert_eq!("r#enum", &to_snake("enum")); + assert_eq!("r#false", &to_snake("false")); + assert_eq!("r#fn", &to_snake("fn")); + assert_eq!("r#for", &to_snake("for")); + assert_eq!("r#if", &to_snake("if")); + assert_eq!("r#impl", &to_snake("impl")); + assert_eq!("r#in", &to_snake("in")); + assert_eq!("r#let", &to_snake("let")); + assert_eq!("r#loop", &to_snake("loop")); + assert_eq!("r#match", &to_snake("match")); + assert_eq!("r#mod", &to_snake("mod")); + assert_eq!("r#move", &to_snake("move")); + assert_eq!("r#mut", &to_snake("mut")); + assert_eq!("r#pub", &to_snake("pub")); + assert_eq!("r#ref", &to_snake("ref")); + assert_eq!("r#return", &to_snake("return")); + assert_eq!("r#static", &to_snake("static")); + assert_eq!("r#struct", &to_snake("struct")); + assert_eq!("r#trait", &to_snake("trait")); + assert_eq!("r#true", &to_snake("true")); + assert_eq!("r#type", &to_snake("type")); + assert_eq!("r#unsafe", &to_snake("unsafe")); + assert_eq!("r#use", &to_snake("use")); + assert_eq!("r#where", &to_snake("where")); + assert_eq!("r#while", &to_snake("while")); + assert_eq!("r#dyn", &to_snake("dyn")); + assert_eq!("r#abstract", &to_snake("abstract")); + assert_eq!("r#become", &to_snake("become")); + assert_eq!("r#box", &to_snake("box")); + assert_eq!("r#do", &to_snake("do")); + assert_eq!("r#final", &to_snake("final")); + assert_eq!("r#macro", &to_snake("macro")); + assert_eq!("r#override", &to_snake("override")); + assert_eq!("r#priv", &to_snake("priv")); + assert_eq!("r#typeof", &to_snake("typeof")); + assert_eq!("r#unsized", &to_snake("unsized")); + assert_eq!("r#virtual", &to_snake("virtual")); + assert_eq!("r#yield", &to_snake("yield")); + assert_eq!("r#async", &to_snake("async")); + assert_eq!("r#await", &to_snake("await")); + assert_eq!("r#try", &to_snake("try")); + } + + #[test] + fn test_to_snake_non_raw_keyword() { + assert_eq!("self_", &to_snake("self")); + assert_eq!("super_", &to_snake("super")); + assert_eq!("extern_", &to_snake("extern")); + assert_eq!("crate_", &to_snake("crate")); + } + + #[test] + fn test_to_upper_camel() { + assert_eq!("", &to_upper_camel("")); + assert_eq!("F", &to_upper_camel("F")); + assert_eq!("Foo", &to_upper_camel("FOO")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR_")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR_")); + assert_eq!("FuzzBuster", &to_upper_camel("fuzzBuster")); + assert_eq!("FuzzBuster", &to_upper_camel("FuzzBuster")); + assert_eq!("Self_", &to_upper_camel("self")); + } +} diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 10b5a2ed..ed5c3615 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -207,10 +207,10 @@ impl Config { })?; // Compile input files into descriptor. - let descriptor_path = &self.output_descriptor_path; //.canonicalize().context("output_descriptor_path.canonicalize()")?; - fs::create_dir_all(&descriptor_path.parent().unwrap()).unwrap(); + fs::create_dir_all(&self.output_descriptor_path.parent().unwrap()).unwrap(); + // TODO: use protox instead, to be able to compress the proto paths. let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); - cmd.arg("-o").arg(&descriptor_path); + cmd.arg("-o").arg(&self.output_descriptor_path); cmd.arg("-I").arg(&self.proto_path.canonicalize()?); let mut pool = prost_reflect::DescriptorPool::new(); @@ -227,8 +227,9 @@ impl Config { // Generate protobuf code from schema. let mut config = prost_build::Config::new(); config.prost_path(self.import("prost")); - config.file_descriptor_set_path(&descriptor_path); + config.file_descriptor_set_path(&self.output_descriptor_path); config.skip_protoc_run(); + // TODO: make it relative to d.proto_package. for d in &self.dependencies { for f in &d.descriptor_proto.file { config.extern_path(format!(".{}",f.package()), format!("{}::{}",&d.rust_module,f.package().split(".").map(ident::to_snake).collect::>().join("::"))); @@ -236,7 +237,7 @@ impl Config { } // Check that messages are compatible with `proto_fmt::canonical`. - let descriptor = fs::read(&descriptor_path)?; + let descriptor = fs::read(&self.output_descriptor_path)?; let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); for f in &descriptor.file { if !f.package().starts_with(&self.proto_package) { @@ -266,12 +267,13 @@ impl Config { let rust_descriptor = self.import("Descriptor"); file += &format!("\ pub static DESCRIPTOR : {rust_lazy}<{rust_descriptor}> = {rust_lazy}::new(|| {{\ - {rust_descriptor}::new(module_path!(), {:?}, vec![{rust_deps}], &include_bytes!({descriptor_path:?}))\ + {rust_descriptor}::new(module_path!(), {:?}, vec![{rust_deps}], &include_bytes!({:?}))\ }});\ - ",self.proto_package); + ",self.proto_package,self.output_descriptor_path); let file = syn::parse_str(&file).unwrap(); fs::write(&self.output_mod_path, prettyplease::unparse(&file))?; + println!("PROTOBUF_DESCRIPTOR={:?}",self.output_descriptor_path); Ok(()) } } From 4747c157e4918745faed9429b57b235793bbc08e Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 1 Nov 2023 17:13:39 +0100 Subject: [PATCH 13/40] protox --- node/Cargo.lock | 232 +++++++++++---------- node/Cargo.toml | 12 +- node/actors/network/src/rpc/sync_blocks.rs | 2 +- node/libs/protobuf/build.rs | 5 +- node/libs/protobuf_build/Cargo.toml | 9 +- node/libs/protobuf_build/src/lib.rs | 57 +++-- node/libs/schema/build.rs | 5 +- 7 files changed, 168 insertions(+), 154 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index 336be26f..056a7ed0 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -130,7 +130,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -166,6 +166,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bindgen" version = "0.65.1" @@ -178,13 +184,13 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease 0.2.12", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.29", + "syn", ] [[package]] @@ -369,7 +375,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -497,7 +503,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -707,7 +713,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1015,7 +1021,7 @@ checksum = "bc28438cad73dcc90ff3466fc329a9252b1b8ba668eb0d5668ba97088cf4eef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1040,6 +1046,38 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn", +] + +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lz4-sys" version = "1.9.4" @@ -1065,6 +1103,29 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1254,7 +1315,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1330,16 +1391,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.12" @@ -1347,7 +1398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.29", + "syn", ] [[package]] @@ -1379,14 +1430,14 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] name = "prost" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" dependencies = [ "bytes", "prost-derive", @@ -1394,70 +1445,60 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck", "itertools", - "lazy_static", "log", "multimap", + "once_cell", "petgraph", - "prettyplease 0.1.25", + "prettyplease", "prost", "prost-types", "regex", - "syn 1.0.109", + "syn", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "prost-reflect" -version = "0.11.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "000e1e05ebf7b26e1eba298e66fe4eee6eb19c567d0ffb35e0dd34231cdac4c8" +checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" dependencies = [ "base64", + "logos", + "miette", "once_cell", "prost", - "prost-reflect-derive", "prost-types", "serde", "serde-value", ] -[[package]] -name = "prost-reflect-derive" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7718375aa8f966df66e583b608a305a45bc87eeb1ffd5db87fae673bea17a7e4" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.29", -] - [[package]] name = "prost-types" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" dependencies = [ "prost", ] @@ -1487,65 +1528,43 @@ dependencies = [ "anyhow", "heck", "once_cell", - "prettyplease 0.2.12", + "prettyplease", "prost", "prost-build", "prost-reflect", "prost-types", - "protoc-bin-vendored", + "protox", + "protox-parse", "rand", - "syn 2.0.29", + "syn", ] [[package]] -name = "protoc-bin-vendored" -version = "3.0.0" +name = "protox" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d" +checksum = "66eb3a834c1ffe362107daab84dd87cfc1e1d2beda30e2eb8e4801f262839364" dependencies = [ - "protoc-bin-vendored-linux-aarch_64", - "protoc-bin-vendored-linux-ppcle_64", - "protoc-bin-vendored-linux-x86_32", - "protoc-bin-vendored-linux-x86_64", - "protoc-bin-vendored-macos-x86_64", - "protoc-bin-vendored-win32", + "bytes", + "miette", + "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", ] [[package]] -name = "protoc-bin-vendored-linux-aarch_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435" - -[[package]] -name = "protoc-bin-vendored-linux-ppcle_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516" - -[[package]] -name = "protoc-bin-vendored-linux-x86_32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0" - -[[package]] -name = "protoc-bin-vendored-linux-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924" - -[[package]] -name = "protoc-bin-vendored-macos-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537" - -[[package]] -name = "protoc-bin-vendored-win32" -version = "3.0.0" +name = "protox-parse" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" +checksum = "7b4581f441c58863525a3e6bec7b8de98188cf75239a56c725a3e7288450a33f" +dependencies = [ + "logos", + "miette", + "prost-types", + "thiserror", +] [[package]] name = "quick-protobuf" @@ -1781,7 +1800,7 @@ checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1945,17 +1964,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.29" @@ -2016,7 +2024,7 @@ checksum = "b4dc4744280091c8760f456b14c5598f5f7afe96851b4da30fe0933725dae0d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2036,7 +2044,7 @@ checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2102,7 +2110,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2162,7 +2170,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2222,6 +2230,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "universal-hash" version = "0.4.0" @@ -2295,7 +2309,7 @@ source = "git+https://github.com/matter-labs/vise.git?rev=8322ddc4bb115a7d111276 dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2435,7 +2449,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] diff --git a/node/Cargo.toml b/node/Cargo.toml index a9a43caa..4be49e55 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -40,12 +40,12 @@ hyper = { version = "0.14.27", features = ["http1", "http2","server","tcp"] } im = "15.1.0" once_cell = "1.17.1" pin-project = "1.1.0" -prost = "0.11.0" -prost-build = "0.11.0" -prost-reflect = { version = "0.11.0", features = ["derive","serde"] } -prost-reflect-build = "0.11.0" -prost-types = "0.11.0" -protoc-bin-vendored = "3.0.0" +prost = "0.12.0" +prost-build = "0.12.0" +prost-reflect = { version = "0.12.0", features = ["serde"] } +prost-types = "0.12.0" +protox = "0.5.0" +protox-parse = "0.5.0" prettyplease = "0.2.6" pretty_assertions = "1.4.0" quick-protobuf = "0.8.1" diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index 0244fa4a..7454a35b 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -121,7 +121,7 @@ impl ProtoFmt for io::GetBlockError { use proto::get_block_response::ErrorReason; let reason = message.reason.context("missing reason")?; - let reason = ErrorReason::from_i32(reason).context("reason")?; + let reason = ErrorReason::try_from(reason).context("reason")?; Ok(match reason { ErrorReason::NotSynced => Self::NotSynced, }) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index d9032337..d90115c2 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -6,8 +6,9 @@ fn main() -> anyhow::Result<()> { let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; protobuf_build::Config { - proto_path: input.join("proto"), - proto_package: "".to_string(), + input_path: input.join("proto"), + proto_path: "".into(), + proto_package: "".into(), dependencies: vec![], output_mod_path: output.join("proto/mod.rs"), diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml index 2ce96d7e..bd4463c8 100644 --- a/node/libs/protobuf_build/Cargo.toml +++ b/node/libs/protobuf_build/Cargo.toml @@ -10,12 +10,13 @@ license.workspace = true anyhow.workspace = true heck.workspace = true once_cell.workspace = true +prettyplease.workspace = true prost.workspace = true +prost-build.workspace = true prost-types.workspace = true prost-reflect.workspace = true +protox.workspace = true +protox-parse.workspace = true rand.workspace = true - syn.workspace = true -protoc-bin-vendored.workspace = true -prost-build.workspace = true -prettyplease.workspace = true + diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index ed5c3615..91f23e36 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -1,7 +1,6 @@ //! Generates rust code from the protobuf use anyhow::Context as _; use std::{collections::BTreeMap, fs, path::{PathBuf,Path}}; -use std::process::Command; use prost::Message as _; use std::collections::HashSet; use std::sync::Mutex; @@ -13,9 +12,9 @@ pub use once_cell::sync::{Lazy}; mod ident; /// Traversed all the files in a directory recursively. -fn traverse_files(path: &Path, f: &mut dyn FnMut(&Path)) -> std::io::Result<()> { +fn traverse_files(path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>) -> anyhow::Result<()> { if !path.is_dir() { - f(&path); + f(path).with_context(||path.to_str().unwrap().to_string())?; return Ok(()); } for entry in fs::read_dir(path)? { @@ -159,6 +158,7 @@ impl Descriptor { } pub struct Config { + pub input_path: PathBuf, pub proto_path: PathBuf, pub proto_package: String, pub dependencies: Vec<&'static Descriptor>, @@ -196,38 +196,35 @@ impl Config { println!("cargo:rerun-if-changed={:?}", self.proto_path); // Find all proto files. - let mut inputs : Vec = vec![]; - traverse_files(&self.proto_path, &mut |path| { - let Some(ext) = path.extension() else { return }; - let Some(ext) = ext.to_str() else { return }; - if ext != "proto" { - return; - }; - inputs.push(path.into()); - })?; - - // Compile input files into descriptor. - fs::create_dir_all(&self.output_descriptor_path.parent().unwrap()).unwrap(); - // TODO: use protox instead, to be able to compress the proto paths. - let mut cmd = Command::new(protoc_bin_vendored::protoc_bin_path().unwrap()); - cmd.arg("-o").arg(&self.output_descriptor_path); - cmd.arg("-I").arg(&self.proto_path.canonicalize()?); - let mut pool = prost_reflect::DescriptorPool::new(); for d in &self.dependencies { d.load(&mut pool).unwrap(); } - let pool_path = "/tmp/pool.binpb"; - fs::write(pool_path, pool.encode_to_vec())?; - cmd.arg("--descriptor_set_in").arg(pool_path); - for input in &inputs { cmd.arg(&input); } - let out = cmd.output().context("protoc execution failed")?; - if !out.status.success() { - anyhow::bail!("protoc_failed:\n{}",String::from_utf8_lossy(&out.stderr)); - } + let mut x = prost_types::FileDescriptorSet::default(); + x.file = pool.file_descriptor_protos().cloned().collect(); + let input_path = self.input_path.canonicalize()?; + let mut new_paths = vec![]; + traverse_files(&input_path, &mut |path| { + let Some(ext) = path.extension() else { return Ok(()) }; + let Some(ext) = ext.to_str() else { return Ok(()) }; + if ext != "proto" { return Ok(()) }; + let rel_path = self.proto_path.join(path.strip_prefix(&input_path).unwrap()); + x.file.push(protox_parse::parse( + rel_path.to_str().unwrap(), + &fs::read_to_string(path).context("fs::read()")?, + ).context("protox_parse::parse()")?); + new_paths.push(rel_path.clone()); + Ok(()) + })?; + let mut compiler = protox::Compiler::with_file_resolver( + protox::file::DescriptorSetFileResolver::new(x) + ); + compiler.open_files(new_paths).unwrap(); + let descriptor = compiler.file_descriptor_set(); + fs::create_dir_all(&self.output_descriptor_path.parent().unwrap()).unwrap(); + fs::write(&self.output_descriptor_path, &descriptor.encode_to_vec())?; // Generate protobuf code from schema. let mut config = prost_build::Config::new(); config.prost_path(self.import("prost")); - config.file_descriptor_set_path(&self.output_descriptor_path); config.skip_protoc_run(); // TODO: make it relative to d.proto_package. for d in &self.dependencies { @@ -243,7 +240,7 @@ impl Config { if !f.package().starts_with(&self.proto_package) { anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),self.proto_package); } - } + } pool.add_file_descriptor_set(descriptor.clone()).unwrap(); let new_messages = get_messages(&descriptor,&pool); diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 2261ff6f..2691da51 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -6,8 +6,9 @@ fn main() -> anyhow::Result<()> { let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; protobuf::build::Config { - proto_path: input.join("proto"), - proto_package: "".to_string(), + input_path: input.join("proto"), + proto_path: "".into(), + proto_package: "".into(), dependencies: vec![&protobuf::proto::DESCRIPTOR], output_mod_path: output.join("proto/mod.rs"), From 70b23d717b934dd81086e3f52efd4d771589617b Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 1 Nov 2023 19:05:11 +0100 Subject: [PATCH 14/40] fixed reflect --- node/libs/protobuf/build.rs | 4 ++-- .../proto/conformance/conformance.proto | 5 +---- .../conformance/test_messages_proto3.proto | 10 +--------- node/libs/protobuf/proto/std.proto | 2 +- node/libs/protobuf/proto/testonly.proto | 2 +- node/libs/protobuf/src/bin/conformance_test.rs | 18 +++++++++--------- node/libs/protobuf_build/src/lib.rs | 13 +++++++++---- node/libs/schema/build.rs | 4 ++-- node/libs/schema/proto/executor/config.proto | 2 +- node/libs/schema/proto/network/consensus.proto | 6 +++--- node/libs/schema/proto/network/gossip.proto | 6 +++--- node/libs/schema/proto/network/mux.proto | 4 ++-- node/libs/schema/proto/network/mux_test.proto | 2 +- node/libs/schema/proto/network/ping.proto | 2 +- node/libs/schema/proto/network/preface.proto | 2 +- node/libs/schema/proto/roles/node.proto | 2 +- node/libs/schema/proto/roles/validator.proto | 16 ++++++++-------- node/libs/schema/proto/storage.proto | 4 ++-- 18 files changed, 49 insertions(+), 55 deletions(-) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index d90115c2..65433f7b 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -7,8 +7,8 @@ fn main() -> anyhow::Result<()> { let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; protobuf_build::Config { input_path: input.join("proto"), - proto_path: "".into(), - proto_package: "".into(), + proto_path: "zksync/protobuf".into(), + proto_package: "zksync.protobuf".into(), dependencies: vec![], output_mod_path: output.join("proto/mod.rs"), diff --git a/node/libs/protobuf/proto/conformance/conformance.proto b/node/libs/protobuf/proto/conformance/conformance.proto index e6403673..fc71bdf4 100644 --- a/node/libs/protobuf/proto/conformance/conformance.proto +++ b/node/libs/protobuf/proto/conformance/conformance.proto @@ -33,10 +33,7 @@ syntax = "proto3"; -package conformance; - -option java_package = "com.google.protobuf.conformance"; -option objc_class_prefix = "Conformance"; +package zksync.protobuf.conformance.conformance; // This defines the conformance testing protocol. This protocol exists between // the conformance test suite itself and the code being tested. For each test, diff --git a/node/libs/protobuf/proto/conformance/test_messages_proto3.proto b/node/libs/protobuf/proto/conformance/test_messages_proto3.proto index d391761b..7e82c545 100644 --- a/node/libs/protobuf/proto/conformance/test_messages_proto3.proto +++ b/node/libs/protobuf/proto/conformance/test_messages_proto3.proto @@ -40,15 +40,7 @@ syntax = "proto3"; -package protobuf_test_messages.proto3; - -option java_package = "com.google.protobuf_test_messages.proto3"; -option objc_class_prefix = "Proto3"; - -// This is the default, but we specify it here explicitly. -option optimize_for = SPEED; - -option cc_enable_arenas = true; +package zksync.protobuf.conformance.test_messages_proto3; // This proto includes every type of field in both singular and repeated // forms. diff --git a/node/libs/protobuf/proto/std.proto b/node/libs/protobuf/proto/std.proto index f5ee5efe..c27646b7 100644 --- a/node/libs/protobuf/proto/std.proto +++ b/node/libs/protobuf/proto/std.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package std; +package zksync.protobuf.std; message Void {} diff --git a/node/libs/protobuf/proto/testonly.proto b/node/libs/protobuf/proto/testonly.proto index 7c5d08cd..b5061bc4 100644 --- a/node/libs/protobuf/proto/testonly.proto +++ b/node/libs/protobuf/proto/testonly.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package testonly; +package zksync.protobuf.testonly; message B { // Recursive union. diff --git a/node/libs/protobuf/src/bin/conformance_test.rs b/node/libs/protobuf/src/bin/conformance_test.rs index 05782feb..b74a6b48 100644 --- a/node/libs/protobuf/src/bin/conformance_test.rs +++ b/node/libs/protobuf/src/bin/conformance_test.rs @@ -26,8 +26,8 @@ async fn main() -> anyhow::Result<()> { let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stdin, &mut msg[..]).await??; - use proto::conformance_response::Result as R; - let req = proto::ConformanceRequest::decode(&msg[..])?; + use proto::conformance::conformance_response::Result as R; + let req = proto::conformance::ConformanceRequest::decode(&msg[..])?; let res = async { let t = req.message_type.context("missing message_type")?; if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { @@ -36,15 +36,15 @@ async fn main() -> anyhow::Result<()> { // Decode. let payload = req.payload.context("missing payload")?; - use protobuf::proto::protobuf_test_messages::proto3::TestAllTypesProto3 as T; + use proto::test_messages_proto3::TestAllTypesProto3 as T; let p = match payload { - proto::conformance_request::Payload::JsonPayload(payload) => { + proto::conformance::conformance_request::Payload::JsonPayload(payload) => { match protobuf::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } } - proto::conformance_request::Payload::ProtobufPayload(payload) => { + proto::conformance::conformance_request::Payload::ProtobufPayload(payload) => { // First filter out incorrect encodings. let Ok(p) = T::decode(&payload[..]) else { return Ok(R::ParseError("parsing failed".to_string())); @@ -62,11 +62,11 @@ async fn main() -> anyhow::Result<()> { let format = req .requested_output_format .context("missing output format")?; - match proto::WireFormat::try_from(format).context("unknown format")? { - proto::WireFormat::Json => { + match proto::conformance::WireFormat::try_from(format).context("unknown format")? { + proto::conformance::WireFormat::Json => { anyhow::Ok(R::JsonPayload(protobuf::encode_json_proto(&p))) } - proto::WireFormat::Protobuf => { + proto::conformance::WireFormat::Protobuf => { // Reencode the parsed proto. anyhow::Ok(R::ProtobufPayload(protobuf::canonical_raw( &p.encode_to_vec(), @@ -77,7 +77,7 @@ async fn main() -> anyhow::Result<()> { } } .await?; - let resp = proto::ConformanceResponse { result: Some(res) }; + let resp = proto::conformance::ConformanceResponse { result: Some(res) }; // Write the response. let msg = resp.encode_to_vec(); diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 91f23e36..728dae88 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -176,7 +176,7 @@ impl Config { fn reflect_impl(&self, m: prost_reflect::MessageDescriptor) -> String { let proto_name = m.full_name(); - let parts : Vec<&str> = proto_name.split(".").collect(); + let parts : Vec<&str> = proto_name.strip_prefix(&(self.proto_package.clone() + ".")).unwrap().split(".").collect(); let mut rust_name : Vec<_> = parts[0..parts.len()-1].iter().map(|p|ident::to_snake(*p)).collect(); rust_name.push(ident::to_upper_camel(parts[parts.len()-1])); let rust_name = rust_name.join("::"); @@ -193,7 +193,7 @@ impl Config { } pub fn generate(&self) -> anyhow::Result<()> { - println!("cargo:rerun-if-changed={:?}", self.proto_path); + println!("cargo:rerun-if-changed={:?}", self.input_path); // Find all proto files. let mut pool = prost_reflect::DescriptorPool::new(); @@ -226,10 +226,11 @@ impl Config { let mut config = prost_build::Config::new(); config.prost_path(self.import("prost")); config.skip_protoc_run(); - // TODO: make it relative to d.proto_package. for d in &self.dependencies { for f in &d.descriptor_proto.file { - config.extern_path(format!(".{}",f.package()), format!("{}::{}",&d.rust_module,f.package().split(".").map(ident::to_snake).collect::>().join("::"))); + let rel = f.package().strip_prefix(&(d.proto_package.clone() + ".")).unwrap(); + let rust_rel = rel.split(".").map(ident::to_snake).collect::>().join("::"); + config.extern_path(format!(".{}",f.package()), format!("{}::{}",d.rust_module,rust_rel)); } } @@ -254,6 +255,10 @@ impl Config { for (name,code) in config.generate(modules).unwrap() { m.insert(name.parts(),&code); } + let mut m = &m; + for name in self.proto_package.split(".") { + m = &m.modules[name]; + } let mut file = m.generate(); for m in &new_messages { file += &self.reflect_impl(m.clone()); diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 2691da51..82c0d329 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -7,8 +7,8 @@ fn main() -> anyhow::Result<()> { let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; protobuf::build::Config { input_path: input.join("proto"), - proto_path: "".into(), - proto_package: "".into(), + proto_path: "zksync/schema".into(), + proto_package: "zksync.schema".into(), dependencies: vec![&protobuf::proto::DESCRIPTOR], output_mod_path: output.join("proto/mod.rs"), diff --git a/node/libs/schema/proto/executor/config.proto b/node/libs/schema/proto/executor/config.proto index 87baafd0..2f58a37b 100644 --- a/node/libs/schema/proto/executor/config.proto +++ b/node/libs/schema/proto/executor/config.proto @@ -36,7 +36,7 @@ // the validator set) or move it to a separate config file. syntax = "proto3"; -package executor.config; +package zksync.schema.executor.config; // (public key, ip address) of a gossip network node. message NodeAddr { diff --git a/node/libs/schema/proto/network/consensus.proto b/node/libs/schema/proto/network/consensus.proto index f5f34dbb..234fbe03 100644 --- a/node/libs/schema/proto/network/consensus.proto +++ b/node/libs/schema/proto/network/consensus.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package network.consensus; +package zksync.schema.network.consensus; -import "roles/validator.proto"; -import "std.proto"; +import "zksync/schema/roles/validator.proto"; +import "zksync/protobuf/std.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/libs/schema/proto/network/gossip.proto b/node/libs/schema/proto/network/gossip.proto index 9423a2c7..723bb100 100644 --- a/node/libs/schema/proto/network/gossip.proto +++ b/node/libs/schema/proto/network/gossip.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package network.gossip; +package zksync.schema.network.gossip; -import "roles/node.proto"; -import "roles/validator.proto"; +import "zksync/schema/roles/node.proto"; +import "zksync/schema/roles/validator.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/libs/schema/proto/network/mux.proto b/node/libs/schema/proto/network/mux.proto index a4bd0eec..91965bfc 100644 --- a/node/libs/schema/proto/network/mux.proto +++ b/node/libs/schema/proto/network/mux.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package network.mux; +package zksync.schema.network.mux; -import "std.proto"; +import "zksync/protobuf/std.proto"; message Handshake { message Capability { diff --git a/node/libs/schema/proto/network/mux_test.proto b/node/libs/schema/proto/network/mux_test.proto index dff0f492..b4cdaf87 100644 --- a/node/libs/schema/proto/network/mux_test.proto +++ b/node/libs/schema/proto/network/mux_test.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.mux_test; +package zksync.schema.network.mux_test; message Req { optional bytes input = 1; diff --git a/node/libs/schema/proto/network/ping.proto b/node/libs/schema/proto/network/ping.proto index 563dcbf9..863022fd 100644 --- a/node/libs/schema/proto/network/ping.proto +++ b/node/libs/schema/proto/network/ping.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.ping; +package zksync.schema.network.ping; message PingReq { optional bytes data = 1; diff --git a/node/libs/schema/proto/network/preface.proto b/node/libs/schema/proto/network/preface.proto index 1325e047..193ca38e 100644 --- a/node/libs/schema/proto/network/preface.proto +++ b/node/libs/schema/proto/network/preface.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.preface; +package zksync.schema.network.preface; // Every connection starts with the following preface: // 1. client sends Encryption msg to server. diff --git a/node/libs/schema/proto/roles/node.proto b/node/libs/schema/proto/roles/node.proto index 178eb914..87b75c1e 100644 --- a/node/libs/schema/proto/roles/node.proto +++ b/node/libs/schema/proto/roles/node.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package roles.node; +package zksync.schema.roles.node; message Msg { oneof t { diff --git a/node/libs/schema/proto/roles/validator.proto b/node/libs/schema/proto/roles/validator.proto index 8d02e1a7..068112af 100644 --- a/node/libs/schema/proto/roles/validator.proto +++ b/node/libs/schema/proto/roles/validator.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package roles.validator; +package zksync.schema.roles.validator; -import "std.proto"; +import "zksync/protobuf/std.proto"; message PayloadHash { optional bytes sha256 = 1; // required @@ -64,20 +64,20 @@ message LeaderCommit { message PrepareQC { repeated ReplicaPrepare msgs = 1; // required - repeated std.BitVector signers = 2; // required + repeated protobuf.std.BitVector signers = 2; // required optional AggregateSignature sig = 3; // required } message CommitQC { optional ReplicaCommit msg = 1; // required - optional std.BitVector signers = 2; // required + optional protobuf.std.BitVector signers = 2; // required optional AggregateSignature sig = 3; // required } message Phase { oneof t { - std.Void prepare = 1; - std.Void commit = 2; + protobuf.std.Void prepare = 1; + protobuf.std.Void commit = 2; } } @@ -88,7 +88,7 @@ message Phase { // network connection to this address. message NetAddress { // Address of the validator. - optional std.SocketAddr addr = 1; // required + optional protobuf.std.SocketAddr addr = 1; // required // Version of the discovery announcement. // Newer (higher) version overrides the older version. // When a validator gets restarted it broadcasts @@ -125,7 +125,7 @@ message NetAddress { // we assume here that it is unlikely for timestamps to collide. // To make this reasoning more strict, we should rather use a random "tie breaker" // instead (replace timestamp with a random nonce, or use a hash of the entire message). - optional std.Timestamp timestamp = 3; // required + optional protobuf.std.Timestamp timestamp = 3; // required } message Msg { diff --git a/node/libs/schema/proto/storage.proto b/node/libs/schema/proto/storage.proto index dbcfa8e2..301a5485 100644 --- a/node/libs/schema/proto/storage.proto +++ b/node/libs/schema/proto/storage.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package storage; +package zksync.schema.storage; -import "roles/validator.proto"; +import "zksync/schema/roles/validator.proto"; message Proposal { optional uint64 number = 1; From 6a9ce92a3b6b1302b9fc1fb8d41a810ace545de5 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 2 Nov 2023 22:41:10 +0100 Subject: [PATCH 15/40] distributed the proto files --- node/libs/protobuf/build.rs | 40 +++++- .../conformance_test_failure_list.txt | 0 .../main.rs} | 21 +-- .../conformance_test/proto}/conformance.proto | 2 +- .../src/bin/conformance_test/proto/mod.rs | 2 + .../proto}/test_messages_proto3.proto | 2 +- node/libs/protobuf/src/lib.rs | 5 +- node/libs/protobuf/src/proto/mod.rs | 2 + node/libs/protobuf/{ => src}/proto/std.proto | 2 +- node/libs/protobuf/src/std_conv.rs | 12 +- .../protobuf/src/{tests.rs => tests/mod.rs} | 12 +- node/libs/protobuf/src/tests/proto/mod.rs | 2 + .../tests/proto/tests.proto} | 2 +- node/libs/protobuf_build/src/lib.rs | 132 ++++++++++++------ node/libs/schema/build.rs | 10 +- .../libs/schema/proto/network/consensus.proto | 2 +- node/libs/schema/proto/network/mux.proto | 2 +- node/libs/schema/proto/roles/validator.proto | 14 +- node/libs/schema/src/lib.rs | 2 +- 19 files changed, 173 insertions(+), 93 deletions(-) rename node/libs/protobuf/src/bin/{ => conformance_test}/conformance_test_failure_list.txt (100%) rename node/libs/protobuf/src/bin/{conformance_test.rs => conformance_test/main.rs} (81%) rename node/libs/protobuf/{proto/conformance => src/bin/conformance_test/proto}/conformance.proto (99%) create mode 100644 node/libs/protobuf/src/bin/conformance_test/proto/mod.rs rename node/libs/protobuf/{proto/conformance => src/bin/conformance_test/proto}/test_messages_proto3.proto (99%) create mode 100644 node/libs/protobuf/src/proto/mod.rs rename node/libs/protobuf/{ => src}/proto/std.proto (98%) rename node/libs/protobuf/src/{tests.rs => tests/mod.rs} (93%) create mode 100644 node/libs/protobuf/src/tests/proto/mod.rs rename node/libs/protobuf/{proto/testonly.proto => src/tests/proto/tests.proto} (91%) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 65433f7b..8cae7566 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -6,14 +6,40 @@ fn main() -> anyhow::Result<()> { let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; protobuf_build::Config { - input_path: input.join("proto"), - proto_path: "zksync/protobuf".into(), - proto_package: "zksync.protobuf".into(), + input_path: input.join("src/proto"), + proto_path: "zksync".into(), + proto_package: "zksync".into(), dependencies: vec![], - output_mod_path: output.join("proto/mod.rs"), - output_descriptor_path: output.join("proto/desc.binpb"), + output_mod_path: output.join("src/proto.gen.rs"), + output_descriptor_path: output.join("src/proto.gen.binpb"), - protobuf_crate: "crate".to_string(), - }.generate().context("protobuf_build::Config::generate()") + protobuf_crate: "crate".into(), + }.generate().context("generate(std)")?; + + protobuf_build::Config { + input_path: input.join("src/tests/proto"), + proto_path: "zksync/protobuf/tests".into(), + proto_package: "zksync.protobuf.tests".into(), + dependencies: vec![], + + output_mod_path: output.join("src/tests/proto.gen.rs"), + output_descriptor_path: output.join("src/tests/proto.gen.binpb"), + + protobuf_crate: "crate".into(), + }.generate().context("generate(test)")?; + + protobuf_build::Config { + input_path: input.join("src/bin/conformance_test/proto"), + proto_path: "zksync/protobuf/conformance_test".into(), + proto_package: "zksync.protobuf.conformance_test".into(), + dependencies: vec![], + + output_mod_path: output.join("src/bin/conformance_test/proto.gen.rs"), + output_descriptor_path: output.join("src/bin/conformance_test/proto.gen.binpb"), + + protobuf_crate: "::protobuf".into(), + }.generate().context("generate(conformance)")?; + + Ok(()) } diff --git a/node/libs/protobuf/src/bin/conformance_test_failure_list.txt b/node/libs/protobuf/src/bin/conformance_test/conformance_test_failure_list.txt similarity index 100% rename from node/libs/protobuf/src/bin/conformance_test_failure_list.txt rename to node/libs/protobuf/src/bin/conformance_test/conformance_test_failure_list.txt diff --git a/node/libs/protobuf/src/bin/conformance_test.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs similarity index 81% rename from node/libs/protobuf/src/bin/conformance_test.rs rename to node/libs/protobuf/src/bin/conformance_test/main.rs index b74a6b48..8e8926e8 100644 --- a/node/libs/protobuf/src/bin/conformance_test.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -9,7 +9,8 @@ use anyhow::Context as _; use concurrency::{ctx, io}; use prost::Message as _; use prost_reflect::ReflectMessage; -use protobuf::proto::conformance as proto; + +mod proto; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -26,8 +27,8 @@ async fn main() -> anyhow::Result<()> { let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stdin, &mut msg[..]).await??; - use proto::conformance::conformance_response::Result as R; - let req = proto::conformance::ConformanceRequest::decode(&msg[..])?; + use proto::conformance_response::Result as R; + let req = proto::ConformanceRequest::decode(&msg[..])?; let res = async { let t = req.message_type.context("missing message_type")?; if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { @@ -36,15 +37,15 @@ async fn main() -> anyhow::Result<()> { // Decode. let payload = req.payload.context("missing payload")?; - use proto::test_messages_proto3::TestAllTypesProto3 as T; + use proto::TestAllTypesProto3 as T; let p = match payload { - proto::conformance::conformance_request::Payload::JsonPayload(payload) => { + proto::conformance_request::Payload::JsonPayload(payload) => { match protobuf::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } } - proto::conformance::conformance_request::Payload::ProtobufPayload(payload) => { + proto::conformance_request::Payload::ProtobufPayload(payload) => { // First filter out incorrect encodings. let Ok(p) = T::decode(&payload[..]) else { return Ok(R::ParseError("parsing failed".to_string())); @@ -62,11 +63,11 @@ async fn main() -> anyhow::Result<()> { let format = req .requested_output_format .context("missing output format")?; - match proto::conformance::WireFormat::try_from(format).context("unknown format")? { - proto::conformance::WireFormat::Json => { + match proto::WireFormat::try_from(format).context("unknown format")? { + proto::WireFormat::Json => { anyhow::Ok(R::JsonPayload(protobuf::encode_json_proto(&p))) } - proto::conformance::WireFormat::Protobuf => { + proto::WireFormat::Protobuf => { // Reencode the parsed proto. anyhow::Ok(R::ProtobufPayload(protobuf::canonical_raw( &p.encode_to_vec(), @@ -77,7 +78,7 @@ async fn main() -> anyhow::Result<()> { } } .await?; - let resp = proto::conformance::ConformanceResponse { result: Some(res) }; + let resp = proto::ConformanceResponse { result: Some(res) }; // Write the response. let msg = resp.encode_to_vec(); diff --git a/node/libs/protobuf/proto/conformance/conformance.proto b/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto similarity index 99% rename from node/libs/protobuf/proto/conformance/conformance.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto index fc71bdf4..819bb61d 100644 --- a/node/libs/protobuf/proto/conformance/conformance.proto +++ b/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto @@ -33,7 +33,7 @@ syntax = "proto3"; -package zksync.protobuf.conformance.conformance; +package zksync.protobuf.conformance_test; // This defines the conformance testing protocol. This protocol exists between // the conformance test suite itself and the code being tested. For each test, diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs new file mode 100644 index 00000000..5763bb1f --- /dev/null +++ b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/bin/conformance_test/proto.gen.rs")); diff --git a/node/libs/protobuf/proto/conformance/test_messages_proto3.proto b/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto similarity index 99% rename from node/libs/protobuf/proto/conformance/test_messages_proto3.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto index 7e82c545..242d8f58 100644 --- a/node/libs/protobuf/proto/conformance/test_messages_proto3.proto +++ b/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto @@ -40,7 +40,7 @@ syntax = "proto3"; -package zksync.protobuf.conformance.test_messages_proto3; +package zksync.protobuf.conformance_test; // This proto includes every type of field in both singular and repeated // forms. diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs index 5f4cd7e0..ffc77094 100644 --- a/node/libs/protobuf/src/lib.rs +++ b/node/libs/protobuf/src/lib.rs @@ -4,6 +4,7 @@ mod proto_fmt; mod std_conv; pub mod testonly; +pub mod proto; pub use proto_fmt::*; pub use protobuf_build as build; @@ -11,7 +12,3 @@ pub use protobuf_build as build; #[cfg(test)] mod tests; -#[allow(warnings)] -pub mod proto { - include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); -} diff --git a/node/libs/protobuf/src/proto/mod.rs b/node/libs/protobuf/src/proto/mod.rs new file mode 100644 index 00000000..08ac1876 --- /dev/null +++ b/node/libs/protobuf/src/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/proto.gen.rs")); diff --git a/node/libs/protobuf/proto/std.proto b/node/libs/protobuf/src/proto/std.proto similarity index 98% rename from node/libs/protobuf/proto/std.proto rename to node/libs/protobuf/src/proto/std.proto index c27646b7..05b6957a 100644 --- a/node/libs/protobuf/proto/std.proto +++ b/node/libs/protobuf/src/proto/std.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.protobuf.std; +package zksync.std; message Void {} diff --git a/node/libs/protobuf/src/std_conv.rs b/node/libs/protobuf/src/std_conv.rs index 2f2fa89b..b961fbf3 100644 --- a/node/libs/protobuf/src/std_conv.rs +++ b/node/libs/protobuf/src/std_conv.rs @@ -1,11 +1,11 @@ //! Proto conversion for messages in std package. -use crate::{proto::std as proto, required, ProtoFmt}; +use crate::{proto, required, ProtoFmt}; use anyhow::Context as _; use concurrency::time; use std::net; impl ProtoFmt for () { - type Proto = proto::Void; + type Proto = proto::std::Void; fn read(_r: &Self::Proto) -> anyhow::Result { Ok(()) } @@ -15,7 +15,7 @@ impl ProtoFmt for () { } impl ProtoFmt for std::net::SocketAddr { - type Proto = proto::SocketAddr; + type Proto = proto::std::SocketAddr; fn read(r: &Self::Proto) -> anyhow::Result { let ip = required(&r.ip).context("ip")?; @@ -41,7 +41,7 @@ impl ProtoFmt for std::net::SocketAddr { } impl ProtoFmt for time::Utc { - type Proto = proto::Timestamp; + type Proto = proto::std::Timestamp; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -59,7 +59,7 @@ impl ProtoFmt for time::Utc { } impl ProtoFmt for time::Duration { - type Proto = proto::Duration; + type Proto = proto::std::Duration; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -82,7 +82,7 @@ impl ProtoFmt for time::Duration { } impl ProtoFmt for bit_vec::BitVec { - type Proto = proto::BitVector; + type Proto = proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { let size = *required(&r.size).context("size")? as usize; diff --git a/node/libs/protobuf/src/tests.rs b/node/libs/protobuf/src/tests/mod.rs similarity index 93% rename from node/libs/protobuf/src/tests.rs rename to node/libs/protobuf/src/tests/mod.rs index d4d8366e..162e71d9 100644 --- a/node/libs/protobuf/src/tests.rs +++ b/node/libs/protobuf/src/tests/mod.rs @@ -1,7 +1,9 @@ use super::*; use anyhow::Context as _; use concurrency::{ctx, time}; -use std::net; +use ::std::net; + +mod proto; #[derive(Debug, PartialEq, Eq)] enum B { @@ -10,16 +12,16 @@ enum B { } impl ProtoFmt for B { - type Proto = proto::testonly::B; + type Proto = proto::B; fn read(r: &Self::Proto) -> anyhow::Result { - use proto::testonly::b::T; + use proto::b::T; Ok(match required(&r.t)? { T::U(x) => Self::U(*x), T::V(x) => Self::V(Box::new(ProtoFmt::read(x.as_ref())?)), }) } fn build(&self) -> Self::Proto { - use proto::testonly::b::T; + use proto::b::T; let t = match self { Self::U(x) => T::U(*x), Self::V(x) => T::V(Box::new(x.build())), @@ -37,7 +39,7 @@ struct A { } impl ProtoFmt for A { - type Proto = proto::testonly::A; + type Proto = proto::A; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { x: required(&r.x).context("x")?.clone(), diff --git a/node/libs/protobuf/src/tests/proto/mod.rs b/node/libs/protobuf/src/tests/proto/mod.rs new file mode 100644 index 00000000..43e7742a --- /dev/null +++ b/node/libs/protobuf/src/tests/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/tests/proto.gen.rs")); diff --git a/node/libs/protobuf/proto/testonly.proto b/node/libs/protobuf/src/tests/proto/tests.proto similarity index 91% rename from node/libs/protobuf/proto/testonly.proto rename to node/libs/protobuf/src/tests/proto/tests.proto index b5061bc4..91cf9e35 100644 --- a/node/libs/protobuf/proto/testonly.proto +++ b/node/libs/protobuf/src/tests/proto/tests.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.protobuf.testonly; +package zksync.protobuf.tests; message B { // Recursive union. diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 728dae88..361f806b 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -1,4 +1,8 @@ //! Generates rust code from the protobuf +//! +//! cargo build --all-targets +//! perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find target/debug/build/*/output -type f` | xargs cat > /tmp/sum.binpb +//! buf breaking /tmp/sum.binpb --against /tmp/sum.binpb use anyhow::Context as _; use std::{collections::BTreeMap, fs, path::{PathBuf,Path}}; use prost::Message as _; @@ -122,17 +126,15 @@ fn get_messages(fds: &prost_types::FileDescriptorSet, pool: &prost_reflect::Desc } pub struct Descriptor { - pub rust_module : String, - pub proto_package : String, - pub descriptor_proto : prost_types::FileDescriptorSet, - pub dependencies : Vec<&'static Descriptor>, + pub proto_package: ProtoPackage, + pub descriptor_proto: prost_types::FileDescriptorSet, + pub dependencies: Vec<&'static Descriptor>, } impl Descriptor { - pub fn new(rust_module: &str, proto_package: &str, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { + pub fn new(proto_package: ProtoPackage, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { Descriptor { - rust_module: rust_module.to_string(), - proto_package: proto_package.to_string(), + proto_package, dependencies, descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()).unwrap(), } @@ -157,35 +159,85 @@ impl Descriptor { } } +#[derive(Clone,PartialEq,Eq)] +pub struct RustName(Vec); + +impl RustName { + fn add(mut self, suffix: impl Into) -> Self { + self.0.extend(suffix.into().0.into_iter()); + self + } + + fn to_string(&self) -> String { self.0.join("::") } +} + +impl From<&str> for RustName { + fn from(s:&str) -> Self { Self(s.split("::").map(String::from).collect()) } +} + +#[derive(Clone,PartialEq,Eq)] +pub struct ProtoPackage(Vec); + +impl ProtoPackage { + fn starts_with(&self, prefix: &Self) -> bool { + let n = prefix.0.len(); + self.0.len() >= n && self.0[0..n] == prefix.0 + } + + fn strip_prefix(&self, prefix: &Self) -> anyhow::Result { + if !self.starts_with(prefix) { + anyhow::bail!("{} is not a prefix of {}",prefix.to_string(),self.to_string()); + } + Ok(Self(self.0[prefix.0.len()..].iter().cloned().collect())) + } + + fn to_rust_module(&self) -> RustName { + RustName(self.0.iter().map(|s|ident::to_snake(s)).collect()) + } + + fn to_rust_type(&self) -> RustName { + let n = self.0.len(); + let mut res = self.to_rust_module(); + res.0[n-1] = ident::to_upper_camel(&self.0[n-1]); + res + } + + fn to_string(&self) -> String { + self.0.join(".") + } +} + +impl From<&str> for ProtoPackage { + fn from(s:&str) -> Self { + Self(s.split(".").map(String::from).collect()) + } +} + pub struct Config { pub input_path: PathBuf, pub proto_path: PathBuf, - pub proto_package: String, - pub dependencies: Vec<&'static Descriptor>, + pub proto_package: ProtoPackage, + pub dependencies: Vec<(&'static str, &'static Descriptor)>, pub output_mod_path: PathBuf, pub output_descriptor_path: PathBuf, - pub protobuf_crate: String, + pub protobuf_crate: RustName, } impl Config { - fn import(&self, elem: &str) -> String { - format!("{}::build::{}",self.protobuf_crate,elem) + fn this_crate(&self) -> RustName { + self.protobuf_crate.clone().add("build") } fn reflect_impl(&self, m: prost_reflect::MessageDescriptor) -> String { let proto_name = m.full_name(); - let parts : Vec<&str> = proto_name.strip_prefix(&(self.proto_package.clone() + ".")).unwrap().split(".").collect(); - let mut rust_name : Vec<_> = parts[0..parts.len()-1].iter().map(|p|ident::to_snake(*p)).collect(); - rust_name.push(ident::to_upper_camel(parts[parts.len()-1])); - let rust_name = rust_name.join("::"); - let rust_reflect = self.import("prost_reflect"); - let rust_lazy = self.import("Lazy"); - format!("impl {rust_reflect}::ReflectMessage for {rust_name} {{\ - fn descriptor(&self) -> {rust_reflect}::MessageDescriptor {{\ - static INIT : {rust_lazy}<{rust_reflect}::MessageDescriptor> = {rust_lazy}::new(|| {{\ - DESCRIPTOR.load_global().get_message_by_name(\"{proto_name}\").unwrap()\ + let rust_name = ProtoPackage::from(proto_name).strip_prefix(&self.proto_package).unwrap().to_rust_type().to_string(); + let this = self.this_crate().to_string(); + format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ + fn descriptor(&self) -> {this}::prost_reflect::MessageDescriptor {{\ + static INIT : {this}::Lazy<{this}::prost_reflect::MessageDescriptor> = {this}::Lazy::new(|| {{\ + DESCRIPTOR.load_global().get_message_by_name({proto_name:?}).unwrap()\ }});\ INIT.clone()\ }}\ @@ -197,7 +249,7 @@ impl Config { // Find all proto files. let mut pool = prost_reflect::DescriptorPool::new(); - for d in &self.dependencies { d.load(&mut pool).unwrap(); } + for d in &self.dependencies { d.1.load(&mut pool).unwrap(); } let mut x = prost_types::FileDescriptorSet::default(); x.file = pool.file_descriptor_protos().cloned().collect(); let input_path = self.input_path.canonicalize()?; @@ -217,32 +269,31 @@ impl Config { let mut compiler = protox::Compiler::with_file_resolver( protox::file::DescriptorSetFileResolver::new(x) ); + // TODO: nice compilation errors. compiler.open_files(new_paths).unwrap(); let descriptor = compiler.file_descriptor_set(); + pool.add_file_descriptor_set(descriptor.clone()).unwrap(); fs::create_dir_all(&self.output_descriptor_path.parent().unwrap()).unwrap(); fs::write(&self.output_descriptor_path, &descriptor.encode_to_vec())?; // Generate protobuf code from schema. let mut config = prost_build::Config::new(); - config.prost_path(self.import("prost")); + config.prost_path(self.this_crate().add("prost").to_string()); config.skip_protoc_run(); for d in &self.dependencies { - for f in &d.descriptor_proto.file { - let rel = f.package().strip_prefix(&(d.proto_package.clone() + ".")).unwrap(); - let rust_rel = rel.split(".").map(ident::to_snake).collect::>().join("::"); - config.extern_path(format!(".{}",f.package()), format!("{}::{}",d.rust_module,rust_rel)); + for f in &d.1.descriptor_proto.file { + let proto_rel = ProtoPackage::from(f.package()).strip_prefix(&d.1.proto_package).unwrap(); + let rust_abs = RustName::from(d.0).add(proto_rel.to_rust_module()); + config.extern_path(format!(".{}",f.package()), rust_abs.to_string()); } } // Check that messages are compatible with `proto_fmt::canonical`. - let descriptor = fs::read(&self.output_descriptor_path)?; - let descriptor = prost_types::FileDescriptorSet::decode(&descriptor[..]).unwrap(); for f in &descriptor.file { - if !f.package().starts_with(&self.proto_package) { - anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),self.proto_package); + if !ProtoPackage::from(f.package()).starts_with(&self.proto_package) { + anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),self.proto_package.to_string()); } } - pool.add_file_descriptor_set(descriptor.clone()).unwrap(); let new_messages = get_messages(&descriptor,&pool); let mut check_state = CanonicalCheckState::default(); @@ -256,22 +307,19 @@ impl Config { m.insert(name.parts(),&code); } let mut m = &m; - for name in self.proto_package.split(".") { - m = &m.modules[name]; - } + for part in &self.proto_package.0 { m = &m.modules[part]; } let mut file = m.generate(); for m in &new_messages { file += &self.reflect_impl(m.clone()); } - let rust_deps = self.dependencies.iter().map(|d|format!("&::{}::DESCRIPTOR",d.rust_module)).collect::>().join(","); - let rust_lazy = self.import("Lazy"); - let rust_descriptor = self.import("Descriptor"); + let rust_deps = self.dependencies.iter().map(|d|format!("&{}::DESCRIPTOR",d.0.to_string())).collect::>().join(","); + let this = self.this_crate().to_string(); file += &format!("\ - pub static DESCRIPTOR : {rust_lazy}<{rust_descriptor}> = {rust_lazy}::new(|| {{\ - {rust_descriptor}::new(module_path!(), {:?}, vec![{rust_deps}], &include_bytes!({:?}))\ + pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ + {this}::Descriptor::new({:?}.into(), vec![{rust_deps}], &include_bytes!({:?}))\ }});\ - ",self.proto_package,self.output_descriptor_path); + ",self.proto_package.to_string(),self.output_descriptor_path); let file = syn::parse_str(&file).unwrap(); fs::write(&self.output_mod_path, prettyplease::unparse(&file))?; diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 82c0d329..7c69a5ce 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -9,11 +9,11 @@ fn main() -> anyhow::Result<()> { input_path: input.join("proto"), proto_path: "zksync/schema".into(), proto_package: "zksync.schema".into(), - dependencies: vec![&protobuf::proto::DESCRIPTOR], + dependencies: vec![("::protobuf::proto",&protobuf::proto::DESCRIPTOR)], - output_mod_path: output.join("proto/mod.rs"), - output_descriptor_path: output.join("proto/desc.binpb"), + output_mod_path: output.join("proto.gen.rs"), + output_descriptor_path: output.join("proto.gen.binpb"), - protobuf_crate: "::protobuf".to_string(), - }.generate().context("protobuf_build::Config::generate()") + protobuf_crate: "::protobuf".into(), + }.generate().context("generate()") } diff --git a/node/libs/schema/proto/network/consensus.proto b/node/libs/schema/proto/network/consensus.proto index 234fbe03..09be8a08 100644 --- a/node/libs/schema/proto/network/consensus.proto +++ b/node/libs/schema/proto/network/consensus.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package zksync.schema.network.consensus; import "zksync/schema/roles/validator.proto"; -import "zksync/protobuf/std.proto"; +import "zksync/std.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/libs/schema/proto/network/mux.proto b/node/libs/schema/proto/network/mux.proto index 91965bfc..3e014c8e 100644 --- a/node/libs/schema/proto/network/mux.proto +++ b/node/libs/schema/proto/network/mux.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package zksync.schema.network.mux; -import "zksync/protobuf/std.proto"; +import "zksync/std.proto"; message Handshake { message Capability { diff --git a/node/libs/schema/proto/roles/validator.proto b/node/libs/schema/proto/roles/validator.proto index 068112af..cf9e15e4 100644 --- a/node/libs/schema/proto/roles/validator.proto +++ b/node/libs/schema/proto/roles/validator.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package zksync.schema.roles.validator; -import "zksync/protobuf/std.proto"; +import "zksync/std.proto"; message PayloadHash { optional bytes sha256 = 1; // required @@ -64,20 +64,20 @@ message LeaderCommit { message PrepareQC { repeated ReplicaPrepare msgs = 1; // required - repeated protobuf.std.BitVector signers = 2; // required + repeated std.BitVector signers = 2; // required optional AggregateSignature sig = 3; // required } message CommitQC { optional ReplicaCommit msg = 1; // required - optional protobuf.std.BitVector signers = 2; // required + optional std.BitVector signers = 2; // required optional AggregateSignature sig = 3; // required } message Phase { oneof t { - protobuf.std.Void prepare = 1; - protobuf.std.Void commit = 2; + std.Void prepare = 1; + std.Void commit = 2; } } @@ -88,7 +88,7 @@ message Phase { // network connection to this address. message NetAddress { // Address of the validator. - optional protobuf.std.SocketAddr addr = 1; // required + optional std.SocketAddr addr = 1; // required // Version of the discovery announcement. // Newer (higher) version overrides the older version. // When a validator gets restarted it broadcasts @@ -125,7 +125,7 @@ message NetAddress { // we assume here that it is unlikely for timestamps to collide. // To make this reasoning more strict, we should rather use a random "tie breaker" // instead (replace timestamp with a random nonce, or use a hash of the entire message). - optional protobuf.std.Timestamp timestamp = 3; // required + optional std.Timestamp timestamp = 3; // required } message Msg { diff --git a/node/libs/schema/src/lib.rs b/node/libs/schema/src/lib.rs index b669be2d..443fd1a2 100644 --- a/node/libs/schema/src/lib.rs +++ b/node/libs/schema/src/lib.rs @@ -2,5 +2,5 @@ #[allow(warnings)] pub mod proto { - include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); + include!(concat!(env!("OUT_DIR"), "/proto.gen.rs")); } From e07753280be63093eaa2b131102c169c43100012 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Thu, 2 Nov 2023 23:38:03 +0100 Subject: [PATCH 16/40] reduced config --- node/libs/protobuf/build.rs | 37 ++-------- .../src/bin/conformance_test/proto/mod.rs | 2 +- node/libs/protobuf/src/proto/mod.rs | 2 +- node/libs/protobuf/src/tests/proto/mod.rs | 2 +- node/libs/protobuf_build/src/lib.rs | 71 ++++++++++++++----- node/libs/schema/build.rs | 16 +---- node/libs/schema/src/lib.rs | 2 +- 7 files changed, 67 insertions(+), 65 deletions(-) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 8cae7566..3dfa8077 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -1,45 +1,22 @@ -//! Generates rust code from the capnp schema files in the `capnp/` directory. -use std::{path::PathBuf, env}; -use anyhow::Context as _; - -fn main() -> anyhow::Result<()> { - let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; - let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; +fn main() { protobuf_build::Config { - input_path: input.join("src/proto"), + input_path: "src/proto".into(), proto_path: "zksync".into(), - proto_package: "zksync".into(), dependencies: vec![], - - output_mod_path: output.join("src/proto.gen.rs"), - output_descriptor_path: output.join("src/proto.gen.binpb"), - protobuf_crate: "crate".into(), - }.generate().context("generate(std)")?; + }.generate().expect("generate(std)"); protobuf_build::Config { - input_path: input.join("src/tests/proto"), + input_path: "src/tests/proto".into(), proto_path: "zksync/protobuf/tests".into(), - proto_package: "zksync.protobuf.tests".into(), dependencies: vec![], - - output_mod_path: output.join("src/tests/proto.gen.rs"), - output_descriptor_path: output.join("src/tests/proto.gen.binpb"), - protobuf_crate: "crate".into(), - }.generate().context("generate(test)")?; + }.generate().expect("generate(test)"); protobuf_build::Config { - input_path: input.join("src/bin/conformance_test/proto"), + input_path: "src/bin/conformance_test/proto".into(), proto_path: "zksync/protobuf/conformance_test".into(), - proto_package: "zksync.protobuf.conformance_test".into(), dependencies: vec![], - - output_mod_path: output.join("src/bin/conformance_test/proto.gen.rs"), - output_descriptor_path: output.join("src/bin/conformance_test/proto.gen.binpb"), - protobuf_crate: "::protobuf".into(), - }.generate().context("generate(conformance)")?; - - Ok(()) + }.generate().expect("generate(conformance)"); } diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs index 5763bb1f..11db6a12 100644 --- a/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs +++ b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs @@ -1,2 +1,2 @@ #![allow(warnings)] -include!(concat!(env!("OUT_DIR"), "/src/bin/conformance_test/proto.gen.rs")); +include!(concat!(env!("OUT_DIR"), "/src/bin/conformance_test/proto/gen.rs")); diff --git a/node/libs/protobuf/src/proto/mod.rs b/node/libs/protobuf/src/proto/mod.rs index 08ac1876..660bf4c5 100644 --- a/node/libs/protobuf/src/proto/mod.rs +++ b/node/libs/protobuf/src/proto/mod.rs @@ -1,2 +1,2 @@ #![allow(warnings)] -include!(concat!(env!("OUT_DIR"), "/src/proto.gen.rs")); +include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs")); diff --git a/node/libs/protobuf/src/tests/proto/mod.rs b/node/libs/protobuf/src/tests/proto/mod.rs index 43e7742a..a4703251 100644 --- a/node/libs/protobuf/src/tests/proto/mod.rs +++ b/node/libs/protobuf/src/tests/proto/mod.rs @@ -1,2 +1,2 @@ #![allow(warnings)] -include!(concat!(env!("OUT_DIR"), "/src/tests/proto.gen.rs")); +include!(concat!(env!("OUT_DIR"), "/src/tests/proto/gen.rs")); diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 361f806b..b2e8f73f 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -159,6 +159,30 @@ impl Descriptor { } } +/// Path relative to $CARGO_MANIFEST_DIR +#[derive(Clone,PartialEq,Eq)] +pub struct InputPath(PathBuf); + +impl From<&str> for InputPath { + fn from(s:&str) -> Self { Self(PathBuf::from(s)) } +} + +impl InputPath { + fn abs(&self) -> PathBuf { + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) + .canonicalize().unwrap().join(&self.0) + } + + /// Path relative to $OUT_DIR with the given extenstion + fn prepare_output_dir(&self) -> PathBuf { + let output = PathBuf::from(std::env::var("OUT_DIR").unwrap()) + .canonicalize().unwrap().join(&self.0); + let _ = fs::remove_dir_all(&output); + fs::create_dir_all(&output).unwrap(); + output + } +} + #[derive(Clone,PartialEq,Eq)] pub struct RustName(Vec); @@ -213,26 +237,31 @@ impl From<&str> for ProtoPackage { } } +impl From<&Path> for ProtoPackage { + fn from(p:&Path) -> Self { + Self(p.iter().map(|c|c.to_str().unwrap().to_string()).collect()) + } +} + pub struct Config { - pub input_path: PathBuf, + pub input_path: InputPath, pub proto_path: PathBuf, - pub proto_package: ProtoPackage, pub dependencies: Vec<(&'static str, &'static Descriptor)>, - - pub output_mod_path: PathBuf, - pub output_descriptor_path: PathBuf, - pub protobuf_crate: RustName, } impl Config { + fn proto_package(&self) -> ProtoPackage { + ProtoPackage::from(&*self.proto_path) + } + fn this_crate(&self) -> RustName { self.protobuf_crate.clone().add("build") } fn reflect_impl(&self, m: prost_reflect::MessageDescriptor) -> String { let proto_name = m.full_name(); - let rust_name = ProtoPackage::from(proto_name).strip_prefix(&self.proto_package).unwrap().to_rust_type().to_string(); + let rust_name = ProtoPackage::from(proto_name).strip_prefix(&self.proto_package()).unwrap().to_rust_type().to_string(); let this = self.this_crate().to_string(); format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ fn descriptor(&self) -> {this}::prost_reflect::MessageDescriptor {{\ @@ -245,14 +274,15 @@ impl Config { } pub fn generate(&self) -> anyhow::Result<()> { - println!("cargo:rerun-if-changed={:?}", self.input_path); + let input_path = self.input_path.abs(); + assert!(input_path.is_dir()); + println!("cargo:rerun-if-changed={input_path:?}"); // Find all proto files. let mut pool = prost_reflect::DescriptorPool::new(); for d in &self.dependencies { d.1.load(&mut pool).unwrap(); } let mut x = prost_types::FileDescriptorSet::default(); x.file = pool.file_descriptor_protos().cloned().collect(); - let input_path = self.input_path.canonicalize()?; let mut new_paths = vec![]; traverse_files(&input_path, &mut |path| { let Some(ext) = path.extension() else { return Ok(()) }; @@ -273,10 +303,15 @@ impl Config { compiler.open_files(new_paths).unwrap(); let descriptor = compiler.file_descriptor_set(); pool.add_file_descriptor_set(descriptor.clone()).unwrap(); - fs::create_dir_all(&self.output_descriptor_path.parent().unwrap()).unwrap(); - fs::write(&self.output_descriptor_path, &descriptor.encode_to_vec())?; // Generate protobuf code from schema. + let output_dir = self.input_path.prepare_output_dir(); + let output_path = output_dir.join("gen.rs"); + let descriptor_path = output_dir.join("gen.binpb"); + println!("PROTOBUF_DESCRIPTOR={descriptor_path:?}"); + + fs::write(&descriptor_path, &descriptor.encode_to_vec())?; + let mut config = prost_build::Config::new(); config.prost_path(self.this_crate().add("prost").to_string()); config.skip_protoc_run(); @@ -289,9 +324,10 @@ impl Config { } // Check that messages are compatible with `proto_fmt::canonical`. + let proto_package = self.proto_package(); for f in &descriptor.file { - if !ProtoPackage::from(f.package()).starts_with(&self.proto_package) { - anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),self.proto_package.to_string()); + if !ProtoPackage::from(f.package()).starts_with(&proto_package) { + anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),proto_package.to_string()); } } let new_messages = get_messages(&descriptor,&pool); @@ -307,7 +343,7 @@ impl Config { m.insert(name.parts(),&code); } let mut m = &m; - for part in &self.proto_package.0 { m = &m.modules[part]; } + for part in &proto_package.0 { m = &m.modules[part]; } let mut file = m.generate(); for m in &new_messages { file += &self.reflect_impl(m.clone()); @@ -317,13 +353,12 @@ impl Config { let this = self.this_crate().to_string(); file += &format!("\ pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ - {this}::Descriptor::new({:?}.into(), vec![{rust_deps}], &include_bytes!({:?}))\ + {this}::Descriptor::new({:?}.into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?}))\ }});\ - ",self.proto_package.to_string(),self.output_descriptor_path); + ",proto_package.to_string()); let file = syn::parse_str(&file).unwrap(); - fs::write(&self.output_mod_path, prettyplease::unparse(&file))?; - println!("PROTOBUF_DESCRIPTOR={:?}",self.output_descriptor_path); + fs::write(&output_path, prettyplease::unparse(&file))?; Ok(()) } } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 7c69a5ce..1c8799fe 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,19 +1,9 @@ //! Generates rust code from the capnp schema files in the `capnp/` directory. -use std::{env, path::PathBuf}; -use anyhow::Context as _; - -fn main() -> anyhow::Result<()> { - let input = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).canonicalize()?; - let output = PathBuf::from(std::env::var("OUT_DIR")?).canonicalize()?; +fn main() { protobuf::build::Config { - input_path: input.join("proto"), + input_path: "proto".into(), proto_path: "zksync/schema".into(), - proto_package: "zksync.schema".into(), dependencies: vec![("::protobuf::proto",&protobuf::proto::DESCRIPTOR)], - - output_mod_path: output.join("proto.gen.rs"), - output_descriptor_path: output.join("proto.gen.binpb"), - protobuf_crate: "::protobuf".into(), - }.generate().context("generate()") + }.generate().expect("generate()"); } diff --git a/node/libs/schema/src/lib.rs b/node/libs/schema/src/lib.rs index 443fd1a2..e1891ab1 100644 --- a/node/libs/schema/src/lib.rs +++ b/node/libs/schema/src/lib.rs @@ -2,5 +2,5 @@ #[allow(warnings)] pub mod proto { - include!(concat!(env!("OUT_DIR"), "/proto.gen.rs")); + include!(concat!(env!("OUT_DIR"), "/proto/gen.rs")); } From d81b815155658783041accbeb6e8d6a6fc4bd471 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 11:36:09 +0100 Subject: [PATCH 17/40] documentation WIP --- node/libs/protobuf_build/src/canonical.rs | 57 ++++ node/libs/protobuf_build/src/lib.rs | 316 ++++++++++------------ node/libs/protobuf_build/src/rust.rs | 57 ++++ 3 files changed, 257 insertions(+), 173 deletions(-) create mode 100644 node/libs/protobuf_build/src/canonical.rs create mode 100644 node/libs/protobuf_build/src/rust.rs diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs new file mode 100644 index 00000000..ab545293 --- /dev/null +++ b/node/libs/protobuf_build/src/canonical.rs @@ -0,0 +1,57 @@ +use anyhow::Context as _; +use std::collections::HashSet; + +#[derive(Default)] +struct Check(HashSet); + +impl Check { + /// Checks if messages of type `m` support canonical encoding. + fn message(&mut self, m: &prost_reflect::MessageDescriptor, check_nested: bool) -> anyhow::Result<()> { + if self.0.contains(m.full_name()) { + return Ok(()); + } + self.0.insert(m.full_name().to_string()); + for f in m.fields() { + self.field(&f).with_context(|| f.name().to_string())?; + } + if check_nested { + for m in m.child_messages() { + self.message(&m,check_nested).with_context(||m.name().to_string())?; + } + } + Ok(()) + } + + /// Checks if field `f` supports canonical encoding. + fn field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + if f.is_map() { + anyhow::bail!("maps unsupported"); + } + if !f.is_list() && !f.supports_presence() { + anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); + } + if let prost_reflect::Kind::Message(msg) = f.kind() { + self.message(&msg,false).with_context(||msg.name().to_string())?; + } + Ok(()) + } + + /// Checks if message types in file `f` support canonical encoding. + fn file(&mut self, f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { + if f.syntax() != prost_reflect::Syntax::Proto3 { + anyhow::bail!("only proto3 syntax is supported"); + } + for m in f.messages() { + self.message(&m,true).with_context(|| m.name().to_string())?; + } + Ok(()) + } +} + +pub fn check(descriptor: &prost_types::FileDescriptorSet, pool: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { + let mut c = Check::default(); + for f in &descriptor.file { + c.file(f).with_context(||f.name())?; + } + Ok(()) +} diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index b2e8f73f..cfc4ffde 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -1,4 +1,46 @@ -//! Generates rust code from the protobuf +//! Generates rust code from the proto files. +//! +//! Basic concepts: +//! * input path - absolute path of the proto file as visible in the file system. Example: +//! $CARGO_MANIFEST_DIR//some/location/abc.proto +//! * output path - absolute path of the output files (generated by this library) as visible in the +//! file system. Output paths are derived from the input paths to keeps things clear. Example: +//! $OUT_DIR//generated_file +//! * proto path - absolute path of the proto file used for importing other proto files. +//! These are derived from field in the config and the input path. Example: +//! /some/location/abc.proto +//! * proto package - in addition to the space of input paths and the space of proto paths, there is also +//! a space of proto packages which is used to reference message types from different proto +//! files. Theoretically it can be totally independent from the proto paths (i.e. you can have +//! "my.favorite.package" proto package defined in "totally/unrelated/file/name.proto", but we +//! recommend the following naming convention: proto package "a.b.c" should be defined either: +//! a) in a single file "a/b/c.proto", or +//! b) in a collection of files under "a/b/c/" directory +//! Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. +//! * rust path - a rust module path that the generated code is available at TODO +//! +//! Protobuf files are collected recursively from $CARGO_MANIFEST_DIR// directory. +//! Corresponding "cargo:rerun-if-changed=..." line is printed to stdout, so that +//! the build script running this function is rerun whenever proto files change. +//! A single rust file is generated and stored at $OUT_DIR//gen.rs file. +//! +//! Protobuf files are compiled to a protobuf descriptor stored at +//! $OUT_DIR//gen.binpb. +//! Additionally a "PROTOBUF_DESCRIPTOR=" line is printed to +//! stdout. This can be used to collect all the descriptors across the build as follows: +//! 1. Checkout the repo to a fresh directory and then run "cargo build --all-targets" +//! We need it fresh so that every crate containing protobufs has only one build in the +//! cargo cache. +//! 2. grep through all target/debug/build/*/output files to find all "PROTOBUF_DESCRIPTOR=..." +//! lines and merge the descriptor files by simply concatenating them. +//! Note that you can run this procedure for 2 revisions of the repo and look for breaking +//! changes by running "buf breaking --against " where before.binpb +//! and after.binpb are the concatenated descriptors from those 2 revisions. +//! +//! The proto files are not expected to be self-contained - to import proto files from +//! different crates you need to specify them as dependencies in the Config.dependencies. +//! It is not possible to depend on a different proto bundle within the same crate (because +//! these are being built simultaneously from the same build script). //! //! cargo build --all-targets //! perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find target/debug/build/*/output -type f` | xargs cat > /tmp/sum.binpb @@ -9,11 +51,14 @@ use prost::Message as _; use std::collections::HashSet; use std::sync::Mutex; +// Imports accessed from the generated code. pub use prost; pub use prost_reflect; -pub use once_cell::sync::{Lazy}; +pub use once_cell::sync::Lazy; mod ident; +mod canonical; +pub mod rust; /// Traversed all the files in a directory recursively. fn traverse_files(path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>) -> anyhow::Result<()> { @@ -27,111 +72,15 @@ fn traverse_files(path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>) Ok(()) } -/// A rust module representation. -/// It is used to collect the generated protobuf code. -#[derive(Default)] -struct Module { - /// Nested modules which transitively contain the generated code. - modules: BTreeMap, - /// Code of the module. - code: String, -} - -impl Module { - /// Inserts a nested generated protobuf module. - /// `name` is a sequence of module names. - fn insert<'a>(&mut self, mut name: impl Iterator, code: &str) { - match name.next() { - None => self.code += code, - Some(module) => self.modules.entry(module.to_string()).or_default().insert(name,code), - } - } - - /// Generates rust code of the module. - fn generate(&self) -> String { - let mut entries = vec![self.code.clone()]; - entries.extend( - self.modules.iter().map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.generate())), - ); - entries.join("") - } -} - -#[derive(Default)] -struct CanonicalCheckState(HashSet); - -impl CanonicalCheckState { - /// Checks if messages of type `m` support canonical encoding. - fn check_message(&mut self, m: &prost_reflect::MessageDescriptor, check_nested: bool) -> anyhow::Result<()> { - if self.0.contains(m.full_name()) { - return Ok(()); - } - self.0.insert(m.full_name().to_string()); - for f in m.fields() { - self.check_field(&f).with_context(|| f.name().to_string())?; - } - if check_nested { - for m in m.child_messages() { - self.check_message(&m,check_nested).with_context(||m.name().to_string())?; - } - } - Ok(()) - } - - /// Checks if field `f` supports canonical encoding. - fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { - if f.is_map() { - anyhow::bail!("maps unsupported"); - } - if !f.is_list() && !f.supports_presence() { - anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); - } - if let prost_reflect::Kind::Message(msg) = f.kind() { - self.check_message(&msg,false).with_context(||msg.name().to_string())?; - } - Ok(()) - } - - /*/// Checks if message types in file `f` support canonical encoding. - fn check_file(&mut self, f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { - if f.syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - for m in f.messages() { - self.check_message(&m,true).with_context(|| m.name().to_string())?; - } - Ok(()) - }*/ -} - -fn get_messages_from_message(out: &mut Vec, m: prost_reflect::MessageDescriptor) { - for m in m.child_messages() { - get_messages_from_message(out,m); - } - out.push(m); -} - -fn get_messages_from_file(out: &mut Vec, f: prost_reflect::FileDescriptor) { - for m in f.messages() { - get_messages_from_message(out,m); - } -} - -fn get_messages(fds: &prost_types::FileDescriptorSet, pool: &prost_reflect::DescriptorPool) -> Vec { - let mut res = vec![]; - for f in &fds.file { - get_messages_from_file(&mut res,pool.get_file_by_name(f.name()).unwrap()); - } - res -} - +/// Protobuf descriptor + info about the mapping to rust code. pub struct Descriptor { - pub proto_package: ProtoPackage, - pub descriptor_proto: prost_types::FileDescriptorSet, - pub dependencies: Vec<&'static Descriptor>, + proto_package: ProtoPackage, + descriptor_proto: prost_types::FileDescriptorSet, + dependencies: Vec<&'static Descriptor>, } impl Descriptor { + /// Constructs a Descriptor. pub fn new(proto_package: ProtoPackage, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { Descriptor { proto_package, @@ -140,6 +89,7 @@ impl Descriptor { } } + /// Loads the descriptor to the pool, if not already loaded. pub fn load(&self, pool: &mut prost_reflect::DescriptorPool) -> anyhow::Result<()> { if self.descriptor_proto.file.iter().all(|f| pool.get_file_by_name(f.name()).is_some()) { return Ok(()); @@ -151,6 +101,7 @@ impl Descriptor { Ok(()) } + /// Loads the descriptor to the global pool and returns a copy of the global pool. pub fn load_global(&self) -> prost_reflect::DescriptorPool { static POOL : Lazy> = Lazy::new(||Mutex::default()); let pool = &mut POOL.lock().unwrap(); @@ -183,26 +134,12 @@ impl InputPath { } } -#[derive(Clone,PartialEq,Eq)] -pub struct RustName(Vec); - -impl RustName { - fn add(mut self, suffix: impl Into) -> Self { - self.0.extend(suffix.into().0.into_iter()); - self - } - - fn to_string(&self) -> String { self.0.join("::") } -} - -impl From<&str> for RustName { - fn from(s:&str) -> Self { Self(s.split("::").map(String::from).collect()) } -} - +/// Represents a (relative) proto package path. #[derive(Clone,PartialEq,Eq)] pub struct ProtoPackage(Vec); impl ProtoPackage { + /// Checks if package path starts with the given prefix. fn starts_with(&self, prefix: &Self) -> bool { let n = prefix.0.len(); self.0.len() >= n && self.0[0..n] == prefix.0 @@ -215,11 +152,11 @@ impl ProtoPackage { Ok(Self(self.0[prefix.0.len()..].iter().cloned().collect())) } - fn to_rust_module(&self) -> RustName { + fn to_rust_module(&self) -> rust::Name { RustName(self.0.iter().map(|s|ident::to_snake(s)).collect()) } - fn to_rust_type(&self) -> RustName { + fn to_rust_type(&self) -> rust::Name { let n = self.0.len(); let mut res = self.to_rust_module(); res.0[n-1] = ident::to_upper_camel(&self.0[n-1]); @@ -244,24 +181,51 @@ impl From<&Path> for ProtoPackage { } pub struct Config { + /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. pub input_path: InputPath, + /// Implicit prefix that should be prepended to proto paths of the proto files in the input directory. pub proto_path: PathBuf, - pub dependencies: Vec<(&'static str, &'static Descriptor)>, - pub protobuf_crate: RustName, + /// Descriptors of the dependencies and the rust absolute paths under which they will be available from the generated code. + pub dependencies: Vec<(rust::Name, &'static Descriptor)>, + /// Rust absolute path under which the protobuf crate will be available from the generated + /// code. + pub protobuf_crate: rust::Name, +} + +/// Extracts names of proto messages defined in the descriptor. +fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { + fn collect(out: &mut Vec, m: &prost_types::DescriptorProto) { + for m in m.child_messages() { + collect(out,m); + } + out.push(m.name().into()); + } + let mut res = vec![]; + for f in &descriptor.file { + for m in &f.message_type { + collect(res,m); + } + } + res } impl Config { + /// Proto package that all compiled proto files are supposed to belong to. + /// It is derived from `proto_path` by simply replacing all '/' with '.'. fn proto_package(&self) -> ProtoPackage { ProtoPackage::from(&*self.proto_path) } - fn this_crate(&self) -> RustName { + /// Location of the protobuf_build crate, visible from the generated code. + fn this_crate(&self) -> rust::Name { self.protobuf_crate.clone().add("build") } - fn reflect_impl(&self, m: prost_reflect::MessageDescriptor) -> String { - let proto_name = m.full_name(); - let rust_name = ProtoPackage::from(proto_name).strip_prefix(&self.proto_package()).unwrap().to_rust_type().to_string(); + /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated + /// from a message of the given `proto_name`. + fn reflect_impl(&self, proto_name: ProtoPackage) -> String { + let rust_name = proto_name.strip_prefix(&self.proto_package()).unwrap().to_rust_type().to_string(); + let proto_name = proto_name.to_string(); let this = self.this_crate().to_string(); format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ fn descriptor(&self) -> {this}::prost_reflect::MessageDescriptor {{\ @@ -273,92 +237,98 @@ impl Config { }}") } + /// Generates rust code from the proto files according to the config. pub fn generate(&self) -> anyhow::Result<()> { let input_path = self.input_path.abs(); - assert!(input_path.is_dir()); + assert!(input_path.is_dir(),"input_path should be a directory"); println!("cargo:rerun-if-changed={input_path:?}"); - // Find all proto files. + // Load dependencies. let mut pool = prost_reflect::DescriptorPool::new(); - for d in &self.dependencies { d.1.load(&mut pool).unwrap(); } - let mut x = prost_types::FileDescriptorSet::default(); - x.file = pool.file_descriptor_protos().cloned().collect(); - let mut new_paths = vec![]; + for d in &self.dependencies { + d.1.load(&mut pool).with_context(||format!("failed to load dependency {}",d.0))?; + } + let mut pool_raw = prost_types::FileDescriptorSet::default(); + pool_raw.file = pool.file_descriptor_protos().cloned().collect(); + + // Load proto files. + let mut proto_paths = vec![]; traverse_files(&input_path, &mut |path| { let Some(ext) = path.extension() else { return Ok(()) }; let Some(ext) = ext.to_str() else { return Ok(()) }; if ext != "proto" { return Ok(()) }; - let rel_path = self.proto_path.join(path.strip_prefix(&input_path).unwrap()); - x.file.push(protox_parse::parse( - rel_path.to_str().unwrap(), - &fs::read_to_string(path).context("fs::read()")?, - ).context("protox_parse::parse()")?); - new_paths.push(rel_path.clone()); + + let file_raw = fs::read_to_string(path).context("fs::read()")?; + // Replace `input_path` with `proto_path` in path. + let path = self.proto_path.join(path.strip_prefix(&input_path).unwrap()); + pool_raw.file.push(protox_parse::parse(path.to_str().unwrap(),&file_raw)) + .context("protox_parse::parse()")?; + proto_paths.push(path); Ok(()) })?; + + // Compile the proto files let mut compiler = protox::Compiler::with_file_resolver( - protox::file::DescriptorSetFileResolver::new(x) + protox::file::DescriptorSetFileResolver::new(pool_raw) ); // TODO: nice compilation errors. compiler.open_files(new_paths).unwrap(); let descriptor = compiler.file_descriptor_set(); pool.add_file_descriptor_set(descriptor.clone()).unwrap(); - - // Generate protobuf code from schema. + + // Check that the compiled proto files belong to the declared proto package + // and that proto messages support canonical encoding. + let proto_package = self.proto_package(); + for f in &descriptor.file { + if !ProtoPackage::from(f.package()).starts_with(&proto_package) { + anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),proto_package.to_string()); + } + } + + // Check that the compiled proto messages support canonical encoding. + canonical::check(&descriptor,&pool).context("canonical::check()")?; + + // Prepare the output directory. let output_dir = self.input_path.prepare_output_dir(); let output_path = output_dir.join("gen.rs"); let descriptor_path = output_dir.join("gen.binpb"); - println!("PROTOBUF_DESCRIPTOR={descriptor_path:?}"); - fs::write(&descriptor_path, &descriptor.encode_to_vec())?; + println!("PROTOBUF_DESCRIPTOR={descriptor_path:?}"); + // Generate code out of compiled proto files. + let mut output = rust::Module::default(); let mut config = prost_build::Config::new(); config.prost_path(self.this_crate().add("prost").to_string()); config.skip_protoc_run(); for d in &self.dependencies { for f in &d.1.descriptor_proto.file { let proto_rel = ProtoPackage::from(f.package()).strip_prefix(&d.1.proto_package).unwrap(); - let rust_abs = RustName::from(d.0).add(proto_rel.to_rust_module()); + let rust_abs = rust::Name::from(d.0).add(proto_rel.to_rust_module()); config.extern_path(format!(".{}",f.package()), rust_abs.to_string()); } } - - // Check that messages are compatible with `proto_fmt::canonical`. - let proto_package = self.proto_package(); - for f in &descriptor.file { - if !ProtoPackage::from(f.package()).starts_with(&proto_package) { - anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),proto_package.to_string()); - } - } - let new_messages = get_messages(&descriptor,&pool); - - let mut check_state = CanonicalCheckState::default(); - for m in &new_messages { - check_state.check_message(m,false)?; - } - let modules : Vec<_> = descriptor.file.iter().map(|d|(prost_build::Module::from_protobuf_package_name(d.package()),d.clone())).collect(); - let mut m = Module::default(); - for (name,code) in config.generate(modules).unwrap() { - m.insert(name.parts(),&code); + for (name,code) in config.generate(modules).expect("generation failed") { + output.insert(&name.into(),&code); } - let mut m = &m; - for part in &proto_package.0 { m = &m.modules[part]; } - let mut file = m.generate(); - for m in &new_messages { - file += &self.reflect_impl(m.clone()); + + // Generate the reflection code. + let output = output.sub(&self.proto_path.to_rust_module()); + for proto_name in extract_message_names(&descriptor) { + output.append_code(&self.reflect_impl(proto_name)); } - let rust_deps = self.dependencies.iter().map(|d|format!("&{}::DESCRIPTOR",d.0.to_string())).collect::>().join(","); + // Generate the descriptor. + let rust_deps = self.dependencies.iter().map(|d|format!("&{}::DESCRIPTOR",d.0)).collect::>().join(","); let this = self.this_crate().to_string(); - file += &format!("\ + output.append_code(&format!("\ pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ {this}::Descriptor::new({:?}.into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?}))\ }});\ - ",proto_package.to_string()); + ",proto_package.to_string())); - let file = syn::parse_str(&file).unwrap(); - fs::write(&output_path, prettyplease::unparse(&file))?; + // Save output. + fs::write(&output_path, output.format())?; Ok(()) } } diff --git a/node/libs/protobuf_build/src/rust.rs b/node/libs/protobuf_build/src/rust.rs new file mode 100644 index 00000000..267b6ce9 --- /dev/null +++ b/node/libs/protobuf_build/src/rust.rs @@ -0,0 +1,57 @@ +use std::collections::BTreeMap; + +type Part = String; + +#[derive(Clone,PartialEq,Eq)] +pub struct Name(Vec); + +impl Name { + fn add(mut self, suffix: impl Into) -> Self { + self.0.extend(suffix.into().0.into_iter()); + self + } + + fn to_string(&self) -> String { self.0.join("::") } +} + +impl From for Name { + fn from(s:prost_build::Module) -> Self { Self(s.parts().map(Part::from).collect()) } +} + +impl From<&str> for Name { + fn from(s:&str) -> Self { Self(s.split("::").map(Part::from).collect()) } +} + +/// A rust module representation. +/// It is used to collect the generated protobuf code. +#[derive(Default)] +pub(super) struct Module { + /// Nested modules which transitively contain the generated code. + modules: BTreeMap, + /// Code of the module. + code: String, +} + +impl Module { + pub fn sub(&self, path: &Name) -> &mut Self { + let mut m = self; + for part in &path.0 { + m = m.modules.entry(part.into()).or_default(); + } + m + } + + /// Appends code to the module. + pub fn append(&mut self, code: &str) { + self.code += code; + } + + /// Collects the code of the module. + pub fn format(&self) -> anyhow::Result { + let mut entries = vec![self.code.clone()]; + entries.extend( + self.modules.iter().map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), + ); + prettyplease::unparse(syn::parse_str(&entries.join("")).context("syn::parse_str()")?) + } +} From 87bae087f59bc5198fc182bdbe6d1eca52059edc Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 13:13:23 +0100 Subject: [PATCH 18/40] snapshot --- node/libs/protobuf_build/src/canonical.rs | 36 ++--- node/libs/protobuf_build/src/lib.rs | 181 +++++---------------- node/libs/protobuf_build/src/rust.rs | 57 ------- node/libs/protobuf_build/src/syntax.rs | 185 ++++++++++++++++++++++ 4 files changed, 236 insertions(+), 223 deletions(-) delete mode 100644 node/libs/protobuf_build/src/rust.rs create mode 100644 node/libs/protobuf_build/src/syntax.rs diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs index ab545293..5ec4ed09 100644 --- a/node/libs/protobuf_build/src/canonical.rs +++ b/node/libs/protobuf_build/src/canonical.rs @@ -1,29 +1,25 @@ use anyhow::Context as _; use std::collections::HashSet; +use super::proto; #[derive(Default)] struct Check(HashSet); impl Check { /// Checks if messages of type `m` support canonical encoding. - fn message(&mut self, m: &prost_reflect::MessageDescriptor, check_nested: bool) -> anyhow::Result<()> { + fn message(&mut self, m: prost_reflect::MessageDescriptor) -> anyhow::Result<()> { if self.0.contains(m.full_name()) { return Ok(()); } self.0.insert(m.full_name().to_string()); for f in m.fields() { - self.field(&f).with_context(|| f.name().to_string())?; - } - if check_nested { - for m in m.child_messages() { - self.message(&m,check_nested).with_context(||m.name().to_string())?; - } + self.field(f).with_context(|| f.name().to_string())?; } Ok(()) } /// Checks if field `f` supports canonical encoding. - fn field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + fn field(&mut self, f: prost_reflect::FieldDescriptor) -> anyhow::Result<()> { if f.is_map() { anyhow::bail!("maps unsupported"); } @@ -31,27 +27,23 @@ impl Check { anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); } if let prost_reflect::Kind::Message(msg) = f.kind() { - self.message(&msg,false).with_context(||msg.name().to_string())?; + self.message(msg).with_context(||msg.name().to_string())?; } Ok(()) - } + } +} - /// Checks if message types in file `f` support canonical encoding. - fn file(&mut self, f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { - if f.syntax() != prost_reflect::Syntax::Proto3 { +pub fn check(descriptor: &prost_types::FileDescriptorSet, pool: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { + for f in &descriptor.file { + if f.syntax() != "proto3" { anyhow::bail!("only proto3 syntax is supported"); } - for m in f.messages() { - self.message(&m,true).with_context(|| m.name().to_string())?; - } - Ok(()) } -} - -pub fn check(descriptor: &prost_types::FileDescriptorSet, pool: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { let mut c = Check::default(); - for f in &descriptor.file { - c.file(f).with_context(||f.name())?; + for msg_name in proto::extract_message_names(descriptor) { + let msg_name = msg_name.to_string(); + let msg = pool.get_message_by_name(&msg_name).with_context(||format!("{msg_name} not found in pool"))?; + c.message(msg).with_context(||msg_name)?; } Ok(()) } diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index cfc4ffde..2d53dff7 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -1,31 +1,12 @@ //! Generates rust code from the proto files. //! -//! Basic concepts: -//! * input path - absolute path of the proto file as visible in the file system. Example: -//! $CARGO_MANIFEST_DIR//some/location/abc.proto -//! * output path - absolute path of the output files (generated by this library) as visible in the -//! file system. Output paths are derived from the input paths to keeps things clear. Example: -//! $OUT_DIR//generated_file -//! * proto path - absolute path of the proto file used for importing other proto files. -//! These are derived from field in the config and the input path. Example: -//! /some/location/abc.proto -//! * proto package - in addition to the space of input paths and the space of proto paths, there is also -//! a space of proto packages which is used to reference message types from different proto -//! files. Theoretically it can be totally independent from the proto paths (i.e. you can have -//! "my.favorite.package" proto package defined in "totally/unrelated/file/name.proto", but we -//! recommend the following naming convention: proto package "a.b.c" should be defined either: -//! a) in a single file "a/b/c.proto", or -//! b) in a collection of files under "a/b/c/" directory -//! Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. -//! * rust path - a rust module path that the generated code is available at TODO -//! -//! Protobuf files are collected recursively from $CARGO_MANIFEST_DIR// directory. +//! Protobuf files are collected recursively from $CARGO_MANIFEST_DIR// directory. //! Corresponding "cargo:rerun-if-changed=..." line is printed to stdout, so that //! the build script running this function is rerun whenever proto files change. -//! A single rust file is generated and stored at $OUT_DIR//gen.rs file. +//! A single rust file is generated and stored at $OUT_DIR//gen.rs file. //! //! Protobuf files are compiled to a protobuf descriptor stored at -//! $OUT_DIR//gen.binpb. +//! $OUT_DIR//gen.binpb. //! Additionally a "PROTOBUF_DESCRIPTOR=" line is printed to //! stdout. This can be used to collect all the descriptors across the build as follows: //! 1. Checkout the repo to a fresh directory and then run "cargo build --all-targets" @@ -46,10 +27,10 @@ //! perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find target/debug/build/*/output -type f` | xargs cat > /tmp/sum.binpb //! buf breaking /tmp/sum.binpb --against /tmp/sum.binpb use anyhow::Context as _; -use std::{collections::BTreeMap, fs, path::{PathBuf,Path}}; +use std::{fs, path::{PathBuf,Path}}; use prost::Message as _; -use std::collections::HashSet; use std::sync::Mutex; +pub use syntax::*; // Imports accessed from the generated code. pub use prost; @@ -58,7 +39,7 @@ pub use once_cell::sync::Lazy; mod ident; mod canonical; -pub mod rust; +mod syntax; /// Traversed all the files in a directory recursively. fn traverse_files(path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>) -> anyhow::Result<()> { @@ -74,14 +55,14 @@ fn traverse_files(path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>) /// Protobuf descriptor + info about the mapping to rust code. pub struct Descriptor { - proto_package: ProtoPackage, + proto_package: ProtoName, descriptor_proto: prost_types::FileDescriptorSet, dependencies: Vec<&'static Descriptor>, } impl Descriptor { /// Constructs a Descriptor. - pub fn new(proto_package: ProtoPackage, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { + pub fn new(proto_package: ProtoName, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { Descriptor { proto_package, dependencies, @@ -110,120 +91,33 @@ impl Descriptor { } } -/// Path relative to $CARGO_MANIFEST_DIR -#[derive(Clone,PartialEq,Eq)] -pub struct InputPath(PathBuf); - -impl From<&str> for InputPath { - fn from(s:&str) -> Self { Self(PathBuf::from(s)) } -} - -impl InputPath { - fn abs(&self) -> PathBuf { - PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) - .canonicalize().unwrap().join(&self.0) - } - - /// Path relative to $OUT_DIR with the given extenstion - fn prepare_output_dir(&self) -> PathBuf { - let output = PathBuf::from(std::env::var("OUT_DIR").unwrap()) - .canonicalize().unwrap().join(&self.0); - let _ = fs::remove_dir_all(&output); - fs::create_dir_all(&output).unwrap(); - output - } -} - -/// Represents a (relative) proto package path. -#[derive(Clone,PartialEq,Eq)] -pub struct ProtoPackage(Vec); - -impl ProtoPackage { - /// Checks if package path starts with the given prefix. - fn starts_with(&self, prefix: &Self) -> bool { - let n = prefix.0.len(); - self.0.len() >= n && self.0[0..n] == prefix.0 - } - - fn strip_prefix(&self, prefix: &Self) -> anyhow::Result { - if !self.starts_with(prefix) { - anyhow::bail!("{} is not a prefix of {}",prefix.to_string(),self.to_string()); - } - Ok(Self(self.0[prefix.0.len()..].iter().cloned().collect())) - } - - fn to_rust_module(&self) -> rust::Name { - RustName(self.0.iter().map(|s|ident::to_snake(s)).collect()) - } - - fn to_rust_type(&self) -> rust::Name { - let n = self.0.len(); - let mut res = self.to_rust_module(); - res.0[n-1] = ident::to_upper_camel(&self.0[n-1]); - res - } - - fn to_string(&self) -> String { - self.0.join(".") - } -} - -impl From<&str> for ProtoPackage { - fn from(s:&str) -> Self { - Self(s.split(".").map(String::from).collect()) - } -} - -impl From<&Path> for ProtoPackage { - fn from(p:&Path) -> Self { - Self(p.iter().map(|c|c.to_str().unwrap().to_string()).collect()) - } -} - pub struct Config { /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. - pub input_path: InputPath, + pub input_root: InputPath, /// Implicit prefix that should be prepended to proto paths of the proto files in the input directory. - pub proto_path: PathBuf, + pub proto_root: ProtoPath, /// Descriptors of the dependencies and the rust absolute paths under which they will be available from the generated code. - pub dependencies: Vec<(rust::Name, &'static Descriptor)>, + pub dependencies: Vec<(RustName, &'static Descriptor)>, /// Rust absolute path under which the protobuf crate will be available from the generated /// code. - pub protobuf_crate: rust::Name, -} - -/// Extracts names of proto messages defined in the descriptor. -fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { - fn collect(out: &mut Vec, m: &prost_types::DescriptorProto) { - for m in m.child_messages() { - collect(out,m); - } - out.push(m.name().into()); - } - let mut res = vec![]; - for f in &descriptor.file { - for m in &f.message_type { - collect(res,m); - } - } - res + pub protobuf_crate: RustName, } impl Config { /// Proto package that all compiled proto files are supposed to belong to. - /// It is derived from `proto_path` by simply replacing all '/' with '.'. - fn proto_package(&self) -> ProtoPackage { - ProtoPackage::from(&*self.proto_path) + /// It is derived from `proto_root` by simply replacing all '/' with '.'. + fn proto_package(&self) -> ProtoName { + ProtoName::from(&*self.proto_root) } /// Location of the protobuf_build crate, visible from the generated code. - fn this_crate(&self) -> rust::Name { + fn this_crate(&self) -> RustName { self.protobuf_crate.clone().add("build") } /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated /// from a message of the given `proto_name`. - fn reflect_impl(&self, proto_name: ProtoPackage) -> String { + fn reflect_impl(&self, proto_name: ProtoName) -> String { let rust_name = proto_name.strip_prefix(&self.proto_package()).unwrap().to_rust_type().to_string(); let proto_name = proto_name.to_string(); let this = self.this_crate().to_string(); @@ -239,30 +133,29 @@ impl Config { /// Generates rust code from the proto files according to the config. pub fn generate(&self) -> anyhow::Result<()> { - let input_path = self.input_path.abs(); - assert!(input_path.is_dir(),"input_path should be a directory"); - println!("cargo:rerun-if-changed={input_path:?}"); + let input_root = self.input_root.abs(); + assert!(input_root.is_dir(),"input_root should be a directory"); + println!("cargo:rerun-if-changed={input_root:?}"); // Load dependencies. let mut pool = prost_reflect::DescriptorPool::new(); for d in &self.dependencies { - d.1.load(&mut pool).with_context(||format!("failed to load dependency {}",d.0))?; + d.1.load(&mut pool).with_context(||format!("failed to load dependency {}",d.0.to_string()))?; } let mut pool_raw = prost_types::FileDescriptorSet::default(); pool_raw.file = pool.file_descriptor_protos().cloned().collect(); // Load proto files. let mut proto_paths = vec![]; - traverse_files(&input_path, &mut |path| { + traverse_files(&input_root, &mut |path| { let Some(ext) = path.extension() else { return Ok(()) }; let Some(ext) = ext.to_str() else { return Ok(()) }; if ext != "proto" { return Ok(()) }; let file_raw = fs::read_to_string(path).context("fs::read()")?; - // Replace `input_path` with `proto_path` in path. - let path = self.proto_path.join(path.strip_prefix(&input_path).unwrap()); - pool_raw.file.push(protox_parse::parse(path.to_str().unwrap(),&file_raw)) - .context("protox_parse::parse()")?; + // Replace `input_root` with `proto_root` in path. + let path = self.proto_root.join(path.strip_prefix(&input_root).unwrap()); + pool_raw.file.push(protox_parse::parse(path.to_str().unwrap(),&file_raw).context("protox_parse::parse()")?); proto_paths.push(path); Ok(()) })?; @@ -272,7 +165,7 @@ impl Config { protox::file::DescriptorSetFileResolver::new(pool_raw) ); // TODO: nice compilation errors. - compiler.open_files(new_paths).unwrap(); + compiler.open_files(proto_paths).unwrap(); let descriptor = compiler.file_descriptor_set(); pool.add_file_descriptor_set(descriptor.clone()).unwrap(); @@ -280,7 +173,7 @@ impl Config { // and that proto messages support canonical encoding. let proto_package = self.proto_package(); for f in &descriptor.file { - if !ProtoPackage::from(f.package()).starts_with(&proto_package) { + if !ProtoName::from(f.package()).starts_with(&proto_package) { anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),proto_package.to_string()); } } @@ -289,7 +182,7 @@ impl Config { canonical::check(&descriptor,&pool).context("canonical::check()")?; // Prepare the output directory. - let output_dir = self.input_path.prepare_output_dir(); + let output_dir = self.input_root.prepare_output_dir(); let output_path = output_dir.join("gen.rs"); let descriptor_path = output_dir.join("gen.binpb"); fs::write(&descriptor_path, &descriptor.encode_to_vec())?; @@ -302,33 +195,33 @@ impl Config { config.skip_protoc_run(); for d in &self.dependencies { for f in &d.1.descriptor_proto.file { - let proto_rel = ProtoPackage::from(f.package()).strip_prefix(&d.1.proto_package).unwrap(); - let rust_abs = rust::Name::from(d.0).add(proto_rel.to_rust_module()); + let proto_rel = ProtoName::from(f.package()).strip_prefix(&d.1.proto_package).unwrap(); + let rust_abs = RustName::from(d.0).add(proto_rel.to_rust_module()); config.extern_path(format!(".{}",f.package()), rust_abs.to_string()); } } let modules : Vec<_> = descriptor.file.iter().map(|d|(prost_build::Module::from_protobuf_package_name(d.package()),d.clone())).collect(); for (name,code) in config.generate(modules).expect("generation failed") { - output.insert(&name.into(),&code); + output.sub(&name.into()).append(&code); } // Generate the reflection code. - let output = output.sub(&self.proto_path.to_rust_module()); - for proto_name in extract_message_names(&descriptor) { - output.append_code(&self.reflect_impl(proto_name)); + let output = output.sub(&self.proto_package().to_rust_module()); + for proto_name in proto::extract_message_names(&descriptor) { + output.append(&self.reflect_impl(proto_name)); } // Generate the descriptor. - let rust_deps = self.dependencies.iter().map(|d|format!("&{}::DESCRIPTOR",d.0)).collect::>().join(","); + let rust_deps = self.dependencies.iter().map(|d|format!("&{}::DESCRIPTOR",d.0.to_string())).collect::>().join(","); let this = self.this_crate().to_string(); - output.append_code(&format!("\ + output.append(&format!("\ pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ {this}::Descriptor::new({:?}.into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?}))\ }});\ ",proto_package.to_string())); // Save output. - fs::write(&output_path, output.format())?; + fs::write(&output_path, output.format().context("output.format()")?)?; Ok(()) } } diff --git a/node/libs/protobuf_build/src/rust.rs b/node/libs/protobuf_build/src/rust.rs deleted file mode 100644 index 267b6ce9..00000000 --- a/node/libs/protobuf_build/src/rust.rs +++ /dev/null @@ -1,57 +0,0 @@ -use std::collections::BTreeMap; - -type Part = String; - -#[derive(Clone,PartialEq,Eq)] -pub struct Name(Vec); - -impl Name { - fn add(mut self, suffix: impl Into) -> Self { - self.0.extend(suffix.into().0.into_iter()); - self - } - - fn to_string(&self) -> String { self.0.join("::") } -} - -impl From for Name { - fn from(s:prost_build::Module) -> Self { Self(s.parts().map(Part::from).collect()) } -} - -impl From<&str> for Name { - fn from(s:&str) -> Self { Self(s.split("::").map(Part::from).collect()) } -} - -/// A rust module representation. -/// It is used to collect the generated protobuf code. -#[derive(Default)] -pub(super) struct Module { - /// Nested modules which transitively contain the generated code. - modules: BTreeMap, - /// Code of the module. - code: String, -} - -impl Module { - pub fn sub(&self, path: &Name) -> &mut Self { - let mut m = self; - for part in &path.0 { - m = m.modules.entry(part.into()).or_default(); - } - m - } - - /// Appends code to the module. - pub fn append(&mut self, code: &str) { - self.code += code; - } - - /// Collects the code of the module. - pub fn format(&self) -> anyhow::Result { - let mut entries = vec![self.code.clone()]; - entries.extend( - self.modules.iter().map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), - ); - prettyplease::unparse(syn::parse_str(&entries.join("")).context("syn::parse_str()")?) - } -} diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs new file mode 100644 index 00000000..50edf250 --- /dev/null +++ b/node/libs/protobuf_build/src/syntax.rs @@ -0,0 +1,185 @@ +//! * input path - absolute path of the proto file as visible in the file system. Example: +//! $CARGO_MANIFEST_DIR//some/location/x.proto +//! * output path - absolute path of the output files (generated by this library) as visible in the +//! file system. Output paths are derived from the input paths by replacing $CARGO_MANIFEST_DIR with $OUT_DIR. +//! Example: +//! $OUT_DIR//generated_file +//! * proto path - absolute path of the proto file used for importing other proto files. +//! These are derived from input path by replacing the $CARGO_MANIFEST_DIR/ with . +//! Example: +//! /some/location/x.proto +//! * proto name - in addition to input paths and proto paths, there are also proto names +//! which are used to reference message types from different proto +//! files. Theoretically proto names can be totally independent from the proto paths (i.e. you can have +//! "my.favorite.package" proto package defined in "totally/unrelated/file/name.proto" file, but we +//! recommend the following naming convention: proto package "a.b.c" should be defined either: +//! a) in a single file "a/b/c.proto", or +//! b) in a collection of files under "a/b/c/" directory +//! Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. +//! * rust name - a rust module/type name that the generated code is available at. Although +//! generated code is location agnostic (it can be embedded in an arbitrary module within the crate), +//! you need to manually (in the Config) specify the rust modules containing the generated code +//! of the dependencies, so that it can be referenced from the newly generated code. Example: +//! ::some::location::x +use std::collections::BTreeMap; +use anyhow::Context as _; +use super::ident; +use std::path::{Path,PathBuf}; + +/// Path relative to $CARGO_MANIFEST_DIR, containing the input files. +#[derive(Clone,PartialEq,Eq)] +pub struct InputPath(PathBuf); + +impl From<&str> for InputPath { + fn from(s:&str) -> Self { Self(PathBuf::from(s)) } +} + +impl InputPath { + /// Converts the relative input path to an absolute path in the local file system. + fn abs(&self) -> PathBuf { + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) + .canonicalize().unwrap().join(&self.0) + } + + /// Derives the output path (under $OUT_DIR) from the input path + /// and re-constructs all the parent directories. + fn prepare_output_dir(&self) -> PathBuf { + let output = PathBuf::from(std::env::var("OUT_DIR").unwrap()) + .canonicalize().unwrap().join(&self.0); + let _ = fs::remove_dir_all(&output); + fs::create_dir_all(&output).unwrap(); + output + } +} + +#[derive(Clone,PartialEq,Eq)] +pub struct ProtoPath(PathBuf); + +impl From<&str> for ProtoPath { + fn from(s:&str) -> Self { Self(PathBuf::from(s)) } +} + +type Part = String; + +#[derive(Clone,PartialEq,Eq)] +pub struct RustName(Vec); + +impl RustName { + pub fn add(mut self, suffix: impl Into) -> Self { + self.0.extend(suffix.into().0.into_iter()); + self + } + + pub fn to_string(&self) -> String { self.0.join("::") } +} + +impl From for RustName { + fn from(s:prost_build::Module) -> Self { Self(s.parts().map(Part::from).collect()) } +} + +impl From<&str> for RustName { + fn from(s:&str) -> Self { Self(s.split("::").map(Part::from).collect()) } +} + +/// A rust module representation. +/// It is used to collect the generated protobuf code. +#[derive(Default)] +pub(super) struct Module { + /// Nested modules which transitively contain the generated code. + modules: BTreeMap, + /// Code of the module. + code: String, +} + +impl Module { + pub fn sub(&mut self, path: &RustName) -> &mut Self { + let mut m = self; + for part in &path.0 { + m = m.modules.entry(part.into()).or_default(); + } + m + } + + /// Appends code to the module. + pub fn append(&mut self, code: &str) { + self.code += code; + } + + fn collect(&self) -> String { + let mut entries = vec![self.code.clone()]; + entries.extend( + self.modules.iter().map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), + ); + entries.join("") + } + + /// Collects the code of the module. + pub fn format(&self) -> anyhow::Result { + Ok(prettyplease::unparse(&syn::parse_str(&self.collect()).context("syn::parse_str()")?)) + } +} + + +/// Represents a (relative) proto package path. +#[derive(Clone,PartialEq,Eq)] +pub(super) struct ProtoName(Vec); + +impl ProtoName { + /// Checks if package path starts with the given prefix. + pub fn starts_with(&self, prefix: &Self) -> bool { + let n = prefix.0.len(); + self.0.len() >= n && self.0[0..n] == prefix.0 + } + + pub fn strip_prefix(&self, prefix: &Self) -> anyhow::Result { + if !self.starts_with(prefix) { + anyhow::bail!("{} is not a prefix of {}",prefix.to_string(),self.to_string()); + } + Ok(Self(self.0[prefix.0.len()..].iter().cloned().collect())) + } + + pub fn to_rust_module(&self) -> rust::Name { + rust::Name::from_parts(self.0.iter().map(|s|ident::to_snake(s)).collect()) + } + + pub fn to_rust_type(&self) -> rust::Name { + let mut parts : Vec<_> = self.0.iter().map(|s|ident::to_snake(s)).collect(); + let n = parts.len(); + parts[n-1] = ident::to_upper_camel(&self.0[n-1]); + rust::Name::from_parts(parts) + } + + pub fn to_string(&self) -> String { + self.0.join(".") + } +} + +impl From<&str> for ProtoName { + fn from(s:&str) -> Self { + Self(s.split(".").map(String::from).collect()) + } +} + +impl From<&Path> for ProtoName { + fn from(p:&Path) -> Self { + Self(p.iter().map(|c|c.to_str().unwrap().to_string()).collect()) + } +} + + +/// Extracts names of proto messages defined in the descriptor. +pub(super) fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { + fn collect(out: &mut Vec, m: &prost_types::DescriptorProto) { + for m in &m.nested_type { + collect(out,m); + } + out.push(m.name().into()); + } + let mut res = vec![]; + for f in &descriptor.file { + for m in &f.message_type { + collect(&mut res,m); + } + } + res +} From b0a22abff2c33750eef36bbf81e31f5ff7b1102e Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 14:24:01 +0100 Subject: [PATCH 19/40] snapshot --- node/libs/protobuf/build.rs | 12 +- node/libs/protobuf_build/src/canonical.rs | 14 +-- node/libs/protobuf_build/src/lib.rs | 46 ++++---- node/libs/protobuf_build/src/syntax.rs | 132 ++++++++++++---------- node/libs/schema/build.rs | 6 +- 5 files changed, 110 insertions(+), 100 deletions(-) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 3dfa8077..7a13b81f 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -1,21 +1,21 @@ fn main() { protobuf_build::Config { - input_path: "src/proto".into(), - proto_path: "zksync".into(), + input_root: "src/proto".into(), + proto_root: "zksync".into(), dependencies: vec![], protobuf_crate: "crate".into(), }.generate().expect("generate(std)"); protobuf_build::Config { - input_path: "src/tests/proto".into(), - proto_path: "zksync/protobuf/tests".into(), + input_root: "src/tests/proto".into(), + proto_root: "zksync/protobuf/tests".into(), dependencies: vec![], protobuf_crate: "crate".into(), }.generate().expect("generate(test)"); protobuf_build::Config { - input_path: "src/bin/conformance_test/proto".into(), - proto_path: "zksync/protobuf/conformance_test".into(), + input_root: "src/bin/conformance_test/proto".into(), + proto_root: "zksync/protobuf/conformance_test".into(), dependencies: vec![], protobuf_crate: "::protobuf".into(), }.generate().expect("generate(conformance)"); diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs index 5ec4ed09..9a190b9a 100644 --- a/node/libs/protobuf_build/src/canonical.rs +++ b/node/libs/protobuf_build/src/canonical.rs @@ -1,32 +1,32 @@ use anyhow::Context as _; use std::collections::HashSet; -use super::proto; +use super::*; #[derive(Default)] struct Check(HashSet); impl Check { /// Checks if messages of type `m` support canonical encoding. - fn message(&mut self, m: prost_reflect::MessageDescriptor) -> anyhow::Result<()> { + fn message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { if self.0.contains(m.full_name()) { return Ok(()); } self.0.insert(m.full_name().to_string()); for f in m.fields() { - self.field(f).with_context(|| f.name().to_string())?; + self.field(&f).with_context(|| f.name().to_string())?; } Ok(()) } /// Checks if field `f` supports canonical encoding. - fn field(&mut self, f: prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + fn field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { if f.is_map() { anyhow::bail!("maps unsupported"); } if !f.is_list() && !f.supports_presence() { anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); } - if let prost_reflect::Kind::Message(msg) = f.kind() { + if let prost_reflect::Kind::Message(msg) = &f.kind() { self.message(msg).with_context(||msg.name().to_string())?; } Ok(()) @@ -40,10 +40,10 @@ pub fn check(descriptor: &prost_types::FileDescriptorSet, pool: &prost_reflect:: } } let mut c = Check::default(); - for msg_name in proto::extract_message_names(descriptor) { + for msg_name in extract_message_names(descriptor) { let msg_name = msg_name.to_string(); let msg = pool.get_message_by_name(&msg_name).with_context(||format!("{msg_name} not found in pool"))?; - c.message(msg).with_context(||msg_name)?; + c.message(&msg).with_context(||msg_name)?; } Ok(()) } diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 2d53dff7..fa90356c 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -27,7 +27,7 @@ //! perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find target/debug/build/*/output -type f` | xargs cat > /tmp/sum.binpb //! buf breaking /tmp/sum.binpb --against /tmp/sum.binpb use anyhow::Context as _; -use std::{fs, path::{PathBuf,Path}}; +use std::{fs, path::Path}; use prost::Message as _; use std::sync::Mutex; pub use syntax::*; @@ -55,16 +55,16 @@ fn traverse_files(path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>) /// Protobuf descriptor + info about the mapping to rust code. pub struct Descriptor { - proto_package: ProtoName, + proto_root: ProtoName, descriptor_proto: prost_types::FileDescriptorSet, dependencies: Vec<&'static Descriptor>, } impl Descriptor { /// Constructs a Descriptor. - pub fn new(proto_package: ProtoName, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { + pub fn new(proto_root: ProtoName, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { Descriptor { - proto_package, + proto_root, dependencies, descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()).unwrap(), } @@ -104,12 +104,6 @@ pub struct Config { } impl Config { - /// Proto package that all compiled proto files are supposed to belong to. - /// It is derived from `proto_root` by simply replacing all '/' with '.'. - fn proto_package(&self) -> ProtoName { - ProtoName::from(&*self.proto_root) - } - /// Location of the protobuf_build crate, visible from the generated code. fn this_crate(&self) -> RustName { self.protobuf_crate.clone().add("build") @@ -118,7 +112,7 @@ impl Config { /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated /// from a message of the given `proto_name`. fn reflect_impl(&self, proto_name: ProtoName) -> String { - let rust_name = proto_name.strip_prefix(&self.proto_package()).unwrap().to_rust_type().to_string(); + let rust_name = proto_name.relative_to(&self.proto_root.to_name()).unwrap().to_rust_type().to_string(); let proto_name = proto_name.to_string(); let this = self.this_crate().to_string(); format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ @@ -153,9 +147,8 @@ impl Config { if ext != "proto" { return Ok(()) }; let file_raw = fs::read_to_string(path).context("fs::read()")?; - // Replace `input_root` with `proto_root` in path. - let path = self.proto_root.join(path.strip_prefix(&input_root).unwrap()); - pool_raw.file.push(protox_parse::parse(path.to_str().unwrap(),&file_raw).context("protox_parse::parse()")?); + let path = ProtoPath::from_input_path(&path,&self.input_root,&self.proto_root); + pool_raw.file.push(protox_parse::parse(path.to_str(),&file_raw).context("protox_parse::parse()")?); proto_paths.push(path); Ok(()) })?; @@ -165,16 +158,15 @@ impl Config { protox::file::DescriptorSetFileResolver::new(pool_raw) ); // TODO: nice compilation errors. - compiler.open_files(proto_paths).unwrap(); + compiler.open_files(proto_paths.iter().map(|p|p.to_path())).unwrap(); let descriptor = compiler.file_descriptor_set(); pool.add_file_descriptor_set(descriptor.clone()).unwrap(); - // Check that the compiled proto files belong to the declared proto package - // and that proto messages support canonical encoding. - let proto_package = self.proto_package(); + // Check that the compiled proto files belong to the declared proto package. + let package_root = self.proto_root.to_name(); for f in &descriptor.file { - if !ProtoName::from(f.package()).starts_with(&proto_package) { - anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),proto_package.to_string()); + if !package_root.contains(&f.package().into()) { + anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),package_root.to_string()); } } @@ -182,21 +174,21 @@ impl Config { canonical::check(&descriptor,&pool).context("canonical::check()")?; // Prepare the output directory. - let output_dir = self.input_root.prepare_output_dir(); + let output_dir = self.input_root.prepare_output_dir().context("prepare_output_dir()")?; let output_path = output_dir.join("gen.rs"); let descriptor_path = output_dir.join("gen.binpb"); fs::write(&descriptor_path, &descriptor.encode_to_vec())?; println!("PROTOBUF_DESCRIPTOR={descriptor_path:?}"); // Generate code out of compiled proto files. - let mut output = rust::Module::default(); + let mut output = RustModule::default(); let mut config = prost_build::Config::new(); config.prost_path(self.this_crate().add("prost").to_string()); config.skip_protoc_run(); for d in &self.dependencies { for f in &d.1.descriptor_proto.file { - let proto_rel = ProtoName::from(f.package()).strip_prefix(&d.1.proto_package).unwrap(); - let rust_abs = RustName::from(d.0).add(proto_rel.to_rust_module()); + let proto_rel = ProtoName::from(f.package()).relative_to(&d.1.proto_root).unwrap(); + let rust_abs = d.0.clone().add(proto_rel.to_rust_module()); config.extern_path(format!(".{}",f.package()), rust_abs.to_string()); } } @@ -206,8 +198,8 @@ impl Config { } // Generate the reflection code. - let output = output.sub(&self.proto_package().to_rust_module()); - for proto_name in proto::extract_message_names(&descriptor) { + let output = output.sub(&self.proto_root.to_name().to_rust_module()); + for proto_name in extract_message_names(&descriptor) { output.append(&self.reflect_impl(proto_name)); } @@ -218,7 +210,7 @@ impl Config { pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ {this}::Descriptor::new({:?}.into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?}))\ }});\ - ",proto_package.to_string())); + ",package_root.to_string())); // Save output. fs::write(&output_path, output.format().context("output.format()")?)?; diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 50edf250..7b29d1e6 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -1,32 +1,10 @@ -//! * input path - absolute path of the proto file as visible in the file system. Example: -//! $CARGO_MANIFEST_DIR//some/location/x.proto -//! * output path - absolute path of the output files (generated by this library) as visible in the -//! file system. Output paths are derived from the input paths by replacing $CARGO_MANIFEST_DIR with $OUT_DIR. -//! Example: -//! $OUT_DIR//generated_file -//! * proto path - absolute path of the proto file used for importing other proto files. -//! These are derived from input path by replacing the $CARGO_MANIFEST_DIR/ with . -//! Example: -//! /some/location/x.proto -//! * proto name - in addition to input paths and proto paths, there are also proto names -//! which are used to reference message types from different proto -//! files. Theoretically proto names can be totally independent from the proto paths (i.e. you can have -//! "my.favorite.package" proto package defined in "totally/unrelated/file/name.proto" file, but we -//! recommend the following naming convention: proto package "a.b.c" should be defined either: -//! a) in a single file "a/b/c.proto", or -//! b) in a collection of files under "a/b/c/" directory -//! Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. -//! * rust name - a rust module/type name that the generated code is available at. Although -//! generated code is location agnostic (it can be embedded in an arbitrary module within the crate), -//! you need to manually (in the Config) specify the rust modules containing the generated code -//! of the dependencies, so that it can be referenced from the newly generated code. Example: -//! ::some::location::x +//! Utilities for handling strings belonging to various namespaces. use std::collections::BTreeMap; use anyhow::Context as _; use super::ident; use std::path::{Path,PathBuf}; -/// Path relative to $CARGO_MANIFEST_DIR, containing the input files. +/// Path relative to $CARGO_MANIFEST_DIR. #[derive(Clone,PartialEq,Eq)] pub struct InputPath(PathBuf); @@ -35,23 +13,25 @@ impl From<&str> for InputPath { } impl InputPath { - /// Converts the relative input path to an absolute path in the local file system. - fn abs(&self) -> PathBuf { + /// Converts the relative input path to an absolute path in the local file system + /// (under $CARGO_MANIFEST_DIR). + pub(super) fn abs(&self) -> PathBuf { PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) .canonicalize().unwrap().join(&self.0) } - /// Derives the output path (under $OUT_DIR) from the input path - /// and re-constructs all the parent directories. - fn prepare_output_dir(&self) -> PathBuf { - let output = PathBuf::from(std::env::var("OUT_DIR").unwrap()) - .canonicalize().unwrap().join(&self.0); - let _ = fs::remove_dir_all(&output); - fs::create_dir_all(&output).unwrap(); - output + /// Output directory path derived from the input path by replacing $CARGO_MANIFEST_DIR with $OUT_DIR. + /// Re-constructs the derived output directory, as a side-effect. + pub(super) fn prepare_output_dir(&self) -> anyhow::Result { + let output = PathBuf::from(std::env::var("OUT_DIR")?) + .canonicalize()?.join(&self.0); + let _ = std::fs::remove_dir_all(&output); + std::fs::create_dir_all(&output)?; + Ok(output) } } +/// Absolute path of the proto file used for importing other proto files. #[derive(Clone,PartialEq,Eq)] pub struct ProtoPath(PathBuf); @@ -59,8 +39,32 @@ impl From<&str> for ProtoPath { fn from(s:&str) -> Self { Self(PathBuf::from(s)) } } +impl ProtoPath { + /// Converts a proto module path to proto package name by replacing all "/" with ".". + pub(super) fn to_name(&self) -> ProtoName { + ProtoName(self.0.iter().map(|p|p.to_str().unwrap().into()).collect()) + } + + /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . + pub(super) fn from_input_path(path: &Path, input_root: &InputPath, proto_root: &ProtoPath) -> ProtoPath { + ProtoPath(proto_root.0.join(&path.strip_prefix(&input_root.abs()).unwrap())) + } + + pub(super) fn to_str(&self) -> &str { + self.0.to_str().unwrap() + } + + pub(super) fn to_path(&self) -> &Path { + &self.0 + } +} + type Part = String; +/// A rust module/type name that the generated code is available at. Although +/// generated code is location agnostic (it can be embedded in an arbitrary module within the crate), +/// you need to manually (in the Config) specify the rust modules containing the generated code +/// of the dependencies, so that it can be referenced from the newly generated code. #[derive(Clone,PartialEq,Eq)] pub struct RustName(Vec); @@ -84,14 +88,15 @@ impl From<&str> for RustName { /// A rust module representation. /// It is used to collect the generated protobuf code. #[derive(Default)] -pub(super) struct Module { +pub(super) struct RustModule { /// Nested modules which transitively contain the generated code. - modules: BTreeMap, + modules: BTreeMap, /// Code of the module. code: String, } -impl Module { +impl RustModule { + /// Returns a reference to a given submodule. pub fn sub(&mut self, path: &RustName) -> &mut Self { let mut m = self; for part in &path.0 { @@ -113,42 +118,52 @@ impl Module { entries.join("") } - /// Collects the code of the module. + /// Collects the code of the module and formats it. pub fn format(&self) -> anyhow::Result { Ok(prettyplease::unparse(&syn::parse_str(&self.collect()).context("syn::parse_str()")?)) } } -/// Represents a (relative) proto package path. +/// In addition to input paths and proto paths, there are also proto names +/// which are used to reference message types from different proto files. +/// Theoretically proto names can be totally independent from the proto paths (i.e. you can have +/// "my.favorite.package" proto package defined in "totally/unrelated/file/name.proto" file, but we +/// recommend the following naming convention: proto package "a.b.c" should be defined either: +/// a) in a single file "a/b/c.proto", or +/// b) in a collection of files under "a/b/c/" directory +/// Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. #[derive(Clone,PartialEq,Eq)] -pub(super) struct ProtoName(Vec); +pub struct ProtoName(Vec); impl ProtoName { /// Checks if package path starts with the given prefix. - pub fn starts_with(&self, prefix: &Self) -> bool { - let n = prefix.0.len(); - self.0.len() >= n && self.0[0..n] == prefix.0 + pub fn contains(&self, elem: &Self) -> bool { + elem.0.len() >= self.0.len() && elem.0[0..self.0.len()] == self.0 } - pub fn strip_prefix(&self, prefix: &Self) -> anyhow::Result { - if !self.starts_with(prefix) { - anyhow::bail!("{} is not a prefix of {}",prefix.to_string(),self.to_string()); + /// Strips a given prefix from the name. + pub fn relative_to(&self, prefix: &Self) -> anyhow::Result { + if !prefix.contains(self) { + anyhow::bail!("{} does not contain {}",self.to_string(),prefix.to_string()); } Ok(Self(self.0[prefix.0.len()..].iter().cloned().collect())) } - pub fn to_rust_module(&self) -> rust::Name { - rust::Name::from_parts(self.0.iter().map(|s|ident::to_snake(s)).collect()) + /// Converts proto package name to rust module name according to prost_build rules. + pub fn to_rust_module(&self) -> RustName { + RustName(self.0.iter().map(|s|ident::to_snake(s)).collect()) } - pub fn to_rust_type(&self) -> rust::Name { - let mut parts : Vec<_> = self.0.iter().map(|s|ident::to_snake(s)).collect(); - let n = parts.len(); - parts[n-1] = ident::to_upper_camel(&self.0[n-1]); - rust::Name::from_parts(parts) + /// Converts proto message name to rust type name according to prost_build rules. + pub fn to_rust_type(&self) -> RustName { + let mut rust = self.to_rust_module(); + let n = rust.0.len(); + rust.0[n-1] = ident::to_upper_camel(&self.0[n-1]); + rust } + /// Converts ProtoName to string. pub fn to_string(&self) -> String { self.0.join(".") } @@ -169,16 +184,19 @@ impl From<&Path> for ProtoName { /// Extracts names of proto messages defined in the descriptor. pub(super) fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { - fn collect(out: &mut Vec, m: &prost_types::DescriptorProto) { + fn collect(out: &mut Vec, prefix: &ProtoName, m: &prost_types::DescriptorProto) { + let mut name = prefix.clone(); + name.0.push(m.name().to_string()); for m in &m.nested_type { - collect(out,m); + collect(out,&name,m); } - out.push(m.name().into()); + out.push(name); } let mut res = vec![]; for f in &descriptor.file { + let name = ProtoName::from(f.package()); for m in &f.message_type { - collect(&mut res,m); + collect(&mut res, &name, m); } } res diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 1c8799fe..d4b17fbf 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,9 +1,9 @@ //! Generates rust code from the capnp schema files in the `capnp/` directory. fn main() { protobuf::build::Config { - input_path: "proto".into(), - proto_path: "zksync/schema".into(), - dependencies: vec![("::protobuf::proto",&protobuf::proto::DESCRIPTOR)], + input_root: "proto".into(), + proto_root: "zksync/schema".into(), + dependencies: vec![("::protobuf::proto".into(),&protobuf::proto::DESCRIPTOR)], protobuf_crate: "::protobuf".into(), }.generate().expect("generate()"); } From d3debe36cca7df27d083e5ce7288eaf786164db1 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 14:29:53 +0100 Subject: [PATCH 20/40] snapshot --- node/libs/protobuf_build/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index fa90356c..52cbdda3 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -192,9 +192,10 @@ impl Config { config.extern_path(format!(".{}",f.package()), rust_abs.to_string()); } } - let modules : Vec<_> = descriptor.file.iter().map(|d|(prost_build::Module::from_protobuf_package_name(d.package()),d.clone())).collect(); - for (name,code) in config.generate(modules).expect("generation failed") { - output.sub(&name.into()).append(&code); + let m = prost_build::Module::from_parts([""]); + for f in &descriptor.file { + let code = config.generate(vec![(m.clone(),f.clone())]).context("generation failed")?; + output.sub(&ProtoName::from(f.package()).to_rust_module()).append(&code[&m]); } // Generate the reflection code. From b195065db18f893bb2dc07d32cf611ecb2ff195f Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 15:27:20 +0100 Subject: [PATCH 21/40] error messages are sufficient now --- node/libs/protobuf_build/src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 52cbdda3..68921e2d 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -148,7 +148,10 @@ impl Config { let file_raw = fs::read_to_string(path).context("fs::read()")?; let path = ProtoPath::from_input_path(&path,&self.input_root,&self.proto_root); - pool_raw.file.push(protox_parse::parse(path.to_str(),&file_raw).context("protox_parse::parse()")?); + pool_raw.file.push(protox_parse::parse(path.to_str(),&file_raw).map_err( + // rewrapping the error, so that source location is included in the error message. + |err|anyhow::anyhow!("{:?}",err) + )?); proto_paths.push(path); Ok(()) })?; @@ -157,8 +160,10 @@ impl Config { let mut compiler = protox::Compiler::with_file_resolver( protox::file::DescriptorSetFileResolver::new(pool_raw) ); - // TODO: nice compilation errors. - compiler.open_files(proto_paths.iter().map(|p|p.to_path())).unwrap(); + compiler.include_source_info(true); + compiler.open_files(proto_paths.iter().map(|p|p.to_path())) + // rewrapping the error, so that source location is included in the error message. + .map_err(|err|anyhow::anyhow!("{:?}",err))?; let descriptor = compiler.file_descriptor_set(); pool.add_file_descriptor_set(descriptor.clone()).unwrap(); From 3808c28fb724ff565075c0c4dc40666929656347 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 16:13:39 +0100 Subject: [PATCH 22/40] fixed rerun, protobuf -> zksync_protobuf --- node/Cargo.lock | 88 ++++++----- node/Cargo.toml | 4 +- node/actors/consensus/Cargo.toml | 2 +- node/actors/consensus/src/inner.rs | 2 +- node/actors/consensus/src/metrics.rs | 2 +- node/actors/executor/Cargo.toml | 2 +- node/actors/executor/src/config/mod.rs | 2 +- node/actors/executor/src/config/tests.rs | 2 +- node/actors/network/Cargo.toml | 2 +- .../network/src/consensus/handshake/mod.rs | 2 +- .../network/src/consensus/handshake/tests.rs | 2 +- node/actors/network/src/frame.rs | 20 +-- .../network/src/gossip/handshake/mod.rs | 2 +- .../network/src/gossip/handshake/tests.rs | 2 +- node/actors/network/src/mux/handshake.rs | 6 +- node/actors/network/src/mux/tests.rs | 4 +- node/actors/network/src/preface.rs | 6 +- node/actors/network/src/rpc/consensus.rs | 6 +- node/actors/network/src/rpc/metrics.rs | 2 +- node/actors/network/src/rpc/mod.rs | 10 +- node/actors/network/src/rpc/ping.rs | 6 +- node/actors/network/src/rpc/sync_blocks.rs | 12 +- .../network/src/rpc/sync_validator_addrs.rs | 6 +- node/actors/network/src/rpc/tests.rs | 4 +- node/deny.toml | 4 - node/libs/protobuf/Cargo.toml | 6 +- node/libs/protobuf/build.rs | 20 ++- .../protobuf/src/bin/conformance_test/main.rs | 14 +- .../src/bin/conformance_test/proto/mod.rs | 5 +- node/libs/protobuf/src/lib.rs | 5 +- node/libs/protobuf/src/tests/mod.rs | 2 +- node/libs/protobuf_build/Cargo.toml | 2 +- node/libs/protobuf_build/src/canonical.rs | 22 ++- node/libs/protobuf_build/src/ident.rs | 4 +- node/libs/protobuf_build/src/lib.rs | 137 ++++++++++++------ node/libs/protobuf_build/src/syntax.rs | 112 +++++++++----- node/libs/roles/Cargo.toml | 2 +- node/libs/roles/src/node/conv.rs | 2 +- node/libs/roles/src/node/messages.rs | 2 +- node/libs/roles/src/node/tests.rs | 2 +- node/libs/roles/src/validator/conv.rs | 8 +- .../roles/src/validator/messages/block.rs | 6 +- node/libs/roles/src/validator/messages/msg.rs | 2 +- node/libs/roles/src/validator/tests.rs | 2 +- node/libs/schema/Cargo.toml | 8 +- node/libs/schema/build.rs | 10 +- node/libs/storage/Cargo.toml | 2 +- node/libs/storage/src/rocksdb.rs | 12 +- node/libs/storage/src/tests/mod.rs | 5 +- node/libs/storage/src/types.rs | 2 +- node/tools/Cargo.toml | 2 +- node/tools/src/bin/localnet_config.rs | 2 +- node/tools/src/config.rs | 4 +- 53 files changed, 347 insertions(+), 255 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index 5b251285..9fc74d80 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -416,7 +416,6 @@ dependencies = [ "crypto", "network", "once_cell", - "protobuf", "rand", "roles", "schema", @@ -426,6 +425,7 @@ dependencies = [ "tracing", "utils", "vise", + "zksync_protobuf", ] [[package]] @@ -612,7 +612,6 @@ dependencies = [ "consensus", "crypto", "network", - "protobuf", "rand", "roles", "schema", @@ -622,6 +621,7 @@ dependencies = [ "tracing", "utils", "vise", + "zksync_protobuf", ] [[package]] @@ -1142,7 +1142,6 @@ dependencies = [ "once_cell", "pin-project", "pretty_assertions", - "protobuf", "rand", "roles", "schema", @@ -1153,6 +1152,7 @@ dependencies = [ "tracing", "utils", "vise", + "zksync_protobuf", ] [[package]] @@ -1474,42 +1474,6 @@ dependencies = [ "prost", ] -[[package]] -name = "protobuf" -version = "0.1.0" -dependencies = [ - "anyhow", - "bit-vec", - "concurrency", - "once_cell", - "prost", - "prost-reflect", - "protobuf_build", - "quick-protobuf", - "rand", - "serde", - "serde_json", - "tokio", -] - -[[package]] -name = "protobuf_build" -version = "0.1.0" -dependencies = [ - "anyhow", - "heck", - "once_cell", - "prettyplease", - "prost", - "prost-build", - "prost-reflect", - "prost-types", - "protox", - "protox-parse", - "rand", - "syn", -] - [[package]] name = "protox" version = "0.5.0" @@ -1666,12 +1630,12 @@ dependencies = [ "concurrency", "crypto", "hex", - "protobuf", "rand", "schema", "serde", "tracing", "utils", + "zksync_protobuf", ] [[package]] @@ -1723,13 +1687,11 @@ dependencies = [ "concurrency", "once_cell", "prost", - "prost-reflect", - "protobuf", - "quick-protobuf", "rand", "serde", "serde_json", "tokio", + "zksync_protobuf", ] [[package]] @@ -1911,7 +1873,6 @@ dependencies = [ "assert_matches", "async-trait", "concurrency", - "protobuf", "rand", "rocksdb", "roles", @@ -1921,6 +1882,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "zksync_protobuf", ] [[package]] @@ -2094,7 +2056,6 @@ dependencies = [ "consensus", "crypto", "executor", - "protobuf", "rand", "roles", "schema", @@ -2104,6 +2065,7 @@ dependencies = [ "tracing-subscriber", "utils", "vise-exporter", + "zksync_protobuf", ] [[package]] @@ -2414,6 +2376,42 @@ dependencies = [ "syn", ] +[[package]] +name = "zksync_protobuf" +version = "0.1.0" +dependencies = [ + "anyhow", + "bit-vec", + "concurrency", + "once_cell", + "prost", + "prost-reflect", + "quick-protobuf", + "rand", + "serde", + "serde_json", + "tokio", + "zksync_protobuf_build", +] + +[[package]] +name = "zksync_protobuf_build" +version = "0.1.0" +dependencies = [ + "anyhow", + "heck", + "once_cell", + "prettyplease", + "prost", + "prost-build", + "prost-reflect", + "prost-types", + "protox", + "protox-parse", + "rand", + "syn", +] + [[package]] name = "zstd-sys" version = "2.0.8+zstd.1.5.5" diff --git a/node/Cargo.toml b/node/Cargo.toml index 9780c544..76b5d8f5 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -23,8 +23,8 @@ homepage = "https://matter-labs.io/" license = "MIT" [workspace.dependencies] -protobuf_build = { path = "libs/protobuf_build" } -protobuf = { path = "libs/protobuf" } +zksync_protobuf_build = { path = "libs/protobuf_build" } +zksync_protobuf = { path = "libs/protobuf" } anyhow = "1" assert_matches = "1.5.0" diff --git a/node/actors/consensus/Cargo.toml b/node/actors/consensus/Cargo.toml index 67cdcae4..5e892a4d 100644 --- a/node/actors/consensus/Cargo.toml +++ b/node/actors/consensus/Cargo.toml @@ -14,7 +14,7 @@ thiserror.workspace = true tracing.workspace = true vise.workspace = true -protobuf.workspace = true +zksync_protobuf.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/consensus/src/inner.rs b/node/actors/consensus/src/inner.rs index 45080e06..4dddc209 100644 --- a/node/actors/consensus/src/inner.rs +++ b/node/actors/consensus/src/inner.rs @@ -23,7 +23,7 @@ pub(crate) struct ConsensusInner { impl ConsensusInner { /// The maximum size of the payload of a block, in bytes. We will /// reject blocks with payloads larger than this. - pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * protobuf::kB; + pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * zksync_protobuf::kB; /// Computes the validator for the given view. #[instrument(level = "trace", ret)] diff --git a/node/actors/consensus/src/metrics.rs b/node/actors/consensus/src/metrics.rs index 5742134b..0327a044 100644 --- a/node/actors/consensus/src/metrics.rs +++ b/node/actors/consensus/src/metrics.rs @@ -4,7 +4,7 @@ use std::time::Duration; use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; const PAYLOAD_SIZE_BUCKETS: Buckets = - Buckets::exponential((4 * protobuf::kB) as f64..=(4 * protobuf::MB) as f64, 4.0); + Buckets::exponential((4 * zksync_protobuf::kB) as f64..=(4 * zksync_protobuf::MB) as f64, 4.0); /// Label for a consensus message. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index 2261b720..8c747cc4 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -11,7 +11,7 @@ anyhow.workspace = true tracing.workspace = true vise.workspace = true -protobuf.workspace = true +zksync_protobuf.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/executor/src/config/mod.rs b/node/actors/executor/src/config/mod.rs index db5ac763..d2f42bb5 100644 --- a/node/actors/executor/src/config/mod.rs +++ b/node/actors/executor/src/config/mod.rs @@ -3,9 +3,9 @@ use anyhow::Context as _; use crypto::{read_required_text, Text, TextFmt}; use network::{consensus, gossip}; +use zksync_protobuf::{read_required, required, ProtoFmt}; use roles::{node, validator}; use schema::proto::executor::config as proto; -use protobuf::{read_required, required, ProtoFmt}; use std::{ collections::{HashMap, HashSet}, net, diff --git a/node/actors/executor/src/config/tests.rs b/node/actors/executor/src/config/tests.rs index bf3bc4d2..98b34367 100644 --- a/node/actors/executor/src/config/tests.rs +++ b/node/actors/executor/src/config/tests.rs @@ -1,11 +1,11 @@ use super::{ConsensusConfig, ExecutorConfig, GossipConfig}; use concurrency::ctx; +use zksync_protobuf::testonly::test_encode_random; use rand::{ distributions::{Distribution, Standard}, Rng, }; use roles::{node, validator}; -use protobuf::testonly::test_encode_random; fn make_addr(rng: &mut R) -> std::net::SocketAddr { std::net::SocketAddr::new(std::net::IpAddr::from(rng.gen::<[u8; 16]>()), rng.gen()) diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index cfc45758..59d2b5be 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -18,7 +18,7 @@ thiserror.workspace = true tracing.workspace = true vise.workspace = true -protobuf.workspace = true +zksync_protobuf.workspace = true concurrency = { path = "../../libs/concurrency" } crypto = { path = "../../libs/crypto" } roles = { path = "../../libs/roles" } diff --git a/node/actors/network/src/consensus/handshake/mod.rs b/node/actors/network/src/consensus/handshake/mod.rs index 024c6a72..f5152a95 100644 --- a/node/actors/network/src/consensus/handshake/mod.rs +++ b/node/actors/network/src/consensus/handshake/mod.rs @@ -2,9 +2,9 @@ use crate::{frame, noise}; use anyhow::Context as _; use concurrency::{ctx, time}; use crypto::{bls12_381, ByteFmt}; +use zksync_protobuf::{read_required, ProtoFmt}; use roles::{node, validator}; use schema::proto::network::consensus as proto; -use protobuf::{read_required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/consensus/handshake/tests.rs b/node/actors/network/src/consensus/handshake/tests.rs index a91b529c..d5ab18a8 100644 --- a/node/actors/network/src/consensus/handshake/tests.rs +++ b/node/actors/network/src/consensus/handshake/tests.rs @@ -7,7 +7,7 @@ use roles::validator; #[test] fn test_schema_encode_decode() { let rng = &mut ctx::test_root(&ctx::RealClock).rng(); - protobuf::testonly::test_encode_random::<_, Handshake>(rng); + zksync_protobuf::testonly::test_encode_random::<_, Handshake>(rng); } #[tokio::test] diff --git a/node/actors/network/src/frame.rs b/node/actors/network/src/frame.rs index 2a7d4464..44b484fc 100644 --- a/node/actors/network/src/frame.rs +++ b/node/actors/network/src/frame.rs @@ -1,5 +1,5 @@ -//! Simple frame encoding format (length ++ value) for protobuf messages, -//! since protobuf messages do not have delimiters. +//! Simple frame encoding format (length ++ value) for zksync_protobuf messages, +//! since zksync_protobuf messages do not have delimiters. use crate::{mux, noise::bytes}; use concurrency::{ctx, io}; @@ -7,7 +7,7 @@ use concurrency::{ctx, io}; /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. /// Returns the decoded proto and the size of the received message in bytes. -pub(crate) async fn mux_recv_proto( +pub(crate) async fn mux_recv_proto( ctx: &ctx::Ctx, stream: &mut mux::ReadStream, ) -> anyhow::Result<(T, usize)> { @@ -25,19 +25,19 @@ pub(crate) async fn mux_recv_proto( if msg.len() < msg_size { anyhow::bail!("end of stream"); } - let msg = protobuf::decode(msg.as_slice())?; + let msg = zksync_protobuf::decode(msg.as_slice())?; Ok((msg, msg_size)) } /// Sends a proto serialized to a raw frame of bytes to the stream. /// It doesn't flush the stream. /// Returns the size of the sent proto in bytes. -pub(crate) async fn mux_send_proto( +pub(crate) async fn mux_send_proto( ctx: &ctx::Ctx, stream: &mut mux::WriteStream, msg: &T, ) -> anyhow::Result { - let msg = protobuf::encode(msg); + let msg = zksync_protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); stream .write_all(ctx, &u32::to_le_bytes(msg.len() as u32)) @@ -49,7 +49,7 @@ pub(crate) async fn mux_send_proto( /// Reads a raw frame of bytes from the stream and interprets it as proto. /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. -pub(crate) async fn recv_proto( +pub(crate) async fn recv_proto( ctx: &ctx::Ctx, stream: &mut S, ) -> anyhow::Result { @@ -61,16 +61,16 @@ pub(crate) async fn recv_proto( } let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stream, &mut msg[..]).await??; - protobuf::decode(&msg) + zksync_protobuf::decode(&msg) } /// Sends a proto serialized to a raw frame of bytes to the stream. -pub(crate) async fn send_proto( +pub(crate) async fn send_proto( ctx: &ctx::Ctx, stream: &mut S, msg: &T, ) -> anyhow::Result<()> { - let msg = protobuf::encode(msg); + let msg = zksync_protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); io::write_all(ctx, stream, &u32::to_le_bytes(msg.len() as u32)).await??; io::write_all(ctx, stream, &msg).await??; diff --git a/node/actors/network/src/gossip/handshake/mod.rs b/node/actors/network/src/gossip/handshake/mod.rs index 20129721..49a27e7c 100644 --- a/node/actors/network/src/gossip/handshake/mod.rs +++ b/node/actors/network/src/gossip/handshake/mod.rs @@ -3,9 +3,9 @@ use crate::{frame, noise}; use anyhow::Context as _; use concurrency::{ctx, time}; use crypto::ByteFmt; +use zksync_protobuf::{read_required, required, ProtoFmt}; use roles::node; use schema::proto::network::gossip as proto; -use protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/gossip/handshake/tests.rs b/node/actors/network/src/gossip/handshake/tests.rs index 26ea257d..8a749240 100644 --- a/node/actors/network/src/gossip/handshake/tests.rs +++ b/node/actors/network/src/gossip/handshake/tests.rs @@ -8,7 +8,7 @@ use std::collections::{HashMap, HashSet}; #[test] fn test_schema_encode_decode() { let rng = &mut ctx::test_root(&ctx::RealClock).rng(); - protobuf::testonly::test_encode_random::<_, Handshake>(rng); + zksync_protobuf::testonly::test_encode_random::<_, Handshake>(rng); } fn make_cfg(rng: &mut R) -> Config { diff --git a/node/actors/network/src/mux/handshake.rs b/node/actors/network/src/mux/handshake.rs index 7f971bca..d2ee6e39 100644 --- a/node/actors/network/src/mux/handshake.rs +++ b/node/actors/network/src/mux/handshake.rs @@ -1,7 +1,7 @@ use super::CapabilityId; use anyhow::Context as _; +use zksync_protobuf::required; use schema::proto::network::mux as proto; -use protobuf::required; use std::collections::HashMap; pub(super) struct Handshake { @@ -37,11 +37,11 @@ fn build_capabilities( .collect() } -impl protobuf::ProtoFmt for Handshake { +impl zksync_protobuf::ProtoFmt for Handshake { type Proto = proto::Handshake; fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { diff --git a/node/actors/network/src/mux/tests.rs b/node/actors/network/src/mux/tests.rs index 59483a5d..ee4c50d7 100644 --- a/node/actors/network/src/mux/tests.rs +++ b/node/actors/network/src/mux/tests.rs @@ -72,7 +72,7 @@ struct Resp { capability_id: mux::CapabilityId, } -impl protobuf::ProtoFmt for Req { +impl zksync_protobuf::ProtoFmt for Req { type Proto = proto::Req; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(r.input.as_ref().unwrap().clone())) @@ -84,7 +84,7 @@ impl protobuf::ProtoFmt for Req { } } -impl protobuf::ProtoFmt for Resp { +impl zksync_protobuf::ProtoFmt for Resp { type Proto = proto::Resp; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { diff --git a/node/actors/network/src/preface.rs b/node/actors/network/src/preface.rs index 88576ff6..6a0f01a9 100644 --- a/node/actors/network/src/preface.rs +++ b/node/actors/network/src/preface.rs @@ -9,8 +9,8 @@ //! and multiplex between mutliple endpoints available on the same TCP port. use crate::{frame, metrics, noise}; use concurrency::{ctx, time}; +use zksync_protobuf::{required, ProtoFmt}; use schema::proto::network::preface as proto; -use protobuf::{required, ProtoFmt}; /// Timeout on executing the preface protocol. const TIMEOUT: time::Duration = time::Duration::seconds(5); @@ -34,7 +34,7 @@ pub(crate) enum Endpoint { impl ProtoFmt for Encryption { type Proto = proto::Encryption; fn max_size() -> usize { - 10 * protobuf::kB + 10 * zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::encryption::T; @@ -54,7 +54,7 @@ impl ProtoFmt for Encryption { impl ProtoFmt for Endpoint { type Proto = proto::Endpoint; fn max_size() -> usize { - 10 * protobuf::kB + 10 * zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::endpoint::T; diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 14857cc8..80f16699 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -1,9 +1,9 @@ //! Defines RPC for passing consensus messages. use crate::mux; use concurrency::{limiter, time}; +use zksync_protobuf::{read_required, ProtoFmt}; use roles::validator; use schema::proto::network::consensus as proto; -use protobuf::{read_required, ProtoFmt}; /// Consensus RPC. pub(crate) struct Rpc; @@ -46,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - protobuf::MB + zksync_protobuf::MB } } @@ -62,6 +62,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } } diff --git a/node/actors/network/src/rpc/metrics.rs b/node/actors/network/src/rpc/metrics.rs index 1a4c2d06..34394e28 100644 --- a/node/actors/network/src/rpc/metrics.rs +++ b/node/actors/network/src/rpc/metrics.rs @@ -86,7 +86,7 @@ pub(super) struct CallLabels { } const MESSAGE_SIZE_BUCKETS: Buckets = - Buckets::exponential(protobuf::kB as f64..=protobuf::MB as f64, 2.0); + Buckets::exponential(zksync_protobuf::kB as f64..=zksync_protobuf::MB as f64, 2.0); #[derive(Debug, Metrics)] #[metrics(prefix = "network_rpc")] diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index e0bb0888..bb4bbe69 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -32,10 +32,10 @@ pub(crate) mod testonly; mod tests; const MUX_CONFIG: mux::Config = mux::Config { - read_buffer_size: 160 * protobuf::kB as u64, - read_frame_size: 16 * protobuf::kB as u64, + read_buffer_size: 160 * zksync_protobuf::kB as u64, + read_frame_size: 16 * zksync_protobuf::kB as u64, read_frame_count: 100, - write_frame_size: 16 * protobuf::kB as u64, + write_frame_size: 16 * zksync_protobuf::kB as u64, }; /// Trait for defining an RPC. @@ -57,9 +57,9 @@ pub(crate) trait Rpc: Sync + Send + 'static { /// Name of the RPC, used in prometheus metrics. const METHOD: &'static str; /// Type of the request message. - type Req: protobuf::ProtoFmt + Send + Sync; + type Req: zksync_protobuf::ProtoFmt + Send + Sync; /// Type of the response message. - type Resp: protobuf::ProtoFmt + Send + Sync; + type Resp: zksync_protobuf::ProtoFmt + Send + Sync; /// Name of the variant of the request message type. /// Useful for collecting metrics with finer than /// per-rpc type granularity. diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index 1310014f..e38b2ae3 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -2,9 +2,9 @@ use crate::{mux, rpc::Rpc as _}; use anyhow::Context as _; use concurrency::{ctx, limiter, time}; +use zksync_protobuf::{required, ProtoFmt}; use rand::Rng; use schema::proto::network::ping as proto; -use protobuf::{required, ProtoFmt}; /// Ping RPC. pub(crate) struct Rpc; @@ -65,7 +65,7 @@ pub(crate) struct Resp(pub(crate) [u8; 32]); impl ProtoFmt for Req { type Proto = proto::PingReq; fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) @@ -80,7 +80,7 @@ impl ProtoFmt for Req { impl ProtoFmt for Resp { type Proto = proto::PingResp; fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index 7454a35b..d59aacfd 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -3,9 +3,9 @@ use crate::{io, mux}; use anyhow::Context; use concurrency::{limiter, time}; +use zksync_protobuf::{read_required, ProtoFmt}; use roles::validator::{BlockNumber, FinalBlock}; use schema::proto::network::gossip as proto; -use protobuf::{read_required, ProtoFmt}; /// `get_sync_state` RPC. #[derive(Debug)] @@ -48,7 +48,7 @@ impl ProtoFmt for io::SyncState { fn max_size() -> usize { // TODO: estimate maximum size more precisely - 100 * protobuf::kB + 100 * zksync_protobuf::kB } } @@ -68,7 +68,7 @@ impl ProtoFmt for SyncStateResponse { } fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } } @@ -110,7 +110,7 @@ impl ProtoFmt for GetBlockRequest { } fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } } @@ -138,7 +138,7 @@ impl ProtoFmt for io::GetBlockError { } fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } } @@ -179,6 +179,6 @@ impl ProtoFmt for GetBlockResponse { } fn max_size() -> usize { - protobuf::MB + zksync_protobuf::MB } } diff --git a/node/actors/network/src/rpc/sync_validator_addrs.rs b/node/actors/network/src/rpc/sync_validator_addrs.rs index 41329381..fd95d002 100644 --- a/node/actors/network/src/rpc/sync_validator_addrs.rs +++ b/node/actors/network/src/rpc/sync_validator_addrs.rs @@ -3,9 +3,9 @@ use crate::mux; use anyhow::Context as _; use concurrency::{limiter, time}; +use zksync_protobuf::ProtoFmt; use roles::validator; use schema::proto::network::gossip as proto; -use protobuf::{ProtoFmt}; use std::sync::Arc; /// SyncValidatorAddrs Rpc. @@ -46,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - protobuf::kB + zksync_protobuf::kB } } @@ -70,6 +70,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - protobuf::MB + zksync_protobuf::MB } } diff --git a/node/actors/network/src/rpc/tests.rs b/node/actors/network/src/rpc/tests.rs index 2d70cb7b..35092250 100644 --- a/node/actors/network/src/rpc/tests.rs +++ b/node/actors/network/src/rpc/tests.rs @@ -21,8 +21,8 @@ fn test_capability_rpc_correspondence() { #[test] fn test_schema_encode_decode() { let rng = &mut ctx::test_root(&ctx::RealClock).rng(); - protobuf::testonly::test_encode_random::<_, consensus::Req>(rng); - protobuf::testonly::test_encode_random::<_, sync_validator_addrs::Resp>(rng); + zksync_protobuf::testonly::test_encode_random::<_, consensus::Req>(rng); + zksync_protobuf::testonly::test_encode_random::<_, sync_validator_addrs::Resp>(rng); } fn expected(res: Result<(), mux::RunError>) -> Result<(), mux::RunError> { diff --git a/node/deny.toml b/node/deny.toml index dd10be9e..2033b549 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -47,10 +47,6 @@ allow = [ multiple-versions = "deny" # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ - # Old versions required by prost. - { name = "prettyplease", version = "=0.1.25" }, - { name = "syn", version = "=1.0.109" }, - # Old versions required by tempfile and prost-build. { name = "bitflags", version = "=1.3.2" }, diff --git a/node/libs/protobuf/Cargo.toml b/node/libs/protobuf/Cargo.toml index 71cc9f50..c2f3f4f0 100644 --- a/node/libs/protobuf/Cargo.toml +++ b/node/libs/protobuf/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "protobuf" +name = "zksync_protobuf" version = "0.1.0" edition.workspace = true authors.workspace = true @@ -21,10 +21,10 @@ serde_json.workspace = true tokio.workspace = true once_cell.workspace = true -protobuf_build.workspace = true +zksync_protobuf_build.workspace = true concurrency = { path = "../concurrency" } [build-dependencies] -protobuf_build.workspace = true +zksync_protobuf_build.workspace = true anyhow.workspace = true diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 7a13b81f..471483a3 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -1,22 +1,28 @@ fn main() { - protobuf_build::Config { + zksync_protobuf_build::Config { input_root: "src/proto".into(), proto_root: "zksync".into(), dependencies: vec![], protobuf_crate: "crate".into(), - }.generate().expect("generate(std)"); + } + .generate() + .expect("generate(std)"); - protobuf_build::Config { + zksync_protobuf_build::Config { input_root: "src/tests/proto".into(), proto_root: "zksync/protobuf/tests".into(), dependencies: vec![], protobuf_crate: "crate".into(), - }.generate().expect("generate(test)"); + } + .generate() + .expect("generate(test)"); - protobuf_build::Config { + zksync_protobuf_build::Config { input_root: "src/bin/conformance_test/proto".into(), proto_root: "zksync/protobuf/conformance_test".into(), dependencies: vec![], - protobuf_crate: "::protobuf".into(), - }.generate().expect("generate(conformance)"); + protobuf_crate: "::zksync_protobuf".into(), + } + .generate() + .expect("generate(conformance)"); } diff --git a/node/libs/protobuf/src/bin/conformance_test/main.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs index 8e8926e8..f1c3fce9 100644 --- a/node/libs/protobuf/src/bin/conformance_test/main.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -1,8 +1,8 @@ //! Conformance test for our canonical encoding implemented according to -//! https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto +//! https://github.com/protocolbuffers/zksync_protobuf/blob/main/conformance/conformance.proto //! Our implementation supports only a subset of proto functionality, so //! `schema/proto/conformance/conformance.proto` and -//! `schema/proto/conformance/protobuf_test_messages.proto` contains only a +//! `schema/proto/conformance/zksync_protobuf_test_messages.proto` contains only a //! subset of original fields. Also we run only proto3 binary -> binary tests. //! conformance_test_failure_list.txt contains tests which are expected to fail. use anyhow::Context as _; @@ -31,7 +31,7 @@ async fn main() -> anyhow::Result<()> { let req = proto::ConformanceRequest::decode(&msg[..])?; let res = async { let t = req.message_type.context("missing message_type")?; - if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { + if t != *"zksync_protobuf_test_messages.proto3.TestAllTypesProto3" { return Ok(R::Skipped("unsupported".to_string())); } @@ -40,7 +40,7 @@ async fn main() -> anyhow::Result<()> { use proto::TestAllTypesProto3 as T; let p = match payload { proto::conformance_request::Payload::JsonPayload(payload) => { - match protobuf::decode_json_proto(&payload) { + match zksync_protobuf::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } @@ -51,7 +51,7 @@ async fn main() -> anyhow::Result<()> { return Ok(R::ParseError("parsing failed".to_string())); }; // Then check if there are any unknown fields in the original payload. - if protobuf::canonical_raw(&payload[..], &p.descriptor()).is_err() { + if zksync_protobuf::canonical_raw(&payload[..], &p.descriptor()).is_err() { return Ok(R::Skipped("unsupported fields".to_string())); } p @@ -65,11 +65,11 @@ async fn main() -> anyhow::Result<()> { .context("missing output format")?; match proto::WireFormat::try_from(format).context("unknown format")? { proto::WireFormat::Json => { - anyhow::Ok(R::JsonPayload(protobuf::encode_json_proto(&p))) + anyhow::Ok(R::JsonPayload(zksync_protobuf::encode_json_proto(&p))) } proto::WireFormat::Protobuf => { // Reencode the parsed proto. - anyhow::Ok(R::ProtobufPayload(protobuf::canonical_raw( + anyhow::Ok(R::ProtobufPayload(zksync_protobuf::canonical_raw( &p.encode_to_vec(), &p.descriptor(), )?)) diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs index 11db6a12..70249255 100644 --- a/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs +++ b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs @@ -1,2 +1,5 @@ #![allow(warnings)] -include!(concat!(env!("OUT_DIR"), "/src/bin/conformance_test/proto/gen.rs")); +include!(concat!( + env!("OUT_DIR"), + "/src/bin/conformance_test/proto/gen.rs" +)); diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs index ffc77094..f461892a 100644 --- a/node/libs/protobuf/src/lib.rs +++ b/node/libs/protobuf/src/lib.rs @@ -1,14 +1,13 @@ //! Code generated from protobuf schema files and //! utilities for serialization. +pub mod proto; mod proto_fmt; mod std_conv; pub mod testonly; -pub mod proto; pub use proto_fmt::*; -pub use protobuf_build as build; +pub use zksync_protobuf_build as build; #[cfg(test)] mod tests; - diff --git a/node/libs/protobuf/src/tests/mod.rs b/node/libs/protobuf/src/tests/mod.rs index 162e71d9..143c47cf 100644 --- a/node/libs/protobuf/src/tests/mod.rs +++ b/node/libs/protobuf/src/tests/mod.rs @@ -1,7 +1,7 @@ use super::*; +use ::std::net; use anyhow::Context as _; use concurrency::{ctx, time}; -use ::std::net; mod proto; diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml index bd4463c8..cf389a8f 100644 --- a/node/libs/protobuf_build/Cargo.toml +++ b/node/libs/protobuf_build/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "protobuf_build" +name = "zksync_protobuf_build" version = "0.1.0" edition.workspace = true authors.workspace = true diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs index 9a190b9a..87ba52f9 100644 --- a/node/libs/protobuf_build/src/canonical.rs +++ b/node/libs/protobuf_build/src/canonical.rs @@ -1,13 +1,14 @@ +//! Checks whether messages in the given file descriptor set support canonical encoding. +use crate::syntax::extract_message_names; use anyhow::Context as _; use std::collections::HashSet; -use super::*; #[derive(Default)] struct Check(HashSet); impl Check { /// Checks if messages of type `m` support canonical encoding. - fn message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { + fn message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { if self.0.contains(m.full_name()) { return Ok(()); } @@ -27,13 +28,18 @@ impl Check { anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); } if let prost_reflect::Kind::Message(msg) = &f.kind() { - self.message(msg).with_context(||msg.name().to_string())?; + self.message(msg).with_context(|| msg.name().to_string())?; } Ok(()) - } + } } -pub fn check(descriptor: &prost_types::FileDescriptorSet, pool: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { +/// Checks whether messages in the given file descriptor set support canonical encoding. +/// pool should contain all transitive dependencies of files in descriptor. +pub(crate) fn check( + descriptor: &prost_types::FileDescriptorSet, + pool: &prost_reflect::DescriptorPool, +) -> anyhow::Result<()> { for f in &descriptor.file { if f.syntax() != "proto3" { anyhow::bail!("only proto3 syntax is supported"); @@ -42,8 +48,10 @@ pub fn check(descriptor: &prost_types::FileDescriptorSet, pool: &prost_reflect:: let mut c = Check::default(); for msg_name in extract_message_names(descriptor) { let msg_name = msg_name.to_string(); - let msg = pool.get_message_by_name(&msg_name).with_context(||format!("{msg_name} not found in pool"))?; - c.message(&msg).with_context(||msg_name)?; + let msg = pool + .get_message_by_name(&msg_name) + .with_context(|| format!("{msg_name} not found in pool"))?; + c.message(&msg).with_context(|| msg_name)?; } Ok(()) } diff --git a/node/libs/protobuf_build/src/ident.rs b/node/libs/protobuf_build/src/ident.rs index 55e8e48a..0e85aa83 100644 --- a/node/libs/protobuf_build/src/ident.rs +++ b/node/libs/protobuf_build/src/ident.rs @@ -4,7 +4,7 @@ use heck::{ToSnakeCase, ToUpperCamelCase}; /// Converts a `camelCase` or `SCREAMING_SNAKE_CASE` identifier to a `lower_snake` case Rust field /// identifier. -pub fn to_snake(s: &str) -> String { +pub(crate) fn to_snake(s: &str) -> String { let mut ident = s.to_snake_case(); // Use a raw identifier if the identifier matches a Rust keyword: @@ -30,7 +30,7 @@ pub fn to_snake(s: &str) -> String { } /// Converts a `snake_case` identifier to an `UpperCamel` case Rust type identifier. -pub fn to_upper_camel(s: &str) -> String { +pub(crate) fn to_upper_camel(s: &str) -> String { let mut ident = s.to_upper_camel_case(); // Suffix an underscore for the `Self` Rust keyword as it is not allowed as raw identifier. diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 68921e2d..179cc619 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -27,24 +27,25 @@ //! perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find target/debug/build/*/output -type f` | xargs cat > /tmp/sum.binpb //! buf breaking /tmp/sum.binpb --against /tmp/sum.binpb use anyhow::Context as _; -use std::{fs, path::Path}; -use prost::Message as _; -use std::sync::Mutex; -pub use syntax::*; - +pub use once_cell::sync::Lazy; // Imports accessed from the generated code. pub use prost; +use prost::Message as _; pub use prost_reflect; -pub use once_cell::sync::Lazy; +use std::{fs, path::Path, sync::Mutex}; +pub use syntax::*; -mod ident; mod canonical; +mod ident; mod syntax; /// Traversed all the files in a directory recursively. -fn traverse_files(path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>) -> anyhow::Result<()> { +fn traverse_files( + path: &Path, + f: &mut impl FnMut(&Path) -> anyhow::Result<()>, +) -> anyhow::Result<()> { if !path.is_dir() { - f(path).with_context(||path.to_str().unwrap().to_string())?; + f(path).with_context(|| path.to_str().unwrap().to_string())?; return Ok(()); } for entry in fs::read_dir(path)? { @@ -62,17 +63,27 @@ pub struct Descriptor { impl Descriptor { /// Constructs a Descriptor. - pub fn new(proto_root: ProtoName, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &impl AsRef<[u8]>) -> Self { + pub fn new( + proto_root: ProtoName, + dependencies: Vec<&'static Descriptor>, + descriptor_bytes: &impl AsRef<[u8]>, + ) -> Self { Descriptor { proto_root, dependencies, - descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()).unwrap(), + descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()) + .unwrap(), } } /// Loads the descriptor to the pool, if not already loaded. pub fn load(&self, pool: &mut prost_reflect::DescriptorPool) -> anyhow::Result<()> { - if self.descriptor_proto.file.iter().all(|f| pool.get_file_by_name(f.name()).is_some()) { + if self + .descriptor_proto + .file + .iter() + .all(|f| pool.get_file_by_name(f.name()).is_some()) + { return Ok(()); } for d in &self.dependencies { @@ -84,7 +95,7 @@ impl Descriptor { /// Loads the descriptor to the global pool and returns a copy of the global pool. pub fn load_global(&self) -> prost_reflect::DescriptorPool { - static POOL : Lazy> = Lazy::new(||Mutex::default()); + static POOL: Lazy> = Lazy::new(Mutex::default); let pool = &mut POOL.lock().unwrap(); self.load(pool).unwrap(); pool.clone() @@ -106,13 +117,17 @@ pub struct Config { impl Config { /// Location of the protobuf_build crate, visible from the generated code. fn this_crate(&self) -> RustName { - self.protobuf_crate.clone().add("build") + self.protobuf_crate.clone().join("build") } /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated /// from a message of the given `proto_name`. fn reflect_impl(&self, proto_name: ProtoName) -> String { - let rust_name = proto_name.relative_to(&self.proto_root.to_name()).unwrap().to_rust_type().to_string(); + let rust_name = proto_name + .relative_to(&self.proto_root.to_name()) + .unwrap() + .to_rust_type() + .to_string(); let proto_name = proto_name.to_string(); let this = self.this_crate().to_string(); format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ @@ -125,61 +140,78 @@ impl Config { }}") } - /// Generates rust code from the proto files according to the config. - pub fn generate(&self) -> anyhow::Result<()> { - let input_root = self.input_root.abs(); - assert!(input_root.is_dir(),"input_root should be a directory"); - println!("cargo:rerun-if-changed={input_root:?}"); - + /// Generates rust code from the proto files according to the config. + pub fn generate(&self) -> anyhow::Result<()> { + assert!(self.input_root.abs().is_dir(), "input_root should be a directory"); + println!("cargo:rerun-if-changed={}",self.input_root.to_str()); + // Load dependencies. let mut pool = prost_reflect::DescriptorPool::new(); for d in &self.dependencies { - d.1.load(&mut pool).with_context(||format!("failed to load dependency {}",d.0.to_string()))?; + d.1.load(&mut pool) + .with_context(|| format!("failed to load dependency {}", d.0.to_string()))?; } let mut pool_raw = prost_types::FileDescriptorSet::default(); pool_raw.file = pool.file_descriptor_protos().cloned().collect(); // Load proto files. let mut proto_paths = vec![]; - traverse_files(&input_root, &mut |path| { - let Some(ext) = path.extension() else { return Ok(()) }; - let Some(ext) = ext.to_str() else { return Ok(()) }; - if ext != "proto" { return Ok(()) }; + traverse_files(&self.input_root.abs(), &mut |path| { + let Some(ext) = path.extension() else { + return Ok(()); + }; + let Some(ext) = ext.to_str() else { + return Ok(()); + }; + if ext != "proto" { + return Ok(()); + }; let file_raw = fs::read_to_string(path).context("fs::read()")?; - let path = ProtoPath::from_input_path(&path,&self.input_root,&self.proto_root); - pool_raw.file.push(protox_parse::parse(path.to_str(),&file_raw).map_err( - // rewrapping the error, so that source location is included in the error message. - |err|anyhow::anyhow!("{:?}",err) - )?); + let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root); + pool_raw + .file + .push(protox_parse::parse(path.to_str(), &file_raw).map_err( + // rewrapping the error, so that source location is included in the error message. + |err| anyhow::anyhow!("{:?}", err), + )?); proto_paths.push(path); Ok(()) })?; // Compile the proto files let mut compiler = protox::Compiler::with_file_resolver( - protox::file::DescriptorSetFileResolver::new(pool_raw) + protox::file::DescriptorSetFileResolver::new(pool_raw), ); compiler.include_source_info(true); - compiler.open_files(proto_paths.iter().map(|p|p.to_path())) + compiler + .open_files(proto_paths.iter().map(|p| p.to_path())) // rewrapping the error, so that source location is included in the error message. - .map_err(|err|anyhow::anyhow!("{:?}",err))?; + .map_err(|err| anyhow::anyhow!("{:?}", err))?; let descriptor = compiler.file_descriptor_set(); pool.add_file_descriptor_set(descriptor.clone()).unwrap(); - + // Check that the compiled proto files belong to the declared proto package. let package_root = self.proto_root.to_name(); for f in &descriptor.file { if !package_root.contains(&f.package().into()) { - anyhow::bail!("{:?} ({:?}) does not belong to package {:?}",f.package(),f.name(),package_root.to_string()); + anyhow::bail!( + "{:?} ({:?}) does not belong to package {:?}", + f.package(), + f.name(), + package_root.to_string() + ); } } // Check that the compiled proto messages support canonical encoding. - canonical::check(&descriptor,&pool).context("canonical::check()")?; + canonical::check(&descriptor, &pool).context("canonical::check()")?; // Prepare the output directory. - let output_dir = self.input_root.prepare_output_dir().context("prepare_output_dir()")?; + let output_dir = self + .input_root + .prepare_output_dir() + .context("prepare_output_dir()")?; let output_path = output_dir.join("gen.rs"); let descriptor_path = output_dir.join("gen.binpb"); fs::write(&descriptor_path, &descriptor.encode_to_vec())?; @@ -188,29 +220,40 @@ impl Config { // Generate code out of compiled proto files. let mut output = RustModule::default(); let mut config = prost_build::Config::new(); - config.prost_path(self.this_crate().add("prost").to_string()); + config.prost_path(self.this_crate().join("prost").to_string()); config.skip_protoc_run(); for d in &self.dependencies { for f in &d.1.descriptor_proto.file { - let proto_rel = ProtoName::from(f.package()).relative_to(&d.1.proto_root).unwrap(); - let rust_abs = d.0.clone().add(proto_rel.to_rust_module()); - config.extern_path(format!(".{}",f.package()), rust_abs.to_string()); + let proto_rel = ProtoName::from(f.package()) + .relative_to(&d.1.proto_root) + .unwrap(); + let rust_abs = d.0.clone().join(proto_rel.to_rust_module()); + config.extern_path(format!(".{}", f.package()), rust_abs.to_string()); } } let m = prost_build::Module::from_parts([""]); for f in &descriptor.file { - let code = config.generate(vec![(m.clone(),f.clone())]).context("generation failed")?; - output.sub(&ProtoName::from(f.package()).to_rust_module()).append(&code[&m]); + let code = config + .generate(vec![(m.clone(), f.clone())]) + .context("generation failed")?; + output + .sub(&ProtoName::from(f.package()).to_rust_module()) + .append(&code[&m]); } // Generate the reflection code. - let output = output.sub(&self.proto_root.to_name().to_rust_module()); + let output = output.sub(&self.proto_root.to_name().to_rust_module()); for proto_name in extract_message_names(&descriptor) { output.append(&self.reflect_impl(proto_name)); } // Generate the descriptor. - let rust_deps = self.dependencies.iter().map(|d|format!("&{}::DESCRIPTOR",d.0.to_string())).collect::>().join(","); + let rust_deps = self + .dependencies + .iter() + .map(|d| format!("&{}::DESCRIPTOR", d.0.to_string())) + .collect::>() + .join(","); let this = self.this_crate().to_string(); output.append(&format!("\ pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ @@ -219,7 +262,7 @@ impl Config { ",package_root.to_string())); // Save output. - fs::write(&output_path, output.format().context("output.format()")?)?; + fs::write(output_path, output.format().context("output.format()")?)?; Ok(()) } } diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 7b29d1e6..84c8f78b 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -1,30 +1,42 @@ //! Utilities for handling strings belonging to various namespaces. -use std::collections::BTreeMap; -use anyhow::Context as _; use super::ident; -use std::path::{Path,PathBuf}; +use anyhow::Context as _; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; /// Path relative to $CARGO_MANIFEST_DIR. -#[derive(Clone,PartialEq,Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct InputPath(PathBuf); impl From<&str> for InputPath { - fn from(s:&str) -> Self { Self(PathBuf::from(s)) } + fn from(s: &str) -> Self { + Self(PathBuf::from(s)) + } } impl InputPath { + /// Converts the relative input path to str + pub(super) fn to_str(&self) -> &str { + self.0.to_str().unwrap() + } + /// Converts the relative input path to an absolute path in the local file system /// (under $CARGO_MANIFEST_DIR). pub(super) fn abs(&self) -> PathBuf { PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) - .canonicalize().unwrap().join(&self.0) + .canonicalize() + .unwrap() + .join(&self.0) } /// Output directory path derived from the input path by replacing $CARGO_MANIFEST_DIR with $OUT_DIR. /// Re-constructs the derived output directory, as a side-effect. pub(super) fn prepare_output_dir(&self) -> anyhow::Result { let output = PathBuf::from(std::env::var("OUT_DIR")?) - .canonicalize()?.join(&self.0); + .canonicalize()? + .join(&self.0); let _ = std::fs::remove_dir_all(&output); std::fs::create_dir_all(&output)?; Ok(output) @@ -32,28 +44,40 @@ impl InputPath { } /// Absolute path of the proto file used for importing other proto files. -#[derive(Clone,PartialEq,Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct ProtoPath(PathBuf); impl From<&str> for ProtoPath { - fn from(s:&str) -> Self { Self(PathBuf::from(s)) } + fn from(s: &str) -> Self { + Self(PathBuf::from(s)) + } } impl ProtoPath { /// Converts a proto module path to proto package name by replacing all "/" with ".". pub(super) fn to_name(&self) -> ProtoName { - ProtoName(self.0.iter().map(|p|p.to_str().unwrap().into()).collect()) + ProtoName(self.0.iter().map(|p| p.to_str().unwrap().into()).collect()) } /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . - pub(super) fn from_input_path(path: &Path, input_root: &InputPath, proto_root: &ProtoPath) -> ProtoPath { - ProtoPath(proto_root.0.join(&path.strip_prefix(&input_root.abs()).unwrap())) - } - + pub(super) fn from_input_path( + path: &Path, + input_root: &InputPath, + proto_root: &ProtoPath, + ) -> ProtoPath { + ProtoPath( + proto_root + .0 + .join(path.strip_prefix(&input_root.abs()).unwrap()), + ) + } + + /// Converts ProtoPath to str. pub(super) fn to_str(&self) -> &str { self.0.to_str().unwrap() } + /// Converts ProtoPath to Path. pub(super) fn to_path(&self) -> &Path { &self.0 } @@ -65,24 +89,30 @@ type Part = String; /// generated code is location agnostic (it can be embedded in an arbitrary module within the crate), /// you need to manually (in the Config) specify the rust modules containing the generated code /// of the dependencies, so that it can be referenced from the newly generated code. -#[derive(Clone,PartialEq,Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct RustName(Vec); impl RustName { - pub fn add(mut self, suffix: impl Into) -> Self { - self.0.extend(suffix.into().0.into_iter()); + pub fn join(mut self, suffix: impl Into) -> Self { + self.0.extend(suffix.into().0); self } - pub fn to_string(&self) -> String { self.0.join("::") } + pub fn to_string(&self) -> String { + self.0.join("::") + } } impl From for RustName { - fn from(s:prost_build::Module) -> Self { Self(s.parts().map(Part::from).collect()) } + fn from(s: prost_build::Module) -> Self { + Self(s.parts().map(Part::from).collect()) + } } impl From<&str> for RustName { - fn from(s:&str) -> Self { Self(s.split("::").map(Part::from).collect()) } + fn from(s: &str) -> Self { + Self(s.split("::").map(Part::from).collect()) + } } /// A rust module representation. @@ -97,7 +127,7 @@ pub(super) struct RustModule { impl RustModule { /// Returns a reference to a given submodule. - pub fn sub(&mut self, path: &RustName) -> &mut Self { + pub(crate) fn sub(&mut self, path: &RustName) -> &mut Self { let mut m = self; for part in &path.0 { m = m.modules.entry(part.into()).or_default(); @@ -106,25 +136,28 @@ impl RustModule { } /// Appends code to the module. - pub fn append(&mut self, code: &str) { + pub(crate) fn append(&mut self, code: &str) { self.code += code; } fn collect(&self) -> String { let mut entries = vec![self.code.clone()]; entries.extend( - self.modules.iter().map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), + self.modules + .iter() + .map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), ); entries.join("") } /// Collects the code of the module and formats it. - pub fn format(&self) -> anyhow::Result { - Ok(prettyplease::unparse(&syn::parse_str(&self.collect()).context("syn::parse_str()")?)) + pub(crate) fn format(&self) -> anyhow::Result { + Ok(prettyplease::unparse( + &syn::parse_str(&self.collect()).context("syn::parse_str()")?, + )) } } - /// In addition to input paths and proto paths, there are also proto names /// which are used to reference message types from different proto files. /// Theoretically proto names can be totally independent from the proto paths (i.e. you can have @@ -133,7 +166,7 @@ impl RustModule { /// a) in a single file "a/b/c.proto", or /// b) in a collection of files under "a/b/c/" directory /// Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. -#[derive(Clone,PartialEq,Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct ProtoName(Vec); impl ProtoName { @@ -145,22 +178,26 @@ impl ProtoName { /// Strips a given prefix from the name. pub fn relative_to(&self, prefix: &Self) -> anyhow::Result { if !prefix.contains(self) { - anyhow::bail!("{} does not contain {}",self.to_string(),prefix.to_string()); + anyhow::bail!( + "{} does not contain {}", + self.to_string(), + prefix.to_string() + ); } - Ok(Self(self.0[prefix.0.len()..].iter().cloned().collect())) + Ok(Self(self.0[prefix.0.len()..].to_vec())) } /// Converts proto package name to rust module name according to prost_build rules. pub fn to_rust_module(&self) -> RustName { - RustName(self.0.iter().map(|s|ident::to_snake(s)).collect()) + RustName(self.0.iter().map(|s| ident::to_snake(s)).collect()) } /// Converts proto message name to rust type name according to prost_build rules. pub fn to_rust_type(&self) -> RustName { let mut rust = self.to_rust_module(); let n = rust.0.len(); - rust.0[n-1] = ident::to_upper_camel(&self.0[n-1]); - rust + rust.0[n - 1] = ident::to_upper_camel(&self.0[n - 1]); + rust } /// Converts ProtoName to string. @@ -170,25 +207,24 @@ impl ProtoName { } impl From<&str> for ProtoName { - fn from(s:&str) -> Self { - Self(s.split(".").map(String::from).collect()) + fn from(s: &str) -> Self { + Self(s.split('.').map(String::from).collect()) } } impl From<&Path> for ProtoName { - fn from(p:&Path) -> Self { - Self(p.iter().map(|c|c.to_str().unwrap().to_string()).collect()) + fn from(p: &Path) -> Self { + Self(p.iter().map(|c| c.to_str().unwrap().to_string()).collect()) } } - /// Extracts names of proto messages defined in the descriptor. pub(super) fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { fn collect(out: &mut Vec, prefix: &ProtoName, m: &prost_types::DescriptorProto) { let mut name = prefix.clone(); name.0.push(m.name().to_string()); for m in &m.nested_type { - collect(out,&name,m); + collect(out, &name, m); } out.push(name); } diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index 0e4b4eb8..402bec2e 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -14,7 +14,7 @@ rand.workspace = true serde.workspace = true tracing.workspace = true -protobuf.workspace = true +zksync_protobuf.workspace = true concurrency = { path = "../concurrency" } crypto = { path = "../crypto" } schema = { path = "../schema" } diff --git a/node/libs/roles/src/node/conv.rs b/node/libs/roles/src/node/conv.rs index ace73f67..f76fdcc8 100644 --- a/node/libs/roles/src/node/conv.rs +++ b/node/libs/roles/src/node/conv.rs @@ -1,7 +1,7 @@ use crate::node; -use protobuf::{read_required, required, ProtoFmt}; use anyhow::Context as _; use crypto::ByteFmt; +use zksync_protobuf::{read_required, required, ProtoFmt}; use utils::enum_util::Variant; impl ProtoFmt for node::Msg { diff --git a/node/libs/roles/src/node/messages.rs b/node/libs/roles/src/node/messages.rs index 77eeb9ce..2db888d7 100644 --- a/node/libs/roles/src/node/messages.rs +++ b/node/libs/roles/src/node/messages.rs @@ -17,7 +17,7 @@ pub enum Msg { impl Msg { /// Get the hash of this message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&protobuf::canonical(self))) + MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } } diff --git a/node/libs/roles/src/node/tests.rs b/node/libs/roles/src/node/tests.rs index 268874eb..adba219f 100644 --- a/node/libs/roles/src/node/tests.rs +++ b/node/libs/roles/src/node/tests.rs @@ -1,7 +1,7 @@ use super::*; -use protobuf::testonly::{test_encode, test_encode_random}; use concurrency::ctx; use crypto::{ByteFmt, Text, TextFmt}; +use zksync_protobuf::testonly::{test_encode, test_encode_random}; use rand::Rng; #[test] diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index 46a4f582..aee21ff6 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -5,9 +5,9 @@ use super::{ Signers, ViewNumber, }; use crate::node::SessionId; -use protobuf::{read_required, required, ProtoFmt}; use anyhow::Context as _; use crypto::ByteFmt; +use zksync_protobuf::{read_required, required, ProtoFmt}; use schema::proto::roles::validator as proto; use std::collections::BTreeMap; use utils::enum_util::Variant; @@ -187,7 +187,7 @@ impl ProtoFmt for LeaderCommit { } impl ProtoFmt for Signers { - type Proto = protobuf::proto::std::BitVector; + type Proto = zksync_protobuf::proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(ProtoFmt::read(r)?)) @@ -266,8 +266,8 @@ impl ProtoFmt for Phase { fn build(&self) -> Self::Proto { use proto::phase::T; let t = match self { - Self::Prepare => T::Prepare(protobuf::proto::std::Void {}), - Self::Commit => T::Commit(protobuf::proto::std::Void {}), + Self::Prepare => T::Prepare(zksync_protobuf::proto::std::Void {}), + Self::Commit => T::Commit(zksync_protobuf::proto::std::Void {}), }; Self::Proto { t: Some(t) } } diff --git a/node/libs/roles/src/validator/messages/block.rs b/node/libs/roles/src/validator/messages/block.rs index 5ae0c09e..ee71d6b2 100644 --- a/node/libs/roles/src/validator/messages/block.rs +++ b/node/libs/roles/src/validator/messages/block.rs @@ -97,7 +97,7 @@ pub struct BlockHeader { impl BlockHeader { /// Returns the hash of the block. pub fn hash(&self) -> BlockHeaderHash { - BlockHeaderHash(sha256::Sha256::new(&protobuf::canonical(self))) + BlockHeaderHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } /// Creates a genesis block. @@ -145,10 +145,10 @@ impl FinalBlock { impl ByteFmt for FinalBlock { fn decode(bytes: &[u8]) -> anyhow::Result { - protobuf::decode(bytes) + zksync_protobuf::decode(bytes) } fn encode(&self) -> Vec { - protobuf::encode(self) + zksync_protobuf::encode(self) } } diff --git a/node/libs/roles/src/validator/messages/msg.rs b/node/libs/roles/src/validator/messages/msg.rs index 2c944d24..a4384184 100644 --- a/node/libs/roles/src/validator/messages/msg.rs +++ b/node/libs/roles/src/validator/messages/msg.rs @@ -20,7 +20,7 @@ pub enum Msg { impl Msg { /// Returns the hash of the message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&protobuf::canonical(self))) + MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } } diff --git a/node/libs/roles/src/validator/tests.rs b/node/libs/roles/src/validator/tests.rs index fafe90c7..8f4438c6 100644 --- a/node/libs/roles/src/validator/tests.rs +++ b/node/libs/roles/src/validator/tests.rs @@ -1,7 +1,7 @@ use super::*; -use protobuf::testonly::test_encode_random; use concurrency::ctx; use crypto::{ByteFmt, Text, TextFmt}; +use zksync_protobuf::testonly::test_encode_random; use rand::Rng; use std::vec; diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index b9a33e99..3f27d5e6 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -11,17 +11,15 @@ anyhow.workspace = true bit-vec.workspace = true serde.workspace = true once_cell.workspace = true -quick-protobuf.workspace = true -prost.workspace = true -prost-reflect.workspace = true rand.workspace = true serde_json.workspace = true tokio.workspace = true -protobuf.workspace = true +prost.workspace = true +zksync_protobuf.workspace = true concurrency = { path = "../concurrency" } [build-dependencies] -protobuf.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index d4b17fbf..1c3f8c01 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,9 +1,11 @@ //! Generates rust code from the capnp schema files in the `capnp/` directory. fn main() { - protobuf::build::Config { + zksync_protobuf::build::Config { input_root: "proto".into(), proto_root: "zksync/schema".into(), - dependencies: vec![("::protobuf::proto".into(),&protobuf::proto::DESCRIPTOR)], - protobuf_crate: "::protobuf".into(), - }.generate().expect("generate()"); + dependencies: vec![("::zksync_protobuf::proto".into(), &zksync_protobuf::proto::DESCRIPTOR)], + protobuf_crate: "::zksync_protobuf".into(), + } + .generate() + .expect("generate()"); } diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index e838faf9..82896c5e 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -14,7 +14,7 @@ rocksdb = { workspace = true, optional = true } thiserror.workspace = true tracing.workspace = true -protobuf.workspace = true +zksync_protobuf.workspace = true concurrency = { path = "../concurrency" } roles = { path = "../roles" } schema = { path = "../schema" } diff --git a/node/libs/storage/src/rocksdb.rs b/node/libs/storage/src/rocksdb.rs index efbdc496..fb95944a 100644 --- a/node/libs/storage/src/rocksdb.rs +++ b/node/libs/storage/src/rocksdb.rs @@ -145,7 +145,7 @@ impl RocksdbStorage { .next() .context("Head block not found")? .context("RocksDB error reading head block")?; - protobuf::decode(&head_block).context("Failed decoding head block bytes") + zksync_protobuf::decode(&head_block).context("Failed decoding head block bytes") } /// Returns a block with the least number stored in this database. @@ -159,7 +159,7 @@ impl RocksdbStorage { .next() .context("First stored block not found")? .context("RocksDB error reading first stored block")?; - protobuf::decode(&first_block).context("Failed decoding first stored block bytes") + zksync_protobuf::decode(&first_block).context("Failed decoding first stored block bytes") } fn last_contiguous_block_number_blocking(&self) -> anyhow::Result { @@ -219,7 +219,7 @@ impl RocksdbStorage { else { return Ok(None); }; - let block = protobuf::decode(&raw_block) + let block = zksync_protobuf::decode(&raw_block) .with_context(|| format!("Failed decoding block #{number}"))?; Ok(Some(block)) } @@ -258,7 +258,7 @@ impl RocksdbStorage { let mut write_batch = rocksdb::WriteBatch::default(); write_batch.put( DatabaseKey::Block(block_number).encode_key(), - protobuf::encode(finalized_block), + zksync_protobuf::encode(finalized_block), ); // Commit the transaction. db.write(write_batch) @@ -277,7 +277,7 @@ impl RocksdbStorage { else { return Ok(None); }; - protobuf::decode(&raw_state) + zksync_protobuf::decode(&raw_state) .map(Some) .context("Failed to decode replica state!") } @@ -286,7 +286,7 @@ impl RocksdbStorage { self.write() .put( DatabaseKey::ReplicaState.encode_key(), - protobuf::encode(replica_state), + zksync_protobuf::encode(replica_state), ) .context("Failed putting ReplicaState to RocksDB") } diff --git a/node/libs/storage/src/tests/mod.rs b/node/libs/storage/src/tests/mod.rs index f5625124..d621201f 100644 --- a/node/libs/storage/src/tests/mod.rs +++ b/node/libs/storage/src/tests/mod.rs @@ -127,7 +127,10 @@ fn test_schema_encode_decode() { let rng = &mut ctx.rng(); let replica = rng.gen::(); - assert_eq!(replica, protobuf::decode(&protobuf::encode(&replica)).unwrap()); + assert_eq!( + replica, + zksync_protobuf::decode(&zksync_protobuf::encode(&replica)).unwrap() + ); } #[test] diff --git a/node/libs/storage/src/types.rs b/node/libs/storage/src/types.rs index 278492f4..881eb0dc 100644 --- a/node/libs/storage/src/types.rs +++ b/node/libs/storage/src/types.rs @@ -1,9 +1,9 @@ //! Defines the schema of the database. use anyhow::Context as _; use concurrency::ctx; +use zksync_protobuf::{read_required, required, ProtoFmt}; use roles::validator::{self, BlockNumber}; use schema::proto::storage as proto; -use protobuf::{read_required, required, ProtoFmt}; use std::{iter, ops}; /// A payload of a proposed block which is not known to be finalized yet. diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index d3411d03..5709f78d 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -17,7 +17,7 @@ tracing.workspace = true tracing-subscriber.workspace = true vise-exporter.workspace = true -protobuf.workspace = true +zksync_protobuf.workspace = true concurrency = { path = "../libs/concurrency" } crypto = { path = "../libs/crypto" } roles = { path = "../libs/roles" } diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 6592c621..72b9b82a 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -110,7 +110,7 @@ fn main() -> anyhow::Result<()> { let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; - fs::write(root.join("config.json"), protobuf::encode_json(&node_cfg)) + fs::write(root.join("config.json"), zksync_protobuf::encode_json(&node_cfg)) .context("fs::write()")?; fs::write( root.join("validator_key"), diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 4b104442..0bd09e96 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -3,9 +3,9 @@ use anyhow::Context as _; use crypto::{read_optional_text, Text, TextFmt}; use executor::{ConsensusConfig, ExecutorConfig}; +use zksync_protobuf::{read_optional, read_required, ProtoFmt}; use roles::{node, validator}; use schema::proto::executor::config as proto; -use protobuf::{read_optional, read_required, ProtoFmt}; use std::{fs, net, path::Path}; /// This struct holds the file path to each of the config files. @@ -77,7 +77,7 @@ impl Configs { args.config.display() ) })?; - let node_config: NodeConfig = protobuf::decode_json(&node_config).with_context(|| { + let node_config: NodeConfig = zksync_protobuf::decode_json(&node_config).with_context(|| { format!( "failed decoding JSON node config at `{}`", args.config.display() From b9c88baff6af76b1919fad59d2ca226d285578a7 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 16:27:31 +0100 Subject: [PATCH 23/40] cargo_fmt --- node/actors/consensus/src/metrics.rs | 6 ++++-- node/actors/executor/src/config/mod.rs | 2 +- node/actors/executor/src/config/tests.rs | 2 +- .../network/src/consensus/handshake/mod.rs | 2 +- .../network/src/gossip/handshake/mod.rs | 2 +- node/actors/network/src/mux/handshake.rs | 2 +- node/actors/network/src/preface.rs | 2 +- node/actors/network/src/rpc/consensus.rs | 2 +- node/actors/network/src/rpc/ping.rs | 2 +- node/actors/network/src/rpc/sync_blocks.rs | 2 +- .../network/src/rpc/sync_validator_addrs.rs | 2 +- node/libs/protobuf/build.rs | 1 + node/libs/protobuf_build/src/lib.rs | 20 ++++++++++++++----- node/libs/protobuf_build/src/syntax.rs | 15 +++++++++----- node/libs/roles/src/node/conv.rs | 2 +- node/libs/roles/src/node/tests.rs | 2 +- node/libs/roles/src/validator/conv.rs | 2 +- node/libs/roles/src/validator/tests.rs | 2 +- node/libs/schema/build.rs | 7 +++++-- node/libs/schema/proto/buf.yaml | 10 ---------- node/libs/storage/src/types.rs | 2 +- node/tools/src/bin/localnet_config.rs | 7 +++++-- node/tools/src/config.rs | 15 +++++++------- 23 files changed, 63 insertions(+), 48 deletions(-) delete mode 100644 node/libs/schema/proto/buf.yaml diff --git a/node/actors/consensus/src/metrics.rs b/node/actors/consensus/src/metrics.rs index 0327a044..8fef767e 100644 --- a/node/actors/consensus/src/metrics.rs +++ b/node/actors/consensus/src/metrics.rs @@ -3,8 +3,10 @@ use std::time::Duration; use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; -const PAYLOAD_SIZE_BUCKETS: Buckets = - Buckets::exponential((4 * zksync_protobuf::kB) as f64..=(4 * zksync_protobuf::MB) as f64, 4.0); +const PAYLOAD_SIZE_BUCKETS: Buckets = Buckets::exponential( + (4 * zksync_protobuf::kB) as f64..=(4 * zksync_protobuf::MB) as f64, + 4.0, +); /// Label for a consensus message. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] diff --git a/node/actors/executor/src/config/mod.rs b/node/actors/executor/src/config/mod.rs index d2f42bb5..4c32cdf6 100644 --- a/node/actors/executor/src/config/mod.rs +++ b/node/actors/executor/src/config/mod.rs @@ -3,13 +3,13 @@ use anyhow::Context as _; use crypto::{read_required_text, Text, TextFmt}; use network::{consensus, gossip}; -use zksync_protobuf::{read_required, required, ProtoFmt}; use roles::{node, validator}; use schema::proto::executor::config as proto; use std::{ collections::{HashMap, HashSet}, net, }; +use zksync_protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod tests; diff --git a/node/actors/executor/src/config/tests.rs b/node/actors/executor/src/config/tests.rs index 98b34367..8baeb573 100644 --- a/node/actors/executor/src/config/tests.rs +++ b/node/actors/executor/src/config/tests.rs @@ -1,11 +1,11 @@ use super::{ConsensusConfig, ExecutorConfig, GossipConfig}; use concurrency::ctx; -use zksync_protobuf::testonly::test_encode_random; use rand::{ distributions::{Distribution, Standard}, Rng, }; use roles::{node, validator}; +use zksync_protobuf::testonly::test_encode_random; fn make_addr(rng: &mut R) -> std::net::SocketAddr { std::net::SocketAddr::new(std::net::IpAddr::from(rng.gen::<[u8; 16]>()), rng.gen()) diff --git a/node/actors/network/src/consensus/handshake/mod.rs b/node/actors/network/src/consensus/handshake/mod.rs index f5152a95..95765d9a 100644 --- a/node/actors/network/src/consensus/handshake/mod.rs +++ b/node/actors/network/src/consensus/handshake/mod.rs @@ -2,9 +2,9 @@ use crate::{frame, noise}; use anyhow::Context as _; use concurrency::{ctx, time}; use crypto::{bls12_381, ByteFmt}; -use zksync_protobuf::{read_required, ProtoFmt}; use roles::{node, validator}; use schema::proto::network::consensus as proto; +use zksync_protobuf::{read_required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/gossip/handshake/mod.rs b/node/actors/network/src/gossip/handshake/mod.rs index 49a27e7c..833f9377 100644 --- a/node/actors/network/src/gossip/handshake/mod.rs +++ b/node/actors/network/src/gossip/handshake/mod.rs @@ -3,9 +3,9 @@ use crate::{frame, noise}; use anyhow::Context as _; use concurrency::{ctx, time}; use crypto::ByteFmt; -use zksync_protobuf::{read_required, required, ProtoFmt}; use roles::node; use schema::proto::network::gossip as proto; +use zksync_protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/mux/handshake.rs b/node/actors/network/src/mux/handshake.rs index d2ee6e39..f2f0d8ea 100644 --- a/node/actors/network/src/mux/handshake.rs +++ b/node/actors/network/src/mux/handshake.rs @@ -1,8 +1,8 @@ use super::CapabilityId; use anyhow::Context as _; -use zksync_protobuf::required; use schema::proto::network::mux as proto; use std::collections::HashMap; +use zksync_protobuf::required; pub(super) struct Handshake { /// Maximal supported number of the accept streams per capability. diff --git a/node/actors/network/src/preface.rs b/node/actors/network/src/preface.rs index 6a0f01a9..46289289 100644 --- a/node/actors/network/src/preface.rs +++ b/node/actors/network/src/preface.rs @@ -9,8 +9,8 @@ //! and multiplex between mutliple endpoints available on the same TCP port. use crate::{frame, metrics, noise}; use concurrency::{ctx, time}; -use zksync_protobuf::{required, ProtoFmt}; use schema::proto::network::preface as proto; +use zksync_protobuf::{required, ProtoFmt}; /// Timeout on executing the preface protocol. const TIMEOUT: time::Duration = time::Duration::seconds(5); diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 80f16699..5038891c 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -1,9 +1,9 @@ //! Defines RPC for passing consensus messages. use crate::mux; use concurrency::{limiter, time}; -use zksync_protobuf::{read_required, ProtoFmt}; use roles::validator; use schema::proto::network::consensus as proto; +use zksync_protobuf::{read_required, ProtoFmt}; /// Consensus RPC. pub(crate) struct Rpc; diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index e38b2ae3..227ef9bb 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -2,9 +2,9 @@ use crate::{mux, rpc::Rpc as _}; use anyhow::Context as _; use concurrency::{ctx, limiter, time}; -use zksync_protobuf::{required, ProtoFmt}; use rand::Rng; use schema::proto::network::ping as proto; +use zksync_protobuf::{required, ProtoFmt}; /// Ping RPC. pub(crate) struct Rpc; diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index d59aacfd..08e8f7e0 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -3,9 +3,9 @@ use crate::{io, mux}; use anyhow::Context; use concurrency::{limiter, time}; -use zksync_protobuf::{read_required, ProtoFmt}; use roles::validator::{BlockNumber, FinalBlock}; use schema::proto::network::gossip as proto; +use zksync_protobuf::{read_required, ProtoFmt}; /// `get_sync_state` RPC. #[derive(Debug)] diff --git a/node/actors/network/src/rpc/sync_validator_addrs.rs b/node/actors/network/src/rpc/sync_validator_addrs.rs index fd95d002..689a8d86 100644 --- a/node/actors/network/src/rpc/sync_validator_addrs.rs +++ b/node/actors/network/src/rpc/sync_validator_addrs.rs @@ -3,10 +3,10 @@ use crate::mux; use anyhow::Context as _; use concurrency::{limiter, time}; -use zksync_protobuf::ProtoFmt; use roles::validator; use schema::proto::network::gossip as proto; use std::sync::Arc; +use zksync_protobuf::ProtoFmt; /// SyncValidatorAddrs Rpc. pub(crate) struct Rpc; diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 471483a3..0348639b 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -1,3 +1,4 @@ +//! Generates rust code from protobufs. fn main() { zksync_protobuf_build::Config { input_root: "src/proto".into(), diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 179cc619..6c5e62a9 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -26,6 +26,7 @@ //! cargo build --all-targets //! perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find target/debug/build/*/output -type f` | xargs cat > /tmp/sum.binpb //! buf breaking /tmp/sum.binpb --against /tmp/sum.binpb +#![allow(clippy::print_stdout)] use anyhow::Context as _; pub use once_cell::sync::Lazy; // Imports accessed from the generated code. @@ -56,8 +57,12 @@ fn traverse_files( /// Protobuf descriptor + info about the mapping to rust code. pub struct Descriptor { + /// Root proto package that all proto files in this descriptor belong to. + /// Rust types have been generated relative to this root. proto_root: ProtoName, + /// Raw descriptor proto. descriptor_proto: prost_types::FileDescriptorSet, + /// Direct dependencies of this descriptor. dependencies: Vec<&'static Descriptor>, } @@ -95,6 +100,7 @@ impl Descriptor { /// Loads the descriptor to the global pool and returns a copy of the global pool. pub fn load_global(&self) -> prost_reflect::DescriptorPool { + /// Global descriptor pool. static POOL: Lazy> = Lazy::new(Mutex::default); let pool = &mut POOL.lock().unwrap(); self.load(pool).unwrap(); @@ -102,6 +108,7 @@ impl Descriptor { } } +/// Code generation config. Use it in build scripts. pub struct Config { /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. pub input_root: InputPath, @@ -142,17 +149,20 @@ impl Config { /// Generates rust code from the proto files according to the config. pub fn generate(&self) -> anyhow::Result<()> { - assert!(self.input_root.abs().is_dir(), "input_root should be a directory"); - println!("cargo:rerun-if-changed={}",self.input_root.to_str()); + assert!( + self.input_root.abs().is_dir(), + "input_root should be a directory" + ); + println!("cargo:rerun-if-changed={}", self.input_root.to_str()); // Load dependencies. let mut pool = prost_reflect::DescriptorPool::new(); for d in &self.dependencies { d.1.load(&mut pool) - .with_context(|| format!("failed to load dependency {}", d.0.to_string()))?; + .with_context(|| format!("failed to load dependency {}", d.0))?; } let mut pool_raw = prost_types::FileDescriptorSet::default(); - pool_raw.file = pool.file_descriptor_protos().cloned().collect(); + pool_raw.file.extend(pool.file_descriptor_protos().cloned()); // Load proto files. let mut proto_paths = vec![]; @@ -251,7 +261,7 @@ impl Config { let rust_deps = self .dependencies .iter() - .map(|d| format!("&{}::DESCRIPTOR", d.0.to_string())) + .map(|d| format!("&{}::DESCRIPTOR", d.0)) .collect::>() .join(","); let this = self.this_crate().to_string(); diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 84c8f78b..dfafd214 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -3,6 +3,7 @@ use super::ident; use anyhow::Context as _; use std::{ collections::BTreeMap, + fmt, path::{Path, PathBuf}, }; @@ -93,13 +94,16 @@ type Part = String; pub struct RustName(Vec); impl RustName { + /// Concatenates 2 rust names. pub fn join(mut self, suffix: impl Into) -> Self { self.0.extend(suffix.into().0); self } +} - pub fn to_string(&self) -> String { - self.0.join("::") +impl fmt::Display for RustName { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&self.0.join("::")) } } @@ -199,10 +203,11 @@ impl ProtoName { rust.0[n - 1] = ident::to_upper_camel(&self.0[n - 1]); rust } +} - /// Converts ProtoName to string. - pub fn to_string(&self) -> String { - self.0.join(".") +impl fmt::Display for ProtoName { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&self.0.join(".")) } } diff --git a/node/libs/roles/src/node/conv.rs b/node/libs/roles/src/node/conv.rs index f76fdcc8..e2314845 100644 --- a/node/libs/roles/src/node/conv.rs +++ b/node/libs/roles/src/node/conv.rs @@ -1,8 +1,8 @@ use crate::node; use anyhow::Context as _; use crypto::ByteFmt; -use zksync_protobuf::{read_required, required, ProtoFmt}; use utils::enum_util::Variant; +use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for node::Msg { type Proto = node::schema::Msg; diff --git a/node/libs/roles/src/node/tests.rs b/node/libs/roles/src/node/tests.rs index adba219f..7d4408c2 100644 --- a/node/libs/roles/src/node/tests.rs +++ b/node/libs/roles/src/node/tests.rs @@ -1,8 +1,8 @@ use super::*; use concurrency::ctx; use crypto::{ByteFmt, Text, TextFmt}; -use zksync_protobuf::testonly::{test_encode, test_encode_random}; use rand::Rng; +use zksync_protobuf::testonly::{test_encode, test_encode_random}; #[test] fn test_byte_encoding() { diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index aee21ff6..9315c35a 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -7,10 +7,10 @@ use super::{ use crate::node::SessionId; use anyhow::Context as _; use crypto::ByteFmt; -use zksync_protobuf::{read_required, required, ProtoFmt}; use schema::proto::roles::validator as proto; use std::collections::BTreeMap; use utils::enum_util::Variant; +use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for BlockHeaderHash { type Proto = proto::BlockHeaderHash; diff --git a/node/libs/roles/src/validator/tests.rs b/node/libs/roles/src/validator/tests.rs index 8f4438c6..6bfb21b4 100644 --- a/node/libs/roles/src/validator/tests.rs +++ b/node/libs/roles/src/validator/tests.rs @@ -1,9 +1,9 @@ use super::*; use concurrency::ctx; use crypto::{ByteFmt, Text, TextFmt}; -use zksync_protobuf::testonly::test_encode_random; use rand::Rng; use std::vec; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_byte_encoding() { diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 1c3f8c01..40c8e5c4 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,9 +1,12 @@ -//! Generates rust code from the capnp schema files in the `capnp/` directory. +//! Generates rust code from protobufs. fn main() { zksync_protobuf::build::Config { input_root: "proto".into(), proto_root: "zksync/schema".into(), - dependencies: vec![("::zksync_protobuf::proto".into(), &zksync_protobuf::proto::DESCRIPTOR)], + dependencies: vec![( + "::zksync_protobuf::proto".into(), + &zksync_protobuf::proto::DESCRIPTOR, + )], protobuf_crate: "::zksync_protobuf".into(), } .generate() diff --git a/node/libs/schema/proto/buf.yaml b/node/libs/schema/proto/buf.yaml deleted file mode 100644 index 94d54ac3..00000000 --- a/node/libs/schema/proto/buf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: v1 -breaking: - use: - # We use FILE rule set for everything, - # althout only protos which we use with JSON format need that. - # All the other protos would be fine with WIRE rule set. - # TODO(gprusak): separate JSON protos from binary protos - # to weaken the compatibility restrictions, in case using FILE - # for everything turns out to be too annoying. - - FILE diff --git a/node/libs/storage/src/types.rs b/node/libs/storage/src/types.rs index 881eb0dc..a9a78f7b 100644 --- a/node/libs/storage/src/types.rs +++ b/node/libs/storage/src/types.rs @@ -1,10 +1,10 @@ //! Defines the schema of the database. use anyhow::Context as _; use concurrency::ctx; -use zksync_protobuf::{read_required, required, ProtoFmt}; use roles::validator::{self, BlockNumber}; use schema::proto::storage as proto; use std::{iter, ops}; +use zksync_protobuf::{read_required, required, ProtoFmt}; /// A payload of a proposed block which is not known to be finalized yet. /// Replicas have to persist such proposed payloads for liveness: diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 72b9b82a..d8dd3f5a 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -110,8 +110,11 @@ fn main() -> anyhow::Result<()> { let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; - fs::write(root.join("config.json"), zksync_protobuf::encode_json(&node_cfg)) - .context("fs::write()")?; + fs::write( + root.join("config.json"), + zksync_protobuf::encode_json(&node_cfg), + ) + .context("fs::write()")?; fs::write( root.join("validator_key"), &TextFmt::encode(&validator_keys[i]), diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 0bd09e96..d8bc44c1 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -3,10 +3,10 @@ use anyhow::Context as _; use crypto::{read_optional_text, Text, TextFmt}; use executor::{ConsensusConfig, ExecutorConfig}; -use zksync_protobuf::{read_optional, read_required, ProtoFmt}; use roles::{node, validator}; use schema::proto::executor::config as proto; use std::{fs, net, path::Path}; +use zksync_protobuf::{read_optional, read_required, ProtoFmt}; /// This struct holds the file path to each of the config files. #[derive(Debug)] @@ -77,12 +77,13 @@ impl Configs { args.config.display() ) })?; - let node_config: NodeConfig = zksync_protobuf::decode_json(&node_config).with_context(|| { - format!( - "failed decoding JSON node config at `{}`", - args.config.display() - ) - })?; + let node_config: NodeConfig = + zksync_protobuf::decode_json(&node_config).with_context(|| { + format!( + "failed decoding JSON node config at `{}`", + args.config.display() + ) + })?; let validator_key: Option = args .validator_key From 1948af46884d153eff2e468a7c825899d7418cf7 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 16:33:41 +0100 Subject: [PATCH 24/40] presubmit --- .github/workflows/protobuf.yaml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index 68daaf44..12043992 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -14,7 +14,12 @@ on: branches: ["main"] env: - PROTOS_DIR: "node/libs/schema/proto" + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + RUSTFLAGS: "-Dwarnings -C linker=clang -C link-arg=-fuse-ld=lld -C link-arg=-Wl,-z,nostart-stop-gc" + RUSTC_WRAPPER: "sccache" + SCCACHE_GHA_ENABLED: "true" + RUST_BACKTRACE: "1" jobs: compatibility: @@ -22,16 +27,23 @@ jobs: steps: # github.base_ref -> github.head_ref for pull_request # github.event.before -> github.event.after for push + - uses: mozilla-actions/sccache-action@v0.0.3 - uses: actions/checkout@v3 with: ref: ${{ github.base_ref || github.event.before }} path: before + - name: compile before + run: cargo build --all-targets + working-directory: ./before/node - uses: actions/checkout@v3 with: ref: ${{ github.head_ref || github.event.after }} path: after - - uses: bufbuild/buf-setup-action@v1 - - uses: bufbuild/buf-breaking-action@v1 - with: - input: "after/$PROTOS_DIR" - against: "before/$PROTOS_DIR" + - name: compile after + run: cargo build --all-targets + working-directory: ./after/node + # - uses: bufbuild/buf-setup-action@v1 + # - uses: bufbuild/buf-breaking-action@v1 + # with: + # input: "after/$PROTOS_DIR" + # against: "before/$PROTOS_DIR" From a5493e5064351714d5f359366c8ecae7f5eb1cbb Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 18:26:05 +0100 Subject: [PATCH 25/40] conformance --- .github/workflows/protobuf_conformance.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/protobuf_conformance.yaml b/.github/workflows/protobuf_conformance.yaml index 6027b84f..86467bd5 100644 --- a/.github/workflows/protobuf_conformance.yaml +++ b/.github/workflows/protobuf_conformance.yaml @@ -28,7 +28,7 @@ jobs: path: "protobuf" - uses: mozilla-actions/sccache-action@v0.0.3 - name: build test - run: cargo build -p schema --bin conformance_test + run: cargo build -p zksync_protobuf --bin conformance_test working-directory: "this/node" - name: Cache Bazel uses: actions/cache@v3 @@ -41,6 +41,6 @@ jobs: - name: run test run: > bazel run //conformance:conformance_test_runner -- - --failure_list "${{ github.workspace }}/this/node/libs/schema/src/bin/conformance_test_failure_list.txt" + --failure_list "${{ github.workspace }}/this/node/libs/protobuf/src/bin/conformance_test/conformance_test_failure_list.txt" "${{ github.workspace }}/this/node/target/debug/conformance_test" working-directory: "protobuf" From e96d0be07f58bac62128fa51c9a84edb30af6948 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 18:28:07 +0100 Subject: [PATCH 26/40] compatibility --- .github/workflows/protobuf.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index 12043992..12ce737d 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -33,14 +33,16 @@ jobs: ref: ${{ github.base_ref || github.event.before }} path: before - name: compile before - run: cargo build --all-targets + run: cargo build --all-targets + with: working-directory: ./before/node - uses: actions/checkout@v3 with: ref: ${{ github.head_ref || github.event.after }} path: after - name: compile after - run: cargo build --all-targets + run: cargo build --all-targets + with: working-directory: ./after/node # - uses: bufbuild/buf-setup-action@v1 # - uses: bufbuild/buf-breaking-action@v1 From d2fc4dd5266211d635f96d2e733cf78f2347fbad Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 18:29:36 +0100 Subject: [PATCH 27/40] compatibility --- .github/workflows/protobuf.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index 12ce737d..da73ef62 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -34,16 +34,14 @@ jobs: path: before - name: compile before run: cargo build --all-targets - with: - working-directory: ./before/node + working-directory: ./before/node - uses: actions/checkout@v3 with: ref: ${{ github.head_ref || github.event.after }} path: after - name: compile after run: cargo build --all-targets - with: - working-directory: ./after/node + working-directory: ./after/node # - uses: bufbuild/buf-setup-action@v1 # - uses: bufbuild/buf-breaking-action@v1 # with: From e37f4f4244f5a96fbac48415d365478d6b9384b7 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 18:48:18 +0100 Subject: [PATCH 28/40] compatibility --- .github/workflows/protobuf.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index da73ef62..0f0f3576 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -35,6 +35,11 @@ jobs: - name: compile before run: cargo build --all-targets working-directory: ./before/node + - name: build before.binpb + run: > + perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' + `find ./before/node/target/debug/build/*/output` + | xargs cat > ./before.binpb - uses: actions/checkout@v3 with: ref: ${{ github.head_ref || github.event.after }} From 64e21c85602b4cef501f79e57c5e9cab371536c3 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 19:41:06 +0100 Subject: [PATCH 29/40] fix for the conformance test --- node/Cargo.lock | 2 ++ node/libs/protobuf/Cargo.toml | 4 ++- ...test_failure_list.txt => failure_list.txt} | 0 .../protobuf/src/bin/conformance_test/main.rs | 34 +++++++++++++++---- 4 files changed, 33 insertions(+), 7 deletions(-) rename node/libs/protobuf/src/bin/conformance_test/{conformance_test_failure_list.txt => failure_list.txt} (100%) diff --git a/node/Cargo.lock b/node/Cargo.lock index 9fc74d80..286a4b6d 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -2391,6 +2391,8 @@ dependencies = [ "serde", "serde_json", "tokio", + "tracing", + "tracing-subscriber", "zksync_protobuf_build", ] diff --git a/node/libs/protobuf/Cargo.toml b/node/libs/protobuf/Cargo.toml index c2f3f4f0..fc4ff927 100644 --- a/node/libs/protobuf/Cargo.toml +++ b/node/libs/protobuf/Cargo.toml @@ -14,12 +14,14 @@ anyhow.workspace = true bit-vec.workspace = true serde.workspace = true quick-protobuf.workspace = true +once_cell.workspace = true prost.workspace = true prost-reflect.workspace = true rand.workspace = true serde_json.workspace = true tokio.workspace = true -once_cell.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true zksync_protobuf_build.workspace = true concurrency = { path = "../concurrency" } diff --git a/node/libs/protobuf/src/bin/conformance_test/conformance_test_failure_list.txt b/node/libs/protobuf/src/bin/conformance_test/failure_list.txt similarity index 100% rename from node/libs/protobuf/src/bin/conformance_test/conformance_test_failure_list.txt rename to node/libs/protobuf/src/bin/conformance_test/failure_list.txt diff --git a/node/libs/protobuf/src/bin/conformance_test/main.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs index f1c3fce9..584c85da 100644 --- a/node/libs/protobuf/src/bin/conformance_test/main.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -1,19 +1,20 @@ //! Conformance test for our canonical encoding implemented according to //! https://github.com/protocolbuffers/zksync_protobuf/blob/main/conformance/conformance.proto //! Our implementation supports only a subset of proto functionality, so -//! `schema/proto/conformance/conformance.proto` and -//! `schema/proto/conformance/zksync_protobuf_test_messages.proto` contains only a +//! `proto/conformance.proto` and +//! `proto/protobuf_test_messages.proto` contains only a //! subset of original fields. Also we run only proto3 binary -> binary tests. -//! conformance_test_failure_list.txt contains tests which are expected to fail. +//! failure_list.txt contains tests which are expected to fail. use anyhow::Context as _; use concurrency::{ctx, io}; use prost::Message as _; use prost_reflect::ReflectMessage; +use std::sync::Mutex; mod proto; -#[tokio::main] -async fn main() -> anyhow::Result<()> { +/// Runs the test server. +async fn run() -> anyhow::Result<()> { let ctx = &ctx::root(); let stdin = &mut tokio::io::stdin(); let stdout = &mut tokio::io::stdout(); @@ -31,7 +32,7 @@ async fn main() -> anyhow::Result<()> { let req = proto::ConformanceRequest::decode(&msg[..])?; let res = async { let t = req.message_type.context("missing message_type")?; - if t != *"zksync_protobuf_test_messages.proto3.TestAllTypesProto3" { + if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { return Ok(R::Skipped("unsupported".to_string())); } @@ -87,3 +88,24 @@ async fn main() -> anyhow::Result<()> { io::flush(ctx, stdout).await??; } } + +#[tokio::main] +async fn main() { + let sub = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()); + match std::env::var("LOG_FILE") { + Err(_) => sub.with_writer(std::io::stderr).init(), + Ok(path) => sub + .with_writer(Mutex::new( + std::fs::File::options() + .create(true) + .append(true) + .open(path) + .unwrap(), + )) + .init(), + }; + if let Err(err) = run().await { + tracing::error!("run(): {err:#}"); + } +} From 7d5a64002d2bae4a5ce70db4ca49d23031e8c489 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 19:43:12 +0100 Subject: [PATCH 30/40] file rename --- .github/workflows/protobuf_conformance.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/protobuf_conformance.yaml b/.github/workflows/protobuf_conformance.yaml index 86467bd5..69efd530 100644 --- a/.github/workflows/protobuf_conformance.yaml +++ b/.github/workflows/protobuf_conformance.yaml @@ -41,6 +41,6 @@ jobs: - name: run test run: > bazel run //conformance:conformance_test_runner -- - --failure_list "${{ github.workspace }}/this/node/libs/protobuf/src/bin/conformance_test/conformance_test_failure_list.txt" + --failure_list "${{ github.workspace }}/this/node/libs/protobuf/src/bin/conformance_test/failure_list.txt" "${{ github.workspace }}/this/node/target/debug/conformance_test" working-directory: "protobuf" From b85485497c3ac913e57d73d6fc8fbb8fbdf30dba Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 19:47:16 +0100 Subject: [PATCH 31/40] compatibility --- .github/workflows/protobuf.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index 0f0f3576..59bd6a36 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -47,8 +47,13 @@ jobs: - name: compile after run: cargo build --all-targets working-directory: ./after/node - # - uses: bufbuild/buf-setup-action@v1 - # - uses: bufbuild/buf-breaking-action@v1 - # with: - # input: "after/$PROTOS_DIR" - # against: "before/$PROTOS_DIR" + - name: build after.binpb + run: > + perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' + `find ./after/node/target/debug/build/*/output` + | xargs cat > ./after.binpb + - uses: bufbuild/buf-setup-action@v1 + - uses: bufbuild/buf-breaking-action@v1 + with: + input: "./after.binpb" + against: "./before.binpb" From f01b63c37bf8b850190ce37bd5e9a70d48badc21 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Fri, 3 Nov 2023 20:58:10 +0100 Subject: [PATCH 32/40] code attribution --- node/libs/protobuf_build/src/ident.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/node/libs/protobuf_build/src/ident.rs b/node/libs/protobuf_build/src/ident.rs index 0e85aa83..9dc4e003 100644 --- a/node/libs/protobuf_build/src/ident.rs +++ b/node/libs/protobuf_build/src/ident.rs @@ -1,4 +1,6 @@ -//! Utility functions for working with identifiers. +//! Copy of https://github.com/tokio-rs/prost/blob/master/prost-build/src/ident.rs. +//! We need it to compute the proto names -> rust names mapping, which is not directly accessible +//! via prost_build public API. use heck::{ToSnakeCase, ToUpperCamelCase}; From 608ed40af2cca46365f69e87cc2789fcfadfa1d5 Mon Sep 17 00:00:00 2001 From: pompon0 Date: Tue, 7 Nov 2023 11:09:13 +0100 Subject: [PATCH 33/40] Update node/libs/protobuf/src/bin/conformance_test/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bruno França --- node/libs/protobuf/src/bin/conformance_test/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/libs/protobuf/src/bin/conformance_test/main.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs index 584c85da..e2e02d9b 100644 --- a/node/libs/protobuf/src/bin/conformance_test/main.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -1,5 +1,5 @@ //! Conformance test for our canonical encoding implemented according to -//! https://github.com/protocolbuffers/zksync_protobuf/blob/main/conformance/conformance.proto +//! https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto //! Our implementation supports only a subset of proto functionality, so //! `proto/conformance.proto` and //! `proto/protobuf_test_messages.proto` contains only a From 66f6dc7bd801b00833ace4e8ac39fc4ce5d07067 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 7 Nov 2023 11:42:34 +0100 Subject: [PATCH 34/40] applied comments --- node/actors/network/src/frame.rs | 4 +- node/libs/protobuf_build/src/canonical.rs | 12 +++--- node/libs/protobuf_build/src/lib.rs | 45 ++++++++++------------- node/libs/protobuf_build/src/syntax.rs | 44 +++++++++++----------- 4 files changed, 49 insertions(+), 56 deletions(-) diff --git a/node/actors/network/src/frame.rs b/node/actors/network/src/frame.rs index 44b484fc..ae62964a 100644 --- a/node/actors/network/src/frame.rs +++ b/node/actors/network/src/frame.rs @@ -1,5 +1,5 @@ -//! Simple frame encoding format (length ++ value) for zksync_protobuf messages, -//! since zksync_protobuf messages do not have delimiters. +//! Simple frame encoding format (length ++ value) for protobuf messages, +//! since protobuf messages do not have delimiters. use crate::{mux, noise::bytes}; use concurrency::{ctx, io}; diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs index 87ba52f9..20055508 100644 --- a/node/libs/protobuf_build/src/canonical.rs +++ b/node/libs/protobuf_build/src/canonical.rs @@ -8,19 +8,19 @@ struct Check(HashSet); impl Check { /// Checks if messages of type `m` support canonical encoding. - fn message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { + fn check_message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { if self.0.contains(m.full_name()) { return Ok(()); } self.0.insert(m.full_name().to_string()); for f in m.fields() { - self.field(&f).with_context(|| f.name().to_string())?; + self.check_field(&f).with_context(|| f.name().to_string())?; } Ok(()) } /// Checks if field `f` supports canonical encoding. - fn field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { if f.is_map() { anyhow::bail!("maps unsupported"); } @@ -28,7 +28,7 @@ impl Check { anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); } if let prost_reflect::Kind::Message(msg) = &f.kind() { - self.message(msg).with_context(|| msg.name().to_string())?; + self.check_message(msg).with_context(|| msg.name().to_string())?; } Ok(()) } @@ -42,7 +42,7 @@ pub(crate) fn check( ) -> anyhow::Result<()> { for f in &descriptor.file { if f.syntax() != "proto3" { - anyhow::bail!("only proto3 syntax is supported"); + anyhow::bail!("{}: only proto3 syntax is supported", f.name()); } } let mut c = Check::default(); @@ -51,7 +51,7 @@ pub(crate) fn check( let msg = pool .get_message_by_name(&msg_name) .with_context(|| format!("{msg_name} not found in pool"))?; - c.message(&msg).with_context(|| msg_name)?; + c.check_message(&msg).with_context(|| msg_name)?; } Ok(()) } diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 6c5e62a9..937fa5d1 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -22,31 +22,28 @@ //! different crates you need to specify them as dependencies in the Config.dependencies. //! It is not possible to depend on a different proto bundle within the same crate (because //! these are being built simultaneously from the same build script). -//! -//! cargo build --all-targets -//! perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find target/debug/build/*/output -type f` | xargs cat > /tmp/sum.binpb -//! buf breaking /tmp/sum.binpb --against /tmp/sum.binpb #![allow(clippy::print_stdout)] use anyhow::Context as _; -pub use once_cell::sync::Lazy; +use prost::Message as _; +use std::{fs, path::Path, sync::Mutex}; + // Imports accessed from the generated code. +pub use once_cell::sync::Lazy; pub use prost; -use prost::Message as _; pub use prost_reflect; -use std::{fs, path::Path, sync::Mutex}; -pub use syntax::*; +pub use self::syntax::*; mod canonical; mod ident; mod syntax; -/// Traversed all the files in a directory recursively. +/// Traverses all the files in a directory recursively. fn traverse_files( path: &Path, f: &mut impl FnMut(&Path) -> anyhow::Result<()>, ) -> anyhow::Result<()> { if !path.is_dir() { - f(path).with_context(|| path.to_str().unwrap().to_string())?; + f(path).with_context(|| path.display().to_string())?; return Ok(()); } for entry in fs::read_dir(path)? { @@ -71,7 +68,7 @@ impl Descriptor { pub fn new( proto_root: ProtoName, dependencies: Vec<&'static Descriptor>, - descriptor_bytes: &impl AsRef<[u8]>, + descriptor_bytes: &[u8], ) -> Self { Descriptor { proto_root, @@ -149,10 +146,9 @@ impl Config { /// Generates rust code from the proto files according to the config. pub fn generate(&self) -> anyhow::Result<()> { - assert!( - self.input_root.abs().is_dir(), - "input_root should be a directory" - ); + if !self.input_root.abs()?.is_dir() { + anyhow::bail!("input_root should be a directory"); + } println!("cargo:rerun-if-changed={}", self.input_root.to_str()); // Load dependencies. @@ -166,7 +162,7 @@ impl Config { // Load proto files. let mut proto_paths = vec![]; - traverse_files(&self.input_root.abs(), &mut |path| { + traverse_files(&self.input_root.abs()?, &mut |path| { let Some(ext) = path.extension() else { return Ok(()); }; @@ -178,12 +174,12 @@ impl Config { }; let file_raw = fs::read_to_string(path).context("fs::read()")?; - let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root); + let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root).context("ProtoPath::from_input_path()")?; pool_raw .file - .push(protox_parse::parse(path.to_str(), &file_raw).map_err( + .push(protox_parse::parse(&path.to_string(), &file_raw).map_err( // rewrapping the error, so that source location is included in the error message. - |err| anyhow::anyhow!("{:?}", err), + |err| anyhow::anyhow!("{err:?}"), )?); proto_paths.push(path); Ok(()) @@ -197,19 +193,18 @@ impl Config { compiler .open_files(proto_paths.iter().map(|p| p.to_path())) // rewrapping the error, so that source location is included in the error message. - .map_err(|err| anyhow::anyhow!("{:?}", err))?; + .map_err(|err| anyhow::anyhow!("{err:?}"))?; let descriptor = compiler.file_descriptor_set(); pool.add_file_descriptor_set(descriptor.clone()).unwrap(); // Check that the compiled proto files belong to the declared proto package. let package_root = self.proto_root.to_name(); for f in &descriptor.file { - if !package_root.contains(&f.package().into()) { + if !ProtoName::from(f.package()).starts_with(&package_root) { anyhow::bail!( - "{:?} ({:?}) does not belong to package {:?}", + "{:?} ({:?}) does not belong to package {package_root}", f.package(), f.name(), - package_root.to_string() ); } } @@ -267,9 +262,9 @@ impl Config { let this = self.this_crate().to_string(); output.append(&format!("\ pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ - {this}::Descriptor::new({:?}.into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?}))\ + {this}::Descriptor::new(\"{package_root}\".into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?})[..])\ }});\ - ",package_root.to_string())); + ")); // Save output. fs::write(output_path, output.format().context("output.format()")?)?; diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index dfafd214..594a4ea5 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -25,11 +25,10 @@ impl InputPath { /// Converts the relative input path to an absolute path in the local file system /// (under $CARGO_MANIFEST_DIR). - pub(super) fn abs(&self) -> PathBuf { - PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) - .canonicalize() - .unwrap() - .join(&self.0) + pub(super) fn abs(&self) -> anyhow::Result { + Ok(PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?) + .canonicalize()? + .join(&self.0)) } /// Output directory path derived from the input path by replacing $CARGO_MANIFEST_DIR with $OUT_DIR. @@ -57,7 +56,7 @@ impl From<&str> for ProtoPath { impl ProtoPath { /// Converts a proto module path to proto package name by replacing all "/" with ".". pub(super) fn to_name(&self) -> ProtoName { - ProtoName(self.0.iter().map(|p| p.to_str().unwrap().into()).collect()) + ProtoName(self.0.iter().map(|p| p.to_string_lossy().to_string()).collect()) } /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . @@ -65,17 +64,12 @@ impl ProtoPath { path: &Path, input_root: &InputPath, proto_root: &ProtoPath, - ) -> ProtoPath { - ProtoPath( + ) -> anyhow::Result { + Ok(ProtoPath( proto_root .0 - .join(path.strip_prefix(&input_root.abs()).unwrap()), - ) - } - - /// Converts ProtoPath to str. - pub(super) fn to_str(&self) -> &str { - self.0.to_str().unwrap() + .join(path.strip_prefix(&input_root.abs()?).unwrap()), + )) } /// Converts ProtoPath to Path. @@ -84,6 +78,14 @@ impl ProtoPath { } } +impl fmt::Display for ProtoPath { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + self.0.display().fmt(fmt) + } +} + + + type Part = String; /// A rust module/type name that the generated code is available at. Although @@ -175,18 +177,14 @@ pub struct ProtoName(Vec); impl ProtoName { /// Checks if package path starts with the given prefix. - pub fn contains(&self, elem: &Self) -> bool { - elem.0.len() >= self.0.len() && elem.0[0..self.0.len()] == self.0 + pub fn starts_with(&self, prefix: &Self) -> bool { + self.0.len() >= prefix.0.len() && self.0[0..prefix.0.len()] == prefix.0 } /// Strips a given prefix from the name. pub fn relative_to(&self, prefix: &Self) -> anyhow::Result { - if !prefix.contains(self) { - anyhow::bail!( - "{} does not contain {}", - self.to_string(), - prefix.to_string() - ); + if !self.starts_with(prefix) { + anyhow::bail!("{self} does not contain {prefix}"); } Ok(Self(self.0[prefix.0.len()..].to_vec())) } From 112f4d1813ff37dab6b069e46f4237e9cbc9ab20 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 7 Nov 2023 12:28:21 +0100 Subject: [PATCH 35/40] removed proto_root --- node/libs/protobuf/build.rs | 3 -- .../protobuf/src/bin/conformance_test/main.rs | 19 +++++++------ .../conformance_test}/conformance.proto | 0 .../test_messages_proto3.proto | 0 .../protobuf/src/proto/{ => zksync}/std.proto | 0 node/libs/protobuf/src/std_conv.rs | 11 ++++---- node/libs/protobuf/src/tests/mod.rs | 9 +++--- .../proto/{ => zksync/protobuf}/tests.proto | 0 node/libs/protobuf_build/src/lib.rs | 28 ++++++------------- node/libs/protobuf_build/src/syntax.rs | 19 +++++++------ node/libs/schema/build.rs | 1 - .../{ => zksync/schema}/executor/config.proto | 0 .../schema}/network/consensus.proto | 0 .../{ => zksync/schema}/network/gossip.proto | 0 .../{ => zksync/schema}/network/mux.proto | 0 .../schema}/network/mux_test.proto | 0 .../{ => zksync/schema}/network/ping.proto | 0 .../{ => zksync/schema}/network/preface.proto | 0 .../{ => zksync/schema}/roles/node.proto | 0 .../{ => zksync/schema}/roles/validator.proto | 0 .../proto/{ => zksync/schema}/storage.proto | 0 21 files changed, 40 insertions(+), 50 deletions(-) rename node/libs/protobuf/src/bin/conformance_test/proto/{ => zksync/protobuf/conformance_test}/conformance.proto (100%) rename node/libs/protobuf/src/bin/conformance_test/proto/{ => zksync/protobuf/conformance_test}/test_messages_proto3.proto (100%) rename node/libs/protobuf/src/proto/{ => zksync}/std.proto (100%) rename node/libs/protobuf/src/tests/proto/{ => zksync/protobuf}/tests.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/executor/config.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/network/consensus.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/network/gossip.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/network/mux.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/network/mux_test.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/network/ping.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/network/preface.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/roles/node.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/roles/validator.proto (100%) rename node/libs/schema/proto/{ => zksync/schema}/storage.proto (100%) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 0348639b..2b4eb17b 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -2,7 +2,6 @@ fn main() { zksync_protobuf_build::Config { input_root: "src/proto".into(), - proto_root: "zksync".into(), dependencies: vec![], protobuf_crate: "crate".into(), } @@ -11,7 +10,6 @@ fn main() { zksync_protobuf_build::Config { input_root: "src/tests/proto".into(), - proto_root: "zksync/protobuf/tests".into(), dependencies: vec![], protobuf_crate: "crate".into(), } @@ -20,7 +18,6 @@ fn main() { zksync_protobuf_build::Config { input_root: "src/bin/conformance_test/proto".into(), - proto_root: "zksync/protobuf/conformance_test".into(), dependencies: vec![], protobuf_crate: "::zksync_protobuf".into(), } diff --git a/node/libs/protobuf/src/bin/conformance_test/main.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs index 584c85da..70bd1a6e 100644 --- a/node/libs/protobuf/src/bin/conformance_test/main.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -12,6 +12,7 @@ use prost_reflect::ReflectMessage; use std::sync::Mutex; mod proto; +use proto::zksync::protobuf::conformance_test as proto_; /// Runs the test server. async fn run() -> anyhow::Result<()> { @@ -28,8 +29,8 @@ async fn run() -> anyhow::Result<()> { let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stdin, &mut msg[..]).await??; - use proto::conformance_response::Result as R; - let req = proto::ConformanceRequest::decode(&msg[..])?; + use proto_::conformance_response::Result as R; + let req = proto_::ConformanceRequest::decode(&msg[..])?; let res = async { let t = req.message_type.context("missing message_type")?; if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { @@ -38,15 +39,15 @@ async fn run() -> anyhow::Result<()> { // Decode. let payload = req.payload.context("missing payload")?; - use proto::TestAllTypesProto3 as T; + use proto_::TestAllTypesProto3 as T; let p = match payload { - proto::conformance_request::Payload::JsonPayload(payload) => { + proto_::conformance_request::Payload::JsonPayload(payload) => { match zksync_protobuf::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } } - proto::conformance_request::Payload::ProtobufPayload(payload) => { + proto_::conformance_request::Payload::ProtobufPayload(payload) => { // First filter out incorrect encodings. let Ok(p) = T::decode(&payload[..]) else { return Ok(R::ParseError("parsing failed".to_string())); @@ -64,11 +65,11 @@ async fn run() -> anyhow::Result<()> { let format = req .requested_output_format .context("missing output format")?; - match proto::WireFormat::try_from(format).context("unknown format")? { - proto::WireFormat::Json => { + match proto_::WireFormat::try_from(format).context("unknown format")? { + proto_::WireFormat::Json => { anyhow::Ok(R::JsonPayload(zksync_protobuf::encode_json_proto(&p))) } - proto::WireFormat::Protobuf => { + proto_::WireFormat::Protobuf => { // Reencode the parsed proto. anyhow::Ok(R::ProtobufPayload(zksync_protobuf::canonical_raw( &p.encode_to_vec(), @@ -79,7 +80,7 @@ async fn run() -> anyhow::Result<()> { } } .await?; - let resp = proto::ConformanceResponse { result: Some(res) }; + let resp = proto_::ConformanceResponse { result: Some(res) }; // Write the response. let msg = resp.encode_to_vec(); diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto b/node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/conformance.proto similarity index 100% rename from node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/conformance.proto diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto b/node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/test_messages_proto3.proto similarity index 100% rename from node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/test_messages_proto3.proto diff --git a/node/libs/protobuf/src/proto/std.proto b/node/libs/protobuf/src/proto/zksync/std.proto similarity index 100% rename from node/libs/protobuf/src/proto/std.proto rename to node/libs/protobuf/src/proto/zksync/std.proto diff --git a/node/libs/protobuf/src/std_conv.rs b/node/libs/protobuf/src/std_conv.rs index b961fbf3..d186c209 100644 --- a/node/libs/protobuf/src/std_conv.rs +++ b/node/libs/protobuf/src/std_conv.rs @@ -3,9 +3,10 @@ use crate::{proto, required, ProtoFmt}; use anyhow::Context as _; use concurrency::time; use std::net; +use proto::zksync as proto_; impl ProtoFmt for () { - type Proto = proto::std::Void; + type Proto = proto_::std::Void; fn read(_r: &Self::Proto) -> anyhow::Result { Ok(()) } @@ -15,7 +16,7 @@ impl ProtoFmt for () { } impl ProtoFmt for std::net::SocketAddr { - type Proto = proto::std::SocketAddr; + type Proto = proto_::std::SocketAddr; fn read(r: &Self::Proto) -> anyhow::Result { let ip = required(&r.ip).context("ip")?; @@ -41,7 +42,7 @@ impl ProtoFmt for std::net::SocketAddr { } impl ProtoFmt for time::Utc { - type Proto = proto::std::Timestamp; + type Proto = proto_::std::Timestamp; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -59,7 +60,7 @@ impl ProtoFmt for time::Utc { } impl ProtoFmt for time::Duration { - type Proto = proto::std::Duration; + type Proto = proto_::std::Duration; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -82,7 +83,7 @@ impl ProtoFmt for time::Duration { } impl ProtoFmt for bit_vec::BitVec { - type Proto = proto::std::BitVector; + type Proto = proto_::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { let size = *required(&r.size).context("size")? as usize; diff --git a/node/libs/protobuf/src/tests/mod.rs b/node/libs/protobuf/src/tests/mod.rs index 143c47cf..24bd96b7 100644 --- a/node/libs/protobuf/src/tests/mod.rs +++ b/node/libs/protobuf/src/tests/mod.rs @@ -4,6 +4,7 @@ use anyhow::Context as _; use concurrency::{ctx, time}; mod proto; +use proto::zksync::protobuf::tests as proto_; #[derive(Debug, PartialEq, Eq)] enum B { @@ -12,16 +13,16 @@ enum B { } impl ProtoFmt for B { - type Proto = proto::B; + type Proto = proto_::B; fn read(r: &Self::Proto) -> anyhow::Result { - use proto::b::T; + use proto_::b::T; Ok(match required(&r.t)? { T::U(x) => Self::U(*x), T::V(x) => Self::V(Box::new(ProtoFmt::read(x.as_ref())?)), }) } fn build(&self) -> Self::Proto { - use proto::b::T; + use proto_::b::T; let t = match self { Self::U(x) => T::U(*x), Self::V(x) => T::V(Box::new(x.build())), @@ -39,7 +40,7 @@ struct A { } impl ProtoFmt for A { - type Proto = proto::A; + type Proto = proto_::A; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { x: required(&r.x).context("x")?.clone(), diff --git a/node/libs/protobuf/src/tests/proto/tests.proto b/node/libs/protobuf/src/tests/proto/zksync/protobuf/tests.proto similarity index 100% rename from node/libs/protobuf/src/tests/proto/tests.proto rename to node/libs/protobuf/src/tests/proto/zksync/protobuf/tests.proto diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 937fa5d1..fb8d9f61 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -54,9 +54,6 @@ fn traverse_files( /// Protobuf descriptor + info about the mapping to rust code. pub struct Descriptor { - /// Root proto package that all proto files in this descriptor belong to. - /// Rust types have been generated relative to this root. - proto_root: ProtoName, /// Raw descriptor proto. descriptor_proto: prost_types::FileDescriptorSet, /// Direct dependencies of this descriptor. @@ -66,12 +63,10 @@ pub struct Descriptor { impl Descriptor { /// Constructs a Descriptor. pub fn new( - proto_root: ProtoName, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &[u8], ) -> Self { Descriptor { - proto_root, dependencies, descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()) .unwrap(), @@ -109,8 +104,6 @@ impl Descriptor { pub struct Config { /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. pub input_root: InputPath, - /// Implicit prefix that should be prepended to proto paths of the proto files in the input directory. - pub proto_root: ProtoPath, /// Descriptors of the dependencies and the rust absolute paths under which they will be available from the generated code. pub dependencies: Vec<(RustName, &'static Descriptor)>, /// Rust absolute path under which the protobuf crate will be available from the generated @@ -128,8 +121,6 @@ impl Config { /// from a message of the given `proto_name`. fn reflect_impl(&self, proto_name: ProtoName) -> String { let rust_name = proto_name - .relative_to(&self.proto_root.to_name()) - .unwrap() .to_rust_type() .to_string(); let proto_name = proto_name.to_string(); @@ -174,7 +165,7 @@ impl Config { }; let file_raw = fs::read_to_string(path).context("fs::read()")?; - let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root).context("ProtoPath::from_input_path()")?; + let path = ProtoPath::from_input_path(path, &self.input_root).context("ProtoPath::from_input_path()")?; pool_raw .file .push(protox_parse::parse(&path.to_string(), &file_raw).map_err( @@ -198,12 +189,13 @@ impl Config { pool.add_file_descriptor_set(descriptor.clone()).unwrap(); // Check that the compiled proto files belong to the declared proto package. - let package_root = self.proto_root.to_name(); for f in &descriptor.file { - if !ProtoName::from(f.package()).starts_with(&package_root) { + let got = ProtoName::from(f.package()); + // Unwrap is ok, because descriptor file here has never an empty name. + let want_prefix = ProtoPath::from(f.name()).parent().unwrap().to_name()?; + if !got.starts_with(&want_prefix) { anyhow::bail!( - "{:?} ({:?}) does not belong to package {package_root}", - f.package(), + "{got} ({:?}) does not belong to package {want_prefix}", f.name(), ); } @@ -229,10 +221,7 @@ impl Config { config.skip_protoc_run(); for d in &self.dependencies { for f in &d.1.descriptor_proto.file { - let proto_rel = ProtoName::from(f.package()) - .relative_to(&d.1.proto_root) - .unwrap(); - let rust_abs = d.0.clone().join(proto_rel.to_rust_module()); + let rust_abs = d.0.clone().join(ProtoName::from(f.package()).to_rust_module()); config.extern_path(format!(".{}", f.package()), rust_abs.to_string()); } } @@ -247,7 +236,6 @@ impl Config { } // Generate the reflection code. - let output = output.sub(&self.proto_root.to_name().to_rust_module()); for proto_name in extract_message_names(&descriptor) { output.append(&self.reflect_impl(proto_name)); } @@ -262,7 +250,7 @@ impl Config { let this = self.this_crate().to_string(); output.append(&format!("\ pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ - {this}::Descriptor::new(\"{package_root}\".into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?})[..])\ + {this}::Descriptor::new(vec![{rust_deps}], &include_bytes!({descriptor_path:?})[..])\ }});\ ")); diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 594a4ea5..0be5ce22 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -55,21 +55,24 @@ impl From<&str> for ProtoPath { impl ProtoPath { /// Converts a proto module path to proto package name by replacing all "/" with ".". - pub(super) fn to_name(&self) -> ProtoName { - ProtoName(self.0.iter().map(|p| p.to_string_lossy().to_string()).collect()) + pub(super) fn to_name(&self) -> anyhow::Result { + let mut parts = vec![]; + for p in self.0.iter() { + parts.push(p.to_str().context("invalid proto path")?.to_string()); + } + Ok(ProtoName(parts)) + } + + pub(super) fn parent(&self) -> Option { + self.0.parent().map(|p|Self(p.into())) } /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . pub(super) fn from_input_path( path: &Path, input_root: &InputPath, - proto_root: &ProtoPath, ) -> anyhow::Result { - Ok(ProtoPath( - proto_root - .0 - .join(path.strip_prefix(&input_root.abs()?).unwrap()), - )) + Ok(ProtoPath(path.strip_prefix(&input_root.abs()?)?.to_path_buf())) } /// Converts ProtoPath to Path. diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 40c8e5c4..997e803b 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -2,7 +2,6 @@ fn main() { zksync_protobuf::build::Config { input_root: "proto".into(), - proto_root: "zksync/schema".into(), dependencies: vec![( "::zksync_protobuf::proto".into(), &zksync_protobuf::proto::DESCRIPTOR, diff --git a/node/libs/schema/proto/executor/config.proto b/node/libs/schema/proto/zksync/schema/executor/config.proto similarity index 100% rename from node/libs/schema/proto/executor/config.proto rename to node/libs/schema/proto/zksync/schema/executor/config.proto diff --git a/node/libs/schema/proto/network/consensus.proto b/node/libs/schema/proto/zksync/schema/network/consensus.proto similarity index 100% rename from node/libs/schema/proto/network/consensus.proto rename to node/libs/schema/proto/zksync/schema/network/consensus.proto diff --git a/node/libs/schema/proto/network/gossip.proto b/node/libs/schema/proto/zksync/schema/network/gossip.proto similarity index 100% rename from node/libs/schema/proto/network/gossip.proto rename to node/libs/schema/proto/zksync/schema/network/gossip.proto diff --git a/node/libs/schema/proto/network/mux.proto b/node/libs/schema/proto/zksync/schema/network/mux.proto similarity index 100% rename from node/libs/schema/proto/network/mux.proto rename to node/libs/schema/proto/zksync/schema/network/mux.proto diff --git a/node/libs/schema/proto/network/mux_test.proto b/node/libs/schema/proto/zksync/schema/network/mux_test.proto similarity index 100% rename from node/libs/schema/proto/network/mux_test.proto rename to node/libs/schema/proto/zksync/schema/network/mux_test.proto diff --git a/node/libs/schema/proto/network/ping.proto b/node/libs/schema/proto/zksync/schema/network/ping.proto similarity index 100% rename from node/libs/schema/proto/network/ping.proto rename to node/libs/schema/proto/zksync/schema/network/ping.proto diff --git a/node/libs/schema/proto/network/preface.proto b/node/libs/schema/proto/zksync/schema/network/preface.proto similarity index 100% rename from node/libs/schema/proto/network/preface.proto rename to node/libs/schema/proto/zksync/schema/network/preface.proto diff --git a/node/libs/schema/proto/roles/node.proto b/node/libs/schema/proto/zksync/schema/roles/node.proto similarity index 100% rename from node/libs/schema/proto/roles/node.proto rename to node/libs/schema/proto/zksync/schema/roles/node.proto diff --git a/node/libs/schema/proto/roles/validator.proto b/node/libs/schema/proto/zksync/schema/roles/validator.proto similarity index 100% rename from node/libs/schema/proto/roles/validator.proto rename to node/libs/schema/proto/zksync/schema/roles/validator.proto diff --git a/node/libs/schema/proto/storage.proto b/node/libs/schema/proto/zksync/schema/storage.proto similarity index 100% rename from node/libs/schema/proto/storage.proto rename to node/libs/schema/proto/zksync/schema/storage.proto From a447ba15488edf9fce9547192725248449385580 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 7 Nov 2023 13:41:33 +0100 Subject: [PATCH 36/40] reverted --- node/libs/protobuf/build.rs | 3 ++ .../protobuf/src/bin/conformance_test/main.rs | 19 ++++++------ .../conformance_test => }/conformance.proto | 0 .../test_messages_proto3.proto | 0 .../protobuf/src/proto/{zksync => }/std.proto | 0 node/libs/protobuf/src/std_conv.rs | 11 ++++--- node/libs/protobuf/src/tests/mod.rs | 9 +++--- .../proto/{zksync/protobuf => }/tests.proto | 0 node/libs/protobuf_build/src/lib.rs | 29 ++++++++++++++----- node/libs/protobuf_build/src/syntax.rs | 7 ++++- node/libs/schema/build.rs | 1 + .../{zksync/schema => }/executor/config.proto | 0 .../schema => }/network/consensus.proto | 0 .../{zksync/schema => }/network/gossip.proto | 0 .../{zksync/schema => }/network/mux.proto | 0 .../schema => }/network/mux_test.proto | 0 .../{zksync/schema => }/network/ping.proto | 0 .../{zksync/schema => }/network/preface.proto | 0 .../{zksync/schema => }/roles/node.proto | 0 .../{zksync/schema => }/roles/validator.proto | 0 .../proto/{zksync/schema => }/storage.proto | 0 21 files changed, 50 insertions(+), 29 deletions(-) rename node/libs/protobuf/src/bin/conformance_test/proto/{zksync/protobuf/conformance_test => }/conformance.proto (100%) rename node/libs/protobuf/src/bin/conformance_test/proto/{zksync/protobuf/conformance_test => }/test_messages_proto3.proto (100%) rename node/libs/protobuf/src/proto/{zksync => }/std.proto (100%) rename node/libs/protobuf/src/tests/proto/{zksync/protobuf => }/tests.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/executor/config.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/network/consensus.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/network/gossip.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/network/mux.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/network/mux_test.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/network/ping.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/network/preface.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/roles/node.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/roles/validator.proto (100%) rename node/libs/schema/proto/{zksync/schema => }/storage.proto (100%) diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 2b4eb17b..0348639b 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -2,6 +2,7 @@ fn main() { zksync_protobuf_build::Config { input_root: "src/proto".into(), + proto_root: "zksync".into(), dependencies: vec![], protobuf_crate: "crate".into(), } @@ -10,6 +11,7 @@ fn main() { zksync_protobuf_build::Config { input_root: "src/tests/proto".into(), + proto_root: "zksync/protobuf/tests".into(), dependencies: vec![], protobuf_crate: "crate".into(), } @@ -18,6 +20,7 @@ fn main() { zksync_protobuf_build::Config { input_root: "src/bin/conformance_test/proto".into(), + proto_root: "zksync/protobuf/conformance_test".into(), dependencies: vec![], protobuf_crate: "::zksync_protobuf".into(), } diff --git a/node/libs/protobuf/src/bin/conformance_test/main.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs index 76d00f79..e2e02d9b 100644 --- a/node/libs/protobuf/src/bin/conformance_test/main.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -12,7 +12,6 @@ use prost_reflect::ReflectMessage; use std::sync::Mutex; mod proto; -use proto::zksync::protobuf::conformance_test as proto_; /// Runs the test server. async fn run() -> anyhow::Result<()> { @@ -29,8 +28,8 @@ async fn run() -> anyhow::Result<()> { let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stdin, &mut msg[..]).await??; - use proto_::conformance_response::Result as R; - let req = proto_::ConformanceRequest::decode(&msg[..])?; + use proto::conformance_response::Result as R; + let req = proto::ConformanceRequest::decode(&msg[..])?; let res = async { let t = req.message_type.context("missing message_type")?; if t != *"protobuf_test_messages.proto3.TestAllTypesProto3" { @@ -39,15 +38,15 @@ async fn run() -> anyhow::Result<()> { // Decode. let payload = req.payload.context("missing payload")?; - use proto_::TestAllTypesProto3 as T; + use proto::TestAllTypesProto3 as T; let p = match payload { - proto_::conformance_request::Payload::JsonPayload(payload) => { + proto::conformance_request::Payload::JsonPayload(payload) => { match zksync_protobuf::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } } - proto_::conformance_request::Payload::ProtobufPayload(payload) => { + proto::conformance_request::Payload::ProtobufPayload(payload) => { // First filter out incorrect encodings. let Ok(p) = T::decode(&payload[..]) else { return Ok(R::ParseError("parsing failed".to_string())); @@ -65,11 +64,11 @@ async fn run() -> anyhow::Result<()> { let format = req .requested_output_format .context("missing output format")?; - match proto_::WireFormat::try_from(format).context("unknown format")? { - proto_::WireFormat::Json => { + match proto::WireFormat::try_from(format).context("unknown format")? { + proto::WireFormat::Json => { anyhow::Ok(R::JsonPayload(zksync_protobuf::encode_json_proto(&p))) } - proto_::WireFormat::Protobuf => { + proto::WireFormat::Protobuf => { // Reencode the parsed proto. anyhow::Ok(R::ProtobufPayload(zksync_protobuf::canonical_raw( &p.encode_to_vec(), @@ -80,7 +79,7 @@ async fn run() -> anyhow::Result<()> { } } .await?; - let resp = proto_::ConformanceResponse { result: Some(res) }; + let resp = proto::ConformanceResponse { result: Some(res) }; // Write the response. let msg = resp.encode_to_vec(); diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/conformance.proto b/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto similarity index 100% rename from node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/conformance.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/test_messages_proto3.proto b/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto similarity index 100% rename from node/libs/protobuf/src/bin/conformance_test/proto/zksync/protobuf/conformance_test/test_messages_proto3.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto diff --git a/node/libs/protobuf/src/proto/zksync/std.proto b/node/libs/protobuf/src/proto/std.proto similarity index 100% rename from node/libs/protobuf/src/proto/zksync/std.proto rename to node/libs/protobuf/src/proto/std.proto diff --git a/node/libs/protobuf/src/std_conv.rs b/node/libs/protobuf/src/std_conv.rs index d186c209..b961fbf3 100644 --- a/node/libs/protobuf/src/std_conv.rs +++ b/node/libs/protobuf/src/std_conv.rs @@ -3,10 +3,9 @@ use crate::{proto, required, ProtoFmt}; use anyhow::Context as _; use concurrency::time; use std::net; -use proto::zksync as proto_; impl ProtoFmt for () { - type Proto = proto_::std::Void; + type Proto = proto::std::Void; fn read(_r: &Self::Proto) -> anyhow::Result { Ok(()) } @@ -16,7 +15,7 @@ impl ProtoFmt for () { } impl ProtoFmt for std::net::SocketAddr { - type Proto = proto_::std::SocketAddr; + type Proto = proto::std::SocketAddr; fn read(r: &Self::Proto) -> anyhow::Result { let ip = required(&r.ip).context("ip")?; @@ -42,7 +41,7 @@ impl ProtoFmt for std::net::SocketAddr { } impl ProtoFmt for time::Utc { - type Proto = proto_::std::Timestamp; + type Proto = proto::std::Timestamp; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -60,7 +59,7 @@ impl ProtoFmt for time::Utc { } impl ProtoFmt for time::Duration { - type Proto = proto_::std::Duration; + type Proto = proto::std::Duration; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -83,7 +82,7 @@ impl ProtoFmt for time::Duration { } impl ProtoFmt for bit_vec::BitVec { - type Proto = proto_::std::BitVector; + type Proto = proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { let size = *required(&r.size).context("size")? as usize; diff --git a/node/libs/protobuf/src/tests/mod.rs b/node/libs/protobuf/src/tests/mod.rs index 24bd96b7..143c47cf 100644 --- a/node/libs/protobuf/src/tests/mod.rs +++ b/node/libs/protobuf/src/tests/mod.rs @@ -4,7 +4,6 @@ use anyhow::Context as _; use concurrency::{ctx, time}; mod proto; -use proto::zksync::protobuf::tests as proto_; #[derive(Debug, PartialEq, Eq)] enum B { @@ -13,16 +12,16 @@ enum B { } impl ProtoFmt for B { - type Proto = proto_::B; + type Proto = proto::B; fn read(r: &Self::Proto) -> anyhow::Result { - use proto_::b::T; + use proto::b::T; Ok(match required(&r.t)? { T::U(x) => Self::U(*x), T::V(x) => Self::V(Box::new(ProtoFmt::read(x.as_ref())?)), }) } fn build(&self) -> Self::Proto { - use proto_::b::T; + use proto::b::T; let t = match self { Self::U(x) => T::U(*x), Self::V(x) => T::V(Box::new(x.build())), @@ -40,7 +39,7 @@ struct A { } impl ProtoFmt for A { - type Proto = proto_::A; + type Proto = proto::A; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { x: required(&r.x).context("x")?.clone(), diff --git a/node/libs/protobuf/src/tests/proto/zksync/protobuf/tests.proto b/node/libs/protobuf/src/tests/proto/tests.proto similarity index 100% rename from node/libs/protobuf/src/tests/proto/zksync/protobuf/tests.proto rename to node/libs/protobuf/src/tests/proto/tests.proto diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index fb8d9f61..4ec196d4 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -54,6 +54,9 @@ fn traverse_files( /// Protobuf descriptor + info about the mapping to rust code. pub struct Descriptor { + /// Root proto package that all proto files in this descriptor belong to. + /// Rust types have been generated relative to this root. + proto_root: ProtoName, /// Raw descriptor proto. descriptor_proto: prost_types::FileDescriptorSet, /// Direct dependencies of this descriptor. @@ -63,10 +66,12 @@ pub struct Descriptor { impl Descriptor { /// Constructs a Descriptor. pub fn new( + proto_root: ProtoName, dependencies: Vec<&'static Descriptor>, descriptor_bytes: &[u8], ) -> Self { Descriptor { + proto_root, dependencies, descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()) .unwrap(), @@ -104,6 +109,8 @@ impl Descriptor { pub struct Config { /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. pub input_root: InputPath, + /// Implicit prefix that should be prepended to proto paths of the proto files in the input directory. + pub proto_root: ProtoPath, /// Descriptors of the dependencies and the rust absolute paths under which they will be available from the generated code. pub dependencies: Vec<(RustName, &'static Descriptor)>, /// Rust absolute path under which the protobuf crate will be available from the generated @@ -119,20 +126,22 @@ impl Config { /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated /// from a message of the given `proto_name`. - fn reflect_impl(&self, proto_name: ProtoName) -> String { + fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { let rust_name = proto_name + .relative_to(&self.proto_root.to_name().context("invalid proto_root")?) + .unwrap() .to_rust_type() .to_string(); let proto_name = proto_name.to_string(); let this = self.this_crate().to_string(); - format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ + Ok(format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ fn descriptor(&self) -> {this}::prost_reflect::MessageDescriptor {{\ static INIT : {this}::Lazy<{this}::prost_reflect::MessageDescriptor> = {this}::Lazy::new(|| {{\ DESCRIPTOR.load_global().get_message_by_name({proto_name:?}).unwrap()\ }});\ INIT.clone()\ }}\ - }}") + }}")) } /// Generates rust code from the proto files according to the config. @@ -165,7 +174,7 @@ impl Config { }; let file_raw = fs::read_to_string(path).context("fs::read()")?; - let path = ProtoPath::from_input_path(path, &self.input_root).context("ProtoPath::from_input_path()")?; + let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root).context("ProtoPath::from_input_path()")?; pool_raw .file .push(protox_parse::parse(&path.to_string(), &file_raw).map_err( @@ -221,7 +230,10 @@ impl Config { config.skip_protoc_run(); for d in &self.dependencies { for f in &d.1.descriptor_proto.file { - let rust_abs = d.0.clone().join(ProtoName::from(f.package()).to_rust_module()); + let proto_rel = ProtoName::from(f.package()) + .relative_to(&d.1.proto_root) + .unwrap(); + let rust_abs = d.0.clone().join(proto_rel.to_rust_module()); config.extern_path(format!(".{}", f.package()), rust_abs.to_string()); } } @@ -236,8 +248,11 @@ impl Config { } // Generate the reflection code. + let package_root = self.proto_root.to_name().context("invalid proto_root")?; + let output = output.sub(&package_root.to_rust_module()); for proto_name in extract_message_names(&descriptor) { - output.append(&self.reflect_impl(proto_name)); + output.append(&self.reflect_impl(&proto_name) + .with_context(||format!("reflect_impl({proto_name})"))?); } // Generate the descriptor. @@ -250,7 +265,7 @@ impl Config { let this = self.this_crate().to_string(); output.append(&format!("\ pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ - {this}::Descriptor::new(vec![{rust_deps}], &include_bytes!({descriptor_path:?})[..])\ + {this}::Descriptor::new(\"{package_root}\".into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?})[..])\ }});\ ")); diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 0be5ce22..36c673e0 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -71,8 +71,13 @@ impl ProtoPath { pub(super) fn from_input_path( path: &Path, input_root: &InputPath, + proto_root: &ProtoPath, ) -> anyhow::Result { - Ok(ProtoPath(path.strip_prefix(&input_root.abs()?)?.to_path_buf())) + Ok(ProtoPath( + proto_root + .0 + .join(path.strip_prefix(&input_root.abs()?).unwrap()), + )) } /// Converts ProtoPath to Path. diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 997e803b..40c8e5c4 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -2,6 +2,7 @@ fn main() { zksync_protobuf::build::Config { input_root: "proto".into(), + proto_root: "zksync/schema".into(), dependencies: vec![( "::zksync_protobuf::proto".into(), &zksync_protobuf::proto::DESCRIPTOR, diff --git a/node/libs/schema/proto/zksync/schema/executor/config.proto b/node/libs/schema/proto/executor/config.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/executor/config.proto rename to node/libs/schema/proto/executor/config.proto diff --git a/node/libs/schema/proto/zksync/schema/network/consensus.proto b/node/libs/schema/proto/network/consensus.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/network/consensus.proto rename to node/libs/schema/proto/network/consensus.proto diff --git a/node/libs/schema/proto/zksync/schema/network/gossip.proto b/node/libs/schema/proto/network/gossip.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/network/gossip.proto rename to node/libs/schema/proto/network/gossip.proto diff --git a/node/libs/schema/proto/zksync/schema/network/mux.proto b/node/libs/schema/proto/network/mux.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/network/mux.proto rename to node/libs/schema/proto/network/mux.proto diff --git a/node/libs/schema/proto/zksync/schema/network/mux_test.proto b/node/libs/schema/proto/network/mux_test.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/network/mux_test.proto rename to node/libs/schema/proto/network/mux_test.proto diff --git a/node/libs/schema/proto/zksync/schema/network/ping.proto b/node/libs/schema/proto/network/ping.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/network/ping.proto rename to node/libs/schema/proto/network/ping.proto diff --git a/node/libs/schema/proto/zksync/schema/network/preface.proto b/node/libs/schema/proto/network/preface.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/network/preface.proto rename to node/libs/schema/proto/network/preface.proto diff --git a/node/libs/schema/proto/zksync/schema/roles/node.proto b/node/libs/schema/proto/roles/node.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/roles/node.proto rename to node/libs/schema/proto/roles/node.proto diff --git a/node/libs/schema/proto/zksync/schema/roles/validator.proto b/node/libs/schema/proto/roles/validator.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/roles/validator.proto rename to node/libs/schema/proto/roles/validator.proto diff --git a/node/libs/schema/proto/zksync/schema/storage.proto b/node/libs/schema/proto/storage.proto similarity index 100% rename from node/libs/schema/proto/zksync/schema/storage.proto rename to node/libs/schema/proto/storage.proto From 0e6f3d7e781d8c166c191e2d593bf71c83019e63 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 7 Nov 2023 14:38:34 +0100 Subject: [PATCH 37/40] declare_descriptor macro --- node/libs/protobuf_build/src/canonical.rs | 3 +- node/libs/protobuf_build/src/lib.rs | 42 +++++++++++++++-------- node/libs/protobuf_build/src/syntax.rs | 4 +-- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs index 20055508..e283663e 100644 --- a/node/libs/protobuf_build/src/canonical.rs +++ b/node/libs/protobuf_build/src/canonical.rs @@ -28,7 +28,8 @@ impl Check { anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); } if let prost_reflect::Kind::Message(msg) = &f.kind() { - self.check_message(msg).with_context(|| msg.name().to_string())?; + self.check_message(msg) + .with_context(|| msg.name().to_string())?; } Ok(()) } diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 4ec196d4..80956f8a 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -23,15 +23,14 @@ //! It is not possible to depend on a different proto bundle within the same crate (because //! these are being built simultaneously from the same build script). #![allow(clippy::print_stdout)] +pub use self::syntax::*; use anyhow::Context as _; -use prost::Message as _; -use std::{fs, path::Path, sync::Mutex}; - // Imports accessed from the generated code. pub use once_cell::sync::Lazy; pub use prost; +use prost::Message as _; pub use prost_reflect; -pub use self::syntax::*; +use std::{fs, path::Path, sync::Mutex}; mod canonical; mod ident; @@ -73,7 +72,7 @@ impl Descriptor { Descriptor { proto_root, dependencies, - descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes.as_ref()) + descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes) .unwrap(), } } @@ -105,6 +104,19 @@ impl Descriptor { } } +#[macro_export] +macro_rules! declare_descriptor { + ($package_root:expr, $descriptor_path:expr, $($rust_deps:path),*) => { + pub static DESCRIPTOR : $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| { + $crate::Descriptor::new( + $package_root.into(), + vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*], + &include_bytes!($descriptor_path)[..], + ) + }); + } +} + /// Code generation config. Use it in build scripts. pub struct Config { /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. @@ -174,7 +186,8 @@ impl Config { }; let file_raw = fs::read_to_string(path).context("fs::read()")?; - let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root).context("ProtoPath::from_input_path()")?; + let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root) + .context("ProtoPath::from_input_path()")?; pool_raw .file .push(protox_parse::parse(&path.to_string(), &file_raw).map_err( @@ -251,23 +264,24 @@ impl Config { let package_root = self.proto_root.to_name().context("invalid proto_root")?; let output = output.sub(&package_root.to_rust_module()); for proto_name in extract_message_names(&descriptor) { - output.append(&self.reflect_impl(&proto_name) - .with_context(||format!("reflect_impl({proto_name})"))?); + output.append( + &self + .reflect_impl(&proto_name) + .with_context(|| format!("reflect_impl({proto_name})"))?, + ); } // Generate the descriptor. let rust_deps = self .dependencies .iter() - .map(|d| format!("&{}::DESCRIPTOR", d.0)) + .map(|d| d.0.to_string()) .collect::>() .join(","); let this = self.this_crate().to_string(); - output.append(&format!("\ - pub static DESCRIPTOR : {this}::Lazy<{this}::Descriptor> = {this}::Lazy::new(|| {{\ - {this}::Descriptor::new(\"{package_root}\".into(), vec![{rust_deps}], &include_bytes!({descriptor_path:?})[..])\ - }});\ - ")); + output.append(&format!( + "{this}::declare_descriptor!(\"{package_root}\",{descriptor_path:?},{rust_deps});" + )); // Save output. fs::write(output_path, output.format().context("output.format()")?)?; diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 36c673e0..a022094a 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -64,7 +64,7 @@ impl ProtoPath { } pub(super) fn parent(&self) -> Option { - self.0.parent().map(|p|Self(p.into())) + self.0.parent().map(|p| Self(p.into())) } /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . @@ -92,8 +92,6 @@ impl fmt::Display for ProtoPath { } } - - type Part = String; /// A rust module/type name that the generated code is available at. Although From 5cb5e41fe1f4372b9e6117a8ba7a399f54e9e0a2 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 7 Nov 2023 17:19:06 +0100 Subject: [PATCH 38/40] cargo fmt --- node/deny.toml | 3 +++ node/libs/protobuf_build/src/syntax.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/node/deny.toml b/node/deny.toml index 4ea57cf8..8bfe4c74 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -57,6 +57,9 @@ skip = [ # Old versions required by hyper. { name = "socket2", version = "=0.4.9" }, { name = "hashbrown", version = "=0.12.3" }, # (hyper -> h2 -> indexmap -> hashbrown) + + # Old versions required by ark-bn254. + { name = "syn", version = "=1.0.109" } ] [sources] diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index a022094a..ca88b2ea 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -63,6 +63,7 @@ impl ProtoPath { Ok(ProtoName(parts)) } + /// Returns path to the parent directory. pub(super) fn parent(&self) -> Option { self.0.parent().map(|p| Self(p.into())) } From 3598b33ef0b79596caa1476d29a87ea54348b420 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Tue, 7 Nov 2023 17:39:22 +0100 Subject: [PATCH 39/40] applied comments --- node/libs/protobuf_build/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 9e24068c..0b1d349f 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -94,15 +94,16 @@ impl Descriptor { } /// Loads the descriptor to the global pool and returns a copy of the global pool. - pub fn load_global(&self) -> prost_reflect::DescriptorPool { + pub fn get_message_by_name(&self, name: &str) -> Option { /// Global descriptor pool. static POOL: Lazy> = Lazy::new(Mutex::default); let pool = &mut POOL.lock().unwrap(); self.load(pool).unwrap(); - pool.clone() + pool.get_message_by_name(name) } } +/// Expands to a descriptor declaration. #[macro_export] macro_rules! declare_descriptor { ($package_root:expr, $descriptor_path:expr, $($rust_deps:path),*) => { @@ -148,7 +149,7 @@ impl Config { Ok(format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ fn descriptor(&self) -> {this}::prost_reflect::MessageDescriptor {{\ static INIT : {this}::Lazy<{this}::prost_reflect::MessageDescriptor> = {this}::Lazy::new(|| {{\ - DESCRIPTOR.load_global().get_message_by_name({proto_name:?}).unwrap()\ + DESCRIPTOR.get_message_by_name({proto_name:?}).unwrap()\ }});\ INIT.clone()\ }}\ From 2c9fb4995734eb21e47e8a956d1841d01addc8b6 Mon Sep 17 00:00:00 2001 From: Grzegorz Prusak Date: Wed, 8 Nov 2023 10:08:37 +0100 Subject: [PATCH 40/40] use quote to generate code --- node/Cargo.lock | 1 + node/Cargo.toml | 1 + node/libs/protobuf_build/Cargo.toml | 1 + node/libs/protobuf_build/src/lib.rs | 41 +++++++++++++++----------- node/libs/protobuf_build/src/syntax.rs | 9 ++---- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index 32201eca..8896fef8 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -2850,6 +2850,7 @@ dependencies = [ "prost-types", "protox", "protox-parse", + "quote", "rand", "syn 2.0.32", ] diff --git a/node/Cargo.toml b/node/Cargo.toml index 4a1a808a..8dc3d506 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -62,6 +62,7 @@ protox-parse = "0.5.0" prettyplease = "0.2.6" pretty_assertions = "1.4.0" quick-protobuf = "0.8.1" +quote = "1.0.33" rand = "0.8.0" rocksdb = "0.21.0" serde = { version = "1.0", features = ["derive"] } diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml index cf389a8f..46129be5 100644 --- a/node/libs/protobuf_build/Cargo.toml +++ b/node/libs/protobuf_build/Cargo.toml @@ -17,6 +17,7 @@ prost-types.workspace = true prost-reflect.workspace = true protox.workspace = true protox-parse.workspace = true +quote.workspace = true rand.workspace = true syn.workspace = true diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 0b1d349f..aa1048f3 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -23,9 +23,9 @@ //! It is not possible to depend on a different proto bundle within the same crate (because //! these are being built simultaneously from the same build script). #![allow(clippy::print_stdout)] +// Imports accessed from the generated code. pub use self::syntax::*; use anyhow::Context as _; -// Imports accessed from the generated code. pub use once_cell::sync::Lazy; pub use prost; use prost::Message as _; @@ -144,16 +144,19 @@ impl Config { .unwrap() .to_rust_type() .to_string(); + let rust_name: syn::Path = syn::parse_str(&rust_name).context("rust_name")?; let proto_name = proto_name.to_string(); - let this = self.this_crate().to_string(); - Ok(format!("impl {this}::prost_reflect::ReflectMessage for {rust_name} {{\ - fn descriptor(&self) -> {this}::prost_reflect::MessageDescriptor {{\ - static INIT : {this}::Lazy<{this}::prost_reflect::MessageDescriptor> = {this}::Lazy::new(|| {{\ - DESCRIPTOR.get_message_by_name({proto_name:?}).unwrap()\ - }});\ - INIT.clone()\ - }}\ - }}")) + let this: syn::Path = syn::parse_str(&self.this_crate().to_string()).context("this")?; + Ok(quote::quote! { + impl #this::prost_reflect::ReflectMessage for #rust_name { + fn descriptor(&self) -> #this::prost_reflect::MessageDescriptor { + static INIT : #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| { + DESCRIPTOR.get_message_by_name(#proto_name).unwrap() + }); + INIT.clone() + } + } + }.to_string()) } /// Generates rust code from the proto files according to the config. @@ -208,6 +211,7 @@ impl Config { // rewrapping the error, so that source location is included in the error message. .map_err(|err| anyhow::anyhow!("{err:?}"))?; let descriptor = compiler.file_descriptor_set(); + // Unwrap is ok, because we add a descriptor from a successful compilation. pool.add_file_descriptor_set(descriptor.clone()).unwrap(); // Check that the compiled proto files belong to the declared proto package. @@ -275,13 +279,16 @@ impl Config { let rust_deps = self .dependencies .iter() - .map(|d| d.0.to_string()) - .collect::>() - .join(","); - let this = self.this_crate().to_string(); - output.append(&format!( - "{this}::declare_descriptor!(\"{package_root}\",{descriptor_path:?},{rust_deps});" - )); + .map(|d| syn::parse_str::(&d.0.to_string()).unwrap()); + let this: syn::Path = syn::parse_str(&self.this_crate().to_string())?; + let package_root = package_root.to_string(); + let descriptor_path = descriptor_path.display().to_string(); + output.append( + "e::quote! { + #this::declare_descriptor!(#package_root,#descriptor_path,#(#rust_deps),*); + } + .to_string(), + ); // Save output. fs::write(output_path, output.format().context("output.format()")?)?; diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index ca88b2ea..80477448 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -116,12 +116,6 @@ impl fmt::Display for RustName { } } -impl From for RustName { - fn from(s: prost_build::Module) -> Self { - Self(s.parts().map(Part::from).collect()) - } -} - impl From<&str> for RustName { fn from(s: &str) -> Self { Self(s.split("::").map(Part::from).collect()) @@ -165,8 +159,9 @@ impl RustModule { /// Collects the code of the module and formats it. pub(crate) fn format(&self) -> anyhow::Result { + let s = self.collect(); Ok(prettyplease::unparse( - &syn::parse_str(&self.collect()).context("syn::parse_str()")?, + &syn::parse_str(&s).with_context(|| format!("syn::parse_str({s:?})"))?, )) } }