diff --git a/node/Cargo.lock b/node/Cargo.lock index 07ce3a74..07cf3024 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -2727,6 +2727,7 @@ dependencies = [ "anyhow", "clap", "rand 0.8.5", + "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/node/Cargo.toml b/node/Cargo.toml index c998681b..ba5ce443 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -72,7 +72,7 @@ thiserror = "1.0.40" time = "0.3.23" # TODO(gprusak): only concurrency crate should depend on tokio. # All the other crates should depend on concurrency. -tokio = { version = "1.28.1", features = ["full"] } +tokio = { version = "1.34.0", features = ["full"] } tracing = { version = "0.1.37", features = ["attributes"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } vise = { version = "0.1.0", git = "https://github.com/matter-labs/vise.git", rev = "dd05139b76ab0843443ab3ff730174942c825dae" } diff --git a/node/deny.toml b/node/deny.toml index 4806b4d4..e7d3b9cd 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -48,18 +48,21 @@ multiple-versions = "deny" # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ # Old versions required by tempfile and prost-build. - { name = "bitflags", version = "=1.3.2" }, + { name = "bitflags", version = "1.3.2" }, # Old version required by tracing-subscriber. - { name = "regex-automata", version = "=0.1.10" }, - { name = "regex-syntax", version = "=0.6.29" }, + { name = "regex-automata", version = "0.1.10" }, + { name = "regex-syntax", version = "0.6.29" }, # Old versions required by hyper. - { name = "socket2", version = "=0.4.10" }, + { name = "socket2", version = "0.4.10" }, # Old versions required by pairing_ce & ff_ce. { name = "rand", version = "0.4" }, { name = "syn", version = "1.0" }, + + # Old versions required by criterion. + { name = "itertools", version = "0.10.5" } ] [sources] diff --git a/node/libs/protobuf/examples/conformance_test/main.rs b/node/libs/protobuf/examples/conformance_test/main.rs index 0b9093ee..e432993b 100644 --- a/node/libs/protobuf/examples/conformance_test/main.rs +++ b/node/libs/protobuf/examples/conformance_test/main.rs @@ -15,6 +15,21 @@ use zksync_concurrency::{ctx, io}; mod proto; +/// Decodes a generated proto message from json for arbitrary ReflectMessage. +fn decode_json_proto(json: &str) -> anyhow::Result { + let mut deserializer = serde_json::Deserializer::from_str(json); + let proto: T = zksync_protobuf::serde::deserialize_proto(&mut deserializer)?; + deserializer.end()?; + Ok(proto) +} + +/// Encodes a generated proto message to json for arbitrary ReflectMessage. +fn encode_json_proto(proto: &T) -> String { + let mut serializer = serde_json::Serializer::pretty(vec![]); + zksync_protobuf::serde::serialize_proto(proto, &mut serializer).unwrap(); + String::from_utf8(serializer.into_inner()).unwrap() +} + impl ConformanceResult { /// Creates a "skipped" result with the specified message. fn skipped(reason: &str) -> Self { @@ -30,7 +45,7 @@ impl ConformanceResult { /// Processes a single conformance request. fn process_request(req: proto::ConformanceRequest) -> anyhow::Result { let message_type = req.message_type.context("missing message_type")?; - if message_type != *"protobuf_test_messages.proto3.TestAllTypesProto3" { + if message_type != "protobuf_test_messages.proto3.TestAllTypesProto3" { return Ok(ConformanceResult::skipped("unsupported")); } @@ -38,7 +53,7 @@ fn process_request(req: proto::ConformanceRequest) -> anyhow::Result { - match zksync_protobuf::decode_json_proto(&payload) { + match decode_json_proto(&payload) { Ok(payload) => payload, Err(_) => return Ok(ConformanceResult::skipped("unsupported fields")), } @@ -62,9 +77,7 @@ fn process_request(req: proto::ConformanceRequest) -> anyhow::Result Ok(ConformanceResult::JsonPayload( - zksync_protobuf::encode_json_proto(&payload), - )), + proto::WireFormat::Json => Ok(ConformanceResult::JsonPayload(encode_json_proto(&payload))), proto::WireFormat::Protobuf => { // Re-encode the parsed proto. let encoded = diff --git a/node/libs/protobuf/src/build.rs b/node/libs/protobuf/src/build.rs index b1258868..75f720f4 100644 --- a/node/libs/protobuf/src/build.rs +++ b/node/libs/protobuf/src/build.rs @@ -5,6 +5,7 @@ pub use prost; use prost::Message as _; pub use prost_reflect; use prost_reflect::prost_types; +pub use serde; use std::sync::RwLock; /// Global descriptor pool. @@ -81,5 +82,36 @@ macro_rules! impl_reflect_message { }; } +/// Implements `serde::Serialize` compatible with protobuf json encoding. +#[macro_export] +macro_rules! impl_serde_serialize { + ($ty:ty) => { + impl $crate::build::serde::Serialize for $ty { + fn serialize( + &self, + s: S, + ) -> Result { + $crate::serde_serialize(self, s) + } + } + }; +} + +/// Implements `serde::Deserialize` compatible with protobuf json encoding. +#[macro_export] +macro_rules! impl_serde_deserialize { + ($ty:ty) => { + impl<'de> $crate::build::serde::Deserialize<'de> for $ty { + fn deserialize>( + d: D, + ) -> Result { + $crate::serde_deserialize(d) + } + } + }; +} + pub use declare_descriptor; pub use impl_reflect_message; +pub use impl_serde_deserialize; +pub use impl_serde_serialize; diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs index 8524d801..49958df5 100644 --- a/node/libs/protobuf/src/lib.rs +++ b/node/libs/protobuf/src/lib.rs @@ -5,6 +5,7 @@ pub mod build; pub mod proto; mod proto_fmt; +pub mod serde; mod std_conv; pub mod testonly; diff --git a/node/libs/protobuf/src/proto_fmt.rs b/node/libs/protobuf/src/proto_fmt.rs index 541b0f0b..53238a64 100644 --- a/node/libs/protobuf/src/proto_fmt.rs +++ b/node/libs/protobuf/src/proto_fmt.rs @@ -265,40 +265,6 @@ pub fn decode(bytes: &[u8]) -> anyhow::Result { T::read(&::Proto::decode(bytes)?) } -/// Encodes a generated proto message to json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn encode_json_proto(x: &T) -> String { - let mut s = serde_json::Serializer::pretty(vec![]); - let opts = prost_reflect::SerializeOptions::new(); - x.transcode_to_dynamic() - .serialize_with_options(&mut s, &opts) - .unwrap(); - String::from_utf8(s.into_inner()).unwrap() -} - -/// Encodes a proto message to json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn encode_json(x: &T) -> String { - encode_json_proto(&x.build()) -} - -/// Decodes a generated proto message from json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn decode_json_proto(json: &str) -> anyhow::Result { - let mut d = serde_json::de::Deserializer::from_str(json); - let mut p = T::default(); - let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), &mut d)?; - d.end()?; - p.merge(msg.encode_to_vec().as_slice()).unwrap(); - Ok(p) -} - -/// Decodes a proto message from json. -/// WARNING: this function uses reflection, so it is not very efficient. -pub fn decode_json(json: &str) -> anyhow::Result { - T::read(&decode_json_proto(json)?) -} - /// Trait defining a proto representation for a type. pub trait ProtoFmt: Sized { /// Proto message type representing Self. diff --git a/node/libs/protobuf/src/serde.rs b/node/libs/protobuf/src/serde.rs new file mode 100644 index 00000000..db11b1cb --- /dev/null +++ b/node/libs/protobuf/src/serde.rs @@ -0,0 +1,37 @@ +//! Implementation of serde traits for structs implementing ProtoFmt. +//! This serde implementation is compatible with protobuf json mapping, +//! therefore it is suitable for version control. +//! WARNING: Currently this serde implementation uses reflection, +//! so it is not very efficient. +use crate::ProtoFmt; +use prost::Message as _; +use prost_reflect::ReflectMessage; + +/// Implementation of serde::Serialize for arbitrary ReflectMessage. +pub fn serialize_proto( + x: &T, + s: S, +) -> Result { + let opts = prost_reflect::SerializeOptions::new(); + x.transcode_to_dynamic().serialize_with_options(s, &opts) +} + +/// Implementation of serde::Serialize for arbitrary ProtoFmt. +pub fn serialize(x: &T, s: S) -> Result { + serialize_proto(&x.build(), s) +} + +/// Implementation of serde::Deserialize for arbitrary ReflectMessage. +pub fn deserialize_proto<'de, T: ReflectMessage + Default, D: serde::Deserializer<'de>>( + d: D, +) -> Result { + let mut p = T::default(); + let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), d)?; + p.merge(msg.encode_to_vec().as_slice()).unwrap(); + Ok(p) +} + +/// Implementation of serde::Deserialize for arbitrary ProtoFmt. +pub fn deserialize<'de, T: ProtoFmt, D: serde::Deserializer<'de>>(d: D) -> Result { + T::read(&deserialize_proto(d)?).map_err(serde::de::Error::custom) +} diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 82ebc065..e6e384cf 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -140,11 +140,6 @@ pub struct Config { } impl Config { - /// Location of the protobuf_build crate, visible from the generated code. - fn this_crate(&self) -> RustName { - self.protobuf_crate.clone().join(RustName::ident("build")) - } - /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated /// from a message of the given `proto_name`. fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { @@ -153,10 +148,10 @@ impl Config { .unwrap() .to_rust_type()?; let proto_name = proto_name.to_string(); - let this = self.this_crate(); - Ok(syn::parse_quote! { - #this::impl_reflect_message!(#rust_name, &DESCRIPTOR, #proto_name); - }) + let protobuf_crate = &self.protobuf_crate; + Ok( + syn::parse_quote! { #protobuf_crate::build::impl_reflect_message!(#rust_name, &DESCRIPTOR, #proto_name); }, + ) } /// Validates this configuration. @@ -319,7 +314,11 @@ impl Config { // Generate code out of compiled proto files. let mut output = RustModule::default(); let mut config = prost_build::Config::new(); - let prost_path = self.this_crate().join(RustName::ident("prost")); + let prost_path = self + .protobuf_crate + .clone() + .join(RustName::ident("build")) + .join(RustName::ident("prost")); config.prost_path(prost_path.to_string()); config.skip_protoc_run(); for (root_path, manifest) in self.dependencies.iter().zip(&dependency_manifests) { @@ -352,18 +351,18 @@ impl Config { // Generate the descriptor. let root_paths_for_deps = self.dependencies.iter(); - let this = self.this_crate(); + let protobuf_crate = &self.protobuf_crate; let descriptor_path = descriptor_path.display().to_string(); output.append_item(syn::parse_quote! { - #this::declare_descriptor!(DESCRIPTOR => #descriptor_path, #(#root_paths_for_deps),*); + #protobuf_crate::build::declare_descriptor!(DESCRIPTOR => #descriptor_path, #(#root_paths_for_deps),*); }); // Generate the reflection code. for proto_name in extract_message_names(&descriptor) { - let impl_item = self + let item = self .reflect_impl(&proto_name) .with_context(|| format!("reflect_impl({proto_name})"))?; - output.append_item(impl_item); + output.append_item(item); } // Save output. diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 677e3df2..055f9c8d 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -21,6 +21,7 @@ zksync_protobuf.workspace = true anyhow.workspace = true clap.workspace = true rand.workspace = true +serde_json.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 69c9f71e..53409e98 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -9,6 +9,13 @@ use zksync_consensus_executor::{ConsensusConfig, ExecutorConfig, GossipConfig}; use zksync_consensus_roles::{node, validator}; use zksync_consensus_tools::NodeConfig; +/// Encodes a generated proto message to json for arbitrary ProtoFmt. +fn encode_json(x: &T) -> String { + let mut s = serde_json::Serializer::pretty(vec![]); + zksync_protobuf::serde::serialize(x, &mut s).unwrap(); + String::from_utf8(s.into_inner()).unwrap() +} + /// Replaces IP of the address with UNSPECIFIED (aka INADDR_ANY) of the corresponding IP type. /// Opening a listener socket with an UNSPECIFIED IP, means that the new connections /// on any network interface of the VM will be accepted. @@ -110,11 +117,7 @@ fn main() -> anyhow::Result<()> { let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; - fs::write( - root.join("config.json"), - zksync_protobuf::encode_json(&node_cfg), - ) - .context("fs::write()")?; + fs::write(root.join("config.json"), encode_json(&node_cfg)).context("fs::write()")?; fs::write( root.join("validator_key"), &TextFmt::encode(&validator_keys[i]), diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 3bdec7c6..14d922bc 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -6,6 +6,14 @@ use zksync_consensus_executor::{proto, ConsensusConfig, ExecutorConfig}; use zksync_consensus_roles::{node, validator}; use zksync_protobuf::{read_optional, read_required, ProtoFmt}; +/// Decodes a proto message from json for arbitrary ProtoFmt. +fn decode_json(json: &str) -> anyhow::Result { + let mut d = serde_json::Deserializer::from_str(json); + let p: T = zksync_protobuf::serde::deserialize(&mut d)?; + d.end()?; + Ok(p) +} + /// This struct holds the file path to each of the config files. #[derive(Debug)] pub struct ConfigPaths<'a> { @@ -75,13 +83,12 @@ impl Configs { args.config.display() ) })?; - let node_config: NodeConfig = - zksync_protobuf::decode_json(&node_config).with_context(|| { - format!( - "failed decoding JSON node config at `{}`", - args.config.display() - ) - })?; + let node_config: NodeConfig = decode_json(&node_config).with_context(|| { + format!( + "failed decoding JSON node config at `{}`", + args.config.display() + ) + })?; let validator_key: Option = args .validator_key