Update from upstream

node/Cargo.toml

@@ -72,7 +72,7 @@ thiserror = "1.0.40"
 time = "0.3.23"
 # TODO(gprusak): only concurrency crate should depend on tokio.
 # All the other crates should depend on concurrency.
-tokio = { version = "1.28.1", features = ["full"] }
+tokio = { version = "1.34.0", features = ["full"] }
 tracing = { version = "0.1.37", features = ["attributes"] }
 tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] }
 vise = { version = "0.1.0", git = "https://github.com/matter-labs/vise.git", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }

node/deny.toml

@@ -48,18 +48,21 @@ multiple-versions = "deny"
 # Certain crates/versions that will be skipped when doing duplicate detection.
 skip = [
     # Old versions required by tempfile and prost-build.
-    { name = "bitflags", version = "=1.3.2" },
+    { name = "bitflags", version = "1.3.2" },
 
     # Old version required by tracing-subscriber.
-    { name = "regex-automata", version = "=0.1.10" },
-    { name = "regex-syntax", version = "=0.6.29" },
+    { name = "regex-automata", version = "0.1.10" },
+    { name = "regex-syntax", version = "0.6.29" },
 
     # Old versions required by hyper.
-    { name = "socket2", version = "=0.4.10" },
+    { name = "socket2", version = "0.4.10" },
 
     # Old versions required by pairing_ce & ff_ce.
     { name = "rand", version = "0.4" },
     { name = "syn", version = "1.0" },
+
+    # Old versions required by criterion.
+    { name = "itertools", version = "0.10.5" }
 ]
 
 [sources]

node/libs/protobuf/examples/conformance_test/main.rs

@@ -15,6 +15,21 @@ use zksync_concurrency::{ctx, io};
 
 mod proto;
 
+/// Decodes a generated proto message from json for arbitrary ReflectMessage.
+fn decode_json_proto<T: ReflectMessage + Default>(json: &str) -> anyhow::Result<T> {
+    let mut deserializer = serde_json::Deserializer::from_str(json);
+    let proto: T = zksync_protobuf::serde::deserialize_proto(&mut deserializer)?;
+    deserializer.end()?;
+    Ok(proto)
+}
+
+/// Encodes a generated proto message to json for arbitrary ReflectMessage.
+fn encode_json_proto<T: ReflectMessage>(proto: &T) -> String {
+    let mut serializer = serde_json::Serializer::pretty(vec![]);
+    zksync_protobuf::serde::serialize_proto(proto, &mut serializer).unwrap();
+    String::from_utf8(serializer.into_inner()).unwrap()
+}
+
 impl ConformanceResult {
     /// Creates a "skipped" result with the specified message.
     fn skipped(reason: &str) -> Self {
@@ -30,15 +45,15 @@ impl ConformanceResult {
 /// Processes a single conformance request.
 fn process_request(req: proto::ConformanceRequest) -> anyhow::Result<ConformanceResult> {
     let message_type = req.message_type.context("missing message_type")?;
-    if message_type != *"protobuf_test_messages.proto3.TestAllTypesProto3" {
+    if message_type != "protobuf_test_messages.proto3.TestAllTypesProto3" {
         return Ok(ConformanceResult::skipped("unsupported"));
     }
 
     // Decode.
     let payload = req.payload.context("missing payload")?;
     let payload = match payload {
         proto::conformance_request::Payload::JsonPayload(payload) => {
-            match zksync_protobuf::decode_json_proto(&payload) {
+            match decode_json_proto(&payload) {
                 Ok(payload) => payload,
                 Err(_) => return Ok(ConformanceResult::skipped("unsupported fields")),
             }
@@ -62,9 +77,7 @@ fn process_request(req: proto::ConformanceRequest) -> anyhow::Result<Conformance
         .requested_output_format
         .context("missing output format")?;
     match proto::WireFormat::try_from(format).context("unknown format")? {
-        proto::WireFormat::Json => Ok(ConformanceResult::JsonPayload(
-            zksync_protobuf::encode_json_proto(&payload),
-        )),
+        proto::WireFormat::Json => Ok(ConformanceResult::JsonPayload(encode_json_proto(&payload))),
         proto::WireFormat::Protobuf => {
             // Re-encode the parsed proto.
             let encoded =

node/libs/protobuf/src/build.rs

@@ -5,6 +5,7 @@ pub use prost;
 use prost::Message as _;
 pub use prost_reflect;
 use prost_reflect::prost_types;
+pub use serde;
 use std::sync::RwLock;
 
 /// Global descriptor pool.
@@ -81,5 +82,36 @@ macro_rules! impl_reflect_message {
     };
 }
 
+/// Implements `serde::Serialize` compatible with protobuf json encoding.
+#[macro_export]
+macro_rules! impl_serde_serialize {
+    ($ty:ty) => {
+        impl $crate::build::serde::Serialize for $ty {
+            fn serialize<S: $crate::build::serde::Serializer>(
+                &self,
+                s: S,
+            ) -> Result<S::Ok, S::Error> {
+                $crate::serde_serialize(self, s)
+            }
+        }
+    };
+}
+
+/// Implements `serde::Deserialize` compatible with protobuf json encoding.
+#[macro_export]
+macro_rules! impl_serde_deserialize {
+    ($ty:ty) => {
+        impl<'de> $crate::build::serde::Deserialize<'de> for $ty {
+            fn deserialize<D: $crate::build::serde::Deserializer<'de>>(
+                d: D,
+            ) -> Result<Self, D::Error> {
+                $crate::serde_deserialize(d)
+            }
+        }
+    };
+}
+
 pub use declare_descriptor;
 pub use impl_reflect_message;
+pub use impl_serde_deserialize;
+pub use impl_serde_serialize;

node/libs/protobuf/src/lib.rs

@@ -5,6 +5,7 @@
 pub mod build;
 pub mod proto;
 mod proto_fmt;
+pub mod serde;
 mod std_conv;
 pub mod testonly;
 

node/libs/protobuf/src/proto_fmt.rs

@@ -265,40 +265,6 @@ pub fn decode<T: ProtoFmt>(bytes: &[u8]) -> anyhow::Result<T> {
     T::read(&<T as ProtoFmt>::Proto::decode(bytes)?)
 }
 
-/// Encodes a generated proto message to json.
-/// WARNING: this function uses reflection, so it is not very efficient.
-pub fn encode_json_proto<T: ReflectMessage>(x: &T) -> String {
-    let mut s = serde_json::Serializer::pretty(vec![]);
-    let opts = prost_reflect::SerializeOptions::new();
-    x.transcode_to_dynamic()
-        .serialize_with_options(&mut s, &opts)
-        .unwrap();
-    String::from_utf8(s.into_inner()).unwrap()
-}
-
-/// Encodes a proto message to json.
-/// WARNING: this function uses reflection, so it is not very efficient.
-pub fn encode_json<T: ProtoFmt>(x: &T) -> String {
-    encode_json_proto(&x.build())
-}
-
-/// Decodes a generated proto message from json.
-/// WARNING: this function uses reflection, so it is not very efficient.
-pub fn decode_json_proto<T: ReflectMessage + Default>(json: &str) -> anyhow::Result<T> {
-    let mut d = serde_json::de::Deserializer::from_str(json);
-    let mut p = T::default();
-    let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), &mut d)?;
-    d.end()?;
-    p.merge(msg.encode_to_vec().as_slice()).unwrap();
-    Ok(p)
-}
-
-/// Decodes a proto message from json.
-/// WARNING: this function uses reflection, so it is not very efficient.
-pub fn decode_json<T: ProtoFmt>(json: &str) -> anyhow::Result<T> {
-    T::read(&decode_json_proto(json)?)
-}
-
 /// Trait defining a proto representation for a type.
 pub trait ProtoFmt: Sized {
     /// Proto message type representing Self.

node/libs/protobuf/src/serde.rs

@@ -0,0 +1,37 @@
+//! Implementation of serde traits for structs implementing ProtoFmt.
+//! This serde implementation is compatible with protobuf json mapping,
+//! therefore it is suitable for version control.
+//! WARNING: Currently this serde implementation uses reflection,
+//! so it is not very efficient.
+use crate::ProtoFmt;
+use prost::Message as _;
+use prost_reflect::ReflectMessage;
+
+/// Implementation of serde::Serialize for arbitrary ReflectMessage.
+pub fn serialize_proto<T: ReflectMessage, S: serde::Serializer>(
+    x: &T,
+    s: S,
+) -> Result<S::Ok, S::Error> {
+    let opts = prost_reflect::SerializeOptions::new();
+    x.transcode_to_dynamic().serialize_with_options(s, &opts)
+}
+
+/// Implementation of serde::Serialize for arbitrary ProtoFmt.
+pub fn serialize<T: ProtoFmt, S: serde::Serializer>(x: &T, s: S) -> Result<S::Ok, S::Error> {
+    serialize_proto(&x.build(), s)
+}
+
+/// Implementation of serde::Deserialize for arbitrary ReflectMessage.
+pub fn deserialize_proto<'de, T: ReflectMessage + Default, D: serde::Deserializer<'de>>(
+    d: D,
+) -> Result<T, D::Error> {
+    let mut p = T::default();
+    let msg = prost_reflect::DynamicMessage::deserialize(p.descriptor(), d)?;
+    p.merge(msg.encode_to_vec().as_slice()).unwrap();
+    Ok(p)
+}
+
+/// Implementation of serde::Deserialize for arbitrary ProtoFmt.
+pub fn deserialize<'de, T: ProtoFmt, D: serde::Deserializer<'de>>(d: D) -> Result<T, D::Error> {
+    T::read(&deserialize_proto(d)?).map_err(serde::de::Error::custom)
+}

node/libs/protobuf_build/src/lib.rs

@@ -140,11 +140,6 @@ pub struct Config {
 }
 
 impl Config {
-    /// Location of the protobuf_build crate, visible from the generated code.
-    fn this_crate(&self) -> RustName {
-        self.protobuf_crate.clone().join(RustName::ident("build"))
-    }
-
     /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated
     /// from a message of the given `proto_name`.
     fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result<syn::Item> {
@@ -153,10 +148,10 @@ impl Config {
             .unwrap()
             .to_rust_type()?;
         let proto_name = proto_name.to_string();
-        let this = self.this_crate();
-        Ok(syn::parse_quote! {
-            #this::impl_reflect_message!(#rust_name, &DESCRIPTOR, #proto_name);
-        })
+        let protobuf_crate = &self.protobuf_crate;
+        Ok(
+            syn::parse_quote! { #protobuf_crate::build::impl_reflect_message!(#rust_name, &DESCRIPTOR, #proto_name); },
+        )
     }
 
     /// Validates this configuration.
@@ -319,7 +314,11 @@ impl Config {
         // Generate code out of compiled proto files.
         let mut output = RustModule::default();
         let mut config = prost_build::Config::new();
-        let prost_path = self.this_crate().join(RustName::ident("prost"));
+        let prost_path = self
+            .protobuf_crate
+            .clone()
+            .join(RustName::ident("build"))
+            .join(RustName::ident("prost"));
         config.prost_path(prost_path.to_string());
         config.skip_protoc_run();
         for (root_path, manifest) in self.dependencies.iter().zip(&dependency_manifests) {
@@ -352,18 +351,18 @@ impl Config {
 
         // Generate the descriptor.
         let root_paths_for_deps = self.dependencies.iter();
-        let this = self.this_crate();
+        let protobuf_crate = &self.protobuf_crate;
         let descriptor_path = descriptor_path.display().to_string();
         output.append_item(syn::parse_quote! {
-            #this::declare_descriptor!(DESCRIPTOR => #descriptor_path, #(#root_paths_for_deps),*);
+            #protobuf_crate::build::declare_descriptor!(DESCRIPTOR => #descriptor_path, #(#root_paths_for_deps),*);
         });
 
         // Generate the reflection code.
         for proto_name in extract_message_names(&descriptor) {
-            let impl_item = self
+            let item = self
                 .reflect_impl(&proto_name)
                 .with_context(|| format!("reflect_impl({proto_name})"))?;
-            output.append_item(impl_item);
+            output.append_item(item);
         }
 
         // Save output.

node/tools/Cargo.toml

@@ -21,6 +21,7 @@ zksync_protobuf.workspace = true
 anyhow.workspace = true
 clap.workspace = true
 rand.workspace = true
+serde_json.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true

node/tools/src/bin/localnet_config.rs

@@ -9,6 +9,13 @@ use zksync_consensus_executor::{ConsensusConfig, ExecutorConfig, GossipConfig};
 use zksync_consensus_roles::{node, validator};
 use zksync_consensus_tools::NodeConfig;
 
+/// Encodes a generated proto message to json for arbitrary ProtoFmt.
+fn encode_json<T: zksync_protobuf::ProtoFmt>(x: &T) -> String {
+    let mut s = serde_json::Serializer::pretty(vec![]);
+    zksync_protobuf::serde::serialize(x, &mut s).unwrap();
+    String::from_utf8(s.into_inner()).unwrap()
+}
+
 /// Replaces IP of the address with UNSPECIFIED (aka INADDR_ANY) of the corresponding IP type.
 /// Opening a listener socket with an UNSPECIFIED IP, means that the new connections
 /// on any network interface of the VM will be accepted.
@@ -110,11 +117,7 @@ fn main() -> anyhow::Result<()> {
         let _ = fs::remove_dir_all(&root);
         fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?;
 
-        fs::write(
-            root.join("config.json"),
-            zksync_protobuf::encode_json(&node_cfg),
-        )
-        .context("fs::write()")?;
+        fs::write(root.join("config.json"), encode_json(&node_cfg)).context("fs::write()")?;
         fs::write(
             root.join("validator_key"),
             &TextFmt::encode(&validator_keys[i]),

node/tools/src/config.rs

@@ -6,6 +6,14 @@ use zksync_consensus_executor::{proto, ConsensusConfig, ExecutorConfig};
 use zksync_consensus_roles::{node, validator};
 use zksync_protobuf::{read_optional, read_required, ProtoFmt};
 
+/// Decodes a proto message from json for arbitrary ProtoFmt.
+fn decode_json<T: ProtoFmt>(json: &str) -> anyhow::Result<T> {
+    let mut d = serde_json::Deserializer::from_str(json);
+    let p: T = zksync_protobuf::serde::deserialize(&mut d)?;
+    d.end()?;
+    Ok(p)
+}
+
 /// This struct holds the file path to each of the config files.
 #[derive(Debug)]
 pub struct ConfigPaths<'a> {
@@ -75,13 +83,12 @@ impl Configs {
                 args.config.display()
             )
         })?;
-        let node_config: NodeConfig =
-            zksync_protobuf::decode_json(&node_config).with_context(|| {
-                format!(
-                    "failed decoding JSON node config at `{}`",
-                    args.config.display()
-                )
-            })?;
+        let node_config: NodeConfig = decode_json(&node_config).with_context(|| {
+            format!(
+                "failed decoding JSON node config at `{}`",
+                args.config.display()
+            )
+        })?;
 
         let validator_key: Option<validator::SecretKey> = args
             .validator_key