diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index 68daaf44..59bd6a36 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -14,7 +14,12 @@ on: branches: ["main"] env: - PROTOS_DIR: "node/libs/schema/proto" + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + RUSTFLAGS: "-Dwarnings -C linker=clang -C link-arg=-fuse-ld=lld -C link-arg=-Wl,-z,nostart-stop-gc" + RUSTC_WRAPPER: "sccache" + SCCACHE_GHA_ENABLED: "true" + RUST_BACKTRACE: "1" jobs: compatibility: @@ -22,16 +27,33 @@ jobs: steps: # github.base_ref -> github.head_ref for pull_request # github.event.before -> github.event.after for push + - uses: mozilla-actions/sccache-action@v0.0.3 - uses: actions/checkout@v3 with: ref: ${{ github.base_ref || github.event.before }} path: before + - name: compile before + run: cargo build --all-targets + working-directory: ./before/node + - name: build before.binpb + run: > + perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' + `find ./before/node/target/debug/build/*/output` + | xargs cat > ./before.binpb - uses: actions/checkout@v3 with: ref: ${{ github.head_ref || github.event.after }} path: after + - name: compile after + run: cargo build --all-targets + working-directory: ./after/node + - name: build after.binpb + run: > + perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' + `find ./after/node/target/debug/build/*/output` + | xargs cat > ./after.binpb - uses: bufbuild/buf-setup-action@v1 - uses: bufbuild/buf-breaking-action@v1 with: - input: "after/$PROTOS_DIR" - against: "before/$PROTOS_DIR" + input: "./after.binpb" + against: "./before.binpb" diff --git a/.github/workflows/protobuf_conformance.yaml b/.github/workflows/protobuf_conformance.yaml index eac65e28..69efd530 100644 --- a/.github/workflows/protobuf_conformance.yaml +++ b/.github/workflows/protobuf_conformance.yaml @@ -28,7 +28,7 @@ jobs: path: "protobuf" - uses: mozilla-actions/sccache-action@v0.0.3 - name: build test - run: cargo build -p zksync_consensus_schema --bin conformance_test + run: cargo build -p zksync_protobuf --bin conformance_test working-directory: "this/node" - name: Cache Bazel uses: actions/cache@v3 @@ -41,6 +41,6 @@ jobs: - name: run test run: > bazel run //conformance:conformance_test_runner -- - --failure_list "${{ github.workspace }}/this/node/libs/schema/src/bin/conformance_test_failure_list.txt" + --failure_list "${{ github.workspace }}/this/node/libs/protobuf/src/bin/conformance_test/failure_list.txt" "${{ github.workspace }}/this/node/target/debug/conformance_test" working-directory: "protobuf" diff --git a/node/Cargo.lock b/node/Cargo.lock index 920b83f9..8896fef8 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -301,6 +301,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bindgen" version = "0.65.1" @@ -313,7 +319,7 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease 0.2.12", + "prettyplease", "proc-macro2", "quote", "regex", @@ -1056,9 +1062,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] @@ -1160,6 +1166,38 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn 2.0.32", +] + +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lz4-sys" version = "1.9.4" @@ -1194,6 +1232,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1494,16 +1555,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.12" @@ -1548,9 +1599,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" dependencies = [ "bytes", "prost-derive", @@ -1558,134 +1609,91 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck", "itertools", - "lazy_static", "log", "multimap", + "once_cell", "petgraph", - "prettyplease 0.1.25", + "prettyplease", "prost", "prost-types", "regex", - "syn 1.0.109", + "syn 2.0.32", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] name = "prost-reflect" -version = "0.11.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "000e1e05ebf7b26e1eba298e66fe4eee6eb19c567d0ffb35e0dd34231cdac4c8" +checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" dependencies = [ "base64", + "logos", + "miette", "once_cell", "prost", - "prost-reflect-derive", "prost-types", "serde", "serde-value", ] [[package]] -name = "prost-reflect-build" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d959f576df574d088e0409c45ef11897f64727158c554ce65edc2d345f8afcf" -dependencies = [ - "prost-build", - "prost-reflect", -] - -[[package]] -name = "prost-reflect-derive" -version = "0.11.0" +name = "prost-types" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7718375aa8f966df66e583b608a305a45bc87eeb1ffd5db87fae673bea17a7e4" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", + "prost", ] [[package]] -name = "prost-types" -version = "0.11.9" +name = "protox" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "66eb3a834c1ffe362107daab84dd87cfc1e1d2beda30e2eb8e4801f262839364" dependencies = [ + "bytes", + "miette", "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", ] [[package]] -name = "protoc-bin-vendored" -version = "3.0.0" +name = "protox-parse" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d" +checksum = "7b4581f441c58863525a3e6bec7b8de98188cf75239a56c725a3e7288450a33f" dependencies = [ - "protoc-bin-vendored-linux-aarch_64", - "protoc-bin-vendored-linux-ppcle_64", - "protoc-bin-vendored-linux-x86_32", - "protoc-bin-vendored-linux-x86_64", - "protoc-bin-vendored-macos-x86_64", - "protoc-bin-vendored-win32", + "logos", + "miette", + "prost-types", + "thiserror", ] -[[package]] -name = "protoc-bin-vendored-linux-aarch_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435" - -[[package]] -name = "protoc-bin-vendored-linux-ppcle_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516" - -[[package]] -name = "protoc-bin-vendored-linux-x86_32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0" - -[[package]] -name = "protoc-bin-vendored-linux-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924" - -[[package]] -name = "protoc-bin-vendored-macos-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537" - -[[package]] -name = "protoc-bin-vendored-win32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" - [[package]] name = "quick-protobuf" version = "0.8.1" @@ -2286,6 +2294,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "universal-hash" version = "0.4.0" @@ -2381,9 +2395,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2391,9 +2405,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", @@ -2406,9 +2420,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2416,9 +2430,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", @@ -2429,15 +2443,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" dependencies = [ "js-sys", "wasm-bindgen", @@ -2559,18 +2573,18 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.15" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ba595b9f2772fbee2312de30eeb80ec773b4cb2f1e8098db024afadda6c06f" +checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.15" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772666c41fb6dceaf520b564b962d738a8e1a83b41bd48945f50837aed78bb1d" +checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" dependencies = [ "proc-macro2", "quote", @@ -2632,6 +2646,7 @@ dependencies = [ "zksync_consensus_schema", "zksync_consensus_storage", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2671,6 +2686,7 @@ dependencies = [ "zksync_consensus_storage", "zksync_consensus_sync_blocks", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2695,6 +2711,7 @@ dependencies = [ "zksync_consensus_roles", "zksync_consensus_schema", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2711,6 +2728,7 @@ dependencies = [ "zksync_consensus_crypto", "zksync_consensus_schema", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2720,19 +2738,13 @@ dependencies = [ "anyhow", "bit-vec", "once_cell", - "prettyplease 0.2.12", "prost", - "prost-build", - "prost-reflect", - "prost-reflect-build", - "protoc-bin-vendored", - "quick-protobuf", "rand", "serde", "serde_json", - "syn 2.0.32", "tokio", "zksync_concurrency", + "zksync_protobuf", ] [[package]] @@ -2752,6 +2764,7 @@ dependencies = [ "zksync_concurrency", "zksync_consensus_roles", "zksync_consensus_schema", + "zksync_protobuf", ] [[package]] @@ -2792,6 +2805,7 @@ dependencies = [ "zksync_consensus_schema", "zksync_consensus_storage", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2802,6 +2816,45 @@ dependencies = [ "zksync_concurrency", ] +[[package]] +name = "zksync_protobuf" +version = "0.1.0" +dependencies = [ + "anyhow", + "bit-vec", + "once_cell", + "prost", + "prost-reflect", + "quick-protobuf", + "rand", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "zksync_concurrency", + "zksync_protobuf_build", +] + +[[package]] +name = "zksync_protobuf_build" +version = "0.1.0" +dependencies = [ + "anyhow", + "heck", + "once_cell", + "prettyplease", + "prost", + "prost-build", + "prost-reflect", + "prost-types", + "protox", + "protox-parse", + "quote", + "rand", + "syn 2.0.32", +] + [[package]] name = "zstd-sys" version = "2.0.8+zstd.1.5.5" diff --git a/node/Cargo.toml b/node/Cargo.toml index 6e2b1185..8dc3d506 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,8 @@ [workspace] members = [ "libs/concurrency", + "libs/protobuf_build", + "libs/protobuf", "actors/bft", "libs/crypto", "actors/executor", @@ -21,6 +23,8 @@ homepage = "https://matter-labs.io/" license = "MIT" [workspace.dependencies] +zksync_protobuf_build = { path = "libs/protobuf_build" } +zksync_protobuf = { path = "libs/protobuf" } zksync_concurrency = { path = "libs/concurrency" } zksync_consensus_bft = { path = "actors/bft" } zksync_consensus_crypto = { path = "libs/crypto" } @@ -43,19 +47,22 @@ ark-ec = "0.4.2" ark-serialize = { version = "0.4.2", features = ["std"] } num-traits = "0.2.17" clap = { version = "4.3.3", features = ["derive"] } +heck = "0.4.1" ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } hex = "0.4.3" im = "15.1.0" once_cell = "1.17.1" pin-project = "1.1.0" -prost = "0.11.0" -prost-build = "0.11.0" -prost-reflect = { version = "0.11.0", features = ["derive", "serde"] } -prost-reflect-build = "0.11.0" -protoc-bin-vendored = "3.0.0" +prost = "0.12.0" +prost-build = "0.12.0" +prost-reflect = { version = "0.12.0", features = ["serde"] } +prost-types = "0.12.0" +protox = "0.5.0" +protox-parse = "0.5.0" prettyplease = "0.2.6" pretty_assertions = "1.4.0" quick-protobuf = "0.8.1" +quote = "1.0.33" rand = "0.8.0" rocksdb = "0.21.0" serde = { version = "1.0", features = ["derive"] } diff --git a/node/actors/bft/Cargo.toml b/node/actors/bft/Cargo.toml index 5f61f2cf..db42209d 100644 --- a/node/actors/bft/Cargo.toml +++ b/node/actors/bft/Cargo.toml @@ -14,6 +14,7 @@ zksync_consensus_roles.workspace = true zksync_consensus_schema.workspace = true zksync_consensus_storage.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true once_cell.workspace = true diff --git a/node/actors/bft/src/inner.rs b/node/actors/bft/src/inner.rs index 56f8d57a..631e90d2 100644 --- a/node/actors/bft/src/inner.rs +++ b/node/actors/bft/src/inner.rs @@ -6,7 +6,6 @@ use crate::{ }; use tracing::instrument; use zksync_consensus_roles::validator; -use zksync_consensus_schema as schema; use zksync_consensus_utils::pipe::ActorPipe; /// The ConsensusInner struct, it contains data to be shared with the state machines. This is never supposed @@ -24,7 +23,7 @@ pub(crate) struct ConsensusInner { impl ConsensusInner { /// The maximum size of the payload of a block, in bytes. We will /// reject blocks with payloads larger than this. - pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * schema::kB; + pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * zksync_protobuf::kB; /// Computes the validator for the given view. #[instrument(level = "trace", ret)] diff --git a/node/actors/bft/src/metrics.rs b/node/actors/bft/src/metrics.rs index 461403f7..8fef767e 100644 --- a/node/actors/bft/src/metrics.rs +++ b/node/actors/bft/src/metrics.rs @@ -2,10 +2,11 @@ use std::time::Duration; use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; -use zksync_consensus_schema as schema; -const PAYLOAD_SIZE_BUCKETS: Buckets = - Buckets::exponential((4 * schema::kB) as f64..=(4 * schema::MB) as f64, 4.0); +const PAYLOAD_SIZE_BUCKETS: Buckets = Buckets::exponential( + (4 * zksync_protobuf::kB) as f64..=(4 * zksync_protobuf::MB) as f64, + 4.0, +); /// Label for a consensus message. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index 9bbe7188..9f94353e 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -16,6 +16,7 @@ zksync_consensus_schema.workspace = true zksync_consensus_storage.workspace = true zksync_consensus_sync_blocks.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true rand.workspace = true diff --git a/node/actors/executor/src/config/mod.rs b/node/actors/executor/src/config/mod.rs index 4ab296fa..0ebfab55 100644 --- a/node/actors/executor/src/config/mod.rs +++ b/node/actors/executor/src/config/mod.rs @@ -1,5 +1,4 @@ //! Module to create the configuration for the consensus node. - use anyhow::Context as _; use std::{ collections::{HashMap, HashSet}, @@ -8,9 +7,8 @@ use std::{ use zksync_consensus_crypto::{read_required_text, Text, TextFmt}; use zksync_consensus_network::{consensus, gossip}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::{ - proto::executor::config as proto, read_required, required, ProtoFmt, -}; +use zksync_consensus_schema::proto::executor::config as proto; +use zksync_protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod tests; diff --git a/node/actors/executor/src/config/tests.rs b/node/actors/executor/src/config/tests.rs index 7b1ab797..3c18f072 100644 --- a/node/actors/executor/src/config/tests.rs +++ b/node/actors/executor/src/config/tests.rs @@ -5,7 +5,7 @@ use rand::{ }; use zksync_concurrency::ctx; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; fn make_addr(rng: &mut R) -> std::net::SocketAddr { std::net::SocketAddr::new(std::net::IpAddr::from(rng.gen::<[u8; 16]>()), rng.gen()) diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index 30fd0e9a..20c85bcb 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -12,6 +12,7 @@ zksync_consensus_crypto.workspace = true zksync_consensus_roles.workspace = true zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true async-trait.workspace = true diff --git a/node/actors/network/src/consensus/handshake/mod.rs b/node/actors/network/src/consensus/handshake/mod.rs index fa304488..65275cc6 100644 --- a/node/actors/network/src/consensus/handshake/mod.rs +++ b/node/actors/network/src/consensus/handshake/mod.rs @@ -3,7 +3,8 @@ use anyhow::Context as _; use zksync_concurrency::{ctx, time}; use zksync_consensus_crypto::ByteFmt; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::{proto::network::consensus as proto, read_required, ProtoFmt}; +use zksync_consensus_schema::proto::network::consensus as proto; +use zksync_protobuf::{read_required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/consensus/handshake/tests.rs b/node/actors/network/src/consensus/handshake/tests.rs index ed49c14d..747844fe 100644 --- a/node/actors/network/src/consensus/handshake/tests.rs +++ b/node/actors/network/src/consensus/handshake/tests.rs @@ -3,7 +3,7 @@ use crate::{frame, noise, testonly}; use rand::Rng; use zksync_concurrency::{ctx, io, scope, testonly::abort_on_panic}; use zksync_consensus_roles::validator; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_schema_encode_decode() { diff --git a/node/actors/network/src/frame.rs b/node/actors/network/src/frame.rs index f865be95..c20969f5 100644 --- a/node/actors/network/src/frame.rs +++ b/node/actors/network/src/frame.rs @@ -2,13 +2,12 @@ //! since protobuf messages do not have delimiters. use crate::{mux, noise::bytes}; use zksync_concurrency::{ctx, io}; -use zksync_consensus_schema as schema; /// Reads a raw frame of bytes from the stream and interprets it as proto. /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. /// Returns the decoded proto and the size of the received message in bytes. -pub(crate) async fn mux_recv_proto( +pub(crate) async fn mux_recv_proto( ctx: &ctx::Ctx, stream: &mut mux::ReadStream, ) -> anyhow::Result<(T, usize)> { @@ -26,19 +25,19 @@ pub(crate) async fn mux_recv_proto( if msg.len() < msg_size { anyhow::bail!("end of stream"); } - let msg = schema::decode(msg.as_slice())?; + let msg = zksync_protobuf::decode(msg.as_slice())?; Ok((msg, msg_size)) } /// Sends a proto serialized to a raw frame of bytes to the stream. /// It doesn't flush the stream. /// Returns the size of the sent proto in bytes. -pub(crate) async fn mux_send_proto( +pub(crate) async fn mux_send_proto( ctx: &ctx::Ctx, stream: &mut mux::WriteStream, msg: &T, ) -> anyhow::Result { - let msg = schema::encode(msg); + let msg = zksync_protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); stream .write_all(ctx, &u32::to_le_bytes(msg.len() as u32)) @@ -50,7 +49,7 @@ pub(crate) async fn mux_send_proto( /// Reads a raw frame of bytes from the stream and interprets it as proto. /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. -pub(crate) async fn recv_proto( +pub(crate) async fn recv_proto( ctx: &ctx::Ctx, stream: &mut S, ) -> anyhow::Result { @@ -62,16 +61,16 @@ pub(crate) async fn recv_proto( } let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stream, &mut msg[..]).await??; - schema::decode(&msg) + zksync_protobuf::decode(&msg) } /// Sends a proto serialized to a raw frame of bytes to the stream. -pub(crate) async fn send_proto( +pub(crate) async fn send_proto( ctx: &ctx::Ctx, stream: &mut S, msg: &T, ) -> anyhow::Result<()> { - let msg = schema::encode(msg); + let msg = zksync_protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); io::write_all(ctx, stream, &u32::to_le_bytes(msg.len() as u32)).await??; io::write_all(ctx, stream, &msg).await??; diff --git a/node/actors/network/src/gossip/handshake/mod.rs b/node/actors/network/src/gossip/handshake/mod.rs index 9114ac0d..45558c53 100644 --- a/node/actors/network/src/gossip/handshake/mod.rs +++ b/node/actors/network/src/gossip/handshake/mod.rs @@ -4,7 +4,8 @@ use anyhow::Context as _; use zksync_concurrency::{ctx, time}; use zksync_consensus_crypto::ByteFmt; use zksync_consensus_roles::node; -use zksync_consensus_schema::{proto::network::gossip as proto, read_required, required, ProtoFmt}; +use zksync_consensus_schema::proto::network::gossip as proto; +use zksync_protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/gossip/handshake/tests.rs b/node/actors/network/src/gossip/handshake/tests.rs index 39c26457..457eadd3 100644 --- a/node/actors/network/src/gossip/handshake/tests.rs +++ b/node/actors/network/src/gossip/handshake/tests.rs @@ -4,7 +4,7 @@ use rand::Rng; use std::collections::{HashMap, HashSet}; use zksync_concurrency::{ctx, io, scope, testonly::abort_on_panic}; use zksync_consensus_roles::node; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_schema_encode_decode() { diff --git a/node/actors/network/src/mux/handshake.rs b/node/actors/network/src/mux/handshake.rs index 83ac6a52..bfee19db 100644 --- a/node/actors/network/src/mux/handshake.rs +++ b/node/actors/network/src/mux/handshake.rs @@ -1,8 +1,8 @@ use super::CapabilityId; use anyhow::Context as _; use std::collections::HashMap; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::mux as proto, required}; +use zksync_consensus_schema::proto::network::mux as proto; +use zksync_protobuf::required; pub(super) struct Handshake { /// Maximal supported number of the accept streams per capability. @@ -37,11 +37,11 @@ fn build_capabilities( .collect() } -impl schema::ProtoFmt for Handshake { +impl zksync_protobuf::ProtoFmt for Handshake { type Proto = proto::Handshake; fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { diff --git a/node/actors/network/src/mux/tests.rs b/node/actors/network/src/mux/tests.rs index 19a82458..2e334964 100644 --- a/node/actors/network/src/mux/tests.rs +++ b/node/actors/network/src/mux/tests.rs @@ -9,7 +9,6 @@ use std::{ }, }; use zksync_concurrency::{ctx, scope, testonly::abort_on_panic}; -use zksync_consensus_schema as schema; use zksync_consensus_schema::proto::network::mux_test as proto; use zksync_consensus_utils::no_copy::NoCopy; @@ -73,7 +72,7 @@ struct Resp { capability_id: mux::CapabilityId, } -impl schema::ProtoFmt for Req { +impl zksync_protobuf::ProtoFmt for Req { type Proto = proto::Req; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(r.input.as_ref().unwrap().clone())) @@ -85,7 +84,7 @@ impl schema::ProtoFmt for Req { } } -impl schema::ProtoFmt for Resp { +impl zksync_protobuf::ProtoFmt for Resp { type Proto = proto::Resp; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { diff --git a/node/actors/network/src/preface.rs b/node/actors/network/src/preface.rs index 5cb53ac9..75a042c1 100644 --- a/node/actors/network/src/preface.rs +++ b/node/actors/network/src/preface.rs @@ -9,8 +9,8 @@ //! and multiplex between mutliple endpoints available on the same TCP port. use crate::{frame, metrics, noise}; use zksync_concurrency::{ctx, time}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::preface as proto, required, ProtoFmt}; +use zksync_consensus_schema::proto::network::preface as proto; +use zksync_protobuf::{required, ProtoFmt}; /// Timeout on executing the preface protocol. const TIMEOUT: time::Duration = time::Duration::seconds(5); @@ -34,7 +34,7 @@ pub(crate) enum Endpoint { impl ProtoFmt for Encryption { type Proto = proto::Encryption; fn max_size() -> usize { - 10 * schema::kB + 10 * zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::encryption::T; @@ -54,7 +54,7 @@ impl ProtoFmt for Encryption { impl ProtoFmt for Endpoint { type Proto = proto::Endpoint; fn max_size() -> usize { - 10 * schema::kB + 10 * zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::endpoint::T; diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 86365eb0..90a20b6e 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -2,8 +2,8 @@ use crate::mux; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::consensus as proto, read_required, ProtoFmt}; +use zksync_consensus_schema::proto::network::consensus as proto; +use zksync_protobuf::{read_required, ProtoFmt}; /// Consensus RPC. pub(crate) struct Rpc; @@ -46,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - schema::MB + zksync_protobuf::MB } } @@ -62,6 +62,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } diff --git a/node/actors/network/src/rpc/metrics.rs b/node/actors/network/src/rpc/metrics.rs index a9111cec..34394e28 100644 --- a/node/actors/network/src/rpc/metrics.rs +++ b/node/actors/network/src/rpc/metrics.rs @@ -6,7 +6,6 @@ use vise::{ Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, LabeledFamily, Metrics, Unit, }; -use zksync_consensus_schema as schema; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] #[metrics(rename_all = "snake_case")] @@ -87,7 +86,7 @@ pub(super) struct CallLabels { } const MESSAGE_SIZE_BUCKETS: Buckets = - Buckets::exponential(schema::kB as f64..=schema::MB as f64, 2.0); + Buckets::exponential(zksync_protobuf::kB as f64..=zksync_protobuf::MB as f64, 2.0); #[derive(Debug, Metrics)] #[metrics(prefix = "network_rpc")] diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index b3173821..77f9a52a 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -20,7 +20,6 @@ use crate::{frame, mux}; use anyhow::Context as _; use std::{collections::BTreeMap, sync::Arc}; use zksync_concurrency::{ctx, io, limiter, metrics::LatencyHistogramExt as _, scope}; -use zksync_consensus_schema as schema; pub(crate) mod consensus; mod metrics; @@ -33,10 +32,10 @@ pub(crate) mod testonly; mod tests; const MUX_CONFIG: mux::Config = mux::Config { - read_buffer_size: 160 * schema::kB as u64, - read_frame_size: 16 * schema::kB as u64, + read_buffer_size: 160 * zksync_protobuf::kB as u64, + read_frame_size: 16 * zksync_protobuf::kB as u64, read_frame_count: 100, - write_frame_size: 16 * schema::kB as u64, + write_frame_size: 16 * zksync_protobuf::kB as u64, }; /// Trait for defining an RPC. @@ -58,9 +57,9 @@ pub(crate) trait Rpc: Sync + Send + 'static { /// Name of the RPC, used in prometheus metrics. const METHOD: &'static str; /// Type of the request message. - type Req: schema::ProtoFmt + Send + Sync; + type Req: zksync_protobuf::ProtoFmt + Send + Sync; /// Type of the response message. - type Resp: schema::ProtoFmt + Send + Sync; + type Resp: zksync_protobuf::ProtoFmt + Send + Sync; /// Name of the variant of the request message type. /// Useful for collecting metrics with finer than /// per-rpc type granularity. diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index 98a292bc..fda4ebf8 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -3,8 +3,8 @@ use crate::{mux, rpc::Rpc as _}; use anyhow::Context as _; use rand::Rng; use zksync_concurrency::{ctx, limiter, time}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::ping as proto, required, ProtoFmt}; +use zksync_consensus_schema::proto::network::ping as proto; +use zksync_protobuf::{required, ProtoFmt}; /// Ping RPC. pub(crate) struct Rpc; @@ -65,7 +65,7 @@ pub(crate) struct Resp(pub(crate) [u8; 32]); impl ProtoFmt for Req { type Proto = proto::PingReq; fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) @@ -80,7 +80,7 @@ impl ProtoFmt for Req { impl ProtoFmt for Resp { type Proto = proto::PingResp; fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index 26454b53..4dd22af7 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -1,11 +1,10 @@ //! Defines RPC for synchronizing blocks. - use crate::{io, mux}; use anyhow::Context; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::gossip as proto, read_required, ProtoFmt}; +use zksync_consensus_schema::proto::network::gossip as proto; +use zksync_protobuf::{read_required, ProtoFmt}; /// `get_sync_state` RPC. #[derive(Debug)] @@ -48,7 +47,7 @@ impl ProtoFmt for io::SyncState { fn max_size() -> usize { // TODO: estimate maximum size more precisely - 100 * schema::kB + 100 * zksync_protobuf::kB } } @@ -68,7 +67,7 @@ impl ProtoFmt for SyncStateResponse { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -110,7 +109,7 @@ impl ProtoFmt for GetBlockRequest { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -121,7 +120,7 @@ impl ProtoFmt for io::GetBlockError { use proto::get_block_response::ErrorReason; let reason = message.reason.context("missing reason")?; - let reason = ErrorReason::from_i32(reason).context("reason")?; + let reason = ErrorReason::try_from(reason).context("reason")?; Ok(match reason { ErrorReason::NotSynced => Self::NotSynced, }) @@ -138,7 +137,7 @@ impl ProtoFmt for io::GetBlockError { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -179,6 +178,6 @@ impl ProtoFmt for GetBlockResponse { } fn max_size() -> usize { - schema::MB + zksync_protobuf::MB } } diff --git a/node/actors/network/src/rpc/sync_validator_addrs.rs b/node/actors/network/src/rpc/sync_validator_addrs.rs index 587c4dbd..d505feda 100644 --- a/node/actors/network/src/rpc/sync_validator_addrs.rs +++ b/node/actors/network/src/rpc/sync_validator_addrs.rs @@ -4,8 +4,8 @@ use anyhow::Context as _; use std::sync::Arc; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::gossip as proto, ProtoFmt}; +use zksync_consensus_schema::proto::network::gossip as proto; +use zksync_protobuf::ProtoFmt; /// SyncValidatorAddrs Rpc. pub(crate) struct Rpc; @@ -45,7 +45,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -69,6 +69,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - schema::MB + zksync_protobuf::MB } } diff --git a/node/actors/network/src/rpc/tests.rs b/node/actors/network/src/rpc/tests.rs index d2e98851..04f28c2a 100644 --- a/node/actors/network/src/rpc/tests.rs +++ b/node/actors/network/src/rpc/tests.rs @@ -6,7 +6,7 @@ use std::{ sync::atomic::{AtomicU64, Ordering}, }; use zksync_concurrency::{ctx, testonly::abort_on_panic, time}; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; /// CAPABILITY_ID should uniquely identify the RPC. #[test] diff --git a/node/deny.toml b/node/deny.toml index 6fdd9bb5..8bfe4c74 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -47,10 +47,6 @@ allow = [ multiple-versions = "deny" # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ - # Old versions required by prost. - { name = "prettyplease", version = "=0.1.25" }, - { name = "syn", version = "=1.0.109" }, - # Old versions required by tempfile and prost-build. { name = "bitflags", version = "=1.3.2" }, @@ -61,6 +57,9 @@ skip = [ # Old versions required by hyper. { name = "socket2", version = "=0.4.9" }, { name = "hashbrown", version = "=0.12.3" }, # (hyper -> h2 -> indexmap -> hashbrown) + + # Old versions required by ark-bn254. + { name = "syn", version = "=1.0.109" } ] [sources] diff --git a/node/libs/protobuf/Cargo.toml b/node/libs/protobuf/Cargo.toml new file mode 100644 index 00000000..83eba563 --- /dev/null +++ b/node/libs/protobuf/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "zksync_protobuf" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[[bin]] +name = "conformance_test" + +[dependencies] +zksync_concurrency.workspace = true +zksync_protobuf_build.workspace = true + +anyhow.workspace = true +bit-vec.workspace = true +serde.workspace = true +quick-protobuf.workspace = true +once_cell.workspace = true +prost.workspace = true +prost-reflect.workspace = true +rand.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[build-dependencies] +zksync_protobuf_build.workspace = true + +anyhow.workspace = true diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs new file mode 100644 index 00000000..0348639b --- /dev/null +++ b/node/libs/protobuf/build.rs @@ -0,0 +1,29 @@ +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf_build::Config { + input_root: "src/proto".into(), + proto_root: "zksync".into(), + dependencies: vec![], + protobuf_crate: "crate".into(), + } + .generate() + .expect("generate(std)"); + + zksync_protobuf_build::Config { + input_root: "src/tests/proto".into(), + proto_root: "zksync/protobuf/tests".into(), + dependencies: vec![], + protobuf_crate: "crate".into(), + } + .generate() + .expect("generate(test)"); + + zksync_protobuf_build::Config { + input_root: "src/bin/conformance_test/proto".into(), + proto_root: "zksync/protobuf/conformance_test".into(), + dependencies: vec![], + protobuf_crate: "::zksync_protobuf".into(), + } + .generate() + .expect("generate(conformance)"); +} diff --git a/node/libs/schema/src/bin/conformance_test_failure_list.txt b/node/libs/protobuf/src/bin/conformance_test/failure_list.txt similarity index 100% rename from node/libs/schema/src/bin/conformance_test_failure_list.txt rename to node/libs/protobuf/src/bin/conformance_test/failure_list.txt diff --git a/node/libs/schema/src/bin/conformance_test.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs similarity index 71% rename from node/libs/schema/src/bin/conformance_test.rs rename to node/libs/protobuf/src/bin/conformance_test/main.rs index 38b9d1bb..4d3a48a3 100644 --- a/node/libs/schema/src/bin/conformance_test.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -1,19 +1,20 @@ //! Conformance test for our canonical encoding implemented according to //! https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto //! Our implementation supports only a subset of proto functionality, so -//! `schema/proto/conformance/conformance.proto` and -//! `schema/proto/conformance/protobuf_test_messages.proto` contains only a +//! `proto/conformance.proto` and +//! `proto/protobuf_test_messages.proto` contains only a //! subset of original fields. Also we run only proto3 binary -> binary tests. -//! conformance_test_failure_list.txt contains tests which are expected to fail. +//! failure_list.txt contains tests which are expected to fail. use anyhow::Context as _; use prost::Message as _; use prost_reflect::ReflectMessage; +use std::sync::Mutex; use zksync_concurrency::{ctx, io}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::proto::conformance as proto; -#[tokio::main] -async fn main() -> anyhow::Result<()> { +mod proto; + +/// Runs the test server. +async fn run() -> anyhow::Result<()> { let ctx = &ctx::root(); let stdin = &mut tokio::io::stdin(); let stdout = &mut tokio::io::stdout(); @@ -37,10 +38,10 @@ async fn main() -> anyhow::Result<()> { // Decode. let payload = req.payload.context("missing payload")?; - use zksync_consensus_schema::proto::protobuf_test_messages::proto3::TestAllTypesProto3 as T; + use proto::TestAllTypesProto3 as T; let p = match payload { proto::conformance_request::Payload::JsonPayload(payload) => { - match schema::decode_json_proto(&payload) { + match zksync_protobuf::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } @@ -51,7 +52,7 @@ async fn main() -> anyhow::Result<()> { return Ok(R::ParseError("parsing failed".to_string())); }; // Then check if there are any unknown fields in the original payload. - if schema::canonical_raw(&payload[..], &p.descriptor()).is_err() { + if zksync_protobuf::canonical_raw(&payload[..], &p.descriptor()).is_err() { return Ok(R::Skipped("unsupported fields".to_string())); } p @@ -63,13 +64,13 @@ async fn main() -> anyhow::Result<()> { let format = req .requested_output_format .context("missing output format")?; - match proto::WireFormat::from_i32(format).context("unknown format")? { + match proto::WireFormat::try_from(format).context("unknown format")? { proto::WireFormat::Json => { - anyhow::Ok(R::JsonPayload(schema::encode_json_proto(&p))) + anyhow::Ok(R::JsonPayload(zksync_protobuf::encode_json_proto(&p))) } proto::WireFormat::Protobuf => { // Reencode the parsed proto. - anyhow::Ok(R::ProtobufPayload(schema::canonical_raw( + anyhow::Ok(R::ProtobufPayload(zksync_protobuf::canonical_raw( &p.encode_to_vec(), &p.descriptor(), )?)) @@ -87,3 +88,24 @@ async fn main() -> anyhow::Result<()> { io::flush(ctx, stdout).await??; } } + +#[tokio::main] +async fn main() { + let sub = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()); + match std::env::var("LOG_FILE") { + Err(_) => sub.with_writer(std::io::stderr).init(), + Ok(path) => sub + .with_writer(Mutex::new( + std::fs::File::options() + .create(true) + .append(true) + .open(path) + .unwrap(), + )) + .init(), + }; + if let Err(err) = run().await { + tracing::error!("run(): {err:#}"); + } +} diff --git a/node/libs/schema/proto/conformance/conformance.proto b/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto similarity index 98% rename from node/libs/schema/proto/conformance/conformance.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto index e6403673..819bb61d 100644 --- a/node/libs/schema/proto/conformance/conformance.proto +++ b/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto @@ -33,10 +33,7 @@ syntax = "proto3"; -package conformance; - -option java_package = "com.google.protobuf.conformance"; -option objc_class_prefix = "Conformance"; +package zksync.protobuf.conformance_test; // This defines the conformance testing protocol. This protocol exists between // the conformance test suite itself and the code being tested. For each test, diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs new file mode 100644 index 00000000..70249255 --- /dev/null +++ b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs @@ -0,0 +1,5 @@ +#![allow(warnings)] +include!(concat!( + env!("OUT_DIR"), + "/src/bin/conformance_test/proto/gen.rs" +)); diff --git a/node/libs/schema/proto/conformance/test_messages_proto3.proto b/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto similarity index 96% rename from node/libs/schema/proto/conformance/test_messages_proto3.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto index d391761b..242d8f58 100644 --- a/node/libs/schema/proto/conformance/test_messages_proto3.proto +++ b/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto @@ -40,15 +40,7 @@ syntax = "proto3"; -package protobuf_test_messages.proto3; - -option java_package = "com.google.protobuf_test_messages.proto3"; -option objc_class_prefix = "Proto3"; - -// This is the default, but we specify it here explicitly. -option optimize_for = SPEED; - -option cc_enable_arenas = true; +package zksync.protobuf.conformance_test; // This proto includes every type of field in both singular and repeated // forms. diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs new file mode 100644 index 00000000..f461892a --- /dev/null +++ b/node/libs/protobuf/src/lib.rs @@ -0,0 +1,13 @@ +//! Code generated from protobuf schema files and +//! utilities for serialization. + +pub mod proto; +mod proto_fmt; +mod std_conv; +pub mod testonly; + +pub use proto_fmt::*; +pub use zksync_protobuf_build as build; + +#[cfg(test)] +mod tests; diff --git a/node/libs/protobuf/src/proto/mod.rs b/node/libs/protobuf/src/proto/mod.rs new file mode 100644 index 00000000..660bf4c5 --- /dev/null +++ b/node/libs/protobuf/src/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs")); diff --git a/node/libs/schema/proto/std.proto b/node/libs/protobuf/src/proto/std.proto similarity index 98% rename from node/libs/schema/proto/std.proto rename to node/libs/protobuf/src/proto/std.proto index f5ee5efe..05b6957a 100644 --- a/node/libs/schema/proto/std.proto +++ b/node/libs/protobuf/src/proto/std.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package std; +package zksync.std; message Void {} diff --git a/node/libs/schema/src/proto_fmt.rs b/node/libs/protobuf/src/proto_fmt.rs similarity index 100% rename from node/libs/schema/src/proto_fmt.rs rename to node/libs/protobuf/src/proto_fmt.rs diff --git a/node/libs/schema/src/std_conv.rs b/node/libs/protobuf/src/std_conv.rs similarity index 92% rename from node/libs/schema/src/std_conv.rs rename to node/libs/protobuf/src/std_conv.rs index 264969b5..76ac4137 100644 --- a/node/libs/schema/src/std_conv.rs +++ b/node/libs/protobuf/src/std_conv.rs @@ -1,11 +1,11 @@ //! Proto conversion for messages in std package. -use crate::{proto::std as proto, required, ProtoFmt}; +use crate::{proto, required, ProtoFmt}; use anyhow::Context as _; use std::net; use zksync_concurrency::time; impl ProtoFmt for () { - type Proto = proto::Void; + type Proto = proto::std::Void; fn read(_r: &Self::Proto) -> anyhow::Result { Ok(()) } @@ -15,7 +15,7 @@ impl ProtoFmt for () { } impl ProtoFmt for std::net::SocketAddr { - type Proto = proto::SocketAddr; + type Proto = proto::std::SocketAddr; fn read(r: &Self::Proto) -> anyhow::Result { let ip = required(&r.ip).context("ip")?; @@ -41,7 +41,7 @@ impl ProtoFmt for std::net::SocketAddr { } impl ProtoFmt for time::Utc { - type Proto = proto::Timestamp; + type Proto = proto::std::Timestamp; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -59,7 +59,7 @@ impl ProtoFmt for time::Utc { } impl ProtoFmt for time::Duration { - type Proto = proto::Duration; + type Proto = proto::std::Duration; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -82,7 +82,7 @@ impl ProtoFmt for time::Duration { } impl ProtoFmt for bit_vec::BitVec { - type Proto = proto::BitVector; + type Proto = proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { let size = *required(&r.size).context("size")? as usize; diff --git a/node/libs/schema/src/testonly.rs b/node/libs/protobuf/src/testonly.rs similarity index 100% rename from node/libs/schema/src/testonly.rs rename to node/libs/protobuf/src/testonly.rs diff --git a/node/libs/schema/src/tests.rs b/node/libs/protobuf/src/tests/mod.rs similarity index 94% rename from node/libs/schema/src/tests.rs rename to node/libs/protobuf/src/tests/mod.rs index ddd49783..a41d6e45 100644 --- a/node/libs/schema/src/tests.rs +++ b/node/libs/protobuf/src/tests/mod.rs @@ -3,6 +3,8 @@ use anyhow::Context as _; use std::net; use zksync_concurrency::{ctx, time}; +mod proto; + #[derive(Debug, PartialEq, Eq)] enum B { U(bool), @@ -10,16 +12,16 @@ enum B { } impl ProtoFmt for B { - type Proto = proto::testonly::B; + type Proto = proto::B; fn read(r: &Self::Proto) -> anyhow::Result { - use proto::testonly::b::T; + use proto::b::T; Ok(match required(&r.t)? { T::U(x) => Self::U(*x), T::V(x) => Self::V(Box::new(ProtoFmt::read(x.as_ref())?)), }) } fn build(&self) -> Self::Proto { - use proto::testonly::b::T; + use proto::b::T; let t = match self { Self::U(x) => T::U(*x), Self::V(x) => T::V(Box::new(x.build())), @@ -37,7 +39,7 @@ struct A { } impl ProtoFmt for A { - type Proto = proto::testonly::A; + type Proto = proto::A; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { x: required(&r.x).context("x")?.clone(), diff --git a/node/libs/protobuf/src/tests/proto/mod.rs b/node/libs/protobuf/src/tests/proto/mod.rs new file mode 100644 index 00000000..a4703251 --- /dev/null +++ b/node/libs/protobuf/src/tests/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/tests/proto/gen.rs")); diff --git a/node/libs/schema/proto/testonly.proto b/node/libs/protobuf/src/tests/proto/tests.proto similarity index 91% rename from node/libs/schema/proto/testonly.proto rename to node/libs/protobuf/src/tests/proto/tests.proto index 7c5d08cd..91cf9e35 100644 --- a/node/libs/schema/proto/testonly.proto +++ b/node/libs/protobuf/src/tests/proto/tests.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package testonly; +package zksync.protobuf.tests; message B { // Recursive union. diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml new file mode 100644 index 00000000..46129be5 --- /dev/null +++ b/node/libs/protobuf_build/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "zksync_protobuf_build" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +heck.workspace = true +once_cell.workspace = true +prettyplease.workspace = true +prost.workspace = true +prost-build.workspace = true +prost-types.workspace = true +prost-reflect.workspace = true +protox.workspace = true +protox-parse.workspace = true +quote.workspace = true +rand.workspace = true +syn.workspace = true + diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs new file mode 100644 index 00000000..e283663e --- /dev/null +++ b/node/libs/protobuf_build/src/canonical.rs @@ -0,0 +1,58 @@ +//! Checks whether messages in the given file descriptor set support canonical encoding. +use crate::syntax::extract_message_names; +use anyhow::Context as _; +use std::collections::HashSet; + +#[derive(Default)] +struct Check(HashSet); + +impl Check { + /// Checks if messages of type `m` support canonical encoding. + fn check_message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { + if self.0.contains(m.full_name()) { + return Ok(()); + } + self.0.insert(m.full_name().to_string()); + for f in m.fields() { + self.check_field(&f).with_context(|| f.name().to_string())?; + } + Ok(()) + } + + /// Checks if field `f` supports canonical encoding. + fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + if f.is_map() { + anyhow::bail!("maps unsupported"); + } + if !f.is_list() && !f.supports_presence() { + anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); + } + if let prost_reflect::Kind::Message(msg) = &f.kind() { + self.check_message(msg) + .with_context(|| msg.name().to_string())?; + } + Ok(()) + } +} + +/// Checks whether messages in the given file descriptor set support canonical encoding. +/// pool should contain all transitive dependencies of files in descriptor. +pub(crate) fn check( + descriptor: &prost_types::FileDescriptorSet, + pool: &prost_reflect::DescriptorPool, +) -> anyhow::Result<()> { + for f in &descriptor.file { + if f.syntax() != "proto3" { + anyhow::bail!("{}: only proto3 syntax is supported", f.name()); + } + } + let mut c = Check::default(); + for msg_name in extract_message_names(descriptor) { + let msg_name = msg_name.to_string(); + let msg = pool + .get_message_by_name(&msg_name) + .with_context(|| format!("{msg_name} not found in pool"))?; + c.check_message(&msg).with_context(|| msg_name)?; + } + Ok(()) +} diff --git a/node/libs/protobuf_build/src/ident.rs b/node/libs/protobuf_build/src/ident.rs new file mode 100644 index 00000000..9dc4e003 --- /dev/null +++ b/node/libs/protobuf_build/src/ident.rs @@ -0,0 +1,163 @@ +//! Copy of https://github.com/tokio-rs/prost/blob/master/prost-build/src/ident.rs. +//! We need it to compute the proto names -> rust names mapping, which is not directly accessible +//! via prost_build public API. + +use heck::{ToSnakeCase, ToUpperCamelCase}; + +/// Converts a `camelCase` or `SCREAMING_SNAKE_CASE` identifier to a `lower_snake` case Rust field +/// identifier. +pub(crate) fn to_snake(s: &str) -> String { + let mut ident = s.to_snake_case(); + + // Use a raw identifier if the identifier matches a Rust keyword: + // https://doc.rust-lang.org/reference/keywords.html. + match ident.as_str() { + // 2015 strict keywords. + | "as" | "break" | "const" | "continue" | "else" | "enum" | "false" + | "fn" | "for" | "if" | "impl" | "in" | "let" | "loop" | "match" | "mod" | "move" | "mut" + | "pub" | "ref" | "return" | "static" | "struct" | "trait" | "true" + | "type" | "unsafe" | "use" | "where" | "while" + // 2018 strict keywords. + | "dyn" + // 2015 reserved keywords. + | "abstract" | "become" | "box" | "do" | "final" | "macro" | "override" | "priv" | "typeof" + | "unsized" | "virtual" | "yield" + // 2018 reserved keywords. + | "async" | "await" | "try" => ident.insert_str(0, "r#"), + // the following keywords are not supported as raw identifiers and are therefore suffixed with an underscore. + "self" | "super" | "extern" | "crate" => ident += "_", + _ => (), + } + ident +} + +/// Converts a `snake_case` identifier to an `UpperCamel` case Rust type identifier. +pub(crate) fn to_upper_camel(s: &str) -> String { + let mut ident = s.to_upper_camel_case(); + + // Suffix an underscore for the `Self` Rust keyword as it is not allowed as raw identifier. + if ident == "Self" { + ident += "_"; + } + ident +} + +#[cfg(test)] +mod tests { + + #![allow(clippy::cognitive_complexity)] + + use super::*; + + #[test] + fn test_to_snake() { + assert_eq!("foo_bar", &to_snake("FooBar")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("xml_http_request", &to_snake("XMLHttpRequest")); + assert_eq!("r#while", &to_snake("While")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_BUSTER")); + assert_eq!("foo_bar_baz", &to_snake("foo_bar_baz")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_buster")); + assert_eq!("fuzz", &to_snake("_FUZZ")); + assert_eq!("fuzz", &to_snake("_fuzz")); + assert_eq!("fuzz", &to_snake("_Fuzz")); + assert_eq!("fuzz", &to_snake("FUZZ_")); + assert_eq!("fuzz", &to_snake("fuzz_")); + assert_eq!("fuzz", &to_snake("Fuzz_")); + assert_eq!("fuz_z", &to_snake("FuzZ_")); + + // From test_messages_proto3.proto. + assert_eq!("fieldname1", &to_snake("fieldname1")); + assert_eq!("field_name2", &to_snake("field_name2")); + assert_eq!("field_name3", &to_snake("_field_name3")); + assert_eq!("field_name4", &to_snake("field__name4_")); + assert_eq!("field0name5", &to_snake("field0name5")); + assert_eq!("field_0_name6", &to_snake("field_0_name6")); + assert_eq!("field_name7", &to_snake("fieldName7")); + assert_eq!("field_name8", &to_snake("FieldName8")); + assert_eq!("field_name9", &to_snake("field_Name9")); + assert_eq!("field_name10", &to_snake("Field_Name10")); + + assert_eq!("field_name11", &to_snake("FIELD_NAME11")); + assert_eq!("field_name12", &to_snake("FIELD_name12")); + assert_eq!("field_name13", &to_snake("__field_name13")); + assert_eq!("field_name14", &to_snake("__Field_name14")); + assert_eq!("field_name15", &to_snake("field__name15")); + assert_eq!("field_name16", &to_snake("field__Name16")); + assert_eq!("field_name17", &to_snake("field_name17__")); + assert_eq!("field_name18", &to_snake("Field_name18__")); + } + + #[test] + fn test_to_snake_raw_keyword() { + assert_eq!("r#as", &to_snake("as")); + assert_eq!("r#break", &to_snake("break")); + assert_eq!("r#const", &to_snake("const")); + assert_eq!("r#continue", &to_snake("continue")); + assert_eq!("r#else", &to_snake("else")); + assert_eq!("r#enum", &to_snake("enum")); + assert_eq!("r#false", &to_snake("false")); + assert_eq!("r#fn", &to_snake("fn")); + assert_eq!("r#for", &to_snake("for")); + assert_eq!("r#if", &to_snake("if")); + assert_eq!("r#impl", &to_snake("impl")); + assert_eq!("r#in", &to_snake("in")); + assert_eq!("r#let", &to_snake("let")); + assert_eq!("r#loop", &to_snake("loop")); + assert_eq!("r#match", &to_snake("match")); + assert_eq!("r#mod", &to_snake("mod")); + assert_eq!("r#move", &to_snake("move")); + assert_eq!("r#mut", &to_snake("mut")); + assert_eq!("r#pub", &to_snake("pub")); + assert_eq!("r#ref", &to_snake("ref")); + assert_eq!("r#return", &to_snake("return")); + assert_eq!("r#static", &to_snake("static")); + assert_eq!("r#struct", &to_snake("struct")); + assert_eq!("r#trait", &to_snake("trait")); + assert_eq!("r#true", &to_snake("true")); + assert_eq!("r#type", &to_snake("type")); + assert_eq!("r#unsafe", &to_snake("unsafe")); + assert_eq!("r#use", &to_snake("use")); + assert_eq!("r#where", &to_snake("where")); + assert_eq!("r#while", &to_snake("while")); + assert_eq!("r#dyn", &to_snake("dyn")); + assert_eq!("r#abstract", &to_snake("abstract")); + assert_eq!("r#become", &to_snake("become")); + assert_eq!("r#box", &to_snake("box")); + assert_eq!("r#do", &to_snake("do")); + assert_eq!("r#final", &to_snake("final")); + assert_eq!("r#macro", &to_snake("macro")); + assert_eq!("r#override", &to_snake("override")); + assert_eq!("r#priv", &to_snake("priv")); + assert_eq!("r#typeof", &to_snake("typeof")); + assert_eq!("r#unsized", &to_snake("unsized")); + assert_eq!("r#virtual", &to_snake("virtual")); + assert_eq!("r#yield", &to_snake("yield")); + assert_eq!("r#async", &to_snake("async")); + assert_eq!("r#await", &to_snake("await")); + assert_eq!("r#try", &to_snake("try")); + } + + #[test] + fn test_to_snake_non_raw_keyword() { + assert_eq!("self_", &to_snake("self")); + assert_eq!("super_", &to_snake("super")); + assert_eq!("extern_", &to_snake("extern")); + assert_eq!("crate_", &to_snake("crate")); + } + + #[test] + fn test_to_upper_camel() { + assert_eq!("", &to_upper_camel("")); + assert_eq!("F", &to_upper_camel("F")); + assert_eq!("Foo", &to_upper_camel("FOO")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR_")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR_")); + assert_eq!("FuzzBuster", &to_upper_camel("fuzzBuster")); + assert_eq!("FuzzBuster", &to_upper_camel("FuzzBuster")); + assert_eq!("Self_", &to_upper_camel("self")); + } +} diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs new file mode 100644 index 00000000..aa1048f3 --- /dev/null +++ b/node/libs/protobuf_build/src/lib.rs @@ -0,0 +1,297 @@ +//! Generates rust code from the proto files. +//! +//! Protobuf files are collected recursively from $CARGO_MANIFEST_DIR// directory. +//! Corresponding "cargo:rerun-if-changed=..." line is printed to stdout, so that +//! the build script running this function is rerun whenever proto files change. +//! A single rust file is generated and stored at $OUT_DIR//gen.rs file. +//! +//! Protobuf files are compiled to a protobuf descriptor stored at +//! $OUT_DIR//gen.binpb. +//! Additionally a "PROTOBUF_DESCRIPTOR=" line is printed to +//! stdout. This can be used to collect all the descriptors across the build as follows: +//! 1. Checkout the repo to a fresh directory and then run "cargo build --all-targets" +//! We need it fresh so that every crate containing protobufs has only one build in the +//! cargo cache. +//! 2. grep through all target/debug/build/*/output files to find all "PROTOBUF_DESCRIPTOR=..." +//! lines and merge the descriptor files by simply concatenating them. +//! Note that you can run this procedure for 2 revisions of the repo and look for breaking +//! changes by running "buf breaking --against " where before.binpb +//! and after.binpb are the concatenated descriptors from those 2 revisions. +//! +//! The proto files are not expected to be self-contained - to import proto files from +//! different crates you need to specify them as dependencies in the Config.dependencies. +//! It is not possible to depend on a different proto bundle within the same crate (because +//! these are being built simultaneously from the same build script). +#![allow(clippy::print_stdout)] +// Imports accessed from the generated code. +pub use self::syntax::*; +use anyhow::Context as _; +pub use once_cell::sync::Lazy; +pub use prost; +use prost::Message as _; +pub use prost_reflect; +use std::{fs, path::Path, sync::Mutex}; + +mod canonical; +mod ident; +mod syntax; + +/// Traverses all the files in a directory recursively. +fn traverse_files( + path: &Path, + f: &mut impl FnMut(&Path) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + if !path.is_dir() { + f(path).with_context(|| path.display().to_string())?; + return Ok(()); + } + for entry in fs::read_dir(path)? { + traverse_files(&entry?.path(), f)?; + } + Ok(()) +} + +/// Protobuf descriptor + info about the mapping to rust code. +pub struct Descriptor { + /// Root proto package that all proto files in this descriptor belong to. + /// Rust types have been generated relative to this root. + proto_root: ProtoName, + /// Raw descriptor proto. + descriptor_proto: prost_types::FileDescriptorSet, + /// Direct dependencies of this descriptor. + dependencies: Vec<&'static Descriptor>, +} + +impl Descriptor { + /// Constructs a Descriptor. + pub fn new( + proto_root: ProtoName, + dependencies: Vec<&'static Descriptor>, + descriptor_bytes: &[u8], + ) -> Self { + Descriptor { + proto_root, + dependencies, + descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes).unwrap(), + } + } + + /// Loads the descriptor to the pool, if not already loaded. + pub fn load(&self, pool: &mut prost_reflect::DescriptorPool) -> anyhow::Result<()> { + if self + .descriptor_proto + .file + .iter() + .all(|f| pool.get_file_by_name(f.name()).is_some()) + { + return Ok(()); + } + for d in &self.dependencies { + d.load(pool)?; + } + pool.add_file_descriptor_set(self.descriptor_proto.clone())?; + Ok(()) + } + + /// Loads the descriptor to the global pool and returns a copy of the global pool. + pub fn get_message_by_name(&self, name: &str) -> Option { + /// Global descriptor pool. + static POOL: Lazy> = Lazy::new(Mutex::default); + let pool = &mut POOL.lock().unwrap(); + self.load(pool).unwrap(); + pool.get_message_by_name(name) + } +} + +/// Expands to a descriptor declaration. +#[macro_export] +macro_rules! declare_descriptor { + ($package_root:expr, $descriptor_path:expr, $($rust_deps:path),*) => { + pub static DESCRIPTOR : $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| { + $crate::Descriptor::new( + $package_root.into(), + vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*], + &include_bytes!($descriptor_path)[..], + ) + }); + } +} + +/// Code generation config. Use it in build scripts. +pub struct Config { + /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. + pub input_root: InputPath, + /// Implicit prefix that should be prepended to proto paths of the proto files in the input directory. + pub proto_root: ProtoPath, + /// Descriptors of the dependencies and the rust absolute paths under which they will be available from the generated code. + pub dependencies: Vec<(RustName, &'static Descriptor)>, + /// Rust absolute path under which the protobuf crate will be available from the generated + /// code. + pub protobuf_crate: RustName, +} + +impl Config { + /// Location of the protobuf_build crate, visible from the generated code. + fn this_crate(&self) -> RustName { + self.protobuf_crate.clone().join("build") + } + + /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated + /// from a message of the given `proto_name`. + fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { + let rust_name = proto_name + .relative_to(&self.proto_root.to_name().context("invalid proto_root")?) + .unwrap() + .to_rust_type() + .to_string(); + let rust_name: syn::Path = syn::parse_str(&rust_name).context("rust_name")?; + let proto_name = proto_name.to_string(); + let this: syn::Path = syn::parse_str(&self.this_crate().to_string()).context("this")?; + Ok(quote::quote! { + impl #this::prost_reflect::ReflectMessage for #rust_name { + fn descriptor(&self) -> #this::prost_reflect::MessageDescriptor { + static INIT : #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| { + DESCRIPTOR.get_message_by_name(#proto_name).unwrap() + }); + INIT.clone() + } + } + }.to_string()) + } + + /// Generates rust code from the proto files according to the config. + pub fn generate(&self) -> anyhow::Result<()> { + if !self.input_root.abs()?.is_dir() { + anyhow::bail!("input_root should be a directory"); + } + println!("cargo:rerun-if-changed={}", self.input_root.to_str()); + + // Load dependencies. + let mut pool = prost_reflect::DescriptorPool::new(); + for d in &self.dependencies { + d.1.load(&mut pool) + .with_context(|| format!("failed to load dependency {}", d.0))?; + } + let mut pool_raw = prost_types::FileDescriptorSet::default(); + pool_raw.file.extend(pool.file_descriptor_protos().cloned()); + + // Load proto files. + let mut proto_paths = vec![]; + traverse_files(&self.input_root.abs()?, &mut |path| { + let Some(ext) = path.extension() else { + return Ok(()); + }; + let Some(ext) = ext.to_str() else { + return Ok(()); + }; + if ext != "proto" { + return Ok(()); + }; + + let file_raw = fs::read_to_string(path).context("fs::read()")?; + let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root) + .context("ProtoPath::from_input_path()")?; + pool_raw + .file + .push(protox_parse::parse(&path.to_string(), &file_raw).map_err( + // rewrapping the error, so that source location is included in the error message. + |err| anyhow::anyhow!("{err:?}"), + )?); + proto_paths.push(path); + Ok(()) + })?; + + // Compile the proto files + let mut compiler = protox::Compiler::with_file_resolver( + protox::file::DescriptorSetFileResolver::new(pool_raw), + ); + compiler.include_source_info(true); + compiler + .open_files(proto_paths.iter().map(|p| p.to_path())) + // rewrapping the error, so that source location is included in the error message. + .map_err(|err| anyhow::anyhow!("{err:?}"))?; + let descriptor = compiler.file_descriptor_set(); + // Unwrap is ok, because we add a descriptor from a successful compilation. + pool.add_file_descriptor_set(descriptor.clone()).unwrap(); + + // Check that the compiled proto files belong to the declared proto package. + for f in &descriptor.file { + let got = ProtoName::from(f.package()); + // Unwrap is ok, because descriptor file here has never an empty name. + let want_prefix = ProtoPath::from(f.name()).parent().unwrap().to_name()?; + if !got.starts_with(&want_prefix) { + anyhow::bail!( + "{got} ({:?}) does not belong to package {want_prefix}", + f.name(), + ); + } + } + + // Check that the compiled proto messages support canonical encoding. + canonical::check(&descriptor, &pool).context("canonical::check()")?; + + // Prepare the output directory. + let output_dir = self + .input_root + .prepare_output_dir() + .context("prepare_output_dir()")?; + let output_path = output_dir.join("gen.rs"); + let descriptor_path = output_dir.join("gen.binpb"); + fs::write(&descriptor_path, &descriptor.encode_to_vec())?; + println!("PROTOBUF_DESCRIPTOR={descriptor_path:?}"); + + // Generate code out of compiled proto files. + let mut output = RustModule::default(); + let mut config = prost_build::Config::new(); + config.prost_path(self.this_crate().join("prost").to_string()); + config.skip_protoc_run(); + for d in &self.dependencies { + for f in &d.1.descriptor_proto.file { + let proto_rel = ProtoName::from(f.package()) + .relative_to(&d.1.proto_root) + .unwrap(); + let rust_abs = d.0.clone().join(proto_rel.to_rust_module()); + config.extern_path(format!(".{}", f.package()), rust_abs.to_string()); + } + } + let m = prost_build::Module::from_parts([""]); + for f in &descriptor.file { + let code = config + .generate(vec![(m.clone(), f.clone())]) + .context("generation failed")?; + output + .sub(&ProtoName::from(f.package()).to_rust_module()) + .append(&code[&m]); + } + + // Generate the reflection code. + let package_root = self.proto_root.to_name().context("invalid proto_root")?; + let output = output.sub(&package_root.to_rust_module()); + for proto_name in extract_message_names(&descriptor) { + output.append( + &self + .reflect_impl(&proto_name) + .with_context(|| format!("reflect_impl({proto_name})"))?, + ); + } + + // Generate the descriptor. + let rust_deps = self + .dependencies + .iter() + .map(|d| syn::parse_str::(&d.0.to_string()).unwrap()); + let this: syn::Path = syn::parse_str(&self.this_crate().to_string())?; + let package_root = package_root.to_string(); + let descriptor_path = descriptor_path.display().to_string(); + output.append( + "e::quote! { + #this::declare_descriptor!(#package_root,#descriptor_path,#(#rust_deps),*); + } + .to_string(), + ); + + // Save output. + fs::write(output_path, output.format().context("output.format()")?)?; + Ok(()) + } +} diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs new file mode 100644 index 00000000..80477448 --- /dev/null +++ b/node/libs/protobuf_build/src/syntax.rs @@ -0,0 +1,244 @@ +//! Utilities for handling strings belonging to various namespaces. +use super::ident; +use anyhow::Context as _; +use std::{ + collections::BTreeMap, + fmt, + path::{Path, PathBuf}, +}; + +/// Path relative to $CARGO_MANIFEST_DIR. +#[derive(Clone, PartialEq, Eq)] +pub struct InputPath(PathBuf); + +impl From<&str> for InputPath { + fn from(s: &str) -> Self { + Self(PathBuf::from(s)) + } +} + +impl InputPath { + /// Converts the relative input path to str + pub(super) fn to_str(&self) -> &str { + self.0.to_str().unwrap() + } + + /// Converts the relative input path to an absolute path in the local file system + /// (under $CARGO_MANIFEST_DIR). + pub(super) fn abs(&self) -> anyhow::Result { + Ok(PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?) + .canonicalize()? + .join(&self.0)) + } + + /// Output directory path derived from the input path by replacing $CARGO_MANIFEST_DIR with $OUT_DIR. + /// Re-constructs the derived output directory, as a side-effect. + pub(super) fn prepare_output_dir(&self) -> anyhow::Result { + let output = PathBuf::from(std::env::var("OUT_DIR")?) + .canonicalize()? + .join(&self.0); + let _ = std::fs::remove_dir_all(&output); + std::fs::create_dir_all(&output)?; + Ok(output) + } +} + +/// Absolute path of the proto file used for importing other proto files. +#[derive(Clone, PartialEq, Eq)] +pub struct ProtoPath(PathBuf); + +impl From<&str> for ProtoPath { + fn from(s: &str) -> Self { + Self(PathBuf::from(s)) + } +} + +impl ProtoPath { + /// Converts a proto module path to proto package name by replacing all "/" with ".". + pub(super) fn to_name(&self) -> anyhow::Result { + let mut parts = vec![]; + for p in self.0.iter() { + parts.push(p.to_str().context("invalid proto path")?.to_string()); + } + Ok(ProtoName(parts)) + } + + /// Returns path to the parent directory. + pub(super) fn parent(&self) -> Option { + self.0.parent().map(|p| Self(p.into())) + } + + /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . + pub(super) fn from_input_path( + path: &Path, + input_root: &InputPath, + proto_root: &ProtoPath, + ) -> anyhow::Result { + Ok(ProtoPath( + proto_root + .0 + .join(path.strip_prefix(&input_root.abs()?).unwrap()), + )) + } + + /// Converts ProtoPath to Path. + pub(super) fn to_path(&self) -> &Path { + &self.0 + } +} + +impl fmt::Display for ProtoPath { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + self.0.display().fmt(fmt) + } +} + +type Part = String; + +/// A rust module/type name that the generated code is available at. Although +/// generated code is location agnostic (it can be embedded in an arbitrary module within the crate), +/// you need to manually (in the Config) specify the rust modules containing the generated code +/// of the dependencies, so that it can be referenced from the newly generated code. +#[derive(Clone, PartialEq, Eq)] +pub struct RustName(Vec); + +impl RustName { + /// Concatenates 2 rust names. + pub fn join(mut self, suffix: impl Into) -> Self { + self.0.extend(suffix.into().0); + self + } +} + +impl fmt::Display for RustName { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&self.0.join("::")) + } +} + +impl From<&str> for RustName { + fn from(s: &str) -> Self { + Self(s.split("::").map(Part::from).collect()) + } +} + +/// A rust module representation. +/// It is used to collect the generated protobuf code. +#[derive(Default)] +pub(super) struct RustModule { + /// Nested modules which transitively contain the generated code. + modules: BTreeMap, + /// Code of the module. + code: String, +} + +impl RustModule { + /// Returns a reference to a given submodule. + pub(crate) fn sub(&mut self, path: &RustName) -> &mut Self { + let mut m = self; + for part in &path.0 { + m = m.modules.entry(part.into()).or_default(); + } + m + } + + /// Appends code to the module. + pub(crate) fn append(&mut self, code: &str) { + self.code += code; + } + + fn collect(&self) -> String { + let mut entries = vec![self.code.clone()]; + entries.extend( + self.modules + .iter() + .map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), + ); + entries.join("") + } + + /// Collects the code of the module and formats it. + pub(crate) fn format(&self) -> anyhow::Result { + let s = self.collect(); + Ok(prettyplease::unparse( + &syn::parse_str(&s).with_context(|| format!("syn::parse_str({s:?})"))?, + )) + } +} + +/// In addition to input paths and proto paths, there are also proto names +/// which are used to reference message types from different proto files. +/// Theoretically proto names can be totally independent from the proto paths (i.e. you can have +/// "my.favorite.package" proto package defined in "totally/unrelated/file/name.proto" file, but we +/// recommend the following naming convention: proto package "a.b.c" should be defined either: +/// a) in a single file "a/b/c.proto", or +/// b) in a collection of files under "a/b/c/" directory +/// Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. +#[derive(Clone, PartialEq, Eq)] +pub struct ProtoName(Vec); + +impl ProtoName { + /// Checks if package path starts with the given prefix. + pub fn starts_with(&self, prefix: &Self) -> bool { + self.0.len() >= prefix.0.len() && self.0[0..prefix.0.len()] == prefix.0 + } + + /// Strips a given prefix from the name. + pub fn relative_to(&self, prefix: &Self) -> anyhow::Result { + if !self.starts_with(prefix) { + anyhow::bail!("{self} does not contain {prefix}"); + } + Ok(Self(self.0[prefix.0.len()..].to_vec())) + } + + /// Converts proto package name to rust module name according to prost_build rules. + pub fn to_rust_module(&self) -> RustName { + RustName(self.0.iter().map(|s| ident::to_snake(s)).collect()) + } + + /// Converts proto message name to rust type name according to prost_build rules. + pub fn to_rust_type(&self) -> RustName { + let mut rust = self.to_rust_module(); + let n = rust.0.len(); + rust.0[n - 1] = ident::to_upper_camel(&self.0[n - 1]); + rust + } +} + +impl fmt::Display for ProtoName { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&self.0.join(".")) + } +} + +impl From<&str> for ProtoName { + fn from(s: &str) -> Self { + Self(s.split('.').map(String::from).collect()) + } +} + +impl From<&Path> for ProtoName { + fn from(p: &Path) -> Self { + Self(p.iter().map(|c| c.to_str().unwrap().to_string()).collect()) + } +} + +/// Extracts names of proto messages defined in the descriptor. +pub(super) fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { + fn collect(out: &mut Vec, prefix: &ProtoName, m: &prost_types::DescriptorProto) { + let mut name = prefix.clone(); + name.0.push(m.name().to_string()); + for m in &m.nested_type { + collect(out, &name, m); + } + out.push(name); + } + let mut res = vec![]; + for f in &descriptor.file { + let name = ProtoName::from(f.package()); + for m in &f.message_type { + collect(&mut res, &name, m); + } + } + res +} diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index fdd1bb84..9b4b16f7 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -11,6 +11,7 @@ zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true bit-vec.workspace = true diff --git a/node/libs/roles/src/node/conv.rs b/node/libs/roles/src/node/conv.rs index 52924ec6..6d7ea973 100644 --- a/node/libs/roles/src/node/conv.rs +++ b/node/libs/roles/src/node/conv.rs @@ -1,8 +1,8 @@ use crate::node; use anyhow::Context as _; use zksync_consensus_crypto::ByteFmt; -use zksync_consensus_schema::{read_required, required, ProtoFmt}; use zksync_consensus_utils::enum_util::Variant; +use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for node::Msg { type Proto = node::schema::Msg; diff --git a/node/libs/roles/src/node/messages.rs b/node/libs/roles/src/node/messages.rs index 9c87f025..7cca89ce 100644 --- a/node/libs/roles/src/node/messages.rs +++ b/node/libs/roles/src/node/messages.rs @@ -1,6 +1,5 @@ use crate::node; use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; -use zksync_consensus_schema as schema; use zksync_consensus_utils::enum_util::{BadVariantError, Variant}; /// The ID for an authentication session. @@ -18,7 +17,7 @@ pub enum Msg { impl Msg { /// Get the hash of this message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&schema::canonical(self))) + MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } } diff --git a/node/libs/roles/src/node/tests.rs b/node/libs/roles/src/node/tests.rs index 2d208796..e6ab6397 100644 --- a/node/libs/roles/src/node/tests.rs +++ b/node/libs/roles/src/node/tests.rs @@ -2,7 +2,7 @@ use super::*; use rand::Rng; use zksync_concurrency::ctx; use zksync_consensus_crypto::{ByteFmt, Text, TextFmt}; -use zksync_consensus_schema::testonly::{test_encode, test_encode_random}; +use zksync_protobuf::testonly::{test_encode, test_encode_random}; #[test] fn test_byte_encoding() { diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index 69d5d179..7e0d030e 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -8,11 +8,9 @@ use crate::node::SessionId; use anyhow::Context as _; use std::collections::BTreeMap; use zksync_consensus_crypto::ByteFmt; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{ - proto::roles::validator as proto, read_required, required, ProtoFmt, -}; +use zksync_consensus_schema::proto::roles::validator as proto; use zksync_consensus_utils::enum_util::Variant; +use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for BlockHeaderHash { type Proto = proto::BlockHeaderHash; @@ -189,7 +187,7 @@ impl ProtoFmt for LeaderCommit { } impl ProtoFmt for Signers { - type Proto = schema::proto::std::BitVector; + type Proto = zksync_protobuf::proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(ProtoFmt::read(r)?)) @@ -268,8 +266,8 @@ impl ProtoFmt for Phase { fn build(&self) -> Self::Proto { use proto::phase::T; let t = match self { - Self::Prepare => T::Prepare(schema::proto::std::Void {}), - Self::Commit => T::Commit(schema::proto::std::Void {}), + Self::Prepare => T::Prepare(zksync_protobuf::proto::std::Void {}), + Self::Commit => T::Commit(zksync_protobuf::proto::std::Void {}), }; Self::Proto { t: Some(t) } } diff --git a/node/libs/roles/src/validator/messages/block.rs b/node/libs/roles/src/validator/messages/block.rs index 6a6d44c1..a7ebb0f8 100644 --- a/node/libs/roles/src/validator/messages/block.rs +++ b/node/libs/roles/src/validator/messages/block.rs @@ -3,7 +3,6 @@ use super::CommitQC; use std::fmt; use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; -use zksync_consensus_schema as schema; /// Payload of the block. Consensus algorithm does not interpret the payload /// (except for imposing a size limit for the payload). Proposing a payload @@ -115,7 +114,7 @@ pub struct BlockHeader { impl BlockHeader { /// Returns the hash of the block. pub fn hash(&self) -> BlockHeaderHash { - BlockHeaderHash(sha256::Sha256::new(&schema::canonical(self))) + BlockHeaderHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } /// Creates a genesis block. @@ -163,11 +162,11 @@ impl FinalBlock { impl ByteFmt for FinalBlock { fn decode(bytes: &[u8]) -> anyhow::Result { - schema::decode(bytes) + zksync_protobuf::decode(bytes) } fn encode(&self) -> Vec { - schema::encode(self) + zksync_protobuf::encode(self) } } diff --git a/node/libs/roles/src/validator/messages/msg.rs b/node/libs/roles/src/validator/messages/msg.rs index c740e9ed..cb6c6ad2 100644 --- a/node/libs/roles/src/validator/messages/msg.rs +++ b/node/libs/roles/src/validator/messages/msg.rs @@ -3,7 +3,6 @@ use super::{ConsensusMsg, NetAddress}; use crate::{node::SessionId, validator, validator::Error}; use std::fmt; use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; -use zksync_consensus_schema as schema; use zksync_consensus_utils::enum_util::{BadVariantError, Variant}; /// Generic message type for a validator. @@ -20,7 +19,7 @@ pub enum Msg { impl Msg { /// Returns the hash of the message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&schema::canonical(self))) + MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } } diff --git a/node/libs/roles/src/validator/tests.rs b/node/libs/roles/src/validator/tests.rs index 8030afbf..4ef8d885 100644 --- a/node/libs/roles/src/validator/tests.rs +++ b/node/libs/roles/src/validator/tests.rs @@ -3,7 +3,7 @@ use rand::Rng; use std::vec; use zksync_concurrency::ctx; use zksync_consensus_crypto::{ByteFmt, Text, TextFmt}; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_byte_encoding() { diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index bdb3ac1c..d0e06888 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -6,28 +6,20 @@ authors.workspace = true homepage.workspace = true license.workspace = true -[[bin]] -name = "conformance_test" - [dependencies] zksync_concurrency.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true bit-vec.workspace = true serde.workspace = true once_cell.workspace = true -quick-protobuf.workspace = true prost.workspace = true -prost-reflect.workspace = true rand.workspace = true serde_json.workspace = true tokio.workspace = true [build-dependencies] +zksync_protobuf.workspace = true + anyhow.workspace = true -syn.workspace = true -protoc-bin-vendored.workspace = true -prost-build.workspace = true -prost-reflect.workspace = true -prost-reflect-build.workspace = true -prettyplease.workspace = true diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 748a0a7e..40c8e5c4 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,159 +1,14 @@ -//! Generates rust code from the capnp schema files in the `capnp/` directory. -use anyhow::Context as _; -use std::{collections::BTreeMap, env, fs, path::PathBuf}; - -/// Traversed all the files in a directory recursively. -fn traverse_files(path: PathBuf, f: &mut dyn FnMut(PathBuf)) -> std::io::Result<()> { - if !path.is_dir() { - f(path); - return Ok(()); - } - for entry in fs::read_dir(path)? { - traverse_files(entry?.path(), f)?; - } - Ok(()) -} - -/// A rust module representation. -/// It is used to collect the generated protobuf code. -#[derive(Default)] -struct Module { - /// Nested modules which transitively contain the generated code. - nested: BTreeMap, - /// Nested modules directly contains the generated code. - include: BTreeMap, -} - -impl Module { - /// Inserts a nested generated protobuf module. - /// `name` is a sequence of module names. - fn insert(&mut self, name: &[String], file: PathBuf) { - println!(" -- {name:?}"); - match name.len() { - 0 => panic!("empty module path"), - 1 => assert!( - self.include.insert(name[0].clone(), file).is_none(), - "duplicate module" - ), - _ => self - .nested - .entry(name[0].clone()) - .or_default() - .insert(&name[1..], file), - } - } - - /// Generates rust code of the module. - fn generate(&self) -> String { - let mut entries = vec![]; - entries.extend( - self.nested - .iter() - .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), - ); - entries.extend( - self.include - .iter() - .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), - ); - entries.join("\n") - } -} - -/// Checks if field `f` supports canonical encoding. -fn check_canonical_field(f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { - if f.is_map() { - anyhow::bail!("maps unsupported"); - } - if !f.is_list() && !f.supports_presence() { - anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); - } - Ok(()) -} - -/// Checks if messages of type `m` support canonical encoding. -fn check_canonical_message(m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { - for m in m.child_messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; - } - for f in m.fields() { - check_canonical_field(&f).with_context(|| f.name().to_string())?; - } - Ok(()) -} - -/// Checks if message types in file `f` support canonical encoding. -fn check_canonical_file(f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { - if f.syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - for m in f.messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; - } - Ok(()) -} - -/// Checks if message types in descriptor pool `d` support canonical encoding. -fn check_canonical_pool(d: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { - for f in d.files() { - check_canonical_file(&f).with_context(|| f.name().to_string())?; - } - Ok(()) -} - -fn main() -> anyhow::Result<()> { - // Prepare input and output root dirs. - let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) - .canonicalize()? - .join("proto"); - let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) - .canonicalize()? - .join("proto"); - println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); - let _ = fs::remove_dir_all(&proto_output); - fs::create_dir_all(&proto_output).unwrap(); - - // Find all proto files. - let mut proto_inputs = vec![]; - traverse_files(proto_include.clone(), &mut |path| { - let Some(ext) = path.extension() else { return }; - let Some(ext) = ext.to_str() else { return }; - if ext != "proto" { - return; - }; - proto_inputs.push(path); - })?; - - // Generate protobuf code from schema (with reflection). - env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); - let mut config = prost_build::Config::new(); - let descriptor_path = proto_output.join("descriptor.bin"); - config.out_dir(&proto_output); - prost_reflect_build::Builder::new() - .file_descriptor_set_path(&descriptor_path) - .descriptor_pool("crate::proto::DESCRIPTOR_POOL") - .compile_protos_with_config(config, &proto_inputs, &[&proto_include]) - .unwrap(); - let descriptor = fs::read(descriptor_path)?; - let pool = prost_reflect::DescriptorPool::decode(descriptor.as_ref()).unwrap(); - - // Check that messages are compatible with `proto_fmt::canonical`. - check_canonical_pool(&pool)?; - - // Generate mod file collecting all proto-generated code. - let mut m = Module::default(); - for entry in fs::read_dir(&proto_output).unwrap() { - let entry = entry.unwrap(); - let name = entry.file_name().into_string().unwrap(); - let Some(name) = name.strip_suffix(".rs") else { - continue; - }; - let name: Vec<_> = name.split('.').map(String::from).collect(); - println!("name = {name:?}"); - m.insert(&name, entry.path()); - } - let file = m.generate(); - let file = syn::parse_str(&file).unwrap(); - fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; - Ok(()) +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf::build::Config { + input_root: "proto".into(), + proto_root: "zksync/schema".into(), + dependencies: vec![( + "::zksync_protobuf::proto".into(), + &zksync_protobuf::proto::DESCRIPTOR, + )], + protobuf_crate: "::zksync_protobuf".into(), + } + .generate() + .expect("generate()"); } diff --git a/node/libs/schema/proto/buf.yaml b/node/libs/schema/proto/buf.yaml deleted file mode 100644 index 94d54ac3..00000000 --- a/node/libs/schema/proto/buf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: v1 -breaking: - use: - # We use FILE rule set for everything, - # althout only protos which we use with JSON format need that. - # All the other protos would be fine with WIRE rule set. - # TODO(gprusak): separate JSON protos from binary protos - # to weaken the compatibility restrictions, in case using FILE - # for everything turns out to be too annoying. - - FILE diff --git a/node/libs/schema/proto/executor/config.proto b/node/libs/schema/proto/executor/config.proto index f273a039..20092d93 100644 --- a/node/libs/schema/proto/executor/config.proto +++ b/node/libs/schema/proto/executor/config.proto @@ -36,7 +36,7 @@ // the validator set) or move it to a separate config file. syntax = "proto3"; -package executor.config; +package zksync.schema.executor.config; // (public key, ip address) of a gossip network node. message NodeAddr { diff --git a/node/libs/schema/proto/network/consensus.proto b/node/libs/schema/proto/network/consensus.proto index f5f34dbb..09be8a08 100644 --- a/node/libs/schema/proto/network/consensus.proto +++ b/node/libs/schema/proto/network/consensus.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package network.consensus; +package zksync.schema.network.consensus; -import "roles/validator.proto"; -import "std.proto"; +import "zksync/schema/roles/validator.proto"; +import "zksync/std.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/libs/schema/proto/network/gossip.proto b/node/libs/schema/proto/network/gossip.proto index 9423a2c7..723bb100 100644 --- a/node/libs/schema/proto/network/gossip.proto +++ b/node/libs/schema/proto/network/gossip.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package network.gossip; +package zksync.schema.network.gossip; -import "roles/node.proto"; -import "roles/validator.proto"; +import "zksync/schema/roles/node.proto"; +import "zksync/schema/roles/validator.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/libs/schema/proto/network/mux.proto b/node/libs/schema/proto/network/mux.proto index a4bd0eec..3e014c8e 100644 --- a/node/libs/schema/proto/network/mux.proto +++ b/node/libs/schema/proto/network/mux.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package network.mux; +package zksync.schema.network.mux; -import "std.proto"; +import "zksync/std.proto"; message Handshake { message Capability { diff --git a/node/libs/schema/proto/network/mux_test.proto b/node/libs/schema/proto/network/mux_test.proto index dff0f492..b4cdaf87 100644 --- a/node/libs/schema/proto/network/mux_test.proto +++ b/node/libs/schema/proto/network/mux_test.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.mux_test; +package zksync.schema.network.mux_test; message Req { optional bytes input = 1; diff --git a/node/libs/schema/proto/network/ping.proto b/node/libs/schema/proto/network/ping.proto index 563dcbf9..863022fd 100644 --- a/node/libs/schema/proto/network/ping.proto +++ b/node/libs/schema/proto/network/ping.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.ping; +package zksync.schema.network.ping; message PingReq { optional bytes data = 1; diff --git a/node/libs/schema/proto/network/preface.proto b/node/libs/schema/proto/network/preface.proto index 1325e047..193ca38e 100644 --- a/node/libs/schema/proto/network/preface.proto +++ b/node/libs/schema/proto/network/preface.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.preface; +package zksync.schema.network.preface; // Every connection starts with the following preface: // 1. client sends Encryption msg to server. diff --git a/node/libs/schema/proto/roles/node.proto b/node/libs/schema/proto/roles/node.proto index 178eb914..87b75c1e 100644 --- a/node/libs/schema/proto/roles/node.proto +++ b/node/libs/schema/proto/roles/node.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package roles.node; +package zksync.schema.roles.node; message Msg { oneof t { diff --git a/node/libs/schema/proto/roles/validator.proto b/node/libs/schema/proto/roles/validator.proto index f622b06d..7419690f 100644 --- a/node/libs/schema/proto/roles/validator.proto +++ b/node/libs/schema/proto/roles/validator.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package roles.validator; +package zksync.schema.roles.validator; -import "std.proto"; +import "zksync/std.proto"; message PayloadHash { optional bytes sha256 = 1; // required diff --git a/node/libs/schema/proto/storage.proto b/node/libs/schema/proto/storage.proto index dbcfa8e2..301a5485 100644 --- a/node/libs/schema/proto/storage.proto +++ b/node/libs/schema/proto/storage.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package storage; +package zksync.schema.storage; -import "roles/validator.proto"; +import "zksync/schema/roles/validator.proto"; message Proposal { optional uint64 number = 1; diff --git a/node/libs/schema/src/lib.rs b/node/libs/schema/src/lib.rs index eeb2ba7e..e1891ab1 100644 --- a/node/libs/schema/src/lib.rs +++ b/node/libs/schema/src/lib.rs @@ -1,23 +1,6 @@ -//! Code generated from protobuf schema files and -//! utilities for serialization. - -mod proto_fmt; -mod std_conv; -pub mod testonly; - -pub use proto_fmt::*; - -#[cfg(test)] -mod tests; +//! Code generated from protobuf schema files. #[allow(warnings)] pub mod proto { - include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); - static DESCRIPTOR_POOL: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| { - prost_reflect::DescriptorPool::decode( - include_bytes!(concat!(env!("OUT_DIR"), "/proto/descriptor.bin")).as_ref(), - ) - .unwrap() - }); + include!(concat!(env!("OUT_DIR"), "/proto/gen.rs")); } diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index 67b685e6..0d034c40 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true zksync_concurrency.workspace = true zksync_consensus_roles.workspace = true zksync_consensus_schema.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true async-trait.workspace = true diff --git a/node/libs/storage/src/rocksdb.rs b/node/libs/storage/src/rocksdb.rs index 1a2ebe7b..ae7879b5 100644 --- a/node/libs/storage/src/rocksdb.rs +++ b/node/libs/storage/src/rocksdb.rs @@ -20,7 +20,6 @@ use std::{ }; use zksync_concurrency::{ctx, scope, sync::watch}; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; -use zksync_consensus_schema as schema; /// Enum used to represent a key in the database. It also acts as a separator between different stores. #[derive(Debug, Clone, PartialEq, Eq)] @@ -145,7 +144,7 @@ impl RocksdbStorage { .next() .context("Head block not found")? .context("RocksDB error reading head block")?; - schema::decode(&head_block).context("Failed decoding head block bytes") + zksync_protobuf::decode(&head_block).context("Failed decoding head block bytes") } /// Returns a block with the least number stored in this database. @@ -159,7 +158,7 @@ impl RocksdbStorage { .next() .context("First stored block not found")? .context("RocksDB error reading first stored block")?; - schema::decode(&first_block).context("Failed decoding first stored block bytes") + zksync_protobuf::decode(&first_block).context("Failed decoding first stored block bytes") } fn last_contiguous_block_number_blocking(&self) -> anyhow::Result { @@ -219,7 +218,7 @@ impl RocksdbStorage { else { return Ok(None); }; - let block = schema::decode(&raw_block) + let block = zksync_protobuf::decode(&raw_block) .with_context(|| format!("Failed decoding block #{number}"))?; Ok(Some(block)) } @@ -258,7 +257,7 @@ impl RocksdbStorage { let mut write_batch = rocksdb::WriteBatch::default(); write_batch.put( DatabaseKey::Block(block_number).encode_key(), - schema::encode(finalized_block), + zksync_protobuf::encode(finalized_block), ); // Commit the transaction. db.write(write_batch) @@ -277,7 +276,7 @@ impl RocksdbStorage { else { return Ok(None); }; - schema::decode(&raw_state) + zksync_protobuf::decode(&raw_state) .map(Some) .context("Failed to decode replica state!") } @@ -286,7 +285,7 @@ impl RocksdbStorage { self.write() .put( DatabaseKey::ReplicaState.encode_key(), - schema::encode(replica_state), + zksync_protobuf::encode(replica_state), ) .context("Failed putting ReplicaState to RocksDB") } diff --git a/node/libs/storage/src/tests/mod.rs b/node/libs/storage/src/tests/mod.rs index 8cd28276..7f840de3 100644 --- a/node/libs/storage/src/tests/mod.rs +++ b/node/libs/storage/src/tests/mod.rs @@ -7,7 +7,6 @@ use zksync_concurrency::ctx; use zksync_consensus_roles::validator::{ testonly::make_block, BlockHeader, BlockNumber, FinalBlock, Payload, }; -use zksync_consensus_schema as schema; #[cfg(feature = "rocksdb")] mod rocksdb; @@ -130,7 +129,10 @@ fn test_schema_encode_decode() { let rng = &mut ctx.rng(); let replica = rng.gen::(); - assert_eq!(replica, schema::decode(&schema::encode(&replica)).unwrap()); + assert_eq!( + replica, + zksync_protobuf::decode(&zksync_protobuf::encode(&replica)).unwrap() + ); } #[test] diff --git a/node/libs/storage/src/types.rs b/node/libs/storage/src/types.rs index 0217a4de..a4ee13bf 100644 --- a/node/libs/storage/src/types.rs +++ b/node/libs/storage/src/types.rs @@ -3,7 +3,8 @@ use anyhow::Context as _; use std::{iter, ops}; use zksync_concurrency::ctx; use zksync_consensus_roles::validator::{self, BlockNumber}; -use zksync_consensus_schema::{proto::storage as proto, read_required, required, ProtoFmt}; +use zksync_consensus_schema::proto::storage as proto; +use zksync_protobuf::{read_required, required, ProtoFmt}; /// A payload of a proposed block which is not known to be finalized yet. /// Replicas have to persist such proposed payloads for liveness: diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 3102f33d..74803009 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -17,6 +17,7 @@ zksync_consensus_roles.workspace = true zksync_consensus_storage = { workspace = true, features = ["rocksdb"] } zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true clap.workspace = true diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 95f74f5a..69c9f71e 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -7,7 +7,6 @@ use zksync_consensus_bft::testonly; use zksync_consensus_crypto::TextFmt; use zksync_consensus_executor::{ConsensusConfig, ExecutorConfig, GossipConfig}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema as schema; use zksync_consensus_tools::NodeConfig; /// Replaces IP of the address with UNSPECIFIED (aka INADDR_ANY) of the corresponding IP type. @@ -111,8 +110,11 @@ fn main() -> anyhow::Result<()> { let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; - fs::write(root.join("config.json"), schema::encode_json(&node_cfg)) - .context("fs::write()")?; + fs::write( + root.join("config.json"), + zksync_protobuf::encode_json(&node_cfg), + ) + .context("fs::write()")?; fs::write( root.join("validator_key"), &TextFmt::encode(&validator_keys[i]), diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index be7b4421..effcd00c 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -4,10 +4,8 @@ use std::{fs, net, path::Path}; use zksync_consensus_crypto::{read_optional_text, Text, TextFmt}; use zksync_consensus_executor::{ConsensusConfig, ExecutorConfig}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{ - proto::executor::config as proto, read_optional, read_required, ProtoFmt, -}; +use zksync_consensus_schema::proto::executor::config as proto; +use zksync_protobuf::{read_optional, read_required, ProtoFmt}; /// This struct holds the file path to each of the config files. #[derive(Debug)] @@ -78,12 +76,13 @@ impl Configs { args.config.display() ) })?; - let node_config: NodeConfig = schema::decode_json(&node_config).with_context(|| { - format!( - "failed decoding JSON node config at `{}`", - args.config.display() - ) - })?; + let node_config: NodeConfig = + zksync_protobuf::decode_json(&node_config).with_context(|| { + format!( + "failed decoding JSON node config at `{}`", + args.config.display() + ) + })?; let validator_key: Option = args .validator_key