From 9b300371512a4e443d086ecd19c1877719c15646 Mon Sep 17 00:00:00 2001 From: pompon0 Date: Wed, 8 Nov 2023 12:54:43 +0100 Subject: [PATCH 1/5] Upgraded protobuf to support cross crate imports. (#24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upgraded protobuf to support cross crate imports. We need it to allow defining protos in zksync-era as well. Proto packages share a single common namespace, so I prefixed our packages with zksync.. I couldn't name the crate "protobuf", because cargo deny has mistaken it for the public crate with the same name. I've named it "zksync_protobuf", to be consistent with zksync-era repo. CI protobuf compatibility check is now more complex, because we need to collect the descriptors from the cargo build artifacts. Currently it is implemented as a bash command, but can be upgraded to a proper github action in the future. --------- Co-authored-by: Bruno França --- .github/workflows/protobuf.yaml | 28 +- .github/workflows/protobuf_conformance.yaml | 4 +- node/Cargo.lock | 279 +++++++++------- node/Cargo.toml | 17 +- node/actors/bft/Cargo.toml | 1 + node/actors/bft/src/inner.rs | 3 +- node/actors/bft/src/metrics.rs | 7 +- node/actors/executor/Cargo.toml | 1 + node/actors/executor/src/config/mod.rs | 6 +- node/actors/executor/src/config/tests.rs | 2 +- node/actors/network/Cargo.toml | 1 + .../network/src/consensus/handshake/mod.rs | 3 +- .../network/src/consensus/handshake/tests.rs | 2 +- node/actors/network/src/frame.rs | 17 +- .../network/src/gossip/handshake/mod.rs | 3 +- .../network/src/gossip/handshake/tests.rs | 2 +- node/actors/network/src/mux/handshake.rs | 8 +- node/actors/network/src/mux/tests.rs | 5 +- node/actors/network/src/preface.rs | 8 +- node/actors/network/src/rpc/consensus.rs | 8 +- node/actors/network/src/rpc/metrics.rs | 3 +- node/actors/network/src/rpc/mod.rs | 11 +- node/actors/network/src/rpc/ping.rs | 8 +- node/actors/network/src/rpc/sync_blocks.rs | 17 +- .../network/src/rpc/sync_validator_addrs.rs | 8 +- node/actors/network/src/rpc/tests.rs | 2 +- node/deny.toml | 7 +- node/libs/protobuf/Cargo.toml | 32 ++ node/libs/protobuf/build.rs | 29 ++ .../bin/conformance_test/failure_list.txt} | 0 .../src/bin/conformance_test/main.rs} | 48 ++- .../conformance_test/proto}/conformance.proto | 5 +- .../src/bin/conformance_test/proto/mod.rs | 5 + .../proto}/test_messages_proto3.proto | 10 +- node/libs/protobuf/src/lib.rs | 13 + node/libs/protobuf/src/proto/mod.rs | 2 + .../{schema => protobuf/src}/proto/std.proto | 2 +- .../{schema => protobuf}/src/proto_fmt.rs | 0 .../libs/{schema => protobuf}/src/std_conv.rs | 12 +- .../libs/{schema => protobuf}/src/testonly.rs | 0 .../tests.rs => protobuf/src/tests/mod.rs} | 10 +- node/libs/protobuf/src/tests/proto/mod.rs | 2 + .../src/tests/proto/tests.proto} | 2 +- node/libs/protobuf_build/Cargo.toml | 23 ++ node/libs/protobuf_build/src/canonical.rs | 58 ++++ node/libs/protobuf_build/src/ident.rs | 163 ++++++++++ node/libs/protobuf_build/src/lib.rs | 297 ++++++++++++++++++ node/libs/protobuf_build/src/syntax.rs | 244 ++++++++++++++ node/libs/roles/Cargo.toml | 1 + node/libs/roles/src/node/conv.rs | 2 +- node/libs/roles/src/node/messages.rs | 3 +- node/libs/roles/src/node/tests.rs | 2 +- node/libs/roles/src/validator/conv.rs | 12 +- .../roles/src/validator/messages/block.rs | 7 +- node/libs/roles/src/validator/messages/msg.rs | 3 +- node/libs/roles/src/validator/tests.rs | 2 +- node/libs/schema/Cargo.toml | 14 +- node/libs/schema/build.rs | 171 +--------- node/libs/schema/proto/buf.yaml | 10 - node/libs/schema/proto/executor/config.proto | 2 +- .../libs/schema/proto/network/consensus.proto | 6 +- node/libs/schema/proto/network/gossip.proto | 6 +- node/libs/schema/proto/network/mux.proto | 4 +- node/libs/schema/proto/network/mux_test.proto | 2 +- node/libs/schema/proto/network/ping.proto | 2 +- node/libs/schema/proto/network/preface.proto | 2 +- node/libs/schema/proto/roles/node.proto | 2 +- node/libs/schema/proto/roles/validator.proto | 4 +- node/libs/schema/proto/storage.proto | 4 +- node/libs/schema/src/lib.rs | 21 +- node/libs/storage/Cargo.toml | 1 + node/libs/storage/src/rocksdb.rs | 13 +- node/libs/storage/src/tests/mod.rs | 6 +- node/libs/storage/src/types.rs | 3 +- node/tools/Cargo.toml | 1 + node/tools/src/bin/localnet_config.rs | 8 +- node/tools/src/config.rs | 19 +- 77 files changed, 1266 insertions(+), 485 deletions(-) create mode 100644 node/libs/protobuf/Cargo.toml create mode 100644 node/libs/protobuf/build.rs rename node/libs/{schema/src/bin/conformance_test_failure_list.txt => protobuf/src/bin/conformance_test/failure_list.txt} (100%) rename node/libs/{schema/src/bin/conformance_test.rs => protobuf/src/bin/conformance_test/main.rs} (71%) rename node/libs/{schema/proto/conformance => protobuf/src/bin/conformance_test/proto}/conformance.proto (98%) create mode 100644 node/libs/protobuf/src/bin/conformance_test/proto/mod.rs rename node/libs/{schema/proto/conformance => protobuf/src/bin/conformance_test/proto}/test_messages_proto3.proto (96%) create mode 100644 node/libs/protobuf/src/lib.rs create mode 100644 node/libs/protobuf/src/proto/mod.rs rename node/libs/{schema => protobuf/src}/proto/std.proto (98%) rename node/libs/{schema => protobuf}/src/proto_fmt.rs (100%) rename node/libs/{schema => protobuf}/src/std_conv.rs (92%) rename node/libs/{schema => protobuf}/src/testonly.rs (100%) rename node/libs/{schema/src/tests.rs => protobuf/src/tests/mod.rs} (94%) create mode 100644 node/libs/protobuf/src/tests/proto/mod.rs rename node/libs/{schema/proto/testonly.proto => protobuf/src/tests/proto/tests.proto} (91%) create mode 100644 node/libs/protobuf_build/Cargo.toml create mode 100644 node/libs/protobuf_build/src/canonical.rs create mode 100644 node/libs/protobuf_build/src/ident.rs create mode 100644 node/libs/protobuf_build/src/lib.rs create mode 100644 node/libs/protobuf_build/src/syntax.rs delete mode 100644 node/libs/schema/proto/buf.yaml diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index 68daaf44..59bd6a36 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -14,7 +14,12 @@ on: branches: ["main"] env: - PROTOS_DIR: "node/libs/schema/proto" + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + RUSTFLAGS: "-Dwarnings -C linker=clang -C link-arg=-fuse-ld=lld -C link-arg=-Wl,-z,nostart-stop-gc" + RUSTC_WRAPPER: "sccache" + SCCACHE_GHA_ENABLED: "true" + RUST_BACKTRACE: "1" jobs: compatibility: @@ -22,16 +27,33 @@ jobs: steps: # github.base_ref -> github.head_ref for pull_request # github.event.before -> github.event.after for push + - uses: mozilla-actions/sccache-action@v0.0.3 - uses: actions/checkout@v3 with: ref: ${{ github.base_ref || github.event.before }} path: before + - name: compile before + run: cargo build --all-targets + working-directory: ./before/node + - name: build before.binpb + run: > + perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' + `find ./before/node/target/debug/build/*/output` + | xargs cat > ./before.binpb - uses: actions/checkout@v3 with: ref: ${{ github.head_ref || github.event.after }} path: after + - name: compile after + run: cargo build --all-targets + working-directory: ./after/node + - name: build after.binpb + run: > + perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' + `find ./after/node/target/debug/build/*/output` + | xargs cat > ./after.binpb - uses: bufbuild/buf-setup-action@v1 - uses: bufbuild/buf-breaking-action@v1 with: - input: "after/$PROTOS_DIR" - against: "before/$PROTOS_DIR" + input: "./after.binpb" + against: "./before.binpb" diff --git a/.github/workflows/protobuf_conformance.yaml b/.github/workflows/protobuf_conformance.yaml index eac65e28..69efd530 100644 --- a/.github/workflows/protobuf_conformance.yaml +++ b/.github/workflows/protobuf_conformance.yaml @@ -28,7 +28,7 @@ jobs: path: "protobuf" - uses: mozilla-actions/sccache-action@v0.0.3 - name: build test - run: cargo build -p zksync_consensus_schema --bin conformance_test + run: cargo build -p zksync_protobuf --bin conformance_test working-directory: "this/node" - name: Cache Bazel uses: actions/cache@v3 @@ -41,6 +41,6 @@ jobs: - name: run test run: > bazel run //conformance:conformance_test_runner -- - --failure_list "${{ github.workspace }}/this/node/libs/schema/src/bin/conformance_test_failure_list.txt" + --failure_list "${{ github.workspace }}/this/node/libs/protobuf/src/bin/conformance_test/failure_list.txt" "${{ github.workspace }}/this/node/target/debug/conformance_test" working-directory: "protobuf" diff --git a/node/Cargo.lock b/node/Cargo.lock index 920b83f9..8896fef8 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -301,6 +301,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bindgen" version = "0.65.1" @@ -313,7 +319,7 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease 0.2.12", + "prettyplease", "proc-macro2", "quote", "regex", @@ -1056,9 +1062,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] @@ -1160,6 +1166,38 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn 2.0.32", +] + +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lz4-sys" version = "1.9.4" @@ -1194,6 +1232,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1494,16 +1555,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.12" @@ -1548,9 +1599,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" dependencies = [ "bytes", "prost-derive", @@ -1558,134 +1609,91 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck", "itertools", - "lazy_static", "log", "multimap", + "once_cell", "petgraph", - "prettyplease 0.1.25", + "prettyplease", "prost", "prost-types", "regex", - "syn 1.0.109", + "syn 2.0.32", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] name = "prost-reflect" -version = "0.11.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "000e1e05ebf7b26e1eba298e66fe4eee6eb19c567d0ffb35e0dd34231cdac4c8" +checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" dependencies = [ "base64", + "logos", + "miette", "once_cell", "prost", - "prost-reflect-derive", "prost-types", "serde", "serde-value", ] [[package]] -name = "prost-reflect-build" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d959f576df574d088e0409c45ef11897f64727158c554ce65edc2d345f8afcf" -dependencies = [ - "prost-build", - "prost-reflect", -] - -[[package]] -name = "prost-reflect-derive" -version = "0.11.0" +name = "prost-types" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7718375aa8f966df66e583b608a305a45bc87eeb1ffd5db87fae673bea17a7e4" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", + "prost", ] [[package]] -name = "prost-types" -version = "0.11.9" +name = "protox" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "66eb3a834c1ffe362107daab84dd87cfc1e1d2beda30e2eb8e4801f262839364" dependencies = [ + "bytes", + "miette", "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", ] [[package]] -name = "protoc-bin-vendored" -version = "3.0.0" +name = "protox-parse" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d" +checksum = "7b4581f441c58863525a3e6bec7b8de98188cf75239a56c725a3e7288450a33f" dependencies = [ - "protoc-bin-vendored-linux-aarch_64", - "protoc-bin-vendored-linux-ppcle_64", - "protoc-bin-vendored-linux-x86_32", - "protoc-bin-vendored-linux-x86_64", - "protoc-bin-vendored-macos-x86_64", - "protoc-bin-vendored-win32", + "logos", + "miette", + "prost-types", + "thiserror", ] -[[package]] -name = "protoc-bin-vendored-linux-aarch_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435" - -[[package]] -name = "protoc-bin-vendored-linux-ppcle_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516" - -[[package]] -name = "protoc-bin-vendored-linux-x86_32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0" - -[[package]] -name = "protoc-bin-vendored-linux-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924" - -[[package]] -name = "protoc-bin-vendored-macos-x86_64" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537" - -[[package]] -name = "protoc-bin-vendored-win32" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" - [[package]] name = "quick-protobuf" version = "0.8.1" @@ -2286,6 +2294,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "universal-hash" version = "0.4.0" @@ -2381,9 +2395,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2391,9 +2405,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", @@ -2406,9 +2420,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2416,9 +2430,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", @@ -2429,15 +2443,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" dependencies = [ "js-sys", "wasm-bindgen", @@ -2559,18 +2573,18 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.15" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ba595b9f2772fbee2312de30eeb80ec773b4cb2f1e8098db024afadda6c06f" +checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.15" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772666c41fb6dceaf520b564b962d738a8e1a83b41bd48945f50837aed78bb1d" +checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" dependencies = [ "proc-macro2", "quote", @@ -2632,6 +2646,7 @@ dependencies = [ "zksync_consensus_schema", "zksync_consensus_storage", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2671,6 +2686,7 @@ dependencies = [ "zksync_consensus_storage", "zksync_consensus_sync_blocks", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2695,6 +2711,7 @@ dependencies = [ "zksync_consensus_roles", "zksync_consensus_schema", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2711,6 +2728,7 @@ dependencies = [ "zksync_consensus_crypto", "zksync_consensus_schema", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2720,19 +2738,13 @@ dependencies = [ "anyhow", "bit-vec", "once_cell", - "prettyplease 0.2.12", "prost", - "prost-build", - "prost-reflect", - "prost-reflect-build", - "protoc-bin-vendored", - "quick-protobuf", "rand", "serde", "serde_json", - "syn 2.0.32", "tokio", "zksync_concurrency", + "zksync_protobuf", ] [[package]] @@ -2752,6 +2764,7 @@ dependencies = [ "zksync_concurrency", "zksync_consensus_roles", "zksync_consensus_schema", + "zksync_protobuf", ] [[package]] @@ -2792,6 +2805,7 @@ dependencies = [ "zksync_consensus_schema", "zksync_consensus_storage", "zksync_consensus_utils", + "zksync_protobuf", ] [[package]] @@ -2802,6 +2816,45 @@ dependencies = [ "zksync_concurrency", ] +[[package]] +name = "zksync_protobuf" +version = "0.1.0" +dependencies = [ + "anyhow", + "bit-vec", + "once_cell", + "prost", + "prost-reflect", + "quick-protobuf", + "rand", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "zksync_concurrency", + "zksync_protobuf_build", +] + +[[package]] +name = "zksync_protobuf_build" +version = "0.1.0" +dependencies = [ + "anyhow", + "heck", + "once_cell", + "prettyplease", + "prost", + "prost-build", + "prost-reflect", + "prost-types", + "protox", + "protox-parse", + "quote", + "rand", + "syn 2.0.32", +] + [[package]] name = "zstd-sys" version = "2.0.8+zstd.1.5.5" diff --git a/node/Cargo.toml b/node/Cargo.toml index 6e2b1185..8dc3d506 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,8 @@ [workspace] members = [ "libs/concurrency", + "libs/protobuf_build", + "libs/protobuf", "actors/bft", "libs/crypto", "actors/executor", @@ -21,6 +23,8 @@ homepage = "https://matter-labs.io/" license = "MIT" [workspace.dependencies] +zksync_protobuf_build = { path = "libs/protobuf_build" } +zksync_protobuf = { path = "libs/protobuf" } zksync_concurrency = { path = "libs/concurrency" } zksync_consensus_bft = { path = "actors/bft" } zksync_consensus_crypto = { path = "libs/crypto" } @@ -43,19 +47,22 @@ ark-ec = "0.4.2" ark-serialize = { version = "0.4.2", features = ["std"] } num-traits = "0.2.17" clap = { version = "4.3.3", features = ["derive"] } +heck = "0.4.1" ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } hex = "0.4.3" im = "15.1.0" once_cell = "1.17.1" pin-project = "1.1.0" -prost = "0.11.0" -prost-build = "0.11.0" -prost-reflect = { version = "0.11.0", features = ["derive", "serde"] } -prost-reflect-build = "0.11.0" -protoc-bin-vendored = "3.0.0" +prost = "0.12.0" +prost-build = "0.12.0" +prost-reflect = { version = "0.12.0", features = ["serde"] } +prost-types = "0.12.0" +protox = "0.5.0" +protox-parse = "0.5.0" prettyplease = "0.2.6" pretty_assertions = "1.4.0" quick-protobuf = "0.8.1" +quote = "1.0.33" rand = "0.8.0" rocksdb = "0.21.0" serde = { version = "1.0", features = ["derive"] } diff --git a/node/actors/bft/Cargo.toml b/node/actors/bft/Cargo.toml index 5f61f2cf..db42209d 100644 --- a/node/actors/bft/Cargo.toml +++ b/node/actors/bft/Cargo.toml @@ -14,6 +14,7 @@ zksync_consensus_roles.workspace = true zksync_consensus_schema.workspace = true zksync_consensus_storage.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true once_cell.workspace = true diff --git a/node/actors/bft/src/inner.rs b/node/actors/bft/src/inner.rs index 56f8d57a..631e90d2 100644 --- a/node/actors/bft/src/inner.rs +++ b/node/actors/bft/src/inner.rs @@ -6,7 +6,6 @@ use crate::{ }; use tracing::instrument; use zksync_consensus_roles::validator; -use zksync_consensus_schema as schema; use zksync_consensus_utils::pipe::ActorPipe; /// The ConsensusInner struct, it contains data to be shared with the state machines. This is never supposed @@ -24,7 +23,7 @@ pub(crate) struct ConsensusInner { impl ConsensusInner { /// The maximum size of the payload of a block, in bytes. We will /// reject blocks with payloads larger than this. - pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * schema::kB; + pub(crate) const PAYLOAD_MAX_SIZE: usize = 500 * zksync_protobuf::kB; /// Computes the validator for the given view. #[instrument(level = "trace", ret)] diff --git a/node/actors/bft/src/metrics.rs b/node/actors/bft/src/metrics.rs index 461403f7..8fef767e 100644 --- a/node/actors/bft/src/metrics.rs +++ b/node/actors/bft/src/metrics.rs @@ -2,10 +2,11 @@ use std::time::Duration; use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; -use zksync_consensus_schema as schema; -const PAYLOAD_SIZE_BUCKETS: Buckets = - Buckets::exponential((4 * schema::kB) as f64..=(4 * schema::MB) as f64, 4.0); +const PAYLOAD_SIZE_BUCKETS: Buckets = Buckets::exponential( + (4 * zksync_protobuf::kB) as f64..=(4 * zksync_protobuf::MB) as f64, + 4.0, +); /// Label for a consensus message. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index 9bbe7188..9f94353e 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -16,6 +16,7 @@ zksync_consensus_schema.workspace = true zksync_consensus_storage.workspace = true zksync_consensus_sync_blocks.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true rand.workspace = true diff --git a/node/actors/executor/src/config/mod.rs b/node/actors/executor/src/config/mod.rs index 4ab296fa..0ebfab55 100644 --- a/node/actors/executor/src/config/mod.rs +++ b/node/actors/executor/src/config/mod.rs @@ -1,5 +1,4 @@ //! Module to create the configuration for the consensus node. - use anyhow::Context as _; use std::{ collections::{HashMap, HashSet}, @@ -8,9 +7,8 @@ use std::{ use zksync_consensus_crypto::{read_required_text, Text, TextFmt}; use zksync_consensus_network::{consensus, gossip}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::{ - proto::executor::config as proto, read_required, required, ProtoFmt, -}; +use zksync_consensus_schema::proto::executor::config as proto; +use zksync_protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod tests; diff --git a/node/actors/executor/src/config/tests.rs b/node/actors/executor/src/config/tests.rs index 7b1ab797..3c18f072 100644 --- a/node/actors/executor/src/config/tests.rs +++ b/node/actors/executor/src/config/tests.rs @@ -5,7 +5,7 @@ use rand::{ }; use zksync_concurrency::ctx; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; fn make_addr(rng: &mut R) -> std::net::SocketAddr { std::net::SocketAddr::new(std::net::IpAddr::from(rng.gen::<[u8; 16]>()), rng.gen()) diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index 30fd0e9a..20c85bcb 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -12,6 +12,7 @@ zksync_consensus_crypto.workspace = true zksync_consensus_roles.workspace = true zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true async-trait.workspace = true diff --git a/node/actors/network/src/consensus/handshake/mod.rs b/node/actors/network/src/consensus/handshake/mod.rs index fa304488..65275cc6 100644 --- a/node/actors/network/src/consensus/handshake/mod.rs +++ b/node/actors/network/src/consensus/handshake/mod.rs @@ -3,7 +3,8 @@ use anyhow::Context as _; use zksync_concurrency::{ctx, time}; use zksync_consensus_crypto::ByteFmt; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::{proto::network::consensus as proto, read_required, ProtoFmt}; +use zksync_consensus_schema::proto::network::consensus as proto; +use zksync_protobuf::{read_required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/consensus/handshake/tests.rs b/node/actors/network/src/consensus/handshake/tests.rs index ed49c14d..747844fe 100644 --- a/node/actors/network/src/consensus/handshake/tests.rs +++ b/node/actors/network/src/consensus/handshake/tests.rs @@ -3,7 +3,7 @@ use crate::{frame, noise, testonly}; use rand::Rng; use zksync_concurrency::{ctx, io, scope, testonly::abort_on_panic}; use zksync_consensus_roles::validator; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_schema_encode_decode() { diff --git a/node/actors/network/src/frame.rs b/node/actors/network/src/frame.rs index f865be95..c20969f5 100644 --- a/node/actors/network/src/frame.rs +++ b/node/actors/network/src/frame.rs @@ -2,13 +2,12 @@ //! since protobuf messages do not have delimiters. use crate::{mux, noise::bytes}; use zksync_concurrency::{ctx, io}; -use zksync_consensus_schema as schema; /// Reads a raw frame of bytes from the stream and interprets it as proto. /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. /// Returns the decoded proto and the size of the received message in bytes. -pub(crate) async fn mux_recv_proto( +pub(crate) async fn mux_recv_proto( ctx: &ctx::Ctx, stream: &mut mux::ReadStream, ) -> anyhow::Result<(T, usize)> { @@ -26,19 +25,19 @@ pub(crate) async fn mux_recv_proto( if msg.len() < msg_size { anyhow::bail!("end of stream"); } - let msg = schema::decode(msg.as_slice())?; + let msg = zksync_protobuf::decode(msg.as_slice())?; Ok((msg, msg_size)) } /// Sends a proto serialized to a raw frame of bytes to the stream. /// It doesn't flush the stream. /// Returns the size of the sent proto in bytes. -pub(crate) async fn mux_send_proto( +pub(crate) async fn mux_send_proto( ctx: &ctx::Ctx, stream: &mut mux::WriteStream, msg: &T, ) -> anyhow::Result { - let msg = schema::encode(msg); + let msg = zksync_protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); stream .write_all(ctx, &u32::to_le_bytes(msg.len() as u32)) @@ -50,7 +49,7 @@ pub(crate) async fn mux_send_proto( /// Reads a raw frame of bytes from the stream and interprets it as proto. /// A `frame : [u8]` is encoded as `L ++ frame`, where `L` is /// a little endian encoding of `frame.len() as u32`. -pub(crate) async fn recv_proto( +pub(crate) async fn recv_proto( ctx: &ctx::Ctx, stream: &mut S, ) -> anyhow::Result { @@ -62,16 +61,16 @@ pub(crate) async fn recv_proto( } let mut msg = vec![0u8; msg_size as usize]; io::read_exact(ctx, stream, &mut msg[..]).await??; - schema::decode(&msg) + zksync_protobuf::decode(&msg) } /// Sends a proto serialized to a raw frame of bytes to the stream. -pub(crate) async fn send_proto( +pub(crate) async fn send_proto( ctx: &ctx::Ctx, stream: &mut S, msg: &T, ) -> anyhow::Result<()> { - let msg = schema::encode(msg); + let msg = zksync_protobuf::encode(msg); assert!(msg.len() <= T::max_size(), "message too large"); io::write_all(ctx, stream, &u32::to_le_bytes(msg.len() as u32)).await??; io::write_all(ctx, stream, &msg).await??; diff --git a/node/actors/network/src/gossip/handshake/mod.rs b/node/actors/network/src/gossip/handshake/mod.rs index 9114ac0d..45558c53 100644 --- a/node/actors/network/src/gossip/handshake/mod.rs +++ b/node/actors/network/src/gossip/handshake/mod.rs @@ -4,7 +4,8 @@ use anyhow::Context as _; use zksync_concurrency::{ctx, time}; use zksync_consensus_crypto::ByteFmt; use zksync_consensus_roles::node; -use zksync_consensus_schema::{proto::network::gossip as proto, read_required, required, ProtoFmt}; +use zksync_consensus_schema::proto::network::gossip as proto; +use zksync_protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] mod testonly; diff --git a/node/actors/network/src/gossip/handshake/tests.rs b/node/actors/network/src/gossip/handshake/tests.rs index 39c26457..457eadd3 100644 --- a/node/actors/network/src/gossip/handshake/tests.rs +++ b/node/actors/network/src/gossip/handshake/tests.rs @@ -4,7 +4,7 @@ use rand::Rng; use std::collections::{HashMap, HashSet}; use zksync_concurrency::{ctx, io, scope, testonly::abort_on_panic}; use zksync_consensus_roles::node; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_schema_encode_decode() { diff --git a/node/actors/network/src/mux/handshake.rs b/node/actors/network/src/mux/handshake.rs index 83ac6a52..bfee19db 100644 --- a/node/actors/network/src/mux/handshake.rs +++ b/node/actors/network/src/mux/handshake.rs @@ -1,8 +1,8 @@ use super::CapabilityId; use anyhow::Context as _; use std::collections::HashMap; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::mux as proto, required}; +use zksync_consensus_schema::proto::network::mux as proto; +use zksync_protobuf::required; pub(super) struct Handshake { /// Maximal supported number of the accept streams per capability. @@ -37,11 +37,11 @@ fn build_capabilities( .collect() } -impl schema::ProtoFmt for Handshake { +impl zksync_protobuf::ProtoFmt for Handshake { type Proto = proto::Handshake; fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { diff --git a/node/actors/network/src/mux/tests.rs b/node/actors/network/src/mux/tests.rs index 19a82458..2e334964 100644 --- a/node/actors/network/src/mux/tests.rs +++ b/node/actors/network/src/mux/tests.rs @@ -9,7 +9,6 @@ use std::{ }, }; use zksync_concurrency::{ctx, scope, testonly::abort_on_panic}; -use zksync_consensus_schema as schema; use zksync_consensus_schema::proto::network::mux_test as proto; use zksync_consensus_utils::no_copy::NoCopy; @@ -73,7 +72,7 @@ struct Resp { capability_id: mux::CapabilityId, } -impl schema::ProtoFmt for Req { +impl zksync_protobuf::ProtoFmt for Req { type Proto = proto::Req; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(r.input.as_ref().unwrap().clone())) @@ -85,7 +84,7 @@ impl schema::ProtoFmt for Req { } } -impl schema::ProtoFmt for Resp { +impl zksync_protobuf::ProtoFmt for Resp { type Proto = proto::Resp; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { diff --git a/node/actors/network/src/preface.rs b/node/actors/network/src/preface.rs index 5cb53ac9..75a042c1 100644 --- a/node/actors/network/src/preface.rs +++ b/node/actors/network/src/preface.rs @@ -9,8 +9,8 @@ //! and multiplex between mutliple endpoints available on the same TCP port. use crate::{frame, metrics, noise}; use zksync_concurrency::{ctx, time}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::preface as proto, required, ProtoFmt}; +use zksync_consensus_schema::proto::network::preface as proto; +use zksync_protobuf::{required, ProtoFmt}; /// Timeout on executing the preface protocol. const TIMEOUT: time::Duration = time::Duration::seconds(5); @@ -34,7 +34,7 @@ pub(crate) enum Endpoint { impl ProtoFmt for Encryption { type Proto = proto::Encryption; fn max_size() -> usize { - 10 * schema::kB + 10 * zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::encryption::T; @@ -54,7 +54,7 @@ impl ProtoFmt for Encryption { impl ProtoFmt for Endpoint { type Proto = proto::Endpoint; fn max_size() -> usize { - 10 * schema::kB + 10 * zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { use proto::endpoint::T; diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 86365eb0..90a20b6e 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -2,8 +2,8 @@ use crate::mux; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::consensus as proto, read_required, ProtoFmt}; +use zksync_consensus_schema::proto::network::consensus as proto; +use zksync_protobuf::{read_required, ProtoFmt}; /// Consensus RPC. pub(crate) struct Rpc; @@ -46,7 +46,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - schema::MB + zksync_protobuf::MB } } @@ -62,6 +62,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } diff --git a/node/actors/network/src/rpc/metrics.rs b/node/actors/network/src/rpc/metrics.rs index a9111cec..34394e28 100644 --- a/node/actors/network/src/rpc/metrics.rs +++ b/node/actors/network/src/rpc/metrics.rs @@ -6,7 +6,6 @@ use vise::{ Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, LabeledFamily, Metrics, Unit, }; -use zksync_consensus_schema as schema; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] #[metrics(rename_all = "snake_case")] @@ -87,7 +86,7 @@ pub(super) struct CallLabels { } const MESSAGE_SIZE_BUCKETS: Buckets = - Buckets::exponential(schema::kB as f64..=schema::MB as f64, 2.0); + Buckets::exponential(zksync_protobuf::kB as f64..=zksync_protobuf::MB as f64, 2.0); #[derive(Debug, Metrics)] #[metrics(prefix = "network_rpc")] diff --git a/node/actors/network/src/rpc/mod.rs b/node/actors/network/src/rpc/mod.rs index b3173821..77f9a52a 100644 --- a/node/actors/network/src/rpc/mod.rs +++ b/node/actors/network/src/rpc/mod.rs @@ -20,7 +20,6 @@ use crate::{frame, mux}; use anyhow::Context as _; use std::{collections::BTreeMap, sync::Arc}; use zksync_concurrency::{ctx, io, limiter, metrics::LatencyHistogramExt as _, scope}; -use zksync_consensus_schema as schema; pub(crate) mod consensus; mod metrics; @@ -33,10 +32,10 @@ pub(crate) mod testonly; mod tests; const MUX_CONFIG: mux::Config = mux::Config { - read_buffer_size: 160 * schema::kB as u64, - read_frame_size: 16 * schema::kB as u64, + read_buffer_size: 160 * zksync_protobuf::kB as u64, + read_frame_size: 16 * zksync_protobuf::kB as u64, read_frame_count: 100, - write_frame_size: 16 * schema::kB as u64, + write_frame_size: 16 * zksync_protobuf::kB as u64, }; /// Trait for defining an RPC. @@ -58,9 +57,9 @@ pub(crate) trait Rpc: Sync + Send + 'static { /// Name of the RPC, used in prometheus metrics. const METHOD: &'static str; /// Type of the request message. - type Req: schema::ProtoFmt + Send + Sync; + type Req: zksync_protobuf::ProtoFmt + Send + Sync; /// Type of the response message. - type Resp: schema::ProtoFmt + Send + Sync; + type Resp: zksync_protobuf::ProtoFmt + Send + Sync; /// Name of the variant of the request message type. /// Useful for collecting metrics with finer than /// per-rpc type granularity. diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index 98a292bc..fda4ebf8 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -3,8 +3,8 @@ use crate::{mux, rpc::Rpc as _}; use anyhow::Context as _; use rand::Rng; use zksync_concurrency::{ctx, limiter, time}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::ping as proto, required, ProtoFmt}; +use zksync_consensus_schema::proto::network::ping as proto; +use zksync_protobuf::{required, ProtoFmt}; /// Ping RPC. pub(crate) struct Rpc; @@ -65,7 +65,7 @@ pub(crate) struct Resp(pub(crate) [u8; 32]); impl ProtoFmt for Req { type Proto = proto::PingReq; fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) @@ -80,7 +80,7 @@ impl ProtoFmt for Req { impl ProtoFmt for Resp { type Proto = proto::PingResp; fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(required(&r.data)?[..].try_into()?)) diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index 26454b53..4dd22af7 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -1,11 +1,10 @@ //! Defines RPC for synchronizing blocks. - use crate::{io, mux}; use anyhow::Context; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::gossip as proto, read_required, ProtoFmt}; +use zksync_consensus_schema::proto::network::gossip as proto; +use zksync_protobuf::{read_required, ProtoFmt}; /// `get_sync_state` RPC. #[derive(Debug)] @@ -48,7 +47,7 @@ impl ProtoFmt for io::SyncState { fn max_size() -> usize { // TODO: estimate maximum size more precisely - 100 * schema::kB + 100 * zksync_protobuf::kB } } @@ -68,7 +67,7 @@ impl ProtoFmt for SyncStateResponse { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -110,7 +109,7 @@ impl ProtoFmt for GetBlockRequest { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -121,7 +120,7 @@ impl ProtoFmt for io::GetBlockError { use proto::get_block_response::ErrorReason; let reason = message.reason.context("missing reason")?; - let reason = ErrorReason::from_i32(reason).context("reason")?; + let reason = ErrorReason::try_from(reason).context("reason")?; Ok(match reason { ErrorReason::NotSynced => Self::NotSynced, }) @@ -138,7 +137,7 @@ impl ProtoFmt for io::GetBlockError { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -179,6 +178,6 @@ impl ProtoFmt for GetBlockResponse { } fn max_size() -> usize { - schema::MB + zksync_protobuf::MB } } diff --git a/node/actors/network/src/rpc/sync_validator_addrs.rs b/node/actors/network/src/rpc/sync_validator_addrs.rs index 587c4dbd..d505feda 100644 --- a/node/actors/network/src/rpc/sync_validator_addrs.rs +++ b/node/actors/network/src/rpc/sync_validator_addrs.rs @@ -4,8 +4,8 @@ use anyhow::Context as _; use std::sync::Arc; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{proto::network::gossip as proto, ProtoFmt}; +use zksync_consensus_schema::proto::network::gossip as proto; +use zksync_protobuf::ProtoFmt; /// SyncValidatorAddrs Rpc. pub(crate) struct Rpc; @@ -45,7 +45,7 @@ impl ProtoFmt for Req { } fn max_size() -> usize { - schema::kB + zksync_protobuf::kB } } @@ -69,6 +69,6 @@ impl ProtoFmt for Resp { } fn max_size() -> usize { - schema::MB + zksync_protobuf::MB } } diff --git a/node/actors/network/src/rpc/tests.rs b/node/actors/network/src/rpc/tests.rs index d2e98851..04f28c2a 100644 --- a/node/actors/network/src/rpc/tests.rs +++ b/node/actors/network/src/rpc/tests.rs @@ -6,7 +6,7 @@ use std::{ sync::atomic::{AtomicU64, Ordering}, }; use zksync_concurrency::{ctx, testonly::abort_on_panic, time}; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; /// CAPABILITY_ID should uniquely identify the RPC. #[test] diff --git a/node/deny.toml b/node/deny.toml index 6fdd9bb5..8bfe4c74 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -47,10 +47,6 @@ allow = [ multiple-versions = "deny" # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ - # Old versions required by prost. - { name = "prettyplease", version = "=0.1.25" }, - { name = "syn", version = "=1.0.109" }, - # Old versions required by tempfile and prost-build. { name = "bitflags", version = "=1.3.2" }, @@ -61,6 +57,9 @@ skip = [ # Old versions required by hyper. { name = "socket2", version = "=0.4.9" }, { name = "hashbrown", version = "=0.12.3" }, # (hyper -> h2 -> indexmap -> hashbrown) + + # Old versions required by ark-bn254. + { name = "syn", version = "=1.0.109" } ] [sources] diff --git a/node/libs/protobuf/Cargo.toml b/node/libs/protobuf/Cargo.toml new file mode 100644 index 00000000..83eba563 --- /dev/null +++ b/node/libs/protobuf/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "zksync_protobuf" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[[bin]] +name = "conformance_test" + +[dependencies] +zksync_concurrency.workspace = true +zksync_protobuf_build.workspace = true + +anyhow.workspace = true +bit-vec.workspace = true +serde.workspace = true +quick-protobuf.workspace = true +once_cell.workspace = true +prost.workspace = true +prost-reflect.workspace = true +rand.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[build-dependencies] +zksync_protobuf_build.workspace = true + +anyhow.workspace = true diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs new file mode 100644 index 00000000..0348639b --- /dev/null +++ b/node/libs/protobuf/build.rs @@ -0,0 +1,29 @@ +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf_build::Config { + input_root: "src/proto".into(), + proto_root: "zksync".into(), + dependencies: vec![], + protobuf_crate: "crate".into(), + } + .generate() + .expect("generate(std)"); + + zksync_protobuf_build::Config { + input_root: "src/tests/proto".into(), + proto_root: "zksync/protobuf/tests".into(), + dependencies: vec![], + protobuf_crate: "crate".into(), + } + .generate() + .expect("generate(test)"); + + zksync_protobuf_build::Config { + input_root: "src/bin/conformance_test/proto".into(), + proto_root: "zksync/protobuf/conformance_test".into(), + dependencies: vec![], + protobuf_crate: "::zksync_protobuf".into(), + } + .generate() + .expect("generate(conformance)"); +} diff --git a/node/libs/schema/src/bin/conformance_test_failure_list.txt b/node/libs/protobuf/src/bin/conformance_test/failure_list.txt similarity index 100% rename from node/libs/schema/src/bin/conformance_test_failure_list.txt rename to node/libs/protobuf/src/bin/conformance_test/failure_list.txt diff --git a/node/libs/schema/src/bin/conformance_test.rs b/node/libs/protobuf/src/bin/conformance_test/main.rs similarity index 71% rename from node/libs/schema/src/bin/conformance_test.rs rename to node/libs/protobuf/src/bin/conformance_test/main.rs index 38b9d1bb..4d3a48a3 100644 --- a/node/libs/schema/src/bin/conformance_test.rs +++ b/node/libs/protobuf/src/bin/conformance_test/main.rs @@ -1,19 +1,20 @@ //! Conformance test for our canonical encoding implemented according to //! https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto //! Our implementation supports only a subset of proto functionality, so -//! `schema/proto/conformance/conformance.proto` and -//! `schema/proto/conformance/protobuf_test_messages.proto` contains only a +//! `proto/conformance.proto` and +//! `proto/protobuf_test_messages.proto` contains only a //! subset of original fields. Also we run only proto3 binary -> binary tests. -//! conformance_test_failure_list.txt contains tests which are expected to fail. +//! failure_list.txt contains tests which are expected to fail. use anyhow::Context as _; use prost::Message as _; use prost_reflect::ReflectMessage; +use std::sync::Mutex; use zksync_concurrency::{ctx, io}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::proto::conformance as proto; -#[tokio::main] -async fn main() -> anyhow::Result<()> { +mod proto; + +/// Runs the test server. +async fn run() -> anyhow::Result<()> { let ctx = &ctx::root(); let stdin = &mut tokio::io::stdin(); let stdout = &mut tokio::io::stdout(); @@ -37,10 +38,10 @@ async fn main() -> anyhow::Result<()> { // Decode. let payload = req.payload.context("missing payload")?; - use zksync_consensus_schema::proto::protobuf_test_messages::proto3::TestAllTypesProto3 as T; + use proto::TestAllTypesProto3 as T; let p = match payload { proto::conformance_request::Payload::JsonPayload(payload) => { - match schema::decode_json_proto(&payload) { + match zksync_protobuf::decode_json_proto(&payload) { Ok(p) => p, Err(_) => return Ok(R::Skipped("unsupported fields".to_string())), } @@ -51,7 +52,7 @@ async fn main() -> anyhow::Result<()> { return Ok(R::ParseError("parsing failed".to_string())); }; // Then check if there are any unknown fields in the original payload. - if schema::canonical_raw(&payload[..], &p.descriptor()).is_err() { + if zksync_protobuf::canonical_raw(&payload[..], &p.descriptor()).is_err() { return Ok(R::Skipped("unsupported fields".to_string())); } p @@ -63,13 +64,13 @@ async fn main() -> anyhow::Result<()> { let format = req .requested_output_format .context("missing output format")?; - match proto::WireFormat::from_i32(format).context("unknown format")? { + match proto::WireFormat::try_from(format).context("unknown format")? { proto::WireFormat::Json => { - anyhow::Ok(R::JsonPayload(schema::encode_json_proto(&p))) + anyhow::Ok(R::JsonPayload(zksync_protobuf::encode_json_proto(&p))) } proto::WireFormat::Protobuf => { // Reencode the parsed proto. - anyhow::Ok(R::ProtobufPayload(schema::canonical_raw( + anyhow::Ok(R::ProtobufPayload(zksync_protobuf::canonical_raw( &p.encode_to_vec(), &p.descriptor(), )?)) @@ -87,3 +88,24 @@ async fn main() -> anyhow::Result<()> { io::flush(ctx, stdout).await??; } } + +#[tokio::main] +async fn main() { + let sub = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()); + match std::env::var("LOG_FILE") { + Err(_) => sub.with_writer(std::io::stderr).init(), + Ok(path) => sub + .with_writer(Mutex::new( + std::fs::File::options() + .create(true) + .append(true) + .open(path) + .unwrap(), + )) + .init(), + }; + if let Err(err) = run().await { + tracing::error!("run(): {err:#}"); + } +} diff --git a/node/libs/schema/proto/conformance/conformance.proto b/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto similarity index 98% rename from node/libs/schema/proto/conformance/conformance.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto index e6403673..819bb61d 100644 --- a/node/libs/schema/proto/conformance/conformance.proto +++ b/node/libs/protobuf/src/bin/conformance_test/proto/conformance.proto @@ -33,10 +33,7 @@ syntax = "proto3"; -package conformance; - -option java_package = "com.google.protobuf.conformance"; -option objc_class_prefix = "Conformance"; +package zksync.protobuf.conformance_test; // This defines the conformance testing protocol. This protocol exists between // the conformance test suite itself and the code being tested. For each test, diff --git a/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs new file mode 100644 index 00000000..70249255 --- /dev/null +++ b/node/libs/protobuf/src/bin/conformance_test/proto/mod.rs @@ -0,0 +1,5 @@ +#![allow(warnings)] +include!(concat!( + env!("OUT_DIR"), + "/src/bin/conformance_test/proto/gen.rs" +)); diff --git a/node/libs/schema/proto/conformance/test_messages_proto3.proto b/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto similarity index 96% rename from node/libs/schema/proto/conformance/test_messages_proto3.proto rename to node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto index d391761b..242d8f58 100644 --- a/node/libs/schema/proto/conformance/test_messages_proto3.proto +++ b/node/libs/protobuf/src/bin/conformance_test/proto/test_messages_proto3.proto @@ -40,15 +40,7 @@ syntax = "proto3"; -package protobuf_test_messages.proto3; - -option java_package = "com.google.protobuf_test_messages.proto3"; -option objc_class_prefix = "Proto3"; - -// This is the default, but we specify it here explicitly. -option optimize_for = SPEED; - -option cc_enable_arenas = true; +package zksync.protobuf.conformance_test; // This proto includes every type of field in both singular and repeated // forms. diff --git a/node/libs/protobuf/src/lib.rs b/node/libs/protobuf/src/lib.rs new file mode 100644 index 00000000..f461892a --- /dev/null +++ b/node/libs/protobuf/src/lib.rs @@ -0,0 +1,13 @@ +//! Code generated from protobuf schema files and +//! utilities for serialization. + +pub mod proto; +mod proto_fmt; +mod std_conv; +pub mod testonly; + +pub use proto_fmt::*; +pub use zksync_protobuf_build as build; + +#[cfg(test)] +mod tests; diff --git a/node/libs/protobuf/src/proto/mod.rs b/node/libs/protobuf/src/proto/mod.rs new file mode 100644 index 00000000..660bf4c5 --- /dev/null +++ b/node/libs/protobuf/src/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs")); diff --git a/node/libs/schema/proto/std.proto b/node/libs/protobuf/src/proto/std.proto similarity index 98% rename from node/libs/schema/proto/std.proto rename to node/libs/protobuf/src/proto/std.proto index f5ee5efe..05b6957a 100644 --- a/node/libs/schema/proto/std.proto +++ b/node/libs/protobuf/src/proto/std.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package std; +package zksync.std; message Void {} diff --git a/node/libs/schema/src/proto_fmt.rs b/node/libs/protobuf/src/proto_fmt.rs similarity index 100% rename from node/libs/schema/src/proto_fmt.rs rename to node/libs/protobuf/src/proto_fmt.rs diff --git a/node/libs/schema/src/std_conv.rs b/node/libs/protobuf/src/std_conv.rs similarity index 92% rename from node/libs/schema/src/std_conv.rs rename to node/libs/protobuf/src/std_conv.rs index 264969b5..76ac4137 100644 --- a/node/libs/schema/src/std_conv.rs +++ b/node/libs/protobuf/src/std_conv.rs @@ -1,11 +1,11 @@ //! Proto conversion for messages in std package. -use crate::{proto::std as proto, required, ProtoFmt}; +use crate::{proto, required, ProtoFmt}; use anyhow::Context as _; use std::net; use zksync_concurrency::time; impl ProtoFmt for () { - type Proto = proto::Void; + type Proto = proto::std::Void; fn read(_r: &Self::Proto) -> anyhow::Result { Ok(()) } @@ -15,7 +15,7 @@ impl ProtoFmt for () { } impl ProtoFmt for std::net::SocketAddr { - type Proto = proto::SocketAddr; + type Proto = proto::std::SocketAddr; fn read(r: &Self::Proto) -> anyhow::Result { let ip = required(&r.ip).context("ip")?; @@ -41,7 +41,7 @@ impl ProtoFmt for std::net::SocketAddr { } impl ProtoFmt for time::Utc { - type Proto = proto::Timestamp; + type Proto = proto::std::Timestamp; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -59,7 +59,7 @@ impl ProtoFmt for time::Utc { } impl ProtoFmt for time::Duration { - type Proto = proto::Duration; + type Proto = proto::std::Duration; fn read(r: &Self::Proto) -> anyhow::Result { let seconds = *required(&r.seconds).context("seconds")?; @@ -82,7 +82,7 @@ impl ProtoFmt for time::Duration { } impl ProtoFmt for bit_vec::BitVec { - type Proto = proto::BitVector; + type Proto = proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { let size = *required(&r.size).context("size")? as usize; diff --git a/node/libs/schema/src/testonly.rs b/node/libs/protobuf/src/testonly.rs similarity index 100% rename from node/libs/schema/src/testonly.rs rename to node/libs/protobuf/src/testonly.rs diff --git a/node/libs/schema/src/tests.rs b/node/libs/protobuf/src/tests/mod.rs similarity index 94% rename from node/libs/schema/src/tests.rs rename to node/libs/protobuf/src/tests/mod.rs index ddd49783..a41d6e45 100644 --- a/node/libs/schema/src/tests.rs +++ b/node/libs/protobuf/src/tests/mod.rs @@ -3,6 +3,8 @@ use anyhow::Context as _; use std::net; use zksync_concurrency::{ctx, time}; +mod proto; + #[derive(Debug, PartialEq, Eq)] enum B { U(bool), @@ -10,16 +12,16 @@ enum B { } impl ProtoFmt for B { - type Proto = proto::testonly::B; + type Proto = proto::B; fn read(r: &Self::Proto) -> anyhow::Result { - use proto::testonly::b::T; + use proto::b::T; Ok(match required(&r.t)? { T::U(x) => Self::U(*x), T::V(x) => Self::V(Box::new(ProtoFmt::read(x.as_ref())?)), }) } fn build(&self) -> Self::Proto { - use proto::testonly::b::T; + use proto::b::T; let t = match self { Self::U(x) => T::U(*x), Self::V(x) => T::V(Box::new(x.build())), @@ -37,7 +39,7 @@ struct A { } impl ProtoFmt for A { - type Proto = proto::testonly::A; + type Proto = proto::A; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { x: required(&r.x).context("x")?.clone(), diff --git a/node/libs/protobuf/src/tests/proto/mod.rs b/node/libs/protobuf/src/tests/proto/mod.rs new file mode 100644 index 00000000..a4703251 --- /dev/null +++ b/node/libs/protobuf/src/tests/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/tests/proto/gen.rs")); diff --git a/node/libs/schema/proto/testonly.proto b/node/libs/protobuf/src/tests/proto/tests.proto similarity index 91% rename from node/libs/schema/proto/testonly.proto rename to node/libs/protobuf/src/tests/proto/tests.proto index 7c5d08cd..91cf9e35 100644 --- a/node/libs/schema/proto/testonly.proto +++ b/node/libs/protobuf/src/tests/proto/tests.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package testonly; +package zksync.protobuf.tests; message B { // Recursive union. diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml new file mode 100644 index 00000000..46129be5 --- /dev/null +++ b/node/libs/protobuf_build/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "zksync_protobuf_build" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +heck.workspace = true +once_cell.workspace = true +prettyplease.workspace = true +prost.workspace = true +prost-build.workspace = true +prost-types.workspace = true +prost-reflect.workspace = true +protox.workspace = true +protox-parse.workspace = true +quote.workspace = true +rand.workspace = true +syn.workspace = true + diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs new file mode 100644 index 00000000..e283663e --- /dev/null +++ b/node/libs/protobuf_build/src/canonical.rs @@ -0,0 +1,58 @@ +//! Checks whether messages in the given file descriptor set support canonical encoding. +use crate::syntax::extract_message_names; +use anyhow::Context as _; +use std::collections::HashSet; + +#[derive(Default)] +struct Check(HashSet); + +impl Check { + /// Checks if messages of type `m` support canonical encoding. + fn check_message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { + if self.0.contains(m.full_name()) { + return Ok(()); + } + self.0.insert(m.full_name().to_string()); + for f in m.fields() { + self.check_field(&f).with_context(|| f.name().to_string())?; + } + Ok(()) + } + + /// Checks if field `f` supports canonical encoding. + fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + if f.is_map() { + anyhow::bail!("maps unsupported"); + } + if !f.is_list() && !f.supports_presence() { + anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); + } + if let prost_reflect::Kind::Message(msg) = &f.kind() { + self.check_message(msg) + .with_context(|| msg.name().to_string())?; + } + Ok(()) + } +} + +/// Checks whether messages in the given file descriptor set support canonical encoding. +/// pool should contain all transitive dependencies of files in descriptor. +pub(crate) fn check( + descriptor: &prost_types::FileDescriptorSet, + pool: &prost_reflect::DescriptorPool, +) -> anyhow::Result<()> { + for f in &descriptor.file { + if f.syntax() != "proto3" { + anyhow::bail!("{}: only proto3 syntax is supported", f.name()); + } + } + let mut c = Check::default(); + for msg_name in extract_message_names(descriptor) { + let msg_name = msg_name.to_string(); + let msg = pool + .get_message_by_name(&msg_name) + .with_context(|| format!("{msg_name} not found in pool"))?; + c.check_message(&msg).with_context(|| msg_name)?; + } + Ok(()) +} diff --git a/node/libs/protobuf_build/src/ident.rs b/node/libs/protobuf_build/src/ident.rs new file mode 100644 index 00000000..9dc4e003 --- /dev/null +++ b/node/libs/protobuf_build/src/ident.rs @@ -0,0 +1,163 @@ +//! Copy of https://github.com/tokio-rs/prost/blob/master/prost-build/src/ident.rs. +//! We need it to compute the proto names -> rust names mapping, which is not directly accessible +//! via prost_build public API. + +use heck::{ToSnakeCase, ToUpperCamelCase}; + +/// Converts a `camelCase` or `SCREAMING_SNAKE_CASE` identifier to a `lower_snake` case Rust field +/// identifier. +pub(crate) fn to_snake(s: &str) -> String { + let mut ident = s.to_snake_case(); + + // Use a raw identifier if the identifier matches a Rust keyword: + // https://doc.rust-lang.org/reference/keywords.html. + match ident.as_str() { + // 2015 strict keywords. + | "as" | "break" | "const" | "continue" | "else" | "enum" | "false" + | "fn" | "for" | "if" | "impl" | "in" | "let" | "loop" | "match" | "mod" | "move" | "mut" + | "pub" | "ref" | "return" | "static" | "struct" | "trait" | "true" + | "type" | "unsafe" | "use" | "where" | "while" + // 2018 strict keywords. + | "dyn" + // 2015 reserved keywords. + | "abstract" | "become" | "box" | "do" | "final" | "macro" | "override" | "priv" | "typeof" + | "unsized" | "virtual" | "yield" + // 2018 reserved keywords. + | "async" | "await" | "try" => ident.insert_str(0, "r#"), + // the following keywords are not supported as raw identifiers and are therefore suffixed with an underscore. + "self" | "super" | "extern" | "crate" => ident += "_", + _ => (), + } + ident +} + +/// Converts a `snake_case` identifier to an `UpperCamel` case Rust type identifier. +pub(crate) fn to_upper_camel(s: &str) -> String { + let mut ident = s.to_upper_camel_case(); + + // Suffix an underscore for the `Self` Rust keyword as it is not allowed as raw identifier. + if ident == "Self" { + ident += "_"; + } + ident +} + +#[cfg(test)] +mod tests { + + #![allow(clippy::cognitive_complexity)] + + use super::*; + + #[test] + fn test_to_snake() { + assert_eq!("foo_bar", &to_snake("FooBar")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("xml_http_request", &to_snake("XMLHttpRequest")); + assert_eq!("r#while", &to_snake("While")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_BUSTER")); + assert_eq!("foo_bar_baz", &to_snake("foo_bar_baz")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_buster")); + assert_eq!("fuzz", &to_snake("_FUZZ")); + assert_eq!("fuzz", &to_snake("_fuzz")); + assert_eq!("fuzz", &to_snake("_Fuzz")); + assert_eq!("fuzz", &to_snake("FUZZ_")); + assert_eq!("fuzz", &to_snake("fuzz_")); + assert_eq!("fuzz", &to_snake("Fuzz_")); + assert_eq!("fuz_z", &to_snake("FuzZ_")); + + // From test_messages_proto3.proto. + assert_eq!("fieldname1", &to_snake("fieldname1")); + assert_eq!("field_name2", &to_snake("field_name2")); + assert_eq!("field_name3", &to_snake("_field_name3")); + assert_eq!("field_name4", &to_snake("field__name4_")); + assert_eq!("field0name5", &to_snake("field0name5")); + assert_eq!("field_0_name6", &to_snake("field_0_name6")); + assert_eq!("field_name7", &to_snake("fieldName7")); + assert_eq!("field_name8", &to_snake("FieldName8")); + assert_eq!("field_name9", &to_snake("field_Name9")); + assert_eq!("field_name10", &to_snake("Field_Name10")); + + assert_eq!("field_name11", &to_snake("FIELD_NAME11")); + assert_eq!("field_name12", &to_snake("FIELD_name12")); + assert_eq!("field_name13", &to_snake("__field_name13")); + assert_eq!("field_name14", &to_snake("__Field_name14")); + assert_eq!("field_name15", &to_snake("field__name15")); + assert_eq!("field_name16", &to_snake("field__Name16")); + assert_eq!("field_name17", &to_snake("field_name17__")); + assert_eq!("field_name18", &to_snake("Field_name18__")); + } + + #[test] + fn test_to_snake_raw_keyword() { + assert_eq!("r#as", &to_snake("as")); + assert_eq!("r#break", &to_snake("break")); + assert_eq!("r#const", &to_snake("const")); + assert_eq!("r#continue", &to_snake("continue")); + assert_eq!("r#else", &to_snake("else")); + assert_eq!("r#enum", &to_snake("enum")); + assert_eq!("r#false", &to_snake("false")); + assert_eq!("r#fn", &to_snake("fn")); + assert_eq!("r#for", &to_snake("for")); + assert_eq!("r#if", &to_snake("if")); + assert_eq!("r#impl", &to_snake("impl")); + assert_eq!("r#in", &to_snake("in")); + assert_eq!("r#let", &to_snake("let")); + assert_eq!("r#loop", &to_snake("loop")); + assert_eq!("r#match", &to_snake("match")); + assert_eq!("r#mod", &to_snake("mod")); + assert_eq!("r#move", &to_snake("move")); + assert_eq!("r#mut", &to_snake("mut")); + assert_eq!("r#pub", &to_snake("pub")); + assert_eq!("r#ref", &to_snake("ref")); + assert_eq!("r#return", &to_snake("return")); + assert_eq!("r#static", &to_snake("static")); + assert_eq!("r#struct", &to_snake("struct")); + assert_eq!("r#trait", &to_snake("trait")); + assert_eq!("r#true", &to_snake("true")); + assert_eq!("r#type", &to_snake("type")); + assert_eq!("r#unsafe", &to_snake("unsafe")); + assert_eq!("r#use", &to_snake("use")); + assert_eq!("r#where", &to_snake("where")); + assert_eq!("r#while", &to_snake("while")); + assert_eq!("r#dyn", &to_snake("dyn")); + assert_eq!("r#abstract", &to_snake("abstract")); + assert_eq!("r#become", &to_snake("become")); + assert_eq!("r#box", &to_snake("box")); + assert_eq!("r#do", &to_snake("do")); + assert_eq!("r#final", &to_snake("final")); + assert_eq!("r#macro", &to_snake("macro")); + assert_eq!("r#override", &to_snake("override")); + assert_eq!("r#priv", &to_snake("priv")); + assert_eq!("r#typeof", &to_snake("typeof")); + assert_eq!("r#unsized", &to_snake("unsized")); + assert_eq!("r#virtual", &to_snake("virtual")); + assert_eq!("r#yield", &to_snake("yield")); + assert_eq!("r#async", &to_snake("async")); + assert_eq!("r#await", &to_snake("await")); + assert_eq!("r#try", &to_snake("try")); + } + + #[test] + fn test_to_snake_non_raw_keyword() { + assert_eq!("self_", &to_snake("self")); + assert_eq!("super_", &to_snake("super")); + assert_eq!("extern_", &to_snake("extern")); + assert_eq!("crate_", &to_snake("crate")); + } + + #[test] + fn test_to_upper_camel() { + assert_eq!("", &to_upper_camel("")); + assert_eq!("F", &to_upper_camel("F")); + assert_eq!("Foo", &to_upper_camel("FOO")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR_")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR_")); + assert_eq!("FuzzBuster", &to_upper_camel("fuzzBuster")); + assert_eq!("FuzzBuster", &to_upper_camel("FuzzBuster")); + assert_eq!("Self_", &to_upper_camel("self")); + } +} diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs new file mode 100644 index 00000000..aa1048f3 --- /dev/null +++ b/node/libs/protobuf_build/src/lib.rs @@ -0,0 +1,297 @@ +//! Generates rust code from the proto files. +//! +//! Protobuf files are collected recursively from $CARGO_MANIFEST_DIR// directory. +//! Corresponding "cargo:rerun-if-changed=..." line is printed to stdout, so that +//! the build script running this function is rerun whenever proto files change. +//! A single rust file is generated and stored at $OUT_DIR//gen.rs file. +//! +//! Protobuf files are compiled to a protobuf descriptor stored at +//! $OUT_DIR//gen.binpb. +//! Additionally a "PROTOBUF_DESCRIPTOR=" line is printed to +//! stdout. This can be used to collect all the descriptors across the build as follows: +//! 1. Checkout the repo to a fresh directory and then run "cargo build --all-targets" +//! We need it fresh so that every crate containing protobufs has only one build in the +//! cargo cache. +//! 2. grep through all target/debug/build/*/output files to find all "PROTOBUF_DESCRIPTOR=..." +//! lines and merge the descriptor files by simply concatenating them. +//! Note that you can run this procedure for 2 revisions of the repo and look for breaking +//! changes by running "buf breaking --against " where before.binpb +//! and after.binpb are the concatenated descriptors from those 2 revisions. +//! +//! The proto files are not expected to be self-contained - to import proto files from +//! different crates you need to specify them as dependencies in the Config.dependencies. +//! It is not possible to depend on a different proto bundle within the same crate (because +//! these are being built simultaneously from the same build script). +#![allow(clippy::print_stdout)] +// Imports accessed from the generated code. +pub use self::syntax::*; +use anyhow::Context as _; +pub use once_cell::sync::Lazy; +pub use prost; +use prost::Message as _; +pub use prost_reflect; +use std::{fs, path::Path, sync::Mutex}; + +mod canonical; +mod ident; +mod syntax; + +/// Traverses all the files in a directory recursively. +fn traverse_files( + path: &Path, + f: &mut impl FnMut(&Path) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + if !path.is_dir() { + f(path).with_context(|| path.display().to_string())?; + return Ok(()); + } + for entry in fs::read_dir(path)? { + traverse_files(&entry?.path(), f)?; + } + Ok(()) +} + +/// Protobuf descriptor + info about the mapping to rust code. +pub struct Descriptor { + /// Root proto package that all proto files in this descriptor belong to. + /// Rust types have been generated relative to this root. + proto_root: ProtoName, + /// Raw descriptor proto. + descriptor_proto: prost_types::FileDescriptorSet, + /// Direct dependencies of this descriptor. + dependencies: Vec<&'static Descriptor>, +} + +impl Descriptor { + /// Constructs a Descriptor. + pub fn new( + proto_root: ProtoName, + dependencies: Vec<&'static Descriptor>, + descriptor_bytes: &[u8], + ) -> Self { + Descriptor { + proto_root, + dependencies, + descriptor_proto: prost_types::FileDescriptorSet::decode(descriptor_bytes).unwrap(), + } + } + + /// Loads the descriptor to the pool, if not already loaded. + pub fn load(&self, pool: &mut prost_reflect::DescriptorPool) -> anyhow::Result<()> { + if self + .descriptor_proto + .file + .iter() + .all(|f| pool.get_file_by_name(f.name()).is_some()) + { + return Ok(()); + } + for d in &self.dependencies { + d.load(pool)?; + } + pool.add_file_descriptor_set(self.descriptor_proto.clone())?; + Ok(()) + } + + /// Loads the descriptor to the global pool and returns a copy of the global pool. + pub fn get_message_by_name(&self, name: &str) -> Option { + /// Global descriptor pool. + static POOL: Lazy> = Lazy::new(Mutex::default); + let pool = &mut POOL.lock().unwrap(); + self.load(pool).unwrap(); + pool.get_message_by_name(name) + } +} + +/// Expands to a descriptor declaration. +#[macro_export] +macro_rules! declare_descriptor { + ($package_root:expr, $descriptor_path:expr, $($rust_deps:path),*) => { + pub static DESCRIPTOR : $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| { + $crate::Descriptor::new( + $package_root.into(), + vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*], + &include_bytes!($descriptor_path)[..], + ) + }); + } +} + +/// Code generation config. Use it in build scripts. +pub struct Config { + /// Input directory relative to $CARGO_MANIFEST_DIR with the proto files to be compiled. + pub input_root: InputPath, + /// Implicit prefix that should be prepended to proto paths of the proto files in the input directory. + pub proto_root: ProtoPath, + /// Descriptors of the dependencies and the rust absolute paths under which they will be available from the generated code. + pub dependencies: Vec<(RustName, &'static Descriptor)>, + /// Rust absolute path under which the protobuf crate will be available from the generated + /// code. + pub protobuf_crate: RustName, +} + +impl Config { + /// Location of the protobuf_build crate, visible from the generated code. + fn this_crate(&self) -> RustName { + self.protobuf_crate.clone().join("build") + } + + /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated + /// from a message of the given `proto_name`. + fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { + let rust_name = proto_name + .relative_to(&self.proto_root.to_name().context("invalid proto_root")?) + .unwrap() + .to_rust_type() + .to_string(); + let rust_name: syn::Path = syn::parse_str(&rust_name).context("rust_name")?; + let proto_name = proto_name.to_string(); + let this: syn::Path = syn::parse_str(&self.this_crate().to_string()).context("this")?; + Ok(quote::quote! { + impl #this::prost_reflect::ReflectMessage for #rust_name { + fn descriptor(&self) -> #this::prost_reflect::MessageDescriptor { + static INIT : #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| { + DESCRIPTOR.get_message_by_name(#proto_name).unwrap() + }); + INIT.clone() + } + } + }.to_string()) + } + + /// Generates rust code from the proto files according to the config. + pub fn generate(&self) -> anyhow::Result<()> { + if !self.input_root.abs()?.is_dir() { + anyhow::bail!("input_root should be a directory"); + } + println!("cargo:rerun-if-changed={}", self.input_root.to_str()); + + // Load dependencies. + let mut pool = prost_reflect::DescriptorPool::new(); + for d in &self.dependencies { + d.1.load(&mut pool) + .with_context(|| format!("failed to load dependency {}", d.0))?; + } + let mut pool_raw = prost_types::FileDescriptorSet::default(); + pool_raw.file.extend(pool.file_descriptor_protos().cloned()); + + // Load proto files. + let mut proto_paths = vec![]; + traverse_files(&self.input_root.abs()?, &mut |path| { + let Some(ext) = path.extension() else { + return Ok(()); + }; + let Some(ext) = ext.to_str() else { + return Ok(()); + }; + if ext != "proto" { + return Ok(()); + }; + + let file_raw = fs::read_to_string(path).context("fs::read()")?; + let path = ProtoPath::from_input_path(path, &self.input_root, &self.proto_root) + .context("ProtoPath::from_input_path()")?; + pool_raw + .file + .push(protox_parse::parse(&path.to_string(), &file_raw).map_err( + // rewrapping the error, so that source location is included in the error message. + |err| anyhow::anyhow!("{err:?}"), + )?); + proto_paths.push(path); + Ok(()) + })?; + + // Compile the proto files + let mut compiler = protox::Compiler::with_file_resolver( + protox::file::DescriptorSetFileResolver::new(pool_raw), + ); + compiler.include_source_info(true); + compiler + .open_files(proto_paths.iter().map(|p| p.to_path())) + // rewrapping the error, so that source location is included in the error message. + .map_err(|err| anyhow::anyhow!("{err:?}"))?; + let descriptor = compiler.file_descriptor_set(); + // Unwrap is ok, because we add a descriptor from a successful compilation. + pool.add_file_descriptor_set(descriptor.clone()).unwrap(); + + // Check that the compiled proto files belong to the declared proto package. + for f in &descriptor.file { + let got = ProtoName::from(f.package()); + // Unwrap is ok, because descriptor file here has never an empty name. + let want_prefix = ProtoPath::from(f.name()).parent().unwrap().to_name()?; + if !got.starts_with(&want_prefix) { + anyhow::bail!( + "{got} ({:?}) does not belong to package {want_prefix}", + f.name(), + ); + } + } + + // Check that the compiled proto messages support canonical encoding. + canonical::check(&descriptor, &pool).context("canonical::check()")?; + + // Prepare the output directory. + let output_dir = self + .input_root + .prepare_output_dir() + .context("prepare_output_dir()")?; + let output_path = output_dir.join("gen.rs"); + let descriptor_path = output_dir.join("gen.binpb"); + fs::write(&descriptor_path, &descriptor.encode_to_vec())?; + println!("PROTOBUF_DESCRIPTOR={descriptor_path:?}"); + + // Generate code out of compiled proto files. + let mut output = RustModule::default(); + let mut config = prost_build::Config::new(); + config.prost_path(self.this_crate().join("prost").to_string()); + config.skip_protoc_run(); + for d in &self.dependencies { + for f in &d.1.descriptor_proto.file { + let proto_rel = ProtoName::from(f.package()) + .relative_to(&d.1.proto_root) + .unwrap(); + let rust_abs = d.0.clone().join(proto_rel.to_rust_module()); + config.extern_path(format!(".{}", f.package()), rust_abs.to_string()); + } + } + let m = prost_build::Module::from_parts([""]); + for f in &descriptor.file { + let code = config + .generate(vec![(m.clone(), f.clone())]) + .context("generation failed")?; + output + .sub(&ProtoName::from(f.package()).to_rust_module()) + .append(&code[&m]); + } + + // Generate the reflection code. + let package_root = self.proto_root.to_name().context("invalid proto_root")?; + let output = output.sub(&package_root.to_rust_module()); + for proto_name in extract_message_names(&descriptor) { + output.append( + &self + .reflect_impl(&proto_name) + .with_context(|| format!("reflect_impl({proto_name})"))?, + ); + } + + // Generate the descriptor. + let rust_deps = self + .dependencies + .iter() + .map(|d| syn::parse_str::(&d.0.to_string()).unwrap()); + let this: syn::Path = syn::parse_str(&self.this_crate().to_string())?; + let package_root = package_root.to_string(); + let descriptor_path = descriptor_path.display().to_string(); + output.append( + "e::quote! { + #this::declare_descriptor!(#package_root,#descriptor_path,#(#rust_deps),*); + } + .to_string(), + ); + + // Save output. + fs::write(output_path, output.format().context("output.format()")?)?; + Ok(()) + } +} diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs new file mode 100644 index 00000000..80477448 --- /dev/null +++ b/node/libs/protobuf_build/src/syntax.rs @@ -0,0 +1,244 @@ +//! Utilities for handling strings belonging to various namespaces. +use super::ident; +use anyhow::Context as _; +use std::{ + collections::BTreeMap, + fmt, + path::{Path, PathBuf}, +}; + +/// Path relative to $CARGO_MANIFEST_DIR. +#[derive(Clone, PartialEq, Eq)] +pub struct InputPath(PathBuf); + +impl From<&str> for InputPath { + fn from(s: &str) -> Self { + Self(PathBuf::from(s)) + } +} + +impl InputPath { + /// Converts the relative input path to str + pub(super) fn to_str(&self) -> &str { + self.0.to_str().unwrap() + } + + /// Converts the relative input path to an absolute path in the local file system + /// (under $CARGO_MANIFEST_DIR). + pub(super) fn abs(&self) -> anyhow::Result { + Ok(PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?) + .canonicalize()? + .join(&self.0)) + } + + /// Output directory path derived from the input path by replacing $CARGO_MANIFEST_DIR with $OUT_DIR. + /// Re-constructs the derived output directory, as a side-effect. + pub(super) fn prepare_output_dir(&self) -> anyhow::Result { + let output = PathBuf::from(std::env::var("OUT_DIR")?) + .canonicalize()? + .join(&self.0); + let _ = std::fs::remove_dir_all(&output); + std::fs::create_dir_all(&output)?; + Ok(output) + } +} + +/// Absolute path of the proto file used for importing other proto files. +#[derive(Clone, PartialEq, Eq)] +pub struct ProtoPath(PathBuf); + +impl From<&str> for ProtoPath { + fn from(s: &str) -> Self { + Self(PathBuf::from(s)) + } +} + +impl ProtoPath { + /// Converts a proto module path to proto package name by replacing all "/" with ".". + pub(super) fn to_name(&self) -> anyhow::Result { + let mut parts = vec![]; + for p in self.0.iter() { + parts.push(p.to_str().context("invalid proto path")?.to_string()); + } + Ok(ProtoName(parts)) + } + + /// Returns path to the parent directory. + pub(super) fn parent(&self) -> Option { + self.0.parent().map(|p| Self(p.into())) + } + + /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . + pub(super) fn from_input_path( + path: &Path, + input_root: &InputPath, + proto_root: &ProtoPath, + ) -> anyhow::Result { + Ok(ProtoPath( + proto_root + .0 + .join(path.strip_prefix(&input_root.abs()?).unwrap()), + )) + } + + /// Converts ProtoPath to Path. + pub(super) fn to_path(&self) -> &Path { + &self.0 + } +} + +impl fmt::Display for ProtoPath { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + self.0.display().fmt(fmt) + } +} + +type Part = String; + +/// A rust module/type name that the generated code is available at. Although +/// generated code is location agnostic (it can be embedded in an arbitrary module within the crate), +/// you need to manually (in the Config) specify the rust modules containing the generated code +/// of the dependencies, so that it can be referenced from the newly generated code. +#[derive(Clone, PartialEq, Eq)] +pub struct RustName(Vec); + +impl RustName { + /// Concatenates 2 rust names. + pub fn join(mut self, suffix: impl Into) -> Self { + self.0.extend(suffix.into().0); + self + } +} + +impl fmt::Display for RustName { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&self.0.join("::")) + } +} + +impl From<&str> for RustName { + fn from(s: &str) -> Self { + Self(s.split("::").map(Part::from).collect()) + } +} + +/// A rust module representation. +/// It is used to collect the generated protobuf code. +#[derive(Default)] +pub(super) struct RustModule { + /// Nested modules which transitively contain the generated code. + modules: BTreeMap, + /// Code of the module. + code: String, +} + +impl RustModule { + /// Returns a reference to a given submodule. + pub(crate) fn sub(&mut self, path: &RustName) -> &mut Self { + let mut m = self; + for part in &path.0 { + m = m.modules.entry(part.into()).or_default(); + } + m + } + + /// Appends code to the module. + pub(crate) fn append(&mut self, code: &str) { + self.code += code; + } + + fn collect(&self) -> String { + let mut entries = vec![self.code.clone()]; + entries.extend( + self.modules + .iter() + .map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), + ); + entries.join("") + } + + /// Collects the code of the module and formats it. + pub(crate) fn format(&self) -> anyhow::Result { + let s = self.collect(); + Ok(prettyplease::unparse( + &syn::parse_str(&s).with_context(|| format!("syn::parse_str({s:?})"))?, + )) + } +} + +/// In addition to input paths and proto paths, there are also proto names +/// which are used to reference message types from different proto files. +/// Theoretically proto names can be totally independent from the proto paths (i.e. you can have +/// "my.favorite.package" proto package defined in "totally/unrelated/file/name.proto" file, but we +/// recommend the following naming convention: proto package "a.b.c" should be defined either: +/// a) in a single file "a/b/c.proto", or +/// b) in a collection of files under "a/b/c/" directory +/// Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. +#[derive(Clone, PartialEq, Eq)] +pub struct ProtoName(Vec); + +impl ProtoName { + /// Checks if package path starts with the given prefix. + pub fn starts_with(&self, prefix: &Self) -> bool { + self.0.len() >= prefix.0.len() && self.0[0..prefix.0.len()] == prefix.0 + } + + /// Strips a given prefix from the name. + pub fn relative_to(&self, prefix: &Self) -> anyhow::Result { + if !self.starts_with(prefix) { + anyhow::bail!("{self} does not contain {prefix}"); + } + Ok(Self(self.0[prefix.0.len()..].to_vec())) + } + + /// Converts proto package name to rust module name according to prost_build rules. + pub fn to_rust_module(&self) -> RustName { + RustName(self.0.iter().map(|s| ident::to_snake(s)).collect()) + } + + /// Converts proto message name to rust type name according to prost_build rules. + pub fn to_rust_type(&self) -> RustName { + let mut rust = self.to_rust_module(); + let n = rust.0.len(); + rust.0[n - 1] = ident::to_upper_camel(&self.0[n - 1]); + rust + } +} + +impl fmt::Display for ProtoName { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&self.0.join(".")) + } +} + +impl From<&str> for ProtoName { + fn from(s: &str) -> Self { + Self(s.split('.').map(String::from).collect()) + } +} + +impl From<&Path> for ProtoName { + fn from(p: &Path) -> Self { + Self(p.iter().map(|c| c.to_str().unwrap().to_string()).collect()) + } +} + +/// Extracts names of proto messages defined in the descriptor. +pub(super) fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { + fn collect(out: &mut Vec, prefix: &ProtoName, m: &prost_types::DescriptorProto) { + let mut name = prefix.clone(); + name.0.push(m.name().to_string()); + for m in &m.nested_type { + collect(out, &name, m); + } + out.push(name); + } + let mut res = vec![]; + for f in &descriptor.file { + let name = ProtoName::from(f.package()); + for m in &f.message_type { + collect(&mut res, &name, m); + } + } + res +} diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index fdd1bb84..9b4b16f7 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -11,6 +11,7 @@ zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true bit-vec.workspace = true diff --git a/node/libs/roles/src/node/conv.rs b/node/libs/roles/src/node/conv.rs index 52924ec6..6d7ea973 100644 --- a/node/libs/roles/src/node/conv.rs +++ b/node/libs/roles/src/node/conv.rs @@ -1,8 +1,8 @@ use crate::node; use anyhow::Context as _; use zksync_consensus_crypto::ByteFmt; -use zksync_consensus_schema::{read_required, required, ProtoFmt}; use zksync_consensus_utils::enum_util::Variant; +use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for node::Msg { type Proto = node::schema::Msg; diff --git a/node/libs/roles/src/node/messages.rs b/node/libs/roles/src/node/messages.rs index 9c87f025..7cca89ce 100644 --- a/node/libs/roles/src/node/messages.rs +++ b/node/libs/roles/src/node/messages.rs @@ -1,6 +1,5 @@ use crate::node; use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; -use zksync_consensus_schema as schema; use zksync_consensus_utils::enum_util::{BadVariantError, Variant}; /// The ID for an authentication session. @@ -18,7 +17,7 @@ pub enum Msg { impl Msg { /// Get the hash of this message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&schema::canonical(self))) + MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } } diff --git a/node/libs/roles/src/node/tests.rs b/node/libs/roles/src/node/tests.rs index 2d208796..e6ab6397 100644 --- a/node/libs/roles/src/node/tests.rs +++ b/node/libs/roles/src/node/tests.rs @@ -2,7 +2,7 @@ use super::*; use rand::Rng; use zksync_concurrency::ctx; use zksync_consensus_crypto::{ByteFmt, Text, TextFmt}; -use zksync_consensus_schema::testonly::{test_encode, test_encode_random}; +use zksync_protobuf::testonly::{test_encode, test_encode_random}; #[test] fn test_byte_encoding() { diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index 69d5d179..7e0d030e 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -8,11 +8,9 @@ use crate::node::SessionId; use anyhow::Context as _; use std::collections::BTreeMap; use zksync_consensus_crypto::ByteFmt; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{ - proto::roles::validator as proto, read_required, required, ProtoFmt, -}; +use zksync_consensus_schema::proto::roles::validator as proto; use zksync_consensus_utils::enum_util::Variant; +use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for BlockHeaderHash { type Proto = proto::BlockHeaderHash; @@ -189,7 +187,7 @@ impl ProtoFmt for LeaderCommit { } impl ProtoFmt for Signers { - type Proto = schema::proto::std::BitVector; + type Proto = zksync_protobuf::proto::std::BitVector; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(ProtoFmt::read(r)?)) @@ -268,8 +266,8 @@ impl ProtoFmt for Phase { fn build(&self) -> Self::Proto { use proto::phase::T; let t = match self { - Self::Prepare => T::Prepare(schema::proto::std::Void {}), - Self::Commit => T::Commit(schema::proto::std::Void {}), + Self::Prepare => T::Prepare(zksync_protobuf::proto::std::Void {}), + Self::Commit => T::Commit(zksync_protobuf::proto::std::Void {}), }; Self::Proto { t: Some(t) } } diff --git a/node/libs/roles/src/validator/messages/block.rs b/node/libs/roles/src/validator/messages/block.rs index 6a6d44c1..a7ebb0f8 100644 --- a/node/libs/roles/src/validator/messages/block.rs +++ b/node/libs/roles/src/validator/messages/block.rs @@ -3,7 +3,6 @@ use super::CommitQC; use std::fmt; use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; -use zksync_consensus_schema as schema; /// Payload of the block. Consensus algorithm does not interpret the payload /// (except for imposing a size limit for the payload). Proposing a payload @@ -115,7 +114,7 @@ pub struct BlockHeader { impl BlockHeader { /// Returns the hash of the block. pub fn hash(&self) -> BlockHeaderHash { - BlockHeaderHash(sha256::Sha256::new(&schema::canonical(self))) + BlockHeaderHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } /// Creates a genesis block. @@ -163,11 +162,11 @@ impl FinalBlock { impl ByteFmt for FinalBlock { fn decode(bytes: &[u8]) -> anyhow::Result { - schema::decode(bytes) + zksync_protobuf::decode(bytes) } fn encode(&self) -> Vec { - schema::encode(self) + zksync_protobuf::encode(self) } } diff --git a/node/libs/roles/src/validator/messages/msg.rs b/node/libs/roles/src/validator/messages/msg.rs index c740e9ed..cb6c6ad2 100644 --- a/node/libs/roles/src/validator/messages/msg.rs +++ b/node/libs/roles/src/validator/messages/msg.rs @@ -3,7 +3,6 @@ use super::{ConsensusMsg, NetAddress}; use crate::{node::SessionId, validator, validator::Error}; use std::fmt; use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; -use zksync_consensus_schema as schema; use zksync_consensus_utils::enum_util::{BadVariantError, Variant}; /// Generic message type for a validator. @@ -20,7 +19,7 @@ pub enum Msg { impl Msg { /// Returns the hash of the message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&schema::canonical(self))) + MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) } } diff --git a/node/libs/roles/src/validator/tests.rs b/node/libs/roles/src/validator/tests.rs index 8030afbf..4ef8d885 100644 --- a/node/libs/roles/src/validator/tests.rs +++ b/node/libs/roles/src/validator/tests.rs @@ -3,7 +3,7 @@ use rand::Rng; use std::vec; use zksync_concurrency::ctx; use zksync_consensus_crypto::{ByteFmt, Text, TextFmt}; -use zksync_consensus_schema::testonly::test_encode_random; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_byte_encoding() { diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml index bdb3ac1c..d0e06888 100644 --- a/node/libs/schema/Cargo.toml +++ b/node/libs/schema/Cargo.toml @@ -6,28 +6,20 @@ authors.workspace = true homepage.workspace = true license.workspace = true -[[bin]] -name = "conformance_test" - [dependencies] zksync_concurrency.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true bit-vec.workspace = true serde.workspace = true once_cell.workspace = true -quick-protobuf.workspace = true prost.workspace = true -prost-reflect.workspace = true rand.workspace = true serde_json.workspace = true tokio.workspace = true [build-dependencies] +zksync_protobuf.workspace = true + anyhow.workspace = true -syn.workspace = true -protoc-bin-vendored.workspace = true -prost-build.workspace = true -prost-reflect.workspace = true -prost-reflect-build.workspace = true -prettyplease.workspace = true diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 748a0a7e..40c8e5c4 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -1,159 +1,14 @@ -//! Generates rust code from the capnp schema files in the `capnp/` directory. -use anyhow::Context as _; -use std::{collections::BTreeMap, env, fs, path::PathBuf}; - -/// Traversed all the files in a directory recursively. -fn traverse_files(path: PathBuf, f: &mut dyn FnMut(PathBuf)) -> std::io::Result<()> { - if !path.is_dir() { - f(path); - return Ok(()); - } - for entry in fs::read_dir(path)? { - traverse_files(entry?.path(), f)?; - } - Ok(()) -} - -/// A rust module representation. -/// It is used to collect the generated protobuf code. -#[derive(Default)] -struct Module { - /// Nested modules which transitively contain the generated code. - nested: BTreeMap, - /// Nested modules directly contains the generated code. - include: BTreeMap, -} - -impl Module { - /// Inserts a nested generated protobuf module. - /// `name` is a sequence of module names. - fn insert(&mut self, name: &[String], file: PathBuf) { - println!(" -- {name:?}"); - match name.len() { - 0 => panic!("empty module path"), - 1 => assert!( - self.include.insert(name[0].clone(), file).is_none(), - "duplicate module" - ), - _ => self - .nested - .entry(name[0].clone()) - .or_default() - .insert(&name[1..], file), - } - } - - /// Generates rust code of the module. - fn generate(&self) -> String { - let mut entries = vec![]; - entries.extend( - self.nested - .iter() - .map(|(name, m)| format!("pub mod {name} {{ {} }}", m.generate())), - ); - entries.extend( - self.include - .iter() - .map(|(name, path)| format!("pub mod {name} {{ include!({path:?}); }}",)), - ); - entries.join("\n") - } -} - -/// Checks if field `f` supports canonical encoding. -fn check_canonical_field(f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { - if f.is_map() { - anyhow::bail!("maps unsupported"); - } - if !f.is_list() && !f.supports_presence() { - anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); - } - Ok(()) -} - -/// Checks if messages of type `m` support canonical encoding. -fn check_canonical_message(m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { - for m in m.child_messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; - } - for f in m.fields() { - check_canonical_field(&f).with_context(|| f.name().to_string())?; - } - Ok(()) -} - -/// Checks if message types in file `f` support canonical encoding. -fn check_canonical_file(f: &prost_reflect::FileDescriptor) -> anyhow::Result<()> { - if f.syntax() != prost_reflect::Syntax::Proto3 { - anyhow::bail!("only proto3 syntax is supported"); - } - for m in f.messages() { - check_canonical_message(&m).with_context(|| m.name().to_string())?; - } - Ok(()) -} - -/// Checks if message types in descriptor pool `d` support canonical encoding. -fn check_canonical_pool(d: &prost_reflect::DescriptorPool) -> anyhow::Result<()> { - for f in d.files() { - check_canonical_file(&f).with_context(|| f.name().to_string())?; - } - Ok(()) -} - -fn main() -> anyhow::Result<()> { - // Prepare input and output root dirs. - let proto_include = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?) - .canonicalize()? - .join("proto"); - let proto_output = PathBuf::from(env::var("OUT_DIR").unwrap()) - .canonicalize()? - .join("proto"); - println!("cargo:rerun-if-changed={}", proto_include.to_str().unwrap()); - let _ = fs::remove_dir_all(&proto_output); - fs::create_dir_all(&proto_output).unwrap(); - - // Find all proto files. - let mut proto_inputs = vec![]; - traverse_files(proto_include.clone(), &mut |path| { - let Some(ext) = path.extension() else { return }; - let Some(ext) = ext.to_str() else { return }; - if ext != "proto" { - return; - }; - proto_inputs.push(path); - })?; - - // Generate protobuf code from schema (with reflection). - env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); - let mut config = prost_build::Config::new(); - let descriptor_path = proto_output.join("descriptor.bin"); - config.out_dir(&proto_output); - prost_reflect_build::Builder::new() - .file_descriptor_set_path(&descriptor_path) - .descriptor_pool("crate::proto::DESCRIPTOR_POOL") - .compile_protos_with_config(config, &proto_inputs, &[&proto_include]) - .unwrap(); - let descriptor = fs::read(descriptor_path)?; - let pool = prost_reflect::DescriptorPool::decode(descriptor.as_ref()).unwrap(); - - // Check that messages are compatible with `proto_fmt::canonical`. - check_canonical_pool(&pool)?; - - // Generate mod file collecting all proto-generated code. - let mut m = Module::default(); - for entry in fs::read_dir(&proto_output).unwrap() { - let entry = entry.unwrap(); - let name = entry.file_name().into_string().unwrap(); - let Some(name) = name.strip_suffix(".rs") else { - continue; - }; - let name: Vec<_> = name.split('.').map(String::from).collect(); - println!("name = {name:?}"); - m.insert(&name, entry.path()); - } - let file = m.generate(); - let file = syn::parse_str(&file).unwrap(); - fs::write(proto_output.join("mod.rs"), prettyplease::unparse(&file))?; - Ok(()) +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf::build::Config { + input_root: "proto".into(), + proto_root: "zksync/schema".into(), + dependencies: vec![( + "::zksync_protobuf::proto".into(), + &zksync_protobuf::proto::DESCRIPTOR, + )], + protobuf_crate: "::zksync_protobuf".into(), + } + .generate() + .expect("generate()"); } diff --git a/node/libs/schema/proto/buf.yaml b/node/libs/schema/proto/buf.yaml deleted file mode 100644 index 94d54ac3..00000000 --- a/node/libs/schema/proto/buf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: v1 -breaking: - use: - # We use FILE rule set for everything, - # althout only protos which we use with JSON format need that. - # All the other protos would be fine with WIRE rule set. - # TODO(gprusak): separate JSON protos from binary protos - # to weaken the compatibility restrictions, in case using FILE - # for everything turns out to be too annoying. - - FILE diff --git a/node/libs/schema/proto/executor/config.proto b/node/libs/schema/proto/executor/config.proto index f273a039..20092d93 100644 --- a/node/libs/schema/proto/executor/config.proto +++ b/node/libs/schema/proto/executor/config.proto @@ -36,7 +36,7 @@ // the validator set) or move it to a separate config file. syntax = "proto3"; -package executor.config; +package zksync.schema.executor.config; // (public key, ip address) of a gossip network node. message NodeAddr { diff --git a/node/libs/schema/proto/network/consensus.proto b/node/libs/schema/proto/network/consensus.proto index f5f34dbb..09be8a08 100644 --- a/node/libs/schema/proto/network/consensus.proto +++ b/node/libs/schema/proto/network/consensus.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package network.consensus; +package zksync.schema.network.consensus; -import "roles/validator.proto"; -import "std.proto"; +import "zksync/schema/roles/validator.proto"; +import "zksync/std.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/libs/schema/proto/network/gossip.proto b/node/libs/schema/proto/network/gossip.proto index 9423a2c7..723bb100 100644 --- a/node/libs/schema/proto/network/gossip.proto +++ b/node/libs/schema/proto/network/gossip.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package network.gossip; +package zksync.schema.network.gossip; -import "roles/node.proto"; -import "roles/validator.proto"; +import "zksync/schema/roles/node.proto"; +import "zksync/schema/roles/validator.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/libs/schema/proto/network/mux.proto b/node/libs/schema/proto/network/mux.proto index a4bd0eec..3e014c8e 100644 --- a/node/libs/schema/proto/network/mux.proto +++ b/node/libs/schema/proto/network/mux.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package network.mux; +package zksync.schema.network.mux; -import "std.proto"; +import "zksync/std.proto"; message Handshake { message Capability { diff --git a/node/libs/schema/proto/network/mux_test.proto b/node/libs/schema/proto/network/mux_test.proto index dff0f492..b4cdaf87 100644 --- a/node/libs/schema/proto/network/mux_test.proto +++ b/node/libs/schema/proto/network/mux_test.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.mux_test; +package zksync.schema.network.mux_test; message Req { optional bytes input = 1; diff --git a/node/libs/schema/proto/network/ping.proto b/node/libs/schema/proto/network/ping.proto index 563dcbf9..863022fd 100644 --- a/node/libs/schema/proto/network/ping.proto +++ b/node/libs/schema/proto/network/ping.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.ping; +package zksync.schema.network.ping; message PingReq { optional bytes data = 1; diff --git a/node/libs/schema/proto/network/preface.proto b/node/libs/schema/proto/network/preface.proto index 1325e047..193ca38e 100644 --- a/node/libs/schema/proto/network/preface.proto +++ b/node/libs/schema/proto/network/preface.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package network.preface; +package zksync.schema.network.preface; // Every connection starts with the following preface: // 1. client sends Encryption msg to server. diff --git a/node/libs/schema/proto/roles/node.proto b/node/libs/schema/proto/roles/node.proto index 178eb914..87b75c1e 100644 --- a/node/libs/schema/proto/roles/node.proto +++ b/node/libs/schema/proto/roles/node.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package roles.node; +package zksync.schema.roles.node; message Msg { oneof t { diff --git a/node/libs/schema/proto/roles/validator.proto b/node/libs/schema/proto/roles/validator.proto index f622b06d..7419690f 100644 --- a/node/libs/schema/proto/roles/validator.proto +++ b/node/libs/schema/proto/roles/validator.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package roles.validator; +package zksync.schema.roles.validator; -import "std.proto"; +import "zksync/std.proto"; message PayloadHash { optional bytes sha256 = 1; // required diff --git a/node/libs/schema/proto/storage.proto b/node/libs/schema/proto/storage.proto index dbcfa8e2..301a5485 100644 --- a/node/libs/schema/proto/storage.proto +++ b/node/libs/schema/proto/storage.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package storage; +package zksync.schema.storage; -import "roles/validator.proto"; +import "zksync/schema/roles/validator.proto"; message Proposal { optional uint64 number = 1; diff --git a/node/libs/schema/src/lib.rs b/node/libs/schema/src/lib.rs index eeb2ba7e..e1891ab1 100644 --- a/node/libs/schema/src/lib.rs +++ b/node/libs/schema/src/lib.rs @@ -1,23 +1,6 @@ -//! Code generated from protobuf schema files and -//! utilities for serialization. - -mod proto_fmt; -mod std_conv; -pub mod testonly; - -pub use proto_fmt::*; - -#[cfg(test)] -mod tests; +//! Code generated from protobuf schema files. #[allow(warnings)] pub mod proto { - include!(concat!(env!("OUT_DIR"), "/proto/mod.rs")); - static DESCRIPTOR_POOL: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| { - prost_reflect::DescriptorPool::decode( - include_bytes!(concat!(env!("OUT_DIR"), "/proto/descriptor.bin")).as_ref(), - ) - .unwrap() - }); + include!(concat!(env!("OUT_DIR"), "/proto/gen.rs")); } diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index 67b685e6..0d034c40 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true zksync_concurrency.workspace = true zksync_consensus_roles.workspace = true zksync_consensus_schema.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true async-trait.workspace = true diff --git a/node/libs/storage/src/rocksdb.rs b/node/libs/storage/src/rocksdb.rs index 1a2ebe7b..ae7879b5 100644 --- a/node/libs/storage/src/rocksdb.rs +++ b/node/libs/storage/src/rocksdb.rs @@ -20,7 +20,6 @@ use std::{ }; use zksync_concurrency::{ctx, scope, sync::watch}; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; -use zksync_consensus_schema as schema; /// Enum used to represent a key in the database. It also acts as a separator between different stores. #[derive(Debug, Clone, PartialEq, Eq)] @@ -145,7 +144,7 @@ impl RocksdbStorage { .next() .context("Head block not found")? .context("RocksDB error reading head block")?; - schema::decode(&head_block).context("Failed decoding head block bytes") + zksync_protobuf::decode(&head_block).context("Failed decoding head block bytes") } /// Returns a block with the least number stored in this database. @@ -159,7 +158,7 @@ impl RocksdbStorage { .next() .context("First stored block not found")? .context("RocksDB error reading first stored block")?; - schema::decode(&first_block).context("Failed decoding first stored block bytes") + zksync_protobuf::decode(&first_block).context("Failed decoding first stored block bytes") } fn last_contiguous_block_number_blocking(&self) -> anyhow::Result { @@ -219,7 +218,7 @@ impl RocksdbStorage { else { return Ok(None); }; - let block = schema::decode(&raw_block) + let block = zksync_protobuf::decode(&raw_block) .with_context(|| format!("Failed decoding block #{number}"))?; Ok(Some(block)) } @@ -258,7 +257,7 @@ impl RocksdbStorage { let mut write_batch = rocksdb::WriteBatch::default(); write_batch.put( DatabaseKey::Block(block_number).encode_key(), - schema::encode(finalized_block), + zksync_protobuf::encode(finalized_block), ); // Commit the transaction. db.write(write_batch) @@ -277,7 +276,7 @@ impl RocksdbStorage { else { return Ok(None); }; - schema::decode(&raw_state) + zksync_protobuf::decode(&raw_state) .map(Some) .context("Failed to decode replica state!") } @@ -286,7 +285,7 @@ impl RocksdbStorage { self.write() .put( DatabaseKey::ReplicaState.encode_key(), - schema::encode(replica_state), + zksync_protobuf::encode(replica_state), ) .context("Failed putting ReplicaState to RocksDB") } diff --git a/node/libs/storage/src/tests/mod.rs b/node/libs/storage/src/tests/mod.rs index 8cd28276..7f840de3 100644 --- a/node/libs/storage/src/tests/mod.rs +++ b/node/libs/storage/src/tests/mod.rs @@ -7,7 +7,6 @@ use zksync_concurrency::ctx; use zksync_consensus_roles::validator::{ testonly::make_block, BlockHeader, BlockNumber, FinalBlock, Payload, }; -use zksync_consensus_schema as schema; #[cfg(feature = "rocksdb")] mod rocksdb; @@ -130,7 +129,10 @@ fn test_schema_encode_decode() { let rng = &mut ctx.rng(); let replica = rng.gen::(); - assert_eq!(replica, schema::decode(&schema::encode(&replica)).unwrap()); + assert_eq!( + replica, + zksync_protobuf::decode(&zksync_protobuf::encode(&replica)).unwrap() + ); } #[test] diff --git a/node/libs/storage/src/types.rs b/node/libs/storage/src/types.rs index 0217a4de..a4ee13bf 100644 --- a/node/libs/storage/src/types.rs +++ b/node/libs/storage/src/types.rs @@ -3,7 +3,8 @@ use anyhow::Context as _; use std::{iter, ops}; use zksync_concurrency::ctx; use zksync_consensus_roles::validator::{self, BlockNumber}; -use zksync_consensus_schema::{proto::storage as proto, read_required, required, ProtoFmt}; +use zksync_consensus_schema::proto::storage as proto; +use zksync_protobuf::{read_required, required, ProtoFmt}; /// A payload of a proposed block which is not known to be finalized yet. /// Replicas have to persist such proposed payloads for liveness: diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 3102f33d..74803009 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -17,6 +17,7 @@ zksync_consensus_roles.workspace = true zksync_consensus_storage = { workspace = true, features = ["rocksdb"] } zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true +zksync_protobuf.workspace = true anyhow.workspace = true clap.workspace = true diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 95f74f5a..69c9f71e 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -7,7 +7,6 @@ use zksync_consensus_bft::testonly; use zksync_consensus_crypto::TextFmt; use zksync_consensus_executor::{ConsensusConfig, ExecutorConfig, GossipConfig}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema as schema; use zksync_consensus_tools::NodeConfig; /// Replaces IP of the address with UNSPECIFIED (aka INADDR_ANY) of the corresponding IP type. @@ -111,8 +110,11 @@ fn main() -> anyhow::Result<()> { let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; - fs::write(root.join("config.json"), schema::encode_json(&node_cfg)) - .context("fs::write()")?; + fs::write( + root.join("config.json"), + zksync_protobuf::encode_json(&node_cfg), + ) + .context("fs::write()")?; fs::write( root.join("validator_key"), &TextFmt::encode(&validator_keys[i]), diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index be7b4421..effcd00c 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -4,10 +4,8 @@ use std::{fs, net, path::Path}; use zksync_consensus_crypto::{read_optional_text, Text, TextFmt}; use zksync_consensus_executor::{ConsensusConfig, ExecutorConfig}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema as schema; -use zksync_consensus_schema::{ - proto::executor::config as proto, read_optional, read_required, ProtoFmt, -}; +use zksync_consensus_schema::proto::executor::config as proto; +use zksync_protobuf::{read_optional, read_required, ProtoFmt}; /// This struct holds the file path to each of the config files. #[derive(Debug)] @@ -78,12 +76,13 @@ impl Configs { args.config.display() ) })?; - let node_config: NodeConfig = schema::decode_json(&node_config).with_context(|| { - format!( - "failed decoding JSON node config at `{}`", - args.config.display() - ) - })?; + let node_config: NodeConfig = + zksync_protobuf::decode_json(&node_config).with_context(|| { + format!( + "failed decoding JSON node config at `{}`", + args.config.display() + ) + })?; let validator_key: Option = args .validator_key From d7361cd7a739be42ac881754de93c5930e6cfa98 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 9 Nov 2023 10:22:32 +0200 Subject: [PATCH 2/5] refactor: Minor `protobuf_build` chores (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ Performs minor cleanup for the protobuf_buildcrate, such as using more intelligent local variable names and relying more on `syn` types. ## Why ❔ Improves maintainability. --- node/Cargo.lock | 1 + node/Cargo.toml | 1 + node/libs/protobuf/build.rs | 6 +- node/libs/protobuf_build/Cargo.toml | 1 + node/libs/protobuf_build/src/canonical.rs | 43 ++--- node/libs/protobuf_build/src/lib.rs | 123 ++++++------- node/libs/protobuf_build/src/syntax.rs | 204 +++++++++++++++------- node/libs/schema/build.rs | 4 +- 8 files changed, 230 insertions(+), 153 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index 8896fef8..184d2f81 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -2844,6 +2844,7 @@ dependencies = [ "heck", "once_cell", "prettyplease", + "proc-macro2", "prost", "prost-build", "prost-reflect", diff --git a/node/Cargo.toml b/node/Cargo.toml index 8dc3d506..d3883596 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -53,6 +53,7 @@ hex = "0.4.3" im = "15.1.0" once_cell = "1.17.1" pin-project = "1.1.0" +proc-macro2 = "1.0.66" prost = "0.12.0" prost-build = "0.12.0" prost-reflect = { version = "0.12.0", features = ["serde"] } diff --git a/node/libs/protobuf/build.rs b/node/libs/protobuf/build.rs index 0348639b..49ba4cb6 100644 --- a/node/libs/protobuf/build.rs +++ b/node/libs/protobuf/build.rs @@ -4,7 +4,7 @@ fn main() { input_root: "src/proto".into(), proto_root: "zksync".into(), dependencies: vec![], - protobuf_crate: "crate".into(), + protobuf_crate: "crate".parse().expect("protobuf_crate"), } .generate() .expect("generate(std)"); @@ -13,7 +13,7 @@ fn main() { input_root: "src/tests/proto".into(), proto_root: "zksync/protobuf/tests".into(), dependencies: vec![], - protobuf_crate: "crate".into(), + protobuf_crate: "crate".parse().expect("protobuf_crate"), } .generate() .expect("generate(test)"); @@ -22,7 +22,7 @@ fn main() { input_root: "src/bin/conformance_test/proto".into(), proto_root: "zksync/protobuf/conformance_test".into(), dependencies: vec![], - protobuf_crate: "::zksync_protobuf".into(), + protobuf_crate: "::zksync_protobuf".parse().expect("protobuf_crate"), } .generate() .expect("generate(conformance)"); diff --git a/node/libs/protobuf_build/Cargo.toml b/node/libs/protobuf_build/Cargo.toml index 46129be5..94e8b9bf 100644 --- a/node/libs/protobuf_build/Cargo.toml +++ b/node/libs/protobuf_build/Cargo.toml @@ -11,6 +11,7 @@ anyhow.workspace = true heck.workspace = true once_cell.workspace = true prettyplease.workspace = true +proc-macro2.workspace = true prost.workspace = true prost-build.workspace = true prost-types.workspace = true diff --git a/node/libs/protobuf_build/src/canonical.rs b/node/libs/protobuf_build/src/canonical.rs index e283663e..fe56fc1c 100644 --- a/node/libs/protobuf_build/src/canonical.rs +++ b/node/libs/protobuf_build/src/canonical.rs @@ -7,29 +7,30 @@ use std::collections::HashSet; struct Check(HashSet); impl Check { - /// Checks if messages of type `m` support canonical encoding. - fn check_message(&mut self, m: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { - if self.0.contains(m.full_name()) { + /// Checks if messages of type `message` support canonical encoding. + fn check_message(&mut self, message: &prost_reflect::MessageDescriptor) -> anyhow::Result<()> { + if self.0.contains(message.full_name()) { return Ok(()); } - self.0.insert(m.full_name().to_string()); - for f in m.fields() { - self.check_field(&f).with_context(|| f.name().to_string())?; + self.0.insert(message.full_name().to_string()); + for field in message.fields() { + self.check_field(&field) + .with_context(|| field.name().to_string())?; } Ok(()) } - /// Checks if field `f` supports canonical encoding. - fn check_field(&mut self, f: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { - if f.is_map() { + /// Checks if field `field` supports canonical encoding. + fn check_field(&mut self, field: &prost_reflect::FieldDescriptor) -> anyhow::Result<()> { + if field.is_map() { anyhow::bail!("maps unsupported"); } - if !f.is_list() && !f.supports_presence() { + if !field.is_list() && !field.supports_presence() { anyhow::bail!("non-repeated, non-oneof fields have to be marked as optional"); } - if let prost_reflect::Kind::Message(msg) = &f.kind() { - self.check_message(msg) - .with_context(|| msg.name().to_string())?; + if let prost_reflect::Kind::Message(message) = &field.kind() { + self.check_message(message) + .with_context(|| message.name().to_string())?; } Ok(()) } @@ -41,18 +42,18 @@ pub(crate) fn check( descriptor: &prost_types::FileDescriptorSet, pool: &prost_reflect::DescriptorPool, ) -> anyhow::Result<()> { - for f in &descriptor.file { - if f.syntax() != "proto3" { - anyhow::bail!("{}: only proto3 syntax is supported", f.name()); + for file in &descriptor.file { + if file.syntax() != "proto3" { + anyhow::bail!("{}: only proto3 syntax is supported", file.name()); } } - let mut c = Check::default(); + let mut check = Check::default(); for msg_name in extract_message_names(descriptor) { - let msg_name = msg_name.to_string(); - let msg = pool - .get_message_by_name(&msg_name) + let message_name = msg_name.to_string(); + let message = pool + .get_message_by_name(&message_name) .with_context(|| format!("{msg_name} not found in pool"))?; - c.check_message(&msg).with_context(|| msg_name)?; + check.check_message(&message).context(message_name)?; } Ok(()) } diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index aa1048f3..0b207628 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -22,6 +22,7 @@ //! different crates you need to specify them as dependencies in the Config.dependencies. //! It is not possible to depend on a different proto bundle within the same crate (because //! these are being built simultaneously from the same build script). + #![allow(clippy::print_stdout)] // Imports accessed from the generated code. pub use self::syntax::*; @@ -39,19 +40,20 @@ mod syntax; /// Traverses all the files in a directory recursively. fn traverse_files( path: &Path, - f: &mut impl FnMut(&Path) -> anyhow::Result<()>, + action: &mut impl FnMut(&Path) -> anyhow::Result<()>, ) -> anyhow::Result<()> { if !path.is_dir() { - f(path).with_context(|| path.display().to_string())?; + action(path).with_context(|| path.display().to_string())?; return Ok(()); } for entry in fs::read_dir(path)? { - traverse_files(&entry?.path(), f)?; + traverse_files(&entry?.path(), action)?; } Ok(()) } /// Protobuf descriptor + info about the mapping to rust code. +#[derive(Debug)] pub struct Descriptor { /// Root proto package that all proto files in this descriptor belong to. /// Rust types have been generated relative to this root. @@ -86,8 +88,8 @@ impl Descriptor { { return Ok(()); } - for d in &self.dependencies { - d.load(pool)?; + for dependency in &self.dependencies { + dependency.load(pool)?; } pool.add_file_descriptor_set(self.descriptor_proto.clone())?; Ok(()) @@ -107,11 +109,11 @@ impl Descriptor { #[macro_export] macro_rules! declare_descriptor { ($package_root:expr, $descriptor_path:expr, $($rust_deps:path),*) => { - pub static DESCRIPTOR : $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| { + pub static DESCRIPTOR: $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| { $crate::Descriptor::new( $package_root.into(), - vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*], - &include_bytes!($descriptor_path)[..], + ::std::vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR })*], + include_bytes!($descriptor_path), ) }); } @@ -133,30 +135,28 @@ pub struct Config { impl Config { /// Location of the protobuf_build crate, visible from the generated code. fn this_crate(&self) -> RustName { - self.protobuf_crate.clone().join("build") + self.protobuf_crate.clone().join(RustName::ident("build")) } /// Generates implementation of `prost_reflect::ReflectMessage` for a rust type generated /// from a message of the given `proto_name`. - fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { + fn reflect_impl(&self, proto_name: &ProtoName) -> anyhow::Result { let rust_name = proto_name .relative_to(&self.proto_root.to_name().context("invalid proto_root")?) .unwrap() - .to_rust_type() - .to_string(); - let rust_name: syn::Path = syn::parse_str(&rust_name).context("rust_name")?; + .to_rust_type()?; let proto_name = proto_name.to_string(); - let this: syn::Path = syn::parse_str(&self.this_crate().to_string()).context("this")?; - Ok(quote::quote! { + let this = self.this_crate(); + Ok(syn::parse_quote! { impl #this::prost_reflect::ReflectMessage for #rust_name { fn descriptor(&self) -> #this::prost_reflect::MessageDescriptor { - static INIT : #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| { + static INIT: #this::Lazy<#this::prost_reflect::MessageDescriptor> = #this::Lazy::new(|| { DESCRIPTOR.get_message_by_name(#proto_name).unwrap() }); INIT.clone() } } - }.to_string()) + }) } /// Generates rust code from the proto files according to the config. @@ -168,9 +168,10 @@ impl Config { // Load dependencies. let mut pool = prost_reflect::DescriptorPool::new(); - for d in &self.dependencies { - d.1.load(&mut pool) - .with_context(|| format!("failed to load dependency {}", d.0))?; + for (root_path, descriptor) in &self.dependencies { + descriptor + .load(&mut pool) + .with_context(|| format!("failed to load dependency `{root_path}`"))?; } let mut pool_raw = prost_types::FileDescriptorSet::default(); pool_raw.file.extend(pool.file_descriptor_protos().cloned()); @@ -207,7 +208,7 @@ impl Config { ); compiler.include_source_info(true); compiler - .open_files(proto_paths.iter().map(|p| p.to_path())) + .open_files(proto_paths) // rewrapping the error, so that source location is included in the error message. .map_err(|err| anyhow::anyhow!("{err:?}"))?; let descriptor = compiler.file_descriptor_set(); @@ -215,16 +216,15 @@ impl Config { pool.add_file_descriptor_set(descriptor.clone()).unwrap(); // Check that the compiled proto files belong to the declared proto package. - for f in &descriptor.file { - let got = ProtoName::from(f.package()); + for file in &descriptor.file { + let got = ProtoName::from(file.package()); // Unwrap is ok, because descriptor file here has never an empty name. - let want_prefix = ProtoPath::from(f.name()).parent().unwrap().to_name()?; - if !got.starts_with(&want_prefix) { - anyhow::bail!( - "{got} ({:?}) does not belong to package {want_prefix}", - f.name(), - ); - } + let want_prefix = ProtoPath::from(file.name()).parent().unwrap().to_name()?; + anyhow::ensure!( + got.starts_with(&want_prefix), + "{got} ({:?}) does not belong to package {want_prefix}", + file.name() + ); } // Check that the compiled proto messages support canonical encoding. @@ -243,55 +243,58 @@ impl Config { // Generate code out of compiled proto files. let mut output = RustModule::default(); let mut config = prost_build::Config::new(); - config.prost_path(self.this_crate().join("prost").to_string()); + let prost_path = self.this_crate().join(RustName::ident("prost")); + config.prost_path(prost_path.to_string()); config.skip_protoc_run(); - for d in &self.dependencies { - for f in &d.1.descriptor_proto.file { - let proto_rel = ProtoName::from(f.package()) - .relative_to(&d.1.proto_root) + for (root_path, descriptor) in &self.dependencies { + for file in &descriptor.descriptor_proto.file { + let proto_rel = ProtoName::from(file.package()) + .relative_to(&descriptor.proto_root) .unwrap(); - let rust_abs = d.0.clone().join(proto_rel.to_rust_module()); - config.extern_path(format!(".{}", f.package()), rust_abs.to_string()); + let rust_path = root_path.clone().join(proto_rel.to_rust_module()?); + config.extern_path(format!(".{}", file.package()), rust_path.to_string()); } } - let m = prost_build::Module::from_parts([""]); - for f in &descriptor.file { + let module = prost_build::Module::from_parts([""]); + for file in &descriptor.file { let code = config - .generate(vec![(m.clone(), f.clone())]) + .generate(vec![(module.clone(), file.clone())]) .context("generation failed")?; + let code = &code[&module]; + let code = syn::parse_str(code).with_context(|| { + format!("prost_build generated invalid code for {}", file.name()) + })?; output - .sub(&ProtoName::from(f.package()).to_rust_module()) - .append(&code[&m]); + .submodule(&ProtoName::from(file.package()).to_rust_module()?) + .extend(code); } // Generate the reflection code. let package_root = self.proto_root.to_name().context("invalid proto_root")?; - let output = output.sub(&package_root.to_rust_module()); + let mut output = output.into_submodule(&package_root.to_rust_module()?); for proto_name in extract_message_names(&descriptor) { - output.append( - &self - .reflect_impl(&proto_name) - .with_context(|| format!("reflect_impl({proto_name})"))?, - ); + let impl_item = self + .reflect_impl(&proto_name) + .with_context(|| format!("reflect_impl({proto_name})"))?; + output.append_item(impl_item.into()); } // Generate the descriptor. - let rust_deps = self - .dependencies - .iter() - .map(|d| syn::parse_str::(&d.0.to_string()).unwrap()); - let this: syn::Path = syn::parse_str(&self.this_crate().to_string())?; + let root_paths_for_deps = self.dependencies.iter().map(|(root_path, _)| root_path); + let this = self.this_crate(); let package_root = package_root.to_string(); let descriptor_path = descriptor_path.display().to_string(); - output.append( - "e::quote! { - #this::declare_descriptor!(#package_root,#descriptor_path,#(#rust_deps),*); - } - .to_string(), - ); + output.append_item(syn::parse_quote! { + #this::declare_descriptor!(#package_root, #descriptor_path, #(#root_paths_for_deps),*); + }); // Save output. - fs::write(output_path, output.format().context("output.format()")?)?; + fs::write(&output_path, output.format()).with_context(|| { + format!( + "failed writing generated code to `{}`", + output_path.display() + ) + })?; Ok(()) } } diff --git a/node/libs/protobuf_build/src/syntax.rs b/node/libs/protobuf_build/src/syntax.rs index 80477448..0d9aee3b 100644 --- a/node/libs/protobuf_build/src/syntax.rs +++ b/node/libs/protobuf_build/src/syntax.rs @@ -1,14 +1,16 @@ //! Utilities for handling strings belonging to various namespaces. + use super::ident; use anyhow::Context as _; use std::{ collections::BTreeMap, fmt, path::{Path, PathBuf}, + str::FromStr, }; -/// Path relative to $CARGO_MANIFEST_DIR. -#[derive(Clone, PartialEq, Eq)] +/// Path relative to `$CARGO_MANIFEST_DIR`. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct InputPath(PathBuf); impl From<&str> for InputPath { @@ -44,7 +46,7 @@ impl InputPath { } /// Absolute path of the proto file used for importing other proto files. -#[derive(Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ProtoPath(PathBuf); impl From<&str> for ProtoPath { @@ -56,16 +58,17 @@ impl From<&str> for ProtoPath { impl ProtoPath { /// Converts a proto module path to proto package name by replacing all "/" with ".". pub(super) fn to_name(&self) -> anyhow::Result { - let mut parts = vec![]; - for p in self.0.iter() { - parts.push(p.to_str().context("invalid proto path")?.to_string()); - } + let parts = self + .0 + .iter() + .map(|part| Ok(part.to_str().context("non-UTF8 proto path")?.to_string())); + let parts = parts.collect::>()?; Ok(ProtoName(parts)) } /// Returns path to the parent directory. pub(super) fn parent(&self) -> Option { - self.0.parent().map(|p| Self(p.into())) + self.0.parent().map(|path| Self(path.to_owned())) } /// Derives a proto path from an input path by replacing the $CARGO_MANIFEST_DIR/ with . @@ -80,89 +83,140 @@ impl ProtoPath { .join(path.strip_prefix(&input_root.abs()?).unwrap()), )) } +} - /// Converts ProtoPath to Path. - pub(super) fn to_path(&self) -> &Path { +impl AsRef for ProtoPath { + fn as_ref(&self) -> &Path { &self.0 } } impl fmt::Display for ProtoPath { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.display().fmt(fmt) } } -type Part = String; - /// A rust module/type name that the generated code is available at. Although /// generated code is location agnostic (it can be embedded in an arbitrary module within the crate), /// you need to manually (in the Config) specify the rust modules containing the generated code /// of the dependencies, so that it can be referenced from the newly generated code. -#[derive(Clone, PartialEq, Eq)] -pub struct RustName(Vec); +#[derive(Clone)] +pub struct RustName(syn::Path); impl RustName { + /// Creates a name consisting of a single identifier. + pub(crate) fn ident(ident: &str) -> Self { + let ident: syn::Ident = syn::parse_str(ident).expect("Invalid identifier"); + Self(syn::Path::from(ident)) + } + /// Concatenates 2 rust names. - pub fn join(mut self, suffix: impl Into) -> Self { - self.0.extend(suffix.into().0); + pub fn join(mut self, suffix: Self) -> Self { + self.0.segments.extend(suffix.0.segments); self } } +impl FromStr for RustName { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let path: syn::Path = + syn::parse_str(s).with_context(|| format!("failed parsing path `{s}`"))?; + for segment in &path.segments { + anyhow::ensure!( + segment.arguments.is_empty(), + "path must be a plain `::`-delimited path (no path arguments)" + ); + } + Ok(Self(path)) + } +} + impl fmt::Display for RustName { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.write_str(&self.0.join("::")) + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let path = &self.0; + fmt::Display::fmt("e::quote!(#path), formatter) } } -impl From<&str> for RustName { - fn from(s: &str) -> Self { - Self(s.split("::").map(Part::from).collect()) +impl quote::ToTokens for RustName { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + self.0.to_tokens(tokens) } } /// A rust module representation. /// It is used to collect the generated protobuf code. -#[derive(Default)] pub(super) struct RustModule { /// Nested modules which transitively contain the generated code. - modules: BTreeMap, + submodules: BTreeMap, /// Code of the module. - code: String, + code: syn::File, +} + +impl Default for RustModule { + fn default() -> Self { + Self { + submodules: BTreeMap::new(), + code: syn::File { + shebang: None, + attrs: vec![], + items: vec![], + }, + } + } } impl RustModule { /// Returns a reference to a given submodule. - pub(crate) fn sub(&mut self, path: &RustName) -> &mut Self { - let mut m = self; - for part in &path.0 { - m = m.modules.entry(part.into()).or_default(); + pub(crate) fn submodule(&mut self, path: &RustName) -> &mut Self { + let mut module = self; + for part in &path.0.segments { + module = module.submodules.entry(part.ident.clone()).or_default(); + } + module + } + + /// Converts this module into a submodule at the specified path. + pub(crate) fn into_submodule(self, path: &RustName) -> Self { + let mut module = self; + for part in &path.0.segments { + module = module.submodules.remove(&part.ident).unwrap_or_default(); } - m + module } - /// Appends code to the module. - pub(crate) fn append(&mut self, code: &str) { - self.code += code; + /// Extends this module with the specified code. + pub(crate) fn extend(&mut self, code: syn::File) { + if self.code.items.is_empty() { + self.code = code; + } else { + self.code.items.extend(code.items); + } + } + + /// Appends the specified item to this module. + pub(crate) fn append_item(&mut self, item: syn::Item) { + self.code.items.push(item); } - fn collect(&self) -> String { - let mut entries = vec![self.code.clone()]; - entries.extend( - self.modules - .iter() - .map(|(name, m)| format!("pub mod {name} {{ {} }}\n", m.collect())), - ); - entries.join("") + fn collect(mut self) -> syn::File { + for (name, submodule) in self.submodules { + let submodule = submodule.collect(); + self.code.items.push(syn::parse_quote! { + pub mod #name { + #submodule + } + }); + } + self.code } /// Collects the code of the module and formats it. - pub(crate) fn format(&self) -> anyhow::Result { - let s = self.collect(); - Ok(prettyplease::unparse( - &syn::parse_str(&s).with_context(|| format!("syn::parse_str({s:?})"))?, - )) + pub(crate) fn format(self) -> String { + prettyplease::unparse(&self.collect()) } } @@ -174,7 +228,7 @@ impl RustModule { /// a) in a single file "a/b/c.proto", or /// b) in a collection of files under "a/b/c/" directory /// Option b) is useful for defining large packages, because there is no equivalent of "pub use" in proto syntax. -#[derive(Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ProtoName(Vec); impl ProtoName { @@ -191,22 +245,33 @@ impl ProtoName { Ok(Self(self.0[prefix.0.len()..].to_vec())) } - /// Converts proto package name to rust module name according to prost_build rules. - pub fn to_rust_module(&self) -> RustName { - RustName(self.0.iter().map(|s| ident::to_snake(s)).collect()) + /// Converts proto package name to rust module name according to `prost_build` rules. + pub fn to_rust_module(&self) -> anyhow::Result { + let segments = self.0.iter().map(|segment| { + let ident: syn::Ident = syn::parse_str(&ident::to_snake(segment)) + .with_context(|| format!("Invalid identifier `{segment}`"))?; + Ok(syn::PathSegment::from(ident)) + }); + Ok(RustName(syn::Path { + leading_colon: None, + segments: segments.collect::>()?, + })) } - /// Converts proto message name to rust type name according to prost_build rules. - pub fn to_rust_type(&self) -> RustName { - let mut rust = self.to_rust_module(); - let n = rust.0.len(); - rust.0[n - 1] = ident::to_upper_camel(&self.0[n - 1]); - rust + /// Converts proto message name to rust type name according to `prost_build` rules. + pub fn to_rust_type(&self) -> anyhow::Result { + let mut rust = self.to_rust_module()?; + let type_name = self.0.last().expect("`ProtoName` cannot be empty"); + let type_name = ident::to_upper_camel(type_name); + let last_segment = rust.0.segments.last_mut().unwrap(); + *last_segment = syn::parse_str(&type_name) + .with_context(|| format!("Invalid identifier `{type_name}`"))?; + Ok(rust) } } impl fmt::Display for ProtoName { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.write_str(&self.0.join(".")) } } @@ -225,20 +290,25 @@ impl From<&Path> for ProtoName { /// Extracts names of proto messages defined in the descriptor. pub(super) fn extract_message_names(descriptor: &prost_types::FileDescriptorSet) -> Vec { - fn collect(out: &mut Vec, prefix: &ProtoName, m: &prost_types::DescriptorProto) { + fn collect( + out: &mut Vec, + prefix: &ProtoName, + message: &prost_types::DescriptorProto, + ) { let mut name = prefix.clone(); - name.0.push(m.name().to_string()); - for m in &m.nested_type { - collect(out, &name, m); + name.0.push(message.name().to_string()); + for nested_message in &message.nested_type { + collect(out, &name, nested_message); } out.push(name); } - let mut res = vec![]; - for f in &descriptor.file { - let name = ProtoName::from(f.package()); - for m in &f.message_type { - collect(&mut res, &name, m); + + let mut names = vec![]; + for file in &descriptor.file { + let name = ProtoName::from(file.package()); + for message in &file.message_type { + collect(&mut names, &name, message); } } - res + names } diff --git a/node/libs/schema/build.rs b/node/libs/schema/build.rs index 40c8e5c4..7e8cc416 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/schema/build.rs @@ -4,10 +4,10 @@ fn main() { input_root: "proto".into(), proto_root: "zksync/schema".into(), dependencies: vec![( - "::zksync_protobuf::proto".into(), + "::zksync_protobuf::proto".parse().expect("dependency_path"), &zksync_protobuf::proto::DESCRIPTOR, )], - protobuf_crate: "::zksync_protobuf".into(), + protobuf_crate: "::zksync_protobuf".parse().expect("protobuf_crate"), } .generate() .expect("generate()"); From 9a3bb210bad6764d5e522c3e489d270b46d746a3 Mon Sep 17 00:00:00 2001 From: Moshe Shababo <17073733+moshababo@users.noreply.github.com> Date: Thu, 9 Nov 2023 02:59:52 -0600 Subject: [PATCH 3/5] Switch from sha256 to keccak256 (#23) Switching from [sha256](https://crates.io/crates/sha2) to [keccak256](https://crates.io/crates/sha3) as the general-purpose hash function, to enhance compatibility with EVM/zkEVM. Closing BFT-361. --------- Co-authored-by: pompon0 --- node/Cargo.lock | 489 ++++++++++-------- node/Cargo.toml | 2 +- node/actors/network/src/noise/stream.rs | 6 +- node/deny.toml | 7 +- node/libs/concurrency/Cargo.toml | 2 +- node/libs/concurrency/src/ctx/rng.rs | 8 +- node/libs/crypto/Cargo.toml | 2 +- node/libs/crypto/src/bn254/hash.rs | 4 +- .../crypto/src/{sha256 => keccak256}/mod.rs | 20 +- node/libs/crypto/src/keccak256/test.rs | 33 ++ node/libs/crypto/src/keccak256/testonly.rs | 13 + node/libs/crypto/src/lib.rs | 2 +- node/libs/crypto/src/sha256/testonly.rs | 13 - node/libs/roles/src/node/messages.rs | 12 +- node/libs/roles/src/validator/conv.rs | 12 +- .../roles/src/validator/messages/block.rs | 25 +- node/libs/roles/src/validator/messages/msg.rs | 12 +- node/libs/schema/proto/roles/validator.proto | 6 +- 18 files changed, 375 insertions(+), 293 deletions(-) rename node/libs/crypto/src/{sha256 => keccak256}/mod.rs (64%) create mode 100644 node/libs/crypto/src/keccak256/test.rs create mode 100644 node/libs/crypto/src/keccak256/testonly.rs delete mode 100644 node/libs/crypto/src/sha256/testonly.rs diff --git a/node/Cargo.lock b/node/Cargo.lock index 184d2f81..054a68cf 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -66,9 +66,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -81,30 +81,29 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.3.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -120,9 +119,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys", @@ -157,7 +156,7 @@ dependencies = [ "ark-std", "derivative", "hashbrown 0.13.2", - "itertools", + "itertools 0.10.5", "num-traits", "zeroize", ] @@ -174,7 +173,7 @@ dependencies = [ "ark-std", "derivative", "digest", - "itertools", + "itertools 0.10.5", "num-bigint", "num-traits", "paste", @@ -259,13 +258,13 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -276,9 +275,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -291,9 +290,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64ct" @@ -325,7 +324,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -342,9 +341,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "bitmaps" @@ -393,15 +392,15 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bzip2-sys" @@ -422,9 +421,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", @@ -519,20 +518,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.22" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b417ae4361bca3f5de378294fc7472d3c4ed86a5ef9f49e93ae722f432aae8d2" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.22" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c90dc0f0e42c64bff177ca9d7be6fcc9ddb0f26a6e062174a61c84dd6c644d4" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstream", "anstyle", @@ -542,21 +540,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -572,9 +570,9 @@ checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -591,7 +589,7 @@ dependencies = [ "clap", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -612,7 +610,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -669,9 +667,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" +checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" dependencies = [ "cfg-if", "cpufeatures", @@ -686,13 +684,13 @@ dependencies = [ [[package]] name = "curve25519-dalek-derive" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -707,9 +705,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.7" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", +] [[package]] name = "derivative" @@ -747,9 +748,9 @@ checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" [[package]] name = "ed25519" -version = "2.2.2" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60f6d271ca33075c88028be6f04d502853d63a5ece419d269c15315d4fc1cf1d" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ "pkcs8", "signature", @@ -785,37 +786,32 @@ dependencies = [ ] [[package]] -name = "errno" -version = "0.3.2" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys", -] +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "errno" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e" dependencies = [ - "cc", "libc", + "windows-sys", ] [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fiat-crypto" -version = "0.1.20" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +checksum = "f69037fe1b785e84986b4f2cbcf647381876a00671d25ceef715d7812dd7e1dd" [[package]] name = "fixedbitset" @@ -831,30 +827,30 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", "futures-task", @@ -874,9 +870,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -895,9 +891,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "glob" @@ -911,12 +907,6 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.13.2" @@ -926,6 +916,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + [[package]] name = "heck" version = "0.4.1" @@ -934,9 +930,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -944,6 +940,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "http" version = "0.2.9" @@ -994,7 +999,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -1017,12 +1022,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ - "autocfg", - "hashbrown 0.12.3", + "equivalent", + "hashbrown 0.14.2", ] [[package]] @@ -1045,6 +1050,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -1053,9 +1067,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -1069,6 +1083,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1083,9 +1106,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libloading" @@ -1126,35 +1149,35 @@ dependencies = [ [[package]] name = "linkme" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f948366ad5bb46b5514ba7a7a80643726eef08b06632592699676748c8bc33b" +checksum = "91ed2ee9464ff9707af8e9ad834cffa4802f072caad90639c583dd3c62e6e608" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc28438cad73dcc90ff3466fc329a9252b1b8ba668eb0d5668ba97088cf4eef0" +checksum = "ba125974b109d512fccbc6c0244e7580143e460895dfd6ea7f8bbb692fd94396" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1186,7 +1209,7 @@ dependencies = [ "proc-macro2", "quote", "regex-syntax 0.6.29", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -1219,9 +1242,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -1252,7 +1275,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -1272,9 +1295,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi", @@ -1349,9 +1372,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] @@ -1376,9 +1399,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] @@ -1401,9 +1424,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", @@ -1426,9 +1449,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", "indexmap", @@ -1451,14 +1474,14 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1484,9 +1507,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platforms" -version = "3.0.2" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" +checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" [[package]] name = "plotters" @@ -1539,6 +1562,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1557,19 +1586,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -1594,7 +1623,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -1615,7 +1644,7 @@ checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -1624,7 +1653,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.32", + "syn 2.0.39", "tempfile", "which", ] @@ -1636,10 +1665,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", - "itertools", + "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -1669,9 +1698,9 @@ dependencies = [ [[package]] name = "protox" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66eb3a834c1ffe362107daab84dd87cfc1e1d2beda30e2eb8e4801f262839364" +checksum = "00bb76c5f6221de491fe2c8f39b106330bbd9762c6511119c07940e10eb9ff11" dependencies = [ "bytes", "miette", @@ -1773,23 +1802,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.9.3" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -1803,13 +1832,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.8.2", ] [[package]] @@ -1820,9 +1849,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rocksdb" @@ -1857,11 +1886,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -1891,15 +1920,15 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.183" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] @@ -1916,20 +1945,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -1938,29 +1967,39 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", "digest", ] +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] name = "shlex" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "signal-hook-registry" @@ -1989,9 +2028,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "snow" @@ -2011,9 +2050,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -2021,9 +2060,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys", @@ -2070,9 +2109,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -2081,9 +2120,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", @@ -2094,42 +2133,42 @@ dependencies = [ [[package]] name = "test-casing" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a368b60fd43dd855b44b15fd27238a569d58b524ef2b5dfdffe013e92e0e0c14" +checksum = "b2378d657757969a2cec9ec4eb616be8f01be98c21c6467991f91cb182e4653b" dependencies = [ "test-casing-macro", ] [[package]] name = "test-casing-macro" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4dc4744280091c8760f456b14c5598f5f7afe96851b4da30fe0933725dae0d3" +checksum = "2cfbe7811249c4c914b06141b8ac0f2cee2733fb883d05eb19668a45fc60c3d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "thiserror" -version = "1.0.47" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.47" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -2153,20 +2192,21 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", + "powerfmt", "serde", "time-core", ] [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "tinytemplate" @@ -2180,9 +2220,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -2192,7 +2232,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.5", "tokio-macros", "windows-sys", ] @@ -2205,7 +2245,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -2216,11 +2256,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2228,20 +2267,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -2249,12 +2288,12 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] @@ -2284,15 +2323,15 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-width" @@ -2365,7 +2404,7 @@ source = "git+https://github.com/matter-labs/vise.git?rev=dd05139b76ab0843443ab3 dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -2414,7 +2453,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", "wasm-bindgen-shared", ] @@ -2436,7 +2475,7 @@ checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2459,13 +2498,14 @@ dependencies = [ [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] @@ -2588,7 +2628,7 @@ checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -2608,7 +2648,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] @@ -2619,7 +2659,7 @@ dependencies = [ "once_cell", "pin-project", "rand", - "sha2", + "sha3", "thiserror", "time", "tokio", @@ -2663,7 +2703,7 @@ dependencies = [ "hex", "num-traits", "rand", - "sha2", + "sha3", "thiserror", "tracing", ] @@ -2853,16 +2893,15 @@ dependencies = [ "protox-parse", "quote", "rand", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/node/Cargo.toml b/node/Cargo.toml index d3883596..a0d72b93 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -68,7 +68,7 @@ rand = "0.8.0" rocksdb = "0.21.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.95" -sha2 = "0.10.6" +sha3 = "0.10.8" snow = "0.9.3" syn = "2.0.17" tempfile = "3" diff --git a/node/actors/network/src/noise/stream.rs b/node/actors/network/src/noise/stream.rs index 205edfaa..79b53109 100644 --- a/node/actors/network/src/noise/stream.rs +++ b/node/actors/network/src/noise/stream.rs @@ -9,7 +9,7 @@ use zksync_concurrency::{ ctx, io, io::{AsyncRead as _, AsyncWrite as _}, }; -use zksync_consensus_crypto::{sha256::Sha256, ByteFmt}; +use zksync_consensus_crypto::{keccak256::Keccak256, ByteFmt}; /// Fixed noise configuration. Nodes need to use the same noise /// configuration to be able to connect to each other. @@ -75,7 +75,7 @@ impl Default for Buffer { pub(crate) struct Stream { /// Hash of the handshake messages. /// Uniquely identifies the noise session. - id: Sha256, + id: Keccak256, /// Underlying TCP stream. #[pin] inner: S, @@ -138,7 +138,7 @@ where /// Returns the noise session id. /// See `Stream::id`. - pub(crate) fn id(&self) -> Sha256 { + pub(crate) fn id(&self) -> Keccak256 { self.id } diff --git a/node/deny.toml b/node/deny.toml index 8bfe4c74..129c4ddf 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -55,11 +55,12 @@ skip = [ { name = "regex-syntax", version = "=0.6.29" }, # Old versions required by hyper. - { name = "socket2", version = "=0.4.9" }, - { name = "hashbrown", version = "=0.12.3" }, # (hyper -> h2 -> indexmap -> hashbrown) + { name = "socket2", version = "0.4.10" }, # Old versions required by ark-bn254. - { name = "syn", version = "=1.0.109" } + { name = "syn", version = "=1.0.109" }, + { name = "hashbrown", version = "=0.13.2" }, + { name = "itertools", version = "=0.10.5" } ] [sources] diff --git a/node/libs/concurrency/Cargo.toml b/node/libs/concurrency/Cargo.toml index 7cfb3e9f..860f5be1 100644 --- a/node/libs/concurrency/Cargo.toml +++ b/node/libs/concurrency/Cargo.toml @@ -11,7 +11,7 @@ anyhow.workspace = true once_cell.workspace = true pin-project.workspace = true rand.workspace = true -sha2.workspace = true +sha3.workspace = true thiserror.workspace = true time.workspace = true tokio.workspace = true diff --git a/node/libs/concurrency/src/ctx/rng.rs b/node/libs/concurrency/src/ctx/rng.rs index 7857e111..a57ac333 100644 --- a/node/libs/concurrency/src/ctx/rng.rs +++ b/node/libs/concurrency/src/ctx/rng.rs @@ -2,7 +2,7 @@ use rand::{ rngs::{OsRng, StdRng}, Rng, SeedableRng, }; -use sha2::{digest::Update as _, Digest as _}; +use sha3::{digest::Update as _, Digest as _}; use std::sync::atomic::{AtomicU64, Ordering}; /// Splittable Pseudorandom Number Generators using Cryptographic Hashing @@ -16,13 +16,13 @@ use std::sync::atomic::{AtomicU64, Ordering}; /// which is not crypto-secure, but for tests should be good enough. pub(super) struct SplitProvider { /// Hash builder of the prefix of the ID. - builder: sha2::Sha256, + builder: sha3::Keccak256, /// Child branches constructed so far from this branch. branch: AtomicU64, } impl SplitProvider { - fn next_builder(&self) -> sha2::Sha256 { + fn next_builder(&self) -> sha3::Keccak256 { let branch = self.branch.fetch_add(1, Ordering::SeqCst); self.builder.clone().chain(branch.to_le_bytes()) } @@ -43,7 +43,7 @@ impl Provider { pub(super) fn test() -> Provider { Self::Split( SplitProvider { - builder: sha2::Sha256::new().chain("Lzr81nDW8eSOMH".as_bytes()), + builder: sha3::Keccak256::new().chain("Lzr81nDW8eSOMH".as_bytes()), branch: 0.into(), } .into(), diff --git a/node/libs/crypto/Cargo.toml b/node/libs/crypto/Cargo.toml index 117dfae2..91eda71f 100644 --- a/node/libs/crypto/Cargo.toml +++ b/node/libs/crypto/Cargo.toml @@ -16,7 +16,7 @@ num-traits.workspace = true ed25519-dalek.workspace = true hex.workspace = true rand.workspace = true -sha2.workspace = true +sha3.workspace = true thiserror.workspace = true tracing.workspace = true diff --git a/node/libs/crypto/src/bn254/hash.rs b/node/libs/crypto/src/bn254/hash.rs index dd7a7b62..e6b93eb9 100644 --- a/node/libs/crypto/src/bn254/hash.rs +++ b/node/libs/crypto/src/bn254/hash.rs @@ -2,13 +2,13 @@ use ark_bn254::{G1Affine, G1Projective}; use ark_ec::AffineRepr as _; -use sha2::Digest as _; +use sha3::Digest as _; /// Hashes an arbitrary message and maps it to an elliptic curve point in G1. pub(crate) fn hash_to_g1(msg: &[u8]) -> G1Projective { for i in 0..256 { // Hash the message with the index as suffix. - let bytes: [u8; 32] = sha2::Sha256::new() + let bytes: [u8; 32] = sha3::Keccak256::new() .chain_update(msg) .chain_update((i as u32).to_be_bytes()) .finalize() diff --git a/node/libs/crypto/src/sha256/mod.rs b/node/libs/crypto/src/keccak256/mod.rs similarity index 64% rename from node/libs/crypto/src/sha256/mod.rs rename to node/libs/crypto/src/keccak256/mod.rs index bc7510dc..1c94c243 100644 --- a/node/libs/crypto/src/sha256/mod.rs +++ b/node/libs/crypto/src/keccak256/mod.rs @@ -1,21 +1,23 @@ -//! Wrappers for the SHA256 cryptographic hash algorithm. +//! Wrappers for the Keccak256 cryptographic hash algorithm. use crate::ByteFmt; -use sha2::{digest::Update as _, Digest as _}; +use sha3::{digest::Update as _, Digest as _}; +#[cfg(test)] +mod test; pub mod testonly; -/// SHA256 hash. +/// Keccak256 hash. #[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Sha256(pub(crate) [u8; 32]); +pub struct Keccak256(pub(crate) [u8; 32]); -impl Sha256 { - /// Computes a SHA-256 hash of a message. +impl Keccak256 { + /// Computes a Keccak256 hash of a message. pub fn new(msg: &[u8]) -> Self { - Self(sha2::Sha256::new().chain(msg).finalize().into()) + Self(sha3::Keccak256::new().chain(msg).finalize().into()) } /// Interprets the specified `bytes` as a hash digest (i.e., a reverse operation to [`Self::as_bytes()`]). - /// It is caller's responsibility to ensure that `bytes` are actually a SHA-256 hash digest. + /// It is caller's responsibility to ensure that `bytes` are actually a Keccak256 hash digest. pub fn from_bytes(bytes: [u8; 32]) -> Self { Self(bytes) } @@ -26,7 +28,7 @@ impl Sha256 { } } -impl ByteFmt for Sha256 { +impl ByteFmt for Keccak256 { fn decode(bytes: &[u8]) -> anyhow::Result { Ok(Self(bytes.try_into()?)) } diff --git a/node/libs/crypto/src/keccak256/test.rs b/node/libs/crypto/src/keccak256/test.rs new file mode 100644 index 00000000..05f71589 --- /dev/null +++ b/node/libs/crypto/src/keccak256/test.rs @@ -0,0 +1,33 @@ +#[test] +fn test_keccak256() -> Result<(), Box> { + use crate::keccak256::Keccak256; + + // Test vectors obtained from https://emn178.github.io/online-tools/keccak_256.html + let test_vectors: Vec<(&[u8], [u8; 32])> = vec![ + ( + b"testing", + hex::decode("5f16f4c7f149ac4f9510d9cf8cf384038ad348b3bcdc01915f95de12df9d1b02")? + .try_into() + .unwrap(), + ), + ( + b"", + hex::decode("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")? + .try_into() + .unwrap(), + ), + ( + &[0x12, 0x34, 0x56], + hex::decode("6adf031833174bbe4c85eafe59ddb54e6584648c2c962c6f94791ab49caa0ad4")? + .try_into() + .unwrap(), + ), + ]; + + for (input, expected_hash) in &test_vectors { + let hash = Keccak256::new(input); + assert_eq!(hash.as_bytes(), expected_hash); + } + + Ok(()) +} diff --git a/node/libs/crypto/src/keccak256/testonly.rs b/node/libs/crypto/src/keccak256/testonly.rs new file mode 100644 index 00000000..f60aeb60 --- /dev/null +++ b/node/libs/crypto/src/keccak256/testonly.rs @@ -0,0 +1,13 @@ +//! Random hash generation, intended for use in testing + +use crate::keccak256::Keccak256; +use rand::{ + distributions::{Distribution, Standard}, + Rng, +}; + +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> Keccak256 { + Keccak256(rng.gen()) + } +} diff --git a/node/libs/crypto/src/lib.rs b/node/libs/crypto/src/lib.rs index 5c22086c..2168af09 100644 --- a/node/libs/crypto/src/lib.rs +++ b/node/libs/crypto/src/lib.rs @@ -8,4 +8,4 @@ pub mod bls12_381; pub mod bn254; pub mod ed25519; mod fmt; -pub mod sha256; +pub mod keccak256; diff --git a/node/libs/crypto/src/sha256/testonly.rs b/node/libs/crypto/src/sha256/testonly.rs deleted file mode 100644 index d306aeb6..00000000 --- a/node/libs/crypto/src/sha256/testonly.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Random hash generation, intended for use in testing - -use crate::sha256::Sha256; -use rand::{ - distributions::{Distribution, Standard}, - Rng, -}; - -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> Sha256 { - Sha256(rng.gen()) - } -} diff --git a/node/libs/roles/src/node/messages.rs b/node/libs/roles/src/node/messages.rs index 7cca89ce..4dbeda28 100644 --- a/node/libs/roles/src/node/messages.rs +++ b/node/libs/roles/src/node/messages.rs @@ -1,5 +1,5 @@ use crate::node; -use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; +use zksync_consensus_crypto::{keccak256, ByteFmt, Text, TextFmt}; use zksync_consensus_utils::enum_util::{BadVariantError, Variant}; /// The ID for an authentication session. @@ -17,7 +17,7 @@ pub enum Msg { impl Msg { /// Get the hash of this message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) + MsgHash(keccak256::Keccak256::new(&zksync_protobuf::canonical(self))) } } @@ -52,7 +52,7 @@ impl + Clone> Signed { } /// The hash of a message. -pub struct MsgHash(pub(super) sha256::Sha256); +pub struct MsgHash(pub(super) keccak256::Keccak256); impl ByteFmt for MsgHash { fn encode(&self) -> Vec { @@ -66,11 +66,13 @@ impl ByteFmt for MsgHash { impl TextFmt for MsgHash { fn encode(&self) -> String { format!( - "validator_msg:sha256:{}", + "validator_msg:keccak256:{}", hex::encode(ByteFmt::encode(&self.0)) ) } fn decode(text: Text) -> anyhow::Result { - text.strip("validator_msg:sha256:")?.decode_hex().map(Self) + text.strip("validator_msg:keccak256:")? + .decode_hex() + .map(Self) } } diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index 7e0d030e..a2e50a7f 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -15,11 +15,11 @@ use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for BlockHeaderHash { type Proto = proto::BlockHeaderHash; fn read(r: &Self::Proto) -> anyhow::Result { - Ok(Self(ByteFmt::decode(required(&r.sha256)?)?)) + Ok(Self(ByteFmt::decode(required(&r.keccak256)?)?)) } fn build(&self) -> Self::Proto { Self::Proto { - sha256: Some(self.0.encode()), + keccak256: Some(self.0.encode()), } } } @@ -27,11 +27,11 @@ impl ProtoFmt for BlockHeaderHash { impl ProtoFmt for PayloadHash { type Proto = proto::PayloadHash; fn read(r: &Self::Proto) -> anyhow::Result { - Ok(Self(ByteFmt::decode(required(&r.sha256)?)?)) + Ok(Self(ByteFmt::decode(required(&r.keccak256)?)?)) } fn build(&self) -> Self::Proto { Self::Proto { - sha256: Some(self.0.encode()), + keccak256: Some(self.0.encode()), } } } @@ -322,12 +322,12 @@ impl ProtoFmt for MsgHash { type Proto = proto::MsgHash; fn read(r: &Self::Proto) -> anyhow::Result { - Ok(Self(ByteFmt::decode(required(&r.sha256)?)?)) + Ok(Self(ByteFmt::decode(required(&r.keccak256)?)?)) } fn build(&self) -> Self::Proto { Self::Proto { - sha256: Some(self.0.encode()), + keccak256: Some(self.0.encode()), } } } diff --git a/node/libs/roles/src/validator/messages/block.rs b/node/libs/roles/src/validator/messages/block.rs index a7ebb0f8..08554690 100644 --- a/node/libs/roles/src/validator/messages/block.rs +++ b/node/libs/roles/src/validator/messages/block.rs @@ -2,7 +2,7 @@ use super::CommitQC; use std::fmt; -use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; +use zksync_consensus_crypto::{keccak256::Keccak256, ByteFmt, Text, TextFmt}; /// Payload of the block. Consensus algorithm does not interpret the payload /// (except for imposing a size limit for the payload). Proposing a payload @@ -13,15 +13,18 @@ pub struct Payload(pub Vec); /// Hash of the Payload. #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct PayloadHash(pub(crate) sha256::Sha256); +pub struct PayloadHash(pub(crate) Keccak256); impl TextFmt for PayloadHash { fn decode(text: Text) -> anyhow::Result { - text.strip("payload:sha256:")?.decode_hex().map(Self) + text.strip("payload:keccak256:")?.decode_hex().map(Self) } fn encode(&self) -> String { - format!("payload:sha256:{}", hex::encode(ByteFmt::encode(&self.0))) + format!( + "payload:keccak256:{}", + hex::encode(ByteFmt::encode(&self.0)) + ) } } @@ -34,7 +37,7 @@ impl fmt::Debug for PayloadHash { impl Payload { /// Hash of the payload. pub fn hash(&self) -> PayloadHash { - PayloadHash(sha256::Sha256::new(&self.0)) + PayloadHash(Keccak256::new(&self.0)) } } @@ -64,13 +67,13 @@ impl fmt::Display for BlockNumber { /// Hash of the block header. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct BlockHeaderHash(pub(crate) sha256::Sha256); +pub struct BlockHeaderHash(pub(crate) Keccak256); impl BlockHeaderHash { /// Interprets the specified `bytes` as a block header hash digest (i.e., a reverse operation to [`Self::as_bytes()`]). /// It is caller's responsibility to ensure that `bytes` are actually a block header hash digest. pub fn from_bytes(bytes: [u8; 32]) -> Self { - Self(sha256::Sha256::from_bytes(bytes)) + Self(Keccak256::from_bytes(bytes)) } /// Returns a reference to the bytes of this hash. @@ -81,14 +84,14 @@ impl BlockHeaderHash { impl TextFmt for BlockHeaderHash { fn decode(text: Text) -> anyhow::Result { - text.strip("block_header_hash:sha256:")? + text.strip("block_header_hash:keccak256:")? .decode_hex() .map(Self) } fn encode(&self) -> String { format!( - "block_header_hash:sha256:{}", + "block_header_hash:keccak256:{}", hex::encode(ByteFmt::encode(&self.0)) ) } @@ -114,13 +117,13 @@ pub struct BlockHeader { impl BlockHeader { /// Returns the hash of the block. pub fn hash(&self) -> BlockHeaderHash { - BlockHeaderHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) + BlockHeaderHash(Keccak256::new(&zksync_protobuf::canonical(self))) } /// Creates a genesis block. pub fn genesis(payload: PayloadHash) -> Self { Self { - parent: BlockHeaderHash(sha256::Sha256::default()), + parent: BlockHeaderHash(Keccak256::default()), number: BlockNumber(0), payload, } diff --git a/node/libs/roles/src/validator/messages/msg.rs b/node/libs/roles/src/validator/messages/msg.rs index cb6c6ad2..a2da4fa4 100644 --- a/node/libs/roles/src/validator/messages/msg.rs +++ b/node/libs/roles/src/validator/messages/msg.rs @@ -2,7 +2,7 @@ use super::{ConsensusMsg, NetAddress}; use crate::{node::SessionId, validator, validator::Error}; use std::fmt; -use zksync_consensus_crypto::{sha256, ByteFmt, Text, TextFmt}; +use zksync_consensus_crypto::{keccak256, ByteFmt, Text, TextFmt}; use zksync_consensus_utils::enum_util::{BadVariantError, Variant}; /// Generic message type for a validator. @@ -19,7 +19,7 @@ pub enum Msg { impl Msg { /// Returns the hash of the message. pub fn hash(&self) -> MsgHash { - MsgHash(sha256::Sha256::new(&zksync_protobuf::canonical(self))) + MsgHash(keccak256::Keccak256::new(&zksync_protobuf::canonical(self))) } } @@ -61,7 +61,7 @@ impl Variant for NetAddress { /// Hash of a message. #[derive(Clone, Copy, PartialEq, Eq)] -pub struct MsgHash(pub(crate) sha256::Sha256); +pub struct MsgHash(pub(crate) keccak256::Keccak256); impl ByteFmt for MsgHash { fn decode(bytes: &[u8]) -> anyhow::Result { @@ -75,12 +75,14 @@ impl ByteFmt for MsgHash { impl TextFmt for MsgHash { fn decode(text: Text) -> anyhow::Result { - text.strip("validator_msg:sha256:")?.decode_hex().map(Self) + text.strip("validator_msg:keccak256:")? + .decode_hex() + .map(Self) } fn encode(&self) -> String { format!( - "validator_msg:sha256:{}", + "validator_msg:keccak256:{}", hex::encode(ByteFmt::encode(&self.0)) ) } diff --git a/node/libs/schema/proto/roles/validator.proto b/node/libs/schema/proto/roles/validator.proto index 7419690f..56ed9b35 100644 --- a/node/libs/schema/proto/roles/validator.proto +++ b/node/libs/schema/proto/roles/validator.proto @@ -5,11 +5,11 @@ package zksync.schema.roles.validator; import "zksync/std.proto"; message PayloadHash { - optional bytes sha256 = 1; // required + optional bytes keccak256 = 1; // required } message BlockHeaderHash { - optional bytes sha256 = 1; // required + optional bytes keccak256 = 1; // required } message BlockHeader { @@ -137,7 +137,7 @@ message Msg { } message MsgHash { - optional bytes sha256 = 1; // required + optional bytes keccak256 = 1; // required } message Signed { From 3f275673ab111fb2b1f62dfe85a30d05e59043b1 Mon Sep 17 00:00:00 2001 From: pompon0 Date: Thu, 9 Nov 2023 11:11:25 +0100 Subject: [PATCH 4/5] Decomposed schema crate. (#30) Now that cross-crate proto imports are supported, we can move the proto files to crates which are actually using them. --- .github/workflows/protobuf.yaml | 6 ++-- node/Cargo.lock | 26 +++-------------- node/Cargo.toml | 2 -- node/actors/bft/Cargo.toml | 1 - node/actors/executor/Cargo.toml | 5 +++- node/actors/executor/build.rs | 11 +++++++ node/actors/executor/src/config/mod.rs | 2 +- .../executor/src/config/proto/mod.proto} | 2 +- node/actors/executor/src/config/proto/mod.rs | 2 ++ node/actors/executor/src/lib.rs | 2 +- node/actors/network/Cargo.toml | 6 +++- node/actors/network/build.rs | 29 +++++++++++++++++++ .../network/src/consensus/handshake/mod.rs | 3 +- .../network/src/gossip/handshake/mod.rs | 3 +- node/actors/network/src/lib.rs | 1 + node/actors/network/src/mux/handshake.rs | 2 +- .../src/mux/{tests.rs => tests/mod.rs} | 2 +- .../network/src/mux/tests/proto/mod.proto} | 2 +- .../actors/network/src/mux/tests/proto/mod.rs | 2 ++ node/actors/network/src/preface.rs | 3 +- .../network/src/proto}/consensus.proto | 4 +-- .../network/src/proto}/gossip.proto | 6 ++-- node/actors/network/src/proto/mod.rs | 2 ++ .../network/src/proto}/mux.proto | 2 +- .../network/src/proto}/ping.proto | 2 +- .../network/src/proto}/preface.proto | 2 +- node/actors/network/src/rpc/consensus.rs | 3 +- node/actors/network/src/rpc/ping.rs | 3 +- node/actors/network/src/rpc/sync_blocks.rs | 3 +- .../network/src/rpc/sync_validator_addrs.rs | 3 +- node/libs/protobuf_build/src/lib.rs | 2 +- node/libs/roles/Cargo.toml | 5 +++- node/libs/{schema => roles}/build.rs | 4 +-- node/libs/roles/src/lib.rs | 1 + node/libs/roles/src/node/conv.rs | 14 ++++----- node/libs/roles/src/node/mod.rs | 2 -- node/libs/roles/src/proto/mod.rs | 2 ++ .../roles => roles/src/proto}/node.proto | 2 +- .../roles => roles/src/proto}/validator.proto | 2 +- node/libs/roles/src/validator/conv.rs | 3 +- node/libs/schema/Cargo.toml | 25 ---------------- node/libs/schema/README.md | 29 ------------------- node/libs/schema/src/lib.rs | 6 ---- node/libs/storage/Cargo.toml | 6 +++- node/libs/storage/build.rs | 14 +++++++++ node/libs/storage/src/lib.rs | 1 + .../src/proto/mod.proto} | 4 +-- node/libs/storage/src/proto/mod.rs | 2 ++ node/libs/storage/src/types.rs | 2 +- node/tools/Cargo.toml | 1 - node/tools/src/config.rs | 3 +- 51 files changed, 130 insertions(+), 142 deletions(-) create mode 100644 node/actors/executor/build.rs rename node/{libs/schema/proto/executor/config.proto => actors/executor/src/config/proto/mod.proto} (99%) create mode 100644 node/actors/executor/src/config/proto/mod.rs create mode 100644 node/actors/network/build.rs rename node/actors/network/src/mux/{tests.rs => tests/mod.rs} (99%) rename node/{libs/schema/proto/network/mux_test.proto => actors/network/src/mux/tests/proto/mod.proto} (78%) create mode 100644 node/actors/network/src/mux/tests/proto/mod.rs rename node/{libs/schema/proto/network => actors/network/src/proto}/consensus.proto (75%) rename node/{libs/schema/proto/network => actors/network/src/proto}/gossip.proto (94%) create mode 100644 node/actors/network/src/proto/mod.rs rename node/{libs/schema/proto/network => actors/network/src/proto}/mux.proto (86%) rename node/{libs/schema/proto/network => actors/network/src/proto}/ping.proto (76%) rename node/{libs/schema/proto/network => actors/network/src/proto}/preface.proto (94%) rename node/libs/{schema => roles}/build.rs (82%) create mode 100644 node/libs/roles/src/proto/mod.rs rename node/libs/{schema/proto/roles => roles/src/proto}/node.proto (89%) rename node/libs/{schema/proto/roles => roles/src/proto}/validator.proto (99%) delete mode 100644 node/libs/schema/Cargo.toml delete mode 100644 node/libs/schema/README.md delete mode 100644 node/libs/schema/src/lib.rs create mode 100644 node/libs/storage/build.rs rename node/libs/{schema/proto/storage.proto => storage/src/proto/mod.proto} (81%) create mode 100644 node/libs/storage/src/proto/mod.rs diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml index 59bd6a36..0f5b7f54 100644 --- a/.github/workflows/protobuf.yaml +++ b/.github/workflows/protobuf.yaml @@ -23,12 +23,12 @@ env: jobs: compatibility: - runs-on: ubuntu-latest + runs-on: [ubuntu-22.04-github-hosted-16core] steps: # github.base_ref -> github.head_ref for pull_request # github.event.before -> github.event.after for push - uses: mozilla-actions/sccache-action@v0.0.3 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.base_ref || github.event.before }} path: before @@ -40,7 +40,7 @@ jobs: perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/' `find ./before/node/target/debug/build/*/output` | xargs cat > ./before.binpb - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.head_ref || github.event.after }} path: after diff --git a/node/Cargo.lock b/node/Cargo.lock index 054a68cf..f60b11ed 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -2683,7 +2683,6 @@ dependencies = [ "zksync_consensus_crypto", "zksync_consensus_network", "zksync_consensus_roles", - "zksync_consensus_schema", "zksync_consensus_storage", "zksync_consensus_utils", "zksync_protobuf", @@ -2713,6 +2712,7 @@ name = "zksync_consensus_executor" version = "0.1.0" dependencies = [ "anyhow", + "prost", "rand", "tokio", "tracing", @@ -2722,7 +2722,6 @@ dependencies = [ "zksync_consensus_crypto", "zksync_consensus_network", "zksync_consensus_roles", - "zksync_consensus_schema", "zksync_consensus_storage", "zksync_consensus_sync_blocks", "zksync_consensus_utils", @@ -2739,6 +2738,7 @@ dependencies = [ "once_cell", "pin-project", "pretty_assertions", + "prost", "rand", "snow", "test-casing", @@ -2749,7 +2749,6 @@ dependencies = [ "zksync_concurrency", "zksync_consensus_crypto", "zksync_consensus_roles", - "zksync_consensus_schema", "zksync_consensus_utils", "zksync_protobuf", ] @@ -2761,32 +2760,16 @@ dependencies = [ "anyhow", "bit-vec", "hex", + "prost", "rand", "serde", "tracing", "zksync_concurrency", "zksync_consensus_crypto", - "zksync_consensus_schema", "zksync_consensus_utils", "zksync_protobuf", ] -[[package]] -name = "zksync_consensus_schema" -version = "0.1.0" -dependencies = [ - "anyhow", - "bit-vec", - "once_cell", - "prost", - "rand", - "serde", - "serde_json", - "tokio", - "zksync_concurrency", - "zksync_protobuf", -] - [[package]] name = "zksync_consensus_storage" version = "0.1.0" @@ -2794,6 +2777,7 @@ dependencies = [ "anyhow", "assert_matches", "async-trait", + "prost", "rand", "rocksdb", "tempfile", @@ -2803,7 +2787,6 @@ dependencies = [ "tracing", "zksync_concurrency", "zksync_consensus_roles", - "zksync_consensus_schema", "zksync_protobuf", ] @@ -2842,7 +2825,6 @@ dependencies = [ "zksync_consensus_crypto", "zksync_consensus_executor", "zksync_consensus_roles", - "zksync_consensus_schema", "zksync_consensus_storage", "zksync_consensus_utils", "zksync_protobuf", diff --git a/node/Cargo.toml b/node/Cargo.toml index a0d72b93..8b92e14b 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -9,7 +9,6 @@ members = [ "actors/network", "actors/sync_blocks", "libs/roles", - "libs/schema", "libs/storage", "tools", "libs/utils", @@ -32,7 +31,6 @@ zksync_consensus_executor = { path = "actors/executor" } zksync_consensus_network = { path = "actors/network" } zksync_consensus_sync_blocks = { path = "actors/sync_blocks" } zksync_consensus_roles = { path = "libs/roles" } -zksync_consensus_schema = { path = "libs/schema" } zksync_consensus_storage = { path = "libs/storage" } zksync_consensus_tools = { path = "tools" } zksync_consensus_utils = { path = "libs/utils" } diff --git a/node/actors/bft/Cargo.toml b/node/actors/bft/Cargo.toml index db42209d..82110ac5 100644 --- a/node/actors/bft/Cargo.toml +++ b/node/actors/bft/Cargo.toml @@ -11,7 +11,6 @@ zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true zksync_consensus_network.workspace = true zksync_consensus_roles.workspace = true -zksync_consensus_schema.workspace = true zksync_consensus_storage.workspace = true zksync_consensus_utils.workspace = true zksync_protobuf.workspace = true diff --git a/node/actors/executor/Cargo.toml b/node/actors/executor/Cargo.toml index 9f94353e..75be56b5 100644 --- a/node/actors/executor/Cargo.toml +++ b/node/actors/executor/Cargo.toml @@ -12,16 +12,19 @@ zksync_consensus_bft.workspace = true zksync_consensus_crypto.workspace = true zksync_consensus_network.workspace = true zksync_consensus_roles.workspace = true -zksync_consensus_schema.workspace = true zksync_consensus_storage.workspace = true zksync_consensus_sync_blocks.workspace = true zksync_consensus_utils.workspace = true zksync_protobuf.workspace = true anyhow.workspace = true +prost.workspace = true rand.workspace = true tracing.workspace = true vise.workspace = true [dev-dependencies] tokio.workspace = true + +[build-dependencies] +zksync_protobuf.workspace = true diff --git a/node/actors/executor/build.rs b/node/actors/executor/build.rs new file mode 100644 index 00000000..8c157605 --- /dev/null +++ b/node/actors/executor/build.rs @@ -0,0 +1,11 @@ +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf::build::Config { + input_root: "src/config/proto".into(), + proto_root: "zksync/executor/config".into(), + dependencies: vec![], + protobuf_crate: "::zksync_protobuf".parse().unwrap(), + } + .generate() + .expect("generate(config)"); +} diff --git a/node/actors/executor/src/config/mod.rs b/node/actors/executor/src/config/mod.rs index 0ebfab55..2824a5cb 100644 --- a/node/actors/executor/src/config/mod.rs +++ b/node/actors/executor/src/config/mod.rs @@ -7,9 +7,9 @@ use std::{ use zksync_consensus_crypto::{read_required_text, Text, TextFmt}; use zksync_consensus_network::{consensus, gossip}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::proto::executor::config as proto; use zksync_protobuf::{read_required, required, ProtoFmt}; +pub mod proto; #[cfg(test)] mod tests; diff --git a/node/libs/schema/proto/executor/config.proto b/node/actors/executor/src/config/proto/mod.proto similarity index 99% rename from node/libs/schema/proto/executor/config.proto rename to node/actors/executor/src/config/proto/mod.proto index 20092d93..8cfce427 100644 --- a/node/libs/schema/proto/executor/config.proto +++ b/node/actors/executor/src/config/proto/mod.proto @@ -36,7 +36,7 @@ // the validator set) or move it to a separate config file. syntax = "proto3"; -package zksync.schema.executor.config; +package zksync.executor.config; // (public key, ip address) of a gossip network node. message NodeAddr { diff --git a/node/actors/executor/src/config/proto/mod.rs b/node/actors/executor/src/config/proto/mod.rs new file mode 100644 index 00000000..2ea69ab4 --- /dev/null +++ b/node/actors/executor/src/config/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/config/proto/gen.rs")); diff --git a/node/actors/executor/src/lib.rs b/node/actors/executor/src/lib.rs index 9de73fdd..a1d779ca 100644 --- a/node/actors/executor/src/lib.rs +++ b/node/actors/executor/src/lib.rs @@ -18,7 +18,7 @@ pub mod testonly; #[cfg(test)] mod tests; -pub use self::config::{ConsensusConfig, ExecutorConfig, GossipConfig}; +pub use self::config::{proto, ConsensusConfig, ExecutorConfig, GossipConfig}; /// Validator-related part of [`Executor`]. #[derive(Debug)] diff --git a/node/actors/network/Cargo.toml b/node/actors/network/Cargo.toml index 20c85bcb..34847a9d 100644 --- a/node/actors/network/Cargo.toml +++ b/node/actors/network/Cargo.toml @@ -10,7 +10,6 @@ license.workspace = true zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true zksync_consensus_roles.workspace = true -zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true zksync_protobuf.workspace = true @@ -19,6 +18,7 @@ async-trait.workspace = true im.workspace = true once_cell.workspace = true pin-project.workspace = true +prost.workspace = true rand.workspace = true snow.workspace = true thiserror.workspace = true @@ -29,3 +29,7 @@ vise.workspace = true pretty_assertions.workspace = true test-casing.workspace = true tokio.workspace = true + +[build-dependencies] +zksync_consensus_roles.workspace = true +zksync_protobuf.workspace = true diff --git a/node/actors/network/build.rs b/node/actors/network/build.rs new file mode 100644 index 00000000..e8ec05eb --- /dev/null +++ b/node/actors/network/build.rs @@ -0,0 +1,29 @@ +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf::build::Config { + input_root: "src/proto".into(), + proto_root: "zksync/network".into(), + dependencies: vec![ + ( + "::zksync_protobuf::proto".parse().unwrap(), + &zksync_protobuf::proto::DESCRIPTOR, + ), + ( + "::zksync_consensus_roles::proto".parse().unwrap(), + &zksync_consensus_roles::proto::DESCRIPTOR, + ), + ], + protobuf_crate: "::zksync_protobuf".parse().unwrap(), + } + .generate() + .expect("generate()"); + + zksync_protobuf::build::Config { + input_root: "src/mux/tests/proto".into(), + proto_root: "zksync/network/mux/tests".into(), + dependencies: vec![], + protobuf_crate: "::zksync_protobuf".parse().unwrap(), + } + .generate() + .expect("generate()"); +} diff --git a/node/actors/network/src/consensus/handshake/mod.rs b/node/actors/network/src/consensus/handshake/mod.rs index 65275cc6..f6218cd9 100644 --- a/node/actors/network/src/consensus/handshake/mod.rs +++ b/node/actors/network/src/consensus/handshake/mod.rs @@ -1,9 +1,8 @@ -use crate::{frame, noise}; +use crate::{frame, noise, proto::consensus as proto}; use anyhow::Context as _; use zksync_concurrency::{ctx, time}; use zksync_consensus_crypto::ByteFmt; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::proto::network::consensus as proto; use zksync_protobuf::{read_required, ProtoFmt}; #[cfg(test)] diff --git a/node/actors/network/src/gossip/handshake/mod.rs b/node/actors/network/src/gossip/handshake/mod.rs index 45558c53..e823baa0 100644 --- a/node/actors/network/src/gossip/handshake/mod.rs +++ b/node/actors/network/src/gossip/handshake/mod.rs @@ -1,10 +1,9 @@ use super::Config; -use crate::{frame, noise}; +use crate::{frame, noise, proto::gossip as proto}; use anyhow::Context as _; use zksync_concurrency::{ctx, time}; use zksync_consensus_crypto::ByteFmt; use zksync_consensus_roles::node; -use zksync_consensus_schema::proto::network::gossip as proto; use zksync_protobuf::{read_required, required, ProtoFmt}; #[cfg(test)] diff --git a/node/actors/network/src/lib.rs b/node/actors/network/src/lib.rs index 011b6e20..0f811d39 100644 --- a/node/actors/network/src/lib.rs +++ b/node/actors/network/src/lib.rs @@ -14,6 +14,7 @@ mod mux; mod noise; mod pool; mod preface; +pub mod proto; mod rpc; mod state; pub mod testonly; diff --git a/node/actors/network/src/mux/handshake.rs b/node/actors/network/src/mux/handshake.rs index bfee19db..1105e079 100644 --- a/node/actors/network/src/mux/handshake.rs +++ b/node/actors/network/src/mux/handshake.rs @@ -1,7 +1,7 @@ use super::CapabilityId; +use crate::proto::mux as proto; use anyhow::Context as _; use std::collections::HashMap; -use zksync_consensus_schema::proto::network::mux as proto; use zksync_protobuf::required; pub(super) struct Handshake { diff --git a/node/actors/network/src/mux/tests.rs b/node/actors/network/src/mux/tests/mod.rs similarity index 99% rename from node/actors/network/src/mux/tests.rs rename to node/actors/network/src/mux/tests/mod.rs index 2e334964..3c91c8ee 100644 --- a/node/actors/network/src/mux/tests.rs +++ b/node/actors/network/src/mux/tests/mod.rs @@ -9,8 +9,8 @@ use std::{ }, }; use zksync_concurrency::{ctx, scope, testonly::abort_on_panic}; -use zksync_consensus_schema::proto::network::mux_test as proto; use zksync_consensus_utils::no_copy::NoCopy; +mod proto; fn assert_partition(sets: &[u16]) { let mut sum = 0; diff --git a/node/libs/schema/proto/network/mux_test.proto b/node/actors/network/src/mux/tests/proto/mod.proto similarity index 78% rename from node/libs/schema/proto/network/mux_test.proto rename to node/actors/network/src/mux/tests/proto/mod.proto index b4cdaf87..59dd4cab 100644 --- a/node/libs/schema/proto/network/mux_test.proto +++ b/node/actors/network/src/mux/tests/proto/mod.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.schema.network.mux_test; +package zksync.network.mux.tests; message Req { optional bytes input = 1; diff --git a/node/actors/network/src/mux/tests/proto/mod.rs b/node/actors/network/src/mux/tests/proto/mod.rs new file mode 100644 index 00000000..bce60e4a --- /dev/null +++ b/node/actors/network/src/mux/tests/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/mux/tests/proto/gen.rs")); diff --git a/node/actors/network/src/preface.rs b/node/actors/network/src/preface.rs index 75a042c1..72cce5eb 100644 --- a/node/actors/network/src/preface.rs +++ b/node/actors/network/src/preface.rs @@ -7,9 +7,8 @@ //! //! Hence, the preface protocol is used to enable encryption //! and multiplex between mutliple endpoints available on the same TCP port. -use crate::{frame, metrics, noise}; +use crate::{frame, metrics, noise, proto::preface as proto}; use zksync_concurrency::{ctx, time}; -use zksync_consensus_schema::proto::network::preface as proto; use zksync_protobuf::{required, ProtoFmt}; /// Timeout on executing the preface protocol. diff --git a/node/libs/schema/proto/network/consensus.proto b/node/actors/network/src/proto/consensus.proto similarity index 75% rename from node/libs/schema/proto/network/consensus.proto rename to node/actors/network/src/proto/consensus.proto index 09be8a08..e86aaeaf 100644 --- a/node/libs/schema/proto/network/consensus.proto +++ b/node/actors/network/src/proto/consensus.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package zksync.schema.network.consensus; +package zksync.network.consensus; -import "zksync/schema/roles/validator.proto"; +import "zksync/roles/validator.proto"; import "zksync/std.proto"; // First message exchanged in the encrypted session. diff --git a/node/libs/schema/proto/network/gossip.proto b/node/actors/network/src/proto/gossip.proto similarity index 94% rename from node/libs/schema/proto/network/gossip.proto rename to node/actors/network/src/proto/gossip.proto index 723bb100..fce8b6a5 100644 --- a/node/libs/schema/proto/network/gossip.proto +++ b/node/actors/network/src/proto/gossip.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package zksync.schema.network.gossip; +package zksync.network.gossip; -import "zksync/schema/roles/node.proto"; -import "zksync/schema/roles/validator.proto"; +import "zksync/roles/node.proto"; +import "zksync/roles/validator.proto"; // First message exchanged in the encrypted session. message Handshake { diff --git a/node/actors/network/src/proto/mod.rs b/node/actors/network/src/proto/mod.rs new file mode 100644 index 00000000..660bf4c5 --- /dev/null +++ b/node/actors/network/src/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs")); diff --git a/node/libs/schema/proto/network/mux.proto b/node/actors/network/src/proto/mux.proto similarity index 86% rename from node/libs/schema/proto/network/mux.proto rename to node/actors/network/src/proto/mux.proto index 3e014c8e..6028679e 100644 --- a/node/libs/schema/proto/network/mux.proto +++ b/node/actors/network/src/proto/mux.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.schema.network.mux; +package zksync.network.mux; import "zksync/std.proto"; diff --git a/node/libs/schema/proto/network/ping.proto b/node/actors/network/src/proto/ping.proto similarity index 76% rename from node/libs/schema/proto/network/ping.proto rename to node/actors/network/src/proto/ping.proto index 863022fd..b63fae15 100644 --- a/node/libs/schema/proto/network/ping.proto +++ b/node/actors/network/src/proto/ping.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.schema.network.ping; +package zksync.network.ping; message PingReq { optional bytes data = 1; diff --git a/node/libs/schema/proto/network/preface.proto b/node/actors/network/src/proto/preface.proto similarity index 94% rename from node/libs/schema/proto/network/preface.proto rename to node/actors/network/src/proto/preface.proto index 193ca38e..a729794a 100644 --- a/node/libs/schema/proto/network/preface.proto +++ b/node/actors/network/src/proto/preface.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.schema.network.preface; +package zksync.network.preface; // Every connection starts with the following preface: // 1. client sends Encryption msg to server. diff --git a/node/actors/network/src/rpc/consensus.rs b/node/actors/network/src/rpc/consensus.rs index 90a20b6e..f266f2b6 100644 --- a/node/actors/network/src/rpc/consensus.rs +++ b/node/actors/network/src/rpc/consensus.rs @@ -1,8 +1,7 @@ //! Defines RPC for passing consensus messages. -use crate::mux; +use crate::{mux, proto::consensus as proto}; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator; -use zksync_consensus_schema::proto::network::consensus as proto; use zksync_protobuf::{read_required, ProtoFmt}; /// Consensus RPC. diff --git a/node/actors/network/src/rpc/ping.rs b/node/actors/network/src/rpc/ping.rs index fda4ebf8..8bd2b79c 100644 --- a/node/actors/network/src/rpc/ping.rs +++ b/node/actors/network/src/rpc/ping.rs @@ -1,9 +1,8 @@ //! Defines an RPC for sending ping messages. -use crate::{mux, rpc::Rpc as _}; +use crate::{mux, proto::ping as proto, rpc::Rpc as _}; use anyhow::Context as _; use rand::Rng; use zksync_concurrency::{ctx, limiter, time}; -use zksync_consensus_schema::proto::network::ping as proto; use zksync_protobuf::{required, ProtoFmt}; /// Ping RPC. diff --git a/node/actors/network/src/rpc/sync_blocks.rs b/node/actors/network/src/rpc/sync_blocks.rs index 4dd22af7..69ba9501 100644 --- a/node/actors/network/src/rpc/sync_blocks.rs +++ b/node/actors/network/src/rpc/sync_blocks.rs @@ -1,9 +1,8 @@ //! Defines RPC for synchronizing blocks. -use crate::{io, mux}; +use crate::{io, mux, proto::gossip as proto}; use anyhow::Context; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator::{BlockNumber, FinalBlock}; -use zksync_consensus_schema::proto::network::gossip as proto; use zksync_protobuf::{read_required, ProtoFmt}; /// `get_sync_state` RPC. diff --git a/node/actors/network/src/rpc/sync_validator_addrs.rs b/node/actors/network/src/rpc/sync_validator_addrs.rs index d505feda..35b31370 100644 --- a/node/actors/network/src/rpc/sync_validator_addrs.rs +++ b/node/actors/network/src/rpc/sync_validator_addrs.rs @@ -1,10 +1,9 @@ //! Defines an Rpc for synchronizing ValidatorAddrs data. -use crate::mux; +use crate::{mux, proto::gossip as proto}; use anyhow::Context as _; use std::sync::Arc; use zksync_concurrency::{limiter, time}; use zksync_consensus_roles::validator; -use zksync_consensus_schema::proto::network::gossip as proto; use zksync_protobuf::ProtoFmt; /// SyncValidatorAddrs Rpc. diff --git a/node/libs/protobuf_build/src/lib.rs b/node/libs/protobuf_build/src/lib.rs index 0b207628..ff933aea 100644 --- a/node/libs/protobuf_build/src/lib.rs +++ b/node/libs/protobuf_build/src/lib.rs @@ -112,7 +112,7 @@ macro_rules! declare_descriptor { pub static DESCRIPTOR: $crate::Lazy<$crate::Descriptor> = $crate::Lazy::new(|| { $crate::Descriptor::new( $package_root.into(), - ::std::vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR })*], + ::std::vec![$({ use $rust_deps as dep; &dep::DESCRIPTOR }),*], include_bytes!($descriptor_path), ) }); diff --git a/node/libs/roles/Cargo.toml b/node/libs/roles/Cargo.toml index 9b4b16f7..9cb2c49d 100644 --- a/node/libs/roles/Cargo.toml +++ b/node/libs/roles/Cargo.toml @@ -9,13 +9,16 @@ license.workspace = true [dependencies] zksync_concurrency.workspace = true zksync_consensus_crypto.workspace = true -zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true zksync_protobuf.workspace = true anyhow.workspace = true bit-vec.workspace = true hex.workspace = true +prost.workspace = true rand.workspace = true serde.workspace = true tracing.workspace = true + +[build-dependencies] +zksync_protobuf.workspace = true diff --git a/node/libs/schema/build.rs b/node/libs/roles/build.rs similarity index 82% rename from node/libs/schema/build.rs rename to node/libs/roles/build.rs index 7e8cc416..b52d64e8 100644 --- a/node/libs/schema/build.rs +++ b/node/libs/roles/build.rs @@ -1,8 +1,8 @@ //! Generates rust code from protobufs. fn main() { zksync_protobuf::build::Config { - input_root: "proto".into(), - proto_root: "zksync/schema".into(), + input_root: "src/proto".into(), + proto_root: "zksync/roles".into(), dependencies: vec![( "::zksync_protobuf::proto".parse().expect("dependency_path"), &zksync_protobuf::proto::DESCRIPTOR, diff --git a/node/libs/roles/src/lib.rs b/node/libs/roles/src/lib.rs index a0ea5190..8f836ca7 100644 --- a/node/libs/roles/src/lib.rs +++ b/node/libs/roles/src/lib.rs @@ -8,4 +8,5 @@ //! every node has this role. pub mod node; +pub mod proto; pub mod validator; diff --git a/node/libs/roles/src/node/conv.rs b/node/libs/roles/src/node/conv.rs index 6d7ea973..f3823d27 100644 --- a/node/libs/roles/src/node/conv.rs +++ b/node/libs/roles/src/node/conv.rs @@ -1,19 +1,19 @@ -use crate::node; +use crate::{node, proto::node as proto}; use anyhow::Context as _; use zksync_consensus_crypto::ByteFmt; use zksync_consensus_utils::enum_util::Variant; use zksync_protobuf::{read_required, required, ProtoFmt}; impl ProtoFmt for node::Msg { - type Proto = node::schema::Msg; + type Proto = proto::Msg; fn read(r: &Self::Proto) -> anyhow::Result { - use node::schema::msg::T; + use proto::msg::T; Ok(match required(&r.t)? { T::SessionId(r) => Self::SessionId(node::SessionId(r.clone())), }) } fn build(&self) -> Self::Proto { - use node::schema::msg::T; + use proto::msg::T; let t = match self { Self::SessionId(x) => T::SessionId(x.0.clone()), }; @@ -22,7 +22,7 @@ impl ProtoFmt for node::Msg { } impl ProtoFmt for node::PublicKey { - type Proto = node::schema::PublicKey; + type Proto = proto::PublicKey; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(ByteFmt::decode(required(&r.ed25519)?)?)) } @@ -34,7 +34,7 @@ impl ProtoFmt for node::PublicKey { } impl ProtoFmt for node::Signature { - type Proto = node::schema::Signature; + type Proto = proto::Signature; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self(ByteFmt::decode(required(&r.ed25519)?)?)) } @@ -46,7 +46,7 @@ impl ProtoFmt for node::Signature { } impl + Clone> ProtoFmt for node::Signed { - type Proto = node::schema::Signed; + type Proto = proto::Signed; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { msg: V::extract(read_required::(&r.msg).context("msg")?)?, diff --git a/node/libs/roles/src/node/mod.rs b/node/libs/roles/src/node/mod.rs index e22bfd04..f1832715 100644 --- a/node/libs/roles/src/node/mod.rs +++ b/node/libs/roles/src/node/mod.rs @@ -1,5 +1,4 @@ //! Node role implementation. - mod conv; mod keys; mod messages; @@ -7,7 +6,6 @@ mod testonly; pub use keys::*; pub use messages::*; -pub use zksync_consensus_schema::proto::roles::node as schema; #[cfg(test)] mod tests; diff --git a/node/libs/roles/src/proto/mod.rs b/node/libs/roles/src/proto/mod.rs new file mode 100644 index 00000000..660bf4c5 --- /dev/null +++ b/node/libs/roles/src/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs")); diff --git a/node/libs/schema/proto/roles/node.proto b/node/libs/roles/src/proto/node.proto similarity index 89% rename from node/libs/schema/proto/roles/node.proto rename to node/libs/roles/src/proto/node.proto index 87b75c1e..6152ec8c 100644 --- a/node/libs/schema/proto/roles/node.proto +++ b/node/libs/roles/src/proto/node.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.schema.roles.node; +package zksync.roles.node; message Msg { oneof t { diff --git a/node/libs/schema/proto/roles/validator.proto b/node/libs/roles/src/proto/validator.proto similarity index 99% rename from node/libs/schema/proto/roles/validator.proto rename to node/libs/roles/src/proto/validator.proto index 56ed9b35..40f7563c 100644 --- a/node/libs/schema/proto/roles/validator.proto +++ b/node/libs/roles/src/proto/validator.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package zksync.schema.roles.validator; +package zksync.roles.validator; import "zksync/std.proto"; diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index a2e50a7f..b7df925f 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -4,11 +4,10 @@ use super::{ PrepareQC, ProtocolVersion, PublicKey, ReplicaCommit, ReplicaPrepare, Signature, Signed, Signers, ViewNumber, }; -use crate::node::SessionId; +use crate::{node::SessionId, proto::validator as proto}; use anyhow::Context as _; use std::collections::BTreeMap; use zksync_consensus_crypto::ByteFmt; -use zksync_consensus_schema::proto::roles::validator as proto; use zksync_consensus_utils::enum_util::Variant; use zksync_protobuf::{read_required, required, ProtoFmt}; diff --git a/node/libs/schema/Cargo.toml b/node/libs/schema/Cargo.toml deleted file mode 100644 index d0e06888..00000000 --- a/node/libs/schema/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "zksync_consensus_schema" -version = "0.1.0" -edition.workspace = true -authors.workspace = true -homepage.workspace = true -license.workspace = true - -[dependencies] -zksync_concurrency.workspace = true -zksync_protobuf.workspace = true - -anyhow.workspace = true -bit-vec.workspace = true -serde.workspace = true -once_cell.workspace = true -prost.workspace = true -rand.workspace = true -serde_json.workspace = true -tokio.workspace = true - -[build-dependencies] -zksync_protobuf.workspace = true - -anyhow.workspace = true diff --git a/node/libs/schema/README.md b/node/libs/schema/README.md deleted file mode 100644 index 99bb305d..00000000 --- a/node/libs/schema/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Schema - -This crate contains all the protobuf schemas used by the node's crates. -The schemas are located in the `proto` directory. -The layout of the `proto` directory is the same as the layout of the node's -workspace: for example the schemas used by the `types` crate reside in -`schema/proto/types` directory. - -`build.rs` generates rust code from the schemas. -Again, the module layout of the generated code is the same as the layout of -the node's workspace: for example the code generated from schemas used by the `types` crate -reside in `schema::types` module. - -Each crate owning the given part of the schema is expected to export an alias to -the generated code, for example `types::validator::schema` is an alias for `schema::types::validator`. -They are also expected to implement a strongly-typed rust representations manually and conversion to -the generated code by implementing `schema::ProtoFmt` trait for them. - -If crate A depends on crate B, crate A is expected to use the strongly-typed representations defined in crate B, -and use the schema-generated types of crate B only to define conversion to schema-generated types of crate A. -For example, crate `network` has a type `network::Message` which contains a field of type `types::validator::PublicKey`. -To define a conversion from `network::Message` to `network::schema::Message`, a conversion -from `types::validator::PublicKey` to `types::validator::schema::PublicKey` is used. - -## Why not place each schema directly to the corresponding crate? - -Generating rust code from schemas scattered over many crates is hard, -because of how cargo works, and how the code-generating library has been -implemented. diff --git a/node/libs/schema/src/lib.rs b/node/libs/schema/src/lib.rs deleted file mode 100644 index e1891ab1..00000000 --- a/node/libs/schema/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Code generated from protobuf schema files. - -#[allow(warnings)] -pub mod proto { - include!(concat!(env!("OUT_DIR"), "/proto/gen.rs")); -} diff --git a/node/libs/storage/Cargo.toml b/node/libs/storage/Cargo.toml index 0d034c40..6ff85f7a 100644 --- a/node/libs/storage/Cargo.toml +++ b/node/libs/storage/Cargo.toml @@ -9,11 +9,11 @@ license.workspace = true [dependencies] zksync_concurrency.workspace = true zksync_consensus_roles.workspace = true -zksync_consensus_schema.workspace = true zksync_protobuf.workspace = true anyhow.workspace = true async-trait.workspace = true +prost.workspace = true rand.workspace = true rocksdb = { workspace = true, optional = true } thiserror.workspace = true @@ -25,6 +25,10 @@ tempfile.workspace = true test-casing.workspace = true tokio.workspace = true +[build-dependencies] +zksync_protobuf.workspace = true +zksync_consensus_roles.workspace = true + [features] default = [] # Enables RocksDB-based storage. diff --git a/node/libs/storage/build.rs b/node/libs/storage/build.rs new file mode 100644 index 00000000..b1c9aabb --- /dev/null +++ b/node/libs/storage/build.rs @@ -0,0 +1,14 @@ +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf::build::Config { + input_root: "src/proto".into(), + proto_root: "zksync/storage".into(), + dependencies: vec![( + "::zksync_consensus_roles::proto".parse().unwrap(), + &zksync_consensus_roles::proto::DESCRIPTOR, + )], + protobuf_crate: "::zksync_protobuf".parse().unwrap(), + } + .generate() + .expect("generate()"); +} diff --git a/node/libs/storage/src/lib.rs b/node/libs/storage/src/lib.rs index 148b1cd2..0afd516d 100644 --- a/node/libs/storage/src/lib.rs +++ b/node/libs/storage/src/lib.rs @@ -2,6 +2,7 @@ //! but this crate only exposes an abstraction of a database, so we can easily switch to a different storage engine in the future. mod in_memory; +pub mod proto; mod replica_state; #[cfg(feature = "rocksdb")] mod rocksdb; diff --git a/node/libs/schema/proto/storage.proto b/node/libs/storage/src/proto/mod.proto similarity index 81% rename from node/libs/schema/proto/storage.proto rename to node/libs/storage/src/proto/mod.proto index 301a5485..594ef1b0 100644 --- a/node/libs/schema/proto/storage.proto +++ b/node/libs/storage/src/proto/mod.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package zksync.schema.storage; +package zksync.storage; -import "zksync/schema/roles/validator.proto"; +import "zksync/roles/validator.proto"; message Proposal { optional uint64 number = 1; diff --git a/node/libs/storage/src/proto/mod.rs b/node/libs/storage/src/proto/mod.rs new file mode 100644 index 00000000..660bf4c5 --- /dev/null +++ b/node/libs/storage/src/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs")); diff --git a/node/libs/storage/src/types.rs b/node/libs/storage/src/types.rs index a4ee13bf..14009e70 100644 --- a/node/libs/storage/src/types.rs +++ b/node/libs/storage/src/types.rs @@ -1,9 +1,9 @@ //! Defines the schema of the database. +use crate::proto; use anyhow::Context as _; use std::{iter, ops}; use zksync_concurrency::ctx; use zksync_consensus_roles::validator::{self, BlockNumber}; -use zksync_consensus_schema::proto::storage as proto; use zksync_protobuf::{read_required, required, ProtoFmt}; /// A payload of a proposed block which is not known to be finalized yet. diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 74803009..677e3df2 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -15,7 +15,6 @@ zksync_consensus_crypto.workspace = true zksync_consensus_executor.workspace = true zksync_consensus_roles.workspace = true zksync_consensus_storage = { workspace = true, features = ["rocksdb"] } -zksync_consensus_schema.workspace = true zksync_consensus_utils.workspace = true zksync_protobuf.workspace = true diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index effcd00c..3bdec7c6 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -2,9 +2,8 @@ use anyhow::Context as _; use std::{fs, net, path::Path}; use zksync_consensus_crypto::{read_optional_text, Text, TextFmt}; -use zksync_consensus_executor::{ConsensusConfig, ExecutorConfig}; +use zksync_consensus_executor::{proto, ConsensusConfig, ExecutorConfig}; use zksync_consensus_roles::{node, validator}; -use zksync_consensus_schema::proto::executor::config as proto; use zksync_protobuf::{read_optional, read_required, ProtoFmt}; /// This struct holds the file path to each of the config files. From 745780aa82030967a236aa2d8398b7d2aa136e22 Mon Sep 17 00:00:00 2001 From: Igor Borodin Date: Thu, 9 Nov 2023 17:10:40 +0100 Subject: [PATCH 5/5] ci: Fix load testing workflow (#22) # What - Pins Ansible Python deps - Adds explicit OS Login to GCP instances to not rely on project metadata propagation - Adds some extra error handling to Ansible inventory parsing - Changes workflow inputs to be PR trigger friendly (to simplify the debug) ## Why - To fix and improve load testing GHA workflow reliability --------- Co-authored-by: Maksym Kryva Co-authored-by: Grzegorz Prusak --- .github/workflows/load_testing.yaml | 56 ++++++++++++------- .../loadtests/ansible/requirements.txt | 4 ++ infrastructure/loadtests/compute.tf | 8 +++ 3 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 infrastructure/loadtests/ansible/requirements.txt diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 412f48e7..b19b6ff8 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -20,9 +20,11 @@ on: default: 8081 env: + NUM_INSTANCES: ${{ github.event.inputs.num_instances || '100' }} + NODE_PORT: ${{ github.event.inputs.node_port || '8080' }} + METRICS_PORT: ${{ github.event.inputs.metrics_port || '8081' }} VM_AUTH_USERNAME: ${{ secrets.VM_AUTH_USERNAME }} VM_AUTH_PASSWORD: ${{ secrets.VM_AUTH_PASSWORD }} - METRICS_PORT: ${{ inputs.metrics_port }} jobs: run_load_tests: @@ -30,14 +32,14 @@ jobs: steps: - name: Fail-fast on incorrect inputs run: | - if [[ "${{ github.event.inputs.num_instances }}" -ge 1 && "${{ github.event.inputs.num_instances }}" -le 200 ]]; then + if [[ "${{ env.NUM_INSTANCES }}" -ge 1 && "${{ env.NUM_INSTANCES }}" -le 200 ]]; then echo "Number of instances is within range." else echo "Error: Number of instances is not within range 1-200." exit 1 fi - if [[ "${{ github.event.inputs.node_port }}" == "${{ github.event.inputs.metrics_port }}" ]]; then + if [[ "${{ env.NODE_PORT }}" == "${{ env.METRICS_PORT }}" ]]; then echo "Error: node_port and metrics_port should not be equal." exit 1 fi @@ -52,16 +54,20 @@ jobs: with: node-version: 16 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 + with: + python-version: '3.12' + - name: Setup Terraform uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.5.6 - - name: Install Ansible + - name: Install Ansible and deps run: | - # to-do pin versions and move to requirements.yaml + sudo apt update && sudo apt install -y openssh-client python -m pip install --upgrade pip - pip install ansible ansible-core google-auth + pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply @@ -69,21 +75,29 @@ jobs: run: | terraform init terraform apply -auto-approve \ - -parallelism=${{ inputs.num_instances }} \ - -var="num_instances=${{ inputs.num_instances }}" \ - -var="node_port=${{ inputs.node_port }}" \ - -var="metrics_port=${{ inputs.metrics_port }}" \ + -parallelism=${{ env.NUM_INSTANCES }} \ + -var="num_instances=${{ env.NUM_INSTANCES }}" \ + -var="node_port=${{ env.NODE_PORT }}" \ + -var="metrics_port=${{ env.METRICS_PORT }}" \ -var="test_id=${{ env.TEST_ID }}" - name: Generate list of host:port for config generator working-directory: node run: | + set -o pipefail + sudo apt update && sudo apt install -y gettext-base tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml) echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml - ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | jq -r '.gcp_loadtest.hosts[]' | \ - awk -v port=${{ inputs.node_port }} -F\" '{print $1 ":" port}' > ips_prts.txt + cat ../infrastructure/loadtests/ansible/gcp.yml + + ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | \ + tee /tmp/ansible_output.txt | \ + grep -q "No inventory was parsed" && { echo "Error: No inventory was parsed."; exit 1; } + + jq -r '.gcp_loadtest.hosts[]' < /tmp/ansible_output.txt | \ + awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt - name: Install node build dependencies run: sudo apt update && sudo apt install -y clang @@ -104,37 +118,37 @@ jobs: - name: Generate node configs working-directory: node run: | - cargo run -p tools \ + cargo run -p zksync_consensus_tools \ --bin localnet_config -- \ --input-addrs ips_prts.txt \ - --metrics-server-port ${{ inputs.metrics_port }} \ + --metrics-server-port ${{ env.METRICS_PORT }} \ --output-dir artifacts/node_configs - name: Build executor binary working-directory: node run: | - build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1 + build_output=$(cargo build --release -p zksync_consensus_tools --bin executor --message-format=json) || exit 1 echo "$build_output" | jq -r 'select(.executable != null) | .executable' \ | while read binary; do cp "$binary" artifacts/binaries/ done - - name: Run ansible + - name: Run Ansible working-directory: infrastructure/loadtests/ansible run: | sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)') ansible-playbook -i gcp.yml \ --user sa_${sa_name} \ --private-key .ssh/google_compute_engine playbook.yml \ - --forks ${{ inputs.num_instances }} + --forks ${{ env.NUM_INSTANCES }} - name: Terraform Destroy working-directory: infrastructure/loadtests if: always() run: | terraform destroy -auto-approve \ - -parallelism=${{ inputs.num_instances }} \ - -var="num_instances=${{ inputs.num_instances }}" \ - -var="node_port=${{ inputs.node_port }}" \ - -var="metrics_port=${{ inputs.metrics_port }}" \ + -parallelism=${{ env.NUM_INSTANCES }} \ + -var="num_instances=${{ env.NUM_INSTANCES }}" \ + -var="node_port=${{ env.NODE_PORT }}" \ + -var="metrics_port=${{ env.METRICS_PORT }}" \ -var="test_id=${{ env.TEST_ID }}" diff --git a/infrastructure/loadtests/ansible/requirements.txt b/infrastructure/loadtests/ansible/requirements.txt new file mode 100644 index 00000000..bb820249 --- /dev/null +++ b/infrastructure/loadtests/ansible/requirements.txt @@ -0,0 +1,4 @@ +ansible==8.5.0 +ansible-core==2.15.5 +google-auth==2.23.4 +requests==2.31.0 diff --git a/infrastructure/loadtests/compute.tf b/infrastructure/loadtests/compute.tf index 9279a4e4..3e671421 100644 --- a/infrastructure/loadtests/compute.tf +++ b/infrastructure/loadtests/compute.tf @@ -30,6 +30,10 @@ resource "google_compute_instance" "zksync_bft_node" { machine_type = "e2-highcpu-8" zone = local.instances_distribution[count.index].zone + metadata = { + enable-oslogin : "TRUE" + } + tags = ["allow-zksync-bft-node-port", "allow-zksync-bft-metrics-port"] labels = { @@ -62,6 +66,10 @@ resource "google_compute_instance" "vmagent" { machine_type = "e2-highcpu-4" zone = "us-central1-a" + metadata = { + enable-oslogin : "TRUE" + } + labels = { repo = "zksync-bft" purpose = "monitoring"