From 6ee7d29e876b75506f58355218e1ea755a315d17 Mon Sep 17 00:00:00 2001 From: Nisheeth Barthwal Date: Sat, 23 Dec 2023 00:53:19 +0100 Subject: [PATCH] feat: move storage view, allow ignoring bootloader execution during tx execution (#251) * move storage view to test node Signed-off-by: Danil * Update era Signed-off-by: Danil * add modified keys and do not execute bootloader Signed-off-by: Danil * Load l1 batch from local Signed-off-by: Danil * Update era Signed-off-by: Danil * Fix nits Signed-off-by: Danil * Adapt tests Signed-off-by: Danil * Introduce bool Signed-off-by: Danil * Update era Signed-off-by: Danil * remove modified keys * fix test * fix block production --------- Signed-off-by: Danil Co-authored-by: Danil --- Cargo.lock | 225 ++++++------------ Cargo.toml | 16 +- .../bootloader/test_infra/src/main.rs | 4 +- src/deps/mod.rs | 47 +--- src/deps/storage_view.rs | 164 +++++++++++++ src/fork.rs | 2 +- src/namespaces/debug.rs | 11 +- src/node/debug.rs | 4 +- src/node/in_memory.rs | 110 ++++++--- src/node/zks.rs | 9 +- src/utils.rs | 6 +- 11 files changed, 336 insertions(+), 262 deletions(-) create mode 100644 src/deps/storage_view.rs diff --git a/Cargo.lock b/Cargo.lock index dabe5d3f..09ddd0f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1530,12 +1530,6 @@ dependencies = [ "serde", ] -[[package]] -name = "const-oid" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" - [[package]] name = "const-oid" version = "0.9.5" @@ -1788,16 +1782,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -[[package]] -name = "crypto-bigint" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" -dependencies = [ - "generic-array 0.14.7", - "subtle", -] - [[package]] name = "crypto-bigint" version = "0.4.9" @@ -1996,24 +1980,13 @@ dependencies = [ "uuid 1.6.1", ] -[[package]] -name = "der" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" -dependencies = [ - "const-oid 0.7.1", - "crypto-bigint 0.3.2", - "pem-rfc7468", -] - [[package]] name = "der" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ - "const-oid 0.9.5", + "const-oid", "zeroize", ] @@ -2023,7 +1996,8 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ - "const-oid 0.9.5", + "const-oid", + "pem-rfc7468", "zeroize", ] @@ -2092,7 +2066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.5", + "const-oid", "crypto-common", "subtle", ] @@ -3340,9 +3314,9 @@ dependencies = [ [[package]] name = "google-cloud-auth" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644f40175857d0b8d7b6cad6cd9594284da5041387fa2ddff30ab6d8faef65eb" +checksum = "af1087f1fbd2dd3f58c17c7574ddd99cd61cbbbc2c4dc81114b8687209b196cb" dependencies = [ "async-trait", "base64 0.21.5", @@ -3362,9 +3336,9 @@ dependencies = [ [[package]] name = "google-cloud-metadata" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96e4ad0802d3f416f62e7ce01ac1460898ee0efc98f8b45cd4aab7611607012f" +checksum = "cc279bfb50487d7bcd900e8688406475fc750fe474a835b2ab9ade9eb1fc90e2" dependencies = [ "reqwest", "thiserror", @@ -3373,11 +3347,12 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "0.12.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "215abab97e07d144428425509c1dad07e57ea72b84b21bcdb6a8a5f12a5c4932" +checksum = "ac04b29849ebdeb9fb008988cc1c4d1f0c9d121b4c7f1ddeb8061df124580e93" dependencies = [ "async-stream", + "async-trait", "base64 0.21.5", "bytes 1.5.0", "futures-util", @@ -3387,10 +3362,10 @@ dependencies = [ "hex", "once_cell", "percent-encoding", + "pkcs8 0.10.2", "regex", "reqwest", - "ring 0.16.20", - "rsa", + "ring 0.17.6", "serde", "serde_json", "sha2 0.10.8", @@ -4396,9 +4371,6 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin 0.5.2", -] [[package]] name = "lazycell" @@ -4561,6 +4533,12 @@ dependencies = [ "logos-codegen", ] +[[package]] +name = "lru" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2994eeba8ed550fd9b47a0b38f0242bc3344e496483c6180b69139cc2fa5d1d7" + [[package]] name = "mach" version = "0.3.2" @@ -4834,7 +4812,7 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "multivm" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "hex", @@ -4989,23 +4967,6 @@ dependencies = [ "serde", ] -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.5", - "smallvec", - "zeroize", -] - [[package]] name = "num-complex" version = "0.3.1" @@ -5338,7 +5299,7 @@ dependencies = [ [[package]] name = "pairing_ce" version = "0.28.5" -source = "git+https://github.com/matter-labs/pairing.git?rev=f55393f#f55393fd366596eac792d78525d26e9c4d6ed1ca" +source = "git+https://github.com/matter-labs/pairing.git?rev=f55393fd366596eac792d78525d26e9c4d6ed1ca#f55393fd366596eac792d78525d26e9c4d6ed1ca" dependencies = [ "byteorder", "cfg-if 1.0.0", @@ -5595,9 +5556,9 @@ dependencies = [ [[package]] name = "pem-rfc7468" -version = "0.3.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01de5d978f34aa4b2296576379fcc416034702fd94117c56ffd8a1a767cefb30" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ "base64ct", ] @@ -5756,28 +5717,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs1" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78f66c04ccc83dd4486fd46c33896f4e17b24a7a3a6400dedc48ed0ddd72320" -dependencies = [ - "der 0.5.1", - "pkcs8 0.8.0", - "zeroize", -] - -[[package]] -name = "pkcs8" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" -dependencies = [ - "der 0.5.1", - "spki 0.5.4", - "zeroize", -] - [[package]] name = "pkcs8" version = "0.9.0" @@ -5996,7 +5935,7 @@ dependencies = [ [[package]] name = "prometheus_exporter" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "metrics", @@ -6725,26 +6664,6 @@ dependencies = [ "librocksdb-sys", ] -[[package]] -name = "rsa" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf22754c49613d2b3b119f0e5d46e34a2c628a937e3024b8762de4e7d8c710b" -dependencies = [ - "byteorder", - "digest 0.10.7", - "num-bigint-dig", - "num-integer", - "num-iter", - "num-traits", - "pkcs1", - "pkcs8 0.8.0", - "rand_core 0.6.4", - "smallvec", - "subtle", - "zeroize", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -7573,16 +7492,6 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -[[package]] -name = "spki" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" -dependencies = [ - "base64ct", - "der 0.5.1", -] - [[package]] name = "spki" version = "0.6.0" @@ -8738,7 +8647,7 @@ dependencies = [ [[package]] name = "vlog" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "chrono", "sentry", @@ -9466,7 +9375,7 @@ dependencies = [ [[package]] name = "zksync_basic_types" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "serde", "serde_json", @@ -9476,7 +9385,7 @@ dependencies = [ [[package]] name = "zksync_circuit_breaker" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "async-trait", @@ -9499,7 +9408,7 @@ dependencies = [ [[package]] name = "zksync_commitment_utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "zkevm_test_harness 1.4.0", "zksync_types", @@ -9509,7 +9418,7 @@ dependencies = [ [[package]] name = "zksync_concurrency" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "once_cell", @@ -9527,7 +9436,7 @@ dependencies = [ [[package]] name = "zksync_config" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "serde", @@ -9537,9 +9446,10 @@ dependencies = [ [[package]] name = "zksync_consensus_bft" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", + "async-trait", "once_cell", "rand 0.8.5", "thiserror", @@ -9557,14 +9467,14 @@ dependencies = [ [[package]] name = "zksync_consensus_crypto" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "blst", "ed25519-dalek", "ff_ce", "hex", - "pairing_ce 0.28.5 (git+https://github.com/matter-labs/pairing.git?rev=f55393f)", + "pairing_ce 0.28.5 (git+https://github.com/matter-labs/pairing.git?rev=f55393fd366596eac792d78525d26e9c4d6ed1ca)", "rand 0.4.6", "rand 0.8.5", "sha3 0.10.8", @@ -9575,7 +9485,7 @@ dependencies = [ [[package]] name = "zksync_consensus_executor" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "prost", @@ -9597,7 +9507,7 @@ dependencies = [ [[package]] name = "zksync_consensus_network" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "async-trait", @@ -9621,7 +9531,7 @@ dependencies = [ [[package]] name = "zksync_consensus_roles" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "bit-vec", @@ -9629,6 +9539,7 @@ dependencies = [ "prost", "rand 0.8.5", "serde", + "thiserror", "tracing", "zksync_concurrency", "zksync_consensus_crypto", @@ -9640,7 +9551,7 @@ dependencies = [ [[package]] name = "zksync_consensus_storage" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "async-trait", @@ -9657,7 +9568,7 @@ dependencies = [ [[package]] name = "zksync_consensus_sync_blocks" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "thiserror", @@ -9672,7 +9583,7 @@ dependencies = [ [[package]] name = "zksync_consensus_utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "thiserror", "zksync_concurrency", @@ -9681,7 +9592,7 @@ dependencies = [ [[package]] name = "zksync_contracts" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "envy", "ethabi 18.0.0", @@ -9695,7 +9606,7 @@ dependencies = [ [[package]] name = "zksync_core" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "actix-cors", "actix-rt", @@ -9717,6 +9628,7 @@ dependencies = [ "jsonrpc-http-server", "jsonrpc-pubsub", "jsonrpc-ws-server", + "lru", "metrics", "multivm", "num 0.3.1", @@ -9738,6 +9650,7 @@ dependencies = [ "zksync_commitment_utils", "zksync_concurrency", "zksync_config", + "zksync_consensus_bft", "zksync_consensus_executor", "zksync_consensus_roles", "zksync_consensus_storage", @@ -9766,7 +9679,7 @@ dependencies = [ [[package]] name = "zksync_crypto" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "base64 0.13.1", "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -9781,7 +9694,7 @@ dependencies = [ [[package]] name = "zksync_dal" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "bigdecimal", @@ -9790,6 +9703,7 @@ dependencies = [ "itertools 0.10.5", "num 0.3.1", "once_cell", + "prost", "rand 0.8.5", "serde", "serde_json", @@ -9800,8 +9714,12 @@ dependencies = [ "tracing", "url", "vise", + "zksync_consensus_roles", + "zksync_consensus_storage", "zksync_contracts", "zksync_health_check", + "zksync_protobuf", + "zksync_protobuf_build", "zksync_system_constants", "zksync_types", "zksync_utils", @@ -9810,7 +9728,7 @@ dependencies = [ [[package]] name = "zksync_eth_client" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "async-trait", @@ -9830,7 +9748,7 @@ dependencies = [ [[package]] name = "zksync_eth_signer" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "async-trait", "hex", @@ -9849,7 +9767,7 @@ dependencies = [ [[package]] name = "zksync_health_check" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "async-trait", "futures 0.3.29", @@ -9862,7 +9780,7 @@ dependencies = [ [[package]] name = "zksync_mempool" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "tracing", "zksync_types", @@ -9871,7 +9789,7 @@ dependencies = [ [[package]] name = "zksync_merkle_tree" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "leb128", "once_cell", @@ -9888,7 +9806,7 @@ dependencies = [ [[package]] name = "zksync_mini_merkle_tree" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "once_cell", "zksync_basic_types", @@ -9898,14 +9816,16 @@ dependencies = [ [[package]] name = "zksync_object_store" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "async-trait", "bincode", + "flate2", "google-cloud-auth", "google-cloud-storage", "http", + "serde_json", "tokio", "tracing", "vise", @@ -9916,7 +9836,7 @@ dependencies = [ [[package]] name = "zksync_protobuf" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "bit-vec", @@ -9934,7 +9854,7 @@ dependencies = [ [[package]] name = "zksync_protobuf_build" version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=ed71b2e817c980a2daffef6a01885219e1dc6fa0#ed71b2e817c980a2daffef6a01885219e1dc6fa0" +source = "git+https://github.com/matter-labs/era-consensus.git?rev=49b1a98f80d0e9f74fdceadece4283e745c71599#49b1a98f80d0e9f74fdceadece4283e745c71599" dependencies = [ "anyhow", "heck 0.4.1", @@ -9950,7 +9870,7 @@ dependencies = [ [[package]] name = "zksync_prover_utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "async-trait", @@ -9970,7 +9890,7 @@ dependencies = [ [[package]] name = "zksync_queued_job_processor" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "async-trait", @@ -9983,7 +9903,7 @@ dependencies = [ [[package]] name = "zksync_state" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "itertools 0.10.5", @@ -10000,7 +9920,7 @@ dependencies = [ [[package]] name = "zksync_storage" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "num_cpus", "once_cell", @@ -10012,7 +9932,7 @@ dependencies = [ [[package]] name = "zksync_system_constants" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "bigdecimal", @@ -10030,7 +9950,7 @@ dependencies = [ [[package]] name = "zksync_types" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -10042,7 +9962,6 @@ dependencies = [ "num_enum 0.6.1", "once_cell", "parity-crypto", - "prost", "rlp", "serde", "serde_json", @@ -10065,7 +9984,7 @@ dependencies = [ [[package]] name = "zksync_utils" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "bigdecimal", @@ -10087,7 +10006,7 @@ dependencies = [ [[package]] name = "zksync_verification_key_generator_and_server" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "anyhow", "bincode", @@ -10107,7 +10026,7 @@ dependencies = [ [[package]] name = "zksync_web3_decl" version = "0.1.0" -source = "git+https://github.com/matter-labs/zksync-era.git?rev=bd268ac02bc3530c1d3247cb9496c3e13c2e52d9#bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" +source = "git+https://github.com/matter-labs/zksync-era.git?rev=3c669e7b0caf6515ad865f4cba5ea6fb36c33811#3c669e7b0caf6515ad865f4cba5ea6fb36c33811" dependencies = [ "bigdecimal", "chrono", diff --git a/Cargo.toml b/Cargo.toml index bbaa1570..48203b0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,14 +11,14 @@ categories = ["cryptography"] publish = false # We don't want to publish our binaries. [dependencies] -zksync_basic_types = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } -zksync_core = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } -multivm = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } -zksync_contracts = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } -zksync_types = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } -zksync_utils = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } -zksync_state = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } -zksync_web3_decl = { git = "https://github.com/matter-labs/zksync-era.git", rev = "bd268ac02bc3530c1d3247cb9496c3e13c2e52d9" } +zksync_basic_types = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } +zksync_core = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } +multivm = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } +zksync_contracts = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } +zksync_types = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } +zksync_utils = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } +zksync_state = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } +zksync_web3_decl = { git = "https://github.com/matter-labs/zksync-era.git", rev = "3c669e7b0caf6515ad865f4cba5ea6fb36c33811" } sha3 = "0.10.6" diff --git a/etc/system-contracts/bootloader/test_infra/src/main.rs b/etc/system-contracts/bootloader/test_infra/src/main.rs index e49e009d..c91a8198 100644 --- a/etc/system-contracts/bootloader/test_infra/src/main.rs +++ b/etc/system-contracts/bootloader/test_infra/src/main.rs @@ -91,7 +91,7 @@ fn execute_internal_bootloader_test() { hash_bytecode, get_system_smart_contracts_from_dir(env::current_dir().unwrap().join("../../")), )) - .to_rc_ptr(); + .into_rc_ptr(); let mut vm = Vm::new( l1_batch_env.clone(), @@ -124,7 +124,7 @@ fn execute_internal_bootloader_test() { hash_bytecode, get_system_smart_contracts_from_dir(env::current_dir().unwrap().join("../../")), )) - .to_rc_ptr(); + .into_rc_ptr(); // We are passing id of the test in location (0) where we normally put the operator. // This is then picked up by the testing framework. diff --git a/src/deps/mod.rs b/src/deps/mod.rs index 2ba82297..0ce904a5 100644 --- a/src/deps/mod.rs +++ b/src/deps/mod.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; pub mod system_contracts; use zksync_types::{ - get_code_key, get_known_code_key, get_system_context_init_logs, L2ChainId, StorageKey, - StorageLog, StorageLogKind, StorageValue, H256, + get_code_key, get_system_context_init_logs, L2ChainId, StorageKey, StorageLog, StorageLogKind, + StorageValue, H256, }; - -use std::fmt; +pub mod storage_view; +use zksync_state::ReadStorage; /// In-memory storage. #[derive(Debug, Default, Clone, PartialEq)] @@ -93,42 +93,3 @@ impl ReadStorage for InMemoryStorage { (&*self).get_enumeration_index(key) } } - -/// Functionality to read from the VM storage. -pub trait ReadStorage: fmt::Debug { - /// Read value of the key. - fn read_value(&mut self, key: &StorageKey) -> StorageValue; - - /// Checks whether a write to this storage at the specified `key` would be an initial write. - /// Roughly speaking, this is the case when the storage doesn't contain `key`, although - /// in case of mutable storages, the caveats apply (a write to a key that is present - /// in the storage but was not committed is still an initial write). - fn is_write_initial(&mut self, key: &StorageKey) -> bool; - - /// Load the factory dependency code by its hash. - fn load_factory_dep(&mut self, hash: H256) -> Option>; - - /// Returns whether a bytecode hash is "known" to the system. - fn is_bytecode_known(&mut self, bytecode_hash: &H256) -> bool { - let code_key = get_known_code_key(bytecode_hash); - self.read_value(&code_key) != H256::zero() - } - - /// Retrieves the enumeration index for a given `key`. - fn get_enumeration_index(&mut self, key: &StorageKey) -> Option; -} - -/// Functionality to write to the VM storage in a batch. -/// -/// So far, this trait is implemented only for [`zksync_state::StorageView`]. -pub trait WriteStorage: ReadStorage { - /// Sets the new value under a given key and returns the previous value. - fn set_value(&mut self, key: StorageKey, value: StorageValue) -> StorageValue; - - /// Returns a map with the key–value pairs updated by this batch. - fn modified_storage_keys(&self) -> &HashMap; - - /// Returns the number of read / write ops for which the value was read from the underlying - /// storage. - fn missed_storage_invocations(&self) -> usize; -} diff --git a/src/deps/storage_view.rs b/src/deps/storage_view.rs new file mode 100644 index 00000000..feb341e0 --- /dev/null +++ b/src/deps/storage_view.rs @@ -0,0 +1,164 @@ +use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc}; + +use zksync_state::{ReadStorage, WriteStorage}; +use zksync_types::{StorageKey, StorageValue, H256}; + +/// `StorageView` is a buffer for `StorageLog`s between storage and transaction execution code. +/// In order to commit transactions logs should be submitted to the underlying storage +/// after a transaction is executed. +/// +/// When executing transactions as a part of miniblock / L1 batch creation, +/// a single `StorageView` is used for the entire L1 batch. +/// One `StorageView` must not be used for multiple L1 batches; +/// otherwise, [`Self::is_write_initial()`] will return incorrect values because of the caching. +/// +/// When executing transactions in the API sandbox, a dedicated view is used for each transaction; +/// the only shared part is the read storage keys cache. +#[derive(Debug)] +pub struct StorageView { + pub storage_handle: S, + // Used for caching and to get the list/count of modified keys + pub modified_storage_keys: HashMap, + // Used purely for caching + read_storage_keys: HashMap, + // Cache for `contains_key()` checks. The cache is only valid within one L1 batch execution. + initial_writes_cache: HashMap, +} + +impl StorageView { + /// Creates a new storage view based on the underlying storage. + pub fn new(storage_handle: S) -> Self { + Self { + storage_handle, + modified_storage_keys: HashMap::new(), + read_storage_keys: HashMap::new(), + initial_writes_cache: HashMap::new(), + } + } + + #[allow(dead_code)] + pub fn clean_cache(&mut self) { + self.modified_storage_keys = Default::default(); + self.read_storage_keys = Default::default(); + self.initial_writes_cache = Default::default(); + } + + fn get_value_no_log(&mut self, key: &StorageKey) -> StorageValue { + let cached_value = self + .modified_storage_keys + .get(key) + .or_else(|| self.read_storage_keys.get(key)); + cached_value.copied().unwrap_or_else(|| { + let value = self.storage_handle.read_value(key); + self.read_storage_keys.insert(*key, value); + value + }) + } + /// Make a Rc RefCell ptr to the storage + pub fn into_rc_ptr(self) -> Rc> { + Rc::new(RefCell::new(self)) + } +} + +impl ReadStorage for StorageView { + fn read_value(&mut self, key: &StorageKey) -> StorageValue { + let value = self.get_value_no_log(key); + + tracing::trace!( + "read value {:?} {:?} ({:?}/{:?})", + key.hashed_key().0, + value.0, + key.address(), + key.key() + ); + + value + } + + /// Only keys contained in the underlying storage will return `false`. If a key was + /// inserted using [`Self::set_value()`], it will still return `true`. + fn is_write_initial(&mut self, key: &StorageKey) -> bool { + if let Some(&is_write_initial) = self.initial_writes_cache.get(key) { + is_write_initial + } else { + let is_write_initial = self.storage_handle.is_write_initial(key); + self.initial_writes_cache.insert(*key, is_write_initial); + is_write_initial + } + } + + fn load_factory_dep(&mut self, hash: H256) -> Option> { + self.storage_handle.load_factory_dep(hash) + } + + fn get_enumeration_index(&mut self, key: &StorageKey) -> Option { + self.storage_handle.get_enumeration_index(key) + } +} + +impl WriteStorage for StorageView { + fn set_value(&mut self, key: StorageKey, value: StorageValue) -> StorageValue { + let original = self.get_value_no_log(&key); + + tracing::trace!( + "write value {:?} value: {:?} original value: {:?} ({:?}/{:?})", + key.hashed_key().0, + value, + original, + key.address(), + key.key() + ); + self.modified_storage_keys.insert(key, value); + + original + } + + fn modified_storage_keys(&self) -> &HashMap { + &self.modified_storage_keys + } + + fn missed_storage_invocations(&self) -> usize { + 0 + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::deps::InMemoryStorage; + use zksync_types::{AccountTreeId, Address, H256}; + + #[test] + fn test_storage_access() { + let account: AccountTreeId = AccountTreeId::new(Address::from([0xfe; 20])); + let key = H256::from_low_u64_be(61); + let value = H256::from_low_u64_be(73); + let key = StorageKey::new(account, key); + + let mut raw_storage = InMemoryStorage::default(); + let mut storage_view = StorageView::new(&raw_storage); + + let default_value = storage_view.read_value(&key); + assert_eq!(default_value, H256::zero()); + + let prev_value = storage_view.set_value(key, value); + assert_eq!(prev_value, H256::zero()); + assert_eq!(storage_view.read_value(&key), value); + assert!(storage_view.is_write_initial(&key)); // key was inserted during the view lifetime + + raw_storage.set_value(key, value); + let mut storage_view = StorageView::new(&raw_storage); + + assert_eq!(storage_view.read_value(&key), value); + assert!(!storage_view.is_write_initial(&key)); // `key` is present in `raw_storage` + + let new_value = H256::from_low_u64_be(74); + storage_view.set_value(key, new_value); + assert_eq!(storage_view.read_value(&key), new_value); + + let new_key = StorageKey::new(account, H256::from_low_u64_be(62)); + storage_view.set_value(new_key, new_value); + assert_eq!(storage_view.read_value(&new_key), new_value); + assert!(storage_view.is_write_initial(&new_key)); + } +} diff --git a/src/fork.rs b/src/fork.rs index 0d51a60a..69aaf089 100644 --- a/src/fork.rs +++ b/src/fork.rs @@ -30,9 +30,9 @@ use zksync_web3_decl::{ }; use zksync_web3_decl::{jsonrpsee::http_client::HttpClientBuilder, namespaces::ZksNamespaceClient}; +use crate::system_contracts; use crate::{cache::CacheConfig, node::TEST_NODE_NETWORK_ID}; use crate::{deps::InMemoryStorage, http_fork_source::HttpForkSource}; -use crate::{deps::ReadStorage as RS, system_contracts}; pub fn block_on(future: F) -> F::Output where diff --git a/src/namespaces/debug.rs b/src/namespaces/debug.rs index 1939ae64..421154a5 100644 --- a/src/namespaces/debug.rs +++ b/src/namespaces/debug.rs @@ -1,11 +1,10 @@ use crate::{ fork::ForkSource, node::{InMemoryNodeInner, MAX_TX_SIZE}, - utils::{create_debug_output, to_real_block_number}, + utils::{create_debug_output, storage_view::StorageView, to_real_block_number}, }; use itertools::Itertools; use jsonrpc_core::{BoxFuture, Result}; -use multivm::vm_latest::HistoryDisabled; use multivm::interface::VmInterface; use multivm::vm_latest::{constants::ETH_CALL_GAS_LIMIT, CallTracer, Vm}; use once_cell::sync::OnceCell; @@ -14,7 +13,6 @@ use zksync_basic_types::H256; use zksync_core::api_server::web3::backend_jsonrpc::{ error::into_jsrpc_error, namespaces::debug::DebugNamespaceT, }; -use zksync_state::StorageView; use zksync_types::{ api::{BlockId, BlockNumber, DebugCall, ResultDebugCall, TracerConfig, TransactionVariant}, l2::L2Tx, @@ -172,7 +170,7 @@ impl DebugNamespaceT }; let execution_mode = multivm::interface::TxExecutionMode::EthCall; - let storage = StorageView::new(&inner.fork_storage).to_rc_ptr(); + let storage = StorageView::new(&inner.fork_storage).into_rc_ptr(); let bootloader_code = inner.system_contracts.contracts_for_l2_call(); @@ -200,10 +198,7 @@ impl DebugNamespaceT let call_tracer_result = Arc::new(OnceCell::default()); let tracer = CallTracer::new(call_tracer_result.clone()).into_tracer_pointer(); - let tx_result = vm.inspect( - tracer.into(), - multivm::interface::VmExecutionMode::OneTx, - ); + let tx_result = vm.inspect(tracer.into(), multivm::interface::VmExecutionMode::OneTx); let call_traces = if only_top { vec![] diff --git a/src/node/debug.rs b/src/node/debug.rs index bccb3830..7bf11179 100644 --- a/src/node/debug.rs +++ b/src/node/debug.rs @@ -9,7 +9,6 @@ use multivm::vm_latest::{constants::ETH_CALL_GAS_LIMIT, ToTracerPointer, Vm}; use zksync_basic_types::H256; use zksync_core::api_server::web3::backend_jsonrpc::error::into_jsrpc_error; -use zksync_state::StorageView; use zksync_types::{ api::{BlockId, BlockNumber, DebugCall, ResultDebugCall, TracerConfig, TransactionVariant}, l2::L2Tx, @@ -18,6 +17,7 @@ use zksync_types::{ }; use zksync_web3_decl::error::Web3Error; +use crate::deps::storage_view::StorageView; use crate::{ fork::ForkSource, namespaces::{DebugNamespaceT, Result, RpcResult}, @@ -162,7 +162,7 @@ impl DebugNames }; let execution_mode = multivm::interface::TxExecutionMode::EthCall; - let storage = StorageView::new(&inner.fork_storage).to_rc_ptr(); + let storage = StorageView::new(&inner.fork_storage).into_rc_ptr(); let bootloader_code = inner.system_contracts.contracts_for_l2_call(); diff --git a/src/node/in_memory.rs b/src/node/in_memory.rs index 4c93e6c5..3fa0f365 100644 --- a/src/node/in_memory.rs +++ b/src/node/in_memory.rs @@ -2,7 +2,7 @@ use crate::{ bootloader_debug::{BootloaderDebug, BootloaderDebugTracer}, console_log::ConsoleLogHandler, - deps::InMemoryStorage, + deps::{storage_view::StorageView, InMemoryStorage}, filters::EthFilters, fork::{ForkDetails, ForkSource, ForkStorage}, formatter, @@ -25,9 +25,12 @@ use std::{ sync::{Arc, RwLock}, }; -use multivm::interface::{ - ExecutionResult, L1BatchEnv, L2BlockEnv, SystemEnv, TxExecutionMode, VmExecutionMode, - VmExecutionResultAndLogs, VmInterface, +use multivm::{ + interface::{ + ExecutionResult, L1BatchEnv, L2BlockEnv, SystemEnv, TxExecutionMode, VmExecutionMode, + VmExecutionResultAndLogs, VmInterface, + }, + vm_latest::L2Block, }; use multivm::{ tracers::CallTracer, @@ -43,14 +46,15 @@ use multivm::{ }, }; use zksync_basic_types::{ - web3::signing::keccak256, Address, Bytes, L1BatchNumber, MiniblockNumber, H160, H256, U256, U64, + web3::signing::keccak256, AccountTreeId, Address, Bytes, L1BatchNumber, MiniblockNumber, H160, + H256, U256, U64, }; use zksync_contracts::BaseSystemContracts; use zksync_core::api_server::web3::backend_jsonrpc::error::into_jsrpc_error; -use zksync_state::{ReadStorage, StoragePtr, StorageView, WriteStorage}; +use zksync_state::{ReadStorage, StoragePtr, WriteStorage}; use zksync_types::{ api::{Block, DebugCall, Log, TransactionReceipt, TransactionVariant}, - block::legacy_miniblock_hash, + block::{unpack_block_info, MiniblockHasher}, fee::Fee, get_nonce_key, l2::L2Tx, @@ -59,6 +63,7 @@ use zksync_types::{ vm_trace::Call, PackedEthSignature, StorageKey, StorageLogQueryType, StorageValue, Transaction, ACCOUNT_CODE_STORAGE_ADDRESS, EIP_712_TX_TYPE, MAX_GAS_PER_PUBDATA_BYTE, MAX_L2_TX_GAS_LIMIT, + SYSTEM_CONTEXT_ADDRESS, SYSTEM_CONTEXT_BLOCK_INFO_POSITION, }; use zksync_utils::{ bytecode::{compress_bytecode, hash_bytecode}, @@ -330,23 +335,35 @@ type L2TxResult = ( ); impl InMemoryNodeInner { + /// Create [L1BatchEnv] to be used in the VM. + /// + /// We compute l1/l2 block details from storage to support fork testing, where the storage + /// can be updated mid execution and no longer matches with the initial node's state. + /// The L1 & L2 timestamps are also compared with node's timestamp to ensure it always increases monotonically. pub fn create_l1_batch_env( &self, storage: StoragePtr, ) -> (L1BatchEnv, BlockContext) { - let last_l2_block_hash = if let Some(last_l2_block) = load_last_l2_block(storage) { - last_l2_block.hash - } else { - // This is the scenario of either the first L2 block ever or - // the first block after the upgrade for support of L2 blocks. - legacy_miniblock_hash(MiniblockNumber(self.current_miniblock as u32)) - }; - let block_ctx = BlockContext::from_current( - self.current_batch, - self.current_miniblock, + let (last_l1_block_num, last_l1_block_ts) = load_last_l1_batch(storage.clone()) + .map(|(num, ts)| (num as u32, ts)) + .unwrap_or_else(|| (self.current_batch, self.current_timestamp)); + let last_l2_block = load_last_l2_block(storage.clone()).unwrap_or_else(|| L2Block { + number: self.current_miniblock as u32, + hash: MiniblockHasher::legacy_hash(MiniblockNumber(self.current_miniblock as u32)), + timestamp: self.current_timestamp, + }); + let latest_timestamp = std::cmp::max( + std::cmp::max(last_l1_block_ts, last_l2_block.timestamp), self.current_timestamp, ); - let block_ctx = block_ctx.new_batch(); + + let block_ctx = BlockContext::from_current( + last_l1_block_num, + last_l2_block.number as u64, + latest_timestamp, + ) + .new_batch(); + let batch_env = L1BatchEnv { // TODO: set the previous batch hash properly (take from fork, when forking, and from local storage, when this is not the first block). previous_batch_hash: None, @@ -361,7 +378,7 @@ impl InMemoryNodeInner { // So the next one should be one higher. number: block_ctx.miniblock as u32, timestamp: block_ctx.timestamp, - prev_block_hash: last_l2_block_hash, + prev_block_hash: last_l2_block.hash, // This is only used during zksyncEra block timestamp/number transition. // In case of starting a new network, it doesn't matter. // In theory , when forking mainnet, we should match this value @@ -496,7 +513,7 @@ impl InMemoryNodeInner { let gas_for_bytecodes_pubdata: u32 = pubdata_for_factory_deps * (gas_per_pubdata_byte as u32); - let storage = storage_view.to_rc_ptr(); + let storage = storage_view.into_rc_ptr(); let execution_mode = TxExecutionMode::EstimateFee; let (mut batch_env, _) = self.create_l1_batch_env(storage.clone()); @@ -702,7 +719,7 @@ impl InMemoryNodeInner { ); l2_tx.common_data.fee.gas_limit = gas_limit_with_overhead.into(); - let storage = StorageView::new(fork_storage).to_rc_ptr(); + let storage = StorageView::new(fork_storage).into_rc_ptr(); // The nonce needs to be updated let nonce = l2_tx.nonce(); @@ -1010,7 +1027,7 @@ impl InMemoryNode { .write() .map_err(|e| format!("Failed to acquire write lock: {}", e))?; - let storage = StorageView::new(&inner.fork_storage).to_rc_ptr(); + let storage = StorageView::new(&inner.fork_storage).into_rc_ptr(); let bootloader_code = inner.system_contracts.contracts_for_l2_call(); @@ -1261,6 +1278,8 @@ impl InMemoryNode { } /// Executes the given L2 transaction and returns all the VM logs. + /// The bootloader can be omitted via specifying the `execute_bootloader` boolean. + /// This causes the VM to produce 1 L2 block per L1 block, instead of the usual 2 blocks per L1 block. /// /// **NOTE** /// @@ -1278,7 +1297,7 @@ impl InMemoryNode { /// * [InMemoryNodeInner::tx_results] /// /// This is because external users of the library may call this function to perform an isolated - /// VM operation with an external storage and get the results back. + /// VM operation (optionally without bootloader execution) with an external storage and get the results back. /// So any data populated in [Self::run_l2_tx] will not be available for the next invocation. pub fn run_l2_tx_raw( &self, @@ -1287,13 +1306,14 @@ impl InMemoryNode { mut tracers: Vec< TracerPointer>, multivm::vm_latest::HistoryDisabled>, >, + execute_bootloader: bool, ) -> Result { let inner = self .inner .write() .map_err(|e| format!("Failed to acquire write lock: {}", e))?; - let storage = StorageView::new(inner.fork_storage.clone()).to_rc_ptr(); + let storage = StorageView::new(inner.fork_storage.clone()).into_rc_ptr(); let (batch_env, block_ctx) = inner.create_l1_batch_env(storage.clone()); @@ -1438,17 +1458,17 @@ impl InMemoryNode { ..Default::default() }; - tracing::info!(""); - - let bytecodes = vm + let bytecodes: HashMap> = vm .get_last_tx_compressed_bytecodes() .iter() .map(|b| bytecode_to_factory_dep(b.original.clone())) .collect(); - - vm.execute(VmExecutionMode::Bootloader); + if execute_bootloader { + vm.execute(VmExecutionMode::Bootloader); + } let modified_keys = storage.borrow().modified_storage_keys().clone(); + Ok(( modified_keys, tx_result, @@ -1484,7 +1504,7 @@ impl InMemoryNode { } let (keys, result, call_traces, block, bytecodes, block_ctx) = - self.run_l2_tx_raw(l2_tx.clone(), execution_mode, vec![])?; + self.run_l2_tx_raw(l2_tx.clone(), execution_mode, vec![], true)?; if let ExecutionResult::Halt { reason } = result.result { // Halt means that something went really bad with the transaction execution (in most cases invalid signature, @@ -1605,7 +1625,10 @@ impl InMemoryNode { inner.current_batch = inner.current_batch.saturating_add(1); - for block in vec![block, empty_block_at_end_of_batch] { + for (i, block) in vec![block, empty_block_at_end_of_batch] + .into_iter() + .enumerate() + { // archive current state before we produce new batch/blocks if let Err(err) = inner.archive_state() { tracing::error!( @@ -1631,7 +1654,7 @@ impl InMemoryNode { if block.number.as_u64() != inner.current_miniblock { panic!( - "expected next block to have miniblock {}, got {}", + "expected next block to have miniblock {}, got {} | {i}", inner.current_miniblock, block.number.as_u64() ); @@ -1639,7 +1662,7 @@ impl InMemoryNode { if block.timestamp.as_u64() != inner.current_timestamp { panic!( - "expected next block to have timestamp {}, got {}", + "expected next block to have timestamp {}, got {} | {i}", inner.current_timestamp, block.timestamp.as_u64() ); @@ -1658,6 +1681,7 @@ impl InMemoryNode { /// Keeps track of a block's batch number, miniblock number and timestamp. /// Useful for keeping track of the current context when creating multiple blocks. +#[derive(Debug, Clone, Default)] pub struct BlockContext { pub batch: u32, pub miniblock: u64, @@ -1812,6 +1836,7 @@ mod tests { testing::TransactionBuilder::new().build(), TxExecutionMode::VerifyExecute, vec![], + true, ) .expect("transaction must pass with external storage"); } @@ -1862,7 +1887,7 @@ mod tests { tx.common_data.transaction_type = TransactionType::LegacyTransaction; tx.set_input(vec![], H256::repeat_byte(0x2)); let (_, result, ..) = node - .run_l2_tx_raw(tx, TxExecutionMode::VerifyExecute, vec![]) + .run_l2_tx_raw(tx, TxExecutionMode::VerifyExecute, vec![], true) .expect("failed tx"); match result.result { @@ -1875,3 +1900,20 @@ mod tests { } } } + +pub fn load_last_l1_batch(storage: StoragePtr) -> Option<(u64, u64)> { + // Get block number and timestamp + let current_l1_batch_info_key = StorageKey::new( + AccountTreeId::new(SYSTEM_CONTEXT_ADDRESS), + SYSTEM_CONTEXT_BLOCK_INFO_POSITION, + ); + let mut storage_ptr = storage.borrow_mut(); + let current_l1_batch_info = storage_ptr.read_value(¤t_l1_batch_info_key); + let (batch_number, batch_timestamp) = unpack_block_info(h256_to_u256(current_l1_batch_info)); + let block_number = batch_number as u32; + if block_number == 0 { + // The block does not exist yet + return None; + } + Some((batch_number, batch_timestamp)) +} diff --git a/src/node/zks.rs b/src/node/zks.rs index 297cb596..4c3834c4 100644 --- a/src/node/zks.rs +++ b/src/node/zks.rs @@ -16,10 +16,7 @@ use zksync_types::{ ExecuteTransactionCommon, ProtocolVersionId, Transaction, L2_ETH_TOKEN_ADDRESS, }; use zksync_utils::h256_to_u256; -use zksync_web3_decl::{ - error::Web3Error, - types::{Filter, Log}, -}; +use zksync_web3_decl::error::Web3Error; use crate::{ fork::ForkSource, @@ -508,10 +505,6 @@ impl ZksNamespa fn get_protocol_version(&self, _version_id: Option) -> RpcResult> { not_implemented("zks_getProtocolVersion") } - - fn get_logs_with_virtual_blocks(&self, _filter: Filter) -> RpcResult> { - not_implemented("zks_getLogs") - } } #[cfg(test)] diff --git a/src/utils.rs b/src/utils.rs index 41177594..197ff03d 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -8,18 +8,18 @@ use multivm::interface::{ExecutionResult, VmExecutionResultAndLogs, VmInterface} use multivm::vm_latest::HistoryDisabled; use multivm::vm_latest::{utils::fee::derive_base_fee_and_gas_per_pubdata, Vm}; use zksync_basic_types::{H256, U256, U64}; -use zksync_state::StorageView; use zksync_state::WriteStorage; use zksync_types::api::{BlockNumber, DebugCall, DebugCallType}; use zksync_types::l2::L2Tx; use zksync_types::vm_trace::Call; use zksync_types::CONTRACT_DEPLOYER_ADDRESS; use zksync_utils::u256_to_h256; +use zksync_utils::{bytecode::hash_bytecode, bytes_to_be_words}; use zksync_web3_decl::error::Web3Error; +use crate::deps::storage_view::StorageView; use crate::node::create_empty_block; use crate::{fork::ForkSource, node::InMemoryNodeInner}; -use zksync_utils::{bytecode::hash_bytecode, bytes_to_be_words}; pub(crate) trait IntoBoxedFuture: Sized + Send + 'static { fn into_boxed_future(self) -> Pin + Send>> { @@ -107,7 +107,7 @@ pub fn mine_empty_blocks( for i in 0..num_blocks { // roll the vm let (keys, bytecodes, block_ctx) = { - let storage = StorageView::new(&node.fork_storage).to_rc_ptr(); + let storage = StorageView::new(&node.fork_storage).into_rc_ptr(); // system_contract.contracts_for_l2_call() will give playground contracts // we need these to use the unsafeOverrideBlock method in SystemContext.sol