Refactor and augment demo rollup benches

Cargo.toml

@@ -34,7 +34,7 @@ members = [
     "module-system/sov-state",
     "module-system/sov-modules-api",
     "module-system/module-schemas",
-    "module-system/utils/sov-data-generators",
+    "module-system/sov-data-generators",
     "module-system/module-implementations/sov-accounts",
     "module-system/module-implementations/sov-bank",
     "module-system/module-implementations/sov-nft-module",

examples/demo-rollup/Cargo.toml

@@ -57,6 +57,7 @@ sov-evm = { path = "../../module-system/module-implementations/sov-evm", feature
 sov-bank = { path = "../../module-system/module-implementations/sov-bank", features = ["native"] }
 sov-nft-module = { path = "../../module-system/module-implementations/sov-nft-module", features = ["native"] }
 sov-zk-cycle-macros = { path = "../../utils/zk-cycle-macros" }
+sov-data-generators = { path = "../../module-system/sov-data-generators" }
 humantime = "2.1"
 
 borsh = { workspace = true }

examples/demo-rollup/benches/prover/README.md

@@ -1,17 +1,22 @@
 # Prover Benchmarks
-* For benchmarking the prover, we measure the number of risc0 vm cycles for each of the major functions.
-* The reason for using the cycles is the assumption that proving works off a cycles/second (KHz, MHz) based on the hardware used
+
+- For benchmarking the prover, we measure the number of risc0 vm cycles for each of the major functions.
+- The reason for using the cycles is the assumption that proving works off a cycles/second (KHz, MHz) based on the hardware used
 
 ## Running the bench
-* From sovereign-sdk
+
+- From sovereign-sdk
+
 ```
 $ cd examples/demo-rollup/benches/prover
 $ cargo bench --features bench --bench prover_bench
 ```
 
 ## Methodology
-* We have `cycle_tracker` macro defined which can be used to annotate a function in zk that we want to measure the cycles for
-* The `cycle_tracker` macro is defined at `sovereign-sdk/zk-cycle-util`
+
+- We have `cycle_tracker` macro defined which can be used to annotate a function in zk that we want to measure the cycles for
+- The `cycle_tracker` macro is defined at `sovereign-sdk/zk-cycle-util`
+
 ```rust
     #[cfg_attr(all(target_os = "zkvm", feature="bench"), cycle_tracker)]
     fn begin_slot(&mut self, witness: Self::Witness) {
@@ -21,7 +26,9 @@ $ cargo bench --features bench --bench prover_bench
         ));
     }
 ```
-* The method we use to track metrics is by registering the `io_callback` syscall when creating the risc0 host.
+
+- The method we use to track metrics is by registering the `io_callback` syscall when creating the risc0 host.
+
 ```
 pub fn get_syscall_name_handler() -> (SyscallName, fn(&[u8]) -> Vec<u8>) {
     let cycle_string = "cycle_metrics\0";
@@ -48,35 +55,41 @@ pub fn get_syscall_name_handler() -> (SyscallName, fn(&[u8]) -> Vec<u8>) {
     default_env.io_callback(metrics_syscall_name, metrics_callback);
 }
 ```
-* The above allows us to use `risc0_zkvm::guest::env::send_recv_slice` which lets the guest pass a slice of raw bytes to host and get back a vector of bytes
-* We use it to pass cycle metrics to the host
-* Cycles are tracked by the macro which gets a cycle count at the beginning and end of the function
+
+- The above allows us to use `risc0_zkvm::guest::env::send_recv_slice` which lets the guest pass a slice of raw bytes to host and get back a vector of bytes
+- We use it to pass cycle metrics to the host
+- Cycles are tracked by the macro which gets a cycle count at the beginning and end of the function
+
 ```rust
 let before = risc0_zkvm::guest::env::get_cycle_count();
 let result = (|| #block)();
 let after = risc0_zkvm::guest::env::get_cycle_count();
 ```
-* We feature gate the application of the macro `cycle_tracker` with both the target_os set to `zkvm` and the feature flag `bench`
-* The reason for using both is that we need conditional compilation to work in all cases
-* For the purpose of this profiling we run the prover without generating the proof
+
+- We feature gate the application of the macro `cycle_tracker` with both the target_os set to `zkvm` and the feature flag `bench`
+- The reason for using both is that we need conditional compilation to work in all cases
+- For the purpose of this profiling we run the prover without generating the proof
 
 ## Input set
-* Unlike demo-prover it's harder to generate fake data since all the proofs and checks need to succeed. 
-* This means the DA samples, hashes, signatures etc need to succeed
-* To make this easier we use a static input set consisting of 3 blocks
-  * we avoid using empty blocks because they skew average metrics
-  * we have 3 blocks
-    * block 1 -> 1 blob containing 1 create token transaction
-    * block 2 -> 1 blob containing 1 transfer transaction
-    * block 3 -> 1 blob containing 2 transfer transactions
-* This dataset is stored at `demo-prover/benches/blocks.hex`
-* The dataset can be substituted with another valid dataset as well from Celestia (TBD: automate parametrized generation of blocks.hex)
-* We can run this on different kinds of workloads to gauge the efficiency of different parts of the code
+
+- Unlike demo-prover it's harder to generate fake data since all the proofs and checks need to succeed.
+- This means the DA samples, hashes, signatures etc need to succeed
+- To make this easier we use a static input set consisting of 3 blocks
+  - we avoid using empty blocks because they skew average metrics
+  - we have 3 blocks
+    - block 1 -> 1 blob containing 1 create token transaction
+    - block 2 -> 1 blob containing 1 transfer transaction
+    - block 3 -> 1 blob containing 2 transfer transactions
+- This dataset is stored at `demo-prover/benches/blocks.hex`
+- The dataset can be substituted with another valid dataset as well from Celestia (TBD: automate parametrized generation of blocks.hex)
+- We can run this on different kinds of workloads to gauge the efficiency of different parts of the code
 
 ## Result
-* Standard hash function patched with risc0/rust_crypto
-* Signature verification currently NOT patched (TBD)
-* Signature verification takes about 60% of the total cycles
+
+- Standard hash function patched with risc0/rust_crypto
+- Signature verification currently NOT patched (TBD)
+- Signature verification takes about 60% of the total cycles
+
 ```
 Block stats
 
@@ -114,37 +127,46 @@ Cycle Metrics
 ```
 
 ## Custom annotations
-* We can also get finer grained information by annotating low level functions, but the process for this isn't straightforward. 
-* For code that we control, it's as simple as adding the `cycle_tracker` annotation to our function and then feature gating it (not feature gating it causes compilation errors)
-* For external dependencies, we need to fork and include a path dependency locally after annotating
-* We did this for the `jmt` jellyfish merkle tree library to measure cycle gains when we use the risc0 accelerated sha function vs without
-* We apply the risc0 patch in the following way in demo-prover/methods/guest/Cargo.toml
+
+- We can also get finer grained information by annotating low level functions, but the process for this isn't straightforward.
+- For code that we control, it's as simple as adding the `cycle_tracker` annotation to our function and then feature gating it (not feature gating it causes compilation errors)
+- For external dependencies, we need to fork and include a path dependency locally after annotating
+- We did this for the `jmt` jellyfish merkle tree library to measure cycle gains when we use the risc0 accelerated sha function vs without
+- We apply the risc0 patch in the following way in demo-prover/methods/guest/Cargo.toml
+
 ```yaml
 [patch.crates-io]
 sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6-risc0" }
 ```
-* Note that the specific tag needs to be pointed to, since master and other branches don't contain acceleration
+
+- Note that the specific tag needs to be pointed to, since master and other branches don't contain acceleration
 
 ## Accelerated vs Non-accelerated libs
-* Accelerated and risc0 optimized crypto libraries give a significant (nearly 10x) cycle gain
-* With sha2 acceleration
+
+- Accelerated and risc0 optimized crypto libraries give a significant (nearly 10x) cycle gain
+- With sha2 acceleration
+
 ```
 =====> hash: 1781
 =====> hash: 1781
 =====> hash: 1781
 =====> hash: 1781
 =====> hash: 1781
 ```
-* Without sha2 acceleration
+
+- Without sha2 acceleration
+
 ```
 =====> hash: 13901
 =====> hash: 13901
 =====> hash: 13901
 =====> hash: 13901
 =====> hash: 13901
 ```
-* Overall performance difference when using sha acceleration vs without for the same dataset (3 blocks, 4 transactions) as described above
-* With sha acceleration
+
+- Overall performance difference when using sha acceleration vs without for the same dataset (3 blocks, 4 transactions) as described above
+- With sha acceleration
+
 ```
 +-------------------------+----------------+-----------+
 | Function                | Average Cycles | Num Calls |
@@ -157,7 +179,9 @@ sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6
 +-------------------------+----------------+-----------+
 Total cycles consumed for test: 20834815
 ```
-* Without sha acceleration
+
+- Without sha acceleration
+
 ```
 +-------------------------+----------------+-----------+
 | Function                | Average Cycles | Num Calls |
@@ -170,10 +194,12 @@ Total cycles consumed for test: 20834815
 +-------------------------+----------------+-----------+
 Total cycles consumed for test: 26152702
 ```
-* There's an overall efficiency of 6 million cycles in total for 3 blocks. 
-* Keep in mind that the above table shows average number of cycles per call, so they give an efficiency per call, but the "Total cycles consumed for test" metric at the bottom shows total for 3 blocks
 
-* With ed25519 acceleration
+- There's an overall efficiency of 6 million cycles in total for 3 blocks.
+- Keep in mind that the above table shows average number of cycles per call, so they give an efficiency per call, but the "Total cycles consumed for test" metric at the bottom shows total for 3 blocks
+
+- With ed25519 acceleration
+
 ```
 +----------------------+---------------------+----------------------+----------+-----------+
 | Function             | Avg Cycles w/o Accel | Avg Cycles w/ Accel | % Change | Num Calls |
@@ -196,10 +222,55 @@ Total cycles consumed for test: 26152702
 +----------------------+----------------------+---------------------+----------+-----------+
 
 ```
-* We can see a ~4x speedup for the `verify` function when using risc0 accelerated ed25519-dalek patch
+
+- We can see a ~4x speedup for the `verify` function when using risc0 accelerated ed25519-dalek patch
+
 ```
 [patch.crates-io]
 sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6-risc0" }
 ed25519-dalek = { git = "https://github.com/risc0/curve25519-dalek", tag = "curve25519-4.1.0-risczero.1" }
 crypto-bigint = {git = "https://github.com/risc0/RustCrypto-crypto-bigint", tag = "v0.5.2-risc0"}
-```
+```
+
+## Augmented input set
+
+- In order to increase the accuracy of the benchmarks, and get estimates closer to real use-cases, we have integrated the data-generation module `sov-data-generators`, to be able to generate transaction data more easily. We have added cycle-tracking methods to have a finer understanding of the system's performances.
+
+For our benchmark, we have used two block types:
+
+- block 1 -> 1 blob containing 1 create token transaction
+- block 2 -> 1 blob containing 100 transfer transaction to random addresses, repeated 10 times
+
+Here are the results (including ed25519 acceleration):
+
+### Block Stats
+
+| Description                              | Value |
+| ---------------------------------------- | ----- |
+| Total blocks                             | 11    |
+| Blocks with transactions                 | 11    |
+| Number of blobs                          | 11    |
+| Total number of transactions             | 1001  |
+| Average number of transactions per block | 91    |
+
+### Cycle Metrics
+
+| Function             | Average Cycles | Num Calls |
+| -------------------- | -------------- | --------- |
+| Cycles per block     | 78,058,312     | 11        |
+| apply_blob           | 74,186,372     | 11        |
+| pre_process_batch    | 71,891,297     | 11        |
+| verify_txs_stateless | 71,555,628     | 11        |
+| apply_txs            | 2,258,064      | 11        |
+| end_slot             | 2,008,051      | 11        |
+| jmt_verify_update    | 1,086,936      | 11        |
+| jmt_verify_existence | 792,805        | 11        |
+| verify               | 734,681        | 1001      |
+| decode_txs           | 238,998        | 11        |
+| begin_slot           | 98,566         | 11        |
+| deserialize_batch    | 88,472         | 11        |
+| deserialize          | 23,515         | 1001      |
+| hash                 | 5,556          | 1001      |
+| commit               | 7              | 11        |
+
+**Total cycles consumed for test: 858,641,427**

examples/demo-rollup/benches/prover/datagen.rs

@@ -1,25 +1,29 @@
+use sov_data_generators::bank_data::BankMessageGenerator;
+use sov_data_generators::MessageGenerator;
+use sov_demo_rollup::MockDemoRollup;
 use sov_mock_da::{MockAddress, MockBlock, MockDaService};
-use sov_rng_da_service::{generate_create_token_payload, generate_transfers};
 use sov_rollup_interface::services::da::DaService;
 
+const DEFAULT_TRANSFER_NUM: u64 = 100;
+
 pub async fn get_bench_blocks() -> Vec<MockBlock> {
     let da_service = MockDaService::new(MockAddress::default());
-
     let mut blocks = vec![];
-    let blob = generate_create_token_payload(0);
+
+    let create_token_message_gen = BankMessageGenerator::default_generate_create_token();
+    let blob = create_token_message_gen.create_blobs::<<MockDemoRollup as sov_modules_rollup_blueprint::RollupBlueprint>::NativeRuntime>();
     da_service.send_transaction(&blob).await.unwrap();
     let block1 = da_service.get_block_at(1).await.unwrap();
     blocks.push(block1);
 
-    let blob = generate_transfers(3, 1);
-    da_service.send_transaction(&blob).await.unwrap();
-    let block2 = da_service.get_block_at(2).await.unwrap();
-    blocks.push(block2);
-
-    let blob = generate_transfers(10, 4);
-    da_service.send_transaction(&blob).await.unwrap();
-    let block2 = da_service.get_block_at(3).await.unwrap();
-    blocks.push(block2);
+    let create_transfer_message_gen =
+        BankMessageGenerator::default_generate_random_transfers(DEFAULT_TRANSFER_NUM);
+    for i in 0..10 {
+        let blob = create_transfer_message_gen.create_blobs::<<MockDemoRollup as sov_modules_rollup_blueprint::RollupBlueprint>::NativeRuntime>();
+        da_service.send_transaction(&blob).await.unwrap();
+        let blocki = da_service.get_block_at(2 + i).await.unwrap();
+        blocks.push(blocki);
+    }
 
     blocks
 }

examples/demo-rollup/stf/Cargo.toml

@@ -48,7 +48,7 @@ sov-evm = { path = "../../../module-system/module-implementations/sov-evm", opti
 demo-stf = { path = ".", features = ["native"] }
 tempfile = { workspace = true }
 rand = { workspace = true }
-sov-data-generators = { path = "../../../module-system/utils/sov-data-generators" }
+sov-data-generators = { path = "../../../module-system/sov-data-generators" }
 sov-mock-zkvm = { path = "../../../adapters/mock-zkvm" }
 sov-prover-storage-manager = { path = "../../../full-node/sov-prover-storage-manager", features = ["test-utils"] }
 

module-system/module-implementations/integration-tests/Cargo.toml

@@ -23,7 +23,7 @@ sov-state = { path = "../../sov-state", features = ["native"] }
 
 sov-mock-zkvm = { path = "../../../adapters/mock-zkvm" }
 sov-schema-db = { path = "../../../full-node/db/sov-schema-db" }
-sov-data-generators = { path = "../../utils/sov-data-generators" }
+sov-data-generators = { path = "../../sov-data-generators" }
 sov-rollup-interface = { path = "../../../rollup-interface", features = ["native"] }
 sov-mock-da = { path = "../../../adapters/mock-da", features = ["native"] }
 sov-modules-stf-blueprint = { path = "../../sov-modules-stf-blueprint", features = ["native"] }

module-system/module-implementations/sov-chain-state/Cargo.toml

@@ -23,7 +23,7 @@ sov-state = { path = "../../sov-state", version = "0.3" }
 
 [dev-dependencies]
 tempfile = { workspace = true }
-sov-data-generators = { path = "../../utils/sov-data-generators" }
+sov-data-generators = { path = "../../sov-data-generators" }
 sov-chain-state = { path = ".", features = ["native"] }
 sov-mock-da = { path = "../../../adapters/mock-da" }
 sov-prover-storage-manager = { path = "../../../full-node/sov-prover-storage-manager", features = ["test-utils"] }

module-system/module-implementations/sov-nft-module/Cargo.toml

@@ -29,7 +29,7 @@ tracing = { workspace = true, optional = true }
 [dev-dependencies]
 sov-nft-module = { version = "*", features = ["native"], path = "." }
 tempfile = { workspace = true }
-sov-data-generators = { path = "../../utils/sov-data-generators" }
+sov-data-generators = { path = "../../sov-data-generators" }
 sov-prover-storage-manager = { path = "../../../full-node/sov-prover-storage-manager", features = ["test-utils"] }
 
 

module-system/sov-data-generators/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "sov-data-generators"
+description = "A set of generator utils used to automatically produce and serialize transaction data"
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+
+version = { workspace = true }
+resolver = "2"
+publish = false
+
+[dependencies]
+sov-modules-api = { path = "../sov-modules-api", features = ["native"] }
+sov-modules-stf-blueprint = { path = "../sov-modules-stf-blueprint", features = ["native"] }
+sov-value-setter = { path = "../module-implementations/examples/sov-value-setter", features = ["native"] }
+sov-bank = { path = "../module-implementations/sov-bank", features = ["native"] }
+sov-state = { path = "../sov-state" }
+sov-mock-da = { path = "../../adapters/mock-da", features = ["native"] }
+
+borsh = { workspace = true }
+
+[dev-dependencies]
+proptest = { workspace = true }