From ebf6a0c355d00db28ff4d7b67d142ed70f31f88f Mon Sep 17 00:00:00 2001 From: Fran Date: Fri, 30 Aug 2024 18:39:07 -0300 Subject: [PATCH 1/4] draft: new setup for criterion --- core/tests/vm-benchmark/benches/criterion.rs | 35 ++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/core/tests/vm-benchmark/benches/criterion.rs b/core/tests/vm-benchmark/benches/criterion.rs index c1b660823b84..e07e0b9b3a19 100644 --- a/core/tests/vm-benchmark/benches/criterion.rs +++ b/core/tests/vm-benchmark/benches/criterion.rs @@ -7,8 +7,8 @@ use criterion::{ use zksync_types::Transaction; use zksync_vm_benchmark_harness::{ cut_to_allowed_bytecode_size, get_deploy_tx, get_heavy_load_test_tx, get_load_test_deploy_tx, - get_load_test_tx, get_realistic_load_test_tx, BenchmarkingVm, BenchmarkingVmFactory, Fast, Lambda, - Legacy, LoadTestParams, + get_load_test_tx, get_realistic_load_test_tx, BenchmarkingVm, BenchmarkingVmFactory, Fast, + Lambda, Legacy, LoadTestParams, }; const SAMPLE_SIZE: usize = 20; @@ -20,7 +20,10 @@ fn benches_in_folder(c: &mut Criter .sample_size(SAMPLE_SIZE) .measurement_time(Duration::from_secs(10)); - let benches = format!("{}/core/tests/vm-benchmark/deployment_benchmarks", ZKSYNC_HOME); + let benches = format!( + "{}/core/tests/vm-benchmark/deployment_benchmarks", + ZKSYNC_HOME + ); for path in std::fs::read_dir(&benches).unwrap() { let path = path.unwrap().path(); @@ -50,6 +53,32 @@ fn benches_in_folder(c: &mut Criter } } +pub fn program_from_file(bin_path: &str) -> Vec { + let program = std::fs::read(bin_path).unwrap(); + let encoded = String::from_utf8(program).unwrap(); + + if &encoded[..2] != "0x" { + panic!("Wrong hex"); + } + + let bin = hex::decode(&encoded[2..]).unwrap(); + + bin +} +// Simpler version +fn benches_in_folder(c: &mut Criterion) { + let bench = format!( + "{}/core/tests/vm-benchmark/deployment_benchmarks/send", + ZKSYNC_HOME + ); + + let mut vm = BenchmarkingVm::::default(); + let code = program_from_file(&bench); + let tx = get_deploy_tx(&code[..]); + let result = vm.run_transaction(&tx); + dbg!(&result.result.is_failed()); +} + fn bench_load_test(c: &mut Criterion) { let mut group = c.benchmark_group(VM::LABEL.as_str()); group From 9c63bd0458844e88aa4334a4f4ff3c2208985084 Mon Sep 17 00:00:00 2001 From: Julian Ventura Date: Mon, 2 Sep 2024 12:09:24 -0300 Subject: [PATCH 2/4] Fix compiling errors --- core/tests/vm-benchmark/Cargo.toml | 1 + core/tests/vm-benchmark/benches/criterion.rs | 78 ++++++++++---------- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/core/tests/vm-benchmark/Cargo.toml b/core/tests/vm-benchmark/Cargo.toml index 27218d79aafe..ea0e3e53ef64 100644 --- a/core/tests/vm-benchmark/Cargo.toml +++ b/core/tests/vm-benchmark/Cargo.toml @@ -13,6 +13,7 @@ zksync_vm_benchmark_harness.workspace = true rand.workspace = true vise.workspace = true tokio.workspace = true +hex.workspace = true [dev-dependencies] criterion.workspace = true diff --git a/core/tests/vm-benchmark/benches/criterion.rs b/core/tests/vm-benchmark/benches/criterion.rs index e07e0b9b3a19..9abda2dcbcb9 100644 --- a/core/tests/vm-benchmark/benches/criterion.rs +++ b/core/tests/vm-benchmark/benches/criterion.rs @@ -14,45 +14,45 @@ use zksync_vm_benchmark_harness::{ const SAMPLE_SIZE: usize = 20; const ZKSYNC_HOME: &str = std::env!("ZKSYNC_HOME"); -fn benches_in_folder(c: &mut Criterion) { - let mut group = c.benchmark_group(VM::LABEL.as_str()); - group - .sample_size(SAMPLE_SIZE) - .measurement_time(Duration::from_secs(10)); - - let benches = format!( - "{}/core/tests/vm-benchmark/deployment_benchmarks", - ZKSYNC_HOME - ); - - for path in std::fs::read_dir(&benches).unwrap() { - let path = path.unwrap().path(); - - let test_contract = std::fs::read(&path).expect("failed to read file"); - - let code = cut_to_allowed_bytecode_size(&test_contract).unwrap(); - let tx = get_deploy_tx(code); - let file_name = path.file_name().unwrap().to_str().unwrap(); - let full_suffix = if FULL { "/full" } else { "" }; - let bench_name = format!("{file_name}{full_suffix}"); - group.bench_function(bench_name, |bencher| { - if FULL { - // Include VM initialization / drop into the measured time - bencher.iter(|| BenchmarkingVm::::default().run_transaction(black_box(&tx))); - } else { - bencher.iter_batched( - BenchmarkingVm::::default, - |mut vm| { - let result = vm.run_transaction(black_box(&tx)); - (vm, result) - }, - BatchSize::LargeInput, // VM can consume significant amount of RAM, especially the new one - ); - } - }); - } -} - +// fn benches_in_folder(c: &mut Criterion) { +// let mut group = c.benchmark_group(VM::LABEL.as_str()); +// group +// .sample_size(SAMPLE_SIZE) +// .measurement_time(Duration::from_secs(10)); +// +// let benches = format!( +// "{}/core/tests/vm-benchmark/deployment_benchmarks", +// ZKSYNC_HOME +// ); +// +// for path in std::fs::read_dir(&benches).unwrap() { +// let path = path.unwrap().path(); +// +// let test_contract = std::fs::read(&path).expect("failed to read file"); +// +// let code = cut_to_allowed_bytecode_size(&test_contract).unwrap(); +// let tx = get_deploy_tx(code); +// let file_name = path.file_name().unwrap().to_str().unwrap(); +// let full_suffix = if FULL { "/full" } else { "" }; +// let bench_name = format!("{file_name}{full_suffix}"); +// group.bench_function(bench_name, |bencher| { +// if FULL { +// // Include VM initialization / drop into the measured time +// bencher.iter(|| BenchmarkingVm::::default().run_transaction(black_box(&tx))); +// } else { +// bencher.iter_batched( +// BenchmarkingVm::::default, +// |mut vm| { +// let result = vm.run_transaction(black_box(&tx)); +// (vm, result) +// }, +// BatchSize::LargeInput, // VM can consume significant amount of RAM, especially the new one +// ); +// } +// }); +// } +// } +// pub fn program_from_file(bin_path: &str) -> Vec { let program = std::fs::read(bin_path).unwrap(); let encoded = String::from_utf8(program).unwrap(); From bc75f6025a07de39962b42952b535d1c12fd7949 Mon Sep 17 00:00:00 2001 From: fkrause98 Date: Mon, 2 Sep 2024 14:58:52 -0300 Subject: [PATCH 3/4] bench-setup: some more fixes --- core/tests/vm-benchmark/benches/criterion.rs | 30 ++++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/core/tests/vm-benchmark/benches/criterion.rs b/core/tests/vm-benchmark/benches/criterion.rs index 9abda2dcbcb9..0cd3bd8f9945 100644 --- a/core/tests/vm-benchmark/benches/criterion.rs +++ b/core/tests/vm-benchmark/benches/criterion.rs @@ -67,16 +67,34 @@ pub fn program_from_file(bin_path: &str) -> Vec { } // Simpler version fn benches_in_folder(c: &mut Criterion) { - let bench = format!( + let mut group = c.benchmark_group(VM::LABEL.as_str()); + + group + .sample_size(SAMPLE_SIZE) + .measurement_time(Duration::from_secs(10)); + + let send_bench = format!( "{}/core/tests/vm-benchmark/deployment_benchmarks/send", ZKSYNC_HOME ); - let mut vm = BenchmarkingVm::::default(); - let code = program_from_file(&bench); - let tx = get_deploy_tx(&code[..]); - let result = vm.run_transaction(&tx); - dbg!(&result.result.is_failed()); + let fibonacci_bench = format!( + "{}/core/tests/vm-benchmark/deployment_benchmarks/fibonacci_rec", + ZKSYNC_HOME + ); + + let benches: Vec = vec![send_bench, fibonacci_bench]; + for bench in benches { + let bench_name = format!("{bench}/full"); + // Only benchmark the tx execution itself + let mut vm = BenchmarkingVm::::default(); + let code = program_from_file(&bench); + let tx = get_deploy_tx(&code[..]); + let result = vm.run_transaction(&tx); + group.bench_function(bench, |bencher| { + bencher.iter(|| vm.run_transaction(black_box(&tx))); + }); + } } fn bench_load_test(c: &mut Criterion) { From 7a95f614a843cbb7f280fb83d684f7711c0dc5e9 Mon Sep 17 00:00:00 2001 From: fkrause98 Date: Mon, 2 Sep 2024 15:49:01 -0300 Subject: [PATCH 4/4] bench-setup: show instances names --- core/tests/vm-benchmark/benches/criterion.rs | 35 ++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/core/tests/vm-benchmark/benches/criterion.rs b/core/tests/vm-benchmark/benches/criterion.rs index 0cd3bd8f9945..8ff771987b2d 100644 --- a/core/tests/vm-benchmark/benches/criterion.rs +++ b/core/tests/vm-benchmark/benches/criterion.rs @@ -72,27 +72,36 @@ fn benches_in_folder(c: &mut Criter group .sample_size(SAMPLE_SIZE) .measurement_time(Duration::from_secs(10)); - + let send_bench_tag = "send"; let send_bench = format!( - "{}/core/tests/vm-benchmark/deployment_benchmarks/send", - ZKSYNC_HOME + "{}/core/tests/vm-benchmark/deployment_benchmarks/{}", + ZKSYNC_HOME, send_bench_tag ); + let fibonacci_bench_tag = "fibonacci_rec"; let fibonacci_bench = format!( - "{}/core/tests/vm-benchmark/deployment_benchmarks/fibonacci_rec", - ZKSYNC_HOME + "{}/core/tests/vm-benchmark/deployment_benchmarks/{}", + ZKSYNC_HOME, fibonacci_bench_tag ); - let benches: Vec = vec![send_bench, fibonacci_bench]; - for bench in benches { - let bench_name = format!("{bench}/full"); + let benches: Vec<(&str, String)> = vec![ + (send_bench_tag, send_bench), + (fibonacci_bench_tag, fibonacci_bench), + ]; + for (bench_tag, bench_path) in benches { + let bench_name = format!("{bench_tag}/full"); // Only benchmark the tx execution itself - let mut vm = BenchmarkingVm::::default(); - let code = program_from_file(&bench); + let code = program_from_file(&bench_path); let tx = get_deploy_tx(&code[..]); - let result = vm.run_transaction(&tx); - group.bench_function(bench, |bencher| { - bencher.iter(|| vm.run_transaction(black_box(&tx))); + group.bench_function(bench_name, |bencher| { + bencher.iter_batched( + BenchmarkingVm::::default, + |mut vm| { + let result = vm.run_transaction(black_box(&tx)); + (vm, result) + }, + BatchSize::LargeInput, + ); }); } }