Skip to content

Commit

Permalink
chore: make benchmarks more reproducible
Browse files Browse the repository at this point in the history
  • Loading branch information
@0xdeafbeef
0xdeafbeef committed Oct 13, 2024
1 parent 8a2410b commit 8610ead
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 153 deletions.
1 change: 1 addition & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ rand = "0.8"
rand_xorshift = "0.3"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
iai-callgrind = "0.14"

[features]
default = ["base64", "serde", "models", "sync"]
Expand Down
75 changes: 21 additions & 54 deletions benches/dict.rs
View file
Original file line number Diff line number Diff line change
@@ -1,68 +1,35 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use everscale_types::cell::*;
use everscale_types::dict::*;
use everscale_types::{cell::*, dict::*};
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
use rand::distributions::{Distribution, Standard};
use rand::{Rng, SeedableRng};
use std::hint::black_box;

fn build_dict_impl<K, V>(id: BenchmarkId, num_elements: usize, c: &mut Criterion)
fn build_dict<K, V>(num_elements: usize) -> Dict<K, V>
where
Standard: Distribution<K> + Distribution<V>,
K: Store + DictKey,
V: Store,
{
let mut rng = rand_xorshift::XorShiftRng::from_seed([0u8; 16]);

let values = (0..num_elements)
.map(|_| (rng.gen::<K>(), rng.gen::<V>()))
.collect::<Vec<_>>();

c.bench_with_input(id, &values, |b, values| {
b.iter(|| {
let mut result = Dict::<K, V>::new();
for (key, value) in values {
result.set(key, value).unwrap();
}
black_box(result);
});
});
}

fn build_dict_group(c: &mut Criterion) {
macro_rules! decl_dict_benches {
($({ $n:literal, $k:ty, $v:ident }),*$(,)?) => {
$({
let id = BenchmarkId::new(
"build_dict",
format!(
"size={}; key={}; value={}",
$n, stringify!($k), stringify!($v)
)
);
build_dict_impl::<$k, $v>(id, $n, c);
});*
};
let mut result = Dict::<K, V>::new();
for _ in 0..num_elements {
let key = rng.gen::<K>();
let value = rng.gen::<V>();
result.set(key, value).unwrap();
}
result
}

decl_dict_benches![
{ 10, u8, u64 },
{ 256, u8, u64 },

{ 10, u16, u64 },
{ 100, u16, u64 },
{ 256, u16, u64 },
{ 10000, u16, u64 },

{ 10, u32, u64 },
{ 100, u32, u64 },
{ 1000, u32, u64 },
{ 100000, u32, u64 },

{ 10, u64, u64 },
{ 100, u64, u64 },
{ 1000, u64, u64 },
{ 100000, u64, u64 },
];
#[library_benchmark]
#[bench::small(10)]
#[bench::medium(100)]
#[bench::large(1000)]
#[bench::xlarge(10000)]
fn bench_build_dict_u64_u64(num_elements: usize) -> Dict<u64, u64> {
black_box(build_dict(num_elements))
}

criterion_group!(build_dict, build_dict_group);
criterion_main!(build_dict);
library_benchmark_group!(name = build_dict; benchmarks = bench_build_dict_u64_u64);

main!(library_benchmark_groups = build_dict);
93 changes: 46 additions & 47 deletions benches/dict_from_slice.rs
View file
Original file line number Diff line number Diff line change
@@ -1,64 +1,63 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use everscale_types::cell::*;
use everscale_types::dict::*;
use everscale_types::{cell::*, dict::*};
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
use rand::distributions::{Distribution, Standard};
use rand::{Rng, SeedableRng};
use std::hint::black_box;

fn build_dict_impl<K, V>(id: impl Into<String>, sizes: &[usize], c: &mut Criterion)
fn build_dict_inserts<K, V>(num_elements: usize) -> Dict<K, V>
where
Standard: Distribution<K> + Distribution<V>,
K: Store + DictKey + Ord,
K: Store + DictKey,
V: Store,
{
let mut rng = rand_xorshift::XorShiftRng::from_seed([0u8; 16]);

let mut group = c.benchmark_group(id);
let mut result = Dict::<K, V>::new();
for _ in 0..num_elements {
let key = rng.gen::<K>();
let value = rng.gen::<V>();
result.add(key, value).unwrap();
}
result
}

for size in sizes {
let mut values = (0..*size)
.map(|_| (rng.gen::<K>(), rng.gen::<V>()))
.collect::<Vec<_>>();
fn build_dict_leaves<K, V>(num_elements: usize) -> Dict<K, V>
where
Standard: Distribution<K> + Distribution<V>,
K: Store + DictKey + Ord,
V: Store,
{
let mut rng = rand_xorshift::XorShiftRng::from_seed([0u8; 16]);

group.bench_with_input(BenchmarkId::new("inserts", size), &values, |b, values| {
b.iter(|| {
let mut result = Dict::<K, V>::new();
for (key, value) in values {
result.add(key, value).unwrap();
}
black_box(result);
});
});
let mut values = (0..num_elements)
.map(|_| (rng.gen::<K>(), rng.gen::<V>()))
.collect::<Vec<_>>();
values.sort_by(|(l, _), (r, _)| l.cmp(r));

values.sort_by(|(l, _), (r, _)| l.cmp(r));
group.bench_with_input(BenchmarkId::new("leaves", size), &values, |b, values| {
b.iter(|| {
let result = Dict::<K, V>::try_from_sorted_slice(values).unwrap();
black_box(result);
});
});
}
Dict::<K, V>::try_from_sorted_slice(&values).unwrap()
}

fn build_dict_group(c: &mut Criterion) {
macro_rules! decl_dict_benches {
($(($k:ty, $v:ident): [$($n:literal),+]),*$(,)?) => {
$({
let id = format!(
"build_dict({},{})",
stringify!($k), stringify!($v)
);
build_dict_impl::<$k, $v>(id, &[$($n),+], c);
});*
};
}
#[library_benchmark]
#[bench::small(10)]
#[bench::medium(100)]
#[bench::large(1000)]
#[bench::xlarge(10000)]
fn bench_build_dict_u64_u64_inserts(num_elements: usize) -> Dict<u64, u64> {
black_box(build_dict_inserts(num_elements))
}

decl_dict_benches![
(u8, u64): [10, 256],
(u16, u64): [10, 100, 256, 1000, 10000],
(u32, u64): [10, 100, 1000, 10000, 100000],
(u64, u64): [10, 100, 1000, 5000, 10000, 20000, 25000, 50000, 75000, 100000],
];
#[library_benchmark]
#[bench::small(10)]
#[bench::medium(100)]
#[bench::large(1000)]
#[bench::xlarge(10000)]
fn bench_build_dict_u64_u64_leaves(num_elements: usize) -> Dict<u64, u64> {
black_box(build_dict_leaves(num_elements))
}

criterion_group!(build_dict, build_dict_group);
criterion_main!(build_dict);
library_benchmark_group!(
name = build_dict;
benchmarks = bench_build_dict_u64_u64_inserts, bench_build_dict_u64_u64_leaves
);

main!(library_benchmark_groups = build_dict);
42 changes: 19 additions & 23 deletions benches/slice_uniform.rs
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,24 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use everscale_types::prelude::*;
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
use std::hint::black_box;

fn test_uniform(c: &mut Criterion) {
let cells = (0..=32)
.chain([
40, 60, 64, 80, 96, 127, 128, 160, 196, 200, 255, 256, 300, 400, 500, 600, 700, 800,
900, 1000, 1023,
])
.map(|bits| {
let mut builder = CellBuilder::new();
builder.store_zeros(bits).unwrap();
builder.build().unwrap()
})
.collect::<Vec<_>>();
#[library_benchmark]
#[bench::small(2)]
#[bench::medium(4)]
#[bench::large(8)]
#[bench::xlarge(10)]
fn test(bits: u32) {
let mut builder = CellBuilder::new();
builder.store_zeros(2u16.pow(bits) - 1u16).unwrap();
let cell = builder.build().unwrap();

for cell in cells {
let slice = cell.as_slice().unwrap();
c.bench_with_input(
BenchmarkId::new("test slice uniform", slice.size_bits()),
&slice,
|b, slice| b.iter(|| black_box(slice.test_uniform())),
);
}
let slice = cell.as_slice().unwrap();
black_box(slice.test_uniform());
}

criterion_group!(benches, test_uniform);
criterion_main!(benches);
library_benchmark_group!(
name = test_uniform;
benchmarks = test
);

main!(library_benchmark_groups = test_uniform);
58 changes: 29 additions & 29 deletions benches/usage_cell.rs
View file
Original file line number Diff line number Diff line change
@@ -1,52 +1,53 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use everscale_types::cell::RefsIter;
use everscale_types::prelude::*;
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
use std::collections::HashSet;
use std::hint::black_box;

const BOC: &str = "te6ccgECCAEAAWQAAnPP9noJKCEBL3oZerOiIcNghuL96V3wIcuYOWQdvNC+2fqCEIJDQAAAAAAAAAAAAAAAAZa8xB6QABNAAgEAUO3QlUyMI4dEepUMw3Ou6oSqq8+1lyHkjOGFK6DAn6TXAAAAAAAAAAABFP8A9KQT9LzyyAsDAgEgBwQC5vJx1wEBwADyeoMI1xjtRNCDB9cB1ws/yPgozxYjzxbJ+QADcdcBAcMAmoMH1wFRE7ry4GTegEDXAYAg1wGAINcBVBZ1+RDyqPgju/J5Zr74I4EHCKCBA+ioUiC8sfJ0AiCCEEzuZGy64w8ByMv/yz/J7VQGBQA+ghAWnj4Ruo4R+AACkyDXSpd41wHUAvsA6NGTMvI84gCYMALXTND6QIMG1wFx1wF41wHXTPgAcIAQBKoCFLHIywVQBc8WUAP6AstpItAhzzEh10mghAm5mDNwAcsAWM8WlzBxAcsAEsziyQH7AAAE0jA=";

fn traverse_cell_ordinary(c: &mut Criterion) {
#[library_benchmark]
fn traverse_cell_ordinary() {
let cell = Boc::decode_base64(BOC).unwrap();

c.bench_function("traverse cell ordinary", |b| {
b.iter(|| {
let mut visitor = Visitor::default();
black_box(visitor.add_cell(cell.as_ref()));
})
});
let mut visitor = Visitor::new();
black_box(visitor.add_cell(cell.as_ref()));
}

fn traverse_cell_storage_cell(c: &mut Criterion) {
#[library_benchmark]
fn traverse_cell_storage_cell() {
let cell = Boc::decode_base64(BOC).unwrap();
let usage_tree = UsageTree::new(UsageTreeMode::OnDataAccess);
let cell = usage_tree.track(&cell);

c.bench_function("traverse cell usage tree", |b| {
b.iter(|| {
let mut visitor = Visitor::default();
black_box(visitor.add_cell(cell.as_ref()));
})
});
let mut visitor = Visitor::new();
black_box(visitor.add_cell(cell.as_ref()));
}

fn traverse_cell_storage_cell_with_capacity(c: &mut Criterion) {
#[library_benchmark]
fn traverse_cell_storage_cell_with_capacity() {
let cell = Boc::decode_base64(BOC).unwrap();
let usage_tree = UsageTree::with_mode_and_capacity(UsageTreeMode::OnDataAccess, 100);
let cell = usage_tree.track(&cell);

c.bench_function("traverse cell usage tree with capacity", |b| {
b.iter(|| {
let mut visitor = Visitor::default();
black_box(visitor.add_cell(cell.as_ref()));
})
});
let mut visitor = Visitor::new();
black_box(visitor.add_cell(cell.as_ref()));
}

#[derive(Default)]
struct Visitor<'a> {
visited: ahash::HashSet<&'a HashBytes>,
stack: Vec<RefsIter<'a>>,
}

impl<'a> Visitor<'a> {
fn new() -> Self {
Self {
visited: HashSet::with_hasher(ahash::RandomState::with_seed(0)),
stack: Vec::new(),
}
}
}

impl<'a> Visitor<'a> {
fn add_cell(&mut self, cell: &'a DynCell) -> bool {
if !self.visited.insert(cell.repr_hash()) {
Expand Down Expand Up @@ -85,10 +86,9 @@ impl<'a> Visitor<'a> {
}
}

criterion_group!(
benches,
traverse_cell_ordinary,
traverse_cell_storage_cell,
traverse_cell_storage_cell_with_capacity
library_benchmark_group!(
name = traverse_cell;
benchmarks = traverse_cell_ordinary, traverse_cell_storage_cell, traverse_cell_storage_cell_with_capacity
);
criterion_main!(benches);

main!(library_benchmark_groups = traverse_cell);

0 comments on commit 8610ead

Please sign in to comment.