-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
benches: initial implementation (#196)
* add readme * readme, basic examples * name changes, bin impl * example, docs * book * add `cuprate-criterion-example` * docs, tracing * fix clippy * docs * lib readme * json-rpc benchmarks * add to crates.md * add `fixme` * fix `cargo b` failing this `cfg()` existing makes a regular workspace `cargo b` fail * fix cargo.toml
- Loading branch information
1 parent
f3c1a5c
commit caa08d5
Showing
37 changed files
with
1,188 additions
and
43 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,5 @@ | ||
| # TODO | ||
| # Benches | ||
| This directory contains Cuprate's benchmarks and benchmarking utilities. | ||
|
|
||
| See the [`Benchmarking` section in the Architecture book](https://architecture.cuprate.org/benchmarking/intro.html) | ||
| to see how to create and run these benchmarks. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| [package] | ||
| name = "cuprate-benchmark" | ||
| version = "0.0.0" | ||
| edition = "2021" | ||
| description = "Cuprate's benchmarking binary" | ||
| license = "MIT" | ||
| authors = ["hinto-janai"] | ||
| repository = "https://github.com/Cuprate/cuprate/tree/main/benches/benchmark/bin" | ||
| keywords = ["cuprate", "benchmarking", "binary"] | ||
|
|
||
| [features] | ||
| # All new benchmarks should be added here! | ||
| all = ["example"] | ||
|
|
||
| # Non-benchmark features. | ||
| default = [] | ||
| json = [] | ||
| trace = [] | ||
| debug = [] | ||
| warn = [] | ||
| info = [] | ||
| error = [] | ||
|
|
||
| # Benchmark features. | ||
| # New benchmarks should be added here! | ||
| example = [ | ||
| "dep:cuprate-benchmark-example" | ||
| ] | ||
|
|
||
| [dependencies] | ||
| cuprate-benchmark-lib = { workspace = true } | ||
| cuprate-benchmark-example = { workspace = true, optional = true } | ||
|
|
||
| cfg-if = { workspace = true } | ||
| serde = { workspace = true, features = ["derive"] } | ||
| serde_json = { workspace = true, features = ["std"] } | ||
| tracing = { workspace = true, features = ["std", "attributes"] } | ||
| tracing-subscriber = { workspace = true, features = ["fmt", "std", "env-filter"] } | ||
|
|
||
| [dev-dependencies] | ||
|
|
||
| [lints] | ||
| workspace = true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| ## `cuprate-benchmark` | ||
| This crate links all benchmarks together into a single binary that can be run as: `cuprate-benchmark`. | ||
|
|
||
| `cuprate-benchmark` will run all enabled benchmarks sequentially and print data at the end. | ||
|
|
||
| ## Benchmarks | ||
| Benchmarks are opt-in and enabled via features. | ||
|
|
||
| | Feature | Enables which benchmark crate? | | ||
| |----------|--------------------------------| | ||
| | example | cuprate-benchmark-example | | ||
| | database | cuprate-benchmark-database | | ||
|
|
||
| ## Features | ||
| These are features that aren't for enabling benchmarks, but rather for other things. | ||
|
|
||
| Since `cuprate-benchmark` is built right before it is ran, | ||
| these features almost act like command line arguments. | ||
|
|
||
| | Features | Does what | | ||
| |----------|-----------| | ||
| | json | Prints JSON timings instead of a markdown table | ||
| | trace | Use the `trace` log-level | ||
| | debug | Use the `debug` log-level | ||
| | warn | Use the `warn` log-level | ||
| | info | Use the `info` log-level (default) | ||
| | error | Use the `error` log-level |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| use cfg_if::cfg_if; | ||
| use tracing::{info, instrument, Level}; | ||
| use tracing_subscriber::FmtSubscriber; | ||
|
|
||
| /// Initializes the `tracing` logger. | ||
| #[instrument] | ||
| pub(crate) fn init_logger() { | ||
| const LOG_LEVEL: Level = { | ||
| cfg_if! { | ||
| if #[cfg(feature = "trace")] { | ||
| Level::TRACE | ||
| } else if #[cfg(feature = "debug")] { | ||
| Level::DEBUG | ||
| } else if #[cfg(feature = "warn")] { | ||
| Level::WARN | ||
| } else if #[cfg(feature = "info")] { | ||
| Level::INFO | ||
| } else if #[cfg(feature = "error")] { | ||
| Level::ERROR | ||
| } else { | ||
| Level::INFO | ||
| } | ||
| } | ||
| }; | ||
|
|
||
| FmtSubscriber::builder().with_max_level(LOG_LEVEL).init(); | ||
|
|
||
| info!("Log level: {LOG_LEVEL}"); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| #![doc = include_str!("../README.md")] | ||
| #![allow( | ||
| unused_crate_dependencies, | ||
| reason = "this crate imports many potentially unused dependencies" | ||
| )] | ||
|
|
||
| mod log; | ||
| mod print; | ||
| mod run; | ||
| mod timings; | ||
|
|
||
| use cfg_if::cfg_if; | ||
|
|
||
| /// What `main()` does: | ||
| /// 1. Run all enabled benchmarks | ||
| /// 2. Record benchmark timings | ||
| /// 3. Print timing data | ||
| /// | ||
| /// To add a new benchmark to be ran here: | ||
| /// 1. Copy + paste a `cfg_if` block | ||
| /// 2. Change it to your benchmark's feature flag | ||
| /// 3. Change it to your benchmark's type | ||
| #[allow( | ||
| clippy::allow_attributes, | ||
| unused_variables, | ||
| unused_mut, | ||
| unreachable_code, | ||
| reason = "clippy does not account for all cfg()s" | ||
| )] | ||
| fn main() { | ||
| log::init_logger(); | ||
|
|
||
| let mut timings = timings::Timings::new(); | ||
|
|
||
| cfg_if! { | ||
| if #[cfg(not(any(feature = "example")))] { | ||
| println!("No feature specified. Use `--features $BENCHMARK_FEATURE` when building."); | ||
| return; | ||
| } | ||
| } | ||
|
|
||
| cfg_if! { | ||
| if #[cfg(feature = "example")] { | ||
| run::run_benchmark::<cuprate_benchmark_example::Example>(&mut timings); | ||
| } | ||
| } | ||
|
|
||
| print::print_timings(&timings); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| #![expect(dead_code, reason = "code hidden behind feature flags")] | ||
|
|
||
| use cfg_if::cfg_if; | ||
|
|
||
| use crate::timings::Timings; | ||
|
|
||
| /// Print the final the final markdown table of benchmark timings. | ||
| pub(crate) fn print_timings(timings: &Timings) { | ||
| println!("\nFinished all benchmarks, printing results:"); | ||
|
|
||
| cfg_if! { | ||
| if #[cfg(feature = "json")] { | ||
| print_timings_json(timings); | ||
| } else { | ||
| print_timings_markdown(timings); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// Default timing formatting. | ||
| pub(crate) fn print_timings_markdown(timings: &Timings) { | ||
| let mut s = String::new(); | ||
| s.push_str("| Benchmark | Time (seconds) |\n"); | ||
| s.push_str("|------------------------------------|----------------|"); | ||
|
|
||
| #[expect(clippy::iter_over_hash_type)] | ||
| for (k, v) in timings { | ||
| s += &format!("\n| {k:<34} | {v:<14} |"); | ||
| } | ||
|
|
||
| println!("\n{s}"); | ||
| } | ||
|
|
||
| /// Enabled via `json` feature. | ||
| pub(crate) fn print_timings_json(timings: &Timings) { | ||
| let json = serde_json::to_string_pretty(timings).unwrap(); | ||
| println!("\n{json}"); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| use tracing::{info, instrument, trace}; | ||
|
|
||
| use cuprate_benchmark_lib::Benchmark; | ||
|
|
||
| use crate::timings::Timings; | ||
|
|
||
| /// Run a [`Benchmark`] and record its timing. | ||
| #[instrument(skip_all)] | ||
| pub(crate) fn run_benchmark<B: Benchmark>(timings: &mut Timings) { | ||
| // Get the benchmark name. | ||
| let name = B::name(); | ||
| trace!("Running benchmark: {name}"); | ||
|
|
||
| // Setup the benchmark input. | ||
| let input = B::SETUP(); | ||
|
|
||
| // Sleep before running the benchmark. | ||
| trace!("Pre-benchmark, sleeping for: {:?}", B::POST_SLEEP_DURATION); | ||
| std::thread::sleep(B::PRE_SLEEP_DURATION); | ||
|
|
||
| // Run/time the benchmark. | ||
| let now = std::time::Instant::now(); | ||
| B::MAIN(input); | ||
| let time = now.elapsed().as_secs_f32(); | ||
|
|
||
| // Print the benchmark timings. | ||
| info!("{name:>34} ... {time}"); | ||
| assert!( | ||
| timings.insert(name, time).is_none(), | ||
| "There were 2 benchmarks with the same name - this collides the final output: {name}", | ||
| ); | ||
|
|
||
| // Sleep for a cooldown period after the benchmark run. | ||
| trace!("Post-benchmark, sleeping for: {:?}", B::POST_SLEEP_DURATION); | ||
| std::thread::sleep(B::POST_SLEEP_DURATION); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| /// Benchmark timing data. | ||
| /// | ||
| /// - Key = benchmark name | ||
| /// - Value = benchmark time in seconds | ||
| pub(crate) type Timings = std::collections::HashMap<&'static str, f32>; |
Oops, something went wrong.