improve benchmark chart rendering

spiraldb · Sep 20, 2024 · 02c9ce5 · 02c9ce5
1 parent 2d8e1af
commit 02c9ce5
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 30 deletions.
diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml
@@ -16,16 +16,16 @@ jobs:
     strategy:
       matrix:
         benchmark:
+          - id: datafusion
+            name: DataFusion
+          - id: random_access
+            name: Random Access
           - id: tpch
-            name: Vortex TPC-H
+            name: TPC-H
           - id: compress_noci
             name: Vortex Compression
           - id: bytes_at
             name: Vortex bytes_at
-          - id: datafusion
-            name: Vortex DataFusion
-          - id: random_access
-            name: Vortex random_access
     runs-on: ubuntu-latest-large
     if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
     steps:

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -14,16 +14,16 @@ jobs:
     strategy:
       matrix:
         benchmark:
+          - id: datafusion
+            name: DataFusion
+          - id: random_access
+            name: Random Access
           - id: tpch
-            name: Vortex TPC-H
+            name: TPC-H
           - id: compress_noci
             name: Vortex Compression
           - id: bytes_at
             name: Vortex bytes_at
-          - id: datafusion
-            name: Vortex DataFusion
-          - id: random_access
-            name: Vortex random_access
     runs-on: ubuntu-latest-large
     steps:
       - uses: actions/checkout@v4

diff --git a/bench-vortex/benches/bytes_at.rs b/bench-vortex/benches/bytes_at.rs
@@ -56,7 +56,7 @@ fn benchmark(c: &mut Criterion) {
     );
 
     c.bench_with_input(
-        BenchmarkId::new("bytes_at", "array_data"),
+        BenchmarkId::new("bytes_at", "array_view"),
         &array_view_fixture(),
         |b, array| {
             b.iter(|| array.bytes_at(3));

diff --git a/bench-vortex/benches/random_access.rs b/bench-vortex/benches/random_access.rs
@@ -25,17 +25,17 @@ const INDICES: [u64; 6] = [10, 11, 12, 13, 100_000, 3_000_000];
 ///
 /// environment variables and assume files to read are already present
 fn random_access_vortex(c: &mut Criterion) {
-    let mut group = c.benchmark_group("vortex");
+    let mut group = c.benchmark_group("random-access");
 
     let taxi_vortex = taxi_data_vortex();
-    group.bench_function("tokio local disk", |b| {
+    group.bench_function("vortex-tokio-local-disk", |b| {
         b.to_async(Runtime::new().unwrap())
             .iter(|| async { black_box(take_vortex_tokio(&taxi_vortex, &INDICES).await.unwrap()) })
     });
 
     let local_fs = Arc::new(LocalFileSystem::new()) as Arc<dyn ObjectStore>;
     let local_fs_path = object_store::path::Path::from_filesystem_path(&taxi_vortex).unwrap();
-    group.bench_function("localfs", |b| {
+    group.bench_function("vortex-local-fs", |b| {
         b.to_async(Runtime::new().unwrap()).iter(|| async {
             black_box(
                 take_vortex_object_store(&local_fs, &local_fs_path, &INDICES)
@@ -45,8 +45,17 @@ fn random_access_vortex(c: &mut Criterion) {
         })
     });
 
+    // everything below here is a lot slower, so we'll run fewer samples
+    group.sample_size(10);
+
+    let taxi_parquet = taxi_data_parquet();
+    group.bench_function("parquet-tokio-local-disk", |b| {
+        b.to_async(Runtime::new().unwrap())
+            .iter(|| async { black_box(take_parquet(&taxi_parquet, &INDICES).await.unwrap()) })
+    });
+
     if env::var("AWS_ACCESS_KEY_ID").is_ok() {
-        group.sample_size(10).bench_function("R2", |b| {
+        group.bench_function("vortex-r2", |b| {
             let r2_fs =
                 Arc::new(AmazonS3Builder::from_env().build().unwrap()) as Arc<dyn ObjectStore>;
             let r2_path = object_store::path::Path::from_url_path(
@@ -62,21 +71,8 @@ fn random_access_vortex(c: &mut Criterion) {
                 )
             })
         });
-    }
-}
 
-fn random_access_parquet(c: &mut Criterion) {
-    let mut group = c.benchmark_group("parquet");
-    group.sample_size(10);
-
-    let taxi_parquet = taxi_data_parquet();
-    group.bench_function("tokio local disk", |b| {
-        b.to_async(Runtime::new().unwrap())
-            .iter(|| async { black_box(take_parquet(&taxi_parquet, &INDICES).await.unwrap()) })
-    });
-
-    if env::var("AWS_ACCESS_KEY_ID").is_ok() {
-        group.bench_function("R2", |b| {
+        group.bench_function("parquet-r2", |b| {
             let r2_fs = Arc::new(AmazonS3Builder::from_env().build().unwrap());
             let r2_parquet_path = object_store::path::Path::from_url_path(
                 taxi_parquet.file_name().unwrap().to_str().unwrap(),
@@ -94,5 +90,5 @@ fn random_access_parquet(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, random_access_vortex, random_access_parquet);
+criterion_group!(benches, random_access_vortex);
 criterion_main!(benches);