From 117e547400603085dd4d18e7fa8af595021c4f23 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 4 Mar 2024 22:59:38 +0000 Subject: [PATCH 1/3] Configure compressor --- bench-vortex/src/lib.rs | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/bench-vortex/src/lib.rs b/bench-vortex/src/lib.rs index aae7427583..107ac296f7 100644 --- a/bench-vortex/src/lib.rs +++ b/bench-vortex/src/lib.rs @@ -1,18 +1,40 @@ use itertools::Itertools; +use vortex::array::bool::BoolEncoding; +use vortex::array::chunked::ChunkedEncoding; +use vortex::array::constant::ConstantEncoding; +use vortex::array::primitive::PrimitiveEncoding; +use vortex::array::sparse::SparseEncoding; +use vortex::array::struct_::StructEncoding; +use vortex::array::typed::TypedEncoding; +use vortex::array::varbin::VarBinEncoding; +use vortex::array::varbinview::VarBinViewEncoding; use vortex::array::Encoding; use vortex_alp::ALPEncoding; use vortex_dict::DictEncoding; -use vortex_fastlanes::{BitPackedEncoding, FoREncoding}; +use vortex_fastlanes::{BitPackedEncoding, DeltaEncoding, FoREncoding}; use vortex_ree::REEEncoding; use vortex_roaring::{RoaringBoolEncoding, RoaringIntEncoding}; use vortex_zigzag::ZigZagEncoding; -pub fn enumerate_arrays() { +pub fn enumerate_arrays() -> Vec<&'static dyn Encoding> { let encodings: Vec<&dyn Encoding> = vec![ + // TODO(ngates): fix https://github.com/fulcrum-so/vortex/issues/35 + // Builtins + &BoolEncoding, + &ChunkedEncoding, + &ConstantEncoding, + &PrimitiveEncoding, + &SparseEncoding, + &StructEncoding, + &TypedEncoding, + &VarBinEncoding, + &VarBinViewEncoding, + // Encodings &ALPEncoding, &DictEncoding, &BitPackedEncoding, + &DeltaEncoding, &FoREncoding, //&FFoREncoding, &REEEncoding, @@ -21,10 +43,12 @@ pub fn enumerate_arrays() { &ZigZagEncoding, ]; println!("{}", encodings.iter().map(|e| e.id()).format(", ")); + encodings } #[cfg(test)] mod test { + use std::collections::HashSet; use std::fs::create_dir_all; use std::fs::File; use std::path::Path; @@ -36,7 +60,7 @@ mod test { use vortex::array::chunked::ChunkedArray; use vortex::array::{Array, ArrayRef}; - use vortex::compress::CompressCtx; + use vortex::compress::{CompressConfig, CompressCtx}; use vortex::dtype::DType; use vortex::error::{VortexError, VortexResult}; @@ -93,7 +117,12 @@ mod test { chunked.chunks().len() ); let array = chunked.boxed(); - let compressed = CompressCtx::default().compress(array.as_ref(), None); + let cfg = CompressConfig::new( + HashSet::from_iter(enumerate_arrays().iter().map(|e| (*e).id())), + HashSet::default(), + ); + println!("Compression config {cfg:?}"); + let compressed = CompressCtx::new(&cfg).compress(array.as_ref(), None); println!("Compressed array {compressed}"); println!( "NBytes {}, Ratio {}", From daa6ce3a26623cfebaee456f95e1ca8470fec48a Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 4 Mar 2024 23:00:11 +0000 Subject: [PATCH 2/3] Configure compressor --- bench-vortex/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-vortex/src/lib.rs b/bench-vortex/src/lib.rs index 107ac296f7..55cd58eb5b 100644 --- a/bench-vortex/src/lib.rs +++ b/bench-vortex/src/lib.rs @@ -34,7 +34,7 @@ pub fn enumerate_arrays() -> Vec<&'static dyn Encoding> { &ALPEncoding, &DictEncoding, &BitPackedEncoding, - &DeltaEncoding, + // &DeltaEncoding, &FoREncoding, //&FFoREncoding, &REEEncoding, From c4593ef3da5b035573b9eb40967f3b46c0103be2 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 4 Mar 2024 23:02:35 +0000 Subject: [PATCH 3/3] Configure compressor --- bench-vortex/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-vortex/src/lib.rs b/bench-vortex/src/lib.rs index 55cd58eb5b..7570287eda 100644 --- a/bench-vortex/src/lib.rs +++ b/bench-vortex/src/lib.rs @@ -12,7 +12,7 @@ use vortex::array::varbinview::VarBinViewEncoding; use vortex::array::Encoding; use vortex_alp::ALPEncoding; use vortex_dict::DictEncoding; -use vortex_fastlanes::{BitPackedEncoding, DeltaEncoding, FoREncoding}; +use vortex_fastlanes::{BitPackedEncoding, FoREncoding}; use vortex_ree::REEEncoding; use vortex_roaring::{RoaringBoolEncoding, RoaringIntEncoding}; use vortex_zigzag::ZigZagEncoding;