Skip to content

Commit

Permalink
Move IDs into encoding implementations (#45)
Browse files Browse the repository at this point in the history
  • Loading branch information
gatesn authored Mar 4, 2024
1 parent 0f30383 commit 9199230
Show file tree
Hide file tree
Showing 33 changed files with 139 additions and 105 deletions.
34 changes: 16 additions & 18 deletions pyvortex/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,13 @@ use vortex::array::typed::TypedArray;
use vortex::array::varbin::VarBinArray;
use vortex::array::varbinview::VarBinViewArray;
use vortex::array::{Array, ArrayKind, ArrayRef};
use vortex_alp::{ALPArray, ALP_ENCODING};
use vortex_dict::{DictArray, DICT_ENCODING};
use vortex_fastlanes::{BitPackedArray, FoRArray, FL_BITPACKED_ENCODING, FL_FOR_ENCODING};
use vortex_ffor::{FFORArray, FFOR_ENCODING};
use vortex_ree::{REEArray, REE_ENCODING};
use vortex_roaring::{
RoaringBoolArray, RoaringIntArray, ROARING_BOOL_ENCODING, ROARING_INT_ENCODING,
};
use vortex_zigzag::{ZigZagArray, ZIGZAG_ENCODING};
use vortex_alp::{ALPArray, ALPEncoding};
use vortex_dict::{DictArray, DictEncoding};
use vortex_fastlanes::{BitPackedArray, BitPackedEncoding, FoRArray, FoREncoding};
use vortex_ffor::{FFORArray, FFoREncoding};
use vortex_ree::{REEArray, REEEncoding};
use vortex_roaring::{RoaringBoolArray, RoaringBoolEncoding, RoaringIntArray, RoaringIntEncoding};
use vortex_zigzag::{ZigZagArray, ZigZagEncoding};

use crate::dtype::PyDType;
use crate::error::PyVortexError;
Expand Down Expand Up @@ -132,42 +130,42 @@ impl PyArray {
ArrayKind::Other(other) => match *other.encoding().id() {
// PyEnc chooses to expose certain encodings as first-class objects.
// For the remainder, we should have a generic EncArray implementation that supports basic functions.
ALP_ENCODING => {
ALPEncoding::ID => {
PyALPArray::wrap(py, inner.into_any().downcast::<ALPArray>().unwrap())?
.extract(py)
}
DICT_ENCODING => {
DictEncoding::ID => {
PyDictArray::wrap(py, inner.into_any().downcast::<DictArray>().unwrap())?
.extract(py)
}
FL_FOR_ENCODING => {
FoREncoding::ID => {
PyFoRArray::wrap(py, inner.into_any().downcast::<FoRArray>().unwrap())?
.extract(py)
}
FL_BITPACKED_ENCODING => PyBitPackedArray::wrap(
BitPackedEncoding::ID => PyBitPackedArray::wrap(
py,
inner.into_any().downcast::<BitPackedArray>().unwrap(),
)?
.extract(py),
FFOR_ENCODING => {
FFoREncoding::ID => {
PyFFORArray::wrap(py, inner.into_any().downcast::<FFORArray>().unwrap())?
.extract(py)
}
REE_ENCODING => {
REEEncoding::ID => {
PyREEArray::wrap(py, inner.into_any().downcast::<REEArray>().unwrap())?
.extract(py)
}
ROARING_BOOL_ENCODING => PyRoaringBoolArray::wrap(
RoaringBoolEncoding::ID => PyRoaringBoolArray::wrap(
py,
inner.into_any().downcast::<RoaringBoolArray>().unwrap(),
)?
.extract(py),
ROARING_INT_ENCODING => PyRoaringIntArray::wrap(
RoaringIntEncoding::ID => PyRoaringIntArray::wrap(
py,
inner.into_any().downcast::<RoaringIntArray>().unwrap(),
)?
.extract(py),
ZIGZAG_ENCODING => {
ZigZagEncoding::ID => {
PyZigZagArray::wrap(py, inner.into_any().downcast::<ZigZagArray>().unwrap())?
.extract(py)
}
Expand Down
6 changes: 4 additions & 2 deletions vortex-alp/src/alp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,13 @@ impl ArrayDisplay for ALPArray {
#[derive(Debug)]
pub struct ALPEncoding;

pub const ALP_ENCODING: EncodingId = EncodingId::new("vortex.alp");
impl ALPEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.alp");
}

impl Encoding for ALPEncoding {
fn id(&self) -> &EncodingId {
&ALP_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-dict/src/dict.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,13 @@ impl ArrayDisplay for DictArray {
#[derive(Debug)]
pub struct DictEncoding;

pub const DICT_ENCODING: EncodingId = EncodingId::new("vortex.dict");
impl DictEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.dict");
}

impl Encoding for DictEncoding {
fn id(&self) -> &EncodingId {
&DICT_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-fastlanes/src/bitpacking/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,13 @@ impl StatsCompute for BitPackedArray {
#[derive(Debug)]
pub struct BitPackedEncoding;

pub const FL_BITPACKED_ENCODING: EncodingId = EncodingId::new("fastlanes.bitpacked");
impl BitPackedEncoding {
pub const ID: EncodingId = EncodingId::new("fastlanes.bitpacked");
}

impl Encoding for BitPackedEncoding {
fn id(&self) -> &EncodingId {
&FL_BITPACKED_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-fastlanes/src/for/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,13 @@ impl StatsCompute for FoRArray {
#[derive(Debug)]
pub struct FoREncoding;

pub const FL_FOR_ENCODING: EncodingId = EncodingId::new("fastlanes.for");
impl FoREncoding {
pub const ID: EncodingId = EncodingId::new("fastlanes.for");
}

impl Encoding for FoREncoding {
fn id(&self) -> &EncodingId {
&FL_FOR_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-ffor/src/ffor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,11 +242,13 @@ impl ArrayDisplay for FFORArray {
#[derive(Debug)]
pub struct FFoREncoding;

pub const FFOR_ENCODING: EncodingId = EncodingId::new("vortex.ffor");
impl FFoREncoding {
pub const ID: EncodingId = EncodingId::new("vortex.ffor");
}

impl Encoding for FFoREncoding {
fn id(&self) -> &EncodingId {
&FFOR_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex-ree/src/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

use codecz::AlignedAllocator;
use vortex::array::downcast::DowncastArrayBuiltin;
use vortex::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING};
use vortex::array::primitive::{PrimitiveArray, PrimitiveEncoding};
use vortex::array::{Array, ArrayRef};
use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
use vortex::dtype::{DType, IntWidth, Nullability};
Expand All @@ -35,7 +35,7 @@ impl EncodingCompression for REEEncoding {
.stats()
.get_or_compute_or::<usize>(array.len(), &Stat::RunCount) as f32;

if array.encoding().id() == &PRIMITIVE_ENCODING
if array.encoding().id() == &PrimitiveEncoding::ID
&& avg_run_length >= config.ree_average_run_threshold
{
return Some(&(ree_compressor as Compressor));
Expand Down
6 changes: 4 additions & 2 deletions vortex-ree/src/ree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -243,11 +243,13 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for REEArray {
#[derive(Debug)]
pub struct REEEncoding;

pub const REE_ENCODING: EncodingId = EncodingId::new("vortex.ree");
impl REEEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.ree");
}

impl Encoding for REEEncoding {
fn id(&self) -> &EncodingId {
&REE_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex-roaring/src/boolean/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

use croaring::Bitmap;

use vortex::array::bool::{BoolArray, BOOL_ENCODING};
use vortex::array::bool::{BoolArray, BoolEncoding};
use vortex::array::downcast::DowncastArrayBuiltin;
use vortex::array::{Array, ArrayRef};
use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
Expand All @@ -30,7 +30,7 @@ impl EncodingCompression for RoaringBoolEncoding {
_config: &CompressConfig,
) -> Option<&'static Compressor> {
// Only support bool enc arrays
if array.encoding().id() != &BOOL_ENCODING {
if array.encoding().id() != &BoolEncoding::ID {
return None;
}

Expand Down
6 changes: 4 additions & 2 deletions vortex-roaring/src/boolean/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,13 @@ impl ArrayDisplay for RoaringBoolArray {
#[derive(Debug)]
pub struct RoaringBoolEncoding;

pub const ROARING_BOOL_ENCODING: EncodingId = EncodingId::new("roaring.bool");
impl RoaringBoolEncoding {
pub const ID: EncodingId = EncodingId::new("roaring.bool");
}

impl Encoding for RoaringBoolEncoding {
fn id(&self) -> &EncodingId {
&ROARING_BOOL_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex-roaring/src/integer/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use log::debug;
use num_traits::NumCast;

use vortex::array::downcast::DowncastArrayBuiltin;
use vortex::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING};
use vortex::array::primitive::{PrimitiveArray, PrimitiveEncoding};
use vortex::array::{Array, ArrayRef};
use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
use vortex::dtype::DType;
Expand All @@ -35,7 +35,7 @@ impl EncodingCompression for RoaringIntEncoding {
_config: &CompressConfig,
) -> Option<&'static Compressor> {
// Only support primitive enc arrays
if array.encoding().id() != &PRIMITIVE_ENCODING {
if array.encoding().id() != &PrimitiveEncoding::ID {
debug!("Skipping roaring int, not primitive");
return None;
}
Expand Down
6 changes: 4 additions & 2 deletions vortex-roaring/src/integer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,11 +163,13 @@ impl ArrayDisplay for RoaringIntArray {
#[derive(Debug)]
pub struct RoaringIntEncoding;

pub const ROARING_INT_ENCODING: EncodingId = EncodingId::new("roaring.int");
impl RoaringIntEncoding {
pub const ID: EncodingId = EncodingId::new("roaring.int");
}

impl Encoding for RoaringIntEncoding {
fn id(&self) -> &EncodingId {
&ROARING_INT_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-zigzag/src/zigzag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,13 @@ impl ArrayDisplay for ZigZagArray {
#[derive(Debug)]
pub struct ZigZagEncoding;

pub const ZIGZAG_ENCODING: EncodingId = EncodingId::new("vortex.zigzag");
impl ZigZagEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.zigzag");
}

impl Encoding for ZigZagEncoding {
fn id(&self) -> &EncodingId {
&ZIGZAG_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex/src/array/bool/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::array::bool::{BoolEncoding, BOOL_ENCODING};
use crate::array::bool::BoolEncoding;
use crate::array::{Array, ArrayRef};
use crate::compress::{
sampled_compression, CompressConfig, CompressCtx, Compressor, EncodingCompression,
Expand All @@ -24,7 +24,7 @@ impl EncodingCompression for BoolEncoding {
array: &dyn Array,
_config: &CompressConfig,
) -> Option<&'static Compressor> {
if array.encoding().id() == &BOOL_ENCODING {
if array.encoding().id() == &BoolEncoding::ID {
Some(&(bool_compressor as Compressor))
} else {
None
Expand Down
8 changes: 5 additions & 3 deletions vortex/src/array/bool/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,16 +175,18 @@ impl Array for BoolArray {
}

#[derive(Debug)]
struct BoolEncoding;
pub struct BoolEncoding;

pub const BOOL_ENCODING: EncodingId = EncodingId::new("vortex.bool");
impl BoolEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.bool");
}

#[distributed_slice(ENCODINGS)]
static ENCODINGS_BOOL: EncodingRef = &BoolEncoding;

impl Encoding for BoolEncoding {
fn id(&self) -> &EncodingId {
&BOOL_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex/src/array/chunked/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

use rayon::prelude::*;

use crate::array::chunked::{ChunkedArray, ChunkedEncoding, CHUNKED_ENCODING};
use crate::array::chunked::{ChunkedArray, ChunkedEncoding};
use crate::array::downcast::DowncastArrayBuiltin;
use crate::array::{Array, ArrayRef};
use crate::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
Expand All @@ -25,7 +25,7 @@ impl EncodingCompression for ChunkedEncoding {
array: &dyn Array,
_config: &CompressConfig,
) -> Option<&'static Compressor> {
if array.encoding().id() == &CHUNKED_ENCODING {
if array.encoding().id() == &Self::ID {
Some(&(chunked_compressor as Compressor))
} else {
None
Expand Down
8 changes: 5 additions & 3 deletions vortex/src/array/chunked/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,16 +225,18 @@ impl ArrayDisplay for ChunkedArray {
}

#[derive(Debug)]
struct ChunkedEncoding;
pub struct ChunkedEncoding;

pub const CHUNKED_ENCODING: EncodingId = EncodingId::new("vortex.chunked");
impl ChunkedEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.chunked");
}

#[distributed_slice(ENCODINGS)]
static ENCODINGS_CHUNKED: EncodingRef = &ChunkedEncoding;

impl Encoding for ChunkedEncoding {
fn id(&self) -> &EncodingId {
&CHUNKED_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex/src/array/constant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,14 +140,16 @@ impl ArrayDisplay for ConstantArray {
#[derive(Debug)]
pub struct ConstantEncoding;

pub const CONSTANT_ENCODING: EncodingId = EncodingId::new("vortex.constant");
impl ConstantEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.constant");
}

#[distributed_slice(ENCODINGS)]
static ENCODINGS_CONSTANT: EncodingRef = &ConstantEncoding;

impl Encoding for ConstantEncoding {
fn id(&self) -> &EncodingId {
&CONSTANT_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
Loading

0 comments on commit 9199230

Please sign in to comment.