Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move IDs into encoding implementations #45

Merged
merged 4 commits into from
Mar 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 16 additions & 18 deletions pyvortex/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,13 @@ use vortex::array::typed::TypedArray;
use vortex::array::varbin::VarBinArray;
use vortex::array::varbinview::VarBinViewArray;
use vortex::array::{Array, ArrayKind, ArrayRef};
use vortex_alp::{ALPArray, ALP_ENCODING};
use vortex_dict::{DictArray, DICT_ENCODING};
use vortex_fastlanes::{BitPackedArray, FoRArray, FL_BITPACKED_ENCODING, FL_FOR_ENCODING};
use vortex_ffor::{FFORArray, FFOR_ENCODING};
use vortex_ree::{REEArray, REE_ENCODING};
use vortex_roaring::{
RoaringBoolArray, RoaringIntArray, ROARING_BOOL_ENCODING, ROARING_INT_ENCODING,
};
use vortex_zigzag::{ZigZagArray, ZIGZAG_ENCODING};
use vortex_alp::{ALPArray, ALPEncoding};
use vortex_dict::{DictArray, DictEncoding};
use vortex_fastlanes::{BitPackedArray, BitPackedEncoding, FoRArray, FoREncoding};
use vortex_ffor::{FFORArray, FFoREncoding};
use vortex_ree::{REEArray, REEEncoding};
use vortex_roaring::{RoaringBoolArray, RoaringBoolEncoding, RoaringIntArray, RoaringIntEncoding};
use vortex_zigzag::{ZigZagArray, ZigZagEncoding};

use crate::dtype::PyDType;
use crate::error::PyVortexError;
Expand Down Expand Up @@ -132,42 +130,42 @@ impl PyArray {
ArrayKind::Other(other) => match *other.encoding().id() {
// PyEnc chooses to expose certain encodings as first-class objects.
// For the remainder, we should have a generic EncArray implementation that supports basic functions.
ALP_ENCODING => {
ALPEncoding::ID => {
PyALPArray::wrap(py, inner.into_any().downcast::<ALPArray>().unwrap())?
.extract(py)
}
DICT_ENCODING => {
DictEncoding::ID => {
PyDictArray::wrap(py, inner.into_any().downcast::<DictArray>().unwrap())?
.extract(py)
}
FL_FOR_ENCODING => {
FoREncoding::ID => {
PyFoRArray::wrap(py, inner.into_any().downcast::<FoRArray>().unwrap())?
.extract(py)
}
FL_BITPACKED_ENCODING => PyBitPackedArray::wrap(
BitPackedEncoding::ID => PyBitPackedArray::wrap(
py,
inner.into_any().downcast::<BitPackedArray>().unwrap(),
)?
.extract(py),
FFOR_ENCODING => {
FFoREncoding::ID => {
PyFFORArray::wrap(py, inner.into_any().downcast::<FFORArray>().unwrap())?
.extract(py)
}
REE_ENCODING => {
REEEncoding::ID => {
PyREEArray::wrap(py, inner.into_any().downcast::<REEArray>().unwrap())?
.extract(py)
}
ROARING_BOOL_ENCODING => PyRoaringBoolArray::wrap(
RoaringBoolEncoding::ID => PyRoaringBoolArray::wrap(
py,
inner.into_any().downcast::<RoaringBoolArray>().unwrap(),
)?
.extract(py),
ROARING_INT_ENCODING => PyRoaringIntArray::wrap(
RoaringIntEncoding::ID => PyRoaringIntArray::wrap(
py,
inner.into_any().downcast::<RoaringIntArray>().unwrap(),
)?
.extract(py),
ZIGZAG_ENCODING => {
ZigZagEncoding::ID => {
PyZigZagArray::wrap(py, inner.into_any().downcast::<ZigZagArray>().unwrap())?
.extract(py)
}
Expand Down
6 changes: 4 additions & 2 deletions vortex-alp/src/alp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,13 @@ impl ArrayDisplay for ALPArray {
#[derive(Debug)]
pub struct ALPEncoding;

pub const ALP_ENCODING: EncodingId = EncodingId::new("vortex.alp");
impl ALPEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.alp");
}

impl Encoding for ALPEncoding {
fn id(&self) -> &EncodingId {
&ALP_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-dict/src/dict.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,13 @@ impl ArrayDisplay for DictArray {
#[derive(Debug)]
pub struct DictEncoding;

pub const DICT_ENCODING: EncodingId = EncodingId::new("vortex.dict");
impl DictEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.dict");
}

impl Encoding for DictEncoding {
fn id(&self) -> &EncodingId {
&DICT_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-fastlanes/src/bitpacking/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,13 @@ impl StatsCompute for BitPackedArray {
#[derive(Debug)]
pub struct BitPackedEncoding;

pub const FL_BITPACKED_ENCODING: EncodingId = EncodingId::new("fastlanes.bitpacked");
impl BitPackedEncoding {
pub const ID: EncodingId = EncodingId::new("fastlanes.bitpacked");
}

impl Encoding for BitPackedEncoding {
fn id(&self) -> &EncodingId {
&FL_BITPACKED_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-fastlanes/src/for/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,13 @@ impl StatsCompute for FoRArray {
#[derive(Debug)]
pub struct FoREncoding;

pub const FL_FOR_ENCODING: EncodingId = EncodingId::new("fastlanes.for");
impl FoREncoding {
pub const ID: EncodingId = EncodingId::new("fastlanes.for");
}

impl Encoding for FoREncoding {
fn id(&self) -> &EncodingId {
&FL_FOR_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-ffor/src/ffor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,11 +242,13 @@ impl ArrayDisplay for FFORArray {
#[derive(Debug)]
pub struct FFoREncoding;

pub const FFOR_ENCODING: EncodingId = EncodingId::new("vortex.ffor");
impl FFoREncoding {
pub const ID: EncodingId = EncodingId::new("vortex.ffor");
}

impl Encoding for FFoREncoding {
fn id(&self) -> &EncodingId {
&FFOR_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex-ree/src/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

use codecz::AlignedAllocator;
use vortex::array::downcast::DowncastArrayBuiltin;
use vortex::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING};
use vortex::array::primitive::{PrimitiveArray, PrimitiveEncoding};
use vortex::array::{Array, ArrayRef};
use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
use vortex::dtype::{DType, IntWidth, Nullability};
Expand All @@ -35,7 +35,7 @@ impl EncodingCompression for REEEncoding {
.stats()
.get_or_compute_or::<usize>(array.len(), &Stat::RunCount) as f32;

if array.encoding().id() == &PRIMITIVE_ENCODING
if array.encoding().id() == &PrimitiveEncoding::ID
&& avg_run_length >= config.ree_average_run_threshold
{
return Some(&(ree_compressor as Compressor));
Expand Down
6 changes: 4 additions & 2 deletions vortex-ree/src/ree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -243,11 +243,13 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for REEArray {
#[derive(Debug)]
pub struct REEEncoding;

pub const REE_ENCODING: EncodingId = EncodingId::new("vortex.ree");
impl REEEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.ree");
}

impl Encoding for REEEncoding {
fn id(&self) -> &EncodingId {
&REE_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex-roaring/src/boolean/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

use croaring::Bitmap;

use vortex::array::bool::{BoolArray, BOOL_ENCODING};
use vortex::array::bool::{BoolArray, BoolEncoding};
use vortex::array::downcast::DowncastArrayBuiltin;
use vortex::array::{Array, ArrayRef};
use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
Expand All @@ -30,7 +30,7 @@ impl EncodingCompression for RoaringBoolEncoding {
_config: &CompressConfig,
) -> Option<&'static Compressor> {
// Only support bool enc arrays
if array.encoding().id() != &BOOL_ENCODING {
if array.encoding().id() != &BoolEncoding::ID {
return None;
}

Expand Down
6 changes: 4 additions & 2 deletions vortex-roaring/src/boolean/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,13 @@ impl ArrayDisplay for RoaringBoolArray {
#[derive(Debug)]
pub struct RoaringBoolEncoding;

pub const ROARING_BOOL_ENCODING: EncodingId = EncodingId::new("roaring.bool");
impl RoaringBoolEncoding {
pub const ID: EncodingId = EncodingId::new("roaring.bool");
}

impl Encoding for RoaringBoolEncoding {
fn id(&self) -> &EncodingId {
&ROARING_BOOL_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex-roaring/src/integer/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use log::debug;
use num_traits::NumCast;

use vortex::array::downcast::DowncastArrayBuiltin;
use vortex::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING};
use vortex::array::primitive::{PrimitiveArray, PrimitiveEncoding};
use vortex::array::{Array, ArrayRef};
use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
use vortex::dtype::DType;
Expand All @@ -35,7 +35,7 @@ impl EncodingCompression for RoaringIntEncoding {
_config: &CompressConfig,
) -> Option<&'static Compressor> {
// Only support primitive enc arrays
if array.encoding().id() != &PRIMITIVE_ENCODING {
if array.encoding().id() != &PrimitiveEncoding::ID {
debug!("Skipping roaring int, not primitive");
return None;
}
Expand Down
6 changes: 4 additions & 2 deletions vortex-roaring/src/integer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,11 +163,13 @@ impl ArrayDisplay for RoaringIntArray {
#[derive(Debug)]
pub struct RoaringIntEncoding;

pub const ROARING_INT_ENCODING: EncodingId = EncodingId::new("roaring.int");
impl RoaringIntEncoding {
pub const ID: EncodingId = EncodingId::new("roaring.int");
}

impl Encoding for RoaringIntEncoding {
fn id(&self) -> &EncodingId {
&ROARING_INT_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex-zigzag/src/zigzag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,13 @@ impl ArrayDisplay for ZigZagArray {
#[derive(Debug)]
pub struct ZigZagEncoding;

pub const ZIGZAG_ENCODING: EncodingId = EncodingId::new("vortex.zigzag");
impl ZigZagEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.zigzag");
}

impl Encoding for ZigZagEncoding {
fn id(&self) -> &EncodingId {
&ZIGZAG_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex/src/array/bool/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::array::bool::{BoolEncoding, BOOL_ENCODING};
use crate::array::bool::BoolEncoding;
use crate::array::{Array, ArrayRef};
use crate::compress::{
sampled_compression, CompressConfig, CompressCtx, Compressor, EncodingCompression,
Expand All @@ -24,7 +24,7 @@ impl EncodingCompression for BoolEncoding {
array: &dyn Array,
_config: &CompressConfig,
) -> Option<&'static Compressor> {
if array.encoding().id() == &BOOL_ENCODING {
if array.encoding().id() == &BoolEncoding::ID {
Some(&(bool_compressor as Compressor))
} else {
None
Expand Down
8 changes: 5 additions & 3 deletions vortex/src/array/bool/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,16 +175,18 @@ impl Array for BoolArray {
}

#[derive(Debug)]
struct BoolEncoding;
pub struct BoolEncoding;

pub const BOOL_ENCODING: EncodingId = EncodingId::new("vortex.bool");
impl BoolEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.bool");
}

#[distributed_slice(ENCODINGS)]
static ENCODINGS_BOOL: EncodingRef = &BoolEncoding;

impl Encoding for BoolEncoding {
fn id(&self) -> &EncodingId {
&BOOL_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
4 changes: 2 additions & 2 deletions vortex/src/array/chunked/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

use rayon::prelude::*;

use crate::array::chunked::{ChunkedArray, ChunkedEncoding, CHUNKED_ENCODING};
use crate::array::chunked::{ChunkedArray, ChunkedEncoding};
use crate::array::downcast::DowncastArrayBuiltin;
use crate::array::{Array, ArrayRef};
use crate::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
Expand All @@ -25,7 +25,7 @@ impl EncodingCompression for ChunkedEncoding {
array: &dyn Array,
_config: &CompressConfig,
) -> Option<&'static Compressor> {
if array.encoding().id() == &CHUNKED_ENCODING {
if array.encoding().id() == &Self::ID {
Some(&(chunked_compressor as Compressor))
} else {
None
Expand Down
8 changes: 5 additions & 3 deletions vortex/src/array/chunked/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,16 +225,18 @@ impl ArrayDisplay for ChunkedArray {
}

#[derive(Debug)]
struct ChunkedEncoding;
pub struct ChunkedEncoding;

pub const CHUNKED_ENCODING: EncodingId = EncodingId::new("vortex.chunked");
impl ChunkedEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.chunked");
}

#[distributed_slice(ENCODINGS)]
static ENCODINGS_CHUNKED: EncodingRef = &ChunkedEncoding;

impl Encoding for ChunkedEncoding {
fn id(&self) -> &EncodingId {
&CHUNKED_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
6 changes: 4 additions & 2 deletions vortex/src/array/constant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,14 +140,16 @@ impl ArrayDisplay for ConstantArray {
#[derive(Debug)]
pub struct ConstantEncoding;

pub const CONSTANT_ENCODING: EncodingId = EncodingId::new("vortex.constant");
impl ConstantEncoding {
pub const ID: EncodingId = EncodingId::new("vortex.constant");
}

#[distributed_slice(ENCODINGS)]
static ENCODINGS_CONSTANT: EncodingRef = &ConstantEncoding;

impl Encoding for ConstantEncoding {
fn id(&self) -> &EncodingId {
&CONSTANT_ENCODING
&Self::ID
}

fn compression(&self) -> Option<&dyn EncodingCompression> {
Expand Down
Loading