diff --git a/Cargo.lock b/Cargo.lock index 8fbd1c7554..ca39fca778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4405,6 +4405,7 @@ dependencies = [ name = "vortex-datetime-parts" version = "0.11.0" dependencies = [ + "itertools 0.13.0", "log", "serde", "vortex-array", diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 4717431557..1950cdcc74 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -9,9 +9,7 @@ use vortex::stats::ArrayStatisticsCompute; use vortex::validity::{ArrayValidity, LogicalValidity, Validity}; use vortex::variants::{ArrayVariants, PrimitiveArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoCanonical, -}; +use vortex::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoCanonical}; use vortex_dtype::{DType, PType}; use vortex_error::{vortex_bail, vortex_panic, VortexExpect as _, VortexResult}; @@ -23,8 +21,6 @@ impl_encoding!("vortex.alp", ids::ALP, ALP); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ALPMetadata { exponents: Exponents, - encoded_dtype: DType, - patches_dtype: Option, } impl ALPArray { @@ -33,7 +29,6 @@ impl ALPArray { exponents: Exponents, patches: Option, ) -> VortexResult { - let encoded_dtype = encoded.dtype().clone(); let dtype = match encoded.dtype() { DType::Primitive(PType::I32, nullability) => DType::Primitive(PType::F32, *nullability), DType::Primitive(PType::I64, nullability) => DType::Primitive(PType::F64, *nullability), @@ -53,21 +48,23 @@ impl ALPArray { } } - let patches_dtype = patches.as_ref().map(|a| a.dtype().as_nullable()); let mut children = Vec::with_capacity(2); children.push(encoded); if let Some(patch) = patches { + if !patch.dtype().eq_ignore_nullability(&dtype) || !patch.dtype().is_nullable() { + vortex_bail!( + "ALP patches dtype, {}, must be nullable version of array dtype, {}", + patch.dtype(), + dtype, + ); + } children.push(patch); } Self::try_from_parts( dtype, length, - ALPMetadata { - exponents, - encoded_dtype, - patches_dtype, - }, + ALPMetadata { exponents }, children.into(), Default::default(), ) @@ -83,7 +80,7 @@ impl ALPArray { pub fn encoded(&self) -> Array { self.as_ref() - .child(0, &self.metadata().encoded_dtype, self.len()) + .child(0, &self.encoded_dtype(), self.len()) .vortex_expect("Missing encoded child in ALPArray") } @@ -93,15 +90,10 @@ impl ALPArray { } pub fn patches(&self) -> Option { - self.metadata().patches_dtype.as_ref().map(|dt| { - self.as_ref().child(1, dt, self.len()).unwrap_or_else(|e| { - vortex_panic!( - e, - "ALPArray: patches child missing: dtype: {}, len: {}", - dt, - self.len() - ) - }) + (self.as_ref().nchildren() > 1).then(|| { + self.as_ref() + .child(1, &self.patches_dtype(), self.len()) + .vortex_expect("Missing patches child in ALPArray") }) } @@ -111,6 +103,24 @@ impl ALPArray { .try_into() .vortex_expect("Failed to convert DType to PType") } + + #[inline] + fn encoded_dtype(&self) -> DType { + match self.dtype() { + DType::Primitive(PType::F32, _) => { + DType::Primitive(PType::I32, self.dtype().nullability()) + } + DType::Primitive(PType::F64, _) => { + DType::Primitive(PType::I64, self.dtype().nullability()) + } + d => vortex_panic!(MismatchedTypes: "f32 or f64", d), + } + } + + #[inline] + fn patches_dtype(&self) -> DType { + self.dtype().as_nullable() + } } impl ArrayTrait for ALPArray {} diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index d942a104ee..ce1ef06e7c 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -4,8 +4,8 @@ use vortex::encoding::ids; use vortex::stats::{ArrayStatisticsCompute, StatsSet}; use vortex::validity::{ArrayValidity, LogicalValidity}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoCanonical}; -use vortex_dtype::{DType, PType}; +use vortex::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoCanonical}; +use vortex_dtype::{DType, Nullability, PType}; use vortex_error::{vortex_bail, VortexExpect, VortexResult}; use crate::alp_rd::alp_rd_decode; @@ -17,7 +17,7 @@ pub struct ALPRDMetadata { right_bit_width: u8, dict_len: u8, dict: [u16; 8], - left_parts_dtype: DType, + left_parts_ptype: PType, has_exceptions: bool, } @@ -34,20 +34,35 @@ impl ALPRDArray { vortex_bail!("ALPRDArray given invalid DType ({dtype})"); } - if left_parts.len() != right_parts.len() { - vortex_bail!("left_parts and right_parts must be of same length"); - } - let len = left_parts.len(); + if right_parts.len() != len { + vortex_bail!( + "left_parts (len {}) and right_parts (len {}) must be of same length", + len, + right_parts.len() + ); + } if !left_parts.dtype().is_unsigned_int() { vortex_bail!("left_parts dtype must be uint"); } + // we delegate array validity to the left_parts child + if dtype.is_nullable() != left_parts.dtype().is_nullable() { + vortex_bail!( + "ALPRDArray dtype nullability ({}) must match left_parts dtype nullability ({})", + dtype, + left_parts.dtype() + ); + } + let left_parts_ptype = + PType::try_from(left_parts.dtype()).vortex_expect("left_parts dtype must be uint"); - let left_parts_dtype = left_parts.dtype().clone(); - - if !right_parts.dtype().is_unsigned_int() { - vortex_bail!("right_parts dtype must be uint"); + // we enforce right_parts to be non-nullable uint + if right_parts.dtype().is_nullable() { + vortex_bail!("right_parts dtype must be non-nullable"); + } + if !right_parts.dtype().is_unsigned_int() || right_parts.dtype().is_nullable() { + vortex_bail!(MismatchedTypes: "non-nullable uint", right_parts.dtype()); } let mut children = vec![left_parts, right_parts]; @@ -73,7 +88,7 @@ impl ALPRDArray { right_bit_width, dict_len: left_parts_dict.as_ref().len() as u8, dict, - left_parts_dtype, + left_parts_ptype, has_exceptions, }, children.into(), @@ -90,30 +105,45 @@ impl ALPRDArray { == PType::F32 } + /// The dtype of the left parts of the array. + #[inline] + fn left_parts_dtype(&self) -> DType { + DType::Primitive(self.metadata().left_parts_ptype, self.dtype().nullability()) + } + + /// The dtype of the right parts of the array. + #[inline] + fn right_parts_dtype(&self) -> DType { + DType::Primitive( + if self.is_f32() { + PType::U32 + } else { + PType::U64 + }, + Nullability::NonNullable, + ) + } + + /// The dtype of the exceptions of the left parts of the array. + #[inline] + fn left_parts_exceptions_dtype(&self) -> DType { + DType::Primitive(self.metadata().left_parts_ptype, Nullability::Nullable) + } + /// The leftmost (most significant) bits of the floating point values stored in the array. /// /// These are bit-packed and dictionary encoded, and cannot directly be interpreted without /// the metadata of this array. pub fn left_parts(&self) -> Array { self.as_ref() - .child(0, &self.metadata().left_parts_dtype, self.len()) + .child(0, &self.left_parts_dtype(), self.len()) .vortex_expect("ALPRDArray: left_parts child") } /// The rightmost (least significant) bits of the floating point values stored in the array. pub fn right_parts(&self) -> Array { - let uint_ptype = if self.is_f32() { - PType::U32 - } else { - PType::U64 - }; - self.as_ref() - .child( - 1, - &DType::Primitive(uint_ptype, self.metadata().left_parts_dtype.nullability()), - self.len(), - ) + .child(1, &self.right_parts_dtype(), self.len()) .vortex_expect("ALPRDArray: right_parts child") } @@ -121,11 +151,7 @@ impl ALPRDArray { pub fn left_parts_exceptions(&self) -> Option { self.metadata().has_exceptions.then(|| { self.as_ref() - .child( - 2, - &self.metadata().left_parts_dtype.as_nullable(), - self.len(), - ) + .child(2, &self.left_parts_exceptions_dtype(), self.len()) .vortex_expect("ALPRDArray: left_parts_exceptions child") }) } diff --git a/encodings/alp/src/alp_rd/mod.rs b/encodings/alp/src/alp_rd/mod.rs index 2c22c00929..7608c97360 100644 --- a/encodings/alp/src/alp_rd/mod.rs +++ b/encodings/alp/src/alp_rd/mod.rs @@ -1,4 +1,5 @@ pub use array::*; +use vortex::validity::Validity; mod array; mod compute; @@ -208,7 +209,7 @@ impl RDEncoder { .into_array() }; - let primitive_right = PrimitiveArray::from_vec(right_parts, array.validity()); + let primitive_right = PrimitiveArray::from(right_parts); // SAFETY: by construction, all values in right_parts are right_bit_width + leading zeros. let packed_right = unsafe { bitpack_encode_unchecked(primitive_right, self.right_bit_width as _) @@ -231,9 +232,7 @@ impl RDEncoder { .into_array() }; - let exc_array = - PrimitiveArray::from_nullable_vec(exceptions.into_iter().map(Some).collect()) - .into_array(); + let exc_array = PrimitiveArray::from_vec(exceptions, Validity::AllValid).into_array(); SparseArray::try_new(packed_pos, exc_array, doubles.len(), ScalarValue::Null) .vortex_expect("ALP-RD: construction of exceptions SparseArray") .into_array() diff --git a/encodings/bytebool/src/lib.rs b/encodings/bytebool/src/lib.rs index 953c62d2ac..1a2afda2a4 100644 --- a/encodings/bytebool/src/lib.rs +++ b/encodings/bytebool/src/lib.rs @@ -9,7 +9,7 @@ use vortex::stats::StatsSet; use vortex::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata}; use vortex::variants::{ArrayVariants, BoolArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{impl_encoding, ArrayDef, ArrayTrait, Canonical, IntoCanonical, TypedArray}; +use vortex::{impl_encoding, ArrayTrait, Canonical, IntoCanonical, TypedArray}; use vortex_buffer::Buffer; use vortex_dtype::DType; use vortex_error::{VortexExpect as _, VortexResult}; diff --git a/encodings/datetime-parts/Cargo.toml b/encodings/datetime-parts/Cargo.toml index 3fd0aa0331..329554d073 100644 --- a/encodings/datetime-parts/Cargo.toml +++ b/encodings/datetime-parts/Cargo.toml @@ -17,6 +17,7 @@ readme = { workspace = true } workspace = true [dependencies] +itertools = { workspace = true } log = { workspace = true } serde = { workspace = true, features = ["derive"] } vortex-array = { workspace = true } diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index b25b506d99..9c7cf5060c 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -2,16 +2,15 @@ use std::fmt::Debug; use serde::{Deserialize, Serialize}; use vortex::array::StructArray; +use vortex::compute::unary::try_cast; use vortex::encoding::ids; use vortex::stats::{ArrayStatisticsCompute, StatsSet}; -use vortex::validity::{ArrayValidity, LogicalValidity}; +use vortex::validity::{ArrayValidity, LogicalValidity, Validity}; use vortex::variants::{ArrayVariants, ExtensionArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoCanonical, -}; -use vortex_dtype::DType; -use vortex_error::{vortex_bail, VortexExpect as _, VortexResult}; +use vortex::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoCanonical}; +use vortex_dtype::{DType, PType}; +use vortex_error::{vortex_bail, VortexExpect as _, VortexResult, VortexUnwrap}; use crate::compute::decode_to_temporal; @@ -21,9 +20,9 @@ impl_encoding!("vortex.datetimeparts", ids::DATE_TIME_PARTS, DateTimeParts); pub struct DateTimePartsMetadata { // Validity lives in the days array // TODO(ngates): we should actually model this with a Tuple array when we have one. - days_dtype: DType, - seconds_dtype: DType, - subseconds_dtype: DType, + days_ptype: PType, + seconds_ptype: PType, + subseconds_ptype: PType, } impl DateTimePartsArray { @@ -33,14 +32,18 @@ impl DateTimePartsArray { seconds: Array, subsecond: Array, ) -> VortexResult { - if !days.dtype().is_int() { - vortex_bail!(MismatchedTypes: "any integer", days.dtype()); + if !days.dtype().is_int() || (dtype.is_nullable() != days.dtype().is_nullable()) { + vortex_bail!( + "Expected integer with nullability {}, got {}", + dtype.is_nullable(), + days.dtype() + ); } - if !seconds.dtype().is_int() { - vortex_bail!(MismatchedTypes: "any integer", seconds.dtype()); + if !seconds.dtype().is_int() || seconds.dtype().is_nullable() { + vortex_bail!(MismatchedTypes: "non-nullable integer", seconds.dtype()); } - if !subsecond.dtype().is_int() { - vortex_bail!(MismatchedTypes: "any integer", subsecond.dtype()); + if !subsecond.dtype().is_int() || subsecond.dtype().is_nullable() { + vortex_bail!(MismatchedTypes: "non-nullable integer", subsecond.dtype()); } let length = days.len(); @@ -53,14 +56,16 @@ impl DateTimePartsArray { ); } + let metadata = DateTimePartsMetadata { + days_ptype: days.dtype().try_into()?, + seconds_ptype: seconds.dtype().try_into()?, + subseconds_ptype: subsecond.dtype().try_into()?, + }; + Self::try_from_parts( dtype, length, - DateTimePartsMetadata { - days_dtype: days.dtype().clone(), - seconds_dtype: seconds.dtype().clone(), - subseconds_dtype: subsecond.dtype().clone(), - }, + metadata, [days, seconds, subsecond].into(), StatsSet::new(), ) @@ -68,21 +73,35 @@ impl DateTimePartsArray { pub fn days(&self) -> Array { self.as_ref() - .child(0, &self.metadata().days_dtype, self.len()) + .child( + 0, + &DType::Primitive(self.metadata().days_ptype, self.dtype().nullability()), + self.len(), + ) .vortex_expect("DatetimePartsArray missing days array") } pub fn seconds(&self) -> Array { self.as_ref() - .child(1, &self.metadata().seconds_dtype, self.len()) + .child(1, &self.metadata().seconds_ptype.into(), self.len()) .vortex_expect("DatetimePartsArray missing seconds array") } pub fn subsecond(&self) -> Array { self.as_ref() - .child(2, &self.metadata().subseconds_dtype, self.len()) + .child(2, &self.metadata().subseconds_ptype.into(), self.len()) .vortex_expect("DatetimePartsArray missing subsecond array") } + + pub fn validity(&self) -> Validity { + if self.dtype().is_nullable() { + self.days() + .with_dyn(|a| a.logical_validity()) + .into_validity() + } else { + Validity::NonNullable + } + } } impl ArrayTrait for DateTimePartsArray {} @@ -96,11 +115,13 @@ impl ArrayVariants for DateTimePartsArray { impl ExtensionArrayTrait for DateTimePartsArray { fn storage_array(&self) -> Array { // FIXME(ngates): this needs to be a tuple array so we can implement Compare + // we don't want to write validity twice, so we pull it up to the top + let days = try_cast(self.days(), &self.days().dtype().as_nonnullable()).vortex_unwrap(); StructArray::try_new( vec!["days".into(), "seconds".into(), "subseconds".into()].into(), - [self.days(), self.seconds(), self.subsecond()].into(), + [days, self.seconds(), self.subsecond()].into(), self.len(), - self.logical_validity().into_validity(), + self.validity(), ) .vortex_expect("Failed to create struct array") .into_array() @@ -115,11 +136,11 @@ impl IntoCanonical for DateTimePartsArray { impl ArrayValidity for DateTimePartsArray { fn is_valid(&self, index: usize) -> bool { - self.days().with_dyn(|a| a.is_valid(index)) + self.validity().is_valid(index) } fn logical_validity(&self) -> LogicalValidity { - self.days().with_dyn(|a| a.logical_validity()) + self.validity().to_logical(self.len()) } } diff --git a/encodings/datetime-parts/src/compress.rs b/encodings/datetime-parts/src/compress.rs index 5c38850a5d..a6038e40f1 100644 --- a/encodings/datetime-parts/src/compress.rs +++ b/encodings/datetime-parts/src/compress.rs @@ -5,17 +5,27 @@ use vortex_datetime_dtype::TimeUnit; use vortex_dtype::{DType, PType}; use vortex_error::{vortex_bail, VortexResult}; +pub struct TemporalParts { + pub days: Array, + pub seconds: Array, + pub subseconds: Array, +} + /// Compress a `TemporalArray` into day, second, and subsecond components. /// /// Splitting the components by granularity creates more small values, which enables better /// cascading compression. -pub fn compress_temporal(array: TemporalArray) -> VortexResult<(Array, Array, Array)> { - // After this operation, timestamps will be PrimitiveArray - let timestamps = array.temporal_values().into_primitive()?; +pub fn split_temporal(array: TemporalArray) -> VortexResult { + let temporal_values = array.temporal_values().into_primitive()?; + let validity = temporal_values.validity().clone(); + + // After this operation, timestamps will be non-nullable PrimitiveArray let timestamps = try_cast( - ×tamps, - &DType::Primitive(PType::I64, timestamps.dtype().nullability()), - )?; + &temporal_values, + &DType::Primitive(PType::I64, temporal_values.dtype().nullability()), + )? + .as_primitive(); + let divisor = match array.temporal_metadata().time_unit() { TimeUnit::Ns => 1_000_000_000, TimeUnit::Us => 1_000_000, @@ -29,15 +39,15 @@ pub fn compress_temporal(array: TemporalArray) -> VortexResult<(Array, Array, Ar let mut seconds = Vec::with_capacity(length); let mut subsecond = Vec::with_capacity(length); - for &t in timestamps.as_primitive().maybe_null_slice::().iter() { + for &t in timestamps.maybe_null_slice::().iter() { days.push(t / (86_400 * divisor)); seconds.push((t % (86_400 * divisor)) / divisor); subsecond.push((t % (86_400 * divisor)) % divisor); } - Ok(( - PrimitiveArray::from_vec(days, timestamps.as_primitive().validity()).into_array(), - PrimitiveArray::from(seconds).into_array(), - PrimitiveArray::from(subsecond).into_array(), - )) + Ok(TemporalParts { + days: PrimitiveArray::from_vec(days, validity).into_array(), + seconds: PrimitiveArray::from(seconds).into_array(), + subseconds: PrimitiveArray::from(subsecond).into_array(), + }) } diff --git a/encodings/datetime-parts/src/compute.rs b/encodings/datetime-parts/src/compute.rs index 48699f249d..5e57800580 100644 --- a/encodings/datetime-parts/src/compute.rs +++ b/encodings/datetime-parts/src/compute.rs @@ -1,10 +1,11 @@ +use itertools::Itertools as _; use vortex::array::{PrimitiveArray, TemporalArray}; use vortex::compute::unary::{scalar_at, ScalarAtFn}; use vortex::compute::{slice, take, ArrayCompute, SliceFn, TakeFn}; use vortex::validity::ArrayValidity; use vortex::{Array, ArrayDType, IntoArray, IntoArrayVariant}; use vortex_datetime_dtype::{TemporalMetadata, TimeUnit}; -use vortex_dtype::DType; +use vortex_dtype::{DType, PType}; use vortex_error::{vortex_bail, VortexResult, VortexUnwrap as _}; use vortex_scalar::Scalar; @@ -61,6 +62,13 @@ impl ScalarAtFn for DateTimePartsArray { vortex_bail!("Metadata must be Timestamp, found {}", ext.id()); }; + if !self.is_valid(index) { + return Ok(Scalar::extension( + ext, + Scalar::null(DType::Primitive(PType::I64, nullability)), + )); + } + let divisor = match time_unit { TimeUnit::Ns => 1_000_000_000, TimeUnit::Us => 1_000_000, @@ -75,7 +83,10 @@ impl ScalarAtFn for DateTimePartsArray { let scalar = days * 86_400 * divisor + seconds * divisor + subseconds; - Ok(Scalar::primitive(scalar, nullability)) + Ok(Scalar::extension( + ext, + Scalar::primitive(scalar, nullability), + )) } fn scalar_at_unchecked(&self, index: usize) -> Scalar { @@ -110,13 +121,13 @@ pub fn decode_to_temporal(array: &DateTimePartsArray) -> VortexResult() .iter() - .zip(seconds_buf.maybe_null_slice::().iter()) - .zip(subsecond_buf.maybe_null_slice::().iter()) + .zip_eq(seconds_buf.maybe_null_slice::().iter()) + .zip_eq(subsecond_buf.maybe_null_slice::().iter()) .map(|((d, s), ss)| d * 86_400 * divisor + s * divisor + ss) .collect::>(); Ok(TemporalArray::new_timestamp( - PrimitiveArray::from_vec(values, array.logical_validity().into_validity()).into_array(), + PrimitiveArray::from_vec(values, array.validity().clone()).into_array(), temporal_metadata.time_unit(), temporal_metadata.time_zone().map(ToString::to_string), )) @@ -125,12 +136,13 @@ pub fn decode_to_temporal(array: &DateTimePartsArray) -> VortexResult(), - raw_values.as_slice() - ); + assert_eq!(primitive_values.maybe_null_slice::(), raw_values); + assert_eq!(primitive_values.validity(), validity); } } diff --git a/encodings/dict/src/dict.rs b/encodings/dict/src/dict.rs index 9710985441..2d418d3e00 100644 --- a/encodings/dict/src/dict.rs +++ b/encodings/dict/src/dict.rs @@ -10,30 +10,31 @@ use vortex::stats::StatsSet; use vortex::validity::{ArrayValidity, LogicalValidity}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoArrayVariant, + impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoArrayVariant, IntoCanonical, }; -use vortex_dtype::{match_each_integer_ptype, DType}; +use vortex_dtype::{match_each_integer_ptype, DType, PType}; use vortex_error::{vortex_bail, vortex_panic, VortexExpect as _, VortexResult}; impl_encoding!("vortex.dict", ids::DICT, Dict); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DictMetadata { - codes_dtype: DType, + codes_ptype: PType, values_len: usize, } impl DictArray { pub fn try_new(codes: Array, values: Array) -> VortexResult { - if !codes.dtype().is_unsigned_int() { - vortex_bail!(MismatchedTypes: "unsigned int", codes.dtype()); + if !codes.dtype().is_unsigned_int() || codes.dtype().is_nullable() { + vortex_bail!(MismatchedTypes: "non-nullable unsigned int", codes.dtype()); } Self::try_from_parts( values.dtype().clone(), codes.len(), DictMetadata { - codes_dtype: codes.dtype().clone(), + codes_ptype: PType::try_from(codes.dtype()) + .vortex_expect("codes dtype must be uint"), values_len: values.len(), }, [values, codes].into(), @@ -51,7 +52,7 @@ impl DictArray { #[inline] pub fn codes(&self) -> Array { self.as_ref() - .child(1, &self.metadata().codes_dtype, self.len()) + .child(1, &DType::from(self.metadata().codes_ptype), self.len()) .vortex_expect("DictArray is missing its codes child array") } } diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index 184a56ee25..ecfe2af76b 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -7,9 +7,7 @@ use vortex::stats::{ArrayStatisticsCompute, StatsSet}; use vortex::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata}; use vortex::variants::{ArrayVariants, PrimitiveArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoCanonical, TypedArray, -}; +use vortex::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoCanonical, TypedArray}; use vortex_buffer::Buffer; use vortex_dtype::{DType, NativePType, Nullability, PType}; use vortex_error::{ diff --git a/encodings/fastlanes/src/delta/mod.rs b/encodings/fastlanes/src/delta/mod.rs index 5c12b7def9..822258fec7 100644 --- a/encodings/fastlanes/src/delta/mod.rs +++ b/encodings/fastlanes/src/delta/mod.rs @@ -8,9 +8,7 @@ use vortex::stats::{ArrayStatisticsCompute, StatsSet}; use vortex::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata}; use vortex::variants::{ArrayVariants, PrimitiveArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoCanonical, -}; +use vortex::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoCanonical}; use vortex_dtype::{match_each_unsigned_integer_ptype, NativePType}; use vortex_error::{vortex_bail, vortex_panic, VortexExpect as _, VortexResult}; diff --git a/encodings/fastlanes/src/for/mod.rs b/encodings/fastlanes/src/for/mod.rs index 7a037dcd95..eec771cac9 100644 --- a/encodings/fastlanes/src/for/mod.rs +++ b/encodings/fastlanes/src/for/mod.rs @@ -7,7 +7,7 @@ use vortex::stats::{ArrayStatisticsCompute, StatsSet}; use vortex::validity::{ArrayValidity, LogicalValidity}; use vortex::variants::{ArrayVariants, PrimitiveArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoCanonical}; +use vortex::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoCanonical}; use vortex_dtype::{DType, PType}; use vortex_error::{vortex_bail, vortex_panic, VortexExpect as _, VortexResult}; use vortex_scalar::Scalar; diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 6130200ac1..f802498c50 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -8,7 +8,7 @@ use vortex::stats::{ArrayStatisticsCompute, StatsSet}; use vortex::validity::{ArrayValidity, LogicalValidity, Validity}; use vortex::variants::{ArrayVariants, BinaryArrayTrait, Utf8ArrayTrait}; use vortex::visitor::AcceptArrayVisitor; -use vortex::{impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, IntoCanonical}; +use vortex::{impl_encoding, Array, ArrayDType, ArrayTrait, IntoCanonical}; use vortex_dtype::{DType, Nullability, PType}; use vortex_error::{vortex_bail, VortexExpect, VortexResult}; diff --git a/encodings/roaring/src/boolean/mod.rs b/encodings/roaring/src/boolean/mod.rs index 458cd0827a..127b56445a 100644 --- a/encodings/roaring/src/boolean/mod.rs +++ b/encodings/roaring/src/boolean/mod.rs @@ -12,9 +12,7 @@ use vortex::stats::{Stat, StatsSet}; use vortex::validity::{ArrayValidity, LogicalValidity, Validity}; use vortex::variants::{ArrayVariants, BoolArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{ - impl_encoding, Array, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoCanonical, TypedArray, -}; +use vortex::{impl_encoding, Array, ArrayTrait, Canonical, IntoArray, IntoCanonical, TypedArray}; use vortex_buffer::Buffer; use vortex_dtype::DType; use vortex_dtype::Nullability::NonNullable; diff --git a/encodings/roaring/src/integer/mod.rs b/encodings/roaring/src/integer/mod.rs index 3444f41bc1..0ccab6f110 100644 --- a/encodings/roaring/src/integer/mod.rs +++ b/encodings/roaring/src/integer/mod.rs @@ -9,9 +9,7 @@ use vortex::stats::{ArrayStatisticsCompute, StatsSet}; use vortex::validity::{ArrayValidity, LogicalValidity}; use vortex::variants::{ArrayVariants, PrimitiveArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use vortex::{ - impl_encoding, Array, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoCanonical, TypedArray, -}; +use vortex::{impl_encoding, Array, ArrayTrait, Canonical, IntoArray, IntoCanonical, TypedArray}; use vortex_buffer::Buffer; use vortex_dtype::Nullability::NonNullable; use vortex_dtype::{DType, PType}; diff --git a/encodings/runend-bool/src/array.rs b/encodings/runend-bool/src/array.rs index 88c5bb2f39..5e8ec946b2 100644 --- a/encodings/runend-bool/src/array.rs +++ b/encodings/runend-bool/src/array.rs @@ -7,8 +7,7 @@ use vortex::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadat use vortex::variants::{ArrayVariants, BoolArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArrayVariant, - IntoCanonical, + impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArrayVariant, IntoCanonical, }; use vortex_dtype::{DType, Nullability}; use vortex_error::{vortex_bail, VortexExpect as _, VortexResult}; diff --git a/encodings/runend/src/runend.rs b/encodings/runend/src/runend.rs index 6bb35e7b0e..8851b79c48 100644 --- a/encodings/runend/src/runend.rs +++ b/encodings/runend/src/runend.rs @@ -10,7 +10,7 @@ use vortex::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadat use vortex::variants::{ArrayVariants, PrimitiveArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoArrayVariant, + impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoArrayVariant, IntoCanonical, }; use vortex_dtype::DType; diff --git a/encodings/zigzag/src/zigzag.rs b/encodings/zigzag/src/zigzag.rs index abd9045bd2..4ce5109eb5 100644 --- a/encodings/zigzag/src/zigzag.rs +++ b/encodings/zigzag/src/zigzag.rs @@ -6,7 +6,7 @@ use vortex::validity::{ArrayValidity, LogicalValidity}; use vortex::variants::{ArrayVariants, PrimitiveArrayTrait}; use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor}; use vortex::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoArrayVariant, + impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoArrayVariant, IntoCanonical, }; use vortex_dtype::{DType, PType}; diff --git a/vortex-array/src/array/bool/mod.rs b/vortex-array/src/array/bool/mod.rs index c40da4d3b7..7e0fb8498f 100644 --- a/vortex-array/src/array/bool/mod.rs +++ b/vortex-array/src/array/bool/mod.rs @@ -11,7 +11,7 @@ use crate::stats::StatsSet; use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata}; use crate::variants::{ArrayVariants, BoolArrayTrait}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{impl_encoding, ArrayDef, ArrayTrait, Canonical, IntoCanonical, TypedArray}; +use crate::{impl_encoding, ArrayTrait, Canonical, IntoCanonical, TypedArray}; mod accessors; mod compute; diff --git a/vortex-array/src/array/chunked/mod.rs b/vortex-array/src/array/chunked/mod.rs index 58bf23a74f..29a9be145c 100644 --- a/vortex-array/src/array/chunked/mod.rs +++ b/vortex-array/src/array/chunked/mod.rs @@ -19,7 +19,7 @@ use crate::stream::{ArrayStream, ArrayStreamAdapter}; use crate::validity::Validity::NonNullable; use crate::validity::{ArrayValidity, LogicalValidity, Validity}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, IntoArray, IntoCanonical}; +use crate::{impl_encoding, Array, ArrayDType, ArrayTrait, IntoArray, IntoCanonical}; mod canonical; mod compute; diff --git a/vortex-array/src/array/constant/mod.rs b/vortex-array/src/array/constant/mod.rs index 48c4ecbc07..a7be493433 100644 --- a/vortex-array/src/array/constant/mod.rs +++ b/vortex-array/src/array/constant/mod.rs @@ -8,7 +8,7 @@ use crate::encoding::ids; use crate::stats::{Stat, StatsSet}; use crate::validity::{ArrayValidity, LogicalValidity}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{impl_encoding, ArrayDType, ArrayDef, ArrayTrait}; +use crate::{impl_encoding, ArrayDType, ArrayTrait}; mod canonical; mod compute; diff --git a/vortex-array/src/array/extension/mod.rs b/vortex-array/src/array/extension/mod.rs index 0a1ec8c805..65dcabee39 100644 --- a/vortex-array/src/array/extension/mod.rs +++ b/vortex-array/src/array/extension/mod.rs @@ -7,7 +7,7 @@ use crate::stats::ArrayStatisticsCompute; use crate::validity::{ArrayValidity, LogicalValidity}; use crate::variants::{ArrayVariants, ExtensionArrayTrait}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoCanonical}; +use crate::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoCanonical}; mod compute; diff --git a/vortex-array/src/array/null/mod.rs b/vortex-array/src/array/null/mod.rs index 4772526d20..c2efd1c4ab 100644 --- a/vortex-array/src/array/null/mod.rs +++ b/vortex-array/src/array/null/mod.rs @@ -7,7 +7,7 @@ use crate::stats::{ArrayStatisticsCompute, Stat, StatsSet}; use crate::validity::{ArrayValidity, LogicalValidity, Validity}; use crate::variants::{ArrayVariants, NullArrayTrait}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{impl_encoding, ArrayDef, ArrayTrait, Canonical, IntoCanonical}; +use crate::{impl_encoding, ArrayTrait, Canonical, IntoCanonical}; mod compute; diff --git a/vortex-array/src/array/primitive/mod.rs b/vortex-array/src/array/primitive/mod.rs index fad3eda471..d4183d33a0 100644 --- a/vortex-array/src/array/primitive/mod.rs +++ b/vortex-array/src/array/primitive/mod.rs @@ -19,8 +19,7 @@ use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata use crate::variants::{ArrayVariants, PrimitiveArrayTrait}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; use crate::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoCanonical, - TypedArray, + impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoCanonical, TypedArray, }; mod accessor; diff --git a/vortex-array/src/array/sparse/mod.rs b/vortex-array/src/array/sparse/mod.rs index fdfb963bac..c0ec5094c0 100644 --- a/vortex-array/src/array/sparse/mod.rs +++ b/vortex-array/src/array/sparse/mod.rs @@ -10,7 +10,7 @@ use crate::encoding::ids; use crate::stats::{ArrayStatisticsCompute, StatsSet}; use crate::validity::{ArrayValidity, LogicalValidity}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, IntoArray, IntoArrayVariant}; +use crate::{impl_encoding, Array, ArrayDType, ArrayTrait, IntoArray, IntoArrayVariant}; mod compute; mod flatten; diff --git a/vortex-array/src/array/struct_/mod.rs b/vortex-array/src/array/struct_/mod.rs index 084dff3b79..44ab782c57 100644 --- a/vortex-array/src/array/struct_/mod.rs +++ b/vortex-array/src/array/struct_/mod.rs @@ -8,9 +8,7 @@ use crate::stats::{ArrayStatisticsCompute, StatsSet}; use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata}; use crate::variants::{ArrayVariants, StructArrayTrait}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArray, IntoCanonical, -}; +use crate::{impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArray, IntoCanonical}; mod compute; diff --git a/vortex-array/src/array/varbin/mod.rs b/vortex-array/src/array/varbin/mod.rs index 150eda6d0f..43be6df45e 100644 --- a/vortex-array/src/array/varbin/mod.rs +++ b/vortex-array/src/array/varbin/mod.rs @@ -18,7 +18,7 @@ use crate::compute::unary::scalar_at; use crate::encoding::ids; use crate::stats::StatsSet; use crate::validity::{Validity, ValidityMetadata}; -use crate::{impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, IntoArrayVariant}; +use crate::{impl_encoding, Array, ArrayDType, ArrayTrait, IntoArrayVariant}; mod accessor; mod array; diff --git a/vortex-array/src/array/varbinview/mod.rs b/vortex-array/src/array/varbinview/mod.rs index 4509b258f8..2d5d2c91a1 100644 --- a/vortex-array/src/array/varbinview/mod.rs +++ b/vortex-array/src/array/varbinview/mod.rs @@ -20,8 +20,7 @@ use crate::stats::StatsSet; use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; use crate::{ - impl_encoding, Array, ArrayDType, ArrayDef, ArrayTrait, Canonical, IntoArrayVariant, - IntoCanonical, + impl_encoding, Array, ArrayDType, ArrayTrait, Canonical, IntoArrayVariant, IntoCanonical, }; mod accessor; diff --git a/vortex-array/src/implementation.rs b/vortex-array/src/implementation.rs index 588b451d54..5c8ec3e4d0 100644 --- a/vortex-array/src/implementation.rs +++ b/vortex-array/src/implementation.rs @@ -120,7 +120,7 @@ macro_rules! impl_encoding { impl $crate::encoding::ArrayEncoding for [<$Name Encoding>] { #[inline] fn id(&self) -> $crate::encoding::EncodingId { - $Name::ID + <$Name as $crate::ArrayDef>::ID } #[inline] diff --git a/vortex-sampling-compressor/src/compressors/date_time_parts.rs b/vortex-sampling-compressor/src/compressors/date_time_parts.rs index b1d1ab5094..5a0608e1b9 100644 --- a/vortex-sampling-compressor/src/compressors/date_time_parts.rs +++ b/vortex-sampling-compressor/src/compressors/date_time_parts.rs @@ -5,7 +5,7 @@ use vortex::encoding::EncodingRef; use vortex::{Array, ArrayDType, ArrayDef, IntoArray}; use vortex_datetime_dtype::TemporalMetadata; use vortex_datetime_parts::{ - compress_temporal, DateTimeParts, DateTimePartsArray, DateTimePartsEncoding, + split_temporal, DateTimeParts, DateTimePartsArray, DateTimePartsEncoding, TemporalParts, }; use vortex_error::VortexResult; @@ -38,7 +38,11 @@ impl EncodingCompressor for DateTimePartsCompressor { like: Option>, ctx: SamplingCompressor<'a>, ) -> VortexResult> { - let (days, seconds, subseconds) = compress_temporal(TemporalArray::try_from(array)?)?; + let TemporalParts { + days, + seconds, + subseconds, + } = split_temporal(TemporalArray::try_from(array)?)?; let days = ctx .named("days")