Validity

vortex-array2/src/array/mod.rs

@@ -1,4 +1,3 @@
 pub mod bool;
 pub mod primitive;
 pub mod ree;
-pub mod validity;

vortex-array2/src/array/primitive/mod.rs

@@ -3,13 +3,10 @@ mod compute;
 use arrow_buffer::Buffer;
 use vortex::ptype::{NativePType, PType};
 use vortex_error::VortexResult;
-use vortex_schema::{DType, Nullability};
+use vortex_schema::DType;
 
-use crate::array::validity::Validity;
-use crate::compute::scalar_at;
-use crate::impl_encoding;
-use crate::validity::ArrayValidity;
-use crate::{Array, IntoArray};
+use crate::validity::{ArrayValidity, Validity, ValidityMetadata};
+use crate::{impl_encoding, IntoArray};
 use crate::{ArrayData, TypedArrayData};
 use crate::{ArrayMetadata, TryFromArrayMetadata};
 use crate::{ArrayView, ToArrayData};
@@ -18,26 +15,39 @@ use crate::{ToArray, TypedArrayView};
 impl_encoding!("vortex.primitive", Primitive);
 
 #[derive(Clone, Debug)]
-pub struct PrimitiveMetadata(PType);
+pub struct PrimitiveMetadata {
+    ptype: PType,
+    validity: ValidityMetadata,
+}
+
 impl PrimitiveMetadata {
     pub fn ptype(&self) -> PType {
-        self.0
+        self.ptype
+    }
+    pub fn validity(&self) -> &ValidityMetadata {
+        &self.validity
     }
 }
 
 pub trait PrimitiveArray {
     fn dtype(&self) -> &DType;
     fn ptype(&self) -> PType;
     fn buffer(&self) -> &Buffer;
-    fn validity(&self) -> Option<Array>;
+    fn len(&self) -> usize {
+        self.buffer().len() / self.ptype().byte_width()
+    }
+    fn validity(&self) -> Option<Validity>;
 }
 
 impl PrimitiveData {
     pub fn from_vec<T: NativePType>(values: Vec<T>) -> Self {
         ArrayData::try_new(
             &PrimitiveEncoding,
             DType::from(T::PTYPE),
-            Arc::new(PrimitiveMetadata(T::PTYPE)),
+            Arc::new(PrimitiveMetadata {
+                ptype: T::PTYPE,
+                validity: ValidityMetadata::NonNullable,
+            }),
             vec![Buffer::from_vec(values)].into(),
             vec![].into(),
         )
@@ -60,11 +70,10 @@ impl PrimitiveArray for PrimitiveData {
         self.data().buffers().first().unwrap()
     }
 
-    fn validity(&self) -> Option<Array> {
-        match self.dtype().nullability() {
-            Nullability::NonNullable => None,
-            Nullability::Nullable => Some(self.data().child(0).unwrap().to_array()),
-        }
+    fn validity(&self) -> Option<Validity> {
+        self.metadata()
+            .validity()
+            .to_validity(self.len(), self.data().child(0).map(|data| data.to_array()))
     }
 }
 
@@ -84,13 +93,13 @@ impl PrimitiveArray for PrimitiveView<'_> {
             .expect("PrimitiveView must have a single buffer")
     }
 
-    fn validity(&self) -> Option<Array> {
-        match self.dtype().nullability() {
-            Nullability::NonNullable => None,
-            Nullability::Nullable => {
-                Some(self.view().child(0, &Validity::DTYPE).unwrap().into_array())
-            }
-        }
+    fn validity(&self) -> Option<Validity> {
+        self.metadata().validity().to_validity(
+            self.len(),
+            self.view()
+                .child(0, &Validity::DTYPE)
+                .map(|view| view.into_array()),
+        )
     }
 }
 
@@ -119,17 +128,13 @@ impl TryFromArrayData for PrimitiveData {
 
 impl ArrayTrait for &dyn PrimitiveArray {
     fn len(&self) -> usize {
-        self.buffer().len() / self.ptype().byte_width()
+        (**self).len()
     }
 }
 
 impl ArrayValidity for &dyn PrimitiveArray {
     fn is_valid(&self, index: usize) -> bool {
-        if let Some(v) = self.validity() {
-            scalar_at(&v, index).unwrap().try_into().unwrap()
-        } else {
-            true
-        }
+        self.validity().map(|v| v.is_valid(index)).unwrap_or(true)
     }
 }
 

vortex-array2/src/validity.rs

@@ -1,7 +1,6 @@
 use vortex_error::{vortex_bail, VortexResult};
 use vortex_schema::{DType, Nullability};
 
-use crate::array::validity::ValidityArray;
 use crate::compute::scalar_at;
 use crate::{Array, ArrayData, ToArrayData, WithArray};
 
@@ -10,12 +9,6 @@ pub trait ArrayValidity {
     // Maybe add to_bool_array() here?
 }
 
-impl ArrayValidity for &dyn ValidityArray {
-    fn is_valid(&self, _index: usize) -> bool {
-        todo!()
-    }
-}
-
 #[derive(Clone, Debug)]
 pub enum ValidityMetadata {
     NonNullable,