Redo REE infers length from ends array (#218)

This somehow got lost in #189 (probably because something was still
broken)

vortex-ree/src/compress.rs

@@ -60,7 +60,6 @@ impl EncodingCompression for REEEncoding {
             compressed_ends,
             compressed_values,
             ctx.compress_validity(primitive_array.validity())?,
-            array.len(),
         )
         .into_array())
     }
@@ -198,7 +197,6 @@ mod test {
             vec![2u32, 5, 10].into_array(),
             vec![1i32, 2, 3].into_array(),
             Some(validity),
-            10,
         );
 
         let decoded = ree_decode(

vortex-ree/src/ree.rs

@@ -2,6 +2,7 @@ use std::sync::{Arc, RwLock};
 
 use vortex::array::{check_slice_bounds, Array, ArrayKind, ArrayRef};
 use vortex::compress::EncodingCompression;
+use vortex::compute::scalar_at::scalar_at;
 use vortex::compute::search_sorted::SearchSortedSide;
 use vortex::compute::ArrayCompute;
 use vortex::encoding::{Encoding, EncodingId, EncodingRef};
@@ -28,34 +29,24 @@ pub struct REEArray {
 }
 
 impl REEArray {
-    pub fn new(
-        ends: ArrayRef,
-        values: ArrayRef,
-        validity: Option<Validity>,
-        length: usize,
-    ) -> Self {
-        Self::try_new(ends, values, validity, length).unwrap()
+    pub fn new(ends: ArrayRef, values: ArrayRef, validity: Option<Validity>) -> Self {
+        Self::try_new(ends, values, validity).unwrap()
     }
 
     pub fn try_new(
         ends: ArrayRef,
         values: ArrayRef,
         validity: Option<Validity>,
-        length: usize,
     ) -> VortexResult<Self> {
+        let length: usize = scalar_at(&ends, ends.len() - 1)?.try_into()?;
         if let Some(v) = &validity {
             assert_eq!(v.len(), length);
         }
 
-        if !ends
-            .stats()
-            .get_as::<bool>(&Stat::IsStrictSorted)
-            .unwrap_or(true)
-        {
+        if !ends.stats().get_as(&Stat::IsStrictSorted).unwrap_or(true) {
             vortex_bail!("Ends array must be strictly sorted",);
         }
 
-        // TODO(ngates): https://github.com/fulcrum-so/spiral/issues/873
         Ok(Self {
             ends,
             values,
@@ -82,7 +73,6 @@ impl REEArray {
                     ends.into_array(),
                     values.into_array(),
                     p.validity().to_owned_view(),
-                    p.len(),
                 )
                 .into_array())
             }
@@ -227,7 +217,6 @@ mod test {
             vec![2u32, 5, 10].into_array(),
             vec![1i32, 2, 3].into_array(),
             None,
-            10,
         );
         assert_eq!(arr.len(), 10);
         assert_eq!(
@@ -250,7 +239,6 @@ mod test {
             vec![2u32, 5, 10].into_array(),
             vec![1i32, 2, 3].into_array(),
             None,
-            10,
         )
         .slice(3, 8)
         .unwrap();
@@ -272,7 +260,6 @@ mod test {
             vec![2u32, 5, 10].into_array(),
             vec![1i32, 2, 3].into_array(),
             None,
-            10,
         );
         assert_eq!(
             flatten_primitive(&arr).unwrap().typed_data::<i32>(),

vortex-ree/src/serde.rs

@@ -7,7 +7,6 @@ use crate::{REEArray, REEEncoding};
 
 impl ArraySerde for REEArray {
     fn write(&self, ctx: &mut WriteCtx) -> VortexResult<()> {
-        ctx.write_usize(self.len())?;
         ctx.write_validity(self.validity())?;
         // TODO(robert): Stop writing this
         ctx.dtype(self.ends().dtype())?;
@@ -22,12 +21,11 @@ impl ArraySerde for REEArray {
 
 impl EncodingSerde for REEEncoding {
     fn read(&self, ctx: &mut ReadCtx) -> VortexResult<ArrayRef> {
-        let len = ctx.read_usize()?;
         let validity = ctx.read_validity()?;
         let ends_dtype = ctx.dtype()?;
         let ends = ctx.with_schema(&ends_dtype).read()?;
         let values = ctx.read()?;
-        Ok(REEArray::new(ends, values, validity, len).into_array())
+        Ok(REEArray::new(ends, values, validity).into_array())
     }
 }
 
@@ -58,22 +56,17 @@ mod test {
             vec![0u8, 9, 20, 32, 49].into_array(),
             vec![-7i64, -13, 17, 23].into_array(),
             None,
-            49,
         );
         let read_arr = roundtrip_array(&arr).unwrap();
         let read_ree = read_arr.as_ree();
 
         assert_eq!(
-            arr.ends().as_primitive().buffer().typed_data::<u8>(),
-            read_ree.ends().as_primitive().buffer().typed_data::<u8>()
+            arr.ends().as_primitive().typed_data::<u8>(),
+            read_ree.ends().as_primitive().typed_data::<u8>()
         );
         assert_eq!(
-            arr.values().as_primitive().buffer().typed_data::<i64>(),
-            read_ree
-                .values()
-                .as_primitive()
-                .buffer()
-                .typed_data::<i64>()
+            arr.values().as_primitive().typed_data::<i64>(),
+            read_ree.values().as_primitive().typed_data::<i64>()
         );
     }
 }