Add an IDX DType

vortex-fastlanes/src/bitpacking/compress.rs

@@ -99,16 +99,13 @@ fn bitpacked_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: Compre
 
     return BitPackedArray::try_new(
         bitpack(parray, bit_width),
-        parray.validity().map(|v| {
-            ctx.compress(
-                v.as_ref(),
-                like_bp.and_then(|bp| bp.validity().map(|a| a.as_ref())),
-            )
-        }),
+        parray
+            .validity()
+            .map(|v| ctx.compress(v.as_ref(), like_bp.and_then(|bp| bp.validity()))),
         if num_exceptions > 0 {
             Some(ctx.compress(
                 bitpack_patches(parray, bit_width, num_exceptions).as_ref(),
-                like_bp.and_then(|bp| bp.patches()).map(|a| a.as_ref()),
+                like_bp.and_then(|bp| bp.patches()),
             ))
         } else {
             None

vortex-fastlanes/src/bitpacking/mod.rs

@@ -14,6 +14,7 @@ use vortex::serde::{ArraySerde, EncodingSerde};
 use vortex::stats::{Stat, Stats, StatsCompute, StatsSet};
 
 mod compress;
+mod serde;
 
 #[derive(Debug, Clone)]
 pub struct BitPackedArray {
@@ -62,13 +63,13 @@ impl BitPackedArray {
     }
 
     #[inline]
-    pub fn validity(&self) -> Option<&ArrayRef> {
-        self.validity.as_ref()
+    pub fn validity(&self) -> Option<&dyn Array> {
+        self.validity.as_deref()
     }
 
     #[inline]
-    pub fn patches(&self) -> Option<&ArrayRef> {
-        self.patches.as_ref()
+    pub fn patches(&self) -> Option<&dyn Array> {
+        self.patches.as_deref()
     }
 
     pub fn is_valid(&self, index: usize) -> bool {
@@ -194,7 +195,6 @@ impl Encoding for BitPackedEncoding {
     }
 
     fn serde(&self) -> Option<&dyn EncodingSerde> {
-        None
-        // Some(self)
+        Some(self)
     }
 }

vortex-fastlanes/src/bitpacking/serde.rs

@@ -0,0 +1,47 @@
+// (c) Copyright 2024 Fulcrum Technologies, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::io;
+
+use vortex::array::{Array, ArrayRef};
+use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx};
+
+use crate::{BitPackedArray, BitPackedEncoding};
+
+impl ArraySerde for BitPackedArray {
+    fn write(&self, ctx: &mut WriteCtx) -> io::Result<()> {
+        ctx.write(self.encoded())?;
+        ctx.write_optional_array(self.validity())?;
+        ctx.write_optional_array(self.patches())?;
+        ctx.write_usize(self.bit_width())?;
+        ctx.dtype(self.dtype())?;
+        ctx.write_usize(self.len())
+    }
+}
+
+impl EncodingSerde for BitPackedEncoding {
+    fn read(&self, ctx: &mut ReadCtx) -> io::Result<ArrayRef> {
+        let encoded = ctx.read()?;
+        let validity = ctx.read_optional_array()?;
+        let patches = ctx.read_optional_array()?;
+        let bit_width = ctx.read_usize()?;
+        let dtype = ctx.dtype()?;
+        let len = ctx.read_usize()?;
+        Ok(
+            BitPackedArray::try_new(encoded, validity, patches, bit_width, dtype, len)
+                .unwrap()
+                .boxed(),
+        )
+    }
+}

vortex-fastlanes/src/for/mod.rs

@@ -12,6 +12,7 @@ use vortex::serde::{ArraySerde, EncodingSerde};
 use vortex::stats::{Stat, Stats, StatsCompute, StatsSet};
 
 mod compress;
+mod serde;
 
 #[derive(Debug, Clone)]
 pub struct FoRArray {
@@ -144,7 +145,6 @@ impl Encoding for FoREncoding {
     }
 
     fn serde(&self) -> Option<&dyn EncodingSerde> {
-        None
-        // Some(self)
+        Some(self)
     }
 }

vortex-fastlanes/src/for/serde.rs

@@ -0,0 +1,66 @@
+// (c) Copyright 2024 Fulcrum Technologies, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::io;
+
+use vortex::array::{Array, ArrayRef};
+use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx};
+
+use crate::{FoRArray, FoREncoding};
+
+impl ArraySerde for FoRArray {
+    fn write(&self, ctx: &mut WriteCtx) -> io::Result<()> {
+        ctx.scalar(self.reference())?;
+        ctx.write(self.child())
+    }
+}
+
+impl EncodingSerde for FoREncoding {
+    fn read(&self, ctx: &mut ReadCtx) -> io::Result<ArrayRef> {
+        let reference = ctx.scalar()?;
+        let child = ctx.read()?;
+        Ok(FoRArray::try_new(child, reference).unwrap().boxed())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::io;
+
+    use vortex::array::primitive::PrimitiveArray;
+    use vortex::array::{Array, ArrayRef};
+    use vortex::scalar::Scalar;
+    use vortex::serde::{ReadCtx, WriteCtx};
+
+    use super::*;
+
+    fn roundtrip_array(array: &dyn Array) -> io::Result<ArrayRef> {
+        let mut buf = Vec::<u8>::new();
+        let mut write_ctx = WriteCtx::new(&mut buf);
+        write_ctx.write(array)?;
+        let mut read = buf.as_slice();
+        let mut read_ctx = ReadCtx::new(array.dtype(), &mut read);
+        read_ctx.read()
+    }
+
+    #[test]
+    fn roundtrip() {
+        let arr = FoRArray::try_new(
+            PrimitiveArray::from_vec(vec![-7i64, -13, 17, 23]).boxed(),
+            <i64 as Into<Box<dyn Scalar>>>::into(-7i64),
+        )
+        .unwrap();
+        roundtrip_array(arr.as_ref()).unwrap();
+    }
+}

vortex-fastlanes/src/lib.rs

@@ -20,7 +20,6 @@ use vortex::array::{EncodingRef, ENCODINGS};
 
 mod bitpacking;
 mod r#for;
-mod serde;
 
 #[distributed_slice(ENCODINGS)]
 static ENCODINGS_FL_BITPACKING: EncodingRef = &BitPackedEncoding;

vortex/src/serde/mod.rs

@@ -119,6 +119,20 @@ impl<'a> ReadCtx<'a> {
             .map(|u| u as usize)
     }
 
+    pub fn read_option_tag(&mut self) -> io::Result<bool> {
+        let mut tag = [0; 1];
+        self.r.read_exact(&mut tag)?;
+        Ok(tag[0] == 0x01)
+    }
+
+    pub fn read_optional_array(&mut self) -> io::Result<Option<ArrayRef>> {
+        if self.read_option_tag()? {
+            self.read().map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+
     pub fn read(&mut self) -> io::Result<ArrayRef> {
         let encoding_id = self.read_usize()?;
         if let Some(serde) = ENCODINGS
@@ -178,6 +192,15 @@ impl<'a> WriteCtx<'a> {
         self.w.write_all(&[if present { 0x01 } else { 0x00 }])
     }
 
+    pub fn write_optional_array(&mut self, array: Option<&dyn Array>) -> io::Result<()> {
+        self.write_option_tag(array.is_some())?;
+        if let Some(array) = array {
+            self.write(array)
+        } else {
+            Ok(())
+        }
+    }
+
     pub fn write(&mut self, array: &dyn Array) -> io::Result<()> {
         let encoding_id = self
             .available_encodings