Array2: Struct Array

Cargo.toml

@@ -68,6 +68,7 @@ pyo3-log = "0.9.0"
 rand = "0.8.5"
 reqwest = { version = "0.12.0", features = ["blocking"] }
 seq-macro = "0.3.5"
+serde = "1.0.197"
 simplelog = { version = "0.12.2", features = ["paris"] }
 thiserror = "1.0.58"
 uninit = "0.6.2"

vortex-array/Cargo.toml

@@ -40,6 +40,7 @@ vortex-alloc = { path = "../vortex-alloc" }
 vortex-error = { path = "../vortex-error" }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }
 vortex-schema = { path = "../vortex-schema" }
+serde = { workspace = true, features = ["derive"], optional = true }
 
 [build-dependencies]
 flatc = { workspace = true }

vortex-array/flatbuffers/array.fbs

@@ -4,11 +4,16 @@ enum Version: uint8 {
   V0 = 0,
 }
 
+// TODO(ngates): figure out if flatbuffers supports optional elements in a vector.
+table ArrayChild {
+    child: Array;
+}
+
 table Array {
     version: Version = V0;
     encoding: uint16;
     metadata: [ubyte];
-    children: [Array];
+    children: [ArrayChild];
     nbuffers: uint16;
 }
 

vortex-array/src/ptype.rs

@@ -11,6 +11,7 @@ use vortex_schema::{DType, FloatWidth, IntWidth};
 
 use crate::scalar::{PScalar, Scalar};
 
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Hash)]
 pub enum PType {
     U8,

vortex-array/src/serde/view.rs

@@ -101,7 +101,12 @@ impl<'a> ArrayView<'a> {
             .children()?
             .iter()
             .take(idx)
-            .map(|child| Self::cumulative_nbuffers(child))
+            .map(|child| {
+                child
+                    .child()
+                    .map(|c| Self::cumulative_nbuffers(c))
+                    .unwrap_or_default()
+            })
             .sum();
         let buffer_count = Self::cumulative_nbuffers(child);
 
@@ -119,7 +124,7 @@ impl<'a> ArrayView<'a> {
     fn array_child(&self, idx: usize) -> Option<fb::Array<'a>> {
         let children = self.array.children()?;
         if idx < children.len() {
-            Some(children.get(idx))
+            children.get(idx).child()
         } else {
             None
         }
@@ -134,7 +139,10 @@ impl<'a> ArrayView<'a> {
     fn cumulative_nbuffers(array: fb::Array) -> usize {
         let mut nbuffers = array.nbuffers() as usize;
         for child in array.children().unwrap_or_default() {
-            nbuffers += Self::cumulative_nbuffers(child);
+            nbuffers += child
+                .child()
+                .map(|c| Self::cumulative_nbuffers(c))
+                .unwrap_or_default();
         }
         nbuffers
     }

vortex-array/src/stats.rs

@@ -10,7 +10,7 @@ use vortex_schema::DType;
 use crate::ptype::NativePType;
 use crate::scalar::{ListScalarVec, Scalar};
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Stat {
     BitWidthFreq,
     TrailingZeroFreq,
@@ -146,7 +146,7 @@ impl StatsSet {
     }
 
     fn merge_scalar_stat(&mut self, other: &Self, stat: &Stat) {
-        match self.0.entry(stat.clone()) {
+        match self.0.entry(*stat) {
             Entry::Occupied(mut e) => {
                 if let Some(other_value) = other.get_as::<usize>(stat).unwrap() {
                     let self_value: usize = e.get().try_into().unwrap();
@@ -228,7 +228,7 @@ impl<'a> Stats<'a> {
     pub fn set_many(&self, other: &Stats, stats: Vec<&Stat>) {
         stats.into_iter().for_each(|stat| {
             if let Some(v) = other.get(stat) {
-                self.cache.write().unwrap().set(stat.clone(), v)
+                self.cache.write().unwrap().set(*stat, v)
             }
         });
     }

vortex-array2/Cargo.toml

@@ -1,24 +1,29 @@
 [package]
 name = "vortex-array2"
-version.workspace = true
-homepage.workspace = true
-repository.workspace = true
-authors.workspace = true
-license.workspace = true
-keywords.workspace = true
-include.workspace = true
-edition.workspace = true
-rust-version.workspace = true
+version = { workspace = true }
+description = "Vortex in memory columnar data format"
+homepage = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+keywords = { workspace = true }
+include = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
 
 [dependencies]
 arrow-buffer = { workspace = true }
 flatbuffers = { workspace = true }
+flexbuffers = { workspace = true }
 half = { workspace = true }
+humansize = { workspace = true }
+linkme = { workspace = true }
 paste = { workspace = true }
-vortex-array = { path = "../vortex-array" }
-vortex-error = { path = "../vortex-error" }
+serde = { workspace = true, features = ["derive"] }
+vortex-array = { path = "../vortex-array", features = ["serde"] }
+vortex-error = { path = "../vortex-error", features = ["flexbuffers"] }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }
-vortex-schema = { path = "../vortex-schema" }
+vortex-schema = { path = "../vortex-schema", features = ["serde"] }
 
 [lints]
 workspace = true