From ff69266e75c5e1b91da4e1b0f173cac19cfa97e3 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sat, 14 Dec 2024 15:52:00 +0000 Subject: [PATCH 1/6] Message cleanup --- Cargo.lock | 1 + vortex-array/src/data/viewed.rs | 6 +- vortex-file/src/read/builder/initial_read.rs | 4 +- vortex-file/src/read/cache.rs | 19 +- vortex-file/src/read/layouts/chunked.rs | 2 +- vortex-file/src/read/layouts/flat.rs | 2 +- vortex-file/src/write/layout.rs | 4 +- vortex-file/src/write/postscript.rs | 4 +- vortex-file/src/write/writer.rs | 34 +- vortex-flatbuffers/Cargo.toml | 4 +- .../flatbuffers/vortex-array/array.fbs | 24 +- .../flatbuffers/vortex-serde/message.fbs | 28 +- vortex-flatbuffers/src/generated/array.rs | 452 ++++++++----- vortex-flatbuffers/src/generated/dtype.rs | 126 ++-- vortex-flatbuffers/src/generated/footer.rs | 30 +- vortex-flatbuffers/src/generated/message.rs | 593 ++---------------- vortex-flatbuffers/src/generated/scalar.rs | 30 +- vortex-flatbuffers/src/lib.rs | 18 + vortex-ipc/src/messages/mod.rs | 166 +++-- vortex-ipc/src/messages/reader.rs | 81 +-- vortex-ipc/src/messages/writer.rs | 63 +- vortex-ipc/src/stream_reader/mod.rs | 2 +- vortex-ipc/src/stream_writer/mod.rs | 6 +- 23 files changed, 661 insertions(+), 1038 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd278c176..5a4c741b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5006,6 +5006,7 @@ name = "vortex-flatbuffers" version = "0.21.0" dependencies = [ "flatbuffers", + "vortex-buffer", ] [[package]] diff --git a/vortex-array/src/data/viewed.rs b/vortex-array/src/data/viewed.rs index bd1f3bad2..cc195316f 100644 --- a/vortex-array/src/data/viewed.rs +++ b/vortex-array/src/data/viewed.rs @@ -105,7 +105,11 @@ impl ViewedArrayData { pub fn buffer(&self) -> Option<&Buffer> { self.flatbuffer() - .buffer_index() + .buffers() + .and_then(|buffers| { + assert!(buffers.len() <= 1, "Array: expected at most one buffer"); + (buffers.len() > 0).then(|| buffers.get(0) as usize) + }) .map(|idx| &self.buffers[usize::try_from(idx).vortex_unwrap()]) } } diff --git a/vortex-file/src/read/builder/initial_read.rs b/vortex-file/src/read/builder/initial_read.rs index 1c2fe4363..c3d531d66 100644 --- a/vortex-file/src/read/builder/initial_read.rs +++ b/vortex-file/src/read/builder/initial_read.rs @@ -3,7 +3,7 @@ use core::ops::Range; use bytes::Bytes; use flatbuffers::{root, root_unchecked}; use vortex_error::{vortex_bail, vortex_err, VortexResult, VortexUnwrap}; -use vortex_flatbuffers::{footer, message}; +use vortex_flatbuffers::{dtype as fbd, footer}; use vortex_io::VortexReadAt; use crate::{LazyDType, EOF_SIZE, INITIAL_READ_SIZE, MAGIC_BYTES, VERSION}; @@ -136,7 +136,7 @@ pub async fn read_initial_bytes( // validate the schema and layout let schema_loc = (schema_offset - initial_read_offset) as usize; let layout_loc = (layout_offset - initial_read_offset) as usize; - root::(&buf[schema_loc..layout_loc])?; + root::(&buf[schema_loc..layout_loc])?; root::(&buf[layout_loc..ps_loc])?; Ok(InitialRead { diff --git a/vortex-file/src/read/cache.rs b/vortex-file/src/read/cache.rs index 561e7b318..8e978ec3b 100644 --- a/vortex-file/src/read/cache.rs +++ b/vortex-file/src/read/cache.rs @@ -9,8 +9,7 @@ use vortex_dtype::field::Field; use vortex_dtype::flatbuffers::{extract_field, project_and_deserialize, resolve_field}; use vortex_dtype::{DType, FieldNames}; use vortex_error::{vortex_bail, vortex_err, vortex_panic, VortexResult}; -use vortex_flatbuffers::dtype::Struct_; -use vortex_flatbuffers::message; +use vortex_flatbuffers::dtype as fbd; use crate::read::projection::Projection; use crate::read::{LayoutPartId, MessageId}; @@ -238,10 +237,7 @@ fn field_names(bytes: &[u8], dtype_field: &SerializedDTypeField) -> VortexResult } fn project_dtype_bytes(bytes: &[u8], dtype_field: &SerializedDTypeField) -> VortexResult { - let fb_dtype = fb_schema(bytes) - .dtype() - .ok_or_else(|| vortex_err!(InvalidSerde: "Schema missing DType"))?; - + let fb_dtype = fb_dtype(bytes); match dtype_field { SerializedDTypeField::Projection(projection) => match projection { Projection::All => DType::try_from(fb_dtype), @@ -251,15 +247,14 @@ fn project_dtype_bytes(bytes: &[u8], dtype_field: &SerializedDTypeField) -> Vort } } -fn fb_struct(bytes: &[u8]) -> VortexResult { - fb_schema(bytes) - .dtype() - .and_then(|d| d.type__as_struct_()) +fn fb_struct(bytes: &[u8]) -> VortexResult { + fb_dtype(bytes) + .type__as_struct_() .ok_or_else(|| vortex_err!("The top-level type should be a struct")) } -fn fb_schema(bytes: &[u8]) -> message::Schema { - unsafe { root_unchecked::(bytes) } +fn fb_dtype(bytes: &[u8]) -> fbd::DType { + unsafe { root_unchecked::(bytes) } } #[derive(Debug, Clone)] diff --git a/vortex-file/src/read/layouts/chunked.rs b/vortex-file/src/read/layouts/chunked.rs index 990807828..bd1c7cf90 100644 --- a/vortex-file/src/read/layouts/chunked.rs +++ b/vortex-file/src/read/layouts/chunked.rs @@ -434,7 +434,7 @@ mod tests { while let Some(chunk) = chunk_stream.try_next().await.unwrap() { row_offset += chunk.len() as u64; row_offsets.push(row_offset); - writer.write_batch(chunk).await.unwrap(); + writer.write_array(chunk).await.unwrap(); byte_offsets.push(writer.tell()); } let flat_layouts = byte_offsets diff --git a/vortex-file/src/read/layouts/flat.rs b/vortex-file/src/read/layouts/flat.rs index 0c1a17689..9a525641a 100644 --- a/vortex-file/src/read/layouts/flat.rs +++ b/vortex-file/src/read/layouts/flat.rs @@ -136,7 +136,7 @@ mod tests { let mut writer = MessageWriter::new(Vec::new()); let array = PrimitiveArray::from((0..100).collect::>()).into_array(); let len = array.len(); - writer.write_batch(array).await.unwrap(); + writer.write_array(array).await.unwrap(); let written = writer.into_inner(); let projection_scan = Scan::empty(); diff --git a/vortex-file/src/write/layout.rs b/vortex-file/src/write/layout.rs index 5ed8cc085..b79bec645 100644 --- a/vortex-file/src/write/layout.rs +++ b/vortex-file/src/write/layout.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use flatbuffers::{FlatBufferBuilder, WIPOffset}; -use vortex_flatbuffers::{footer as fb, WriteFlatBuffer}; +use vortex_flatbuffers::{footer as fb, FlatBufferRoot, WriteFlatBuffer}; use vortex_ipc::stream_writer::ByteRange; use crate::{LayoutId, CHUNKED_LAYOUT_ID, COLUMNAR_LAYOUT_ID, FLAT_LAYOUT_ID}; @@ -49,6 +49,8 @@ impl LayoutSpec { } } +impl FlatBufferRoot for LayoutSpec {} + impl WriteFlatBuffer for LayoutSpec { type Target<'a> = fb::Layout<'a>; diff --git a/vortex-file/src/write/postscript.rs b/vortex-file/src/write/postscript.rs index 080d07fdc..4a7db7250 100644 --- a/vortex-file/src/write/postscript.rs +++ b/vortex-file/src/write/postscript.rs @@ -1,6 +1,6 @@ use flatbuffers::{FlatBufferBuilder, WIPOffset}; use vortex_error::{vortex_bail, VortexResult}; -use vortex_flatbuffers::{footer as fb, WriteFlatBuffer}; +use vortex_flatbuffers::{footer as fb, FlatBufferRoot, WriteFlatBuffer}; #[derive(Debug)] pub struct Postscript { @@ -24,6 +24,8 @@ impl Postscript { } } +impl FlatBufferRoot for Postscript {} + impl WriteFlatBuffer for Postscript { type Target<'a> = fb::Postscript<'a>; diff --git a/vortex-file/src/write/writer.rs b/vortex-file/src/write/writer.rs index 6d845f38b..0c48616b9 100644 --- a/vortex-file/src/write/writer.rs +++ b/vortex-file/src/write/writer.rs @@ -3,20 +3,17 @@ use std::{io, iter, mem}; use bytes::Bytes; -use flatbuffers::FlatBufferBuilder; use futures::TryStreamExt; use itertools::Itertools; use vortex_array::array::{ChunkedArray, StructArray}; use vortex_array::stats::{as_stat_bitset_bytes, ArrayStatistics, Stat}; use vortex_array::stream::ArrayStream; use vortex_array::{ArrayData, ArrayLen}; -use vortex_buffer::io_buf::IoBuf; use vortex_dtype::DType; use vortex_error::{vortex_bail, vortex_err, VortexExpect as _, VortexResult}; -use vortex_flatbuffers::WriteFlatBuffer; +use vortex_flatbuffers::{FlatBufferRoot, FlatBufferToBytes, WriteFlatBuffer}; use vortex_io::VortexWrite; use vortex_ipc::messages::writer::MessageWriter; -use vortex_ipc::messages::IPCSchema; use vortex_ipc::stream_writer::ByteRange; use crate::write::postscript::Postscript; @@ -138,7 +135,7 @@ impl VortexFileWriter { pub async fn finalize(mut self) -> VortexResult { let top_level_layout = self.write_metadata_arrays().await?; - let schema_offset = self.msgs.tell(); + let dtype_offset = self.msgs.tell(); // we want to write raw flatbuffers from here on out, not messages let mut writer = self.msgs.into_inner(); @@ -152,15 +149,14 @@ impl VortexFileWriter { // we write an IPCSchema instead of a DType, which allows us to evolve / add to the schema later // these bytes get deserialized as message::Schema // NB: we don't wrap the IPCSchema in an IPCMessage, because we record the lengths/offsets in the footer - let schema = IPCSchema(&dtype); - let schema_len = write_fb_raw(&mut writer, schema).await?; - schema_offset + schema_len + let dtype_len = write_fb_raw(&mut writer, dtype).await?; + dtype_offset + dtype_len }; // write the layout write_fb_raw(&mut writer, top_level_layout).await?; - let footer = Postscript::try_new(schema_offset, layout_offset)?; + let footer = Postscript::try_new(dtype_offset, layout_offset)?; let footer_len = write_fb_raw(&mut writer, footer).await?; if footer_len > MAX_FOOTER_SIZE as u64 { vortex_bail!( @@ -182,20 +178,14 @@ impl VortexFileWriter { } /// Write a flatbuffer to a writer and return the number of bytes written. -async fn write_fb_raw( +async fn write_fb_raw( writer: &mut W, fb: F, ) -> io::Result { - let mut fbb = FlatBufferBuilder::new(); - let ps_fb = fb.write_flatbuffer(&mut fbb); - fbb.finish_minimal(ps_fb); - - let (buffer, buffer_begin) = fbb.collapse(); - let buffer_end = buffer.len(); - - let bytes = buffer.slice_owned(buffer_begin..buffer_end); - writer.write_all(bytes).await?; - Ok((buffer_end - buffer_begin) as u64) + let buffer = fb.write_flatbuffer_bytes(); + let buffer_len = buffer.len(); + writer.write_all(buffer).await?; + Ok(buffer_len as u64) } struct ColumnWriter { @@ -240,7 +230,7 @@ impl ColumnWriter { // clear the stats that we don't want to serialize into the file retain_only_stats(&chunk, STATS_TO_WRITE); - msgs.write_batch(chunk).await?; + msgs.write_array(chunk).await?; offsets.push(msgs.tell()); row_offsets.push(rows_written); } @@ -280,7 +270,7 @@ impl ColumnWriter { let stat_bitset = as_stat_bitset_bytes(&present_stats); let metadata_array_begin = msgs.tell(); - msgs.write_batch(metadata_array).await?; + msgs.write_array(metadata_array).await?; let metadata_array_end = msgs.tell(); let layouts = iter::once(LayoutSpec::flat( diff --git a/vortex-flatbuffers/Cargo.toml b/vortex-flatbuffers/Cargo.toml index 15ddd73e6..4444baf7f 100644 --- a/vortex-flatbuffers/Cargo.toml +++ b/vortex-flatbuffers/Cargo.toml @@ -18,10 +18,12 @@ dtype = [] scalar = ["dtype"] array = ["dtype", "scalar"] ipc = ["array"] -file = ["ipc"] +file = ["array", "layout"] +layout = [] [dependencies] flatbuffers = { workspace = true } +vortex-buffer = { workspace = true } [lints] workspace = true diff --git a/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs b/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs index 133b00774..5d0148d7f 100644 --- a/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs +++ b/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs @@ -1,16 +1,22 @@ include "vortex-scalar/scalar.fbs"; -enum Version: uint8 { - V0 = 0, +/// An ArrayData describes the hierarchy of an array as well as the locations of the data buffers that appear +/// immediately after the message in the byte stream. +table ArrayData { + /// The array's hierarchical definition. + array: Array; + /// The row count of the array. + row_count: uint64; + /// The locations of the data buffers of the array, in ascending order of offset. + buffers: [Buffer]; } table Array { - version: Version = V0; - buffer_index: uint64 = null; encoding: uint16; metadata: [ubyte]; - stats: ArrayStats; children: [Array]; + buffers: [uint16]; + stats: ArrayStats; } table ArrayStats { @@ -27,5 +33,11 @@ table ArrayStats { uncompressed_size_in_bytes: uint64 = null; } +table Buffer { + /// The length of the buffer in bytes. + length: uint64; + /// The length of any padding bytes written immediately following the buffer. + padding: uint16; +} -root_type Array; +root_type ArrayData; \ No newline at end of file diff --git a/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs b/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs index b30572f7b..65baf7b63 100644 --- a/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs +++ b/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs @@ -9,32 +9,10 @@ enum Compression: uint8 { None = 0, } -table Schema { - dtype: DType; -} - -struct Buffer { - offset: uint64; - padding: uint16; - compression: Compression; -} - -table Batch { - array: Array; - length: uint64; - buffers: [Buffer]; - buffer_size: uint64; -} - -table Page { - buffer_size: uint32; - padding: uint16; -} - union MessageHeader { - Schema, - Batch, - Page, + ArrayData, + Buffer, + DType, } table Message { diff --git a/vortex-flatbuffers/src/generated/array.rs b/vortex-flatbuffers/src/generated/array.rs index 18c00caed..c2c4e346c 100644 --- a/vortex-flatbuffers/src/generated/array.rs +++ b/vortex-flatbuffers/src/generated/array.rs @@ -11,87 +11,142 @@ use core::cmp::Ordering; extern crate flatbuffers; use self::flatbuffers::{EndianScalar, Follow}; -#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")] -pub const ENUM_MIN_VERSION: u8 = 0; -#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")] -pub const ENUM_MAX_VERSION: u8 = 0; -#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")] -#[allow(non_camel_case_types)] -pub const ENUM_VALUES_VERSION: [Version; 1] = [ - Version::V0, -]; - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -#[repr(transparent)] -pub struct Version(pub u8); -#[allow(non_upper_case_globals)] -impl Version { - pub const V0: Self = Self(0); - - pub const ENUM_MIN: u8 = 0; - pub const ENUM_MAX: u8 = 0; - pub const ENUM_VALUES: &'static [Self] = &[ - Self::V0, - ]; - /// Returns the variant's name or "" if unknown. - pub fn variant_name(self) -> Option<&'static str> { - match self { - Self::V0 => Some("V0"), - _ => None, - } - } -} -impl core::fmt::Debug for Version { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - if let Some(name) = self.variant_name() { - f.write_str(name) - } else { - f.write_fmt(format_args!("", self.0)) - } - } +pub enum ArrayDataOffset {} +#[derive(Copy, Clone, PartialEq)] + +/// An ArrayData describes the hierarchy of an array as well as the locations of the data buffers that appear +/// immediately after the message in the byte stream. +pub struct ArrayData<'a> { + pub _tab: flatbuffers::Table<'a>, } -impl<'a> flatbuffers::Follow<'a> for Version { - type Inner = Self; + +impl<'a> flatbuffers::Follow<'a> for ArrayData<'a> { + type Inner = ArrayData<'a>; #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - let b = flatbuffers::read_scalar_at::(buf, loc); - Self(b) + Self { _tab: flatbuffers::Table::new(buf, loc) } } } -impl flatbuffers::Push for Version { - type Output = Version; - #[inline] - unsafe fn push(&self, dst: &mut [u8], _written_len: usize) { - flatbuffers::emplace_scalar::(dst, self.0); - } -} +impl<'a> ArrayData<'a> { + pub const VT_ARRAY: flatbuffers::VOffsetT = 4; + pub const VT_ROW_COUNT: flatbuffers::VOffsetT = 6; + pub const VT_BUFFERS: flatbuffers::VOffsetT = 8; -impl flatbuffers::EndianScalar for Version { - type Scalar = u8; #[inline] - fn to_little_endian(self) -> u8 { - self.0.to_le() + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ArrayData { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ArrayDataArgs<'args> + ) -> flatbuffers::WIPOffset> { + let mut builder = ArrayDataBuilder::new(_fbb); + builder.add_row_count(args.row_count); + if let Some(x) = args.buffers { builder.add_buffers(x); } + if let Some(x) = args.array { builder.add_array(x); } + builder.finish() + } + + + /// The array's hierarchical definition. + #[inline] + pub fn array(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::>(ArrayData::VT_ARRAY, None)} + } + /// The row count of the array. + #[inline] + pub fn row_count(&self) -> u64 { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::(ArrayData::VT_ROW_COUNT, Some(0)).unwrap()} } + /// The locations of the data buffers of the array, in ascending order of offset. #[inline] - #[allow(clippy::wrong_self_convention)] - fn from_little_endian(v: u8) -> Self { - let b = u8::from_le(v); - Self(b) + pub fn buffers(&self) -> Option>>> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::>>>(ArrayData::VT_BUFFERS, None)} } } -impl<'a> flatbuffers::Verifiable for Version { +impl flatbuffers::Verifiable for ArrayData<'_> { #[inline] fn run_verifier( v: &mut flatbuffers::Verifier, pos: usize ) -> Result<(), flatbuffers::InvalidFlatbuffer> { use self::flatbuffers::Verifiable; - u8::run_verifier(v, pos) + v.visit_table(pos)? + .visit_field::>("array", Self::VT_ARRAY, false)? + .visit_field::("row_count", Self::VT_ROW_COUNT, false)? + .visit_field::>>>("buffers", Self::VT_BUFFERS, false)? + .finish(); + Ok(()) + } +} +pub struct ArrayDataArgs<'a> { + pub array: Option>>, + pub row_count: u64, + pub buffers: Option>>>>, +} +impl<'a> Default for ArrayDataArgs<'a> { + #[inline] + fn default() -> Self { + ArrayDataArgs { + array: None, + row_count: 0, + buffers: None, + } + } +} + +pub struct ArrayDataBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> ArrayDataBuilder<'a, 'b> { + #[inline] + pub fn add_array(&mut self, array: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(ArrayData::VT_ARRAY, array); + } + #[inline] + pub fn add_row_count(&mut self, row_count: u64) { + self.fbb_.push_slot::(ArrayData::VT_ROW_COUNT, row_count, 0); + } + #[inline] + pub fn add_buffers(&mut self, buffers: flatbuffers::WIPOffset>>>) { + self.fbb_.push_slot_always::>(ArrayData::VT_BUFFERS, buffers); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ArrayDataBuilder<'a, 'b> { + let start = _fbb.start_table(); + ArrayDataBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) } } -impl flatbuffers::SimpleToVerifyInSlice for Version {} +impl core::fmt::Debug for ArrayData<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("ArrayData"); + ds.field("array", &self.array()); + ds.field("row_count", &self.row_count()); + ds.field("buffers", &self.buffers()); + ds.finish() + } +} pub enum ArrayOffset {} #[derive(Copy, Clone, PartialEq)] @@ -108,60 +163,58 @@ impl<'a> flatbuffers::Follow<'a> for Array<'a> { } impl<'a> Array<'a> { - pub const VT_VERSION: flatbuffers::VOffsetT = 4; - pub const VT_BUFFER_INDEX: flatbuffers::VOffsetT = 6; - pub const VT_ENCODING: flatbuffers::VOffsetT = 8; - pub const VT_METADATA: flatbuffers::VOffsetT = 10; + pub const VT_ENCODING: flatbuffers::VOffsetT = 4; + pub const VT_METADATA: flatbuffers::VOffsetT = 6; + pub const VT_CHILDREN: flatbuffers::VOffsetT = 8; + pub const VT_BUFFERS: flatbuffers::VOffsetT = 10; pub const VT_STATS: flatbuffers::VOffsetT = 12; - pub const VT_CHILDREN: flatbuffers::VOffsetT = 14; #[inline] pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { Array { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args ArrayArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ArrayBuilder::new(_fbb); - if let Some(x) = args.buffer_index { builder.add_buffer_index(x); } - if let Some(x) = args.children { builder.add_children(x); } if let Some(x) = args.stats { builder.add_stats(x); } + if let Some(x) = args.buffers { builder.add_buffers(x); } + if let Some(x) = args.children { builder.add_children(x); } if let Some(x) = args.metadata { builder.add_metadata(x); } builder.add_encoding(args.encoding); - builder.add_version(args.version); builder.finish() } #[inline] - pub fn version(&self) -> Version { + pub fn encoding(&self) -> u16 { // Safety: // Created from valid Table for this object // which contains a valid value in this slot - unsafe { self._tab.get::(Array::VT_VERSION, Some(Version::V0)).unwrap()} + unsafe { self._tab.get::(Array::VT_ENCODING, Some(0)).unwrap()} } #[inline] - pub fn buffer_index(&self) -> Option { + pub fn metadata(&self) -> Option> { // Safety: // Created from valid Table for this object // which contains a valid value in this slot - unsafe { self._tab.get::(Array::VT_BUFFER_INDEX, None)} + unsafe { self._tab.get::>>(Array::VT_METADATA, None)} } #[inline] - pub fn encoding(&self) -> u16 { + pub fn children(&self) -> Option>>> { // Safety: // Created from valid Table for this object // which contains a valid value in this slot - unsafe { self._tab.get::(Array::VT_ENCODING, Some(0)).unwrap()} + unsafe { self._tab.get::>>>(Array::VT_CHILDREN, None)} } #[inline] - pub fn metadata(&self) -> Option> { + pub fn buffers(&self) -> Option> { // Safety: // Created from valid Table for this object // which contains a valid value in this slot - unsafe { self._tab.get::>>(Array::VT_METADATA, None)} + unsafe { self._tab.get::>>(Array::VT_BUFFERS, None)} } #[inline] pub fn stats(&self) -> Option> { @@ -170,13 +223,6 @@ impl<'a> Array<'a> { // which contains a valid value in this slot unsafe { self._tab.get::>(Array::VT_STATS, None)} } - #[inline] - pub fn children(&self) -> Option>>> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::>>>(Array::VT_CHILDREN, None)} - } } impl flatbuffers::Verifiable for Array<'_> { @@ -186,51 +232,40 @@ impl flatbuffers::Verifiable for Array<'_> { ) -> Result<(), flatbuffers::InvalidFlatbuffer> { use self::flatbuffers::Verifiable; v.visit_table(pos)? - .visit_field::("version", Self::VT_VERSION, false)? - .visit_field::("buffer_index", Self::VT_BUFFER_INDEX, false)? .visit_field::("encoding", Self::VT_ENCODING, false)? .visit_field::>>("metadata", Self::VT_METADATA, false)? - .visit_field::>("stats", Self::VT_STATS, false)? .visit_field::>>>("children", Self::VT_CHILDREN, false)? + .visit_field::>>("buffers", Self::VT_BUFFERS, false)? + .visit_field::>("stats", Self::VT_STATS, false)? .finish(); Ok(()) } } pub struct ArrayArgs<'a> { - pub version: Version, - pub buffer_index: Option, pub encoding: u16, pub metadata: Option>>, - pub stats: Option>>, pub children: Option>>>>, + pub buffers: Option>>, + pub stats: Option>>, } impl<'a> Default for ArrayArgs<'a> { #[inline] fn default() -> Self { ArrayArgs { - version: Version::V0, - buffer_index: None, encoding: 0, metadata: None, - stats: None, children: None, + buffers: None, + stats: None, } } } -pub struct ArrayBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct ArrayBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayBuilder<'a, 'b, A> { - #[inline] - pub fn add_version(&mut self, version: Version) { - self.fbb_.push_slot::(Array::VT_VERSION, version, Version::V0); - } - #[inline] - pub fn add_buffer_index(&mut self, buffer_index: u64) { - self.fbb_.push_slot_always::(Array::VT_BUFFER_INDEX, buffer_index); - } +impl<'a: 'b, 'b> ArrayBuilder<'a, 'b> { #[inline] pub fn add_encoding(&mut self, encoding: u16) { self.fbb_.push_slot::(Array::VT_ENCODING, encoding, 0); @@ -240,15 +275,19 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayBuilder<'a, 'b, A> { self.fbb_.push_slot_always::>(Array::VT_METADATA, metadata); } #[inline] - pub fn add_stats(&mut self, stats: flatbuffers::WIPOffset>) { - self.fbb_.push_slot_always::>(Array::VT_STATS, stats); - } - #[inline] pub fn add_children(&mut self, children: flatbuffers::WIPOffset>>>) { self.fbb_.push_slot_always::>(Array::VT_CHILDREN, children); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ArrayBuilder<'a, 'b, A> { + pub fn add_buffers(&mut self, buffers: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(Array::VT_BUFFERS, buffers); + } + #[inline] + pub fn add_stats(&mut self, stats: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(Array::VT_STATS, stats); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ArrayBuilder<'a, 'b> { let start = _fbb.start_table(); ArrayBuilder { fbb_: _fbb, @@ -265,12 +304,11 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayBuilder<'a, 'b, A> { impl core::fmt::Debug for Array<'_> { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let mut ds = f.debug_struct("Array"); - ds.field("version", &self.version()); - ds.field("buffer_index", &self.buffer_index()); ds.field("encoding", &self.encoding()); ds.field("metadata", &self.metadata()); - ds.field("stats", &self.stats()); ds.field("children", &self.children()); + ds.field("buffers", &self.buffers()); + ds.field("stats", &self.stats()); ds.finish() } } @@ -307,8 +345,8 @@ impl<'a> ArrayStats<'a> { ArrayStats { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args ArrayStatsArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ArrayStatsBuilder::new(_fbb); @@ -460,11 +498,11 @@ impl<'a> Default for ArrayStatsArgs<'a> { } } -pub struct ArrayStatsBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct ArrayStatsBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayStatsBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> ArrayStatsBuilder<'a, 'b> { #[inline] pub fn add_min(&mut self, min: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(ArrayStats::VT_MIN, min); @@ -510,7 +548,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayStatsBuilder<'a, 'b, A> { self.fbb_.push_slot_always::(ArrayStats::VT_UNCOMPRESSED_SIZE_IN_BYTES, uncompressed_size_in_bytes); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ArrayStatsBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ArrayStatsBuilder<'a, 'b> { let start = _fbb.start_table(); ArrayStatsBuilder { fbb_: _fbb, @@ -541,74 +579,190 @@ impl core::fmt::Debug for ArrayStats<'_> { ds.finish() } } +pub enum BufferOffset {} +#[derive(Copy, Clone, PartialEq)] + +pub struct Buffer<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for Buffer<'a> { + type Inner = Buffer<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { _tab: flatbuffers::Table::new(buf, loc) } + } +} + +impl<'a> Buffer<'a> { + pub const VT_LENGTH: flatbuffers::VOffsetT = 4; + pub const VT_PADDING: flatbuffers::VOffsetT = 6; + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Buffer { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args BufferArgs + ) -> flatbuffers::WIPOffset> { + let mut builder = BufferBuilder::new(_fbb); + builder.add_length(args.length); + builder.add_padding(args.padding); + builder.finish() + } + + + /// The length of the buffer in bytes. + #[inline] + pub fn length(&self) -> u64 { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::(Buffer::VT_LENGTH, Some(0)).unwrap()} + } + /// The length of any padding bytes written immediately following the buffer. + #[inline] + pub fn padding(&self) -> u16 { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::(Buffer::VT_PADDING, Some(0)).unwrap()} + } +} + +impl flatbuffers::Verifiable for Buffer<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, pos: usize + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::("length", Self::VT_LENGTH, false)? + .visit_field::("padding", Self::VT_PADDING, false)? + .finish(); + Ok(()) + } +} +pub struct BufferArgs { + pub length: u64, + pub padding: u16, +} +impl<'a> Default for BufferArgs { + #[inline] + fn default() -> Self { + BufferArgs { + length: 0, + padding: 0, + } + } +} + +pub struct BufferBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> BufferBuilder<'a, 'b> { + #[inline] + pub fn add_length(&mut self, length: u64) { + self.fbb_.push_slot::(Buffer::VT_LENGTH, length, 0); + } + #[inline] + pub fn add_padding(&mut self, padding: u16) { + self.fbb_.push_slot::(Buffer::VT_PADDING, padding, 0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BufferBuilder<'a, 'b> { + let start = _fbb.start_table(); + BufferBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +impl core::fmt::Debug for Buffer<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("Buffer"); + ds.field("length", &self.length()); + ds.field("padding", &self.padding()); + ds.finish() + } +} #[inline] -/// Verifies that a buffer of bytes contains a `Array` +/// Verifies that a buffer of bytes contains a `ArrayData` /// and returns it. /// Note that verification is still experimental and may not /// catch every error, or be maximally performant. For the /// previous, unchecked, behavior use -/// `root_as_array_unchecked`. -pub fn root_as_array(buf: &[u8]) -> Result { - flatbuffers::root::(buf) +/// `root_as_array_data_unchecked`. +pub fn root_as_array_data(buf: &[u8]) -> Result { + flatbuffers::root::(buf) } #[inline] /// Verifies that a buffer of bytes contains a size prefixed -/// `Array` and returns it. +/// `ArrayData` and returns it. /// Note that verification is still experimental and may not /// catch every error, or be maximally performant. For the /// previous, unchecked, behavior use -/// `size_prefixed_root_as_array_unchecked`. -pub fn size_prefixed_root_as_array(buf: &[u8]) -> Result { - flatbuffers::size_prefixed_root::(buf) +/// `size_prefixed_root_as_array_data_unchecked`. +pub fn size_prefixed_root_as_array_data(buf: &[u8]) -> Result { + flatbuffers::size_prefixed_root::(buf) } #[inline] /// Verifies, with the given options, that a buffer of bytes -/// contains a `Array` and returns it. +/// contains a `ArrayData` and returns it. /// Note that verification is still experimental and may not /// catch every error, or be maximally performant. For the /// previous, unchecked, behavior use -/// `root_as_array_unchecked`. -pub fn root_as_array_with_opts<'b, 'o>( +/// `root_as_array_data_unchecked`. +pub fn root_as_array_data_with_opts<'b, 'o>( opts: &'o flatbuffers::VerifierOptions, buf: &'b [u8], -) -> Result, flatbuffers::InvalidFlatbuffer> { - flatbuffers::root_with_opts::>(opts, buf) +) -> Result, flatbuffers::InvalidFlatbuffer> { + flatbuffers::root_with_opts::>(opts, buf) } #[inline] /// Verifies, with the given verifier options, that a buffer of -/// bytes contains a size prefixed `Array` and returns +/// bytes contains a size prefixed `ArrayData` and returns /// it. Note that verification is still experimental and may not /// catch every error, or be maximally performant. For the /// previous, unchecked, behavior use -/// `root_as_array_unchecked`. -pub fn size_prefixed_root_as_array_with_opts<'b, 'o>( +/// `root_as_array_data_unchecked`. +pub fn size_prefixed_root_as_array_data_with_opts<'b, 'o>( opts: &'o flatbuffers::VerifierOptions, buf: &'b [u8], -) -> Result, flatbuffers::InvalidFlatbuffer> { - flatbuffers::size_prefixed_root_with_opts::>(opts, buf) +) -> Result, flatbuffers::InvalidFlatbuffer> { + flatbuffers::size_prefixed_root_with_opts::>(opts, buf) } #[inline] -/// Assumes, without verification, that a buffer of bytes contains a Array and returns it. +/// Assumes, without verification, that a buffer of bytes contains a ArrayData and returns it. /// # Safety -/// Callers must trust the given bytes do indeed contain a valid `Array`. -pub unsafe fn root_as_array_unchecked(buf: &[u8]) -> Array { - flatbuffers::root_unchecked::(buf) +/// Callers must trust the given bytes do indeed contain a valid `ArrayData`. +pub unsafe fn root_as_array_data_unchecked(buf: &[u8]) -> ArrayData { + flatbuffers::root_unchecked::(buf) } #[inline] -/// Assumes, without verification, that a buffer of bytes contains a size prefixed Array and returns it. +/// Assumes, without verification, that a buffer of bytes contains a size prefixed ArrayData and returns it. /// # Safety -/// Callers must trust the given bytes do indeed contain a valid size prefixed `Array`. -pub unsafe fn size_prefixed_root_as_array_unchecked(buf: &[u8]) -> Array { - flatbuffers::size_prefixed_root_unchecked::(buf) +/// Callers must trust the given bytes do indeed contain a valid size prefixed `ArrayData`. +pub unsafe fn size_prefixed_root_as_array_data_unchecked(buf: &[u8]) -> ArrayData { + flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_array_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - root: flatbuffers::WIPOffset>) { +pub fn finish_array_data_buffer<'a, 'b>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_array_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_array_data_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/dtype.rs b/vortex-flatbuffers/src/generated/dtype.rs index 695eee413..af311ad69 100644 --- a/vortex-flatbuffers/src/generated/dtype.rs +++ b/vortex-flatbuffers/src/generated/dtype.rs @@ -271,8 +271,8 @@ impl<'a> Null<'a> { Null { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, _args: &'args NullArgs ) -> flatbuffers::WIPOffset> { let mut builder = NullBuilder::new(_fbb); @@ -302,13 +302,13 @@ impl<'a> Default for NullArgs { } } -pub struct NullBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct NullBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> NullBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> NullBuilder<'a, 'b> { #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> NullBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> NullBuilder<'a, 'b> { let start = _fbb.start_table(); NullBuilder { fbb_: _fbb, @@ -351,8 +351,8 @@ impl<'a> Bool<'a> { Bool { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args BoolArgs ) -> flatbuffers::WIPOffset> { let mut builder = BoolBuilder::new(_fbb); @@ -394,17 +394,17 @@ impl<'a> Default for BoolArgs { } } -pub struct BoolBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct BoolBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> BoolBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> BoolBuilder<'a, 'b> { #[inline] pub fn add_nullable(&mut self, nullable: bool) { self.fbb_.push_slot::(Bool::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> BoolBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BoolBuilder<'a, 'b> { let start = _fbb.start_table(); BoolBuilder { fbb_: _fbb, @@ -449,8 +449,8 @@ impl<'a> Primitive<'a> { Primitive { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args PrimitiveArgs ) -> flatbuffers::WIPOffset> { let mut builder = PrimitiveBuilder::new(_fbb); @@ -503,11 +503,11 @@ impl<'a> Default for PrimitiveArgs { } } -pub struct PrimitiveBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct PrimitiveBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PrimitiveBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> PrimitiveBuilder<'a, 'b> { #[inline] pub fn add_ptype(&mut self, ptype: PType) { self.fbb_.push_slot::(Primitive::VT_PTYPE, ptype, PType::U8); @@ -517,7 +517,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PrimitiveBuilder<'a, 'b, A> { self.fbb_.push_slot::(Primitive::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PrimitiveBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PrimitiveBuilder<'a, 'b> { let start = _fbb.start_table(); PrimitiveBuilder { fbb_: _fbb, @@ -564,8 +564,8 @@ impl<'a> Decimal<'a> { Decimal { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args DecimalArgs ) -> flatbuffers::WIPOffset> { let mut builder = DecimalBuilder::new(_fbb); @@ -631,11 +631,11 @@ impl<'a> Default for DecimalArgs { } } -pub struct DecimalBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct DecimalBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecimalBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> DecimalBuilder<'a, 'b> { #[inline] pub fn add_precision(&mut self, precision: u8) { self.fbb_.push_slot::(Decimal::VT_PRECISION, precision, 0); @@ -649,7 +649,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecimalBuilder<'a, 'b, A> { self.fbb_.push_slot::(Decimal::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecimalBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DecimalBuilder<'a, 'b> { let start = _fbb.start_table(); DecimalBuilder { fbb_: _fbb, @@ -695,8 +695,8 @@ impl<'a> Utf8<'a> { Utf8 { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args Utf8Args ) -> flatbuffers::WIPOffset> { let mut builder = Utf8Builder::new(_fbb); @@ -738,17 +738,17 @@ impl<'a> Default for Utf8Args { } } -pub struct Utf8Builder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct Utf8Builder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> Utf8Builder<'a, 'b, A> { +impl<'a: 'b, 'b> Utf8Builder<'a, 'b> { #[inline] pub fn add_nullable(&mut self, nullable: bool) { self.fbb_.push_slot::(Utf8::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> Utf8Builder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> Utf8Builder<'a, 'b> { let start = _fbb.start_table(); Utf8Builder { fbb_: _fbb, @@ -792,8 +792,8 @@ impl<'a> Binary<'a> { Binary { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args BinaryArgs ) -> flatbuffers::WIPOffset> { let mut builder = BinaryBuilder::new(_fbb); @@ -835,17 +835,17 @@ impl<'a> Default for BinaryArgs { } } -pub struct BinaryBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct BinaryBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> BinaryBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> BinaryBuilder<'a, 'b> { #[inline] pub fn add_nullable(&mut self, nullable: bool) { self.fbb_.push_slot::(Binary::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> BinaryBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BinaryBuilder<'a, 'b> { let start = _fbb.start_table(); BinaryBuilder { fbb_: _fbb, @@ -891,8 +891,8 @@ impl<'a> Struct_<'a> { Struct_ { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args Struct_Args<'args> ) -> flatbuffers::WIPOffset> { let mut builder = Struct_Builder::new(_fbb); @@ -956,11 +956,11 @@ impl<'a> Default for Struct_Args<'a> { } } -pub struct Struct_Builder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct Struct_Builder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> Struct_Builder<'a, 'b, A> { +impl<'a: 'b, 'b> Struct_Builder<'a, 'b> { #[inline] pub fn add_names(&mut self, names: flatbuffers::WIPOffset>>) { self.fbb_.push_slot_always::>(Struct_::VT_NAMES, names); @@ -974,7 +974,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> Struct_Builder<'a, 'b, A> { self.fbb_.push_slot::(Struct_::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> Struct_Builder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> Struct_Builder<'a, 'b> { let start = _fbb.start_table(); Struct_Builder { fbb_: _fbb, @@ -1021,8 +1021,8 @@ impl<'a> List<'a> { List { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args ListArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ListBuilder::new(_fbb); @@ -1075,11 +1075,11 @@ impl<'a> Default for ListArgs<'a> { } } -pub struct ListBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct ListBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ListBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> ListBuilder<'a, 'b> { #[inline] pub fn add_element_type(&mut self, element_type: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(List::VT_ELEMENT_TYPE, element_type); @@ -1089,7 +1089,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ListBuilder<'a, 'b, A> { self.fbb_.push_slot::(List::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ListBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ListBuilder<'a, 'b> { let start = _fbb.start_table(); ListBuilder { fbb_: _fbb, @@ -1136,8 +1136,8 @@ impl<'a> Extension<'a> { Extension { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args ExtensionArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ExtensionBuilder::new(_fbb); @@ -1201,11 +1201,11 @@ impl<'a> Default for ExtensionArgs<'a> { } } -pub struct ExtensionBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct ExtensionBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ExtensionBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> ExtensionBuilder<'a, 'b> { #[inline] pub fn add_id(&mut self, id: flatbuffers::WIPOffset<&'b str>) { self.fbb_.push_slot_always::>(Extension::VT_ID, id); @@ -1219,7 +1219,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ExtensionBuilder<'a, 'b, A> { self.fbb_.push_slot_always::>(Extension::VT_METADATA, metadata); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ExtensionBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ExtensionBuilder<'a, 'b> { let start = _fbb.start_table(); ExtensionBuilder { fbb_: _fbb, @@ -1266,8 +1266,8 @@ impl<'a> DType<'a> { DType { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args DTypeArgs ) -> flatbuffers::WIPOffset> { let mut builder = DTypeBuilder::new(_fbb); @@ -1467,11 +1467,11 @@ impl<'a> Default for DTypeArgs { } } -pub struct DTypeBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct DTypeBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DTypeBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> DTypeBuilder<'a, 'b> { #[inline] pub fn add_type_type(&mut self, type_type: Type) { self.fbb_.push_slot::(DType::VT_TYPE_TYPE, type_type, Type::NONE); @@ -1481,7 +1481,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DTypeBuilder<'a, 'b, A> { self.fbb_.push_slot_always::>(DType::VT_TYPE_, type_); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DTypeBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DTypeBuilder<'a, 'b> { let start = _fbb.start_table(); DTypeBuilder { fbb_: _fbb, @@ -1632,13 +1632,13 @@ pub unsafe fn size_prefixed_root_as_dtype_unchecked(buf: &[u8]) -> DType { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_dtype_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub fn finish_dtype_buffer<'a, 'b>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_dtype_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_dtype_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/footer.rs b/vortex-flatbuffers/src/generated/footer.rs index 33551bacb..d242f5bc6 100644 --- a/vortex-flatbuffers/src/generated/footer.rs +++ b/vortex-flatbuffers/src/generated/footer.rs @@ -176,8 +176,8 @@ impl<'a> Layout<'a> { Layout { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args LayoutArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = LayoutBuilder::new(_fbb); @@ -263,11 +263,11 @@ impl<'a> Default for LayoutArgs<'a> { } } -pub struct LayoutBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct LayoutBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> LayoutBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> LayoutBuilder<'a, 'b> { #[inline] pub fn add_encoding(&mut self, encoding: u16) { self.fbb_.push_slot::(Layout::VT_ENCODING, encoding, 0); @@ -289,7 +289,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> LayoutBuilder<'a, 'b, A> { self.fbb_.push_slot_always::>(Layout::VT_METADATA, metadata); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> LayoutBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> LayoutBuilder<'a, 'b> { let start = _fbb.start_table(); LayoutBuilder { fbb_: _fbb, @@ -350,8 +350,8 @@ impl<'a> Postscript<'a> { Postscript { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args PostscriptArgs ) -> flatbuffers::WIPOffset> { let mut builder = PostscriptBuilder::new(_fbb); @@ -404,11 +404,11 @@ impl<'a> Default for PostscriptArgs { } } -pub struct PostscriptBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct PostscriptBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PostscriptBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> PostscriptBuilder<'a, 'b> { #[inline] pub fn add_schema_offset(&mut self, schema_offset: u64) { self.fbb_.push_slot::(Postscript::VT_SCHEMA_OFFSET, schema_offset, 0); @@ -418,7 +418,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PostscriptBuilder<'a, 'b, A> { self.fbb_.push_slot::(Postscript::VT_LAYOUT_OFFSET, layout_offset, 0); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PostscriptBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PostscriptBuilder<'a, 'b> { let start = _fbb.start_table(); PostscriptBuilder { fbb_: _fbb, @@ -501,13 +501,13 @@ pub unsafe fn size_prefixed_root_as_postscript_unchecked(buf: &[u8]) -> Postscri flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_postscript_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub fn finish_postscript_buffer<'a, 'b>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_postscript_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_postscript_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/message.rs b/vortex-flatbuffers/src/generated/message.rs index 6aaaf0919..9c73bff11 100644 --- a/vortex-flatbuffers/src/generated/message.rs +++ b/vortex-flatbuffers/src/generated/message.rs @@ -182,9 +182,9 @@ pub const ENUM_MAX_MESSAGE_HEADER: u8 = 3; #[allow(non_camel_case_types)] pub const ENUM_VALUES_MESSAGE_HEADER: [MessageHeader; 4] = [ MessageHeader::NONE, - MessageHeader::Schema, - MessageHeader::Batch, - MessageHeader::Page, + MessageHeader::ArrayData, + MessageHeader::Buffer, + MessageHeader::DType, ]; #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] @@ -193,25 +193,25 @@ pub struct MessageHeader(pub u8); #[allow(non_upper_case_globals)] impl MessageHeader { pub const NONE: Self = Self(0); - pub const Schema: Self = Self(1); - pub const Batch: Self = Self(2); - pub const Page: Self = Self(3); + pub const ArrayData: Self = Self(1); + pub const Buffer: Self = Self(2); + pub const DType: Self = Self(3); pub const ENUM_MIN: u8 = 0; pub const ENUM_MAX: u8 = 3; pub const ENUM_VALUES: &'static [Self] = &[ Self::NONE, - Self::Schema, - Self::Batch, - Self::Page, + Self::ArrayData, + Self::Buffer, + Self::DType, ]; /// Returns the variant's name or "" if unknown. pub fn variant_name(self) -> Option<&'static str> { match self { Self::NONE => Some("NONE"), - Self::Schema => Some("Schema"), - Self::Batch => Some("Batch"), - Self::Page => Some("Page"), + Self::ArrayData => Some("ArrayData"), + Self::Buffer => Some("Buffer"), + Self::DType => Some("DType"), _ => None, } } @@ -269,521 +269,6 @@ impl<'a> flatbuffers::Verifiable for MessageHeader { impl flatbuffers::SimpleToVerifyInSlice for MessageHeader {} pub struct MessageHeaderUnionTableOffset {} -// struct Buffer, aligned to 8 -#[repr(transparent)] -#[derive(Clone, Copy, PartialEq)] -pub struct Buffer(pub [u8; 16]); -impl Default for Buffer { - fn default() -> Self { - Self([0; 16]) - } -} -impl core::fmt::Debug for Buffer { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - f.debug_struct("Buffer") - .field("offset", &self.offset()) - .field("padding", &self.padding()) - .field("compression", &self.compression()) - .finish() - } -} - -impl flatbuffers::SimpleToVerifyInSlice for Buffer {} -impl<'a> flatbuffers::Follow<'a> for Buffer { - type Inner = &'a Buffer; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - <&'a Buffer>::follow(buf, loc) - } -} -impl<'a> flatbuffers::Follow<'a> for &'a Buffer { - type Inner = &'a Buffer; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - flatbuffers::follow_cast_ref::(buf, loc) - } -} -impl<'b> flatbuffers::Push for Buffer { - type Output = Buffer; - #[inline] - unsafe fn push(&self, dst: &mut [u8], _written_len: usize) { - let src = ::core::slice::from_raw_parts(self as *const Buffer as *const u8, Self::size()); - dst.copy_from_slice(src); - } -} - -impl<'a> flatbuffers::Verifiable for Buffer { - #[inline] - fn run_verifier( - v: &mut flatbuffers::Verifier, pos: usize - ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.in_buffer::(pos) - } -} - -impl<'a> Buffer { - #[allow(clippy::too_many_arguments)] - pub fn new( - offset: u64, - padding: u16, - compression: Compression, - ) -> Self { - let mut s = Self([0; 16]); - s.set_offset(offset); - s.set_padding(padding); - s.set_compression(compression); - s - } - - pub fn offset(&self) -> u64 { - let mut mem = core::mem::MaybeUninit::<::Scalar>::uninit(); - // Safety: - // Created from a valid Table for this object - // Which contains a valid value in this slot - EndianScalar::from_little_endian(unsafe { - core::ptr::copy_nonoverlapping( - self.0[0..].as_ptr(), - mem.as_mut_ptr() as *mut u8, - core::mem::size_of::<::Scalar>(), - ); - mem.assume_init() - }) - } - - pub fn set_offset(&mut self, x: u64) { - let x_le = x.to_little_endian(); - // Safety: - // Created from a valid Table for this object - // Which contains a valid value in this slot - unsafe { - core::ptr::copy_nonoverlapping( - &x_le as *const _ as *const u8, - self.0[0..].as_mut_ptr(), - core::mem::size_of::<::Scalar>(), - ); - } - } - - pub fn padding(&self) -> u16 { - let mut mem = core::mem::MaybeUninit::<::Scalar>::uninit(); - // Safety: - // Created from a valid Table for this object - // Which contains a valid value in this slot - EndianScalar::from_little_endian(unsafe { - core::ptr::copy_nonoverlapping( - self.0[8..].as_ptr(), - mem.as_mut_ptr() as *mut u8, - core::mem::size_of::<::Scalar>(), - ); - mem.assume_init() - }) - } - - pub fn set_padding(&mut self, x: u16) { - let x_le = x.to_little_endian(); - // Safety: - // Created from a valid Table for this object - // Which contains a valid value in this slot - unsafe { - core::ptr::copy_nonoverlapping( - &x_le as *const _ as *const u8, - self.0[8..].as_mut_ptr(), - core::mem::size_of::<::Scalar>(), - ); - } - } - - pub fn compression(&self) -> Compression { - let mut mem = core::mem::MaybeUninit::<::Scalar>::uninit(); - // Safety: - // Created from a valid Table for this object - // Which contains a valid value in this slot - EndianScalar::from_little_endian(unsafe { - core::ptr::copy_nonoverlapping( - self.0[10..].as_ptr(), - mem.as_mut_ptr() as *mut u8, - core::mem::size_of::<::Scalar>(), - ); - mem.assume_init() - }) - } - - pub fn set_compression(&mut self, x: Compression) { - let x_le = x.to_little_endian(); - // Safety: - // Created from a valid Table for this object - // Which contains a valid value in this slot - unsafe { - core::ptr::copy_nonoverlapping( - &x_le as *const _ as *const u8, - self.0[10..].as_mut_ptr(), - core::mem::size_of::<::Scalar>(), - ); - } - } - -} - -pub enum SchemaOffset {} -#[derive(Copy, Clone, PartialEq)] - -pub struct Schema<'a> { - pub _tab: flatbuffers::Table<'a>, -} - -impl<'a> flatbuffers::Follow<'a> for Schema<'a> { - type Inner = Schema<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { _tab: flatbuffers::Table::new(buf, loc) } - } -} - -impl<'a> Schema<'a> { - pub const VT_DTYPE: flatbuffers::VOffsetT = 4; - - #[inline] - pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { - Schema { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args SchemaArgs<'args> - ) -> flatbuffers::WIPOffset> { - let mut builder = SchemaBuilder::new(_fbb); - if let Some(x) = args.dtype { builder.add_dtype(x); } - builder.finish() - } - - - #[inline] - pub fn dtype(&self) -> Option> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::>(Schema::VT_DTYPE, None)} - } -} - -impl flatbuffers::Verifiable for Schema<'_> { - #[inline] - fn run_verifier( - v: &mut flatbuffers::Verifier, pos: usize - ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.visit_table(pos)? - .visit_field::>("dtype", Self::VT_DTYPE, false)? - .finish(); - Ok(()) - } -} -pub struct SchemaArgs<'a> { - pub dtype: Option>>, -} -impl<'a> Default for SchemaArgs<'a> { - #[inline] - fn default() -> Self { - SchemaArgs { - dtype: None, - } - } -} - -pub struct SchemaBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - start_: flatbuffers::WIPOffset, -} -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> SchemaBuilder<'a, 'b, A> { - #[inline] - pub fn add_dtype(&mut self, dtype: flatbuffers::WIPOffset>) { - self.fbb_.push_slot_always::>(Schema::VT_DTYPE, dtype); - } - #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> SchemaBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - SchemaBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - flatbuffers::WIPOffset::new(o.value()) - } -} - -impl core::fmt::Debug for Schema<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut ds = f.debug_struct("Schema"); - ds.field("dtype", &self.dtype()); - ds.finish() - } -} -pub enum BatchOffset {} -#[derive(Copy, Clone, PartialEq)] - -pub struct Batch<'a> { - pub _tab: flatbuffers::Table<'a>, -} - -impl<'a> flatbuffers::Follow<'a> for Batch<'a> { - type Inner = Batch<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { _tab: flatbuffers::Table::new(buf, loc) } - } -} - -impl<'a> Batch<'a> { - pub const VT_ARRAY: flatbuffers::VOffsetT = 4; - pub const VT_LENGTH: flatbuffers::VOffsetT = 6; - pub const VT_BUFFERS: flatbuffers::VOffsetT = 8; - pub const VT_BUFFER_SIZE: flatbuffers::VOffsetT = 10; - - #[inline] - pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { - Batch { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args BatchArgs<'args> - ) -> flatbuffers::WIPOffset> { - let mut builder = BatchBuilder::new(_fbb); - builder.add_buffer_size(args.buffer_size); - builder.add_length(args.length); - if let Some(x) = args.buffers { builder.add_buffers(x); } - if let Some(x) = args.array { builder.add_array(x); } - builder.finish() - } - - - #[inline] - pub fn array(&self) -> Option> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::>(Batch::VT_ARRAY, None)} - } - #[inline] - pub fn length(&self) -> u64 { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::(Batch::VT_LENGTH, Some(0)).unwrap()} - } - #[inline] - pub fn buffers(&self) -> Option> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::>>(Batch::VT_BUFFERS, None)} - } - #[inline] - pub fn buffer_size(&self) -> u64 { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::(Batch::VT_BUFFER_SIZE, Some(0)).unwrap()} - } -} - -impl flatbuffers::Verifiable for Batch<'_> { - #[inline] - fn run_verifier( - v: &mut flatbuffers::Verifier, pos: usize - ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.visit_table(pos)? - .visit_field::>("array", Self::VT_ARRAY, false)? - .visit_field::("length", Self::VT_LENGTH, false)? - .visit_field::>>("buffers", Self::VT_BUFFERS, false)? - .visit_field::("buffer_size", Self::VT_BUFFER_SIZE, false)? - .finish(); - Ok(()) - } -} -pub struct BatchArgs<'a> { - pub array: Option>>, - pub length: u64, - pub buffers: Option>>, - pub buffer_size: u64, -} -impl<'a> Default for BatchArgs<'a> { - #[inline] - fn default() -> Self { - BatchArgs { - array: None, - length: 0, - buffers: None, - buffer_size: 0, - } - } -} - -pub struct BatchBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - start_: flatbuffers::WIPOffset, -} -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> BatchBuilder<'a, 'b, A> { - #[inline] - pub fn add_array(&mut self, array: flatbuffers::WIPOffset>) { - self.fbb_.push_slot_always::>(Batch::VT_ARRAY, array); - } - #[inline] - pub fn add_length(&mut self, length: u64) { - self.fbb_.push_slot::(Batch::VT_LENGTH, length, 0); - } - #[inline] - pub fn add_buffers(&mut self, buffers: flatbuffers::WIPOffset>) { - self.fbb_.push_slot_always::>(Batch::VT_BUFFERS, buffers); - } - #[inline] - pub fn add_buffer_size(&mut self, buffer_size: u64) { - self.fbb_.push_slot::(Batch::VT_BUFFER_SIZE, buffer_size, 0); - } - #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> BatchBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - BatchBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - flatbuffers::WIPOffset::new(o.value()) - } -} - -impl core::fmt::Debug for Batch<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut ds = f.debug_struct("Batch"); - ds.field("array", &self.array()); - ds.field("length", &self.length()); - ds.field("buffers", &self.buffers()); - ds.field("buffer_size", &self.buffer_size()); - ds.finish() - } -} -pub enum PageOffset {} -#[derive(Copy, Clone, PartialEq)] - -pub struct Page<'a> { - pub _tab: flatbuffers::Table<'a>, -} - -impl<'a> flatbuffers::Follow<'a> for Page<'a> { - type Inner = Page<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { _tab: flatbuffers::Table::new(buf, loc) } - } -} - -impl<'a> Page<'a> { - pub const VT_BUFFER_SIZE: flatbuffers::VOffsetT = 4; - pub const VT_PADDING: flatbuffers::VOffsetT = 6; - - #[inline] - pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { - Page { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args PageArgs - ) -> flatbuffers::WIPOffset> { - let mut builder = PageBuilder::new(_fbb); - builder.add_buffer_size(args.buffer_size); - builder.add_padding(args.padding); - builder.finish() - } - - - #[inline] - pub fn buffer_size(&self) -> u32 { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::(Page::VT_BUFFER_SIZE, Some(0)).unwrap()} - } - #[inline] - pub fn padding(&self) -> u16 { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::(Page::VT_PADDING, Some(0)).unwrap()} - } -} - -impl flatbuffers::Verifiable for Page<'_> { - #[inline] - fn run_verifier( - v: &mut flatbuffers::Verifier, pos: usize - ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.visit_table(pos)? - .visit_field::("buffer_size", Self::VT_BUFFER_SIZE, false)? - .visit_field::("padding", Self::VT_PADDING, false)? - .finish(); - Ok(()) - } -} -pub struct PageArgs { - pub buffer_size: u32, - pub padding: u16, -} -impl<'a> Default for PageArgs { - #[inline] - fn default() -> Self { - PageArgs { - buffer_size: 0, - padding: 0, - } - } -} - -pub struct PageBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - start_: flatbuffers::WIPOffset, -} -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PageBuilder<'a, 'b, A> { - #[inline] - pub fn add_buffer_size(&mut self, buffer_size: u32) { - self.fbb_.push_slot::(Page::VT_BUFFER_SIZE, buffer_size, 0); - } - #[inline] - pub fn add_padding(&mut self, padding: u16) { - self.fbb_.push_slot::(Page::VT_PADDING, padding, 0); - } - #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PageBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - PageBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - flatbuffers::WIPOffset::new(o.value()) - } -} - -impl core::fmt::Debug for Page<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut ds = f.debug_struct("Page"); - ds.field("buffer_size", &self.buffer_size()); - ds.field("padding", &self.padding()); - ds.finish() - } -} pub enum MessageOffset {} #[derive(Copy, Clone, PartialEq)] @@ -809,8 +294,8 @@ impl<'a> Message<'a> { Message { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args MessageArgs ) -> flatbuffers::WIPOffset> { let mut builder = MessageBuilder::new(_fbb); @@ -844,13 +329,13 @@ impl<'a> Message<'a> { } #[inline] #[allow(non_snake_case)] - pub fn header_as_schema(&self) -> Option> { - if self.header_type() == MessageHeader::Schema { + pub fn header_as_array_data(&self) -> Option> { + if self.header_type() == MessageHeader::ArrayData { self.header().map(|t| { // Safety: // Created from a valid Table for this object // Which contains a valid union in this slot - unsafe { Schema::init_from_table(t) } + unsafe { ArrayData::init_from_table(t) } }) } else { None @@ -859,13 +344,13 @@ impl<'a> Message<'a> { #[inline] #[allow(non_snake_case)] - pub fn header_as_batch(&self) -> Option> { - if self.header_type() == MessageHeader::Batch { + pub fn header_as_buffer(&self) -> Option> { + if self.header_type() == MessageHeader::Buffer { self.header().map(|t| { // Safety: // Created from a valid Table for this object // Which contains a valid union in this slot - unsafe { Batch::init_from_table(t) } + unsafe { Buffer::init_from_table(t) } }) } else { None @@ -874,13 +359,13 @@ impl<'a> Message<'a> { #[inline] #[allow(non_snake_case)] - pub fn header_as_page(&self) -> Option> { - if self.header_type() == MessageHeader::Page { + pub fn header_as_dtype(&self) -> Option> { + if self.header_type() == MessageHeader::DType { self.header().map(|t| { // Safety: // Created from a valid Table for this object // Which contains a valid union in this slot - unsafe { Page::init_from_table(t) } + unsafe { DType::init_from_table(t) } }) } else { None @@ -899,9 +384,9 @@ impl flatbuffers::Verifiable for Message<'_> { .visit_field::("version", Self::VT_VERSION, false)? .visit_union::("header_type", Self::VT_HEADER_TYPE, "header", Self::VT_HEADER, false, |key, v, pos| { match key { - MessageHeader::Schema => v.verify_union_variant::>("MessageHeader::Schema", pos), - MessageHeader::Batch => v.verify_union_variant::>("MessageHeader::Batch", pos), - MessageHeader::Page => v.verify_union_variant::>("MessageHeader::Page", pos), + MessageHeader::ArrayData => v.verify_union_variant::>("MessageHeader::ArrayData", pos), + MessageHeader::Buffer => v.verify_union_variant::>("MessageHeader::Buffer", pos), + MessageHeader::DType => v.verify_union_variant::>("MessageHeader::DType", pos), _ => Ok(()), } })? @@ -925,11 +410,11 @@ impl<'a> Default for MessageArgs { } } -pub struct MessageBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct MessageBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> MessageBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> MessageBuilder<'a, 'b> { #[inline] pub fn add_version(&mut self, version: MessageVersion) { self.fbb_.push_slot::(Message::VT_VERSION, version, MessageVersion::V0); @@ -943,7 +428,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> MessageBuilder<'a, 'b, A> { self.fbb_.push_slot_always::>(Message::VT_HEADER, header); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> MessageBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MessageBuilder<'a, 'b> { let start = _fbb.start_table(); MessageBuilder { fbb_: _fbb, @@ -963,22 +448,22 @@ impl core::fmt::Debug for Message<'_> { ds.field("version", &self.version()); ds.field("header_type", &self.header_type()); match self.header_type() { - MessageHeader::Schema => { - if let Some(x) = self.header_as_schema() { + MessageHeader::ArrayData => { + if let Some(x) = self.header_as_array_data() { ds.field("header", &x) } else { ds.field("header", &"InvalidFlatbuffer: Union discriminant does not match value.") } }, - MessageHeader::Batch => { - if let Some(x) = self.header_as_batch() { + MessageHeader::Buffer => { + if let Some(x) = self.header_as_buffer() { ds.field("header", &x) } else { ds.field("header", &"InvalidFlatbuffer: Union discriminant does not match value.") } }, - MessageHeader::Page => { - if let Some(x) = self.header_as_page() { + MessageHeader::DType => { + if let Some(x) = self.header_as_dtype() { ds.field("header", &x) } else { ds.field("header", &"InvalidFlatbuffer: Union discriminant does not match value.") @@ -1053,13 +538,13 @@ pub unsafe fn size_prefixed_root_as_message_unchecked(buf: &[u8]) -> Message { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_message_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub fn finish_message_buffer<'a, 'b>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_message_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_message_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/scalar.rs b/vortex-flatbuffers/src/generated/scalar.rs index 5864130da..300c17119 100644 --- a/vortex-flatbuffers/src/generated/scalar.rs +++ b/vortex-flatbuffers/src/generated/scalar.rs @@ -34,8 +34,8 @@ impl<'a> Scalar<'a> { Scalar { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args ScalarArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ScalarBuilder::new(_fbb); @@ -88,11 +88,11 @@ impl<'a> Default for ScalarArgs<'a> { } } -pub struct ScalarBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct ScalarBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ScalarBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> ScalarBuilder<'a, 'b> { #[inline] pub fn add_dtype(&mut self, dtype: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(Scalar::VT_DTYPE, dtype); @@ -102,7 +102,7 @@ impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ScalarBuilder<'a, 'b, A> { self.fbb_.push_slot_always::>(Scalar::VT_VALUE, value); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ScalarBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ScalarBuilder<'a, 'b> { let start = _fbb.start_table(); ScalarBuilder { fbb_: _fbb, @@ -149,8 +149,8 @@ impl<'a> ScalarValue<'a> { ScalarValue { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, args: &'args ScalarValueArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ScalarValueBuilder::new(_fbb); @@ -192,17 +192,17 @@ impl<'a> Default for ScalarValueArgs<'a> { } } -pub struct ScalarValueBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub struct ScalarValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ScalarValueBuilder<'a, 'b, A> { +impl<'a: 'b, 'b> ScalarValueBuilder<'a, 'b> { #[inline] pub fn add_flex(&mut self, flex: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(ScalarValue::VT_FLEX, flex); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ScalarValueBuilder<'a, 'b, A> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ScalarValueBuilder<'a, 'b> { let start = _fbb.start_table(); ScalarValueBuilder { fbb_: _fbb, @@ -285,13 +285,13 @@ pub unsafe fn size_prefixed_root_as_scalar_unchecked(buf: &[u8]) -> Scalar { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_scalar_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, +pub fn finish_scalar_buffer<'a, 'b>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_scalar_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_scalar_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/lib.rs b/vortex-flatbuffers/src/lib.rs index bb99fb601..b45a50c18 100644 --- a/vortex-flatbuffers/src/lib.rs +++ b/vortex-flatbuffers/src/lib.rs @@ -115,6 +115,7 @@ pub mod footer; pub mod message; use flatbuffers::{root, FlatBufferBuilder, Follow, InvalidFlatbuffer, Verifiable, WIPOffset}; +use vortex_buffer::Buffer; pub trait FlatBufferRoot {} @@ -145,10 +146,27 @@ pub trait WriteFlatBuffer { } pub trait FlatBufferToBytes { + /// Write the flatbuffer to a byte vector and return the vector along with the starting + /// position of the flatbuffer in the vector. + fn write_flatbuffer_bytes(&self) -> Buffer; + + #[deprecated( + note = "Use `write_flatbuffer_bytes` instead. This method will be removed in a future release." + )] fn with_flatbuffer_bytes R>(&self, f: Fn) -> R; } impl FlatBufferToBytes for F { + /// Write the flatbuffer into a [`vortex_buffer::Buffer`]. + fn write_flatbuffer_bytes(&self) -> Buffer { + let mut fbb = FlatBufferBuilder::new(); + let root_offset = self.write_flatbuffer(&mut fbb); + fbb.finish_minimal(root_offset); + let (vec, start) = fbb.collapse(); + let end = vec.len(); + Buffer::from(vec).slice(start..end) + } + fn with_flatbuffer_bytes R>(&self, f: Fn) -> R { let mut fbb = FlatBufferBuilder::new(); let root_offset = self.write_flatbuffer(&mut fbb); diff --git a/vortex-ipc/src/messages/mod.rs b/vortex-ipc/src/messages/mod.rs index 9b505e92b..2f30d81a7 100644 --- a/vortex-ipc/src/messages/mod.rs +++ b/vortex-ipc/src/messages/mod.rs @@ -4,8 +4,7 @@ use vortex_array::stats::ArrayStatistics; use vortex_array::{flatbuffers as fba, ArrayData}; use vortex_buffer::Buffer; use vortex_dtype::DType; -use vortex_error::{VortexExpect as _, VortexUnwrap}; -use vortex_flatbuffers::message::Compression; +use vortex_error::VortexExpect; use vortex_flatbuffers::{message as fb, FlatBufferRoot, WriteFlatBuffer}; use crate::ALIGNMENT; @@ -13,20 +12,15 @@ use crate::ALIGNMENT; pub mod reader; pub mod writer; -pub enum IPCMessage<'a> { - Schema(IPCSchema<'a>), - Batch(IPCBatch<'a>), - Page(IPCPage<'a>), +pub enum IPCMessage { + Array(ArrayData), + Buffer(Buffer), + DType(DType), } -pub struct IPCSchema<'a>(pub &'a DType); -pub struct IPCBatch<'a>(pub &'a ArrayData); -pub struct IPCArray<'a>(pub &'a ArrayData, usize); -pub struct IPCPage<'a>(pub &'a Buffer); +impl FlatBufferRoot for IPCMessage {} -impl FlatBufferRoot for IPCMessage<'_> {} - -impl WriteFlatBuffer for IPCMessage<'_> { +impl WriteFlatBuffer for IPCMessage { type Target<'a> = fb::Message<'a>; fn write_flatbuffer<'fb>( @@ -34,87 +28,98 @@ impl WriteFlatBuffer for IPCMessage<'_> { fbb: &mut FlatBufferBuilder<'fb>, ) -> WIPOffset> { let header = match self { - Self::Schema(f) => f.write_flatbuffer(fbb).as_union_value(), - Self::Batch(f) => f.write_flatbuffer(fbb).as_union_value(), - Self::Page(f) => f.write_flatbuffer(fbb).as_union_value(), + Self::Array(array) => ArrayDataWriter { array } + .write_flatbuffer(fbb) + .as_union_value(), + Self::Buffer(buffer) => { + let aligned_len = buffer.len().next_multiple_of(ALIGNMENT); + let padding = aligned_len - buffer.len(); + fba::Buffer::create( + fbb, + &fba::BufferArgs { + length: buffer.len() as u64, + padding: padding as u16, + }, + ) + .as_union_value() + } + Self::DType(dtype) => dtype.write_flatbuffer(fbb).as_union_value(), }; let mut msg = fb::MessageBuilder::new(fbb); msg.add_version(Default::default()); msg.add_header_type(match self { - Self::Schema(_) => fb::MessageHeader::Schema, - Self::Batch(_) => fb::MessageHeader::Batch, - Self::Page(_) => fb::MessageHeader::Page, + Self::Array(_) => fb::MessageHeader::ArrayData, + Self::Buffer(_) => fb::MessageHeader::Buffer, + Self::DType(_) => fb::MessageHeader::DType, }); msg.add_header(header); msg.finish() } } -impl WriteFlatBuffer for IPCSchema<'_> { - type Target<'t> = fb::Schema<'t>; - - fn write_flatbuffer<'fb>( - &self, - fbb: &mut FlatBufferBuilder<'fb>, - ) -> WIPOffset> { - let dtype = Some(self.0.write_flatbuffer(fbb)); - fb::Schema::create(fbb, &fb::SchemaArgs { dtype }) - } +struct ArrayDataWriter<'a> { + array: &'a ArrayData, } -impl WriteFlatBuffer for IPCBatch<'_> { - type Target<'t> = fb::Batch<'t>; +impl WriteFlatBuffer for ArrayDataWriter<'_> { + type Target<'t> = fba::ArrayData<'t>; fn write_flatbuffer<'fb>( &self, fbb: &mut FlatBufferBuilder<'fb>, ) -> WIPOffset> { - let array_data = self.0; - let array = Some(IPCArray(array_data, 0).write_flatbuffer(fbb)); - - let length = array_data.len() as u64; + let array = Some( + ArrayWriter { + array: self.array, + buffer_idx: 0, + } + .write_flatbuffer(fbb), + ); - // Walk the ColumnData depth-first to compute the buffer offsets. + // Walk the ColumnData depth-first to compute the buffer lengths. let mut buffers = vec![]; - let mut offset = 0; - - for array_data in array_data.depth_first_traversal() { + for array_data in self.array.depth_first_traversal() { if let Some(buffer) = array_data.buffer() { - let aligned_size = (buffer.len() + (ALIGNMENT - 1)) & !(ALIGNMENT - 1); - buffers.push(fb::Buffer::new( - offset as u64, - (aligned_size - buffer.len()).try_into().vortex_unwrap(), - Compression::None, + let aligned_size = buffer.len().next_multiple_of(ALIGNMENT); + let padding = (aligned_size - buffer.len()) as u16; + buffers.push(fba::Buffer::create( + fbb, + &fba::BufferArgs { + length: buffer.len() as u64, + padding, + }, )); - offset += aligned_size; } } let buffers = Some(fbb.create_vector(&buffers)); - fb::Batch::create( + fba::ArrayData::create( fbb, - &fb::BatchArgs { + &fba::ArrayDataArgs { array, - length, + row_count: self.array.len() as u64, buffers, - buffer_size: offset as u64, }, ) } } -impl WriteFlatBuffer for IPCArray<'_> { +struct ArrayWriter<'a> { + array: &'a ArrayData, + buffer_idx: u16, +} + +impl WriteFlatBuffer for ArrayWriter<'_> { type Target<'t> = fba::Array<'t>; fn write_flatbuffer<'fb>( &self, fbb: &mut FlatBufferBuilder<'fb>, ) -> WIPOffset> { - let column_data = self.0; - - let encoding = column_data.encoding().id().code(); - let metadata = column_data + let encoding = self.array.encoding().id().code(); + let metadata = self + .array .metadata_bytes() .vortex_expect("IPCArray is missing metadata during serialization"); let metadata = Some(fbb.create_vector(metadata.as_ref())); @@ -123,52 +128,45 @@ impl WriteFlatBuffer for IPCArray<'_> { // The second tuple element holds the buffer_index for this Array subtree. If this array // has a buffer, that is its buffer index. If it does not, that buffer index belongs // to one of the children. - let child_buffer_offset = self.1 + if self.0.buffer().is_some() { 1 } else { 0 }; + let child_buffer_idx = self.buffer_idx + if self.array.buffer().is_some() { 1 } else { 0 }; - let children = column_data + let children = self + .array .children() .iter() - .scan(child_buffer_offset, |buffer_offset, child| { + .scan(child_buffer_idx, |buffer_idx, child| { // Update the number of buffers required. - let msg = IPCArray(child, *buffer_offset).write_flatbuffer(fbb); - *buffer_offset += child.cumulative_nbuffers(); + let msg = ArrayWriter { + array: child, + buffer_idx: *buffer_idx, + } + .write_flatbuffer(fbb); + *buffer_idx = u16::try_from(child.cumulative_nbuffers()) + .ok() + .and_then(|nbuffers| nbuffers.checked_add(*buffer_idx)) + .vortex_expect("Too many buffers (u16) for ArrayData"); Some(msg) }) .collect_vec(); let children = Some(fbb.create_vector(&children)); - let stats = Some(column_data.statistics().write_flatbuffer(fbb)); + let buffers = self + .array + .buffer() + .is_some() + .then(|| self.buffer_idx) + .map(|buffer_idx| fbb.create_vector_from_iter(std::iter::once(buffer_idx))); + + let stats = Some(self.array.statistics().write_flatbuffer(fbb)); fba::Array::create( fbb, &fba::ArrayArgs { - version: Default::default(), - buffer_index: self.0.buffer().is_some().then_some(self.1 as u64), encoding, metadata, - stats, children, - }, - ) - } -} - -impl WriteFlatBuffer for IPCPage<'_> { - type Target<'t> = fb::Page<'t>; - - fn write_flatbuffer<'fb>( - &self, - fbb: &mut FlatBufferBuilder<'fb>, - ) -> WIPOffset> { - let buffer_size = self.0.len(); - let aligned_size = (buffer_size + (ALIGNMENT - 1)) & !(ALIGNMENT - 1); - let padding_size = aligned_size - buffer_size; - - fb::Page::create( - fbb, - &fb::PageArgs { - buffer_size: buffer_size.try_into().vortex_unwrap(), - padding: padding_size.try_into().vortex_unwrap(), + buffers, + stats, }, ) } diff --git a/vortex-ipc/src/messages/reader.rs b/vortex-ipc/src/messages/reader.rs index f2a8ab3c3..242b1fae9 100644 --- a/vortex-ipc/src/messages/reader.rs +++ b/vortex-ipc/src/messages/reader.rs @@ -5,10 +5,10 @@ use bytes::{Buf, Bytes}; use flatbuffers::{root, root_unchecked}; use futures_util::stream::try_unfold; use vortex_array::stream::{ArrayStream, ArrayStreamAdapter}; -use vortex_array::{ArrayData, Context}; +use vortex_array::{flatbuffers as fba, ArrayData, Context}; use vortex_buffer::Buffer; use vortex_dtype::DType; -use vortex_error::{vortex_bail, vortex_err, VortexExpect, VortexResult, VortexUnwrap}; +use vortex_error::{vortex_bail, vortex_err, VortexExpect, VortexResult}; use vortex_flatbuffers::message as fb; use vortex_io::{VortexBufReader, VortexReadAt}; @@ -95,21 +95,18 @@ impl MessageReader { } pub async fn read_dtype(&mut self) -> VortexResult { - if self.peek().and_then(|m| m.header_as_schema()).is_none() { - vortex_bail!("Expected schema message") + if self.peek().and_then(|m| m.header_as_dtype()).is_none() { + vortex_bail!("Expected DType message") } let buf = self.next().await?; let msg = unsafe { root_unchecked::(&buf) } - .header_as_schema() + .header_as_dtype() .ok_or_else(|| { vortex_err!("Expected schema message; this was checked earlier in the function") })?; - DType::try_from( - msg.dtype() - .ok_or_else(|| vortex_err!(InvalidSerde: "Schema missing DType"))?, - ) + DType::try_from(msg) } pub async fn maybe_read_chunk( @@ -117,9 +114,14 @@ impl MessageReader { ctx: Arc, dtype: DType, ) -> VortexResult> { - let all_buffers_size: usize = match self.peek().and_then(|m| m.header_as_batch()) { + let all_buffers_size: usize = match self.peek().and_then(|m| m.header_as_array_data()) { None => return Ok(None), - Some(chunk) => chunk.buffer_size().try_into().vortex_unwrap(), + Some(array_data) => array_data + .buffers() + .unwrap_or_default() + .iter() + .map(|b| b.length() as usize + (b.padding() as usize)) + .sum(), }; let mut array_reader = ArrayMessageReader::from_fb_bytes(Buffer::from( @@ -193,13 +195,13 @@ impl MessageReader { ) } - pub async fn maybe_read_page(&mut self) -> VortexResult> { - let Some(page_msg) = self.peek().and_then(|m| m.header_as_page()) else { + pub async fn maybe_read_buffer(&mut self) -> VortexResult> { + let Some(buffer_msg) = self.peek().and_then(|m| m.header_as_buffer()) else { return Ok(None); }; - let buffer_len = page_msg.buffer_size() as u64; - let total_len = buffer_len + (page_msg.padding() as u64); + let buffer_len = buffer_msg.length(); + let total_len = buffer_len + (buffer_msg.padding() as u64); let buffer = self.read.read_bytes(total_len).await?; let page_buffer = Ok(Some(Buffer::from( @@ -251,6 +253,7 @@ impl ArrayMessageReader { } } + /// Parse the given bytes and optionally request more from the input. pub fn read(&mut self, mut bytes: Bytes) -> VortexResult> { match self.state { ReadState::Init => { @@ -264,40 +267,40 @@ impl ArrayMessageReader { ReadState::ReadingFb => { // SAFETY: Assumes that any flatbuffer bytes passed have been validated. // This is currently the case in stream and file implementations. - let batch = unsafe { + let array_data = unsafe { root_unchecked::(&bytes) - .header_as_batch() + .header_as_array_data() .ok_or_else(|| vortex_err!("Message was not a batch"))? }; - let buffer_size: usize = batch.buffer_size().try_into().vortex_unwrap(); + + let buffers_size = array_data + .buffers() + .map(|buffers| { + buffers + .iter() + .map(|buffer| buffer.length() + buffer.padding() as u64) + .sum::() + }) + .unwrap_or_default(); + self.fb_msg = Some(Buffer::from(bytes)); self.state = ReadState::ReadingBuffers; - Ok(Some(buffer_size)) + Ok(Some(buffers_size as usize)) } ReadState::ReadingBuffers => { // Split out into individual buffers // Initialize the column's buffers for a vectored read. // To start with, we include the padding and then truncate the buffers after. - let batch_msg = self.fb_bytes_as_batch()?; - let all_buffers_size = batch_msg.buffer_size(); + let batch_msg = self.fb_bytes_as_array_data()?; + let ipc_buffers = batch_msg.buffers().unwrap_or_default(); let buffers = ipc_buffers .iter() - .zip( - ipc_buffers - .iter() - .map(vortex_flatbuffers::message::Buffer::offset) - .skip(1) - .chain([all_buffers_size]), - ) - .map(|(buffer, next_offset)| { - let len = next_offset - buffer.offset() - buffer.padding() as u64; - + .map(|buffer| { // Grab the buffer - let data_buffer = bytes.split_to(len.try_into().vortex_unwrap()); + let data_buffer = bytes.split_to(buffer.length() as usize); // Strip off any padding from the previous buffer bytes.advance(buffer.padding() as usize); - Buffer::from(data_buffer) }) .collect::>(); @@ -310,7 +313,7 @@ impl ArrayMessageReader { } } - fn fb_bytes_as_batch(&self) -> VortexResult { + fn fb_bytes_as_array_data(&self) -> VortexResult { unsafe { root_unchecked::( self.fb_msg @@ -318,24 +321,24 @@ impl ArrayMessageReader { .ok_or_else(|| vortex_err!("Populated in previous step"))?, ) } - .header_as_batch() + .header_as_array_data() .ok_or_else(|| vortex_err!("Checked in previous step")) } /// Produce the array buffered in the reader pub fn into_array(self, ctx: Arc, dtype: DType) -> VortexResult { - let length: usize = self.fb_bytes_as_batch()?.length().try_into()?; + let row_count: usize = self.fb_bytes_as_array_data()?.row_count().try_into()?; let fb_msg = self .fb_msg .ok_or_else(|| vortex_err!("Populated in previous step"))?; ArrayData::try_new_viewed( ctx, dtype, - length, + row_count, fb_msg, |flatbuffer| { unsafe { root_unchecked::(flatbuffer) } - .header_as_batch() + .header_as_array_data() .ok_or_else(|| vortex_err!("Failed to get root header as batch"))? .array() .ok_or_else(|| vortex_err!("Chunk missing Array")) @@ -369,7 +372,7 @@ mod test { let mut reader = block_on(async { MessageReader::try_new(VortexBufReader::new(written)).await }) .unwrap(); - let read_page = block_on(async { reader.maybe_read_page().await }) + let read_page = block_on(async { reader.maybe_read_buffer().await }) .unwrap() .unwrap(); assert_eq!(read_page, Buffer::from(Bytes::from("somevalue"))); diff --git a/vortex-ipc/src/messages/writer.rs b/vortex-ipc/src/messages/writer.rs index c28996f02..3ae80a0be 100644 --- a/vortex-ipc/src/messages/writer.rs +++ b/vortex-ipc/src/messages/writer.rs @@ -3,16 +3,15 @@ use std::io; use bytes::Bytes; use flatbuffers::FlatBufferBuilder; -use itertools::Itertools; use vortex_array::ArrayData; use vortex_buffer::io_buf::IoBuf; use vortex_buffer::Buffer; use vortex_dtype::DType; use vortex_error::VortexUnwrap; -use vortex_flatbuffers::WriteFlatBuffer; +use vortex_flatbuffers::{FlatBufferToBytes, WriteFlatBuffer}; use vortex_io::VortexWrite; -use crate::messages::{IPCBatch, IPCMessage, IPCPage, IPCSchema}; +use crate::messages::IPCMessage; use crate::ALIGNMENT; static ZEROS: [u8; 512] = [0; 512]; @@ -47,17 +46,9 @@ impl MessageWriter { } pub async fn write_dtype_raw(&mut self, dtype: &DType) -> io::Result<()> { - let fb = IPCSchema(dtype); - let mut fbb = FlatBufferBuilder::new(); - let ps_fb = fb.write_flatbuffer(&mut fbb); - fbb.finish_minimal(ps_fb); - - let (buffer, buffer_begin) = fbb.collapse(); - let buffer_end = buffer.len(); - - let written_len = buffer_end - buffer_begin; - let bytes = buffer.slice_owned(buffer_begin..buffer_end); - self.write_all(bytes).await?; + let buffer = dtype.write_flatbuffer_bytes(); + let written_len = buffer.len(); + self.write_all(buffer).await?; let aligned_size = written_len.next_multiple_of(self.alignment); let padding = aligned_size - written_len; @@ -67,44 +58,32 @@ impl MessageWriter { Ok(()) } - pub async fn write_dtype(&mut self, dtype: &DType) -> io::Result<()> { - self.write_message(IPCMessage::Schema(IPCSchema(dtype))) - .await + pub async fn write_dtype(&mut self, dtype: DType) -> io::Result<()> { + self.write_message(IPCMessage::DType(dtype)).await } - pub async fn write_batch(&mut self, chunk: ArrayData) -> io::Result<()> { - let buffer_offsets = chunk.all_buffer_offsets(self.alignment); - - // Serialize the Chunk message. - self.write_message(IPCMessage::Batch(IPCBatch(&chunk))) - .await?; - - // Keep track of the offset to add padding after each buffer. - let mut current_offset = 0; - for (buffer, buffer_end) in chunk - .depth_first_traversal() - .flat_map(|data| data.into_buffer().into_iter()) - .zip_eq(buffer_offsets.into_iter().skip(1)) - { - let buffer_len = buffer.len(); - let buffer_end: usize = buffer_end.try_into().vortex_unwrap(); - self.write_all(buffer).await?; - let padding = buffer_end - current_offset - buffer_len; - self.write_all(Bytes::from(&ZEROS[..padding])).await?; - current_offset = buffer_end; + pub async fn write_array(&mut self, array: ArrayData) -> io::Result<()> { + let msg = IPCMessage::Array(array.clone()).write_flatbuffer_bytes(); + self.write_all(msg).await?; + + for array in array.depth_first_traversal() { + if let Some(buffer) = array.buffer() { + let buffer_len = buffer.len(); + let padding = buffer_len.next_multiple_of(self.alignment) - buffer_len; + self.write_all(buffer.clone()).await?; + self.write_all(Bytes::from(&ZEROS[..padding])).await?; + } } Ok(()) } pub async fn write_page(&mut self, buffer: Buffer) -> io::Result<()> { - self.write_message(IPCMessage::Page(IPCPage(&buffer))) - .await?; let buffer_len = buffer.len(); + let padding = buffer_len.next_multiple_of(self.alignment) - buffer_len; + self.write_message(IPCMessage::Buffer(buffer.clone())) + .await?; self.write_all(buffer).await?; - - let aligned_size = buffer_len.next_multiple_of(self.alignment); - let padding = aligned_size - buffer_len; self.write_all(Bytes::from(&ZEROS[..padding])).await?; Ok(()) diff --git a/vortex-ipc/src/stream_reader/mod.rs b/vortex-ipc/src/stream_reader/mod.rs index fbdf529fd..c7414b5f2 100644 --- a/vortex-ipc/src/stream_reader/mod.rs +++ b/vortex-ipc/src/stream_reader/mod.rs @@ -62,7 +62,7 @@ impl StreamArrayReader { /// Reads a single page from the stream. pub async fn next_page(&mut self) -> VortexResult> { - self.msgs.maybe_read_page().await + self.msgs.maybe_read_buffer().await } /// Reads consecutive pages from the stream until the message type changes. diff --git a/vortex-ipc/src/stream_writer/mod.rs b/vortex-ipc/src/stream_writer/mod.rs index c991ab887..ef6ac38a1 100644 --- a/vortex-ipc/src/stream_writer/mod.rs +++ b/vortex-ipc/src/stream_writer/mod.rs @@ -43,7 +43,7 @@ impl StreamArrayWriter { self.msgs.into_inner() } - async fn write_dtype(&mut self, dtype: &DType) -> VortexResult { + async fn write_dtype(&mut self, dtype: DType) -> VortexResult { let begin = self.msgs.tell(); self.msgs.write_dtype(dtype).await?; let end = self.msgs.tell(); @@ -61,7 +61,7 @@ impl StreamArrayWriter { while let Some(chunk) = stream.try_next().await? { row_offset += chunk.len() as u64; row_offsets.push(row_offset); - self.msgs.write_batch(chunk).await?; + self.msgs.write_array(chunk).await?; byte_offsets.push(self.msgs.tell()); } @@ -72,7 +72,7 @@ impl StreamArrayWriter { mut self, mut array_stream: S, ) -> VortexResult { - let dtype_pos = self.write_dtype(array_stream.dtype()).await?; + let dtype_pos = self.write_dtype(array_stream.dtype().clone()).await?; let chunk_pos = self.write_array_chunks(&mut array_stream).await?; self.array_layouts.push(ArrayLayout { dtype: dtype_pos, From d1cd36259acba8b4f7324a2c181c1b2d95f37d48 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sat, 14 Dec 2024 15:56:34 +0000 Subject: [PATCH 2/6] Move expr --- vortex-ipc/src/messages/writer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vortex-ipc/src/messages/writer.rs b/vortex-ipc/src/messages/writer.rs index 3ae80a0be..205a63a61 100644 --- a/vortex-ipc/src/messages/writer.rs +++ b/vortex-ipc/src/messages/writer.rs @@ -63,8 +63,7 @@ impl MessageWriter { } pub async fn write_array(&mut self, array: ArrayData) -> io::Result<()> { - let msg = IPCMessage::Array(array.clone()).write_flatbuffer_bytes(); - self.write_all(msg).await?; + self.write_message(IPCMessage::Array(array.clone())).await?; for array in array.depth_first_traversal() { if let Some(buffer) = array.buffer() { From 54ddfd8eec4e86dc2be050a6f6b3955dfd14bf8d Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sat, 14 Dec 2024 16:02:11 +0000 Subject: [PATCH 3/6] Clean up --- vortex-array/src/data/viewed.rs | 6 +++--- vortex-dtype/src/serde/flatbuffers/mod.rs | 2 +- vortex-flatbuffers/src/lib.rs | 12 ------------ vortex-ipc/src/messages/mod.rs | 8 ++++---- vortex-ipc/src/messages/reader.rs | 19 ++++++++++++++----- 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/vortex-array/src/data/viewed.rs b/vortex-array/src/data/viewed.rs index cc195316f..32f53ffa5 100644 --- a/vortex-array/src/data/viewed.rs +++ b/vortex-array/src/data/viewed.rs @@ -5,7 +5,7 @@ use enum_iterator::all; use itertools::Itertools; use vortex_buffer::Buffer; use vortex_dtype::{DType, Nullability, PType}; -use vortex_error::{vortex_err, VortexExpect as _, VortexResult, VortexUnwrap}; +use vortex_error::{vortex_err, VortexExpect as _, VortexResult}; use vortex_scalar::{Scalar, ScalarValue}; use crate::encoding::opaque::OpaqueEncoding; @@ -108,9 +108,9 @@ impl ViewedArrayData { .buffers() .and_then(|buffers| { assert!(buffers.len() <= 1, "Array: expected at most one buffer"); - (buffers.len() > 0).then(|| buffers.get(0) as usize) + (!buffers.is_empty()).then(|| buffers.get(0) as usize) }) - .map(|idx| &self.buffers[usize::try_from(idx).vortex_unwrap()]) + .map(|idx| &self.buffers[idx]) } } diff --git a/vortex-dtype/src/serde/flatbuffers/mod.rs b/vortex-dtype/src/serde/flatbuffers/mod.rs index 49f8fa949..6236982b3 100644 --- a/vortex-dtype/src/serde/flatbuffers/mod.rs +++ b/vortex-dtype/src/serde/flatbuffers/mod.rs @@ -265,7 +265,7 @@ mod test { use crate::{flatbuffers as fb, DType, PType, StructDType}; fn roundtrip_dtype(dtype: DType) { - let bytes = dtype.with_flatbuffer_bytes(|bytes| bytes.to_vec()); + let bytes = dtype.write_flatbuffer_bytes(); let deserialized = DType::try_from(root::(&bytes).unwrap()).unwrap(); assert_eq!(dtype, deserialized); } diff --git a/vortex-flatbuffers/src/lib.rs b/vortex-flatbuffers/src/lib.rs index b45a50c18..118825603 100644 --- a/vortex-flatbuffers/src/lib.rs +++ b/vortex-flatbuffers/src/lib.rs @@ -149,11 +149,6 @@ pub trait FlatBufferToBytes { /// Write the flatbuffer to a byte vector and return the vector along with the starting /// position of the flatbuffer in the vector. fn write_flatbuffer_bytes(&self) -> Buffer; - - #[deprecated( - note = "Use `write_flatbuffer_bytes` instead. This method will be removed in a future release." - )] - fn with_flatbuffer_bytes R>(&self, f: Fn) -> R; } impl FlatBufferToBytes for F { @@ -166,11 +161,4 @@ impl FlatBufferToBytes for F { let end = vec.len(); Buffer::from(vec).slice(start..end) } - - fn with_flatbuffer_bytes R>(&self, f: Fn) -> R { - let mut fbb = FlatBufferBuilder::new(); - let root_offset = self.write_flatbuffer(&mut fbb); - fbb.finish_minimal(root_offset); - f(fbb.finished_data()) - } } diff --git a/vortex-ipc/src/messages/mod.rs b/vortex-ipc/src/messages/mod.rs index 2f30d81a7..dd00afa7a 100644 --- a/vortex-ipc/src/messages/mod.rs +++ b/vortex-ipc/src/messages/mod.rs @@ -38,7 +38,7 @@ impl WriteFlatBuffer for IPCMessage { fbb, &fba::BufferArgs { length: buffer.len() as u64, - padding: padding as u16, + padding: padding.try_into().vortex_expect("padding must fit in u16"), }, ) .as_union_value() @@ -82,12 +82,12 @@ impl WriteFlatBuffer for ArrayDataWriter<'_> { for array_data in self.array.depth_first_traversal() { if let Some(buffer) = array_data.buffer() { let aligned_size = buffer.len().next_multiple_of(ALIGNMENT); - let padding = (aligned_size - buffer.len()) as u16; + let padding = aligned_size - buffer.len(); buffers.push(fba::Buffer::create( fbb, &fba::BufferArgs { length: buffer.len() as u64, - padding, + padding: padding.try_into().vortex_expect("padding must fit in u16"), }, )); } @@ -154,7 +154,7 @@ impl WriteFlatBuffer for ArrayWriter<'_> { .array .buffer() .is_some() - .then(|| self.buffer_idx) + .then_some(self.buffer_idx) .map(|buffer_idx| fbb.create_vector_from_iter(std::iter::once(buffer_idx))); let stats = Some(self.array.statistics().write_flatbuffer(fbb)); diff --git a/vortex-ipc/src/messages/reader.rs b/vortex-ipc/src/messages/reader.rs index 242b1fae9..2a4d72abb 100644 --- a/vortex-ipc/src/messages/reader.rs +++ b/vortex-ipc/src/messages/reader.rs @@ -114,13 +114,13 @@ impl MessageReader { ctx: Arc, dtype: DType, ) -> VortexResult> { - let all_buffers_size: usize = match self.peek().and_then(|m| m.header_as_array_data()) { + let all_buffers_size = match self.peek().and_then(|m| m.header_as_array_data()) { None => return Ok(None), Some(array_data) => array_data .buffers() .unwrap_or_default() .iter() - .map(|b| b.length() as usize + (b.padding() as usize)) + .map(|b| b.length() + (b.padding() as u64)) .sum(), }; @@ -129,7 +129,7 @@ impl MessageReader { )); // Issue a single read to grab all buffers - let all_buffers = self.read.read_bytes(all_buffers_size as u64).await?; + let all_buffers = self.read.read_bytes(all_buffers_size).await?; if array_reader.read(all_buffers)?.is_some() { unreachable!("This is an implementation bug") @@ -285,7 +285,11 @@ impl ArrayMessageReader { self.fb_msg = Some(Buffer::from(bytes)); self.state = ReadState::ReadingBuffers; - Ok(Some(buffers_size as usize)) + Ok(Some( + buffers_size + .try_into() + .vortex_expect("Cannot cast to usize"), + )) } ReadState::ReadingBuffers => { // Split out into individual buffers @@ -298,7 +302,12 @@ impl ArrayMessageReader { .iter() .map(|buffer| { // Grab the buffer - let data_buffer = bytes.split_to(buffer.length() as usize); + let data_buffer = bytes.split_to( + buffer + .length() + .try_into() + .vortex_expect("Buffer size does not fit into usize"), + ); // Strip off any padding from the previous buffer bytes.advance(buffer.padding() as usize); Buffer::from(data_buffer) From b368ec0bc6ddc3cd4ff8a1263a5d57b758f1bc9d Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sat, 14 Dec 2024 16:04:23 +0000 Subject: [PATCH 4/6] Clean up --- vortex-flatbuffers/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vortex-flatbuffers/Cargo.toml b/vortex-flatbuffers/Cargo.toml index 4444baf7f..8aa2c3500 100644 --- a/vortex-flatbuffers/Cargo.toml +++ b/vortex-flatbuffers/Cargo.toml @@ -18,8 +18,7 @@ dtype = [] scalar = ["dtype"] array = ["dtype", "scalar"] ipc = ["array"] -file = ["array", "layout"] -layout = [] +file = ["ipc"] [dependencies] flatbuffers = { workspace = true } From 34edbf5755e18f838cf18e1f24f49b76e470b39d Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sat, 14 Dec 2024 16:09:05 +0000 Subject: [PATCH 5/6] Clean up --- vortex-flatbuffers/src/generated/array.rs | 54 ++++----- vortex-flatbuffers/src/generated/dtype.rs | 126 ++++++++++---------- vortex-flatbuffers/src/generated/footer.rs | 30 ++--- vortex-flatbuffers/src/generated/message.rs | 18 +-- vortex-flatbuffers/src/generated/scalar.rs | 30 ++--- 5 files changed, 129 insertions(+), 129 deletions(-) diff --git a/vortex-flatbuffers/src/generated/array.rs b/vortex-flatbuffers/src/generated/array.rs index c2c4e346c..3e841ab31 100644 --- a/vortex-flatbuffers/src/generated/array.rs +++ b/vortex-flatbuffers/src/generated/array.rs @@ -38,8 +38,8 @@ impl<'a> ArrayData<'a> { ArrayData { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args ArrayDataArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ArrayDataBuilder::new(_fbb); @@ -106,11 +106,11 @@ impl<'a> Default for ArrayDataArgs<'a> { } } -pub struct ArrayDataBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct ArrayDataBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> ArrayDataBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayDataBuilder<'a, 'b, A> { #[inline] pub fn add_array(&mut self, array: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(ArrayData::VT_ARRAY, array); @@ -124,7 +124,7 @@ impl<'a: 'b, 'b> ArrayDataBuilder<'a, 'b> { self.fbb_.push_slot_always::>(ArrayData::VT_BUFFERS, buffers); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ArrayDataBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ArrayDataBuilder<'a, 'b, A> { let start = _fbb.start_table(); ArrayDataBuilder { fbb_: _fbb, @@ -174,8 +174,8 @@ impl<'a> Array<'a> { Array { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args ArrayArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ArrayBuilder::new(_fbb); @@ -261,11 +261,11 @@ impl<'a> Default for ArrayArgs<'a> { } } -pub struct ArrayBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct ArrayBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> ArrayBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayBuilder<'a, 'b, A> { #[inline] pub fn add_encoding(&mut self, encoding: u16) { self.fbb_.push_slot::(Array::VT_ENCODING, encoding, 0); @@ -287,7 +287,7 @@ impl<'a: 'b, 'b> ArrayBuilder<'a, 'b> { self.fbb_.push_slot_always::>(Array::VT_STATS, stats); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ArrayBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ArrayBuilder<'a, 'b, A> { let start = _fbb.start_table(); ArrayBuilder { fbb_: _fbb, @@ -345,8 +345,8 @@ impl<'a> ArrayStats<'a> { ArrayStats { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args ArrayStatsArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ArrayStatsBuilder::new(_fbb); @@ -498,11 +498,11 @@ impl<'a> Default for ArrayStatsArgs<'a> { } } -pub struct ArrayStatsBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct ArrayStatsBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> ArrayStatsBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ArrayStatsBuilder<'a, 'b, A> { #[inline] pub fn add_min(&mut self, min: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(ArrayStats::VT_MIN, min); @@ -548,7 +548,7 @@ impl<'a: 'b, 'b> ArrayStatsBuilder<'a, 'b> { self.fbb_.push_slot_always::(ArrayStats::VT_UNCOMPRESSED_SIZE_IN_BYTES, uncompressed_size_in_bytes); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ArrayStatsBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ArrayStatsBuilder<'a, 'b, A> { let start = _fbb.start_table(); ArrayStatsBuilder { fbb_: _fbb, @@ -603,8 +603,8 @@ impl<'a> Buffer<'a> { Buffer { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args BufferArgs ) -> flatbuffers::WIPOffset> { let mut builder = BufferBuilder::new(_fbb); @@ -659,11 +659,11 @@ impl<'a> Default for BufferArgs { } } -pub struct BufferBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct BufferBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> BufferBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> BufferBuilder<'a, 'b, A> { #[inline] pub fn add_length(&mut self, length: u64) { self.fbb_.push_slot::(Buffer::VT_LENGTH, length, 0); @@ -673,7 +673,7 @@ impl<'a: 'b, 'b> BufferBuilder<'a, 'b> { self.fbb_.push_slot::(Buffer::VT_PADDING, padding, 0); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BufferBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> BufferBuilder<'a, 'b, A> { let start = _fbb.start_table(); BufferBuilder { fbb_: _fbb, @@ -756,13 +756,13 @@ pub unsafe fn size_prefixed_root_as_array_data_unchecked(buf: &[u8]) -> ArrayDat flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_array_data_buffer<'a, 'b>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub fn finish_array_data_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_array_data_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_array_data_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/dtype.rs b/vortex-flatbuffers/src/generated/dtype.rs index af311ad69..695eee413 100644 --- a/vortex-flatbuffers/src/generated/dtype.rs +++ b/vortex-flatbuffers/src/generated/dtype.rs @@ -271,8 +271,8 @@ impl<'a> Null<'a> { Null { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, _args: &'args NullArgs ) -> flatbuffers::WIPOffset> { let mut builder = NullBuilder::new(_fbb); @@ -302,13 +302,13 @@ impl<'a> Default for NullArgs { } } -pub struct NullBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct NullBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> NullBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> NullBuilder<'a, 'b, A> { #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> NullBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> NullBuilder<'a, 'b, A> { let start = _fbb.start_table(); NullBuilder { fbb_: _fbb, @@ -351,8 +351,8 @@ impl<'a> Bool<'a> { Bool { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args BoolArgs ) -> flatbuffers::WIPOffset> { let mut builder = BoolBuilder::new(_fbb); @@ -394,17 +394,17 @@ impl<'a> Default for BoolArgs { } } -pub struct BoolBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct BoolBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> BoolBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> BoolBuilder<'a, 'b, A> { #[inline] pub fn add_nullable(&mut self, nullable: bool) { self.fbb_.push_slot::(Bool::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BoolBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> BoolBuilder<'a, 'b, A> { let start = _fbb.start_table(); BoolBuilder { fbb_: _fbb, @@ -449,8 +449,8 @@ impl<'a> Primitive<'a> { Primitive { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args PrimitiveArgs ) -> flatbuffers::WIPOffset> { let mut builder = PrimitiveBuilder::new(_fbb); @@ -503,11 +503,11 @@ impl<'a> Default for PrimitiveArgs { } } -pub struct PrimitiveBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct PrimitiveBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> PrimitiveBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PrimitiveBuilder<'a, 'b, A> { #[inline] pub fn add_ptype(&mut self, ptype: PType) { self.fbb_.push_slot::(Primitive::VT_PTYPE, ptype, PType::U8); @@ -517,7 +517,7 @@ impl<'a: 'b, 'b> PrimitiveBuilder<'a, 'b> { self.fbb_.push_slot::(Primitive::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PrimitiveBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PrimitiveBuilder<'a, 'b, A> { let start = _fbb.start_table(); PrimitiveBuilder { fbb_: _fbb, @@ -564,8 +564,8 @@ impl<'a> Decimal<'a> { Decimal { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args DecimalArgs ) -> flatbuffers::WIPOffset> { let mut builder = DecimalBuilder::new(_fbb); @@ -631,11 +631,11 @@ impl<'a> Default for DecimalArgs { } } -pub struct DecimalBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct DecimalBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> DecimalBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecimalBuilder<'a, 'b, A> { #[inline] pub fn add_precision(&mut self, precision: u8) { self.fbb_.push_slot::(Decimal::VT_PRECISION, precision, 0); @@ -649,7 +649,7 @@ impl<'a: 'b, 'b> DecimalBuilder<'a, 'b> { self.fbb_.push_slot::(Decimal::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DecimalBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecimalBuilder<'a, 'b, A> { let start = _fbb.start_table(); DecimalBuilder { fbb_: _fbb, @@ -695,8 +695,8 @@ impl<'a> Utf8<'a> { Utf8 { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args Utf8Args ) -> flatbuffers::WIPOffset> { let mut builder = Utf8Builder::new(_fbb); @@ -738,17 +738,17 @@ impl<'a> Default for Utf8Args { } } -pub struct Utf8Builder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct Utf8Builder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> Utf8Builder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> Utf8Builder<'a, 'b, A> { #[inline] pub fn add_nullable(&mut self, nullable: bool) { self.fbb_.push_slot::(Utf8::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> Utf8Builder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> Utf8Builder<'a, 'b, A> { let start = _fbb.start_table(); Utf8Builder { fbb_: _fbb, @@ -792,8 +792,8 @@ impl<'a> Binary<'a> { Binary { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args BinaryArgs ) -> flatbuffers::WIPOffset> { let mut builder = BinaryBuilder::new(_fbb); @@ -835,17 +835,17 @@ impl<'a> Default for BinaryArgs { } } -pub struct BinaryBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct BinaryBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> BinaryBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> BinaryBuilder<'a, 'b, A> { #[inline] pub fn add_nullable(&mut self, nullable: bool) { self.fbb_.push_slot::(Binary::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BinaryBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> BinaryBuilder<'a, 'b, A> { let start = _fbb.start_table(); BinaryBuilder { fbb_: _fbb, @@ -891,8 +891,8 @@ impl<'a> Struct_<'a> { Struct_ { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args Struct_Args<'args> ) -> flatbuffers::WIPOffset> { let mut builder = Struct_Builder::new(_fbb); @@ -956,11 +956,11 @@ impl<'a> Default for Struct_Args<'a> { } } -pub struct Struct_Builder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct Struct_Builder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> Struct_Builder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> Struct_Builder<'a, 'b, A> { #[inline] pub fn add_names(&mut self, names: flatbuffers::WIPOffset>>) { self.fbb_.push_slot_always::>(Struct_::VT_NAMES, names); @@ -974,7 +974,7 @@ impl<'a: 'b, 'b> Struct_Builder<'a, 'b> { self.fbb_.push_slot::(Struct_::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> Struct_Builder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> Struct_Builder<'a, 'b, A> { let start = _fbb.start_table(); Struct_Builder { fbb_: _fbb, @@ -1021,8 +1021,8 @@ impl<'a> List<'a> { List { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args ListArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ListBuilder::new(_fbb); @@ -1075,11 +1075,11 @@ impl<'a> Default for ListArgs<'a> { } } -pub struct ListBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct ListBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> ListBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ListBuilder<'a, 'b, A> { #[inline] pub fn add_element_type(&mut self, element_type: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(List::VT_ELEMENT_TYPE, element_type); @@ -1089,7 +1089,7 @@ impl<'a: 'b, 'b> ListBuilder<'a, 'b> { self.fbb_.push_slot::(List::VT_NULLABLE, nullable, false); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ListBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ListBuilder<'a, 'b, A> { let start = _fbb.start_table(); ListBuilder { fbb_: _fbb, @@ -1136,8 +1136,8 @@ impl<'a> Extension<'a> { Extension { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args ExtensionArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ExtensionBuilder::new(_fbb); @@ -1201,11 +1201,11 @@ impl<'a> Default for ExtensionArgs<'a> { } } -pub struct ExtensionBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct ExtensionBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> ExtensionBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ExtensionBuilder<'a, 'b, A> { #[inline] pub fn add_id(&mut self, id: flatbuffers::WIPOffset<&'b str>) { self.fbb_.push_slot_always::>(Extension::VT_ID, id); @@ -1219,7 +1219,7 @@ impl<'a: 'b, 'b> ExtensionBuilder<'a, 'b> { self.fbb_.push_slot_always::>(Extension::VT_METADATA, metadata); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ExtensionBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ExtensionBuilder<'a, 'b, A> { let start = _fbb.start_table(); ExtensionBuilder { fbb_: _fbb, @@ -1266,8 +1266,8 @@ impl<'a> DType<'a> { DType { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args DTypeArgs ) -> flatbuffers::WIPOffset> { let mut builder = DTypeBuilder::new(_fbb); @@ -1467,11 +1467,11 @@ impl<'a> Default for DTypeArgs { } } -pub struct DTypeBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct DTypeBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> DTypeBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DTypeBuilder<'a, 'b, A> { #[inline] pub fn add_type_type(&mut self, type_type: Type) { self.fbb_.push_slot::(DType::VT_TYPE_TYPE, type_type, Type::NONE); @@ -1481,7 +1481,7 @@ impl<'a: 'b, 'b> DTypeBuilder<'a, 'b> { self.fbb_.push_slot_always::>(DType::VT_TYPE_, type_); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DTypeBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DTypeBuilder<'a, 'b, A> { let start = _fbb.start_table(); DTypeBuilder { fbb_: _fbb, @@ -1632,13 +1632,13 @@ pub unsafe fn size_prefixed_root_as_dtype_unchecked(buf: &[u8]) -> DType { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_dtype_buffer<'a, 'b>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub fn finish_dtype_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_dtype_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_dtype_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/footer.rs b/vortex-flatbuffers/src/generated/footer.rs index d242f5bc6..33551bacb 100644 --- a/vortex-flatbuffers/src/generated/footer.rs +++ b/vortex-flatbuffers/src/generated/footer.rs @@ -176,8 +176,8 @@ impl<'a> Layout<'a> { Layout { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args LayoutArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = LayoutBuilder::new(_fbb); @@ -263,11 +263,11 @@ impl<'a> Default for LayoutArgs<'a> { } } -pub struct LayoutBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct LayoutBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> LayoutBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> LayoutBuilder<'a, 'b, A> { #[inline] pub fn add_encoding(&mut self, encoding: u16) { self.fbb_.push_slot::(Layout::VT_ENCODING, encoding, 0); @@ -289,7 +289,7 @@ impl<'a: 'b, 'b> LayoutBuilder<'a, 'b> { self.fbb_.push_slot_always::>(Layout::VT_METADATA, metadata); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> LayoutBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> LayoutBuilder<'a, 'b, A> { let start = _fbb.start_table(); LayoutBuilder { fbb_: _fbb, @@ -350,8 +350,8 @@ impl<'a> Postscript<'a> { Postscript { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args PostscriptArgs ) -> flatbuffers::WIPOffset> { let mut builder = PostscriptBuilder::new(_fbb); @@ -404,11 +404,11 @@ impl<'a> Default for PostscriptArgs { } } -pub struct PostscriptBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct PostscriptBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> PostscriptBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PostscriptBuilder<'a, 'b, A> { #[inline] pub fn add_schema_offset(&mut self, schema_offset: u64) { self.fbb_.push_slot::(Postscript::VT_SCHEMA_OFFSET, schema_offset, 0); @@ -418,7 +418,7 @@ impl<'a: 'b, 'b> PostscriptBuilder<'a, 'b> { self.fbb_.push_slot::(Postscript::VT_LAYOUT_OFFSET, layout_offset, 0); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PostscriptBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PostscriptBuilder<'a, 'b, A> { let start = _fbb.start_table(); PostscriptBuilder { fbb_: _fbb, @@ -501,13 +501,13 @@ pub unsafe fn size_prefixed_root_as_postscript_unchecked(buf: &[u8]) -> Postscri flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_postscript_buffer<'a, 'b>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub fn finish_postscript_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_postscript_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_postscript_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/message.rs b/vortex-flatbuffers/src/generated/message.rs index 9c73bff11..a4a743317 100644 --- a/vortex-flatbuffers/src/generated/message.rs +++ b/vortex-flatbuffers/src/generated/message.rs @@ -294,8 +294,8 @@ impl<'a> Message<'a> { Message { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args MessageArgs ) -> flatbuffers::WIPOffset> { let mut builder = MessageBuilder::new(_fbb); @@ -410,11 +410,11 @@ impl<'a> Default for MessageArgs { } } -pub struct MessageBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct MessageBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> MessageBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> MessageBuilder<'a, 'b, A> { #[inline] pub fn add_version(&mut self, version: MessageVersion) { self.fbb_.push_slot::(Message::VT_VERSION, version, MessageVersion::V0); @@ -428,7 +428,7 @@ impl<'a: 'b, 'b> MessageBuilder<'a, 'b> { self.fbb_.push_slot_always::>(Message::VT_HEADER, header); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MessageBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> MessageBuilder<'a, 'b, A> { let start = _fbb.start_table(); MessageBuilder { fbb_: _fbb, @@ -538,13 +538,13 @@ pub unsafe fn size_prefixed_root_as_message_unchecked(buf: &[u8]) -> Message { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_message_buffer<'a, 'b>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub fn finish_message_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_message_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_message_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } diff --git a/vortex-flatbuffers/src/generated/scalar.rs b/vortex-flatbuffers/src/generated/scalar.rs index 300c17119..5864130da 100644 --- a/vortex-flatbuffers/src/generated/scalar.rs +++ b/vortex-flatbuffers/src/generated/scalar.rs @@ -34,8 +34,8 @@ impl<'a> Scalar<'a> { Scalar { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args ScalarArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ScalarBuilder::new(_fbb); @@ -88,11 +88,11 @@ impl<'a> Default for ScalarArgs<'a> { } } -pub struct ScalarBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct ScalarBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> ScalarBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ScalarBuilder<'a, 'b, A> { #[inline] pub fn add_dtype(&mut self, dtype: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(Scalar::VT_DTYPE, dtype); @@ -102,7 +102,7 @@ impl<'a: 'b, 'b> ScalarBuilder<'a, 'b> { self.fbb_.push_slot_always::>(Scalar::VT_VALUE, value); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ScalarBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ScalarBuilder<'a, 'b, A> { let start = _fbb.start_table(); ScalarBuilder { fbb_: _fbb, @@ -149,8 +149,8 @@ impl<'a> ScalarValue<'a> { ScalarValue { _tab: table } } #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, args: &'args ScalarValueArgs<'args> ) -> flatbuffers::WIPOffset> { let mut builder = ScalarValueBuilder::new(_fbb); @@ -192,17 +192,17 @@ impl<'a> Default for ScalarValueArgs<'a> { } } -pub struct ScalarValueBuilder<'a: 'b, 'b> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub struct ScalarValueBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, start_: flatbuffers::WIPOffset, } -impl<'a: 'b, 'b> ScalarValueBuilder<'a, 'b> { +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> ScalarValueBuilder<'a, 'b, A> { #[inline] pub fn add_flex(&mut self, flex: flatbuffers::WIPOffset>) { self.fbb_.push_slot_always::>(ScalarValue::VT_FLEX, flex); } #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ScalarValueBuilder<'a, 'b> { + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> ScalarValueBuilder<'a, 'b, A> { let start = _fbb.start_table(); ScalarValueBuilder { fbb_: _fbb, @@ -285,13 +285,13 @@ pub unsafe fn size_prefixed_root_as_scalar_unchecked(buf: &[u8]) -> Scalar { flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] -pub fn finish_scalar_buffer<'a, 'b>( - fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, +pub fn finish_scalar_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish(root, None); } #[inline] -pub fn finish_size_prefixed_scalar_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { +pub fn finish_size_prefixed_scalar_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset>) { fbb.finish_size_prefixed(root, None); } From 75fd1e050becd65debf69068ed5525d54d1b2f91 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 16 Dec 2024 17:09:18 +0000 Subject: [PATCH 6/6] Message Codecs --- vortex-dtype/src/serde/flatbuffers/mod.rs | 2 +- vortex-file/src/write/writer.rs | 2 +- vortex-flatbuffers/src/lib.rs | 8 +++----- vortex-ipc/src/messages/writer.rs | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/vortex-dtype/src/serde/flatbuffers/mod.rs b/vortex-dtype/src/serde/flatbuffers/mod.rs index 6236982b3..4c36d2266 100644 --- a/vortex-dtype/src/serde/flatbuffers/mod.rs +++ b/vortex-dtype/src/serde/flatbuffers/mod.rs @@ -259,7 +259,7 @@ mod test { use std::sync::Arc; use flatbuffers::root; - use vortex_flatbuffers::FlatBufferToBytes; + use vortex_flatbuffers::WriteFlatBufferExt; use crate::nullability::Nullability; use crate::{flatbuffers as fb, DType, PType, StructDType}; diff --git a/vortex-file/src/write/writer.rs b/vortex-file/src/write/writer.rs index 0c48616b9..164f1b2db 100644 --- a/vortex-file/src/write/writer.rs +++ b/vortex-file/src/write/writer.rs @@ -11,7 +11,7 @@ use vortex_array::stream::ArrayStream; use vortex_array::{ArrayData, ArrayLen}; use vortex_dtype::DType; use vortex_error::{vortex_bail, vortex_err, VortexExpect as _, VortexResult}; -use vortex_flatbuffers::{FlatBufferRoot, FlatBufferToBytes, WriteFlatBuffer}; +use vortex_flatbuffers::{FlatBufferRoot, WriteFlatBuffer, WriteFlatBufferExt}; use vortex_io::VortexWrite; use vortex_ipc::messages::writer::MessageWriter; use vortex_ipc::stream_writer::ByteRange; diff --git a/vortex-flatbuffers/src/lib.rs b/vortex-flatbuffers/src/lib.rs index 118825603..6b8860c3d 100644 --- a/vortex-flatbuffers/src/lib.rs +++ b/vortex-flatbuffers/src/lib.rs @@ -145,14 +145,12 @@ pub trait WriteFlatBuffer { ) -> WIPOffset>; } -pub trait FlatBufferToBytes { - /// Write the flatbuffer to a byte vector and return the vector along with the starting - /// position of the flatbuffer in the vector. +pub trait WriteFlatBufferExt: WriteFlatBuffer + FlatBufferRoot { + /// Write the flatbuffer into a [`Buffer`]. fn write_flatbuffer_bytes(&self) -> Buffer; } -impl FlatBufferToBytes for F { - /// Write the flatbuffer into a [`vortex_buffer::Buffer`]. +impl WriteFlatBufferExt for F { fn write_flatbuffer_bytes(&self) -> Buffer { let mut fbb = FlatBufferBuilder::new(); let root_offset = self.write_flatbuffer(&mut fbb); diff --git a/vortex-ipc/src/messages/writer.rs b/vortex-ipc/src/messages/writer.rs index 205a63a61..c7941fe68 100644 --- a/vortex-ipc/src/messages/writer.rs +++ b/vortex-ipc/src/messages/writer.rs @@ -8,7 +8,7 @@ use vortex_buffer::io_buf::IoBuf; use vortex_buffer::Buffer; use vortex_dtype::DType; use vortex_error::VortexUnwrap; -use vortex_flatbuffers::{FlatBufferToBytes, WriteFlatBuffer}; +use vortex_flatbuffers::{WriteFlatBuffer, WriteFlatBufferExt}; use vortex_io::VortexWrite; use crate::messages::IPCMessage;