update file format diagram docstring and make version a u16 again

spiraldb · Oct 19, 2024 · 40a5298 · 40a5298
1 parent 1ea6c57
commit 40a5298
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 11 deletions.
diff --git a/vortex-serde/src/layouts/mod.rs b/vortex-serde/src/layouts/mod.rs
@@ -5,7 +5,7 @@ mod pruning;
 #[cfg(test)]
 mod tests;
 
-pub const VERSION: u32 = 1;
+pub const VERSION: u16 = 1;
 pub const MAGIC_BYTES: [u8; 4] = *b"VRTX";
 // Size of serialized Postscript Flatbuffer
 pub const FOOTER_POSTSCRIPT_SIZE: usize = 32;

diff --git a/vortex-serde/src/layouts/read/footer.rs b/vortex-serde/src/layouts/read/footer.rs
@@ -16,26 +16,34 @@ use crate::FLATBUFFER_SIZE_LENGTH;
 /// Wrapper around serialized file footer. Provides handle on file schema and
 /// layout metadata to read the contents.
 ///
-/// # Footer format
+/// # File Format
 /// ┌────────────────────────────┐
 /// │                            │
-///              ...
+/// │            Data            │
+/// │    (Array IPC Messages)    │
+/// │                            │
+/// ├────────────────────────────┤
+/// │                            │
+/// │     Per-Column Metadata    │
+/// │    (Array IPC Messages)    │
+/// │                            │
 /// ├────────────────────────────┤
 /// │                            │
 /// │          Schema            │
 /// │                            │
 /// ├────────────────────────────┤
 /// │                            │
-/// │         Layouts            │
+/// │          Footer            │
+/// │    (Layouts + Row Count)   │
 /// │                            │
 /// ├────────────────────────────┤
 /// │                            │
 /// │        Postscript          │
-/// │  (Schema + Layout offset)  │
+/// │  (Schema/Footer Offsets)   │
 /// │        (32 bytes)          │
 /// │                            │
 /// ├────────────────────────────┤
-/// │      Version (4 bytes)     │
+/// │   Version Info (4 bytes)   │
 /// ├────────────────────────────┤
 /// │    Magic bytes (4 bytes)   │
 /// └────────────────────────────┘
@@ -152,14 +160,13 @@ impl LayoutDescriptorReader {
         let eof_loc = read_size - EOF_SIZE;
 
         let magic_bytes_loc = eof_loc + (EOF_SIZE - MAGIC_BYTES.len());
-
         let magic_number = &buf[magic_bytes_loc..];
         if magic_number != MAGIC_BYTES {
             vortex_bail!("Malformed file, invalid magic bytes, got {magic_number:?}")
         }
 
-        let version = u32::from_le_bytes(
-            buf[eof_loc..eof_loc + 4]
+        let version = u16::from_le_bytes(
+            buf[eof_loc..eof_loc + 2]
                 .try_into()
                 .map_err(|e| vortex_err!("Version was not a u16 {e}"))?,
         );

diff --git a/vortex-serde/src/layouts/write/writer.rs b/vortex-serde/src/layouts/write/writer.rs
@@ -15,7 +15,7 @@ use vortex_flatbuffers::WriteFlatBuffer;
 use crate::io::VortexWrite;
 use crate::layouts::write::footer::{Footer, Postscript};
 use crate::layouts::write::layouts::Layout;
-use crate::layouts::{EOF_SIZE, MAGIC_BYTES, VERSION};
+use crate::layouts::{EOF_SIZE, FOOTER_POSTSCRIPT_SIZE, MAGIC_BYTES, VERSION};
 use crate::MessageWriter;
 
 pub struct LayoutWriter<W> {
@@ -176,7 +176,7 @@ impl<W: VortexWrite> LayoutWriter<W> {
         w = write_fb_raw(w, ps).await?;
 
         let mut eof = [0u8; EOF_SIZE];
-        eof[0..4].copy_from_slice(&VERSION.to_le_bytes());
+        eof[0..2].copy_from_slice(&VERSION.to_le_bytes());
         eof[4..8].copy_from_slice(&MAGIC_BYTES);
         w.write_all(eof).await?;
         Ok(w)