handle zero bytes in input properly

rust-toolchain.toml

@@ -1,4 +1,4 @@
 [toolchain]
-channel = "stable"
+channel = "nightly-2024-08-14"
 components = ["rust-src", "rustfmt", "clippy"]
 profile = "minimal"

src/lib.rs

@@ -58,7 +58,14 @@ impl Symbol {
         // For little-endian platforms, this counts the number of *trailing* zeros
         let null_bytes = (numeric.leading_zeros() >> 3) as usize;
 
-        size_of::<Self>() - null_bytes
+        // Special case handling of a symbol with all-zeros. This is actually
+        // a 1-byte symbol containing 0x00.
+        let len = size_of::<Self>() - null_bytes;
+        if len == 0 {
+            1
+        } else {
+            len
+        }
     }
 
     /// Returns true if the symbol does not encode any bytes.
@@ -298,9 +305,9 @@ impl SymbolTable {
     ///
     /// # Safety
     ///
-    /// `in_ptr` and `out_ptr` must never be NULL or otherwise point to invalid memory.
+    /// `out_ptr` must never be NULL or otherwise point to invalid memory.
     // NOTE(aduffy): uncomment this line to make the function appear in profiles
-    // #[inline(never)]
+    #[inline(never)]
     pub(crate) unsafe fn compress_word(&self, word: u64, out_ptr: *mut u8) -> (usize, usize) {
         // Speculatively write the first byte of `word` at offset 1. This is necessary if it is an escape, and
         // if it isn't, it will be overwritten anyway.

tests/correctness.rs

@@ -29,17 +29,17 @@ fn test_train_on_empty() {
     );
 }
 
-// #[test]
-// fn test_zeros() {
-//     println!("training zeros");
-//     let training_data: Vec<u8> = vec![0, 1, 2, 3, 4];
-//     let trained = fsst_rs::train(&training_data);
-//     println!("compressing with zeros");
-//     let compressed = trained.compress(&[0, 4]);
-//     println!("decomperssing with zeros");
-//     assert_eq!(trained.decompress(&compressed), &[0, 4]);
-//     println!("done");
-// }
+#[test]
+fn test_zeros() {
+    println!("training zeros");
+    let training_data: Vec<u8> = vec![0, 1, 2, 3, 4];
+    let trained = fsst_rs::train(&training_data);
+    println!("compressing with zeros");
+    let compressed = trained.compress(&[0, 4]);
+    println!("decomperssing with zeros");
+    assert_eq!(trained.decompress(&compressed), &[0, 4]);
+    println!("done");
+}
 
 #[test]
 fn test_large() {