Optimize ByteArray serialization (#310)

packages/consensus/src/codec.cairo

@@ -4,6 +4,7 @@ use super::types::transaction::{Transaction, TxIn, TxOut, OutPoint};
 use utils::hash::Digest;
 use utils::word_array::{WordArray, WordArrayTrait, WordSpan, WordSpanTrait};
 use core::traits::DivRem;
+use core::serde::Serde;
 
 pub trait Encode<T> {
     /// Encodes using Bitcoin codec and appends to the buffer.
@@ -31,15 +32,26 @@ pub impl EncodeSpan<T, +Encode<T>> of Encode<Span<T>> {
 }
 
 /// `Encode` trait implementation for `ByteArray`.
-/// TODO: use WordArray for arguments instead of ByteArray.
-/// Extra optimization: predict word array offset
 pub impl EncodeByteArray of Encode<ByteArray> {
     fn encode_to(self: @ByteArray, ref dest: WordArray) {
         encode_compact_size(self.len(), ref dest);
-        let num_bytes = self.len();
-        for i in 0..num_bytes {
-            dest.append_u8(self[i]);
-        }
+
+        // We cannot get access to ByteArray internals, but we can use Serde to convert
+        // the inner triple (Array<bytes31>, last_word, last_word_len) into array of felts.
+        // WARNING: these assumptions might be broken if ByteArray implementation changes
+        // in corelib. A better approach would be introducing IntoIterator<felt252> for ByteArray.
+        let mut out: Array<felt252> = Default::default();
+        self.serialize(ref out);
+
+        let mut num_bytes31 = out.pop_front().unwrap();
+        while num_bytes31 != 0 {
+            dest.append_bytes31(out.pop_front().unwrap());
+            num_bytes31 -= 1;
+        };
+
+        let last_word = out.pop_front().unwrap();
+        let last_word_len = out.pop_front().unwrap();
+        dest.append_bytes(last_word, last_word_len.try_into().unwrap());
     }
 }
 

packages/utils/src/bit_shifts.cairo

@@ -90,6 +90,29 @@ pub fn pow2(exponent: u32) -> u64 {
     *hardcoded_results.span()[exponent]
 }
 
+pub fn pow256(exponent: u32) -> NonZero<u256> {
+    let hardcoded_results: [u256; 32] = [
+        0x1, 0x100, 0x10000, 0x1000000, 0x100000000, 0x10000000000, 0x1000000000000,
+        0x100000000000000, 0x10000000000000000, 0x1000000000000000000, 0x100000000000000000000,
+        0x10000000000000000000000, 0x1000000000000000000000000, 0x100000000000000000000000000,
+        0x10000000000000000000000000000, 0x1000000000000000000000000000000,
+        0x100000000000000000000000000000000, 0x10000000000000000000000000000000000,
+        0x1000000000000000000000000000000000000, 0x100000000000000000000000000000000000000,
+        0x10000000000000000000000000000000000000000, 0x1000000000000000000000000000000000000000000,
+        0x100000000000000000000000000000000000000000000,
+        0x10000000000000000000000000000000000000000000000,
+        0x1000000000000000000000000000000000000000000000000,
+        0x100000000000000000000000000000000000000000000000000,
+        0x10000000000000000000000000000000000000000000000000000,
+        0x1000000000000000000000000000000000000000000000000000000,
+        0x100000000000000000000000000000000000000000000000000000000,
+        0x10000000000000000000000000000000000000000000000000000000000,
+        0x1000000000000000000000000000000000000000000000000000000000000,
+        0x100000000000000000000000000000000000000000000000000000000000000,
+    ];
+    (*hardcoded_results.span()[exponent]).try_into().unwrap()
+}
+
 #[cfg(test)]
 mod tests {
     use super::{fast_pow, pow2, shr_u64};

packages/utils/src/word_array.cairo

@@ -5,6 +5,7 @@
 //! SHA256 hash function which operates on 4-byte words.
 
 use core::traits::DivRem;
+use utils::bit_shifts::pow256;
 
 /// Array of 4-byte words where the last word can be partial.
 #[derive(Drop, Debug, Default, PartialEq)]
@@ -179,6 +180,100 @@ pub impl WordArrayImpl of WordArrayTrait {
         }
     }
 
+    /// Append 31 bytes (max number of full bytes that single field element can store)
+    fn append_bytes31(ref self: WordArray, value: felt252) {
+        let bytes31: u256 = value.into(); // DivRem is not implemented for felt252
+        let bytes28 = if self.last_input_num_bytes == 0 {
+            let (bytes28, last_word) = DivRem::div_rem(bytes31, 0x1000000);
+            self.last_input_word = last_word.try_into().expect('append_bytes31/1');
+            self.last_input_num_bytes = 3;
+            bytes28
+        } else if self.last_input_num_bytes == 1 {
+            let (first_word, bytes28) = DivRem::div_rem(
+                bytes31, 0x100000000000000000000000000000000000000000000000000000000,
+            );
+            self.append_word(first_word.try_into().expect('append_bytes31/2'), 3);
+            bytes28
+        } else if self.last_input_num_bytes == 2 {
+            let (bytes29, last_word) = DivRem::div_rem(bytes31, 0x100);
+            let (first_word, bytes28) = DivRem::div_rem(
+                bytes29, 0x100000000000000000000000000000000000000000000000000000000,
+            );
+            self.append_word(first_word.try_into().expect('append_bytes31/3'), 2);
+            self.last_input_word = last_word.try_into().expect('4');
+            self.last_input_num_bytes = 1;
+            bytes28
+        } else {
+            let (bytes30, last_word) = DivRem::div_rem(bytes31, 0x10000);
+            let (first_word, bytes28) = DivRem::div_rem(
+                bytes30, 0x100000000000000000000000000000000000000000000000000000000,
+            );
+            self.append_word(first_word.try_into().expect('append_bytes31/5'), 1);
+            self.last_input_word = last_word.try_into().expect('append_bytes31/6');
+            self.last_input_num_bytes = 2;
+            bytes28
+        };
+
+        let (q0, r0) = DivRem::div_rem(bytes28, 0x100000000);
+        let (q1, r1) = DivRem::div_rem(q0, 0x100000000);
+        let (q2, r2) = DivRem::div_rem(q1, 0x100000000);
+        let (q3, r3) = DivRem::div_rem(q2, 0x100000000);
+        let (q4, r4) = DivRem::div_rem(q3, 0x100000000);
+        let (q5, r5) = DivRem::div_rem(q4, 0x100000000);
+        self.input.append(q5.try_into().expect('append_bytes31/7'));
+        self.input.append(r5.try_into().expect('append_bytes31/8'));
+        self.input.append(r4.try_into().expect('append_bytes31/9'));
+        self.input.append(r3.try_into().expect('append_bytes31/10'));
+        self.input.append(r2.try_into().expect('append_bytes31/11'));
+        self.input.append(r1.try_into().expect('append_bytes31/12'));
+        self.input.append(r0.try_into().expect('append_bytes31/13'));
+    }
+
+    /// Append up to 31 bytes (byte length provided), packed in a single field element.
+    fn append_bytes(ref self: WordArray, value: felt252, num_bytes: u32) {
+        let bytes: u256 = value.into(); // DivRem is not implemented for felt252
+        let (num_full_words, last_input_num_bytes) = DivRem::div_rem(
+            self.last_input_num_bytes + num_bytes, 4,
+        );
+
+        if num_full_words != 0 {
+            let (head, last_word) = if last_input_num_bytes == 0 {
+                (bytes, 0)
+            } else {
+                DivRem::div_rem(bytes, pow256(last_input_num_bytes))
+            };
+
+            let (mut full_words, mut full_words_num_bytes) = if self.last_input_num_bytes == 0 {
+                (head, num_bytes - last_input_num_bytes)
+            } else {
+                let first_word_num_bytes = 4 - self.last_input_num_bytes;
+                let full_words_num_bytes = num_bytes - last_input_num_bytes - first_word_num_bytes;
+                let (first_word, full_words) = DivRem::div_rem(head, pow256(full_words_num_bytes));
+                self
+                    .append_word(
+                        first_word.try_into().expect('append_bytes/0'), first_word_num_bytes,
+                    );
+                (full_words, full_words_num_bytes)
+            };
+
+            if full_words_num_bytes != 0 {
+                full_words_num_bytes -= 4;
+                while full_words_num_bytes != 0 {
+                    let (word, r) = DivRem::div_rem(full_words, pow256(full_words_num_bytes));
+                    self.input.append(word.try_into().expect('append_bytes/1'));
+                    full_words = r;
+                    full_words_num_bytes -= 4;
+                };
+                self.input.append(full_words.try_into().expect('append_bytes/2'));
+            }
+
+            self.last_input_word = last_word.try_into().expect('append_bytes/3');
+            self.last_input_num_bytes = last_input_num_bytes;
+        } else {
+            self.append_word(bytes.try_into().expect('append_bytes/3'), num_bytes);
+        }
+    }
+
     /// Split word array into components:
     /// (array of full 4-byte words, last word, number of bytes in the last word)
     fn into_components(self: WordArray) -> (Array<u32>, u32, u32) {
@@ -263,7 +358,7 @@ pub mod hex {
 #[cfg(test)]
 mod tests {
     use super::WordSpanTrait;
-    use super::hex::words_to_hex;
+    use super::hex::{words_to_hex};
     use super::{WordArray, WordArrayTrait};
 
     #[test]
@@ -342,4 +437,74 @@ mod tests {
         assert_eq!((0, 1), span.pop_back().unwrap());
         assert_eq!(Option::None, span.pop_back());
     }
+
+    #[test]
+    fn append_bytes31() {
+        let mut words: WordArray = Default::default();
+        words.append_bytes31(0x01020304050607080910111213141516171819202122232425262728293031);
+        assert_eq!(
+            "01020304050607080910111213141516171819202122232425262728293031",
+            words_to_hex(words.span()),
+        );
+
+        let mut words: WordArray = Default::default();
+        words.append_word(0xff, 1);
+        words.append_bytes31(0x01020304050607080910111213141516171819202122232425262728293031);
+        assert_eq!(
+            "ff01020304050607080910111213141516171819202122232425262728293031",
+            words_to_hex(words.span()),
+        );
+
+        let mut words: WordArray = Default::default();
+        words.append_word(0xfffe, 2);
+        words.append_bytes31(0x01020304050607080910111213141516171819202122232425262728293031);
+        assert_eq!(
+            "fffe01020304050607080910111213141516171819202122232425262728293031",
+            words_to_hex(words.span()),
+        );
+
+        let mut words: WordArray = Default::default();
+        words.append_word(0xfffefd, 3);
+        words.append_bytes31(0x01020304050607080910111213141516171819202122232425262728293031);
+        assert_eq!(
+            "fffefd01020304050607080910111213141516171819202122232425262728293031",
+            words_to_hex(words.span()),
+        );
+    }
+
+    #[test]
+    fn append_bytes() {
+        let mut words: WordArray = Default::default();
+        words.append_bytes(0x010203040506070809101112131415161718192021222324252627282930, 30);
+        assert_eq!(
+            "010203040506070809101112131415161718192021222324252627282930",
+            words_to_hex(words.span()),
+        );
+
+        let mut words: WordArray = Default::default();
+        words.append_word(0xff, 1);
+        words.append_bytes(0x01020304050607080910111213141516171819202122232425262728, 28);
+        assert_eq!(
+            "ff01020304050607080910111213141516171819202122232425262728",
+            words_to_hex(words.span()),
+        );
+
+        let mut words: WordArray = Default::default();
+        words.append_word(0xfffe, 2);
+        words.append_bytes(0x010203040506070809101112131415161718192021222324252627, 27);
+        assert_eq!(
+            "fffe010203040506070809101112131415161718192021222324252627",
+            words_to_hex(words.span()),
+        );
+
+        let mut words: WordArray = Default::default();
+        words.append_word(0xfffefd, 3);
+        words.append_bytes(0x01, 1);
+        assert_eq!("fffefd01", words_to_hex(words.span()));
+
+        let mut words: WordArray = Default::default();
+        words.append_word(0xfffefd, 3);
+        words.append_bytes(0x0102, 2);
+        assert_eq!("fffefd0102", words_to_hex(words.span()));
+    }
 }

scripts/data/integration_tests.sh

@@ -30,25 +30,6 @@ for arg in "$@"; do
 done
 
 ignored_files=(
-    "tests/data/full_209999.json", #cairo-run dies, to be investigated
-    "tests/data/full_403199.json", #cairo-run dies, to be investigated
-    "tests/data/full_481823.json", #cairo-run dies, to be investigated
-    "tests/data/full_489888.json", #cairo-run dies, to be investigated
-    "tests/data/full_491406.json", #cairo-run dies, to be investigated
-    "tests/data/full_629999.json", #cairo-run dies, to be investigated
-    "tests/data/full_709631.json", #cairo-run dies, to be investigated
-    "tests/data/full_774627.json", # Couldn't compute operand op1. Unknown value for memory cell 1:131082
-    "tests/data/full_839999.json", # Couldn't compute operand op1. Unknown value for memory cell 1:262154
-    "tests/data/full_116927.json", # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_150012.json", # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_2015.json",   # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_24834.json",  # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_32255.json",  # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_478557.json", # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_57042.json",  # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_72575.json",  # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_757752.json", # Run panicked with [108217864776563 ('blocks'), ].
-    # "tests/data/full_478557.json", #runs on server
 )
 
 ignore_file="tests/data/ignore"