Update integration test for generic extraction.

mp2-common/src/eth.rs

@@ -237,6 +237,9 @@ impl StorageSlot {
     }
 }
 impl ProofQuery {
+    pub fn new(contract: Address, slot: StorageSlot) -> Self {
+        Self { contract, slot }
+    }
     pub fn new_simple_slot(address: Address, slot: usize) -> Self {
         Self {
             contract: address,

mp2-common/src/types.rs

@@ -112,6 +112,12 @@ impl From<HashOut<F>> for HashOutput {
     }
 }
 
+impl From<&HashOut<F>> for HashOutput {
+    fn from(value: &HashOut<F>) -> Self {
+        value.to_bytes().try_into().unwrap()
+    }
+}
+
 impl From<HashOutput> for HashOut<F> {
     fn from(value: HashOutput) -> Self {
         Self::from_bytes(&value.0)

mp2-v1/src/api.rs

@@ -21,6 +21,7 @@ use crate::{
 use alloy::primitives::Address;
 use anyhow::Result;
 use itertools::Itertools;
+use log::debug;
 use mp2_common::{
     digest::Digest,
     group_hashing::map_to_curve_point,
@@ -209,7 +210,7 @@ pub enum SlotInputs {
     MappingWithLength(Vec<SlotInput>, u8),
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
 pub struct SlotInput {
     /// Slot information of the variable
     pub(crate) slot: u8,
@@ -338,7 +339,7 @@ fn value_metadata<const MAX_COLUMNS: usize, const MAX_FIELD_PER_EVM: usize>(
 }
 
 /// Compute the table information for the value columns.
-fn compute_table_info(
+pub fn compute_table_info(
     inputs: Vec<SlotInput>,
     address: &Address,
     chain_id: u64,
@@ -438,8 +439,16 @@ pub fn metadata_hash<const MAX_COLUMNS: usize, const MAX_FIELD_PER_EVM: usize>(
         chain_id,
         extra,
     );
+    // Correspond to the computation of final extraction base circuit.
+    let value_digest = map_to_curve_point(&value_digest.to_fields());
     // add contract digest
     let contract_digest = contract_metadata_digest(contract_address);
+    debug!(
+        "METADATA_HASH ->\n\tvalues_ext_md = {:?}\n\tcontract_md = {:?}\n\tfinal_ex_md(contract + values_ex) = {:?}",
+        value_digest.to_weierstrass(),
+        contract_digest.to_weierstrass(),
+        (contract_digest + value_digest).to_weierstrass(),
+    );
     // compute final hash
     combine_digest_and_block(contract_digest + value_digest)
 }

mp2-v1/src/final_extraction/public_inputs.rs

@@ -6,7 +6,7 @@ use mp2_common::{
     public_inputs::{PublicInputCommon, PublicInputRange},
     types::{CBuilder, CURVE_TARGET_LEN},
     u256::{self, UInt256Target},
-    utils::{FromFields, FromTargets, ToTargets},
+    utils::{FromFields, FromTargets, ToTargets, TryIntoBool},
     F,
 };
 use plonky2::iop::target::{BoolTarget, Target};
@@ -110,6 +110,10 @@ impl<'a> PublicInputs<'a, F> {
     pub fn block_number(&self) -> u64 {
         U256::from_fields(self.bn).to()
     }
+    /// Get the merge flag
+    pub fn merge_flag(&self) -> bool {
+        self.merge[0].try_into_bool().unwrap()
+    }
 }
 
 impl<'a, T> PublicInputs<'a, T> {

mp2-v1/src/indexing/row.rs

@@ -172,6 +172,8 @@ pub struct RowPayload<PrimaryIndex: PartialEq + Eq + Default> {
     pub max: U256,
     /// Hash of this node
     pub hash: HashOutput,
+    /// Row unique data
+    pub row_unique_data: HashOutput,
 }
 
 impl<PrimaryIndex: std::fmt::Debug + Clone + Default + PartialEq + Eq> RowPayload<PrimaryIndex> {
@@ -199,6 +201,9 @@ impl<PrimaryIndex: std::fmt::Debug + Clone + Default + PartialEq + Eq> RowPayloa
         }
     }
 
+    pub fn column_value(&self, column_id: ColumnID) -> Option<U256> {
+        self.cells.get(&column_id).map(|c| c.value)
+    }
     pub fn secondary_index_value(&self) -> U256 {
         self.cells
             .get(&self.secondary_index_column)

mp2-v1/src/values_extraction/api.rs

@@ -899,11 +899,12 @@ mod tests {
             }
             // Mapping variable
             StorageSlot::Mapping(mapping_key, slot) => {
+                let outer_key_id = test_slot.outer_key_id.unwrap();
                 let metadata_digest = compute_leaf_mapping_metadata_digest::<
                     TEST_MAX_COLUMNS,
                     TEST_MAX_FIELD_PER_EVM,
                 >(
-                    table_info.clone(), slot as u8, test_slot.outer_key_id
+                    table_info.clone(), slot as u8, outer_key_id
                 );
 
                 let values_digest = compute_leaf_mapping_values_digest::<TEST_MAX_FIELD_PER_EVM>(
@@ -912,14 +913,14 @@ mod tests {
                     value,
                     mapping_key.clone(),
                     evm_word,
-                    test_slot.outer_key_id,
+                    outer_key_id,
                 );
 
                 let circuit_input = CircuitInput::new_mapping_variable_leaf(
                     node,
                     slot as u8,
                     mapping_key,
-                    test_slot.outer_key_id,
+                    outer_key_id,
                     metadata,
                 );
 
@@ -946,27 +947,27 @@ mod tests {
                 }
                 // Mapping Struct
                 StorageSlot::Mapping(mapping_key, slot) => {
-                    let metadata_digest = compute_leaf_mapping_metadata_digest::<
-                        TEST_MAX_COLUMNS,
-                        TEST_MAX_FIELD_PER_EVM,
-                    >(
-                        table_info.clone(), slot as u8, test_slot.outer_key_id
-                    );
+                    let outer_key_id = test_slot.outer_key_id.unwrap();
+                    let metadata_digest =
+                        compute_leaf_mapping_metadata_digest::<
+                            TEST_MAX_COLUMNS,
+                            TEST_MAX_FIELD_PER_EVM,
+                        >(table_info.clone(), slot as u8, outer_key_id);
 
                     let values_digest = compute_leaf_mapping_values_digest::<TEST_MAX_FIELD_PER_EVM>(
                         table_info,
                         &extracted_column_identifiers,
                         value,
                         mapping_key.clone(),
                         evm_word,
-                        test_slot.outer_key_id,
+                        outer_key_id,
                     );
 
                     let circuit_input = CircuitInput::new_mapping_variable_leaf(
                         node,
                         slot as u8,
                         mapping_key,
-                        test_slot.outer_key_id,
+                        outer_key_id,
                         metadata,
                     );
 
@@ -976,15 +977,15 @@ mod tests {
                 StorageSlot::Node(StorageSlotNode::Mapping(grand, inner_mapping_key)) => {
                     match *grand {
                         StorageSlot::Mapping(outer_mapping_key, slot) => {
-                            let metadata_digest = compute_leaf_mapping_of_mappings_metadata_digest::<
-                                TEST_MAX_COLUMNS,
-                                TEST_MAX_FIELD_PER_EVM,
-                            >(
-                                table_info.clone(),
-                                slot as u8,
-                                test_slot.outer_key_id,
-                                test_slot.inner_key_id,
-                            );
+                            let outer_key_id = test_slot.outer_key_id.unwrap();
+                            let inner_key_id = test_slot.inner_key_id.unwrap();
+                            let metadata_digest =
+                                compute_leaf_mapping_of_mappings_metadata_digest::<
+                                    TEST_MAX_COLUMNS,
+                                    TEST_MAX_FIELD_PER_EVM,
+                                >(
+                                    table_info.clone(), slot as u8, outer_key_id, inner_key_id
+                                );
 
                             let values_digest = compute_leaf_mapping_of_mappings_values_digest::<
                                 TEST_MAX_FIELD_PER_EVM,
@@ -995,17 +996,17 @@ mod tests {
                                 evm_word,
                                 outer_mapping_key.clone(),
                                 inner_mapping_key.clone(),
-                                test_slot.outer_key_id,
-                                test_slot.inner_key_id,
+                                outer_key_id,
+                                inner_key_id,
                             );
 
                             let circuit_input = CircuitInput::new_mapping_of_mappings_leaf(
                                 node,
                                 slot as u8,
                                 outer_mapping_key,
                                 inner_mapping_key,
-                                test_slot.outer_key_id,
-                                test_slot.inner_key_id,
+                                outer_key_id,
+                                inner_key_id,
                                 metadata,
                             );
 

mp2-v1/src/values_extraction/gadgets/column_gadget.rs

@@ -56,12 +56,17 @@ impl<'a, const MAX_FIELD_PER_EVM: usize> ColumnGadget<'a, MAX_FIELD_PER_EVM> {
         // as a big-endian integer.
         let bytes = &(0..=u8::MAX as u16).collect_vec();
         let mut lookup_inputs = [bytes; NUM_BITS_LOOKUP_TABLES];
-        let last_bits_lookup_indexes = add_last_bits_lookup_tables(b, lookup_inputs);
-        // The maxiumn lookup value is `u8::MAX + 8`, since the maxiumn `info.length` is 256,
+        // This maxiumn lookup value is `u8::MAX + 8`, since the maxiumn `info.length` is 256,
         // and we need to compute `first_bits_5(info.length + 7)`.
         let first_bits_5_input = (0..=u8::MAX as u16 + 8).collect_vec();
         lookup_inputs[4] = &first_bits_5_input;
         let first_bits_lookup_indexes = add_first_bits_lookup_tables(b, lookup_inputs);
+        lookup_inputs[4] = bytes;
+        // This maxiumn lookup value is `256`, since the maxiumn `info.length` is 256,
+        // and we need to compute `last_bits_3(info.length)`.
+        let last_bits_3_input = (0..=u8::MAX as u16 + 1).collect_vec();
+        lookup_inputs[2] = &last_bits_3_input;
+        let last_bits_lookup_indexes = add_last_bits_lookup_tables(b, lookup_inputs);
 
         // Accumulate to compute the value digest.
         let mut value_digest = b.curve_zero();
@@ -72,7 +77,7 @@ impl<'a, const MAX_FIELD_PER_EVM: usize> ColumnGadget<'a, MAX_FIELD_PER_EVM> {
             let is_extracted = self.is_extracted_columns[i];
 
             // Extract the value by column info.
-            let extracted_value = extract_value(
+            let extracted_value = extract_value_target(
                 b,
                 info,
                 self.value,
@@ -150,7 +155,7 @@ fn add_last_bits_lookup_tables(
 }
 
 /// Extract the value by the column info.
-fn extract_value(
+fn extract_value_target(
     b: &mut CBuilder,
     info: &ColumnInfoTarget,
     value_bytes: &[Target; MAPPING_LEAF_VALUE_LEN],
@@ -310,65 +315,72 @@ impl<const MAX_FIELD_PER_EVM: usize> ColumnGadgetData<MAX_FIELD_PER_EVM> {
 
     /// Compute the values digest.
     pub fn digest(&self) -> Point {
+        let value = self
+            .value
+            .map(|f| u8::try_from(f.to_canonical_u64()).unwrap());
         self.table_info[..self.num_extracted_columns]
             .iter()
             .fold(Point::NEUTRAL, |acc, info| {
-                let extracted_value = self.extract_value(info);
+                let extracted_value = extract_value(&value, info);
 
                 // digest = D(info.identifier || pack(extracted_value))
                 let inputs = once(info.identifier)
-                    .chain(extracted_value.pack(Endianness::Big))
+                    .chain(
+                        extracted_value
+                            .pack(Endianness::Big)
+                            .into_iter()
+                            .map(F::from_canonical_u32),
+                    )
                     .collect_vec();
                 let digest = map_to_curve_point(&inputs);
 
                 acc + digest
             })
     }
+}
 
-    fn extract_value(&self, info: &ColumnInfo) -> [F; MAPPING_LEAF_VALUE_LEN] {
-        let bit_offset = u8::try_from(info.bit_offset.to_canonical_u64()).unwrap();
-        assert!(bit_offset <= 8);
-        let [byte_offset, length] =
-            [info.byte_offset, info.length].map(|f| usize::try_from(f.to_canonical_u64()).unwrap());
-
-        let value_bytes = self
-            .value
-            .map(|f| u8::try_from(f.to_canonical_u64()).unwrap());
-
-        // last_byte_offset = info.byte_offset + ceil(info.length / 8) - 1
-        let last_byte_offset = byte_offset + length.div_ceil(8) - 1;
+pub fn extract_value(
+    value_bytes: &[u8; MAPPING_LEAF_VALUE_LEN],
+    info: &ColumnInfo,
+) -> [u8; MAPPING_LEAF_VALUE_LEN] {
+    let bit_offset = u8::try_from(info.bit_offset.to_canonical_u64()).unwrap();
+    assert!(bit_offset <= 8);
+    let [byte_offset, length] =
+        [info.byte_offset, info.length].map(|f| usize::try_from(f.to_canonical_u64()).unwrap());
 
-        // Extract all the bits of the field aligined with bytes.
-        let mut result_bytes = Vec::with_capacity(last_byte_offset - byte_offset + 1);
-        for i in byte_offset..=last_byte_offset {
-            // Get the current and next bytes.
-            let current_byte = u16::from(value_bytes[i]);
-            let next_byte = if i < MAPPING_LEAF_VALUE_LEN - 1 {
-                u16::from(value_bytes[i + 1])
-            } else {
-                0
-            };
+    // last_byte_offset = info.byte_offset + ceil(info.length / 8) - 1
+    let last_byte_offset = (byte_offset + length.div_ceil(8) - 1).min(MAPPING_LEAF_VALUE_LEN - 1);
 
-            // actual_byte = last_bits(current_byte, 8 - bit_offset) * 2^bit_offset + first_bits(next_byte, bit_offset)
-            let actual_byte = (last_bits(current_byte, 8 - bit_offset) << bit_offset)
-                + first_bits(next_byte, bit_offset);
+    // Extract all the bits of the field aligined with bytes.
+    let mut result_bytes = Vec::with_capacity(last_byte_offset - byte_offset + 1);
+    for i in byte_offset..=last_byte_offset {
+        // Get the current and next bytes.
+        let current_byte = u16::from(value_bytes[i]);
+        let next_byte = if i < MAPPING_LEAF_VALUE_LEN - 1 {
+            u16::from(value_bytes[i + 1])
+        } else {
+            0
+        };
 
-            result_bytes.push(u8::try_from(actual_byte).unwrap());
-        }
+        // actual_byte = last_bits(current_byte, 8 - bit_offset) * 2^bit_offset + first_bits(next_byte, bit_offset)
+        let actual_byte = (last_bits(current_byte, 8 - bit_offset) << bit_offset)
+            + first_bits(next_byte, bit_offset);
 
-        // At last we need to retain only the first `info.length % 8` bits for
-        // the last byte of result.
-        let mut last_byte = u16::from(*result_bytes.last().unwrap());
-        let length_mod_8 = length % 8;
-        if length_mod_8 > 0 {
-            // If length_mod_8 == 0, we don't need to cut any bit.
-            last_byte = first_bits(last_byte, u8::try_from(length_mod_8).unwrap());
-        }
-        *result_bytes.last_mut().unwrap() = u8::try_from(last_byte).unwrap();
+        result_bytes.push(u8::try_from(actual_byte).unwrap());
+    }
 
-        // Normalize left.
-        left_pad32(&result_bytes).map(F::from_canonical_u8)
+    // At last we need to retain only the first `info.length % 8` bits for
+    // the last byte of result.
+    let mut last_byte = u16::from(*result_bytes.last().unwrap());
+    let length_mod_8 = length % 8;
+    if length_mod_8 > 0 {
+        // If length_mod_8 == 0, we don't need to cut any bit.
+        last_byte = first_bits(last_byte, u8::try_from(length_mod_8).unwrap());
     }
+    *result_bytes.last_mut().unwrap() = u8::try_from(last_byte).unwrap();
+
+    // Normalize left.
+    left_pad32(&result_bytes)
 }
 
 #[cfg(test)]