From a38724c6f7594965b767999d603c1d18f129d6d0 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Thu, 24 Oct 2024 09:54:03 +0800 Subject: [PATCH] Update integration test for generic extraction. --- mp2-common/src/eth.rs | 3 + mp2-v1/src/api.rs | 13 +- mp2-v1/src/final_extraction/public_inputs.rs | 6 +- mp2-v1/src/values_extraction/api.rs | 49 +- .../gadgets/column_gadget.rs | 14 +- .../values_extraction/gadgets/column_info.rs | 24 +- mp2-v1/src/values_extraction/gadgets/mod.rs | 2 +- mp2-v1/src/values_extraction/mod.rs | 81 +- mp2-v1/test-contracts/src/Simple.sol | 43 + mp2-v1/tests/common/bindings/simple.rs | 862 ++++++++++++- mp2-v1/tests/common/cases/contract.rs | 13 +- mp2-v1/tests/common/cases/indexing.rs | 683 +++++++--- .../common/cases/query/aggregated_queries.rs | 8 +- mp2-v1/tests/common/cases/query/mod.rs | 4 +- mp2-v1/tests/common/cases/table_source.rs | 1126 ++++++++++++++--- mp2-v1/tests/common/celltree.rs | 8 +- mp2-v1/tests/common/final_extraction.rs | 44 +- mp2-v1/tests/common/index_tree.rs | 6 +- mp2-v1/tests/common/mod.rs | 83 +- mp2-v1/tests/common/rowtree.rs | 73 +- mp2-v1/tests/common/storage_trie.rs | 21 +- mp2-v1/tests/common/table.rs | 48 +- mp2-v1/tests/common/values_extraction.rs | 126 +- mp2-v1/tests/integrated_tests.rs | 21 + 24 files changed, 2698 insertions(+), 663 deletions(-) diff --git a/mp2-common/src/eth.rs b/mp2-common/src/eth.rs index 66bff6c03..c33d08c69 100644 --- a/mp2-common/src/eth.rs +++ b/mp2-common/src/eth.rs @@ -237,6 +237,9 @@ impl StorageSlot { } } impl ProofQuery { + pub fn new(contract: Address, slot: StorageSlot) -> Self { + Self { contract, slot } + } pub fn new_simple_slot(address: Address, slot: usize) -> Self { Self { contract: address, diff --git a/mp2-v1/src/api.rs b/mp2-v1/src/api.rs index 31e49bba1..d1fcce953 100644 --- a/mp2-v1/src/api.rs +++ b/mp2-v1/src/api.rs @@ -21,6 +21,7 @@ use crate::{ use alloy::primitives::Address; use anyhow::Result; use itertools::Itertools; +use log::debug; use mp2_common::{ digest::Digest, group_hashing::map_to_curve_point, @@ -209,7 +210,7 @@ pub enum SlotInputs { MappingWithLength(Vec, u8), } -#[derive(Debug)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct SlotInput { /// Slot information of the variable pub(crate) slot: u8, @@ -338,7 +339,7 @@ fn value_metadata( } /// Compute the table information for the value columns. -fn compute_table_info( +pub fn compute_table_info( inputs: Vec, address: &Address, chain_id: u64, @@ -438,8 +439,16 @@ pub fn metadata_hash( chain_id, extra, ); + // Correspond to the computation of final extraction base circuit. + let value_digest = map_to_curve_point(&value_digest.to_fields()); // add contract digest let contract_digest = contract_metadata_digest(contract_address); + debug!( + "METADATA_HASH ->\n\tvalues_ext_md = {:?}\n\tcontract_md = {:?}\n\tfinal_ex_md(contract + values_ex) = {:?}", + value_digest.to_weierstrass(), + contract_digest.to_weierstrass(), + (contract_digest + value_digest).to_weierstrass(), + ); // compute final hash combine_digest_and_block(contract_digest + value_digest) } diff --git a/mp2-v1/src/final_extraction/public_inputs.rs b/mp2-v1/src/final_extraction/public_inputs.rs index 4b60dbb36..08bbf5b75 100644 --- a/mp2-v1/src/final_extraction/public_inputs.rs +++ b/mp2-v1/src/final_extraction/public_inputs.rs @@ -6,7 +6,7 @@ use mp2_common::{ public_inputs::{PublicInputCommon, PublicInputRange}, types::{CBuilder, CURVE_TARGET_LEN}, u256::{self, UInt256Target}, - utils::{FromFields, FromTargets, ToTargets}, + utils::{FromFields, FromTargets, ToTargets, TryIntoBool}, F, }; use plonky2::iop::target::{BoolTarget, Target}; @@ -110,6 +110,10 @@ impl<'a> PublicInputs<'a, F> { pub fn block_number(&self) -> u64 { U256::from_fields(self.bn).to() } + /// Get the merge flag + pub fn merge_flag(&self) -> bool { + self.merge[0].try_into_bool().unwrap() + } } impl<'a, T> PublicInputs<'a, T> { diff --git a/mp2-v1/src/values_extraction/api.rs b/mp2-v1/src/values_extraction/api.rs index dd1d9794f..ca9f1523e 100644 --- a/mp2-v1/src/values_extraction/api.rs +++ b/mp2-v1/src/values_extraction/api.rs @@ -899,11 +899,12 @@ mod tests { } // Mapping variable StorageSlot::Mapping(mapping_key, slot) => { + let outer_key_id = test_slot.outer_key_id.unwrap(); let metadata_digest = compute_leaf_mapping_metadata_digest::< TEST_MAX_COLUMNS, TEST_MAX_FIELD_PER_EVM, >( - table_info.clone(), slot as u8, test_slot.outer_key_id + table_info.clone(), slot as u8, outer_key_id ); let values_digest = compute_leaf_mapping_values_digest::( @@ -912,14 +913,14 @@ mod tests { value, mapping_key.clone(), evm_word, - test_slot.outer_key_id, + outer_key_id, ); let circuit_input = CircuitInput::new_mapping_variable_leaf( node, slot as u8, mapping_key, - test_slot.outer_key_id, + outer_key_id, metadata, ); @@ -946,12 +947,12 @@ mod tests { } // Mapping Struct StorageSlot::Mapping(mapping_key, slot) => { - let metadata_digest = compute_leaf_mapping_metadata_digest::< - TEST_MAX_COLUMNS, - TEST_MAX_FIELD_PER_EVM, - >( - table_info.clone(), slot as u8, test_slot.outer_key_id - ); + let outer_key_id = test_slot.outer_key_id.unwrap(); + let metadata_digest = + compute_leaf_mapping_metadata_digest::< + TEST_MAX_COLUMNS, + TEST_MAX_FIELD_PER_EVM, + >(table_info.clone(), slot as u8, outer_key_id); let values_digest = compute_leaf_mapping_values_digest::( table_info, @@ -959,14 +960,14 @@ mod tests { value, mapping_key.clone(), evm_word, - test_slot.outer_key_id, + outer_key_id, ); let circuit_input = CircuitInput::new_mapping_variable_leaf( node, slot as u8, mapping_key, - test_slot.outer_key_id, + outer_key_id, metadata, ); @@ -976,15 +977,15 @@ mod tests { StorageSlot::Node(StorageSlotNode::Mapping(grand, inner_mapping_key)) => { match *grand { StorageSlot::Mapping(outer_mapping_key, slot) => { - let metadata_digest = compute_leaf_mapping_of_mappings_metadata_digest::< - TEST_MAX_COLUMNS, - TEST_MAX_FIELD_PER_EVM, - >( - table_info.clone(), - slot as u8, - test_slot.outer_key_id, - test_slot.inner_key_id, - ); + let outer_key_id = test_slot.outer_key_id.unwrap(); + let inner_key_id = test_slot.inner_key_id.unwrap(); + let metadata_digest = + compute_leaf_mapping_of_mappings_metadata_digest::< + TEST_MAX_COLUMNS, + TEST_MAX_FIELD_PER_EVM, + >( + table_info.clone(), slot as u8, outer_key_id, inner_key_id + ); let values_digest = compute_leaf_mapping_of_mappings_values_digest::< TEST_MAX_FIELD_PER_EVM, @@ -995,8 +996,8 @@ mod tests { evm_word, outer_mapping_key.clone(), inner_mapping_key.clone(), - test_slot.outer_key_id, - test_slot.inner_key_id, + outer_key_id, + inner_key_id, ); let circuit_input = CircuitInput::new_mapping_of_mappings_leaf( @@ -1004,8 +1005,8 @@ mod tests { slot as u8, outer_mapping_key, inner_mapping_key, - test_slot.outer_key_id, - test_slot.inner_key_id, + outer_key_id, + inner_key_id, metadata, ); diff --git a/mp2-v1/src/values_extraction/gadgets/column_gadget.rs b/mp2-v1/src/values_extraction/gadgets/column_gadget.rs index 65aead5b9..4b388d891 100644 --- a/mp2-v1/src/values_extraction/gadgets/column_gadget.rs +++ b/mp2-v1/src/values_extraction/gadgets/column_gadget.rs @@ -56,12 +56,17 @@ impl<'a, const MAX_FIELD_PER_EVM: usize> ColumnGadget<'a, MAX_FIELD_PER_EVM> { // as a big-endian integer. let bytes = &(0..=u8::MAX as u16).collect_vec(); let mut lookup_inputs = [bytes; NUM_BITS_LOOKUP_TABLES]; - let last_bits_lookup_indexes = add_last_bits_lookup_tables(b, lookup_inputs); - // The maxiumn lookup value is `u8::MAX + 8`, since the maxiumn `info.length` is 256, + // This maxiumn lookup value is `u8::MAX + 8`, since the maxiumn `info.length` is 256, // and we need to compute `first_bits_5(info.length + 7)`. let first_bits_5_input = (0..=u8::MAX as u16 + 8).collect_vec(); lookup_inputs[4] = &first_bits_5_input; let first_bits_lookup_indexes = add_first_bits_lookup_tables(b, lookup_inputs); + lookup_inputs[4] = bytes; + // This maxiumn lookup value is `256`, since the maxiumn `info.length` is 256, + // and we need to compute `last_bits_3(info.length)`. + let last_bits_3_input = (0..=u8::MAX as u16 + 1).collect_vec(); + lookup_inputs[2] = &last_bits_3_input; + let last_bits_lookup_indexes = add_last_bits_lookup_tables(b, lookup_inputs); // Accumulate to compute the value digest. let mut value_digest = b.curve_zero(); @@ -325,7 +330,7 @@ impl ColumnGadgetData { }) } - fn extract_value(&self, info: &ColumnInfo) -> [F; MAPPING_LEAF_VALUE_LEN] { + pub fn extract_value(&self, info: &ColumnInfo) -> [F; MAPPING_LEAF_VALUE_LEN] { let bit_offset = u8::try_from(info.bit_offset.to_canonical_u64()).unwrap(); assert!(bit_offset <= 8); let [byte_offset, length] = @@ -336,7 +341,8 @@ impl ColumnGadgetData { .map(|f| u8::try_from(f.to_canonical_u64()).unwrap()); // last_byte_offset = info.byte_offset + ceil(info.length / 8) - 1 - let last_byte_offset = byte_offset + length.div_ceil(8) - 1; + let last_byte_offset = + (byte_offset + length.div_ceil(8) - 1).min(MAPPING_LEAF_VALUE_LEN - 1); // Extract all the bits of the field aligined with bytes. let mut result_bytes = Vec::with_capacity(last_byte_offset - byte_offset + 1); diff --git a/mp2-v1/src/values_extraction/gadgets/column_info.rs b/mp2-v1/src/values_extraction/gadgets/column_info.rs index b233a55be..7e6d4a406 100644 --- a/mp2-v1/src/values_extraction/gadgets/column_info.rs +++ b/mp2-v1/src/values_extraction/gadgets/column_info.rs @@ -1,5 +1,6 @@ //! Column information for values extraction +use crate::api::SlotInput; use itertools::{zip_eq, Itertools}; use mp2_common::{ group_hashing::map_to_curve_point, @@ -9,6 +10,7 @@ use mp2_common::{ }; use plonky2::{ field::types::{Field, Sample}, + hash::hash_types::HashOut, iop::{target::Target, witness::WitnessWrite}, plonk::config::Hasher, }; @@ -62,6 +64,17 @@ impl ColumnInfo { } } + pub fn new_from_slot_input(identifier: u64, slot_input: &SlotInput) -> Self { + Self::new( + slot_input.slot, + identifier, + slot_input.byte_offset, + slot_input.bit_offset, + slot_input.length, + slot_input.evm_word, + ) + } + /// Create a sample column info. It could be used in integration tests. pub fn sample() -> Self { let rng = &mut thread_rng(); @@ -82,8 +95,8 @@ impl ColumnInfo { evm_word, } } - /// Compute the column information digest. - pub fn digest(&self) -> Point { + /// Compute the MPT metadata. + pub fn mpt_metadata(&self) -> HashOut { // metadata = H(info.slot || info.evm_word || info.byte_offset || info.bit_offset || info.length) let inputs = vec![ self.slot, @@ -92,7 +105,12 @@ impl ColumnInfo { self.bit_offset, self.length, ]; - let metadata = H::hash_no_pad(&inputs); + H::hash_no_pad(&inputs) + } + + /// Compute the column information digest. + pub fn digest(&self) -> Point { + let metadata = self.mpt_metadata(); // digest = D(mpt_metadata || info.identifier) let inputs = metadata diff --git a/mp2-v1/src/values_extraction/gadgets/mod.rs b/mp2-v1/src/values_extraction/gadgets/mod.rs index 483f92142..08059cda0 100644 --- a/mp2-v1/src/values_extraction/gadgets/mod.rs +++ b/mp2-v1/src/values_extraction/gadgets/mod.rs @@ -1,3 +1,3 @@ -pub(crate) mod column_gadget; +pub mod column_gadget; pub mod column_info; pub mod metadata_gadget; diff --git a/mp2-v1/src/values_extraction/mod.rs b/mp2-v1/src/values_extraction/mod.rs index f5508acc8..f91fb18fd 100644 --- a/mp2-v1/src/values_extraction/mod.rs +++ b/mp2-v1/src/values_extraction/mod.rs @@ -14,6 +14,7 @@ use mp2_common::{ }; use plonky2::{ field::types::{Field, PrimeField64}, + hash::hash_types::HashOut, plonk::config::Hasher, }; use plonky2_ecgfp5::curve::{curve::Point as Digest, scalar_field::Scalar}; @@ -42,13 +43,13 @@ pub(crate) const OUTER_KEY_ID_PREFIX: &[u8] = b"\0OUT_KEY"; pub(crate) const BLOCK_ID_DST: &[u8] = b"BLOCK_NUMBER"; -/// Storage slot information for generating the proof +/// Storage slot information for generating the extraction proof #[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct StorageSlotInfo { slot: StorageSlot, metadata: MetadataGadget, - outer_key_id: u64, - inner_key_id: u64, + outer_key_id: Option, + inner_key_id: Option, } impl @@ -60,8 +61,6 @@ impl outer_key_id: Option, inner_key_id: Option, ) -> Self { - let [outer_key_id, inner_key_id] = - [outer_key_id, inner_key_id].map(|key_id| key_id.unwrap_or_default()); Self { slot, metadata, @@ -78,13 +77,21 @@ impl &self.metadata } - pub fn outer_key_id(&self) -> u64 { + pub fn outer_key_id(&self) -> Option { self.outer_key_id } - pub fn inner_key_id(&self) -> u64 { + pub fn inner_key_id(&self) -> Option { self.inner_key_id } + + pub fn slot_inputs(&self) -> Vec { + self.metadata() + .extracted_table_info() + .iter() + .map(Into::into) + .collect_vec() + } } pub fn identifier_block_column() -> u64 { @@ -93,7 +100,8 @@ pub fn identifier_block_column() -> u64 { } /// Compute identifier for value column. -/// The value column could be either simple value or mapping value. +/// +/// The value column could be either simple or mapping slot. /// `id = H(slot || byte_offset || bit_offset || length || evm_word || contract_address || chain_id)[0]` pub fn identifier_for_value_column( input: &SlotInput, @@ -169,6 +177,39 @@ fn compute_id_with_prefix( H::hash_no_pad(&inputs).elements[0].to_canonical_u64() } +/// Compute the row unique data for single leaf. +pub fn row_unique_data_for_single_leaf() -> HashOut { + *empty_poseidon_hash() +} + +/// Compute the row unique data for mapping leaf. +pub fn row_unique_data_for_mapping_leaf(mapping_key: &[u8]) -> HashOut { + // row_unique_data = H(pack(left_pad32(key)) + let packed_mapping_key = left_pad32(mapping_key) + .pack(Endianness::Big) + .into_iter() + .map(F::from_canonical_u32) + .collect_vec(); + H::hash_no_pad(&packed_mapping_key) +} + +/// Compute the row unique data for mapping of mappings leaf. +pub fn row_unique_data_for_mapping_of_mappings_leaf( + outer_mapping_key: &[u8], + inner_mapping_key: &[u8], +) -> HashOut { + let [packed_outer_key, packed_inner_key] = [outer_mapping_key, inner_mapping_key].map(|key| { + left_pad32(key) + .pack(Endianness::Big) + .into_iter() + .map(F::from_canonical_u32) + }); + // Compute the unique data to identify a row is the mapping key: + // row_unique_data = H(outer_key || inner_key) + let inputs = packed_outer_key.chain(packed_inner_key).collect_vec(); + H::hash_no_pad(&inputs) +} + /// Compute the metadata digest for single variable leaf. pub fn compute_leaf_single_metadata_digest< const MAX_COLUMNS: usize, @@ -192,7 +233,7 @@ pub fn compute_leaf_single_values_digest( .digest(); // row_id = H2int(H("") || num_actual_columns) - let inputs = empty_poseidon_hash() + let inputs = row_unique_data_for_single_leaf() .to_fields() .into_iter() .chain(once(num_actual_columns)) @@ -263,8 +304,7 @@ pub fn compute_leaf_mapping_values_digest( let values_key_digest = map_to_curve_point(&inputs); values_digest += values_key_digest; } - // row_unique_data = H(pack(left_pad32(key)) - let row_unique_data = H::hash_no_pad(&packed_mapping_key.collect_vec()); + let row_unique_data = row_unique_data_for_mapping_leaf(&mapping_key); // row_id = H2int(row_unique_data || num_actual_columns) let inputs = row_unique_data .to_fields() @@ -336,12 +376,13 @@ pub fn compute_leaf_mapping_of_mappings_values_digest::RustType, + } + #[allow(non_camel_case_types, non_snake_case, clippy::style)] + const _: () = { + use alloy::sol_types as alloy_sol_types; + #[doc(hidden)] + type UnderlyingSolTuple<'a> = ( + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<128>, + alloy::sol_types::sol_data::Uint<128>, + MappingOperation, + ); + #[doc(hidden)] + type UnderlyingRustTuple<'a> = ( + alloy::sol_types::private::U256, + alloy::sol_types::private::U256, + u128, + u128, + ::RustType, + ); + #[cfg(test)] + #[allow(dead_code, unreachable_patterns)] + fn _type_assertion(_t: alloy_sol_types::private::AssertTypeEq) { + match _t { + alloy_sol_types::private::AssertTypeEq::< + ::RustType, + >(_) => {} + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From for UnderlyingRustTuple<'_> { + fn from(value: MappingStructChange) -> Self { + ( + value.key, + value.field1, + value.field2, + value.field3, + value.operation, + ) + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From> for MappingStructChange { + fn from(tuple: UnderlyingRustTuple<'_>) -> Self { + Self { + key: tuple.0, + field1: tuple.1, + field2: tuple.2, + field3: tuple.3, + operation: tuple.4, + } + } + } + #[automatically_derived] + impl alloy_sol_types::SolValue for MappingStructChange { + type SolType = Self; + } + #[automatically_derived] + impl alloy_sol_types::private::SolTypeValue for MappingStructChange { + #[inline] + fn stv_to_tokens(&self) -> ::Token<'_> { + ( + as alloy_sol_types::SolType>::tokenize( + &self.key, + ), + as alloy_sol_types::SolType>::tokenize( + &self.field1, + ), + as alloy_sol_types::SolType>::tokenize( + &self.field2, + ), + as alloy_sol_types::SolType>::tokenize( + &self.field3, + ), + ::tokenize(&self.operation), + ) + } + #[inline] + fn stv_abi_encoded_size(&self) -> usize { + if let Some(size) = ::ENCODED_SIZE { + return size; + } + let tuple = + as ::core::convert::From>::from(self.clone()); + as alloy_sol_types::SolType>::abi_encoded_size(&tuple) + } + #[inline] + fn stv_eip712_data_word(&self) -> alloy_sol_types::Word { + ::eip712_hash_struct(self) + } + #[inline] + fn stv_abi_encode_packed_to(&self, out: &mut alloy_sol_types::private::Vec) { + let tuple = + as ::core::convert::From>::from(self.clone()); + as alloy_sol_types::SolType>::abi_encode_packed_to( + &tuple, out, + ) + } + #[inline] + fn stv_abi_packed_encoded_size(&self) -> usize { + if let Some(size) = ::PACKED_ENCODED_SIZE { + return size; + } + let tuple = + as ::core::convert::From>::from(self.clone()); + as alloy_sol_types::SolType>::abi_packed_encoded_size( + &tuple, + ) + } + } + #[automatically_derived] + impl alloy_sol_types::SolType for MappingStructChange { + type RustType = Self; + type Token<'a> = as alloy_sol_types::SolType>::Token<'a>; + const SOL_NAME: &'static str = ::NAME; + const ENCODED_SIZE: Option = + as alloy_sol_types::SolType>::ENCODED_SIZE; + const PACKED_ENCODED_SIZE: Option = + as alloy_sol_types::SolType>::PACKED_ENCODED_SIZE; + #[inline] + fn valid_token(token: &Self::Token<'_>) -> bool { + as alloy_sol_types::SolType>::valid_token(token) + } + #[inline] + fn detokenize(token: Self::Token<'_>) -> Self::RustType { + let tuple = as alloy_sol_types::SolType>::detokenize(token); + >>::from(tuple) + } + } + #[automatically_derived] + impl alloy_sol_types::SolStruct for MappingStructChange { + const NAME: &'static str = "MappingStructChange"; + #[inline] + fn eip712_root_type() -> alloy_sol_types::private::Cow<'static, str> { + alloy_sol_types::private::Cow::Borrowed( + "MappingStructChange(uint256 key,uint256 field1,uint128 field2,uint128 field3,uint8 operation)", + ) + } + #[inline] + fn eip712_components( + ) -> alloy_sol_types::private::Vec> + { + alloy_sol_types::private::Vec::new() + } + #[inline] + fn eip712_encode_type() -> alloy_sol_types::private::Cow<'static, str> { + ::eip712_root_type() + } + #[inline] + fn eip712_encode_data(&self) -> alloy_sol_types::private::Vec { + [ + as alloy_sol_types::SolType>::eip712_data_word(&self.key) + .0, + as alloy_sol_types::SolType>::eip712_data_word(&self.field1) + .0, + as alloy_sol_types::SolType>::eip712_data_word(&self.field2) + .0, + as alloy_sol_types::SolType>::eip712_data_word(&self.field3) + .0, + ::eip712_data_word( + &self.operation, + ) + .0, + ] + .concat() + } + } + #[automatically_derived] + impl alloy_sol_types::EventTopic for MappingStructChange { + #[inline] + fn topic_preimage_length(rust: &Self::RustType) -> usize { + 0usize + + as alloy_sol_types::EventTopic>::topic_preimage_length(&rust.key) + + as alloy_sol_types::EventTopic>::topic_preimage_length( + &rust.field1, + ) + + as alloy_sol_types::EventTopic>::topic_preimage_length( + &rust.field2, + ) + + as alloy_sol_types::EventTopic>::topic_preimage_length( + &rust.field3, + ) + + ::topic_preimage_length( + &rust.operation, + ) + } + #[inline] + fn encode_topic_preimage( + rust: &Self::RustType, + out: &mut alloy_sol_types::private::Vec, + ) { + out.reserve(::topic_preimage_length(rust)); + as alloy_sol_types::EventTopic>::encode_topic_preimage(&rust.key, out); + as alloy_sol_types::EventTopic>::encode_topic_preimage( + &rust.field1, + out, + ); + as alloy_sol_types::EventTopic>::encode_topic_preimage( + &rust.field2, + out, + ); + as alloy_sol_types::EventTopic>::encode_topic_preimage( + &rust.field3, + out, + ); + ::encode_topic_preimage( + &rust.operation, + out, + ); + } + #[inline] + fn encode_topic(rust: &Self::RustType) -> alloy_sol_types::abi::token::WordToken { + let mut out = alloy_sol_types::private::Vec::new(); + ::encode_topic_preimage(rust, &mut out); + alloy_sol_types::abi::token::WordToken(alloy_sol_types::private::keccak256(out)) + } + } + }; /**Function with signature `addToArray(uint256)` and selector `0xd15ec851`. ```solidity function addToArray(uint256 value) external; @@ -962,7 +1318,123 @@ pub mod Simple { fn tokenize(&self) -> Self::Token<'_> { ( as alloy_sol_types::SolType>::tokenize(&self.changes), + ) + } + #[inline] + fn abi_decode_returns( + data: &[u8], + validate: bool, + ) -> alloy_sol_types::Result { + as alloy_sol_types::SolType>::abi_decode_sequence( + data, validate, + ) + .map(Into::into) + } + } + }; + /**Function with signature `changeMappingStruct((uint256,uint256,uint128,uint128,uint8)[])` and selector `0xc7bf4db5`. + ```solidity + function changeMappingStruct(MappingStructChange[] memory changes) external; + ```*/ + #[allow(non_camel_case_types, non_snake_case)] + #[derive(Clone)] + pub struct changeMappingStructCall { + pub changes: alloy::sol_types::private::Vec< + ::RustType, + >, + } + ///Container type for the return parameters of the [`changeMappingStruct((uint256,uint256,uint128,uint128,uint8)[])`](changeMappingStructCall) function. + #[allow(non_camel_case_types, non_snake_case)] + #[derive(Clone)] + pub struct changeMappingStructReturn {} + #[allow(non_camel_case_types, non_snake_case, clippy::style)] + const _: () = { + use alloy::sol_types as alloy_sol_types; + { + #[doc(hidden)] + type UnderlyingSolTuple<'a> = (alloy::sol_types::sol_data::Array,); + #[doc(hidden)] + type UnderlyingRustTuple<'a> = ( + alloy::sol_types::private::Vec< + ::RustType, + >, + ); + #[cfg(test)] + #[allow(dead_code, unreachable_patterns)] + fn _type_assertion(_t: alloy_sol_types::private::AssertTypeEq) { + match _t { + alloy_sol_types::private::AssertTypeEq::< + ::RustType, + >(_) => {} + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From for UnderlyingRustTuple<'_> { + fn from(value: changeMappingStructCall) -> Self { + (value.changes,) + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From> for changeMappingStructCall { + fn from(tuple: UnderlyingRustTuple<'_>) -> Self { + Self { changes: tuple.0 } + } + } + } + { + #[doc(hidden)] + type UnderlyingSolTuple<'a> = (); + #[doc(hidden)] + type UnderlyingRustTuple<'a> = (); + #[cfg(test)] + #[allow(dead_code, unreachable_patterns)] + fn _type_assertion(_t: alloy_sol_types::private::AssertTypeEq) { + match _t { + alloy_sol_types::private::AssertTypeEq::< + ::RustType, + >(_) => {} + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From for UnderlyingRustTuple<'_> { + fn from(value: changeMappingStructReturn) -> Self { + () + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From> for changeMappingStructReturn { + fn from(tuple: UnderlyingRustTuple<'_>) -> Self { + Self {} + } + } + } + #[automatically_derived] + impl alloy_sol_types::SolCall for changeMappingStructCall { + type Parameters<'a> = (alloy::sol_types::sol_data::Array,); + type Token<'a> = as alloy_sol_types::SolType>::Token<'a>; + type Return = changeMappingStructReturn; + type ReturnTuple<'a> = (); + type ReturnToken<'a> = as alloy_sol_types::SolType>::Token<'a>; + const SIGNATURE: &'static str = + "changeMappingStruct((uint256,uint256,uint128,uint128,uint8)[])"; + const SELECTOR: [u8; 4] = [199u8, 191u8, 77u8, 181u8]; + #[inline] + fn new<'a>( + tuple: as alloy_sol_types::SolType>::RustType, + ) -> Self { + tuple.into() + } + #[inline] + fn tokenize(&self) -> Self::Token<'_> { + ( + as alloy_sol_types::SolType>::tokenize(&self.changes), ) } @@ -1775,6 +2247,147 @@ pub mod Simple { } } }; + /**Function with signature `setMappingStruct(uint256,uint256,uint128,uint128)` and selector `0x8026de31`. + ```solidity + function setMappingStruct(uint256 _key, uint256 _field1, uint128 _field2, uint128 _field3) external; + ```*/ + #[allow(non_camel_case_types, non_snake_case)] + #[derive(Clone)] + pub struct setMappingStructCall { + pub _key: alloy::sol_types::private::U256, + pub _field1: alloy::sol_types::private::U256, + pub _field2: u128, + pub _field3: u128, + } + ///Container type for the return parameters of the [`setMappingStruct(uint256,uint256,uint128,uint128)`](setMappingStructCall) function. + #[allow(non_camel_case_types, non_snake_case)] + #[derive(Clone)] + pub struct setMappingStructReturn {} + #[allow(non_camel_case_types, non_snake_case, clippy::style)] + const _: () = { + use alloy::sol_types as alloy_sol_types; + { + #[doc(hidden)] + type UnderlyingSolTuple<'a> = ( + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<128>, + alloy::sol_types::sol_data::Uint<128>, + ); + #[doc(hidden)] + type UnderlyingRustTuple<'a> = ( + alloy::sol_types::private::U256, + alloy::sol_types::private::U256, + u128, + u128, + ); + #[cfg(test)] + #[allow(dead_code, unreachable_patterns)] + fn _type_assertion(_t: alloy_sol_types::private::AssertTypeEq) { + match _t { + alloy_sol_types::private::AssertTypeEq::< + ::RustType, + >(_) => {} + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From for UnderlyingRustTuple<'_> { + fn from(value: setMappingStructCall) -> Self { + (value._key, value._field1, value._field2, value._field3) + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From> for setMappingStructCall { + fn from(tuple: UnderlyingRustTuple<'_>) -> Self { + Self { + _key: tuple.0, + _field1: tuple.1, + _field2: tuple.2, + _field3: tuple.3, + } + } + } + } + { + #[doc(hidden)] + type UnderlyingSolTuple<'a> = (); + #[doc(hidden)] + type UnderlyingRustTuple<'a> = (); + #[cfg(test)] + #[allow(dead_code, unreachable_patterns)] + fn _type_assertion(_t: alloy_sol_types::private::AssertTypeEq) { + match _t { + alloy_sol_types::private::AssertTypeEq::< + ::RustType, + >(_) => {} + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From for UnderlyingRustTuple<'_> { + fn from(value: setMappingStructReturn) -> Self { + () + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From> for setMappingStructReturn { + fn from(tuple: UnderlyingRustTuple<'_>) -> Self { + Self {} + } + } + } + #[automatically_derived] + impl alloy_sol_types::SolCall for setMappingStructCall { + type Parameters<'a> = ( + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<128>, + alloy::sol_types::sol_data::Uint<128>, + ); + type Token<'a> = as alloy_sol_types::SolType>::Token<'a>; + type Return = setMappingStructReturn; + type ReturnTuple<'a> = (); + type ReturnToken<'a> = as alloy_sol_types::SolType>::Token<'a>; + const SIGNATURE: &'static str = "setMappingStruct(uint256,uint256,uint128,uint128)"; + const SELECTOR: [u8; 4] = [128u8, 38u8, 222u8, 49u8]; + #[inline] + fn new<'a>( + tuple: as alloy_sol_types::SolType>::RustType, + ) -> Self { + tuple.into() + } + #[inline] + fn tokenize(&self) -> Self::Token<'_> { + ( + as alloy_sol_types::SolType>::tokenize( + &self._key, + ), + as alloy_sol_types::SolType>::tokenize( + &self._field1, + ), + as alloy_sol_types::SolType>::tokenize( + &self._field2, + ), + as alloy_sol_types::SolType>::tokenize( + &self._field3, + ), + ) + } + #[inline] + fn abi_decode_returns( + data: &[u8], + validate: bool, + ) -> alloy_sol_types::Result { + as alloy_sol_types::SolType>::abi_decode_sequence( + data, validate, + ) + .map(Into::into) + } + } + }; /**Function with signature `setS2(uint256)` and selector `0xf25d54f5`. ```solidity function setS2(uint256 newS2) external; @@ -1884,6 +2497,135 @@ pub mod Simple { } } }; + /**Function with signature `setSimpleStruct(uint256,uint128,uint128)` and selector `0x1417a4f0`. + ```solidity + function setSimpleStruct(uint256 _field1, uint128 _field2, uint128 _field3) external; + ```*/ + #[allow(non_camel_case_types, non_snake_case)] + #[derive(Clone)] + pub struct setSimpleStructCall { + pub _field1: alloy::sol_types::private::U256, + pub _field2: u128, + pub _field3: u128, + } + ///Container type for the return parameters of the [`setSimpleStruct(uint256,uint128,uint128)`](setSimpleStructCall) function. + #[allow(non_camel_case_types, non_snake_case)] + #[derive(Clone)] + pub struct setSimpleStructReturn {} + #[allow(non_camel_case_types, non_snake_case, clippy::style)] + const _: () = { + use alloy::sol_types as alloy_sol_types; + { + #[doc(hidden)] + type UnderlyingSolTuple<'a> = ( + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<128>, + alloy::sol_types::sol_data::Uint<128>, + ); + #[doc(hidden)] + type UnderlyingRustTuple<'a> = (alloy::sol_types::private::U256, u128, u128); + #[cfg(test)] + #[allow(dead_code, unreachable_patterns)] + fn _type_assertion(_t: alloy_sol_types::private::AssertTypeEq) { + match _t { + alloy_sol_types::private::AssertTypeEq::< + ::RustType, + >(_) => {} + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From for UnderlyingRustTuple<'_> { + fn from(value: setSimpleStructCall) -> Self { + (value._field1, value._field2, value._field3) + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From> for setSimpleStructCall { + fn from(tuple: UnderlyingRustTuple<'_>) -> Self { + Self { + _field1: tuple.0, + _field2: tuple.1, + _field3: tuple.2, + } + } + } + } + { + #[doc(hidden)] + type UnderlyingSolTuple<'a> = (); + #[doc(hidden)] + type UnderlyingRustTuple<'a> = (); + #[cfg(test)] + #[allow(dead_code, unreachable_patterns)] + fn _type_assertion(_t: alloy_sol_types::private::AssertTypeEq) { + match _t { + alloy_sol_types::private::AssertTypeEq::< + ::RustType, + >(_) => {} + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From for UnderlyingRustTuple<'_> { + fn from(value: setSimpleStructReturn) -> Self { + () + } + } + #[automatically_derived] + #[doc(hidden)] + impl ::core::convert::From> for setSimpleStructReturn { + fn from(tuple: UnderlyingRustTuple<'_>) -> Self { + Self {} + } + } + } + #[automatically_derived] + impl alloy_sol_types::SolCall for setSimpleStructCall { + type Parameters<'a> = ( + alloy::sol_types::sol_data::Uint<256>, + alloy::sol_types::sol_data::Uint<128>, + alloy::sol_types::sol_data::Uint<128>, + ); + type Token<'a> = as alloy_sol_types::SolType>::Token<'a>; + type Return = setSimpleStructReturn; + type ReturnTuple<'a> = (); + type ReturnToken<'a> = as alloy_sol_types::SolType>::Token<'a>; + const SIGNATURE: &'static str = "setSimpleStruct(uint256,uint128,uint128)"; + const SELECTOR: [u8; 4] = [20u8, 23u8, 164u8, 240u8]; + #[inline] + fn new<'a>( + tuple: as alloy_sol_types::SolType>::RustType, + ) -> Self { + tuple.into() + } + #[inline] + fn tokenize(&self) -> Self::Token<'_> { + ( + as alloy_sol_types::SolType>::tokenize( + &self._field1, + ), + as alloy_sol_types::SolType>::tokenize( + &self._field2, + ), + as alloy_sol_types::SolType>::tokenize( + &self._field3, + ), + ) + } + #[inline] + fn abi_decode_returns( + data: &[u8], + validate: bool, + ) -> alloy_sol_types::Result { + as alloy_sol_types::SolType>::abi_decode_sequence( + data, validate, + ) + .map(Into::into) + } + } + }; /**Function with signature `setSimples(bool,uint256,string,address)` and selector `0x0200225c`. ```solidity function setSimples(bool newS1, uint256 newS2, string memory newS3, address newS4) external; @@ -2274,6 +3016,7 @@ pub mod Simple { addToArray(addToArrayCall), arr1(arr1Call), changeMapping(changeMappingCall), + changeMappingStruct(changeMappingStructCall), m1(m1Call), mappingOfMappings(mappingOfMappingsCall), s1(s1Call), @@ -2281,7 +3024,9 @@ pub mod Simple { s3(s3Call), s4(s4Call), setMapping(setMappingCall), + setMappingStruct(setMappingStructCall), setS2(setS2Call), + setSimpleStruct(setSimpleStructCall), setSimples(setSimplesCall), simpleStruct(simpleStructCall), structMapping(structMappingCall), @@ -2298,13 +3043,16 @@ pub mod Simple { [2u8, 0u8, 34u8, 92u8], [10u8, 77u8, 4u8, 247u8], [12u8, 22u8, 22u8, 201u8], + [20u8, 23u8, 164u8, 240u8], [28u8, 19u8, 67u8, 21u8], [42u8, 228u8, 38u8, 134u8], [105u8, 135u8, 177u8, 251u8], [108u8, 192u8, 20u8, 222u8], + [128u8, 38u8, 222u8, 49u8], [136u8, 223u8, 221u8, 198u8], [163u8, 20u8, 21u8, 15u8], [165u8, 214u8, 102u8, 169u8], + [199u8, 191u8, 77u8, 181u8], [200u8, 175u8, 58u8, 166u8], [209u8, 94u8, 200u8, 81u8], [234u8, 209u8, 132u8, 0u8], @@ -2315,13 +3063,16 @@ pub mod Simple { impl alloy_sol_types::SolInterface for SimpleCalls { const NAME: &'static str = "SimpleCalls"; const MIN_DATA_LENGTH: usize = 0usize; - const COUNT: usize = 14usize; + const COUNT: usize = 17usize; #[inline] fn selector(&self) -> [u8; 4] { match self { Self::addToArray(_) => ::SELECTOR, Self::arr1(_) => ::SELECTOR, Self::changeMapping(_) => ::SELECTOR, + Self::changeMappingStruct(_) => { + ::SELECTOR + } Self::m1(_) => ::SELECTOR, Self::mappingOfMappings(_) => { ::SELECTOR @@ -2331,7 +3082,13 @@ pub mod Simple { Self::s3(_) => ::SELECTOR, Self::s4(_) => ::SELECTOR, Self::setMapping(_) => ::SELECTOR, + Self::setMappingStruct(_) => { + ::SELECTOR + } Self::setS2(_) => ::SELECTOR, + Self::setSimpleStruct(_) => { + ::SELECTOR + } Self::setSimples(_) => ::SELECTOR, Self::simpleStruct(_) => ::SELECTOR, Self::structMapping(_) => ::SELECTOR, @@ -2387,6 +3144,18 @@ pub mod Simple { } changeMapping }, + { + fn setSimpleStruct( + data: &[u8], + validate: bool, + ) -> alloy_sol_types::Result { + ::abi_decode_raw( + data, validate, + ) + .map(SimpleCalls::setSimpleStruct) + } + setSimpleStruct + }, { fn setMapping( data: &[u8], @@ -2418,6 +3187,18 @@ pub mod Simple { } s1 }, + { + fn setMappingStruct( + data: &[u8], + validate: bool, + ) -> alloy_sol_types::Result { + ::abi_decode_raw( + data, validate, + ) + .map(SimpleCalls::setMappingStruct) + } + setMappingStruct + }, { fn structMapping( data: &[u8], @@ -2444,6 +3225,18 @@ pub mod Simple { } s3 }, + { + fn changeMappingStruct( + data: &[u8], + validate: bool, + ) -> alloy_sol_types::Result { + ::abi_decode_raw( + data, validate, + ) + .map(SimpleCalls::changeMappingStruct) + } + changeMappingStruct + }, { fn s4(data: &[u8], validate: bool) -> alloy_sol_types::Result { ::abi_decode_raw(data, validate) @@ -2501,6 +3294,9 @@ pub mod Simple { Self::changeMapping(inner) => { ::abi_encoded_size(inner) } + Self::changeMappingStruct(inner) => { + ::abi_encoded_size(inner) + } Self::m1(inner) => ::abi_encoded_size(inner), Self::mappingOfMappings(inner) => { ::abi_encoded_size(inner) @@ -2512,9 +3308,15 @@ pub mod Simple { Self::setMapping(inner) => { ::abi_encoded_size(inner) } + Self::setMappingStruct(inner) => { + ::abi_encoded_size(inner) + } Self::setS2(inner) => { ::abi_encoded_size(inner) } + Self::setSimpleStruct(inner) => { + ::abi_encoded_size(inner) + } Self::setSimples(inner) => { ::abi_encoded_size(inner) } @@ -2538,6 +3340,11 @@ pub mod Simple { Self::changeMapping(inner) => { ::abi_encode_raw(inner, out) } + Self::changeMappingStruct(inner) => { + ::abi_encode_raw( + inner, out, + ) + } Self::m1(inner) => ::abi_encode_raw(inner, out), Self::mappingOfMappings(inner) => { ::abi_encode_raw(inner, out) @@ -2549,9 +3356,15 @@ pub mod Simple { Self::setMapping(inner) => { ::abi_encode_raw(inner, out) } + Self::setMappingStruct(inner) => { + ::abi_encode_raw(inner, out) + } Self::setS2(inner) => { ::abi_encode_raw(inner, out) } + Self::setSimpleStruct(inner) => { + ::abi_encode_raw(inner, out) + } Self::setSimples(inner) => { ::abi_encode_raw(inner, out) } @@ -2750,6 +3563,15 @@ pub mod Simple { ) -> alloy_contract::SolCallBuilder { self.call_builder(&changeMappingCall { changes }) } + ///Creates a new call builder for the [`changeMappingStruct`] function. + pub fn changeMappingStruct( + &self, + changes: alloy::sol_types::private::Vec< + ::RustType, + >, + ) -> alloy_contract::SolCallBuilder { + self.call_builder(&changeMappingStructCall { changes }) + } ///Creates a new call builder for the [`m1`] function. pub fn m1( &self, @@ -2789,6 +3611,21 @@ pub mod Simple { ) -> alloy_contract::SolCallBuilder { self.call_builder(&setMappingCall { key, value }) } + ///Creates a new call builder for the [`setMappingStruct`] function. + pub fn setMappingStruct( + &self, + _key: alloy::sol_types::private::U256, + _field1: alloy::sol_types::private::U256, + _field2: u128, + _field3: u128, + ) -> alloy_contract::SolCallBuilder { + self.call_builder(&setMappingStructCall { + _key, + _field1, + _field2, + _field3, + }) + } ///Creates a new call builder for the [`setS2`] function. pub fn setS2( &self, @@ -2796,6 +3633,19 @@ pub mod Simple { ) -> alloy_contract::SolCallBuilder { self.call_builder(&setS2Call { newS2 }) } + ///Creates a new call builder for the [`setSimpleStruct`] function. + pub fn setSimpleStruct( + &self, + _field1: alloy::sol_types::private::U256, + _field2: u128, + _field3: u128, + ) -> alloy_contract::SolCallBuilder { + self.call_builder(&setSimpleStructCall { + _field1, + _field2, + _field3, + }) + } ///Creates a new call builder for the [`setSimples`] function. pub fn setSimples( &self, diff --git a/mp2-v1/tests/common/cases/contract.rs b/mp2-v1/tests/common/cases/contract.rs index c25b3aa44..5b2122612 100644 --- a/mp2-v1/tests/common/cases/contract.rs +++ b/mp2-v1/tests/common/cases/contract.rs @@ -4,7 +4,7 @@ use log::info; use crate::common::{bindings::simple::Simple, TestContext}; -use super::indexing::{SimpleSingleValue, UpdateSimpleStorage}; +use super::indexing::{LargeStruct, SimpleSingleValue, UpdateSimpleStorage}; pub struct Contract { pub address: Address, @@ -27,6 +27,17 @@ impl Contract { s4: contract.s4().call().await.unwrap()._0, }) } + pub async fn current_single_struct(&self, ctx: &TestContext) -> Result { + let provider = ProviderBuilder::new() + .with_recommended_fillers() + .wallet(ctx.wallet()) + .on_http(ctx.rpc_url.parse()?); + + let contract = Simple::new(self.address, &provider); + let res = contract.simpleStruct().call().await?; + + Ok(res.into()) + } // Returns the table updated pub async fn apply_update( &self, diff --git a/mp2-v1/tests/common/cases/indexing.rs b/mp2-v1/tests/common/cases/indexing.rs index e9c454d03..e5a58e7e4 100644 --- a/mp2-v1/tests/common/cases/indexing.rs +++ b/mp2-v1/tests/common/cases/indexing.rs @@ -5,7 +5,7 @@ use anyhow::Result; use itertools::Itertools; use log::{debug, info}; use mp2_v1::{ - api::SlotInput, + api::{compute_table_info, SlotInput}, contract_extraction, indexing::{ block::BlockPrimaryIndex, @@ -13,18 +13,24 @@ use mp2_v1::{ row::{CellCollection, CellInfo, Row, RowTreeKey}, ColumnID, }, - values_extraction::{identifier_block_column, identifier_for_value_column}, + values_extraction::{ + gadgets::column_info::ColumnInfo, identifier_block_column, identifier_for_value_column, + }, }; use ryhope::storage::RoEpochKvStorage; use crate::common::{ - bindings::simple::Simple::{self, MappingChange, MappingOperation}, + bindings::simple::Simple::{ + self, simpleStructReturn, structMappingReturn, MappingChange, MappingOperation, + MappingStructChange, SimpleInstance, + }, cases::{ contract::Contract, identifier_for_mapping_key_column, table_source::{ - single_var_slot_info, LengthExtractionArgs, MappingIndex, MappingValuesExtractionArgs, - MergeSource, SingleValuesExtractionArgs, UniqueMappingEntry, DEFAULT_ADDRESS, + LengthExtractionArgs, MappingIndex, MappingStructExtractionArgs, + MappingValuesExtractionArgs, MergeSource, SingleStructExtractionArgs, + SingleValuesExtractionArgs, DEFAULT_ADDRESS, }, }, proof_storage::{ProofKey, ProofStorage}, @@ -33,35 +39,31 @@ use crate::common::{ CellsUpdate, IndexType, IndexUpdate, Table, TableColumn, TableColumns, TreeRowUpdate, TreeUpdateType, }, - MetadataGadget, StorageSlotInfo, TableInfo, TestContext, TEST_MAX_COLUMNS, - TEST_MAX_FIELD_PER_EVM, + MetadataGadget, TableInfo, TestContext, }; -use super::{ - super::bindings::simple::Simple::SimpleInstance, ContractExtractionArgs, TableIndexing, - TableSource, -}; +use super::{ContractExtractionArgs, TableIndexing, TableSource}; use alloy::{ contract::private::{Network, Provider, Transport}, primitives::{Address, U256}, providers::ProviderBuilder, }; use mp2_common::{ - eth::{ProofQuery, StorageSlot}, + eth::StorageSlot, proof::ProofWithVK, - types::{HashOutput, ADDRESS_LEN}, + types::{HashOutput, MAPPING_LEAF_VALUE_LEN}, F, }; -use plonky2::field::types::Field; -use std::{assert_matches::assert_matches, str::FromStr, sync::atomic::AtomicU64}; +use plonky2::field::types::PrimeField64; /// Test slots for single values extraction -const SINGLE_SLOTS: [u8; 4] = [0, 1, 2, 3]; +pub(crate) const SINGLE_SLOTS: [u8; 4] = [0, 1, 2, 3]; + /// Define which slots is the secondary index. In this case, it's the U256 const INDEX_SLOT: u8 = 1; /// Test slot for mapping values extraction -const MAPPING_SLOT: u8 = 4; +pub(crate) const MAPPING_SLOT: u8 = 4; /// Test slot for length extraction const LENGTH_SLOT: u8 = 1; @@ -72,14 +74,14 @@ const LENGTH_VALUE: u8 = 2; /// Test slot for contract extraction const CONTRACT_SLOT: usize = 1; -/// Test slot for single Struct extractin -const SINGLE_STRUCT_SLOT: usize = 6; +/// Test slot for single Struct extraction +pub(crate) const SINGLE_STRUCT_SLOT: usize = 6; /// Test slot for mapping Struct extraction -const MAPPING_STRUCT_SLOT: usize = 7; +pub(crate) const MAPPING_STRUCT_SLOT: usize = 7; /// Test slot for mapping of mappings extraction -const MAPPING_OF_MAPPINGS_SLOT: usize = 8; +pub(crate) const MAPPING_OF_MAPPINGS_SLOT: usize = 8; /// human friendly name about the column containing the block number pub(crate) const BLOCK_COLUMN_NAME: &str = "block_number"; @@ -110,89 +112,79 @@ impl TableIndexing { address: *contract_address, chain_id, }; - let single_source = SingleValuesExtractionArgs { - // this test puts the mapping value as secondary index so there is no index for the - // single variable slots. - index_slot: None, - slots: single_var_slot_info(contract_address, chain_id), - }; - // to toggle off and on - let value_as_index = true; - let slot_input = SlotInput::new( - MAPPING_SLOT, - // byte_offset - 0, - // bit_offset - 0, - // length - 0, - // evm_word - 0, - ); - let value_id = identifier_for_value_column(&slot_input, contract_address, chain_id, vec![]); - let key_id = + // this test puts the mapping value as secondary index so there is no index for the + // single variable slots. + let single_source = SingleValuesExtractionArgs::new(None); + let mapping_key_id = identifier_for_mapping_key_column(MAPPING_SLOT, contract_address, chain_id, vec![]); + let mapping_value_id = identifier_for_value_column( + &MappingValuesExtractionArgs::slot_input(), + contract_address, + chain_id, + vec![], + ); + // to toggle off and on + let value_as_index = false; let (mapping_index_id, mapping_index, mapping_cell_id) = match value_as_index { - true => (value_id, MappingIndex::Value(value_id), key_id), - false => (key_id, MappingIndex::Key(key_id), value_id), - }; - - let mapping_source = MappingValuesExtractionArgs { - slot: MAPPING_SLOT, - index: mapping_index, - // at the beginning there is no mapping key inserted - // NOTE: This array is a convenience to handle smart contract updates - // manually, but does not need to be stored explicitely by dist system. - mapping_keys: vec![], + true => ( + mapping_value_id, + MappingIndex::Value(mapping_value_id), + mapping_key_id, + ), + false => ( + mapping_key_id, + MappingIndex::Key(mapping_key_id), + mapping_value_id, + ), }; + let mapping_source = MappingValuesExtractionArgs::new(mapping_index); let mut source = TableSource::Merge(MergeSource::new(single_source, mapping_source)); let genesis_change = source.init_contract_data(ctx, &contract).await; - let single_columns = SINGLE_SLOTS + let single_columns = SingleValuesExtractionArgs::slot_inputs() .iter() .enumerate() - .filter_map(|(i, slot)| { - let slot_input = SlotInput::new( - *slot, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, - ); + .map(|(i, slot_input)| { let identifier = - identifier_for_value_column(&slot_input, contract_address, chain_id, vec![]); - Some(TableColumn { + identifier_for_value_column(slot_input, contract_address, chain_id, vec![]); + let info = ColumnInfo::new_from_slot_input(identifier, slot_input); + TableColumn { name: format!("column_{}", i), - identifier, index: IndexType::None, // ALL single columns are "multiplier" since we do tableA * D(tableB), i.e. all // entries of table A are repeated for each entry of table B. multiplier: true, - }) + info, + } }) - .collect::>(); - let mapping_column = vec![TableColumn { - name: if value_as_index { - MAPPING_KEY_COLUMN - } else { - MAPPING_VALUE_COLUMN - } - .to_string(), - identifier: mapping_cell_id, - index: IndexType::None, - // here is it important to specify false to mean that the entries of table B are - // not repeated. - multiplier: false, - }]; + .collect_vec(); + let mapping_slot_input = MappingValuesExtractionArgs::slot_input(); + let mapping_column = { + let info = ColumnInfo::new_from_slot_input(mapping_cell_id, &mapping_slot_input); + vec![TableColumn { + name: if value_as_index { + MAPPING_KEY_COLUMN + } else { + MAPPING_VALUE_COLUMN + } + .to_string(), + index: IndexType::None, + // here is it important to specify false to mean that the entries of table B are + // not repeated. + multiplier: false, + info, + }] + }; let value_column = mapping_column[0].name.clone(); let all_columns = [single_columns.as_slice(), mapping_column.as_slice()].concat(); let columns = TableColumns { primary: TableColumn { name: BLOCK_COLUMN_NAME.to_string(), - identifier: identifier_block_column(), index: IndexType::Primary, // it doesn't matter for this one since block is "outside" of the table definition // really, it is a special column we add multiplier: true, + // Only valid for the identifier of block column, others are dummy. + info: ColumnInfo::new(0, identifier_block_column(), 0, 0, 0, 0), }, secondary: TableColumn { name: if value_as_index { @@ -201,11 +193,11 @@ impl TableIndexing { MAPPING_KEY_COLUMN } .to_string(), - identifier: mapping_index_id, index: IndexType::Secondary, // here is it important to specify false to mean that the entries of table B are // not repeated. multiplier: false, + info: ColumnInfo::new_from_slot_input(mapping_index_id, &mapping_slot_input), }, rest: all_columns, }; @@ -251,72 +243,149 @@ impl TableIndexing { chain_id, }; - let mut source = TableSource::SingleValues(SingleValuesExtractionArgs { - index_slot: Some(INDEX_SLOT), - slots: single_var_slot_info(contract_address, chain_id), - }); + let mut source = + TableSource::SingleValues(SingleValuesExtractionArgs::new(Some(INDEX_SLOT))); let genesis_updates = source.init_contract_data(ctx, &contract).await; - let indexing_genesis_block = ctx.block_number().await; + let mut slot_inputs = SingleValuesExtractionArgs::slot_inputs(); + let pos = slot_inputs + .iter() + .position(|slot_input| slot_input.slot() == INDEX_SLOT) + .unwrap(); + let secondary_index_slot_input = slot_inputs.remove(pos); + // Defining the columns structure of the table from the source slots // This is depending on what is our data source, mappings and CSV both have their o // own way of defining their table. let columns = TableColumns { primary: TableColumn { name: BLOCK_COLUMN_NAME.to_string(), - identifier: identifier_block_column(), index: IndexType::Primary, multiplier: false, + // Only valid for the identifier of block column, others are dummy. + info: ColumnInfo::new(0, identifier_block_column(), 0, 0, 0, 0), }, secondary: TableColumn { name: "column_value".to_string(), - identifier: identifier_for_value_column( - &SlotInput::new( - INDEX_SLOT, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, + index: IndexType::Secondary, + // here we put false always since these are not coming from a "merged" table + multiplier: false, + info: ColumnInfo::new_from_slot_input( + identifier_for_value_column( + &secondary_index_slot_input, + contract_address, + chain_id, + vec![], ), - contract_address, - chain_id, - vec![], + &secondary_index_slot_input, ), + }, + rest: slot_inputs + .iter() + .enumerate() + .map(|(i, slot_input)| { + let identifier = + identifier_for_value_column(slot_input, contract_address, chain_id, vec![]); + let info = ColumnInfo::new_from_slot_input(identifier, slot_input); + TableColumn { + name: format!("column_{}", i), + index: IndexType::None, + multiplier: false, + info, + } + }) + .collect_vec(), + }; + let table = Table::new(indexing_genesis_block, "single_table".to_string(), columns).await; + Ok(( + Self { + value_column: "".to_string(), + source: source.clone(), + table, + contract, + contract_extraction: ContractExtractionArgs { + slot: StorageSlot::Simple(CONTRACT_SLOT), + }, + }, + genesis_updates, + )) + } + + pub(crate) async fn single_struct_test_case( + ctx: &mut TestContext, + ) -> Result<(Self, Vec>)> { + // Create a provider with the wallet for contract deployment and interaction. + let provider = ProviderBuilder::new() + .with_recommended_fillers() + .wallet(ctx.wallet()) + .on_http(ctx.rpc_url.parse().unwrap()); + + let contract = Simple::deploy(&provider).await.unwrap(); + info!( + "Deployed Simple contract at address: {}", + contract.address() + ); + let contract_address = contract.address(); + let chain_id = ctx.rpc.get_chain_id().await.unwrap(); + let contract = Contract { + address: *contract_address, + chain_id, + }; + + let mut source = TableSource::SingleStruct(SingleStructExtractionArgs::new(&contract)); + let genesis_updates = source.init_contract_data(ctx, &contract).await; + let indexing_genesis_block = ctx.block_number().await; + let secondary_index_slot_input = SingleStructExtractionArgs::secondary_index_slot_input(); + let rest_slot_inputs = SingleStructExtractionArgs::rest_slot_inputs(); + + // Defining the columns structure of the table from the source slots + // This is depending on what is our data source, mappings and CSV both have their o + // own way of defining their table. + let columns = TableColumns { + primary: TableColumn { + name: BLOCK_COLUMN_NAME.to_string(), + index: IndexType::Primary, + multiplier: false, + // Only valid for the identifier of block column, others are dummy. + info: ColumnInfo::new(0, identifier_block_column(), 0, 0, 0, 0), + }, + secondary: TableColumn { + name: "column_value".to_string(), index: IndexType::Secondary, // here we put false always since these are not coming from a "merged" table multiplier: false, + info: ColumnInfo::new_from_slot_input( + identifier_for_value_column( + &secondary_index_slot_input, + contract_address, + chain_id, + vec![], + ), + &secondary_index_slot_input, + ), }, - rest: SINGLE_SLOTS + rest: rest_slot_inputs .iter() .enumerate() - .filter_map(|(i, slot)| match i { - _ if *slot == INDEX_SLOT => None, - _ => { - let slot_input = SlotInput::new( - *slot, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, - ); - let identifier = identifier_for_value_column( - &slot_input, - contract_address, - chain_id, - vec![], - ); - Some(TableColumn { - name: format!("column_{}", i), - identifier, - index: IndexType::None, - // here we put false always since these are not coming from a "merged" table - multiplier: false, - }) + .map(|(i, slot_input)| { + let identifier = + identifier_for_value_column(slot_input, contract_address, chain_id, vec![]); + let info = ColumnInfo::new_from_slot_input(identifier, slot_input); + TableColumn { + name: format!("column_{}", i), + index: IndexType::None, + multiplier: false, + info, } }) - .collect::>(), + .collect_vec(), }; - let table = Table::new(indexing_genesis_block, "single_table".to_string(), columns).await; + let table = Table::new( + indexing_genesis_block, + "single_struct_table".to_string(), + columns, + ) + .await; Ok(( Self { value_column: "".to_string(), @@ -347,38 +416,19 @@ impl TableIndexing { ); let contract_address = contract.address(); let chain_id = ctx.rpc.get_chain_id().await.unwrap(); - // to toggle off and on - let value_as_index = true; - let slot_input = SlotInput::new( - MAPPING_SLOT, - // byte_offset - 0, - // bit_offset - 0, - // length - 0, - // evm_word - 0, - ); - let value_id = identifier_for_value_column(&slot_input, contract_address, chain_id, vec![]); + let slot_input = MappingValuesExtractionArgs::slot_input(); let key_id = identifier_for_mapping_key_column(MAPPING_SLOT, contract_address, chain_id, vec![]); + let value_id = identifier_for_value_column(&slot_input, contract_address, chain_id, vec![]); + // to toggle off and on + let value_as_index = false; let (index_identifier, mapping_index, cell_identifier) = match value_as_index { true => (value_id, MappingIndex::Value(value_id), key_id), false => (key_id, MappingIndex::Key(key_id), value_id), }; - // mapping(uint256 => address) public m1 - let mapping_args = MappingValuesExtractionArgs { - slot: MAPPING_SLOT, - index: mapping_index, - // at the beginning there is no mapping key inserted - // NOTE: This array is a convenience to handle smart contract updates - // manually, but does not need to be stored explicitely by dist system. - mapping_keys: vec![], - }; - - let mut source = TableSource::Mapping(( + let mapping_args = MappingValuesExtractionArgs::new(mapping_index); + let mut source = TableSource::MappingValues(( mapping_args, Some(LengthExtractionArgs { slot: LENGTH_SLOT, @@ -397,9 +447,10 @@ impl TableIndexing { let columns = TableColumns { primary: TableColumn { name: BLOCK_COLUMN_NAME.to_string(), - identifier: identifier_block_column(), index: IndexType::Primary, multiplier: false, + // Only valid for the identifier of block column, others are dummy. + info: ColumnInfo::new(0, identifier_block_column(), 0, 0, 0, 0), }, secondary: TableColumn { name: if value_as_index { @@ -408,10 +459,10 @@ impl TableIndexing { MAPPING_KEY_COLUMN } .to_string(), - identifier: index_identifier, index: IndexType::Secondary, // here important to put false since these are not coming from any "merged" table multiplier: false, + info: ColumnInfo::new_from_slot_input(index_identifier, &slot_input), }, rest: vec![TableColumn { name: if value_as_index { @@ -420,10 +471,10 @@ impl TableIndexing { MAPPING_VALUE_COLUMN } .to_string(), - identifier: cell_identifier, index: IndexType::None, // here important to put false since these are not coming from any "merged" table multiplier: false, + info: ColumnInfo::new_from_slot_input(cell_identifier, &slot_input), }], }; let value_column = columns.rest[0].name.clone(); @@ -445,6 +496,114 @@ impl TableIndexing { )) } + pub(crate) async fn mapping_struct_test_case( + ctx: &mut TestContext, + ) -> Result<(Self, Vec>)> { + // Create a provider with the wallet for contract deployment and interaction. + let provider = ProviderBuilder::new() + .with_recommended_fillers() + .wallet(ctx.wallet()) + .on_http(ctx.rpc_url.parse().unwrap()); + + let contract = Simple::deploy(&provider).await.unwrap(); + info!( + "Deployed MAPPING Simple contract at address: {}", + contract.address() + ); + let contract_address = contract.address(); + let chain_id = ctx.rpc.get_chain_id().await.unwrap(); + let contract = Contract { + address: *contract_address, + chain_id, + }; + let secondary_index_slot_input = SingleStructExtractionArgs::secondary_index_slot_input(); + let rest_slot_inputs = SingleStructExtractionArgs::rest_slot_inputs(); + let key_id = identifier_for_mapping_key_column( + MAPPING_STRUCT_SLOT as u8, + contract_address, + chain_id, + vec![], + ); + let value_id = identifier_for_value_column( + &secondary_index_slot_input, + contract_address, + chain_id, + vec![], + ); + // to toggle off and on + let value_as_index = false; + let (index_identifier, mapping_index, cell_identifier) = match value_as_index { + true => (value_id, MappingIndex::Value(value_id), key_id), + false => (key_id, MappingIndex::Key(key_id), value_id), + }; + let mapping_args = MappingStructExtractionArgs::new(mapping_index, &contract); + let mut source = TableSource::MappingStruct((mapping_args, None)); + + let table_row_updates = source.init_contract_data(ctx, &contract).await; + // Defining the columns structure of the table from the source slots + // This is depending on what is our data source, mappings and CSV both have their o + // own way of defining their table. + let columns = TableColumns { + primary: TableColumn { + name: BLOCK_COLUMN_NAME.to_string(), + index: IndexType::Primary, + multiplier: false, + // Only valid for the identifier of block column, others are dummy. + info: ColumnInfo::new(0, identifier_block_column(), 0, 0, 0, 0), + }, + secondary: TableColumn { + name: if value_as_index { + MAPPING_VALUE_COLUMN + } else { + MAPPING_KEY_COLUMN + } + .to_string(), + index: IndexType::Secondary, + // here important to put false since these are not coming from any "merged" table + multiplier: false, + info: ColumnInfo::new_from_slot_input( + index_identifier, + &secondary_index_slot_input, + ), + }, + rest: rest_slot_inputs + .iter() + .enumerate() + .map(|(i, slot_input)| { + let info = ColumnInfo::new_from_slot_input(cell_identifier, slot_input); + TableColumn { + name: format!("mapping_column_{}", i), + index: IndexType::None, + multiplier: false, + info, + } + }) + .collect_vec(), + }; + let value_column = columns.rest[0].name.clone(); + debug!("MAPPING STRUCT ZK COLUMNS -> {:?}", columns); + let index_genesis_block = ctx.block_number().await; + let table = Table::new( + index_genesis_block, + "mapping_struct_table".to_string(), + columns, + ) + .await; + + Ok(( + Self { + value_column, + contract_extraction: ContractExtractionArgs { + slot: StorageSlot::Simple(CONTRACT_SLOT), + }, + contract, + source, + table, + }, + table_row_updates, + )) + } + pub async fn run( &mut self, ctx: &mut TestContext, @@ -517,7 +676,7 @@ impl TableIndexing { false => Row::default(), }; let new_cell_collection = row_update.updated_cells_collection( - self.table.columns.secondary_column().identifier, + self.table.columns.secondary_column().identifier(), bn, &previous_row.payload.cells, ); @@ -550,7 +709,7 @@ impl TableIndexing { .await .expect("unable to find previous row"); let new_cell_collection = row_update.updated_cells_collection( - self.table.columns.secondary_column().identifier, + self.table.columns.secondary_column().identifier(), bn, &old_row.cells, ); @@ -657,10 +816,9 @@ impl TableIndexing { debug!( " CONTRACT storage root pis.storage_root() {:?}", hex::encode( - &pis.root_hash_field() + pis.root_hash_field() .into_iter() - .map(|u| u.to_be_bytes()) - .flatten() + .flat_map(|u| u.to_be_bytes()) .collect::>() ) ); @@ -704,6 +862,7 @@ impl TableIndexing { .source .generate_extraction_proof_inputs(ctx, &self.contract, value_key) .await?; + // no need to generate it if it's already present if ctx.storage.get_proof_exact(&final_key).is_err() { let proof = ctx @@ -722,13 +881,15 @@ impl TableIndexing { #[derive(Clone, Debug)] pub enum UpdateSimpleStorage { - Single(SimpleSingleValue), - Mapping(Vec), + SingleValues(SimpleSingleValue), + MappingValues(Vec), + SingleStruct(LargeStruct), + MappingStruct(Vec), } /// Represents the update that can come from the chain #[derive(Clone, Debug)] -pub enum MappingUpdate { +pub enum MappingValuesUpdate { // key, value Deletion(U256, U256), // key, previous_value, new_value @@ -738,12 +899,12 @@ pub enum MappingUpdate { } /// passing form the rust type to the solidity type -impl From<&MappingUpdate> for MappingOperation { - fn from(value: &MappingUpdate) -> Self { +impl From<&MappingValuesUpdate> for MappingOperation { + fn from(value: &MappingValuesUpdate) -> Self { Self::from(match value { - MappingUpdate::Deletion(_, _) => 0, - MappingUpdate::Update(_, _, _) => 1, - MappingUpdate::Insertion(_, _) => 2, + MappingValuesUpdate::Deletion(_, _) => 0, + MappingValuesUpdate::Update(_, _, _) => 1, + MappingValuesUpdate::Insertion(_, _) => 2, }) } } @@ -756,6 +917,120 @@ pub struct SimpleSingleValue { pub(crate) s4: Address, } +#[derive(Clone, Debug, Default, Eq, PartialEq, Hash)] +pub struct LargeStruct { + pub(crate) field1: U256, + pub(crate) field2: u128, + pub(crate) field3: u128, +} + +impl LargeStruct { + pub const FIELD_NUM: usize = 3; + + pub fn new(field1: U256, field2: u128, field3: u128) -> Self { + Self { + field1, + field2, + field3, + } + } + + pub fn slot_inputs(slot: u8) -> Vec { + vec![ + SlotInput::new(slot, 0, 0, 256, 0), + SlotInput::new(slot, 0, 0, 128, 1), + SlotInput::new(slot, 16, 0, 128, 1), + ] + } + + pub fn to_bytes(&self) -> Vec { + self.field1 + .to_be_bytes::<{ U256::BYTES }>() + .into_iter() + .chain(self.field2.to_be_bytes()) + .chain(self.field3.to_be_bytes()) + .collect() + } + + pub fn metadata(slot: u8, chain_id: u64, contract_address: &Address) -> Vec { + let table_info = + compute_table_info(Self::slot_inputs(slot), contract_address, chain_id, vec![]); + let ids1 = table_info[..1] + .iter() + .map(|c| c.identifier().to_canonical_u64()) + .collect_vec(); + let ids2 = table_info[1..] + .iter() + .map(|c| c.identifier().to_canonical_u64()) + .collect_vec(); + vec![ + MetadataGadget::new(table_info.clone(), &ids1, 0), + MetadataGadget::new(table_info, &ids2, 1), + ] + } +} + +impl From for LargeStruct { + fn from(res: simpleStructReturn) -> Self { + Self { + field1: res.field1, + field2: res.field2, + field3: res.field3, + } + } +} + +impl From for LargeStruct { + fn from(res: structMappingReturn) -> Self { + Self { + field1: res.field1, + field2: res.field2, + field3: res.field3, + } + } +} + +impl From<&[[F; MAPPING_LEAF_VALUE_LEN]]> for LargeStruct { + fn from(fields: &[[F; MAPPING_LEAF_VALUE_LEN]]) -> Self { + assert_eq!(fields.len(), Self::FIELD_NUM); + + let fields = fields + .iter() + .map(|bytes| { + U256::from_be_bytes(bytes.map(|b| u8::try_from(b.to_canonical_u64()).unwrap())) + }) + .collect_vec(); + + let field1 = fields[0]; + let field2 = fields[1].to(); + let field3 = fields[2].to(); + Self { + field1, + field2, + field3, + } + } +} +#[derive(Clone, Debug)] +pub enum MappingStructUpdate { + // key, struct value + Deletion(U256, LargeStruct), + // key, previous struct value, new struct value + Update(U256, LargeStruct, LargeStruct), + // key, struct value + Insertion(U256, LargeStruct), +} + +impl From<&MappingStructUpdate> for MappingOperation { + fn from(mapping: &MappingStructUpdate) -> Self { + Self::from(match mapping { + MappingStructUpdate::Deletion(_, _) => 0, + MappingStructUpdate::Update(_, _, _) => 1, + MappingStructUpdate::Insertion(_, _) => 2, + }) + } +} + impl UpdateSimpleStorage { // This function applies the update in _one_ transaction so that Anvil only moves by one block // so we can test the "subsequent block" @@ -764,12 +1039,18 @@ impl UpdateSimpleStorage { contract: &SimpleInstance, ) { match self { - UpdateSimpleStorage::Single(ref single) => { + UpdateSimpleStorage::SingleValues(ref single) => { Self::update_single_values(contract, single).await } - UpdateSimpleStorage::Mapping(ref updates) => { + UpdateSimpleStorage::MappingValues(ref updates) => { Self::update_mapping_values(contract, updates).await } + UpdateSimpleStorage::SingleStruct(ref single) => { + Self::update_single_struct(contract, single).await + } + UpdateSimpleStorage::MappingStruct(ref updates) => { + Self::update_mapping_struct(contract, updates).await + } } } @@ -784,15 +1065,15 @@ impl UpdateSimpleStorage { async fn update_mapping_values, N: Network>( contract: &SimpleInstance, - values: &[MappingUpdate], + values: &[MappingValuesUpdate], ) { let contract_changes = values .iter() .map(|tuple| { let op: MappingOperation = tuple.into(); let (k, v) = match tuple { - MappingUpdate::Deletion(k, _) => (*k, DEFAULT_ADDRESS.clone()), - MappingUpdate::Update(k, _, v) | MappingUpdate::Insertion(k, v) => { + MappingValuesUpdate::Deletion(k, _) => (*k, DEFAULT_ADDRESS.clone()), + MappingValuesUpdate::Update(k, _, v) | MappingValuesUpdate::Insertion(k, v) => { (*k, Address::from_slice(&v.to_be_bytes_trimmed_vec())) } }; @@ -810,19 +1091,19 @@ impl UpdateSimpleStorage { // sanity check for op in values { match op { - MappingUpdate::Deletion(k, _) => { + MappingValuesUpdate::Deletion(k, _) => { let res = contract.m1(*k).call().await.unwrap(); let vu: U256 = res._0.into_word().into(); let is_correct = vu == U256::from(0); assert!(is_correct, "key deletion not correct on contract"); } - MappingUpdate::Insertion(k, v) => { + MappingValuesUpdate::Insertion(k, v) => { let res = contract.m1(*k).call().await.unwrap(); let newv: U256 = res._0.into_word().into(); let is_correct = newv == *v; assert!(is_correct, "key insertion not correct on contract"); } - MappingUpdate::Update(k, _, v) => { + MappingValuesUpdate::Update(k, _, v) => { let res = contract.m1(*k).call().await.unwrap(); let newv: U256 = res._0.into_word().into(); let is_correct = newv == *v; @@ -833,6 +1114,62 @@ impl UpdateSimpleStorage { } log::info!("Updated simple contract single values"); } + + async fn update_single_struct, N: Network>( + contract: &SimpleInstance, + single: &LargeStruct, + ) { + let b = contract.setSimpleStruct(single.field1, single.field2, single.field3); + b.send().await.unwrap().watch().await.unwrap(); + log::info!("Updated simple contract for single struct"); + } + + async fn update_mapping_struct, N: Network>( + contract: &SimpleInstance, + values: &[MappingStructUpdate], + ) { + let contract_changes = values + .iter() + .map(|tuple| { + let op: MappingOperation = tuple.into(); + let (key, field1, field2, field3) = match tuple { + MappingStructUpdate::Deletion(k, v) => (*k, v.field1, v.field2, v.field3), + MappingStructUpdate::Update(k, _, v) | MappingStructUpdate::Insertion(k, v) => { + (*k, v.field1, v.field2, v.field3) + } + }; + MappingStructChange { + key, + field1, + field2, + field3, + operation: op.into(), + } + }) + .collect_vec(); + + let b = contract.changeMappingStruct(contract_changes); + b.send().await.unwrap().watch().await.unwrap(); + { + // sanity check + for op in values { + match op { + MappingStructUpdate::Deletion(k, _) => { + let res = contract.structMapping(*k).call().await.unwrap(); + assert_eq!( + LargeStruct::from(res), + LargeStruct::new(U256::from(0), 0, 0) + ); + } + MappingStructUpdate::Insertion(k, v) | MappingStructUpdate::Update(k, _, v) => { + let res = contract.structMapping(*k).call().await.unwrap(); + assert_eq!(&LargeStruct::from(res), v); + } + } + } + } + log::info!("Updated simple contract for single struct"); + } } #[derive(Clone, Debug)] diff --git a/mp2-v1/tests/common/cases/query/aggregated_queries.rs b/mp2-v1/tests/common/cases/query/aggregated_queries.rs index 92b31ea5d..57f5d7afc 100644 --- a/mp2-v1/tests/common/cases/query/aggregated_queries.rs +++ b/mp2-v1/tests/common/cases/query/aggregated_queries.rs @@ -703,8 +703,8 @@ pub fn generate_non_existence_proof<'a>( is_rows_tree_node: bool, ) -> Result> { let index_ids = [ - planner.table.columns.primary_column().identifier, - planner.table.columns.secondary_column().identifier, + planner.table.columns.primary_column().identifier(), + planner.table.columns.secondary_column().identifier(), ]; assert_eq!(index_ids[0], identifier_block_column()); let column_ids = ColumnIDs::new( @@ -715,7 +715,7 @@ pub fn generate_non_existence_proof<'a>( .columns .non_indexed_columns() .iter() - .map(|column| column.identifier) + .map(|column| column.identifier()) .collect_vec(), ); let query_hashes = QueryHashNonExistenceCircuits::new::< @@ -1126,7 +1126,7 @@ pub async fn prove_single_row Result<()> { match &t.source { - TableSource::Mapping(_) | TableSource::Merge(_) => query_mapping(ctx, &table, &t).await?, + TableSource::MappingValues(_) | TableSource::Merge(_) => { + query_mapping(ctx, &table, &t).await? + } _ => unimplemented!("yet"), } Ok(()) diff --git a/mp2-v1/tests/common/cases/table_source.rs b/mp2-v1/tests/common/cases/table_source.rs index f26c7307a..b0af76268 100644 --- a/mp2-v1/tests/common/cases/table_source.rs +++ b/mp2-v1/tests/common/cases/table_source.rs @@ -1,6 +1,5 @@ use std::{ assert_matches::assert_matches, - slice, str::FromStr, sync::atomic::{AtomicU64, AtomicUsize}, }; @@ -8,7 +7,6 @@ use std::{ use alloy::{ eips::BlockNumberOrTag, primitives::{Address, U256}, - providers::Provider, }; use anyhow::{bail, Result}; use futures::{future::BoxFuture, FutureExt}; @@ -16,9 +14,11 @@ use itertools::Itertools; use log::{debug, info}; use mp2_common::{ digest::TableDimension, - eth::{ProofQuery, StorageSlot}, + eth::{ProofQuery, StorageSlot, StorageSlotNode}, + group_hashing::map_to_curve_point, proof::ProofWithVK, types::HashOutput, + utils::ToFields, }; use mp2_v1::{ api::{merge_metadata_hash, metadata_hash, SlotInput, SlotInputs}, @@ -28,7 +28,7 @@ use mp2_v1::{ row::{RowTreeKey, ToNonce}, }, values_extraction::{ - gadgets::{column_info::ColumnInfo, metadata_gadget::MetadataGadget}, + compute_leaf_single_metadata_digest, gadgets::column_info::ColumnInfo, identifier_for_mapping_key_column, identifier_for_value_column, }, }; @@ -37,17 +37,23 @@ use rand::{Rng, SeedableRng}; use serde::{Deserialize, Serialize}; use crate::common::{ - cases::indexing::{MappingUpdate, SimpleSingleValue, TableRowValues}, + cases::indexing::{ + LargeStruct, MappingStructUpdate, MappingValuesUpdate, SimpleSingleValue, TableRowValues, + }, final_extraction::{ExtractionProofInput, ExtractionTableProof, MergeExtractionProof}, proof_storage::{ProofKey, ProofStorage}, rowtree::SecondaryIndexCell, table::CellsUpdate, - StorageSlotInfo, TestContext, TEST_MAX_COLUMNS, TEST_MAX_FIELD_PER_EVM, + ColumnGadgetData, MetadataGadget, StorageSlotInfo, TestContext, TEST_MAX_COLUMNS, + TEST_MAX_FIELD_PER_EVM, }; use super::{ contract::Contract, - indexing::{ChangeType, TableRowUpdate, UpdateSimpleStorage, UpdateType}, + indexing::{ + ChangeType, TableRowUpdate, UpdateSimpleStorage, UpdateType, MAPPING_SLOT, + MAPPING_STRUCT_SLOT, SINGLE_SLOTS, SINGLE_STRUCT_SLOT, + }, }; /// The key,value such that the combination is unique. This can be turned into a RowTreeKey. @@ -86,13 +92,13 @@ impl UniqueMappingEntry { &self, block_number: BlockPrimaryIndex, mapping_index: &MappingIndex, - slot: u8, + slot_input: &SlotInput, contract: &Address, chain_id: u64, previous_row_key: Option, ) -> (CellsUpdate, SecondaryIndexCell) { let row_value = - self.to_table_row_value(block_number, mapping_index, slot, contract, chain_id); + self.to_table_row_value(block_number, mapping_index, slot_input, contract, chain_id); let cells_update = CellsUpdate { previous_row_key: previous_row_key.unwrap_or_default(), new_row_key: self.to_row_key(mapping_index), @@ -108,7 +114,7 @@ impl UniqueMappingEntry { &self, block_number: BlockPrimaryIndex, index: &MappingIndex, - slot: u8, + slot_input: &SlotInput, contract: &Address, chain_id: u64, ) -> TableRowValues { @@ -116,20 +122,14 @@ impl UniqueMappingEntry { // a SecondaryIndexCell depending on the secondary index type we have chosen // for this mapping. let extract_key = MappingIndex::Key(identifier_for_mapping_key_column( - slot, + slot_input.slot(), contract, chain_id, vec![], )); let key_cell = self.to_cell(extract_key); let extract_key = MappingIndex::Value(identifier_for_value_column( - &SlotInput::new( - slot, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, - ), + slot_input, contract, chain_id, vec![], @@ -191,44 +191,137 @@ impl UniqueMappingEntry { } } +/// The combination of key and struct value is unique. +/// This can be turned into a RowTreeKey to store in the row tree. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct UniqueMappingStructEntry { + key: U256, + value: LargeStruct, +} + +impl From<(U256, LargeStruct)> for UniqueMappingStructEntry { + fn from(pair: (U256, LargeStruct)) -> Self { + Self { + key: pair.0, + value: pair.1, + } + } +} + +impl UniqueMappingStructEntry { + pub fn new(k: &U256, v: &LargeStruct) -> Self { + Self { + key: *k, + value: v.clone(), + } + } + + pub fn to_update( + &self, + block_number: BlockPrimaryIndex, + contract: &Contract, + previous_row_key: Option, + ) -> (CellsUpdate, SecondaryIndexCell) { + let row_value = self.to_table_row_value(block_number, contract); + let cells_update = CellsUpdate { + previous_row_key: previous_row_key.unwrap_or_default(), + new_row_key: self.to_row_key(), + updated_cells: row_value.current_cells, + primary: block_number, + }; + let index_cell = row_value.current_secondary.unwrap_or_default(); + (cells_update, index_cell) + } + + /// Return a row given this mapping entry, depending on the chosen index + pub fn to_table_row_value( + &self, + block_number: BlockPrimaryIndex, + contract: &Contract, + ) -> TableRowValues { + // we construct the key and value cells in the table. One of them will become + // a SecondaryIndexCell depending on the secondary index type we have chosen + // for this mapping. + let key_cell = { + let key_id = identifier_for_mapping_key_column( + MAPPING_STRUCT_SLOT as u8, + &contract.address, + contract.chain_id, + vec![], + ); + + Cell::new(key_id, self.key) + }; + let value_ids = LargeStruct::slot_inputs(MAPPING_STRUCT_SLOT as u8) + .iter() + .map(|slot_input| { + identifier_for_value_column( + slot_input, + &contract.address, + contract.chain_id, + vec![], + ) + }) + .collect_vec(); + assert_eq!(value_ids.len(), LargeStruct::FIELD_NUM); + let current_cells = vec![ + Cell::new(value_ids[0], self.value.field1), + Cell::new(value_ids[1], U256::from(self.value.field2)), + Cell::new(value_ids[2], U256::from(self.value.field3)), + ]; + + // then we look at which one is must be the secondary cell, if any + // by definition, mapping key is unique, so there is no need for a specific + // nonce for the tree in that case + let current_secondary = Some(SecondaryIndexCell::new_from(key_cell, U256::from(0))); + debug!( + " --- MAPPING STRUCT: to row: secondary index {:?} -- cell {:?}", + current_secondary, current_cells, + ); + TableRowValues { + current_cells, + current_secondary, + primary: block_number, + } + } + + pub fn to_row_key(&self) -> RowTreeKey { + RowTreeKey { + // tree key indexed by mapping key + value: self.key, + rest: self.value.to_bytes().to_nonce(), + } + } +} + #[derive(Serialize, Deserialize, Debug, Hash, Clone, PartialEq, Eq)] pub(crate) enum TableSource { /// Test arguments for single values extraction (C.1) SingleValues(SingleValuesExtractionArgs), /// Test arguments for mapping values extraction (C.1) /// We can test with and without the length - Mapping((MappingValuesExtractionArgs, Option)), + MappingValues((MappingValuesExtractionArgs, Option)), + /// Test arguments for single struct extraction + SingleStruct(SingleStructExtractionArgs), + /// Test arguments for mapping struct extraction + MappingStruct((MappingStructExtractionArgs, Option)), Merge(MergeSource), } impl TableSource { pub fn slot_input(&self) -> SlotInputs { match self { - TableSource::SingleValues(single) => { - let inputs = single - .slots - .iter() - .flat_map(|slot_info| { - slot_info - .metadata() - .extracted_table_info() - .iter() - .map(Into::into) - .collect_vec() - }) - .collect(); - SlotInputs::Simple(inputs) + TableSource::SingleValues(_) => { + SlotInputs::Simple(SingleValuesExtractionArgs::slot_inputs()) } - TableSource::Mapping((m, _)) => { - // TODO: Support for mapping to Struct. - let slot_input = SlotInput::new( - m.slot, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, - ); - SlotInputs::Mapping(vec![slot_input]) + TableSource::MappingValues(_) => { + SlotInputs::Mapping(vec![MappingValuesExtractionArgs::slot_input()]) + } + TableSource::SingleStruct(_) => { + SlotInputs::Simple(SingleStructExtractionArgs::slot_inputs()) + } + TableSource::MappingStruct(_) => { + SlotInputs::Mapping(MappingStructExtractionArgs::slot_inputs()) } // TODO: Support for mapping of mappings. TableSource::Merge(_) => panic!("can't call slot inputs on merge table"), @@ -243,7 +336,13 @@ impl TableSource { async move { match self { TableSource::SingleValues(ref mut s) => s.init_contract_data(ctx, contract).await, - TableSource::Mapping((ref mut m, _)) => m.init_contract_data(ctx, contract).await, + TableSource::MappingValues((ref mut m, _)) => { + m.init_contract_data(ctx, contract).await + } + TableSource::SingleStruct(ref mut s) => s.init_contract_data(ctx, contract).await, + TableSource::MappingStruct((ref mut m, _)) => { + m.init_contract_data(ctx, contract).await + } TableSource::Merge(ref mut merge) => merge.init_contract_data(ctx, contract).await, } } @@ -257,14 +356,21 @@ impl TableSource { value_key: ProofKey, ) -> Result<(ExtractionProofInput, HashOutput)> { match self { + TableSource::SingleValues(ref s) => { + s.generate_extraction_proof_inputs(ctx, contract, value_key) + .await + } // first lets do without length - TableSource::Mapping((ref mapping, _)) => { - mapping - .generate_extraction_proof_inputs(ctx, contract, value_key) + TableSource::MappingValues((ref m, _)) => { + m.generate_extraction_proof_inputs(ctx, contract, value_key) + .await + } + TableSource::SingleStruct(ref s) => { + s.generate_extraction_proof_inputs(ctx, contract, value_key) .await } - TableSource::SingleValues(ref args) => { - args.generate_extraction_proof_inputs(ctx, contract, value_key) + TableSource::MappingStruct((ref m, _)) => { + m.generate_extraction_proof_inputs(ctx, contract, value_key) .await } TableSource::Merge(ref merge) => { @@ -283,11 +389,17 @@ impl TableSource { ) -> BoxFuture>> { async move { match self { - TableSource::Mapping((ref mut mapping, _)) => { - mapping.random_contract_update(ctx, contract, c).await + TableSource::SingleValues(ref s) => { + s.random_contract_update(ctx, contract, c).await + } + TableSource::MappingValues((ref mut m, _)) => { + m.random_contract_update(ctx, contract, c).await } - TableSource::SingleValues(ref v) => { - v.random_contract_update(ctx, contract, c).await + TableSource::SingleStruct(ref s) => { + s.random_contract_update(ctx, contract, c).await + } + TableSource::MappingStruct((ref mut m, _)) => { + m.random_contract_update(ctx, contract, c).await } TableSource::Merge(ref mut merge) => { merge.random_contract_update(ctx, contract, c).await @@ -301,13 +413,40 @@ impl TableSource { /// Single values extraction arguments (C.1) #[derive(Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Clone)] pub(crate) struct SingleValuesExtractionArgs { - /// Simple slots - pub(crate) slots: Vec, // in case of merge table, there might not be any index slot for single table pub(crate) index_slot: Option, } impl SingleValuesExtractionArgs { + pub fn new(index_slot: Option) -> Self { + Self { index_slot } + } + pub fn slot_inputs() -> Vec { + vec![ + // bool + SlotInput::new(SINGLE_SLOTS[0], 0, 0, 256, 0), + // uint256 + SlotInput::new(SINGLE_SLOTS[1], 0, 0, 256, 0), + // string + SlotInput::new(SINGLE_SLOTS[2], 0, 0, 256, 0), + // address + SlotInput::new(SINGLE_SLOTS[3], 0, 0, 256, 0), + ] + } + pub fn table_info(contract: &Contract) -> Vec { + Self::slot_inputs() + .iter() + .map(|slot_input| { + let id = identifier_for_value_column( + slot_input, + &contract.address, + contract.chain_id, + vec![], + ); + ColumnInfo::new_from_slot_input(id, slot_input) + }) + .collect_vec() + } async fn init_contract_data( &mut self, ctx: &mut TestContext, @@ -324,7 +463,7 @@ impl SingleValuesExtractionArgs { // phase let old_table_values = TableRowValues::default(); contract - .apply_update(ctx, &UpdateSimpleStorage::Single(contract_update)) + .apply_update(ctx, &UpdateSimpleStorage::SingleValues(contract_update)) .await .unwrap(); let new_table_values = self.current_table_row_values(ctx, contract).await; @@ -372,7 +511,7 @@ impl SingleValuesExtractionArgs { }, }; - let contract_update = UpdateSimpleStorage::Single(current_values); + let contract_update = UpdateSimpleStorage::SingleValues(current_values); contract.apply_update(ctx, &contract_update).await.unwrap(); let new_table_values = self.current_table_row_values(ctx, contract).await; assert!( @@ -381,6 +520,7 @@ impl SingleValuesExtractionArgs { ); old_table_values.compute_update(&new_table_values[0]) } + // construct a row of the table from the actual value in the contract by fetching from MPT async fn current_table_row_values( &self, @@ -389,15 +529,12 @@ impl SingleValuesExtractionArgs { ) -> Vec> { let mut secondary_cell = None; let mut rest_cells = Vec::new(); - for slot_info in self.slots.iter() { - let slot = slot_info.slot().slot(); - let query = ProofQuery::new_simple_slot(contract.address, slot as usize); - // TODO: Support for the Struct. - let slot_input = (&slot_info.metadata().extracted_table_info()[0]).into(); + for slot_input in Self::slot_inputs().iter() { + let query = ProofQuery::new_simple_slot(contract.address, slot_input.slot() as usize); let id = identifier_for_value_column( - &slot_input, + slot_input, &contract.address, - ctx.rpc.get_chain_id().await.unwrap(), + contract.chain_id, vec![], ); // Instead of manually setting the value to U256, we really extract from the @@ -411,7 +548,7 @@ impl SingleValuesExtractionArgs { let cell = Cell::new(id, value); // make sure we separate the secondary cells and rest of the cells separately. if let Some(index) = self.index_slot - && index == slot + && index == slot_input.slot() { // we put 0 since we know there are no other rows with that secondary value since we are dealing // we single values, so only 1 row. @@ -435,18 +572,34 @@ impl SingleValuesExtractionArgs { contract: &Contract, proof_key: ProofKey, ) -> Result<(ExtractionProofInput, HashOutput)> { - let chain_id = ctx.rpc.get_chain_id().await?; - let ProofKey::ValueExtraction((id, bn)) = proof_key.clone() else { + let ProofKey::ValueExtraction((_, bn)) = proof_key.clone() else { bail!("invalid proof key"); }; let single_value_proof = match ctx.storage.get_proof_exact(&proof_key) { Ok(p) => p, Err(_) => { + let table_info = Self::table_info(contract); + let metadata_digest = compute_leaf_single_metadata_digest::< + TEST_MAX_COLUMNS, + TEST_MAX_FIELD_PER_EVM, + >(table_info.clone()); + let metadata_digest = metadata_digest.to_weierstrass(); + debug!("SINGLE VALUE metadata digest: {metadata_digest:?}"); + let storage_slot_info = table_info + .iter() + .map(|c| { + let id = c.identifier().to_canonical_u64(); + let evm_word = c.evm_word().to_canonical_u64() as u32; + let slot = StorageSlot::Simple(c.slot().to_canonical_u64() as usize); + let metadata = MetadataGadget::new(table_info.clone(), &[id], evm_word); + StorageSlotInfo::new(slot, metadata, None, None) + }) + .collect_vec(); let single_values_proof = ctx - .prove_single_values_extraction( + .prove_values_extraction( &contract.address, BlockNumberOrTag::Number(bn as u64), - &self.slots, + &storage_slot_info, ) .await; ctx.storage @@ -463,39 +616,26 @@ impl SingleValuesExtractionArgs { debug!( "[--] SINGLE FINAL ROOT HASH --> {:?} ", hex::encode( - &pi.root_hash() + pi.root_hash() .into_iter() - .map(|u| u.to_be_bytes()) - .flatten() - .collect::>() + .flat_map(|u| u.to_be_bytes()) + .collect_vec(), ) ); } single_values_proof } }; - let inputs = self - .slots - .iter() - .flat_map(|slot_info| { - slot_info - .metadata() - .extracted_table_info() - .iter() - .map(Into::into) - .collect_vec() - }) - .collect(); - let slot_input = SlotInputs::Simple(inputs); + let slot_inputs = SlotInputs::Simple(Self::slot_inputs()); let metadata_hash = metadata_hash::( - slot_input, + slot_inputs, &contract.address, - chain_id, + contract.chain_id, vec![], ); // we're just proving a single set of a value let input = ExtractionProofInput::Single(ExtractionTableProof { - dimension: TableDimension::Single, + dimension: TableDimension::Compound, value_proof: single_value_proof, length_proof: None, }); @@ -506,8 +646,7 @@ impl SingleValuesExtractionArgs { /// Mapping values extraction arguments (C.1) #[derive(Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Clone)] pub(crate) struct MappingValuesExtractionArgs { - /// Mapping slot number - pub(crate) slot: u8, + /// Mapping index pub(crate) index: MappingIndex, /// Mapping keys: they are useful for two things: /// * doing some controlled changes on the smart contract, since if we want to do an update we @@ -518,13 +657,21 @@ pub(crate) struct MappingValuesExtractionArgs { } impl MappingValuesExtractionArgs { + pub fn new(index: MappingIndex) -> Self { + Self { + index, + mapping_keys: vec![], + } + } + pub fn slot_input() -> SlotInput { + SlotInput::new(MAPPING_SLOT, 0, 0, 160, 0) + } pub async fn init_contract_data( &mut self, ctx: &mut TestContext, contract: &Contract, ) -> Vec> { let index = self.index.clone(); - let slot = self.slot; let init_pair = (next_value(), next_address()); // NOTE: here is the same address but for different mapping key (10,11) let pair2 = (next_value(), init_pair.1); @@ -540,15 +687,18 @@ impl MappingValuesExtractionArgs { ); let mapping_updates = init_state .iter() - .map(|u| MappingUpdate::Insertion(u.0, u.1.into_word().into())) + .map(|u| MappingValuesUpdate::Insertion(u.0, u.1.into_word().into())) .collect::>(); contract - .apply_update(ctx, &UpdateSimpleStorage::Mapping(mapping_updates.clone())) + .apply_update( + ctx, + &UpdateSimpleStorage::MappingValues(mapping_updates.clone()), + ) .await .unwrap(); let new_block_number = ctx.block_number().await as BlockPrimaryIndex; - self.mapping_to_table_update(new_block_number, mapping_updates, index, slot, contract) + self.mapping_to_table_update(new_block_number, mapping_updates, index, contract) } async fn random_contract_update( @@ -581,10 +731,9 @@ impl MappingValuesExtractionArgs { // of it. let idx = 0; let mkey = &self.mapping_keys[idx].clone(); - let slot = self.slot as usize; let index_type = self.index.clone(); let address = &contract.address.clone(); - let query = ProofQuery::new_mapping_slot(*address, slot, mkey.to_owned()); + let query = ProofQuery::new_mapping_slot(*address, MAPPING_SLOT as usize, mkey.to_owned()); let response = ctx .query_mpt_proof(&query, BlockNumberOrTag::Number(ctx.block_number().await)) .await; @@ -595,14 +744,14 @@ impl MappingValuesExtractionArgs { let mapping_updates = match c { ChangeType::Silent => vec![], ChangeType::Insertion => { - vec![MappingUpdate::Insertion(new_key, new_value)] + vec![MappingValuesUpdate::Insertion(new_key, new_value)] } ChangeType::Deletion => { // NOTE: We care about the value here since that allows _us_ to pinpoint the // correct row in the table and delete it since for a mpping, we uniquely // identify row per (mapping_key,mapping_value) (in the order dictated by // the secondary index) - vec![MappingUpdate::Deletion(current_key, current_value)] + vec![MappingValuesUpdate::Deletion(current_key, current_value)] } ChangeType::Update(u) => { match u { @@ -612,15 +761,19 @@ impl MappingValuesExtractionArgs { match index_type { MappingIndex::Key(_) => { // we simply change the mapping value since the key is the secondary index - vec![MappingUpdate::Update(current_key, current_value, new_value)] + vec![MappingValuesUpdate::Update( + current_key, + current_value, + new_value, + )] } MappingIndex::Value(_) => { // TRICKY: in this case, the mapping key must change. But from the // onchain perspective, it means a transfer // mapping(old_key -> new_key,value) vec![ - MappingUpdate::Deletion(current_key, current_value), - MappingUpdate::Insertion(new_key, current_value), + MappingValuesUpdate::Deletion(current_key, current_value), + MappingValuesUpdate::Insertion(new_key, current_value), ] } MappingIndex::None => { @@ -628,7 +781,11 @@ impl MappingValuesExtractionArgs { // not impacting the secondary index of the table since the mapping // doesn't contain the column which is the secondary index, in case // of the merge table case. - vec![MappingUpdate::Update(current_key, current_value, new_value)] + vec![MappingValuesUpdate::Update( + current_key, + current_value, + new_value, + )] } } } @@ -638,14 +795,18 @@ impl MappingValuesExtractionArgs { // TRICKY: if the mapping key changes, it's a deletion then // insertion from onchain perspective vec![ - MappingUpdate::Deletion(current_key, current_value), + MappingValuesUpdate::Deletion(current_key, current_value), // we insert the same value but with a new mapping key - MappingUpdate::Insertion(new_key, current_value), + MappingValuesUpdate::Insertion(new_key, current_value), ] } MappingIndex::Value(_) => { // if the value changes, it's a simple update in mapping - vec![MappingUpdate::Update(current_key, current_value, new_value)] + vec![MappingValuesUpdate::Update( + current_key, + current_value, + new_value, + )] } MappingIndex::None => { // empty vec since this table has no secondary index so it should @@ -660,35 +821,32 @@ impl MappingValuesExtractionArgs { // small iteration to always have a good updated list of mapping keys for update in mapping_updates.iter() { match update { - MappingUpdate::Deletion(mkey, _) => { + MappingValuesUpdate::Deletion(mkey, _) => { info!("Removing key {} from mappping keys tracking", mkey); let key_stored = mkey.to_be_bytes_trimmed_vec(); self.mapping_keys.retain(|u| u != &key_stored); } - MappingUpdate::Insertion(mkey, _) => { + MappingValuesUpdate::Insertion(mkey, _) => { info!("Inserting key {} to mappping keys tracking", mkey); self.mapping_keys.push(mkey.to_be_bytes_trimmed_vec()); } // the mapping key doesn't change here so no need to update the list - MappingUpdate::Update(_, _, _) => {} + MappingValuesUpdate::Update(_, _, _) => {} } } contract - .apply_update(ctx, &UpdateSimpleStorage::Mapping(mapping_updates.clone())) + .apply_update( + ctx, + &UpdateSimpleStorage::MappingValues(mapping_updates.clone()), + ) .await .unwrap(); let new_block_number = ctx.block_number().await as BlockPrimaryIndex; // NOTE HERE is the interesting bit for dist system as this is the logic to execute // on receiving updates from scapper. This only needs to have the relevant // information from update and it will translate that to changes in the tree. - self.mapping_to_table_update( - new_block_number, - mapping_updates, - index_type, - slot as u8, - contract, - ) + self.mapping_to_table_update(new_block_number, mapping_updates, index_type, contract) } pub async fn generate_extraction_proof_inputs( @@ -697,23 +855,37 @@ impl MappingValuesExtractionArgs { contract: &Contract, proof_key: ProofKey, ) -> Result<(ExtractionProofInput, HashOutput)> { - let chain_id = ctx.rpc.get_chain_id().await?; - let slot_input = SlotInput::new( - self.slot, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, + let ProofKey::ValueExtraction((_, bn)) = proof_key.clone() else { + bail!("invalid proof key"); + }; + let slot_input = Self::slot_input(); + let key_id = identifier_for_mapping_key_column( + MAPPING_SLOT, + &contract.address, + contract.chain_id, + vec![], ); + let value_id = + identifier_for_value_column(&slot_input, &contract.address, contract.chain_id, vec![]); + let column_info = ColumnInfo::new_from_slot_input(value_id, &slot_input); let mapping_root_proof = match ctx.storage.get_proof_exact(&proof_key) { Ok(p) => p, Err(_) => { + let storage_slot_info = self + .mapping_keys + .iter() + .map(|mapping_key| { + let slot = StorageSlot::Mapping(mapping_key.clone(), MAPPING_SLOT as usize); + let metadata = + MetadataGadget::new(vec![column_info.clone()], &[value_id], 0); + StorageSlotInfo::new(slot, metadata, Some(key_id), None) + }) + .collect_vec(); let mapping_values_proof = ctx - .prove_mapping_values_extraction( + .prove_values_extraction( &contract.address, - chain_id, - &slot_input, - self.mapping_keys.clone(), + BlockNumberOrTag::Number(bn as u64), + &storage_slot_info, ) .await; @@ -731,11 +903,10 @@ impl MappingValuesExtractionArgs { debug!( "[--] MAPPING FINAL ROOT HASH --> {:?} ", hex::encode( - &pi.root_hash() + pi.root_hash() .into_iter() - .map(|u| u.to_be_bytes()) - .flatten() - .collect::>() + .flat_map(|u| u.to_be_bytes()) + .collect_vec(), ) ); } @@ -743,9 +914,9 @@ impl MappingValuesExtractionArgs { } }; let metadata_hash = metadata_hash::( - SlotInputs::Mapping(vec![slot_input]), + SlotInputs::Mapping(vec![Self::slot_input()]), &contract.address, - chain_id, + contract.chain_id, vec![], ); // it's a compoound value type of proof since we're not using the length @@ -760,16 +931,16 @@ impl MappingValuesExtractionArgs { pub fn mapping_to_table_update( &self, block_number: BlockPrimaryIndex, - updates: Vec, + updates: Vec, index: MappingIndex, - slot: u8, contract: &Contract, ) -> Vec> { + let slot_input = Self::slot_input(); updates .iter() .flat_map(|mapping_change| { match mapping_change { - MappingUpdate::Deletion(mkey, mvalue) => { + MappingValuesUpdate::Deletion(mkey, mvalue) => { // find the associated row key tree to that value // HERE: there are multiple possibilities: // * search for the entry at the previous block instead @@ -779,20 +950,20 @@ impl MappingValuesExtractionArgs { let entry = UniqueMappingEntry::new(mkey, mvalue); vec![TableRowUpdate::Deletion(entry.to_row_key(&index))] } - MappingUpdate::Insertion(mkey, mvalue) => { + MappingValuesUpdate::Insertion(mkey, mvalue) => { // we transform the mapping entry into the "table notion" of row let entry = UniqueMappingEntry::new(mkey, mvalue); let (cells, index) = entry.to_update( block_number, &index, - slot, + &slot_input, &contract.address, contract.chain_id, None, ); vec![TableRowUpdate::Insertion(cells, index)] } - MappingUpdate::Update(mkey, old_value, mvalue) => { + MappingValuesUpdate::Update(mkey, old_value, mvalue) => { // NOTE: we need here to (a) delete current row and (b) insert new row // Regardless of the change if it's on the mapping key or value, since a // row is uniquely identified by its pair (key,value) then if one of those @@ -805,7 +976,7 @@ impl MappingValuesExtractionArgs { let (mut cells, mut secondary_index) = new_entry.to_update( block_number, &index, - slot, + &slot_input, &contract.address, contract.chain_id, // NOTE: here we provide the previous key such that we can @@ -942,12 +1113,15 @@ impl MergeSource { let single_updates = self.single.random_contract_update(ctx, contract, c).await; let rsu = &single_updates; let bn = ctx.block_number().await; - let mslot = self.mapping.slot as usize; let address = &contract.address.clone(); // we fetch the value of all mapping entries, and let mut all_updates = Vec::new(); for mk in &self.mapping.mapping_keys { - let query = ProofQuery::new_mapping_slot(*address, mslot, mk.to_owned()); + let query = ProofQuery::new_mapping_slot( + *address, + MAPPING_SLOT as usize, + mk.to_owned(), + ); let response = ctx .query_mpt_proof(&query, BlockNumberOrTag::Number(ctx.block_number().await)) .await; @@ -1049,7 +1223,7 @@ impl MergeSource { contract.chain_id, vec![], TableSource::SingleValues(self.single.clone()).slot_input(), - TableSource::Mapping((self.mapping.clone(), None)).slot_input(), + TableSource::MappingValues((self.mapping.clone(), None)).slot_input(), ); assert!(extract_a != extract_b); Ok(( @@ -1063,6 +1237,7 @@ impl MergeSource { .boxed() } } + /// Length extraction arguments (C.2) #[derive(Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Clone)] pub(crate) struct LengthExtractionArgs { @@ -1108,52 +1283,621 @@ pub fn next_value() -> U256 { bv + U256::from(shift) } -/// Construct the storage slot information for the simple variable slots. -// bool public s1 -// uint256 public s2 -// string public s3 -// address public s4 -pub(crate) fn single_var_slot_info( - contract_address: &Address, - chain_id: u64, -) -> Vec { - const SINGLE_SLOTS: [u8; 4] = [0, 1, 2, 3]; - // bool, uint256, string, address - const SINGLE_SLOT_LENGTHS: [usize; 4] = [1, 32, 32, 20]; - - let table_info = SINGLE_SLOTS - .into_iter() - .zip_eq(SINGLE_SLOT_LENGTHS) - .map(|(slot, length)| { - let slot_input = SlotInput::new( - slot, // byte_offset - 0, // bit_offset - length, // length - 0, // evm_word - 0, - ); - let identifier = - identifier_for_value_column(&slot_input, contract_address, chain_id, vec![]); - - ColumnInfo::new(slot, identifier, 0, 0, length, 0) - }) - .collect_vec(); - - SINGLE_SLOTS - .into_iter() - .enumerate() - .map(|(i, slot)| { - // Create the simple slot. - let slot = StorageSlot::Simple(slot as usize); - - // Create the metadata gadget. - let metadata = MetadataGadget::new( - table_info.clone(), - slice::from_ref(&table_info[i].identifier().to_canonical_u64()), - 0, - ); +/// Single struct extraction arguments +#[derive(Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Clone)] +pub(crate) struct SingleStructExtractionArgs { + /// Metadata information + metadata: Vec, +} + +impl SingleStructExtractionArgs { + pub fn new(contract: &Contract) -> Self { + let metadata = LargeStruct::metadata( + SINGLE_STRUCT_SLOT as u8, + contract.chain_id, + &contract.address, + ); + + Self { metadata } + } - StorageSlotInfo::new(slot, metadata, None, None) - }) - .collect_vec() + pub fn slot_inputs() -> Vec { + LargeStruct::slot_inputs(SINGLE_STRUCT_SLOT as u8) + } + + pub fn secondary_index_slot_input() -> SlotInput { + let mut slot_inputs = Self::slot_inputs(); + slot_inputs.remove(1) + } + + pub fn rest_slot_inputs() -> Vec { + let mut slot_inputs = Self::slot_inputs(); + slot_inputs.remove(1); + + slot_inputs + } + + async fn init_contract_data( + &mut self, + ctx: &mut TestContext, + contract: &Contract, + ) -> Vec> { + let contract_update = LargeStruct { + field1: U256::from(1234), + field2: 1234, + field3: 5678, + }; + let old_table_values = TableRowValues::default(); + contract + .apply_update(ctx, &UpdateSimpleStorage::SingleStruct(contract_update)) + .await + .unwrap(); + let new_table_values = self.current_table_row_values(ctx, contract).await; + assert!( + new_table_values.len() == 1, + "single struct case should only have one row" + ); + let update = old_table_values.compute_update(&new_table_values[0]); + assert!(update.len() == 1, "one row at a time"); + assert_matches!( + update[0], + TableRowUpdate::Insertion(_, _), + "initialization of the contract's table should be init" + ); + update + } + + pub async fn random_contract_update( + &self, + ctx: &mut TestContext, + contract: &Contract, + c: ChangeType, + ) -> Vec> { + let old_table_values = self.current_table_row_values(ctx, contract).await; + let old_table_values = &old_table_values[0]; + let mut current_struct = contract.current_single_struct(ctx).await.unwrap(); + match c { + ChangeType::Silent => {} + ChangeType::Deletion => { + panic!("can't remove a single row from blockchain data over single values") + } + ChangeType::Insertion => { + panic!("can't add a new row for blockchain data over single values") + } + ChangeType::Update(u) => match u { + UpdateType::Rest => current_struct.field3 += 1, + UpdateType::SecondaryIndex => current_struct.field2 += 1, + }, + }; + + let contract_update = UpdateSimpleStorage::SingleStruct(current_struct); + contract.apply_update(ctx, &contract_update).await.unwrap(); + let new_table_values = self.current_table_row_values(ctx, contract).await; + assert!( + new_table_values.len() == 1, + "there should be only a single row for single struct case" + ); + old_table_values.compute_update(&new_table_values[0]) + } + async fn current_table_row_values( + &self, + ctx: &mut TestContext, + contract: &Contract, + ) -> Vec> { + let mut rest_cells = Vec::new(); + let parent_slot = StorageSlot::Simple(SINGLE_STRUCT_SLOT); + for metadata in &self.metadata { + let storage_slot = StorageSlot::Node(StorageSlotNode::new_struct( + parent_slot.clone(), + metadata.evm_word(), + )); + let query = ProofQuery::new(contract.address, storage_slot); + let value = ctx + .query_mpt_proof(&query, BlockNumberOrTag::Number(ctx.block_number().await)) + .await + .storage_proof[0] + .value; + metadata + .extracted_table_info() + .iter() + .for_each(|column_info| { + let cell = Cell::new(column_info.identifier().to_canonical_u64(), value); + rest_cells.push(cell); + }); + } + vec![TableRowValues { + current_cells: rest_cells, + current_secondary: None, + primary: ctx.block_number().await as BlockPrimaryIndex, + }] + } + + pub async fn generate_extraction_proof_inputs( + &self, + ctx: &mut TestContext, + contract: &Contract, + proof_key: ProofKey, + ) -> Result<(ExtractionProofInput, HashOutput)> { + let ProofKey::ValueExtraction((_, bn)) = proof_key.clone() else { + bail!("invalid proof key"); + }; + let single_struct_proof = match ctx.storage.get_proof_exact(&proof_key) { + Ok(p) => p, + Err(_) => { + let parent_slot = StorageSlot::Simple(SINGLE_STRUCT_SLOT); + let storage_slot_info = self + .metadata + .iter() + .map(|metadata| { + let storage_slot = StorageSlot::Node(StorageSlotNode::new_struct( + parent_slot.clone(), + metadata.evm_word(), + )); + StorageSlotInfo::new(storage_slot, metadata.clone(), None, None) + }) + .collect_vec(); + let single_struct_proof = ctx + .prove_values_extraction( + &contract.address, + BlockNumberOrTag::Number(bn as u64), + &storage_slot_info, + ) + .await; + ctx.storage + .store_proof(proof_key, single_struct_proof.clone())?; + info!("Generated Values Extraction (C.1) proof for single struct"); + { + let pproof = ProofWithVK::deserialize(&single_struct_proof).unwrap(); + let pi = + mp2_v1::values_extraction::PublicInputs::new(&pproof.proof().public_inputs); + debug!( + "[--] SINGLE STRUCT FINAL MPT DIGEST VALUE --> {:?} ", + pi.values_digest() + ); + debug!( + "[--] SINGLE STRUCT FINAL ROOT HASH --> {:?} ", + hex::encode( + &pi.root_hash() + .into_iter() + .map(|u| u.to_be_bytes()) + .flatten() + .collect::>() + ) + ); + } + single_struct_proof + } + }; + let metadata_hash = metadata_hash::( + SlotInputs::Simple(Self::slot_inputs()), + &contract.address, + contract.chain_id, + vec![], + ); + // we're just proving a single set of a value + let input = ExtractionProofInput::Single(ExtractionTableProof { + dimension: TableDimension::Compound, + value_proof: single_struct_proof, + length_proof: None, + }); + Ok((input, metadata_hash)) + } +} + +/// Mapping struct extraction arguments +#[derive(Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Clone)] +pub(crate) struct MappingStructExtractionArgs { + /// Mapping index type + index: MappingIndex, + /// Metadata information + metadata: Vec, + /// Mapping keys: they are useful for two things: + /// * doing some controlled changes on the smart contract, since if we want to do an update we + /// need to know an existing key + /// * doing the MPT proofs over, since this test doesn't implement the copy on write for MPT + /// (yet), we're just recomputing all the proofs at every block and we need the keys for that. + mapping_keys: Vec>, +} + +impl MappingStructExtractionArgs { + pub fn new(index: MappingIndex, contract: &Contract) -> Self { + let metadata = LargeStruct::metadata( + MAPPING_STRUCT_SLOT as u8, + contract.chain_id, + &contract.address, + ); + + Self { + index, + metadata, + mapping_keys: vec![], + } + } + + pub fn slot_inputs() -> Vec { + LargeStruct::slot_inputs(MAPPING_STRUCT_SLOT as u8) + } + + pub fn secondary_index_slot_input() -> SlotInput { + let mut slot_inputs = Self::slot_inputs(); + slot_inputs.remove(1) + } + + pub fn rest_slot_inputs() -> Vec { + let mut slot_inputs = Self::slot_inputs(); + slot_inputs.remove(1); + + slot_inputs + } + + pub async fn init_contract_data( + &mut self, + ctx: &mut TestContext, + contract: &Contract, + ) -> Vec> { + let struct_value1 = LargeStruct { + field1: U256::from(1234), + field2: 1234, + field3: 5678, + }; + let mut struct_value2 = struct_value1.clone(); + struct_value2.field2 += 1; + let mut struct_value3 = struct_value2.clone(); + struct_value3.field3 += 1; + let mapping_pairs = [ + (next_value(), struct_value1), + (next_value(), struct_value2), + (next_value(), struct_value3), + ]; + // Save the update mapping keys. + self.mapping_keys.extend( + mapping_pairs + .iter() + .map(|u| u.0.to_be_bytes_trimmed_vec()) + .collect_vec(), + ); + let mapping_updates = mapping_pairs + .into_iter() + .map(|u| MappingStructUpdate::Insertion(u.0, u.1)) + .collect_vec(); + + contract + .apply_update( + ctx, + &UpdateSimpleStorage::MappingStruct(mapping_updates.clone()), + ) + .await + .unwrap(); + let new_block_number = ctx.block_number().await as BlockPrimaryIndex; + self.mapping_to_table_update(new_block_number, mapping_updates, contract) + } + + async fn random_contract_update( + &mut self, + ctx: &mut TestContext, + contract: &Contract, + c: ChangeType, + ) -> Vec> { + // NOTE 1: The first part is just trying to construct the right input to simulate any + // changes on a mapping. This is mostly irrelevant for dist system but needs to manually + // construct our test cases here. The second part is more interesting as it looks at + // "what to do when receiving an update from scrapper". The core of the function is in + // `from_mapping_to_table_update` + // + // NOTE 2: This implementation tries to emulate as much as possible what happens in dist + // system. To compute the set of updates, it first simulate an update on the contract + // and creates the signal "MappingUpdate" corresponding to the update. From that point + // onwards, the table row updates are manually created. + // Note this can actually lead to more work than necessary in some cases. + // Take an example where the mapping is storing (10->A), (11->A), and where the + // secondary index value is the value, i.e. A. + // Our table initially looks like `A | 10`, `A | 11`. + // Imagine an update where we want to change the first row to `A | 12`. In the "table" + // world, this is only a simple update of a simple cell, no index even involved. But + // from the perspective of mapping, the "scrapper" can only tells us : + // * Key 10 has been deleted + // * Key 12 has been added with value A + // In the backend, we translate that in the "table world" to a deletion and an insertion. + // Having such optimization could be done later on, need to properly evaluate the cost + // of it. + let mkey = &self.mapping_keys[0]; + let parent_slot = StorageSlot::Mapping(mkey.clone(), MAPPING_STRUCT_SLOT); + let mut fields = vec![]; + for metadata in &self.metadata { + let storage_slot = StorageSlot::Node(StorageSlotNode::new_struct( + parent_slot.clone(), + metadata.evm_word(), + )); + let query = ProofQuery::new(contract.address, storage_slot); + let value = ctx + .query_mpt_proof(&query, BlockNumberOrTag::Number(ctx.block_number().await)) + .await + .storage_proof[0] + .value; + + let table_info = metadata.table_info(); + let table_data = ColumnGadgetData::new( + metadata.table_info().to_vec(), + &metadata.extracted_column_identifiers(), + value.to_be_bytes(), + ); + table_info + .iter() + .for_each(|column_info| fields.push(table_data.extract_value(column_info))); + } + assert_eq!(fields.len(), LargeStruct::FIELD_NUM); + let current_value = LargeStruct::from(fields.as_slice()); + let current_key = U256::from_be_slice(mkey); + let new_key = next_mapping_key(); + let new_value = LargeStruct::new(U256::from(4321), 4321, 8765); + let mapping_updates = match c { + ChangeType::Silent => vec![], + ChangeType::Insertion => { + vec![MappingStructUpdate::Insertion(new_key, new_value)] + } + ChangeType::Deletion => { + vec![MappingStructUpdate::Deletion( + current_key, + LargeStruct::default(), + )] + } + ChangeType::Update(u) => { + match u { + UpdateType::Rest => { + match self.index { + MappingIndex::Key(_) => { + // we simply change the mapping value since the key is the secondary index + vec![MappingStructUpdate::Update( + current_key, + current_value, + new_value, + )] + } + MappingIndex::Value(_) => { + // TRICKY: in this case, the mapping key must change. But from the + // onchain perspective, it means a transfer mapping(old_key -> new_key,value) + vec![ + MappingStructUpdate::Deletion( + current_key, + current_value.clone(), + ), + MappingStructUpdate::Insertion(new_key, current_value), + ] + } + MappingIndex::None => { + // a random update of the mapping, we don't care which since it is + // not impacting the secondary index of the table since the mapping + // doesn't contain the column which is the secondary index, in case + // of the merge table case. + vec![MappingStructUpdate::Update( + current_key, + current_value, + new_value, + )] + } + } + } + UpdateType::SecondaryIndex => { + match self.index { + MappingIndex::Key(_) => { + // TRICKY: if the mapping key changes, it's a deletion then + // insertion from onchain perspective + vec![ + MappingStructUpdate::Deletion( + current_key, + current_value.clone(), + ), + // we insert the same value but with a new mapping key + MappingStructUpdate::Insertion(new_key, current_value), + ] + } + MappingIndex::Value(_) => { + // if the value changes, it's a simple update in mapping + vec![MappingStructUpdate::Update( + current_key, + current_value, + new_value, + )] + } + MappingIndex::None => { + // empty vec since this table has no secondary index so it should + // give no updates + vec![] + } + } + } + } + } + }; + // small iteration to always have a good updated list of mapping keys + for update in mapping_updates.iter() { + match update { + MappingStructUpdate::Deletion(mkey, _) => { + info!("Removing key {} from mappping keys tracking", mkey); + let key_stored = mkey.to_be_bytes_trimmed_vec(); + self.mapping_keys.retain(|u| u != &key_stored); + } + MappingStructUpdate::Insertion(mkey, _) => { + info!("Inserting key {} to mappping keys tracking", mkey); + self.mapping_keys.push(mkey.to_be_bytes_trimmed_vec()); + } + // the mapping key doesn't change here so no need to update the list + MappingStructUpdate::Update(_, _, _) => {} + } + } + + contract + .apply_update( + ctx, + &UpdateSimpleStorage::MappingStruct(mapping_updates.clone()), + ) + .await + .unwrap(); + let new_block_number = ctx.block_number().await as BlockPrimaryIndex; + // NOTE HERE is the interesting bit for dist system as this is the logic to execute + // on receiving updates from scapper. This only needs to have the relevant + // information from update and it will translate that to changes in the tree. + self.mapping_to_table_update(new_block_number, mapping_updates, contract) + } + + pub async fn generate_extraction_proof_inputs( + &self, + ctx: &mut TestContext, + contract: &Contract, + proof_key: ProofKey, + ) -> Result<(ExtractionProofInput, HashOutput)> { + let ProofKey::ValueExtraction((_, bn)) = proof_key.clone() else { + bail!("invalid proof key"); + }; + let key_id = identifier_for_mapping_key_column( + MAPPING_STRUCT_SLOT as u8, + &contract.address, + contract.chain_id, + vec![], + ); + let storage_slot_info = self + .metadata + .iter() + .cartesian_product(self.mapping_keys.iter()) + .map(|(metadata, mapping_key)| { + let parent_slot = StorageSlot::Mapping(mapping_key.clone(), MAPPING_STRUCT_SLOT); + let storage_slot = StorageSlot::Node(StorageSlotNode::new_struct( + parent_slot.clone(), + metadata.evm_word(), + )); + StorageSlotInfo::new(storage_slot, metadata.clone(), Some(key_id), None) + }) + .collect_vec(); + let mapping_root_proof = match ctx.storage.get_proof_exact(&proof_key) { + Ok(p) => p, + Err(_) => { + let mapping_values_proof = ctx + .prove_values_extraction( + &contract.address, + BlockNumberOrTag::Number(bn as u64), + &storage_slot_info, + ) + .await; + ctx.storage + .store_proof(proof_key, mapping_values_proof.clone())?; + info!("Generated Values Extraction proof for mapping struct slots"); + { + let pproof = ProofWithVK::deserialize(&mapping_values_proof).unwrap(); + let pi = + mp2_v1::values_extraction::PublicInputs::new(&pproof.proof().public_inputs); + debug!( + "[--] MAPPING FINAL MPT DIGEST VALUE --> {:?} ", + pi.values_digest() + ); + debug!( + "[--] MAPPING FINAL ROOT HASH --> {:?} ", + hex::encode( + &pi.root_hash() + .into_iter() + .map(|u| u.to_be_bytes()) + .flatten() + .collect::>() + ) + ); + } + mapping_values_proof + } + }; + let metadata_hash = metadata_hash::( + SlotInputs::Mapping(Self::slot_inputs()), + &contract.address, + contract.chain_id, + vec![], + ); + // it's a compoound value type of proof since we're not using the length + let input = ExtractionProofInput::Single(ExtractionTableProof { + dimension: TableDimension::Compound, + value_proof: mapping_root_proof, + length_proof: None, + }); + Ok((input, metadata_hash)) + } + + pub fn mapping_to_table_update( + &self, + block_number: BlockPrimaryIndex, + updates: Vec, + contract: &Contract, + ) -> Vec> { + updates + .iter() + .flat_map(|mapping_change| { + match mapping_change { + MappingStructUpdate::Deletion(mkey, mvalue) => { + // find the associated row key tree to that value + // HERE: there are multiple possibilities: + // * search for the entry at the previous block instead + // * passing inside the deletion the value deleted as well, so we can + // reconstruct the row key + // * or have this extra list of mapping keys + let entry = UniqueMappingStructEntry::new(mkey, mvalue); + vec![TableRowUpdate::Deletion(entry.to_row_key())] + } + MappingStructUpdate::Insertion(mkey, mvalue) => { + // we transform the mapping entry into the "table notion" of row + let entry = UniqueMappingStructEntry::new(mkey, mvalue); + let (cells, index) = entry.to_update(block_number, &contract, None); + vec![TableRowUpdate::Insertion(cells, index)] + } + MappingStructUpdate::Update(mkey, old_value, mvalue) => { + // NOTE: we need here to (a) delete current row and (b) insert new row + // Regardless of the change if it's on the mapping key or value, since a + // row is uniquely identified by its pair (key,value) then if one of those + // change, that means the row tree key needs to change as well, i.e. it's a + // deletion and addition. + let previous_entry = UniqueMappingStructEntry::new(mkey, old_value); + let previous_row_key = previous_entry.to_row_key(); + let new_entry = UniqueMappingStructEntry::new(mkey, mvalue); + + let (mut cells, mut secondary_index) = new_entry.to_update( + block_number, + &contract, + // NOTE: here we provide the previous key such that we can + // reconstruct the cells tree as it was before and then apply + // the update and put it in a new row. Otherwise we don't know + // the update plan since we don't have a base tree to deal + // with. + // In the case the key is the cell, that's good, we don't need to do + // anything to the tree then since the doesn't change. + // In the case it's the value, then we'll have to reprove the cell. + Some(previous_row_key.clone()), + ); + match self.index { + MappingIndex::Key(_) => { + // in this case, the mapping value changed, so the cells changed so + // we need to start from scratch. Telling there was no previous row + // key means it's treated as a full new cells tree. + cells.previous_row_key = Default::default(); + } + MappingIndex::Value(_) => { + // This is a bit hacky way but essentially it means that there is + // no update in the cells tree to apply, even tho it's still a new + // insertion of a new row, since we pick up the cells tree form the + // previous location, and that cells tree didn't change (since it's + // based on the mapping key), then no need to update anything. + // TODO: maybe make a better API to express the different + // possibilities: + // * insertion with new cells tree + // * insertion without modification to cells tree + // * update with modification to cells tree (default) + cells.updated_cells = vec![]; + } + MappingIndex::None => { + secondary_index = Default::default(); + } + }; + vec![ + TableRowUpdate::Deletion(previous_row_key), + TableRowUpdate::Insertion(cells, secondary_index), + ] + } + } + }) + .collect_vec() + } } diff --git a/mp2-v1/tests/common/celltree.rs b/mp2-v1/tests/common/celltree.rs index 483f630f1..355a76dc6 100644 --- a/mp2-v1/tests/common/celltree.rs +++ b/mp2-v1/tests/common/celltree.rs @@ -10,7 +10,7 @@ use mp2_v1::{ row::{CellCollection, Row, RowPayload, RowTreeKey}, }, }; -use plonky2::{field::types::Sample, hash::hash_types::HashOut, plonk::config::GenericHashOut}; +use plonky2::plonk::config::GenericHashOut; use ryhope::storage::{ updatetree::{Next, UpdateTree}, RoEpochKvStorage, @@ -257,7 +257,7 @@ impl TestContext { // only move the cells tree proof of the actual cells, not the secondary index ! // CellsCollection is a bit weird because it has to contain as well the secondary // index to be able to search in it in JSON - if *id == table.columns.secondary_column().identifier { + if *id == table.columns.secondary_column().identifier() { return (*id, new_cell); } @@ -266,7 +266,7 @@ impl TestContext { " --- CELL TREE key {} index of {id} vs secondary id {} vs table.secondary_id {}", tree_key, previous_row.payload.secondary_index_column, - table.columns.secondary.identifier + table.columns.secondary.identifier() ); // we need to update the primary on the impacted cells at least, OR on all the cells if // we are moving all the proofs to a new row key which happens when doing an DELETE + @@ -310,7 +310,7 @@ impl TestContext { ); RowPayload { - secondary_index_column: table.columns.secondary_column().identifier, + secondary_index_column: table.columns.secondary_column().identifier(), cell_root_key: Some(root_key), cell_root_hash: Some(tree_hash), cell_root_column: Some( diff --git a/mp2-v1/tests/common/final_extraction.rs b/mp2-v1/tests/common/final_extraction.rs index 0ae8db58a..a4925f6b6 100644 --- a/mp2-v1/tests/common/final_extraction.rs +++ b/mp2-v1/tests/common/final_extraction.rs @@ -1,8 +1,12 @@ use log::debug; -use mp2_common::{digest::TableDimension, proof::ProofWithVK, types::HashOutput, utils::ToFields}; +use mp2_common::{ + digest::TableDimension, group_hashing::weierstrass_to_point, proof::ProofWithVK, + types::HashOutput, utils::ToFields, F, +}; use mp2_v1::{ - api, + api, contract_extraction, final_extraction::{CircuitInput, PublicInputs}, + values_extraction, }; use super::TestContext; @@ -44,12 +48,30 @@ impl TestContext { inputs.length_proof.unwrap(), ) } - ExtractionProofInput::Single(inputs) => CircuitInput::new_simple_input( - block_proof, - contract_proof, - inputs.value_proof, - inputs.dimension, - ), + ExtractionProofInput::Single(inputs) => { + { + let value_proof = ProofWithVK::deserialize(&inputs.value_proof).unwrap(); + let value_pi = values_extraction::PublicInputs::::new( + &value_proof.proof().public_inputs, + ); + let contract_proof = ProofWithVK::deserialize(&contract_proof).unwrap(); + let contract_pi = contract_extraction::PublicInputs::from_slice( + &contract_proof.proof().public_inputs, + ); + debug!( + "BEFORE proving final extraction:\n\tvalues_ex_md = {:?}\n\tcontract_md = {:?}\n\texpected_final_md = {:?}", + value_pi.metadata_digest(), + contract_pi.metadata_point(), + (weierstrass_to_point(&value_pi.metadata_digest()) + weierstrass_to_point(&contract_pi.metadata_point())).to_weierstrass(), + ); + } + CircuitInput::new_simple_input( + block_proof, + contract_proof, + inputs.value_proof, + inputs.dimension, + ) + } // NOTE hardcoded for single and mapping right now ExtractionProofInput::Merge(inputs) => CircuitInput::new_merge_single_and_mapping( block_proof, @@ -76,7 +98,11 @@ impl TestContext { assert_eq!(pis.block_number(), block.header.number); assert_eq!(pis.block_hash_raw(), block_hash.to_fields()); assert_eq!(pis.prev_block_hash_raw(), prev_block_hash.to_fields()); - debug!(" FINAL EXTRACTION MPT - digest: {:?}", pis.value_point()); + debug!( + " FINAL EXTRACTION MPT -\n\tvalues digest: {:?}\n\tmetadata digest: {:?}", + pis.value_point(), + pis.metadata_point(), + ); Ok(proof) } diff --git a/mp2-v1/tests/common/index_tree.rs b/mp2-v1/tests/common/index_tree.rs index 63e043e1e..3063b13a9 100644 --- a/mp2-v1/tests/common/index_tree.rs +++ b/mp2-v1/tests/common/index_tree.rs @@ -83,14 +83,16 @@ impl TestContext { let ext_pi = mp2_v1::final_extraction::PublicInputs::from_slice( &ext_proof.proof().public_inputs, ); - // TODO: Fix the rows digest in rows tree according to values extraction update. - // assert_eq!( row_pi.individual_digest_point(), ext_pi.value_point(), "values extracted vs value in db don't match (left row, right mpt (block {})", node.value.0.to::() ); + debug!( + "NodeIndex Proving - multiplier digest: {:?}", + row_pi.multiplier_digest_point(), + ); } let proof = if context.is_leaf() { info!( diff --git a/mp2-v1/tests/common/mod.rs b/mp2-v1/tests/common/mod.rs index 11409fecc..558c1d433 100644 --- a/mp2-v1/tests/common/mod.rs +++ b/mp2-v1/tests/common/mod.rs @@ -1,9 +1,11 @@ //! Utility structs and functions used for integration tests use alloy::primitives::Address; use anyhow::Result; -use cases::table_source::TableSource; -use itertools::Itertools; -use mp2_v1::api::{merge_metadata_hash, metadata_hash, MetadataHash, SlotInput, SlotInputs}; +use cases::table_source::{ + MappingStructExtractionArgs, MappingValuesExtractionArgs, SingleStructExtractionArgs, + SingleValuesExtractionArgs, TableSource, +}; +use mp2_v1::api::{merge_metadata_hash, metadata_hash, MetadataHash, SlotInputs}; use serde::{Deserialize, Serialize}; use table::TableColumns; pub mod benchmarker; @@ -43,6 +45,8 @@ type MetadataGadget = mp2_v1::values_extraction::gadgets::metadata_gadget::Metad TEST_MAX_COLUMNS, TEST_MAX_FIELD_PER_EVM, >; +type ColumnGadgetData = + mp2_v1::values_extraction::gadgets::column_gadget::ColumnGadgetData; type PublicParameters = mp2_v1::api::PublicParameters; fn cell_tree_proof_to_hash(proof: &[u8]) -> HashOutput { @@ -91,36 +95,19 @@ pub struct TableInfo { impl TableInfo { pub fn metadata_hash(&self) -> MetadataHash { match &self.source { - TableSource::Mapping((mapping, _)) => { - let slot_input = SlotInputs::Mapping(vec![SlotInput::new( - mapping.slot, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, - )]); + TableSource::MappingValues(_) => { + let slot_inputs = + SlotInputs::Mapping(vec![(MappingValuesExtractionArgs::slot_input())]); metadata_hash::( - slot_input, + slot_inputs, &self.contract_address, self.chain_id, vec![], ) } // mapping with length not tested right now - TableSource::SingleValues(args) => { - let inputs = args - .slots - .iter() - .flat_map(|slot_info| { - slot_info - .metadata() - .extracted_table_info() - .iter() - .map(Into::into) - .collect_vec() - }) - .collect(); - let slot = SlotInputs::Simple(inputs); + TableSource::SingleValues(_) => { + let slot = SlotInputs::Simple(SingleValuesExtractionArgs::slot_inputs()); metadata_hash::( slot, &self.contract_address, @@ -128,28 +115,28 @@ impl TableInfo { vec![], ) } - TableSource::Merge(merge) => { - let inputs = merge - .single - .slots - .iter() - .flat_map(|slot_info| { - slot_info - .metadata() - .extracted_table_info() - .iter() - .map(Into::into) - .collect_vec() - }) - .collect(); - let single = SlotInputs::Simple(inputs); - let mapping = SlotInputs::Mapping(vec![SlotInput::new( - merge.mapping.slot, // byte_offset - 0, // bit_offset - 0, // length - 0, // evm_word - 0, - )]); + TableSource::SingleStruct(_) => { + let slot = SlotInputs::Simple(SingleStructExtractionArgs::slot_inputs()); + metadata_hash::( + slot, + &self.contract_address, + self.chain_id, + vec![], + ) + } + TableSource::MappingStruct(_) => { + let slot_inputs = SlotInputs::Mapping(MappingStructExtractionArgs::slot_inputs()); + metadata_hash::( + slot_inputs, + &self.contract_address, + self.chain_id, + vec![], + ) + } + TableSource::Merge(_) => { + let single = SlotInputs::Simple(SingleValuesExtractionArgs::slot_inputs()); + let mapping = + SlotInputs::Mapping(vec![(MappingValuesExtractionArgs::slot_input())]); merge_metadata_hash::( self.contract_address, self.chain_id, diff --git a/mp2-v1/tests/common/rowtree.rs b/mp2-v1/tests/common/rowtree.rs index 329966885..5907ec141 100644 --- a/mp2-v1/tests/common/rowtree.rs +++ b/mp2-v1/tests/common/rowtree.rs @@ -1,7 +1,13 @@ use alloy::primitives::U256; use anyhow::*; +use itertools::Itertools; use log::debug; -use mp2_common::proof::ProofWithVK; +use mp2_common::{ + poseidon::H, + proof::ProofWithVK, + utils::{Endianness, Packer}, + F, +}; use mp2_v1::{ api::{self, CircuitInput}, indexing::{ @@ -11,7 +17,10 @@ use mp2_v1::{ row::{RowPayload, RowTree, RowTreeKey, ToNonce}, }, }; -use plonky2::{field::types::Sample, hash::hash_types::HashOut, plonk::config::GenericHashOut}; +use plonky2::{ + field::types::Field, + plonk::config::{GenericHashOut, Hasher}, +}; use ryhope::{ storage::{ pgsql::PgsqlStorage, @@ -91,7 +100,19 @@ impl TestContext { let id = row.secondary_index_column; // Sec. index value let value = row.secondary_index_value(); - let multiplier = table.columns.column_info(id).multiplier; + let column_info = table.columns.column_info(id); + let multiplier = column_info.multiplier; + // TODO + let packed_mapping_key = k + .value + .to_be_bytes::<{ U256::BYTES }>() + .pack(Endianness::Big) + .into_iter() + .map(F::from_canonical_u32) + .collect_vec(); + let row_unique_data = H::hash_no_pad(&packed_mapping_key); + // TODO: gupeng + let row_unique_data = *mp2_common::poseidon::empty_poseidon_hash(); // NOTE remove that when playing more with sec. index assert!(!multiplier, "secondary index should be individual type"); // find where the root cells proof has been stored. This comes from looking up the @@ -127,15 +148,15 @@ impl TestContext { row.cells, ); - { - let pvk = ProofWithVK::deserialize(&cell_tree_proof)?; - let pis = cells_tree::PublicInputs::from_slice(&pvk.proof().public_inputs); - debug!( - " Cell Root SPLIT digest: multiplier {:?}, individual {:?}", - pis.multiplier_values_digest_point(), - pis.individual_values_digest_point() - ); - } + let cells_tree_proof_with_vk = ProofWithVK::deserialize(&cell_tree_proof)?; + let cells_tree_pi = cells_tree::PublicInputs::from_slice( + &cells_tree_proof_with_vk.proof().public_inputs, + ); + debug!( + " Cell Root SPLIT digest:\n\tindividual_value {:?}\n\tmultiplier_value {:?}", + cells_tree_pi.individual_values_digest_point(), + cells_tree_pi.multiplier_values_digest_point(), + ); let proof = if context.is_leaf() { // Prove a leaf @@ -146,23 +167,35 @@ impl TestContext { hex::encode(cell_root_hash_from_proof.clone()), hex::encode(row.cell_root_hash.unwrap().0) ); + let cell = cells_tree::Cell::new(F::from_canonical_u64(id), value, multiplier); let inputs = CircuitInput::RowsTree( verifiable_db::row_tree::CircuitInput::leaf_multiplier( id, value, multiplier, - // TODO: row_unique_data - HashOut::rand(), + row_unique_data, cell_tree_proof, ) .unwrap(), ); debug!("Before proving leaf node row tree key {:?}", k); - self.b + let proof = self + .b .bench("indexing::row_tree::leaf", || { api::generate_proof(self.params(), inputs) }) - .expect("while proving leaf") + .expect("while proving leaf"); + let pproof = ProofWithVK::deserialize(&proof).unwrap(); + let pi = verifiable_db::row_tree::PublicInputs::from_slice( + &pproof.proof().public_inputs, + ); + debug!( + "FINISH PROVING ROW -->\n\tid = {:?}\n\tindividual digest = {:?}\n\tmultiplier digest = {:?}", + id, + pi.individual_digest_point(), + pi.multiplier_digest_point(), + ); + proof } else if context.is_partial() { let child_key = context .left @@ -189,8 +222,7 @@ impl TestContext { value, multiplier, context.left.is_some(), - // TODO: row_unique_data - HashOut::rand(), + row_unique_data, child_proof, cell_tree_proof, ) @@ -233,8 +265,7 @@ impl TestContext { id, value, multiplier, - // TODO: row_unique_data - HashOut::rand(), + row_unique_data, left_proof, right_proof, cell_tree_proof, @@ -321,7 +352,7 @@ impl TestContext { ); Ok(IndexNode { - identifier: table.columns.primary_column().identifier, + identifier: table.columns.primary_column().identifier(), value: U256::from(primary).into(), row_tree_root_key: root_proof_key.tree_key, row_tree_hash: table.row.root_data().await.unwrap().hash, diff --git a/mp2-v1/tests/common/storage_trie.rs b/mp2-v1/tests/common/storage_trie.rs index 1cd0fc5e1..505beb1bf 100644 --- a/mp2-v1/tests/common/storage_trie.rs +++ b/mp2-v1/tests/common/storage_trie.rs @@ -5,7 +5,6 @@ use alloy::{ eips::BlockNumberOrTag, primitives::{Address, U256}, }; -use itertools::Itertools; use log::debug; use mp2_common::{ eth::{ProofQuery, StorageSlot, StorageSlotNode}, @@ -17,7 +16,6 @@ use mp2_v1::{ api::{generate_proof, CircuitInput}, length_extraction, values_extraction, }; -use plonky2::field::types::PrimeField64; use rlp::{Prototype, Rlp}; use std::collections::HashMap; @@ -221,7 +219,7 @@ impl TrieNode { node.clone(), *slot as u8, mapping_key.clone(), - slot_info.outer_key_id(), + slot_info.outer_key_id().unwrap(), metadata, ), ), @@ -242,7 +240,7 @@ impl TrieNode { node.clone(), *slot as u8, mapping_key.clone(), - slot_info.outer_key_id(), + slot_info.outer_key_id().unwrap(), metadata, ), ), @@ -256,8 +254,8 @@ impl TrieNode { *slot as u8, outer_mapping_key.clone(), inner_mapping_key.clone(), - slot_info.outer_key_id(), - slot_info.inner_key_id(), + slot_info.outer_key_id().unwrap(), + slot_info.inner_key_id().unwrap(), metadata, ), ), @@ -431,10 +429,10 @@ impl TestStorageTrie { bn: BlockNumberOrTag, slot_info: StorageSlotInfo, ) { - let slot = slot_info.slot().slot() as usize; - log::debug!("Querying the simple slot `{slot:?}` of the contract `{contract_address}` from the test context's RPC"); + let storage_slot = slot_info.slot(); + log::debug!("Querying the slot `{storage_slot:?}` of the contract `{contract_address}` from the test context's RPC"); - let query = ProofQuery::new_simple_slot(*contract_address, slot); + let query = ProofQuery::new(*contract_address, storage_slot.clone()); let response = ctx.query_mpt_proof(&query, bn).await; // Get the nodes to prove. Reverse to the sequence from leaf to root. @@ -445,10 +443,8 @@ impl TestStorageTrie { .map(|node| node.to_vec()) .collect(); - let slot = StorageSlot::Simple(slot); - log::debug!( - "Simple slot {slot:?} queried, appending `{}` proof nodes to the trie", + "Storage slot {storage_slot:?} queried, appending `{}` proof nodes to the trie", nodes.len() ); @@ -505,6 +501,7 @@ impl TestStorageTrie { // Must have the same slot number for the mapping type. assert_eq!(slot, new_slot); } + (&StorageSlot::Node(_), &StorageSlot::Node(_)) => (), _ => panic!("Add the different type of storage slots: {slot:?}, {new_slot:?}"), } } diff --git a/mp2-v1/tests/common/table.rs b/mp2-v1/tests/common/table.rs index ae6d3ccd6..0a45a1b26 100644 --- a/mp2-v1/tests/common/table.rs +++ b/mp2-v1/tests/common/table.rs @@ -8,14 +8,18 @@ use futures::{ }; use itertools::Itertools; use log::{debug, info}; -use mp2_v1::indexing::{ - block::BlockPrimaryIndex, - cell::{self, Cell, CellTreeKey, MerkleCell, MerkleCellTree}, - index::IndexNode, - row::{CellCollection, Row, RowTreeKey}, - ColumnID, +use mp2_v1::{ + indexing::{ + block::BlockPrimaryIndex, + cell::{self, Cell, CellTreeKey, MerkleCell, MerkleCellTree}, + index::IndexNode, + row::{CellCollection, Row, RowTreeKey}, + ColumnID, + }, + values_extraction::gadgets::column_info::ColumnInfo, }; use parsil::symbols::{ColumnKind, ContextProvider, ZkColumn, ZkTable}; +use plonky2::field::types::PrimeField64; use ryhope::{ storage::{ pgsql::{SqlServerConnection, SqlStorageSettings}, @@ -58,13 +62,19 @@ impl IndexType { #[derive(Serialize, Deserialize, Clone, Debug)] pub struct TableColumn { pub name: String, - pub identifier: ColumnID, + pub info: ColumnInfo, pub index: IndexType, /// multiplier means if this columns come from a "merged" table, then it either come from a /// table a or table b. One of these table is the "multiplier" table, the other is not. pub multiplier: bool, } +impl TableColumn { + pub fn identifier(&self) -> ColumnID { + self.info.identifier().to_canonical_u64() + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TableColumns { pub primary: TableColumn, @@ -82,13 +92,13 @@ impl TableColumns { self.rest.clone() } pub fn column_id_of_cells_index(&self, key: CellTreeKey) -> Option { - self.rest.get(key - 1).map(|tc| tc.identifier) + self.rest.get(key - 1).map(|tc| tc.identifier()) } pub fn column_info(&self, identifier: ColumnIdentifier) -> TableColumn { self.rest .iter() .chain(once(&self.secondary)) - .find(|c| c.identifier == identifier) + .find(|c| c.identifier() == identifier) .expect(&format!("can't find cell from identifier {}", identifier)) .clone() } @@ -105,17 +115,17 @@ impl TableColumns { pub fn cells_tree_index_of(&self, identifier: ColumnIdentifier) -> usize { match identifier { // TODO this will be problematic in the CSV case - _ if identifier == self.primary.identifier => panic!( + _ if identifier == self.primary.identifier() => panic!( "should not call the position on primary index since should not be included in cells tree" ), - _ if identifier == self.secondary.identifier => panic!( + _ if identifier == self.secondary.identifier() => panic!( "should not call the position on secondary index since should not be included in cells tree" ), _ => self .rest .iter() .enumerate() - .find(|(_, c)| c.identifier == identifier) + .find(|(_, c)| c.identifier() == identifier) // + 1 because sbbst starts at 1 not zero .map(|(i, _)| i+1) .expect("can't find index of identfier"), @@ -123,9 +133,9 @@ impl TableColumns { } pub fn self_assert(&self) { for column in self.non_indexed_columns() { - let idx = self.cells_tree_index_of(column.identifier); + let idx = self.cells_tree_index_of(column.identifier()); let id = self.column_id_of_cells_index(idx).unwrap(); - assert!(column.identifier == id); + assert!(column.identifier() == id); } } } @@ -133,12 +143,12 @@ impl TableColumns { impl From<&TableColumns> for ColumnIDs { fn from(columns: &TableColumns) -> Self { ColumnIDs::new( - columns.primary.identifier, - columns.secondary.identifier, + columns.primary.identifier(), + columns.secondary.identifier(), columns .non_indexed_columns() .into_iter() - .map(|column| column.identifier) + .map(|column| column.identifier()) .collect_vec(), ) } @@ -263,7 +273,7 @@ impl Table { .columns .non_indexed_columns() .iter() - .map(|tc| tc.identifier) + .map(|tc| tc.identifier()) .filter_map(|id| cells.find_by_column(id).map(|info| (id, info))) .map(|(id, info)| cell::MerkleCell::new(id, info.value, info.primary)) .collect::>(); @@ -625,7 +635,7 @@ impl TableColumns { impl TableColumn { pub fn to_zkcolumn(&self) -> ZkColumn { ZkColumn { - id: self.identifier, + id: self.identifier(), kind: match self.index { IndexType::Primary => ColumnKind::PrimaryIndex, IndexType::Secondary => ColumnKind::SecondaryIndex, diff --git a/mp2-v1/tests/common/values_extraction.rs b/mp2-v1/tests/common/values_extraction.rs index dec732e99..0bccb0fd8 100644 --- a/mp2-v1/tests/common/values_extraction.rs +++ b/mp2-v1/tests/common/values_extraction.rs @@ -2,32 +2,17 @@ use super::{storage_trie::TestStorageTrie, TestContext}; use crate::common::StorageSlotInfo; -use alloy::{ - eips::BlockNumberOrTag, - primitives::{Address, U256}, - providers::Provider, -}; +use alloy::{eips::BlockNumberOrTag, primitives::Address, providers::Provider}; use log::info; -use mp2_common::{ - eth::{ProofQuery, StorageSlot}, - mpt_sequential::utils::bytes_to_nibbles, - F, -}; -use mp2_v1::{ - api::SlotInput, - values_extraction::{ - gadgets::{column_info::ColumnInfo, metadata_gadget::MetadataGadget}, - identifier_for_value_column, - public_inputs::PublicInputs, - }, -}; +use mp2_common::F; +use mp2_v1::values_extraction::public_inputs::PublicInputs; use plonky2::field::types::Field; type MappingKey = Vec; impl TestContext { - /// Generate the Values Extraction (C.1) proof for single variables. - pub(crate) async fn prove_single_values_extraction( + /// Generate the Values Extraction proof for single or mapping variables. + pub(crate) async fn prove_values_extraction( &self, contract_address: &Address, bn: BlockNumberOrTag, @@ -44,110 +29,19 @@ impl TestContext { } let chain_id = self.rpc.get_chain_id().await.unwrap(); - info!("Prove the test storage trie including the simple slots {slots:?}"); let proof_value = trie.prove_value(contract_address, chain_id, self.params(), &self.b); // Check the public inputs. let pi = PublicInputs::new(&proof_value.proof().public_inputs); - assert_eq!(pi.n(), F::from_canonical_usize(slots.len())); assert_eq!(pi.root_hash(), trie.root_hash()); { - let exp_key = StorageSlot::Simple(slots[0].slot().slot() as usize).mpt_key_vec(); - let exp_key: Vec<_> = bytes_to_nibbles(&exp_key) - .into_iter() - .map(F::from_canonical_u8) - .collect(); - - let (key, ptr) = pi.mpt_key_info(); - assert_eq!(key, exp_key); - assert_eq!(ptr, F::NEG_ONE); - } - - proof_value.serialize().unwrap() - } - - /// Generate the Values Extraction (C.1) proof for mapping variables. - pub(crate) async fn prove_mapping_values_extraction( - &self, - contract_address: &Address, - chain_id: u64, - slot_input: &SlotInput, - mapping_keys: Vec, - ) -> Vec { - let first_mapping_key = mapping_keys[0].clone(); - let storage_slot_number = mapping_keys.len(); - - // Initialize the test trie. - let mut trie = TestStorageTrie::new(); - info!("mapping mpt proving: Initialized the test storage trie"); - - // Compute the column identifier for the value column. - let column_identifier = - identifier_for_value_column(slot_input, contract_address, chain_id, vec![]); - // Compute the table metadata information. - let slot = slot_input.slot(); - let evm_word = slot_input.evm_word(); - let table_info = vec![ColumnInfo::new( - slot, - column_identifier, - slot_input.byte_offset(), - slot_input.bit_offset(), - slot_input.length(), - evm_word, - )]; - let metadata = MetadataGadget::new(table_info, &[column_identifier], evm_word); - - // Query the slot and add the node path to the trie. - let slot = slot as usize; - for mapping_key in mapping_keys { - let query = ProofQuery::new_mapping_slot(*contract_address, slot, mapping_key.clone()); - let response = self - .query_mpt_proof(&query, BlockNumberOrTag::Number(self.block_number().await)) - .await; - - // Get the nodes to prove. Reverse to the sequence from leaf to root. - let nodes: Vec<_> = response.storage_proof[0] - .proof + let n = slots .iter() - .rev() - .map(|node| node.to_vec()) - .collect(); - - let sslot = StorageSlot::Mapping(mapping_key.clone(), slot); - info!( - "Save the mapping key {:?} (value {}) on slot {} to the test storage trie", - U256::from_be_slice(&mapping_key), - response.storage_proof[0].value, - slot - ); - - // TODO: Check if we could use the column identifier as the - // outer key ID for mapping values. - let outer_key_id = Some(column_identifier); - let slot_info = StorageSlotInfo::new(sslot, metadata.clone(), outer_key_id, None); - trie.add_slot(slot_info, nodes); - } - - let chain_id = self.rpc.get_chain_id().await.unwrap(); - info!("Prove the test storage trie including the mapping slots ({slot}, ...)"); - let proof = trie.prove_value(contract_address, chain_id, self.params(), &self.b); - - // Check the public inputs. - let pi = PublicInputs::new(&proof.proof().public_inputs); - assert_eq!(pi.n(), F::from_canonical_usize(storage_slot_number)); - assert_eq!(pi.root_hash(), trie.root_hash()); - { - let exp_key = StorageSlot::Mapping(first_mapping_key, slot).mpt_key_vec(); - let exp_key: Vec<_> = bytes_to_nibbles(&exp_key) - .into_iter() - .map(F::from_canonical_u8) - .collect(); - - let (key, ptr) = pi.mpt_key_info(); - assert_eq!(key, exp_key); - assert_eq!(ptr, F::NEG_ONE); + .map(|slot_info| slot_info.slot_inputs().len()) + .sum(); + assert_eq!(pi.n(), F::from_canonical_usize(n)); } - proof.serialize().expect("can't serialize mpt proof") + proof_value.serialize().unwrap() } } diff --git a/mp2-v1/tests/integrated_tests.rs b/mp2-v1/tests/integrated_tests.rs index 6e8d9631d..4efe33ccd 100644 --- a/mp2-v1/tests/integrated_tests.rs +++ b/mp2-v1/tests/integrated_tests.rs @@ -91,6 +91,7 @@ async fn integrated_indexing() -> Result<()> { ChangeType::Update(UpdateType::SecondaryIndex), ]; single.run(&mut ctx, genesis, changes.clone()).await?; + let (mut mapping, genesis) = TableIndexing::mapping_test_case(&mut ctx).await?; let changes = vec![ ChangeType::Insertion, @@ -101,6 +102,25 @@ async fn integrated_indexing() -> Result<()> { ]; mapping.run(&mut ctx, genesis, changes).await?; + let (mut single_struct, genesis) = TableIndexing::single_struct_test_case(&mut ctx).await?; + let changes = vec![ + ChangeType::Update(UpdateType::Rest), + ChangeType::Silent, + ChangeType::Update(UpdateType::SecondaryIndex), + ]; + single_struct + .run(&mut ctx, genesis, changes.clone()) + .await?; + let (mut mapping_struct, genesis) = TableIndexing::mapping_struct_test_case(&mut ctx).await?; + let changes = vec![ + ChangeType::Insertion, + ChangeType::Update(UpdateType::Rest), + ChangeType::Silent, + ChangeType::Update(UpdateType::SecondaryIndex), + ChangeType::Deletion, + ]; + mapping_struct.run(&mut ctx, genesis, changes).await?; + let (mut merged, genesis) = TableIndexing::merge_table_test_case(&mut ctx).await?; let changes = vec![ ChangeType::Insertion, @@ -114,6 +134,7 @@ async fn integrated_indexing() -> Result<()> { // save columns information and table information in JSON so querying test can pick up write_table_info(MAPPING_TABLE_INFO_FILE, mapping.table_info())?; write_table_info(MERGE_TABLE_INFO_FILE, merged.table_info())?; + Ok(()) }