Integration test for off-chain data

mp2-v1/src/api.rs

@@ -1,11 +1,12 @@
 //! Main APIs and related structures
 
-use std::iter::once;
+use std::{collections::BTreeSet, iter::once};
 
 use crate::{
     block_extraction,
     contract_extraction::{self, compute_metadata_digest as contract_metadata_digest},
     final_extraction,
+    indexing::ColumnID,
     length_extraction::{
         self, compute_metadata_digest as length_metadata_digest, LengthCircuitInput,
     },
@@ -33,7 +34,6 @@ use plonky2::{
 };
 use plonky2_ecgfp5::curve::curve::Point;
 use serde::{Deserialize, Serialize};
-use verifiable_db::query::computational_hash_ids::ColumnIDs;
 
 /// Struct containing the expected input MPT Extension/Branch node
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -253,19 +253,23 @@ pub fn metadata_hash(
 
 // compute metadata digest for a table including no provable extraction data:
 // it corresponds to the digest of the column identifiers
-pub(crate) fn no_provable_metadata_digest(column_ids: &ColumnIDs) -> Digest {
+pub(crate) fn no_provable_metadata_digest<I: IntoIterator<Item = ColumnID>>(
+    column_ids: I,
+) -> Digest {
     map_to_curve_point(
-        &vec![column_ids.primary_column(), column_ids.secondary_column()]
+        &column_ids
             .into_iter()
-            .chain(column_ids.non_indexed_columns())
-            .map(|id| F::from_canonical_u64(id))
+            .collect::<BTreeSet<_>>() // collect into a BTreeSet to ensure they are hashed
+            // in a deterministic order
+            .into_iter()
+            .map(F::from_canonical_u64)
             .collect_vec(),
     )
 }
 
 /// Compute the metadata hash for a table including no provable extraction data.
 /// The input is the set of the column identifiers of the table
-pub fn no_provable_metadata_hash(column_ids: &ColumnIDs) -> MetadataHash {
+pub fn no_provable_metadata_hash<I: IntoIterator<Item = ColumnID>>(column_ids: I) -> MetadataHash {
     let metadata_digest = no_provable_metadata_digest(column_ids);
     // Add the prefix to the metadata digest to ensure the metadata digest
     // will keep track of whether we use this dummy circuit or not.

mp2-v1/src/final_extraction/api.rs

@@ -11,9 +11,11 @@ use recursion_framework::{
     framework::{prepare_recursive_circuit_for_circuit_set, RecursiveCircuits},
 };
 use serde::{Deserialize, Serialize};
-use verifiable_db::query::universal_circuit::universal_circuit_inputs::RowCells;
 
-use crate::{api::no_provable_metadata_digest, values_extraction::compute_table_row_digest};
+use crate::{
+    api::no_provable_metadata_digest, indexing::row::CellCollection,
+    values_extraction::compute_table_row_digest,
+};
 
 use super::{
     base_circuit::BaseCircuitInput,
@@ -252,11 +254,11 @@ impl CircuitInput {
         Ok(Self::Lengthed(LengthedCircuitInput { base, length_proof }))
     }
     /// Instantiate inputs for the dummy circuit dealing with no provable extraction case
-    pub fn new_no_provable_input(
+    pub fn new_no_provable_input<PrimaryIndex: PartialEq + Eq + Default + Clone>(
         block_number: U256,
         block_hash: HashOutput,
         prev_block_hash: HashOutput,
-        table_rows: &[RowCells],
+        table_rows: &[CellCollection<PrimaryIndex>],
     ) -> Result<Self> {
         let [block_hash, prev_block_hash] = [block_hash, prev_block_hash].map(|h| {
             h.pack(mp2_common::utils::Endianness::Little)
@@ -271,7 +273,7 @@ impl CircuitInput {
             "At least one row should be provided as input to construct a table"
         );
         let column_ids = table_rows[0].column_ids();
-        let metadata_digest = no_provable_metadata_digest(&column_ids);
+        let metadata_digest = no_provable_metadata_digest(column_ids);
         let row_digest = compute_table_row_digest(table_rows);
 
         Ok(Self::NoProvable(DummyCircuit::new(

mp2-v1/src/indexing/row.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use super::{cell::CellTreeKey, ColumnID};
 use alloy::primitives::U256;
 use anyhow::Result;
@@ -117,6 +119,12 @@ impl<PrimaryIndex: PartialEq + Eq + Default + Clone> CellCollection<PrimaryIndex
     pub fn find_by_column(&self, id: ColumnID) -> Option<&CellInfo<PrimaryIndex>> {
         self.0.get(&id)
     }
+
+    // Return the set of all the column identifiers for cells in `self`
+    pub fn column_ids(&self) -> HashSet<ColumnID> {
+        self.0.keys().cloned().collect()
+    }
+
     // take all the cells ids on both collections, take the value present in the updated one
     // if it exists, otherwise take from self.
     pub fn merge_with_update(&self, updated_cells: &Self) -> Self {

mp2-v1/src/values_extraction/mod.rs

@@ -14,7 +14,6 @@ use plonky2::{
 };
 use plonky2_ecgfp5::curve::curve::Point as Digest;
 use std::iter::{self, once};
-use verifiable_db::query::universal_circuit::universal_circuit_inputs::RowCells;
 
 pub mod api;
 mod branch;
@@ -26,13 +25,24 @@ pub mod public_inputs;
 pub use api::{build_circuits_params, generate_proof, CircuitInput, PublicParameters};
 pub use public_inputs::PublicInputs;
 
+use crate::indexing::row::CellCollection;
+
 /// Constant prefixes for key and value IDs. Restrict both prefixes to 3-bytes,
 /// so `prefix + slot (u8)` could be converted to an U32.
 pub(crate) const KEY_ID_PREFIX: &[u8] = b"KEY";
 pub(crate) const VALUE_ID_PREFIX: &[u8] = b"VAL";
 
 pub(crate) const BLOCK_ID_DST: &[u8] = b"BLOCK_NUMBER";
 
+/// Compute the identifier for a column of a table containing off-chain data
+pub fn identifier_offchain_column(table_name: &str, column_name: &str) -> u64 {
+    let inputs: Vec<F> = vec![table_name, column_name]
+        .into_iter()
+        .flat_map(|name| name.as_bytes().to_fields())
+        .collect_vec();
+    H::hash_no_pad(&inputs).elements[0].to_canonical_u64()
+}
+
 pub fn identifier_block_column() -> u64 {
     let inputs: Vec<F> = BLOCK_ID_DST.to_fields();
     H::hash_no_pad(&inputs).elements[0].to_canonical_u64()
@@ -162,26 +172,28 @@ pub fn compute_leaf_mapping_metadata_digest(key_id: u64, value_id: u64, slot: u8
 }
 
 /// Compute the row value digest of one table, whose rows are provided as input
-pub fn compute_table_row_digest(table_rows: &[RowCells]) -> Digest {
+pub fn compute_table_row_digest<PrimaryIndex: PartialEq + Eq + Default + Clone>(
+    table_rows: &[CellCollection<PrimaryIndex>],
+) -> Digest {
     let column_ids = table_rows[0].column_ids();
     table_rows
-        .into_iter()
+        .iter()
         .enumerate()
         .fold(Digest::NEUTRAL, |acc, (i, row)| {
+            let current_column_ids = row.column_ids();
             // check that column ids are the same for each row
             assert_eq!(
-                row.column_ids(),
-                column_ids,
+                current_column_ids, column_ids,
                 "row {i} has different column ids than other rows"
             );
-            let cells = row.to_cells();
             let current_row_digest = map_to_curve_point(
-                &cells
+                &current_column_ids
                     .into_iter()
-                    .skip(1) // we skip primary index column
-                    .fold(Digest::NEUTRAL, |acc, row| {
+                    .fold(Digest::NEUTRAL, |acc, id| {
                         acc + map_to_curve_point(
-                            &once(row.id).chain(row.value.to_fields()).collect_vec(),
+                            &once(F::from_canonical_u64(id))
+                                .chain(row.find_by_column(id).unwrap().value.to_fields())
+                                .collect_vec(),
                         )
                     })
                     .to_fields(),