From cd46b4488d53de3dfabf8eb8e4d8d30515ed5eb6 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 09:44:30 -0700 Subject: [PATCH 001/124] added 25519 prime field --- Cargo.toml | 1 + src/ff/ec_prime_field.rs | 194 +++++++++++++++++++++++++++++++++++++++ src/ff/mod.rs | 1 + 3 files changed, 196 insertions(+) create mode 100644 src/ff/ec_prime_field.rs diff --git a/Cargo.toml b/Cargo.toml index e264d503f..11e15fdc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ clap = { version = "4.3.2", optional = true, features = ["derive"] } comfy-table = { version = "7.0", optional = true } config = "0.13.2" criterion = { version = "0.5.1", optional = true, default-features = false, features = ["async_tokio", "plotters", "html_reports"] } +curve25519-dalek = "4.1.1" dashmap = "5.4" dhat = "0.3.2" embed-doc-image = "0.1.4" diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs new file mode 100644 index 000000000..c87dbfa94 --- /dev/null +++ b/src/ff/ec_prime_field.rs @@ -0,0 +1,194 @@ +use generic_array::GenericArray; +use curve25519_dalek::scalar::Scalar; +use rand_core::RngCore; +use typenum::U32; + +use crate::{ + ff::Serializable, + secret_sharing::{Block, SharedValue}, +}; + +impl Block for Scalar { + type Size = U32; +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct Fp25519(::Storage); + +//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? + + +impl SharedValue for Fp25519 { + type Storage = Scalar; + const BITS: u32 = 256; + const ZERO: Self = Self(Scalar::ZERO); +} + +impl Serializable for Fp25519 { + type Size = <::Storage as Block>::Size; + + fn serialize(&self, buf: &mut GenericArray) { + let raw = &self.0.as_bytes()[..buf.len()] ; + buf.copy_from_slice(raw); + } + + fn deserialize(buf: &GenericArray) -> Self { + let mut buf_to = [0u8; 32]; + buf_to[..buf.len()].copy_from_slice(buf); + + Fp25519(Scalar::from_bytes_mod_order(buf_to)) + } +} + + +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> Fp25519 { + //Fp25519(Scalar::random(rng: &mut R)) + let mut scalar_bytes = [0u8; 64]; + rng.fill_bytes(&mut scalar_bytes); + Fp25519(Scalar::from_bytes_mod_order_wide(&scalar_bytes)) + } +} + + +impl std::ops::Add for Fp25519 { +type Output = Self; + +fn add(self, rhs: Self) -> Self::Output { + Self(self.0+rhs.0) +} +} + +impl std::ops::AddAssign for Fp25519 { +#[allow(clippy::assign_op_pattern)] +fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; +} +} + +impl std::ops::Neg for Fp25519 { +type Output = Self; + +fn neg(self) -> Self::Output { + Self(self.0.neg()) +} +} + +impl std::ops::Sub for Fp25519 { +type Output = Self; + +fn sub(self, rhs: Self) -> Self::Output { + Self(self.0- rhs.0) +} +} + +impl std::ops::SubAssign for Fp25519 { +#[allow(clippy::assign_op_pattern)] +fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; +} +} + +impl std::ops::Mul for Fp25519 { +type Output = Self; + +fn mul(self, rhs: Self) -> Self::Output { + Self(self.0 * rhs.0) +} +} + +impl std::ops::MulAssign for Fp25519 { +#[allow(clippy::assign_op_pattern)] +fn mul_assign(&mut self, rhs: Self) { + *self = *self * rhs; +} +} + +//must not use with ZERO +impl Fp25519 { + pub fn invert(&self) -> Fp25519 { + Fp25519(self.0.invert()) + } +} + +#[cfg(all(test, unit_test))] +mod test { + use generic_array::GenericArray; + use crate::ff::ec_prime_field::Fp25519; + use crate::ff::Serializable; + use typenum::U32; + use curve25519_dalek::scalar::Scalar; + use rand::{thread_rng, Rng}; + use crate::secret_sharing::SharedValue; + + #[test] + fn serde_25519() { + let input:[u8;32] = [ + 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + ]; + let mut output: GenericArray = [0u8;32].into(); + let a = Fp25519::deserialize(&input.into()); + assert_eq!(a.0.as_bytes()[..32],input); + a.serialize(&mut output); + assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); + assert_eq!(input,output.as_slice()[..32]); + } + + // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek + #[test] + fn simple_arithmetics_25519() { + let a = Fp25519(Scalar::from_bytes_mod_order([ + 0x02, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let b = Fp25519(Scalar::from_bytes_mod_order([ + 0x03, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let c = Fp25519(Scalar::from_bytes_mod_order([ + 0x01, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let d = Fp25519(Scalar::from_bytes_mod_order([ + 0x05, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let e = Fp25519(Scalar::from_bytes_mod_order([ + 0x06, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let cc = b-a; + let dc = a+b; + let ec = a*b; + assert_eq!(cc,c); + assert_eq!(dc,d); + assert_eq!(ec,e); + } + + #[test] + fn simple_random_25519(){ + let mut rng = thread_rng(); + assert_ne!(Fp25519::ZERO,rng.gen::()); + } + + #[test] + fn invert_25519(){ + let mut rng = thread_rng(); + let a=rng.gen::(); + let ia=a.invert(); + assert_eq!(a*ia, Fp25519(Scalar::ONE)); + } +} \ No newline at end of file diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 453876175..2afd4a602 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -5,6 +5,7 @@ mod field; mod galois_field; mod prime_field; +mod ec_prime_field; use std::ops::{Add, AddAssign, Mul, MulAssign, Neg, Sub, SubAssign}; From e59b214c7b7e7e33485eeaa565bbd944f35c719b Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 11:49:49 -0700 Subject: [PATCH 002/124] add files, make ec pub --- src/ff/mod.rs | 2 +- src/protocol/mod.rs | 1 + src/protocol/prf_eval/eval.rs | 0 src/protocol/prf_eval/keygen.rs | 0 src/protocol/prf_eval/mod.rs | 0 5 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 src/protocol/prf_eval/eval.rs create mode 100644 src/protocol/prf_eval/keygen.rs create mode 100644 src/protocol/prf_eval/mod.rs diff --git a/src/ff/mod.rs b/src/ff/mod.rs index b89208c43..c4d287e7a 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -5,7 +5,7 @@ mod field; mod galois_field; mod prime_field; -mod ec_prime_field; +pub mod ec_prime_field; use std::ops::{Add, AddAssign, Sub, SubAssign}; diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 7931a1e37..530c12f74 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -6,6 +6,7 @@ pub mod context; pub mod dp; pub mod ipa; pub mod modulus_conversion; +pub mod prf_eval; #[cfg(feature = "descriptive-gate")] pub mod prf_sharding; pub mod prss; diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/protocol/prf_eval/keygen.rs b/src/protocol/prf_eval/keygen.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs new file mode 100644 index 000000000..e69de29bb From 1b552f4c94381496b76f89994c06feddd3b27648 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 14:20:19 -0700 Subject: [PATCH 003/124] last commit before working on shared curve points --- src/protocol/prf_eval/eval.rs | 28 ++++++++++++++++++++++++++++ src/protocol/prf_eval/mod.rs | 2 ++ 2 files changed, 30 insertions(+) diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs index e69de29bb..30c5c1d38 100644 --- a/src/protocol/prf_eval/eval.rs +++ b/src/protocol/prf_eval/eval.rs @@ -0,0 +1,28 @@ +use crate::{ + error::Error, + helpers::Direction, + ff::ec_prime_field::Fp25519, + protocol::{context::Context, step::BitOpStep, BasicProtocols, RecordId}, + secret_sharing::{Linear as LinearSecretSharing, LinearRefOps}, +}; + +/// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) +/// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 +/// PRF key k is generated using keygen +/// In 3IPA, x is the match key +/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output +pub async fn eval_dy( + ctx: C, + record_id: RecordId, + k: &[S], + x: &[S], +) -> Result + where + C: Context, + S: LinearSecretSharing + BasicProtocols, + for<'a> &'a S: LinearRefOps<'a, S, Fp25519>, +{ + let role = ctx.role(); + + Ok(5u64) +} \ No newline at end of file diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index e69de29bb..6183868b8 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -0,0 +1,2 @@ +mod eval; +mod keygen; \ No newline at end of file From 9dc50c47d6263852baa4e123c56a3754747c40cc Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 16:48:31 -0700 Subject: [PATCH 004/124] adding curve points + conversion --- src/ff/curve_points.rs | 170 +++++++++++++++++++++++++++++++++++++++ src/ff/ec_prime_field.rs | 6 ++ src/ff/mod.rs | 1 + 3 files changed, 177 insertions(+) create mode 100644 src/ff/curve_points.rs diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs new file mode 100644 index 000000000..3351efc51 --- /dev/null +++ b/src/ff/curve_points.rs @@ -0,0 +1,170 @@ +use std::ops::Mul; +use generic_array::GenericArray; +use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; +use rand_core::RngCore; +use typenum::U32; + +use crate::{ + ff::{Serializable,ec_prime_field::Fp25519}, + secret_sharing::{Block, SharedValue}, +}; + +impl Block for CompressedRistretto { + type Size = U32; +} + +///ristretto point for curve 25519 +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct RP25519(::Storage); + +//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? + +/// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 +impl SharedValue for RP25519 { + type Storage = CompressedRistretto; + const BITS: u32 = 256; + const ZERO: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); +} + +impl Serializable for RP25519 { + type Size = <::Storage as Block>::Size; + + fn serialize(&self, buf: &mut GenericArray) { + let raw = &self.0.as_bytes()[..buf.len()] ; + buf.copy_from_slice(raw); + } + + fn deserialize(buf: &GenericArray) -> Self { + RP25519(CompressedRistretto::from_slice(buf).unwrap()) + } +} + + +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> RP25519 { + //Fp25519(Scalar::random(rng: &mut R)) + let mut scalar_bytes = [0u8; 64]; + rng.fill_bytes(&mut scalar_bytes); + RP25519(RistrettoPoint::from_uniform_bytes(&scalar_bytes).compress()) + } +} + + +impl std::ops::Add for RP25519 { +type Output = Self; + +fn add(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap()+rhs.0.decompress().unwrap()).compress()) +} +} + +impl std::ops::AddAssign for RP25519 { +#[allow(clippy::assign_op_pattern)] +fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; +} +} + +impl std::ops::Neg for RP25519 { +type Output = Self; + +fn neg(self) -> Self::Output { + Self(self.0.decompress().unwrap().neg().compress()) +} +} + +impl std::ops::Sub for RP25519 { +type Output = Self; + +fn sub(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap()-rhs.0.decompress().unwrap()).compress()) +} +} + +impl std::ops::SubAssign for RP25519 { +#[allow(clippy::assign_op_pattern)] +fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; +} +} + + +///Scalar Multiplication +///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a +impl RP25519 { + +fn smul(self, rhs: Fp25519) -> RP25519 { + RP25519((self.0.decompress().unwrap() * >::into(rhs)).compress()) +} +} + +///<'a> std::ops::MulAssign<&'a Fp25519> for +impl RP25519 { +#[allow(clippy::assign_op_pattern)] +fn smul_assign(&mut self, rhs: Fp25519) { + *self = self.smul(rhs); +} +} + +///do not use +impl std::ops::Mul for RP25519 { + type Output = Self; + + fn mul(self, rhs: RP25519) -> Self::Output { + Self::ZERO + } +} + +///do not use +impl std::ops::MulAssign for RP25519 { + + fn mul_assign(& mut self, rhs: RP25519) { + *self=Self::ZERO; + } +} + +impl From for RP25519 { + fn from(s: Scalar) -> Self { + RP25519(RistrettoPoint::mul_base(&s).compress()) + } +} + +impl From for RP25519 { + fn from(s: Fp25519) -> Self { + RP25519(RistrettoPoint::mul_base(&s.into()).compress()) + } +} + +// impl Into for Fp25519 { +// fn into(self) -> RP25519 { +// RP25519(RistrettoPoint::mul_base(self.into()).compress()) +// } +// } + +#[cfg(all(test, unit_test))] +mod test { + use generic_array::GenericArray; + use crate::ff::curve_points::RP25519; + use crate::ff::Serializable; + use typenum::U32; + use curve25519_dalek::scalar::Scalar; + use rand::{thread_rng, Rng}; + use crate::secret_sharing::SharedValue; + + #[test] + fn serde_25519() { + let input:[u8;32] = [ + 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + ]; + let mut output: GenericArray = [0u8;32].into(); + let a = RP25519::deserialize(&input.into()); + assert_eq!(a.0.as_bytes()[..32],input); + a.serialize(&mut output); + assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); + assert_eq!(input,output.as_slice()[..32]); + } + +} \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index c87dbfa94..5e2d13fa5 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -24,6 +24,12 @@ impl SharedValue for Fp25519 { const ZERO: Self = Self(Scalar::ZERO); } +impl Into for Fp25519 { + fn into(self) -> Scalar { + self.0 + } +} + impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; diff --git a/src/ff/mod.rs b/src/ff/mod.rs index c4d287e7a..b26596086 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -6,6 +6,7 @@ mod field; mod galois_field; mod prime_field; pub mod ec_prime_field; +pub mod curve_points; use std::ops::{Add, AddAssign, Sub, SubAssign}; From 4f1f605b539fe8752c9332dd3653ba2f9b660998 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Wed, 4 Oct 2023 00:48:47 +0800 Subject: [PATCH 005/124] Feature vector label vector dot product --- .../prf_sharding/feature_label_dot_product.rs | 469 ++++++++++++++++++ src/protocol/prf_sharding/mod.rs | 3 + 2 files changed, 472 insertions(+) create mode 100644 src/protocol/prf_sharding/feature_label_dot_product.rs diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs new file mode 100644 index 000000000..bdea62dad --- /dev/null +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -0,0 +1,469 @@ +use futures::{stream::iter as stream_iter, TryStreamExt}; +use futures_util::future::try_join; +use ipa_macros::Step; + +use crate::{ + error::Error, + ff::{Field, GaloisField, Gf2, PrimeField, Serializable}, + protocol::{ + basics::{SecureMul, ShareKnownValue}, + boolean::or::or, + context::{Context, UpgradableContext, UpgradedContext, Validator}, + modulus_conversion::convert_bits, + step::BitOpStep, + RecordId, + }, + secret_sharing::{ + replicated::{ + malicious::ExtendableField, semi_honest::AdditiveShare as Replicated, + ReplicatedSecretSharing, + }, + BitDecomposed, Linear as LinearSecretSharing, + }, + seq_join::seq_try_join_all, +}; + +pub struct PrfShardedIpaInputRow { + prf_of_match_key: u64, + is_trigger_bit: Replicated, + feature_vector: Replicated, +} + +struct InputsRequiredFromPrevRow { + ever_encountered_a_trigger_event: Replicated, + is_saturated: Replicated, +} + +impl InputsRequiredFromPrevRow { + /// + /// This function contains the main logic for the per-user attribution circuit. + /// Multiple rows of data about a single user are processed in-order from newest to oldest. + /// + /// Summary: + /// - Last touch attribution + /// - Every source event which has a subsequent trigger event receives attribution + /// - Per user capping + /// - A cumulative count of "Source Events Receiving Attribution" is maintained + /// - Bitwise addition is used, and a single bit indicates if the sum is "saturated" + /// - The only available values for "cap" are powers of 2 (i.e. 1, 2, 4, 8, 16, 32, ...) + /// - Prior to saturation, feature vectors of source events receiving attribution contribute to the dot-product. + /// - All subsequent rows contribute zero + /// - Outputs + /// - If a user has `N` input rows, they will generate `N-1` output rows. (The first row cannot possibly contribute any value to the output) + /// - Each output row is a vector, either the feature vector or zeroes. + pub async fn compute_row_with_previous( + &mut self, + ctx: C, + record_id: RecordId, + input_row: &PrfShardedIpaInputRow, + ) -> Result>, Error> + where + C: UpgradedContext>, + FV: GaloisField, + { + let share_of_one = Replicated::share_known_value(&ctx, Gf2::ONE); + let is_source_event = &share_of_one - &input_row.is_trigger_bit; + + let (ever_encountered_a_trigger_event, did_source_get_attributed) = try_join( + or( + ctx.narrow(&Step::EverEncounteredTriggerEvent), + record_id, + &input_row.is_trigger_bit, + &self.ever_encountered_a_trigger_event, + ), + is_source_event.multiply( + &self.ever_encountered_a_trigger_event, + ctx.narrow(&Step::DidSourceReceiveAttribution), + record_id, + ), + ) + .await?; + + let (updated_is_saturated, capped_label) = try_join( + or( + ctx.narrow(&Step::ComputeSaturatingSum), + record_id, + &self.is_saturated, + &did_source_get_attributed, + ), + did_source_get_attributed.multiply( + &(share_of_one - &self.is_saturated), + ctx.narrow(&Step::IsAttributedSourceAndPrevRowNotSaturated), + record_id, + ), + ) + .await?; + + let unbitpacked_feature_vector = BitDecomposed::decompose(FV::BITS, |i| { + input_row.feature_vector.map(|v| Gf2::truncate_from(v[i])) + }); + + let capped_attributed_feature_vector = compute_capped_feature_vector( + ctx.narrow(&Step::ComputedCappedFeatureVector), + record_id, + &capped_label, + &unbitpacked_feature_vector, + ) + .await?; + + self.ever_encountered_a_trigger_event = ever_encountered_a_trigger_event; + self.is_saturated = updated_is_saturated; + + Ok(capped_attributed_feature_vector) + } +} + +#[derive(Step)] +pub enum UserNthRowStep { + #[dynamic] + Row(usize), +} + +impl From for UserNthRowStep { + fn from(v: usize) -> Self { + Self::Row(v) + } +} + +#[derive(Step)] +pub(crate) enum Step { + BinaryValidator, + PrimeFieldValidator, + EverEncounteredTriggerEvent, + DidSourceReceiveAttribution, + ComputeSaturatingSum, + IsAttributedSourceAndPrevRowNotSaturated, + ComputedCappedFeatureVector, + ModulusConvertFeatureVectorBits, +} + +fn compute_histogram_of_users_with_row_count(rows_chunked_by_user: &[Vec]) -> Vec { + let mut output = vec![]; + for user_rows in rows_chunked_by_user { + for j in 0..user_rows.len() { + if j >= output.len() { + output.push(0); + } + output[j] += 1; + } + } + output +} + +fn set_up_contexts(root_ctx: &C, histogram: &[usize]) -> Vec +where + C: UpgradedContext>, +{ + let mut context_per_row_depth = Vec::with_capacity(histogram.len()); + for (row_number, num_users_having_that_row_number) in histogram.iter().enumerate() { + if row_number == 0 { + // no multiplications needed for each user's row 0. No context needed + } else { + let ctx_for_row_number = root_ctx + .narrow(&UserNthRowStep::from(row_number)) + .set_total_records(*num_users_having_that_row_number); + context_per_row_depth.push(ctx_for_row_number); + } + } + context_per_row_depth +} + +fn chunk_rows_by_user( + input_rows: Vec>, +) -> Vec>> +where + FV: GaloisField, +{ + let mut rows_for_user: Vec> = vec![]; + + let mut rows_chunked_by_user = vec![]; + for row in input_rows { + if rows_for_user.is_empty() || row.prf_of_match_key == rows_for_user[0].prf_of_match_key { + rows_for_user.push(row); + } else { + rows_chunked_by_user.push(rows_for_user); + rows_for_user = vec![row]; + } + } + if !rows_for_user.is_empty() { + rows_chunked_by_user.push(rows_for_user); + } + + rows_chunked_by_user +} + +/// Sub-protocol of the PRF-sharded IPA Protocol +/// +/// After the computation of the per-user PRF, addition of dummy records and shuffling, +/// the PRF column can be revealed. After that, all of the records corresponding to a single +/// device can be processed together. +/// +/// This circuit expects to receive records from multiple users, +/// but with all of the records from a given user adjacent to one another, and in time order. +/// +/// This circuit will compute attribution, and per-user capping. +/// +/// The output of this circuit is the input to the next stage: Aggregation. +/// +/// # Errors +/// Propagates errors from multiplications +/// # Panics +/// Propagates errors from multiplications +pub async fn compute_feature_label_dot_product( + sh_ctx: C, + input_rows: Vec>, +) -> Result, Error> +where + C: UpgradableContext, + C::UpgradedContext: UpgradedContext>, + C::UpgradedContext: UpgradedContext, + S: LinearSecretSharing + Serializable + SecureMul>, + FV: GaloisField, + F: PrimeField + ExtendableField, +{ + assert!(FV::BITS > 0); + + let rows_chunked_by_user = chunk_rows_by_user(input_rows); + let histogram = compute_histogram_of_users_with_row_count(&rows_chunked_by_user); + let binary_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); + let binary_m_ctx = binary_validator.context(); + let mut num_users_who_encountered_row_depth = Vec::with_capacity(histogram.len()); + let ctx_for_row_number = set_up_contexts(&binary_m_ctx, &histogram); + let mut futures = Vec::with_capacity(rows_chunked_by_user.len()); + for rows_for_user in rows_chunked_by_user { + for i in 0..rows_for_user.len() { + if i >= num_users_who_encountered_row_depth.len() { + num_users_who_encountered_row_depth.push(0); + } + num_users_who_encountered_row_depth[i] += 1; + } + + futures.push(evaluate_per_user_attribution_circuit( + &ctx_for_row_number, + num_users_who_encountered_row_depth + .iter() + .take(rows_for_user.len()) + .map(|x| RecordId(x - 1)) + .collect(), + rows_for_user, + )); + } + let outputs_chunked_by_user = seq_try_join_all(sh_ctx.active_work(), futures) + .await? + .into_iter() + .flatten() + .collect::>>>(); + + let prime_field_validator = sh_ctx.narrow(&Step::PrimeFieldValidator).validator::(); + let prime_field_ctx = prime_field_validator.context(); + + // modulus convert feature vector bits + let converted_feature_vector_bits = convert_bits( + prime_field_ctx + .narrow(&Step::ModulusConvertFeatureVectorBits) + .set_total_records(outputs_chunked_by_user.len()), + stream_iter(outputs_chunked_by_user), + 0..FV::BITS, + ); + + converted_feature_vector_bits + .try_fold( + vec![S::ZERO; 1 << FV::BITS], + |mut running_sums, row_contribution| async move { + for (i, contribution) in row_contribution.iter().enumerate() { + running_sums[i] += contribution; + } + Ok(running_sums) + }, + ) + .await +} + +async fn evaluate_per_user_attribution_circuit( + ctx_for_row_number: &[C], + record_id_for_each_depth: Vec, + rows_for_user: Vec>, +) -> Result>>, Error> +where + C: UpgradedContext>, + FV: GaloisField, +{ + assert!(!rows_for_user.is_empty()); + if rows_for_user.len() == 1 { + return Ok(Vec::new()); + } + let first_row = &rows_for_user[0]; + let mut prev_row_inputs = initialize_new_device_attribution_variables(first_row); + + let mut output = Vec::with_capacity(rows_for_user.len() - 1); + for (i, row) in rows_for_user.iter().skip(1).enumerate() { + let ctx_for_this_row_depth = ctx_for_row_number[i].clone(); // no context was created for row 0 + let record_id_for_this_row_depth = record_id_for_each_depth[i + 1]; // skip row 0 + + let capped_attribution_outputs = prev_row_inputs + .compute_row_with_previous(ctx_for_this_row_depth, record_id_for_this_row_depth, row) + .await?; + + output.push(capped_attribution_outputs); + } + + Ok(output) +} + +/// +/// Upon encountering the first row of data from a new user (as distinguished by a different OPRF of the match key) +/// this function encapsulates the variables that must be initialized. No communication is required for this first row. +/// +fn initialize_new_device_attribution_variables( + input_row: &PrfShardedIpaInputRow, +) -> InputsRequiredFromPrevRow +where + FV: GaloisField, +{ + InputsRequiredFromPrevRow { + ever_encountered_a_trigger_event: input_row.is_trigger_bit.clone(), + is_saturated: Replicated::ZERO, + } +} + +async fn compute_capped_feature_vector( + ctx: C, + record_id: RecordId, + capped_label: &Replicated, + feature_vector: &BitDecomposed>, +) -> Result>, Error> +where + C: UpgradedContext>, +{ + Ok(BitDecomposed::new( + ctx.parallel_join(feature_vector.iter().enumerate().map(|(i, bit)| { + let c1 = ctx.narrow(&BitOpStep::from(i)); + async move { capped_label.multiply(bit, c1, record_id).await } + })) + .await?, + )) +} + +#[cfg(all(test, unit_test))] +pub mod tests { + use crate::{ + ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf32Bit}, + protocol::prf_sharding::feature_label_dot_product::{ + compute_feature_label_dot_product, PrfShardedIpaInputRow, + }, + rand::Rng, + secret_sharing::{ + replicated::semi_honest::AdditiveShare as Replicated, IntoShares, SharedValue, + }, + test_executor::run, + test_fixture::{Reconstruct, Runner, TestWorld}, + }; + + struct PreShardedAndSortedOPRFTestInput { + prf_of_match_key: u64, + is_trigger_bit: Gf2, + feature_vector: FV, + } + + fn test_input( + prf_of_match_key: u64, + is_trigger: bool, + feature_vector: u32, + ) -> PreShardedAndSortedOPRFTestInput { + let is_trigger_bit = if is_trigger { Gf2::ONE } else { Gf2::ZERO }; + + PreShardedAndSortedOPRFTestInput { + prf_of_match_key, + is_trigger_bit, + feature_vector: Gf32Bit::truncate_from(feature_vector), + } + } + + impl IntoShares> for PreShardedAndSortedOPRFTestInput + where + FV: GaloisField + IntoShares>, + { + fn share_with(self, rng: &mut R) -> [PrfShardedIpaInputRow; 3] { + let PreShardedAndSortedOPRFTestInput { + prf_of_match_key, + is_trigger_bit, + feature_vector, + } = self; + + let [is_trigger_bit0, is_trigger_bit1, is_trigger_bit2] = + is_trigger_bit.share_with(rng); + let [feature_vector0, feature_vector1, feature_vector2] = + feature_vector.share_with(rng); + + [ + PrfShardedIpaInputRow { + prf_of_match_key, + is_trigger_bit: is_trigger_bit0, + feature_vector: feature_vector0, + }, + PrfShardedIpaInputRow { + prf_of_match_key, + is_trigger_bit: is_trigger_bit1, + feature_vector: feature_vector1, + }, + PrfShardedIpaInputRow { + prf_of_match_key, + is_trigger_bit: is_trigger_bit2, + feature_vector: feature_vector2, + }, + ] + } + } + + #[test] + fn semi_honest() { + run(|| async move { + let world = TestWorld::default(); + + let records: Vec> = vec![ + /* First User */ + test_input(123, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(123, false, 0b1101_0100_1111_0001_0111_0010_1010_1011), // this source DOES receive attribution + test_input(123, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(123, false, 0b0110_1101_0001_0100_1011_0100_1010_1001), // this source does not receive attribution (capped) + /* Second User */ + test_input(234, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(234, false, 0b0001_1010_0011_0111_0110_0010_1111_0000), // this source DOES receive attribution + /* Third User */ + test_input(345, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(345, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(345, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(345, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(345, false, 0b0111_0101_0001_0000_0111_0100_0101_0011), // this source DOES receive attribution + test_input(345, false, 0b1001_1000_1011_1101_0100_0110_0001_0100), // this source does not receive attribution (capped) + test_input(345, true, 0b0000_0000_0000_0000_0000_0000_0000_0000), // trigger + test_input(345, false, 0b1000_1001_0100_0011_0111_0010_0000_1101), // this source does not receive attribution (capped) + ]; + + let expected: [u128; 32] = [ + // 1101_0100_1111_0001_0111_0010_1010_1011 + // 0001_1010_0011_0111_0110_0010_1111_0000 + // + 0111_0101_0001_0000_0111_0100_0101_0011 + // ------------------------------------------- + // 1213_1211_1123_0112_0332_0120_2222_1022 + 1, 2, 1, 3, 1, 2, 1, 1, 1, 1, 2, 3, 0, 1, 1, 2, 0, 3, 3, 2, 0, 1, 2, 0, 2, 2, 2, 2, + 1, 0, 2, 2, + ]; + + let result: Vec<_> = world + .semi_honest(records.into_iter(), |ctx, input_rows| async move { + compute_feature_label_dot_product::< + _, + Gf32Bit, + Fp32BitPrime, + Replicated, + >(ctx, input_rows) + .await + .unwrap() + }) + .await + .reconstruct(); + assert_eq!(result, &expected); + }); + } +} diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 0e0b55304..fdd3a6440 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -27,6 +27,9 @@ use crate::{ }, seq_join::{seq_join, seq_try_join_all}, }; + +pub mod feature_label_dot_product; + pub struct PrfShardedIpaInputRow { prf_of_match_key: u64, is_trigger_bit: Replicated, From 94597b1b5d97a12f364e9ed1b22d5bc004be72b4 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 3 Oct 2023 10:17:02 -0700 Subject: [PATCH 006/124] from, into scalar for RP25519 + test --- src/ff/curve_points.rs | 22 +++++++++++++++++++--- src/ff/ec_prime_field.rs | 6 ++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 3351efc51..c0236c04a 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -93,7 +93,7 @@ fn sub_assign(&mut self, rhs: Self) { ///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a impl RP25519 { -fn smul(self, rhs: Fp25519) -> RP25519 { +fn s_mul(self, rhs: Fp25519) -> RP25519 { RP25519((self.0.decompress().unwrap() * >::into(rhs)).compress()) } } @@ -101,8 +101,8 @@ fn smul(self, rhs: Fp25519) -> RP25519 { ///<'a> std::ops::MulAssign<&'a Fp25519> for impl RP25519 { #[allow(clippy::assign_op_pattern)] -fn smul_assign(&mut self, rhs: Fp25519) { - *self = self.smul(rhs); +fn s_mul_assign(&mut self, rhs: Fp25519) { + *self = self.s_mul(rhs); } } @@ -149,6 +149,7 @@ mod test { use typenum::U32; use curve25519_dalek::scalar::Scalar; use rand::{thread_rng, Rng}; + use crate::ff::ec_prime_field::Fp25519; use crate::secret_sharing::SharedValue; #[test] @@ -167,4 +168,19 @@ mod test { assert_eq!(input,output.as_slice()[..32]); } + #[test] + fn scalar_to_point() { + let a = Scalar::ONE; + let b : RP25519 = a.clone().into(); + let d : Fp25519 = a.into(); + let c : RP25519 = RP25519::from(d); + assert_eq!(b,RP25519::ZERO); + assert_eq!(c,RP25519::ZERO); + } + + #[test] + fn curve_arithmetics() { + + } + } \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 5e2d13fa5..30ee3955e 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -117,6 +117,12 @@ impl Fp25519 { } } +impl From for Fp25519 { + fn from(s: Scalar) -> Self { + Fp25519(s) + } +} + #[cfg(all(test, unit_test))] mod test { use generic_array::GenericArray; From 4da801413bf8e055061ad107af0429e93f394888 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 3 Oct 2023 11:03:32 -0700 Subject: [PATCH 007/124] test aritmetics for RP25519 --- src/ff/curve_points.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index c0236c04a..1678210d7 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -180,7 +180,19 @@ mod test { #[test] fn curve_arithmetics() { - + let mut rng = rand::thread_rng(); + let a = rng.gen::(); + let b = rng.gen::(); + let c = a+b; + let d = RP25519::from(a)+RP25519::from(b); + assert_eq!(d, RP25519::from(c)); + assert_ne!(d, RP25519::ZERO); + let e = rng.gen::(); + let f=rng.gen::(); + let g =e*f; + let h = RP25519::from(e).s_mul(f); + assert_eq!(h,RP25519::from(g)); + assert_ne!(h, RP25519::ZERO); } } \ No newline at end of file From 094eb28346c1e5fc6d22fe09bca59f4cc6111679 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Wed, 4 Oct 2023 21:40:17 +0800 Subject: [PATCH 008/124] It doesn't really compile yet --- Cargo.toml | 1 + .../prf_sharding/feature_label_dot_product.rs | 43 ++++++++++--------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e264d503f..a72bfcfbf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,7 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } typenum = "1.16" # hpke is pinned to it x25519-dalek = "2.0.0-pre.0" +stream-flatten-iters = "0.2.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.5.0" diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index bdea62dad..66dca764e 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -1,7 +1,9 @@ use futures::{stream::iter as stream_iter, TryStreamExt}; -use futures_util::future::try_join; +use futures_util::{future::try_join, StreamExt}; use ipa_macros::Step; +use stream_flatten_iters::StreamExt as _; + use crate::{ error::Error, ff::{Field, GaloisField, Gf2, PrimeField, Serializable}, @@ -20,7 +22,7 @@ use crate::{ }, BitDecomposed, Linear as LinearSecretSharing, }, - seq_join::seq_try_join_all, + seq_join::seq_join, }; pub struct PrfShardedIpaInputRow { @@ -223,36 +225,34 @@ where { assert!(FV::BITS > 0); + let mut num_outputs = input_rows.len(); let rows_chunked_by_user = chunk_rows_by_user(input_rows); + num_outputs -= rows_chunked_by_user.len(); let histogram = compute_histogram_of_users_with_row_count(&rows_chunked_by_user); let binary_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); let binary_m_ctx = binary_validator.context(); - let mut num_users_who_encountered_row_depth = Vec::with_capacity(histogram.len()); + let mut num_users_who_encountered_row_depth = vec![0_u32; histogram.len()]; let ctx_for_row_number = set_up_contexts(&binary_m_ctx, &histogram); let mut futures = Vec::with_capacity(rows_chunked_by_user.len()); for rows_for_user in rows_chunked_by_user { - for i in 0..rows_for_user.len() { - if i >= num_users_who_encountered_row_depth.len() { - num_users_who_encountered_row_depth.push(0); - } - num_users_who_encountered_row_depth[i] += 1; - } - + let num_user_rows = rows_for_user.len(); futures.push(evaluate_per_user_attribution_circuit( &ctx_for_row_number, num_users_who_encountered_row_depth .iter() - .take(rows_for_user.len()) - .map(|x| RecordId(x - 1)) + .take(num_user_rows) + .map(|x| RecordId(*x)) .collect(), rows_for_user, )); + for i in 0..num_user_rows { + num_users_who_encountered_row_depth[i] += 1; + } } - let outputs_chunked_by_user = seq_try_join_all(sh_ctx.active_work(), futures) - .await? - .into_iter() - .flatten() - .collect::>>>(); + + let flattenned_stream = seq_join(sh_ctx.active_work(), stream_iter(futures)) + .map(|x| x.unwrap().into_iter()) + .flatten_iters(); let prime_field_validator = sh_ctx.narrow(&Step::PrimeFieldValidator).validator::(); let prime_field_ctx = prime_field_validator.context(); @@ -261,14 +261,14 @@ where let converted_feature_vector_bits = convert_bits( prime_field_ctx .narrow(&Step::ModulusConvertFeatureVectorBits) - .set_total_records(outputs_chunked_by_user.len()), - stream_iter(outputs_chunked_by_user), + .set_total_records(num_outputs), + flattenned_stream, 0..FV::BITS, ); converted_feature_vector_bits .try_fold( - vec![S::ZERO; 1 << FV::BITS], + vec![S::ZERO; usize::try_from(FV::BITS).unwrap()], |mut running_sums, row_contribution| async move { for (i, contribution) in row_contribution.iter().enumerate() { running_sums[i] += contribution; @@ -440,7 +440,7 @@ pub mod tests { test_input(345, false, 0b1000_1001_0100_0011_0111_0010_0000_1101), // this source does not receive attribution (capped) ]; - let expected: [u128; 32] = [ + let mut expected: [u128; 32] = [ // 1101_0100_1111_0001_0111_0010_1010_1011 // 0001_1010_0011_0111_0110_0010_1111_0000 // + 0111_0101_0001_0000_0111_0100_0101_0011 @@ -449,6 +449,7 @@ pub mod tests { 1, 2, 1, 3, 1, 2, 1, 1, 1, 1, 2, 3, 0, 1, 1, 2, 0, 3, 3, 2, 0, 1, 2, 0, 2, 2, 2, 2, 1, 0, 2, 2, ]; + expected.reverse(); // convert to little-endian order let result: Vec<_> = world .semi_honest(records.into_iter(), |ctx, input_rows| async move { From fd6c28e4ba7aade1945dc5d2407f561a4675146e Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 4 Oct 2023 11:29:56 -0700 Subject: [PATCH 009/124] upgrade share to share --- src/ff/curve_points.rs | 3 +-- src/ff/ec_prime_field.rs | 7 ++++--- src/protocol/prf_eval/eval.rs | 13 +++++++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 1678210d7..084b1a48d 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,4 +1,3 @@ -use std::ops::Mul; use generic_array::GenericArray; use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; use rand_core::RngCore; @@ -180,7 +179,7 @@ mod test { #[test] fn curve_arithmetics() { - let mut rng = rand::thread_rng(); + let mut rng = thread_rng(); let a = rng.gen::(); let b = rng.gen::(); let c = a+b; diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 30ee3955e..313cb9753 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -49,10 +49,11 @@ impl Serializable for Fp25519 { impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> Fp25519 { - //Fp25519(Scalar::random(rng: &mut R)) - let mut scalar_bytes = [0u8; 64]; + let mut scalar_bytes = [0u8; 32]; rng.fill_bytes(&mut scalar_bytes); - Fp25519(Scalar::from_bytes_mod_order_wide(&scalar_bytes)) + Fp25519(Scalar::from_bytes_mod_order(scalar_bytes)) + //not needed since above has sufficiently small bias + //Fp25519(Scalar::from_bytes_mod_order(&scalar_bytes)) } } diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs index 30c5c1d38..0307f42b8 100644 --- a/src/protocol/prf_eval/eval.rs +++ b/src/protocol/prf_eval/eval.rs @@ -1,11 +1,20 @@ use crate::{ error::Error, helpers::Direction, - ff::ec_prime_field::Fp25519, + ff::{ec_prime_field::Fp25519, curve_points::RP25519}, protocol::{context::Context, step::BitOpStep, BasicProtocols, RecordId}, - secret_sharing::{Linear as LinearSecretSharing, LinearRefOps}, + secret_sharing::{Linear as LinearSecretSharing, LinearRefOps, SharedValue, replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, }; + +impl Into> for AdditiveShare { + fn into(self) -> AdditiveShare + { + AdditiveShare::new(RP25519::from(self.left()),RP25519::from(self.right())) + } +} + + /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 /// PRF key k is generated using keygen From 3c5683e46e185b80141e316b148637ea3882b38b Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 4 Oct 2023 11:57:10 -0700 Subject: [PATCH 010/124] remove warning --- src/ff/ec_prime_field.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 313cb9753..cedad411a 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -1,6 +1,6 @@ use generic_array::GenericArray; use curve25519_dalek::scalar::Scalar; -use rand_core::RngCore; +//use rand_core::RngCore; use typenum::U32; use crate::{ From b2164819941d9ec9bfb50c01eb33f81db2333425 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 4 Oct 2023 15:47:45 -0700 Subject: [PATCH 011/124] add hash curve points --- src/ff/curve_points.rs | 65 +++++++++++++++++++++++++++++----------- src/ff/ec_prime_field.rs | 31 +++++++++---------- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 084b1a48d..eed23e5ec 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,7 +1,9 @@ use generic_array::GenericArray; use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; -use rand_core::RngCore; use typenum::U32; +use sha2::Sha256; +use hkdf::Hkdf; + use crate::{ ff::{Serializable,ec_prime_field::Fp25519}, @@ -93,32 +95,25 @@ fn sub_assign(&mut self, rhs: Self) { impl RP25519 { fn s_mul(self, rhs: Fp25519) -> RP25519 { - RP25519((self.0.decompress().unwrap() * >::into(rhs)).compress()) + RP25519((self.0.decompress().unwrap() * Scalar::from(rhs)).compress()) } } -///<'a> std::ops::MulAssign<&'a Fp25519> for -impl RP25519 { -#[allow(clippy::assign_op_pattern)] -fn s_mul_assign(&mut self, rhs: Fp25519) { - *self = self.s_mul(rhs); -} -} ///do not use impl std::ops::Mul for RP25519 { type Output = Self; - fn mul(self, rhs: RP25519) -> Self::Output { - Self::ZERO + fn mul(self, _rhs: RP25519) -> Self::Output { + panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); } } ///do not use impl std::ops::MulAssign for RP25519 { - fn mul_assign(& mut self, rhs: RP25519) { - *self=Self::ZERO; + fn mul_assign(& mut self, _rhs: RP25519) { + panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); } } @@ -134,11 +129,36 @@ impl From for RP25519 { } } -// impl Into for Fp25519 { -// fn into(self) -> RP25519 { -// RP25519(RistrettoPoint::mul_base(self.into()).compress()) -// } -// } +macro_rules! cp_hash_impl { + ( $u_type:ty, $byte_size:literal) => { + impl From for $u_type { + fn from(s: RP25519) -> Self { + let hk = Hkdf::::new(None, s.0.as_bytes()); + let mut okm = [0u8; $byte_size]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + <$u_type>::from_le_bytes(okm) + } + } + } +} + +cp_hash_impl!( + u128, + 16 +); + +cp_hash_impl!( + u64, + 8 +); + +cp_hash_impl!( + u32, + 4 +); + + #[cfg(all(test, unit_test))] mod test { @@ -194,4 +214,13 @@ mod test { assert_ne!(h, RP25519::ZERO); } + #[test] + fn curve_point_to_hash() { + let mut rng = thread_rng(); + let a = rng.gen::(); + assert_ne!(0u128,u128::from(a)); + assert_ne!(0u64,u64::from(a)); + assert_ne!(0u32,u32::from(a)); + } + } \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index cedad411a..3e4f762d8 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -15,7 +15,16 @@ impl Block for Scalar { #[derive(Clone, Copy, PartialEq, Debug)] pub struct Fp25519(::Storage); -//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? + +impl Fp25519 { + const ONE: Self = Self(Scalar::ONE); + + //must not use with ZERO + pub fn invert(&self) -> Fp25519 { + Fp25519(self.0.invert()) + } + +} impl SharedValue for Fp25519 { @@ -24,9 +33,9 @@ impl SharedValue for Fp25519 { const ZERO: Self = Self(Scalar::ZERO); } -impl Into for Fp25519 { - fn into(self) -> Scalar { - self.0 +impl From for Scalar { + fn from(s: Fp25519) -> Self { + s.0 } } @@ -111,12 +120,6 @@ fn mul_assign(&mut self, rhs: Self) { } } -//must not use with ZERO -impl Fp25519 { - pub fn invert(&self) -> Fp25519 { - Fp25519(self.0.invert()) - } -} impl From for Fp25519 { fn from(s: Scalar) -> Self { @@ -165,12 +168,6 @@ mod test { 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 ])); - let c = Fp25519(Scalar::from_bytes_mod_order([ - 0x01, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 - ])); let d = Fp25519(Scalar::from_bytes_mod_order([ 0x05, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, @@ -186,7 +183,7 @@ mod test { let cc = b-a; let dc = a+b; let ec = a*b; - assert_eq!(cc,c); + assert_eq!(cc,Fp25519::ONE); assert_eq!(dc,d); assert_eq!(ec,e); } From 0106fb3526a2c3aaa8a8465b55272968a1d47020 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Thu, 5 Oct 2023 08:26:08 +0800 Subject: [PATCH 012/124] Now it compiles. thank you martin --- .../prf_sharding/feature_label_dot_product.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index 66dca764e..3fb35835d 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -2,6 +2,8 @@ use futures::{stream::iter as stream_iter, TryStreamExt}; use futures_util::{future::try_join, StreamExt}; use ipa_macros::Step; +use std::iter::zip; + use stream_flatten_iters::StreamExt as _; use crate::{ @@ -237,7 +239,7 @@ where for rows_for_user in rows_chunked_by_user { let num_user_rows = rows_for_user.len(); futures.push(evaluate_per_user_attribution_circuit( - &ctx_for_row_number, + ctx_for_row_number[..rows_for_user.len() - 1].to_owned(), num_users_who_encountered_row_depth .iter() .take(num_user_rows) @@ -280,7 +282,7 @@ where } async fn evaluate_per_user_attribution_circuit( - ctx_for_row_number: &[C], + ctx_for_row_number: Vec, record_id_for_each_depth: Vec, rows_for_user: Vec>, ) -> Result>>, Error> @@ -296,12 +298,15 @@ where let mut prev_row_inputs = initialize_new_device_attribution_variables(first_row); let mut output = Vec::with_capacity(rows_for_user.len() - 1); - for (i, row) in rows_for_user.iter().skip(1).enumerate() { - let ctx_for_this_row_depth = ctx_for_row_number[i].clone(); // no context was created for row 0 + // skip the first row as it requires no multiplications + // no context was created for the first row + for (i, (row, ctx)) in + zip(rows_for_user.iter().skip(1), ctx_for_row_number.into_iter()).enumerate() + { let record_id_for_this_row_depth = record_id_for_each_depth[i + 1]; // skip row 0 let capped_attribution_outputs = prev_row_inputs - .compute_row_with_previous(ctx_for_this_row_depth, record_id_for_this_row_depth, row) + .compute_row_with_previous(ctx, record_id_for_this_row_depth, row) .await?; output.push(capped_attribution_outputs); From 9ec0e779915d6f60aad29dfd3bc6726c23af8d22 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Thu, 5 Oct 2023 11:46:57 +0800 Subject: [PATCH 013/124] Adding test for aggregation alone --- src/protocol/prf_sharding/mod.rs | 174 +++++++++++++++++++++++++------ 1 file changed, 145 insertions(+), 29 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 0e0b55304..91fa0aef3 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -709,14 +709,21 @@ pub mod tests { use super::{attribution_and_capping, CappedAttributionOutputs, PrfShardedIpaInputRow}; use crate::{ ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf3Bit, Gf5Bit}, - protocol::prf_sharding::attribution_and_capping_and_aggregation, + protocol::{ + context::{Context, UpgradableContext, Validator}, + prf_sharding::{ + attribution_and_capping_and_aggregation, do_aggregation, + move_single_value_to_bucket, + }, + RecordId, + }, rand::Rng, secret_sharing::{ replicated::semi_honest::AdditiveShare as Replicated, BitDecomposed, IntoShares, SharedValue, }, test_executor::run, - test_fixture::{Reconstruct, Runner, TestWorld}, + test_fixture::{get_bits, Reconstruct, Runner, TestWorld}, }; struct PreShardedAndSortedOPRFTestInput { @@ -742,20 +749,30 @@ pub mod tests { } } - fn test_output( + fn test_output( attributed_breakdown_key: u128, capped_attributed_trigger_value: u128, - ) -> PreAggregationTestOutput { + ) -> PreAggregationTestOutput + where + BK: GaloisField, + TV: GaloisField, + { PreAggregationTestOutput { - attributed_breakdown_key, - capped_attributed_trigger_value, + attributed_breakdown_key: get_bits::( + attributed_breakdown_key.try_into().unwrap(), + BK::BITS, + ), + capped_attributed_trigger_value: get_bits::( + capped_attributed_trigger_value.try_into().unwrap(), + TV::BITS, + ), } } #[derive(Debug, PartialEq)] struct PreAggregationTestOutput { - attributed_breakdown_key: u128, - capped_attributed_trigger_value: u128, + attributed_breakdown_key: BitDecomposed, + capped_attributed_trigger_value: BitDecomposed, } impl IntoShares> for PreShardedAndSortedOPRFTestInput @@ -799,6 +816,40 @@ pub mod tests { } } + impl IntoShares for PreAggregationTestOutput { + fn share_with(self, rng: &mut R) -> [CappedAttributionOutputs; 3] { + let PreAggregationTestOutput { + attributed_breakdown_key, + capped_attributed_trigger_value, + } = self; + + let did_trigger_get_attributed = Gf2::ONE; + let [attributed_breakdown_key0, attributed_breakdown_key1, attributed_breakdown_key2] = + attributed_breakdown_key.share_with(rng); + let [capped_attributed_trigger_value0, capped_attributed_trigger_value1, capped_attributed_trigger_value2] = + capped_attributed_trigger_value.share_with(rng); + let [did_trigger_get_attributed0, did_trigger_get_attributed1, did_trigger_get_attributed2] = + did_trigger_get_attributed.share_with(rng); + + [ + CappedAttributionOutputs { + did_trigger_get_attributed: did_trigger_get_attributed0, + attributed_breakdown_key_bits: attributed_breakdown_key0, + capped_attributed_trigger_value: capped_attributed_trigger_value0, + }, + CappedAttributionOutputs { + did_trigger_get_attributed: did_trigger_get_attributed1, + attributed_breakdown_key_bits: attributed_breakdown_key1, + capped_attributed_trigger_value: capped_attributed_trigger_value1, + }, + CappedAttributionOutputs { + did_trigger_get_attributed: did_trigger_get_attributed2, + attributed_breakdown_key_bits: attributed_breakdown_key2, + capped_attributed_trigger_value: capped_attributed_trigger_value2, + }, + ] + } + } impl Reconstruct for [&CappedAttributionOutputs; 3] { fn reconstruct(&self) -> PreAggregationTestOutput { let [s0, s1, s2] = self; @@ -820,16 +871,8 @@ pub mod tests { ); PreAggregationTestOutput { - attributed_breakdown_key: attributed_breakdown_key_bits - .iter() - .map(Field::as_u128) - .enumerate() - .fold(0_u128, |acc, (i, x)| acc + (x << i)), - capped_attributed_trigger_value: capped_attributed_trigger_value_bits - .iter() - .map(Field::as_u128) - .enumerate() - .fold(0_u128, |acc, (i, x)| acc + (x << i)), + attributed_breakdown_key: attributed_breakdown_key_bits, + capped_attributed_trigger_value: capped_attributed_trigger_value_bits, } } } @@ -860,17 +903,17 @@ pub mod tests { ]; let expected: [PreAggregationTestOutput; 11] = [ - test_output(17, 7), - test_output(20, 0), - test_output(20, 3), - test_output(12, 5), - test_output(20, 7), - test_output(18, 0), - test_output(12, 0), - test_output(12, 7), - test_output(12, 7), - test_output(12, 7), - test_output(12, 4), + test_output::(17, 7), + test_output::(20, 0), + test_output::(20, 3), + test_output::(12, 5), + test_output::(20, 7), + test_output::(18, 0), + test_output::(12, 0), + test_output::(12, 7), + test_output::(12, 7), + test_output::(12, 7), + test_output::(12, 4), ]; let num_saturating_bits: usize = 5; @@ -940,4 +983,77 @@ pub mod tests { assert_eq!(result, &expected); }); } + + #[test] + fn semi_honest_aggregation() { + run(|| async move { + let world = TestWorld::default(); + + let records: Vec = vec![ + test_output::(17, 7), + test_output::(20, 0), + test_output::(20, 3), + test_output::(12, 5), + test_output::(20, 7), + test_output::(18, 0), + test_output::(12, 0), + test_output::(12, 7), + test_output::(12, 7), + test_output::(12, 7), + test_output::(12, 4), + ]; + + let mut expected = [0_u128; 32]; + expected[12] = 30; + expected[17] = 7; + expected[20] = 10; + + let result: Vec<_> = world + .semi_honest(records.into_iter(), |ctx, input_rows| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + do_aggregation::<_, Gf5Bit, Gf3Bit, Fp32BitPrime, _>(ctx, input_rows) + .await + .unwrap() + }) + .await + .reconstruct(); + assert_eq!(result, &expected); + }); + } + + #[test] + fn semi_honest_move_value_to_single_bucket() { + run(|| async move { + let world = TestWorld::default(); + + let breakdown_key = 5_u128; + let value = Fp32BitPrime::truncate_from(10_u128); + let mut expected = [0_u128; 8]; + expected[5] = 10; + + let result: Vec<_> = world + .semi_honest( + ( + get_bits::(breakdown_key.try_into().unwrap(), Gf3Bit::BITS), + value, + ), + |ctx, (breakdown_key_share, value_share)| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + move_single_value_to_bucket::( + ctx.set_total_records(1), + RecordId::from(0), + breakdown_key_share, + value_share, + ) + .await + .unwrap() + }, + ) + .await + .reconstruct(); + assert_eq!(result, &expected); + }); + } } From 06f6eb1725ab740c0de941fb8598955f2da4c9a2 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Thu, 5 Oct 2023 13:46:24 -0700 Subject: [PATCH 014/124] implementation of PRF eval protocol --- src/ff/curve_points.rs | 72 ++++++++++++++++++++++++++---- src/ff/ec_prime_field.rs | 79 ++++++++++++++++++++++++++++++++- src/protocol/prf_eval/eval.rs | 37 --------------- src/protocol/prf_eval/keygen.rs | 0 src/protocol/prf_eval/mod.rs | 67 +++++++++++++++++++++++++++- 5 files changed, 206 insertions(+), 49 deletions(-) delete mode 100644 src/protocol/prf_eval/eval.rs delete mode 100644 src/protocol/prf_eval/keygen.rs diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index eed23e5ec..fad671dd0 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -6,7 +6,7 @@ use hkdf::Hkdf; use crate::{ - ff::{Serializable,ec_prime_field::Fp25519}, + ff::{Serializable,ec_prime_field::Fp25519,Field}, secret_sharing::{Block, SharedValue}, }; @@ -93,8 +93,9 @@ fn sub_assign(&mut self, rhs: Self) { ///Scalar Multiplication ///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a impl RP25519 { + pub const ONE: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); -fn s_mul(self, rhs: Fp25519) -> RP25519 { +pub fn s_mul(self, rhs: Fp25519) -> RP25519 { RP25519((self.0.decompress().unwrap() * Scalar::from(rhs)).compress()) } } @@ -129,6 +130,18 @@ impl From for RP25519 { } } +impl From for RP25519 { + fn from(s: CompressedRistretto) -> Self { + RP25519(s) + } +} + +impl From for CompressedRistretto { + fn from(s: RP25519) -> Self { + s.0 + } +} + macro_rules! cp_hash_impl { ( $u_type:ty, $byte_size:literal) => { impl From for $u_type { @@ -140,14 +153,19 @@ macro_rules! cp_hash_impl { <$u_type>::from_le_bytes(okm) } } + + impl From<$u_type> for RP25519 { + fn from(s: $u_type) -> Self { + let hk = Hkdf::::new(None, &s.to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + RP25519::deserialize(&okm.into()) + } + } } } -cp_hash_impl!( - u128, - 16 -); - cp_hash_impl!( u64, 8 @@ -158,6 +176,45 @@ cp_hash_impl!( 4 ); +/// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is +/// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed +impl Field for RP25519 { + const ONE: RP25519= RP25519::ONE; + + ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 + /// from or into u128. However it is sufficient to generate random elements in Fp25519 + fn as_u128(&self) -> u128 { + let hk = Hkdf::::new(None, self.0.as_bytes()); + let mut okm = [0u8; 16]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + u128::from_le_bytes(okm) + } + + ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + fn truncate_from>(v: T) -> Self { + let hk = Hkdf::::new(None, &v.into().to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + RP25519::deserialize(&okm.into()) + } + +} + +impl TryFrom for RP25519 { + type Error = crate::error::Error; + + fn try_from(v: u128) -> Result { + let mut bits = [0u8; 32]; + bits[..].copy_from_slice(&v.to_le_bytes()); + let f: RP25519=RP25519::ONE; + f.serialize((&mut bits).into()); + Ok(f) + } +} + + #[cfg(all(test, unit_test))] @@ -218,7 +275,6 @@ mod test { fn curve_point_to_hash() { let mut rng = thread_rng(); let a = rng.gen::(); - assert_ne!(0u128,u128::from(a)); assert_ne!(0u64,u64::from(a)); assert_ne!(0u32,u32::from(a)); } diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 3e4f762d8..2f88b07cc 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -2,9 +2,11 @@ use generic_array::GenericArray; use curve25519_dalek::scalar::Scalar; //use rand_core::RngCore; use typenum::U32; +use sha2::Sha256; +use hkdf::Hkdf; use crate::{ - ff::Serializable, + ff::{Serializable, Field}, secret_sharing::{Block, SharedValue}, }; @@ -17,7 +19,7 @@ pub struct Fp25519(::Storage); impl Fp25519 { - const ONE: Self = Self(Scalar::ONE); + pub const ONE: Self = Self(Scalar::ONE); //must not use with ZERO pub fn invert(&self) -> Fp25519 { @@ -127,6 +129,79 @@ impl From for Fp25519 { } } +macro_rules! sc_hash_impl { + ( $u_type:ty, $byte_size:literal) => { + impl From for $u_type { + fn from(s: Fp25519) -> Self { + let hk = Hkdf::::new(None, s.0.as_bytes()); + let mut okm = [0u8; $byte_size]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + <$u_type>::from_le_bytes(okm) + } + } + + impl From<$u_type> for Fp25519 { + fn from(s: $u_type) -> Self { + let hk = Hkdf::::new(None, &s.to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + Fp25519::deserialize(&okm.into()) + } + } + } +} + + +sc_hash_impl!( + u64, + 8 +); + +sc_hash_impl!( + u32, + 4 +); + + +/// Daniel had to implement this since PRSS wants it, prefer not to +impl Field for Fp25519 { + const ONE: Fp25519= Fp25519::ONE; + + ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 + /// from or into u128. However it is sufficient to generate random elements in Fp25519 + fn as_u128(&self) -> u128 { + let hk = Hkdf::::new(None, self.0.as_bytes()); + let mut okm = [0u8; 16]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + u128::from_le_bytes(okm) + } + + ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + fn truncate_from>(v: T) -> Self { + let hk = Hkdf::::new(None, &v.into().to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + Fp25519::deserialize(&okm.into()) + } + +} + +impl TryFrom for Fp25519 { + type Error = crate::error::Error; + + fn try_from(v: u128) -> Result { + let mut bits = [0u8; 32]; + bits[..].copy_from_slice(&v.to_le_bytes()); + let f: Fp25519=Fp25519::ONE; + f.serialize((&mut bits).into()); + Ok(f) + } +} + #[cfg(all(test, unit_test))] mod test { use generic_array::GenericArray; diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs deleted file mode 100644 index 0307f42b8..000000000 --- a/src/protocol/prf_eval/eval.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::{ - error::Error, - helpers::Direction, - ff::{ec_prime_field::Fp25519, curve_points::RP25519}, - protocol::{context::Context, step::BitOpStep, BasicProtocols, RecordId}, - secret_sharing::{Linear as LinearSecretSharing, LinearRefOps, SharedValue, replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, -}; - - -impl Into> for AdditiveShare { - fn into(self) -> AdditiveShare - { - AdditiveShare::new(RP25519::from(self.left()),RP25519::from(self.right())) - } -} - - -/// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) -/// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 -/// PRF key k is generated using keygen -/// In 3IPA, x is the match key -/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output -pub async fn eval_dy( - ctx: C, - record_id: RecordId, - k: &[S], - x: &[S], -) -> Result - where - C: Context, - S: LinearSecretSharing + BasicProtocols, - for<'a> &'a S: LinearRefOps<'a, S, Fp25519>, -{ - let role = ctx.role(); - - Ok(5u64) -} \ No newline at end of file diff --git a/src/protocol/prf_eval/keygen.rs b/src/protocol/prf_eval/keygen.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index 6183868b8..193cb360c 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -1,2 +1,65 @@ -mod eval; -mod keygen; \ No newline at end of file +use crate::{ + error::Error, + ff::{ec_prime_field::Fp25519, curve_points::RP25519}, + protocol::{context::Context, RecordId, prss::SharedRandomness, basics::Reveal, basics::SecureMul}, + secret_sharing::{replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, +}; + + +impl From> for AdditiveShare { + fn from(s: AdditiveShare) -> Self { + AdditiveShare::new(RP25519::from(s.left()),RP25519::from(s.right())) + } +} + +/// generates PRF key k as secret sharing over Fp25519 +pub fn gen_prf_key (ctx: C) -> AdditiveShare + where + C: Context, +{ + ctx.prss().generate_replicated(u128::MAX-100u128) +} + + +/// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) +/// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 +/// PRF key k is generated using keygen +/// In 3IPA, x is the match key +/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output +pub async fn eval_dy_prf( + ctx: C, + record_id: RecordId, + k: &AdditiveShare, + x: &AdditiveShare, +) -> Result + where + C: Context, +{ + // generate random shares using shared randomness, use index max/2 to not clash with multiply + let sh_r: AdditiveShare = ctx.prss().generate_replicated(u128::MAX/2+u128::from(record_id)); + + //compute (g^left, g^right) + let sh_gr = AdditiveShare::::from(sh_r.clone()); + + //compute x+k + let y =x+k; + + //compute y <- r*y + //Daniel: probably shouldn't use ctx anymore? why doesn't y need to be mutable? + y.multiply(&sh_r, ctx.clone(), record_id).await?; + + //reconstruct (z,R) + let mut gr: RP25519 = sh_gr.reveal(ctx, record_id).await?; + + //invert z + let mut z: Fp25519 = Fp25519::ONE; + z=z.invert(); + //compute R^z + gr=gr.s_mul(z); + Ok(u64::from(gr)) +} + +#[cfg(all(test, unit_test))] +mod test { + +} \ No newline at end of file From 346240966a00f57016bbe974b5b0addc559f3466 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Fri, 6 Oct 2023 10:28:18 +0800 Subject: [PATCH 015/124] issue with unpin --- .../prf_sharding/feature_label_dot_product.rs | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index 3fb35835d..32615d1cc 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -1,5 +1,5 @@ use futures::{stream::iter as stream_iter, TryStreamExt}; -use futures_util::{future::try_join, StreamExt}; +use futures_util::{future::try_join, stream, StreamExt}; use ipa_macros::Step; use std::iter::zip; @@ -235,24 +235,36 @@ where let binary_m_ctx = binary_validator.context(); let mut num_users_who_encountered_row_depth = vec![0_u32; histogram.len()]; let ctx_for_row_number = set_up_contexts(&binary_m_ctx, &histogram); - let mut futures = Vec::with_capacity(rows_chunked_by_user.len()); - for rows_for_user in rows_chunked_by_user { - let num_user_rows = rows_for_user.len(); - futures.push(evaluate_per_user_attribution_circuit( - ctx_for_row_number[..rows_for_user.len() - 1].to_owned(), - num_users_who_encountered_row_depth - .iter() - .take(num_user_rows) - .map(|x| RecordId(*x)) - .collect(), - rows_for_user, - )); - for i in 0..num_user_rows { - num_users_who_encountered_row_depth[i] += 1; - } - } + let stream_of_per_user_circuits = stream::unfold( + ( + num_users_who_encountered_row_depth, + ctx_for_row_number, + stream_iter(rows_chunked_by_user), + ), + |state| async move { + let (mut count_by_row_depth, contexts, s) = state; + if let Some(rows_for_user) = s.next().await { + let num_user_rows = rows_for_user.len(); + let yielded = evaluate_per_user_attribution_circuit( + contexts[..num_user_rows - 1].to_owned(), + num_users_who_encountered_row_depth + .iter() + .take(num_user_rows) + .map(|x| RecordId(*x)) + .collect(), + rows_for_user, + ); + for i in 0..num_user_rows { + count_by_row_depth[i] += 1; + } + Some((yielded, (count_by_row_depth, contexts, s))) + } else { + None + } + }, + ); - let flattenned_stream = seq_join(sh_ctx.active_work(), stream_iter(futures)) + let flattenned_stream = seq_join(sh_ctx.active_work(), stream_of_per_user_circuits) .map(|x| x.unwrap().into_iter()) .flatten_iters(); From 1d5cace41057c42ef60c217c85dbedb061d6bfcc Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Fri, 6 Oct 2023 15:24:42 +0800 Subject: [PATCH 016/124] Adding robust --- src/protocol/prf_sharding/mod.rs | 82 ++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 91fa0aef3..8beb04cfa 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -631,7 +631,8 @@ where let record_id: RecordId = RecordId::from(i); let bd_key = bk_bits.unwrap(); async move { - move_single_value_to_bucket::(ctx, record_id, bd_key, value).await + move_single_value_to_bucket::(ctx, record_id, bd_key, value, None) + .await } }); @@ -669,6 +670,7 @@ async fn move_single_value_to_bucket( record_id: RecordId, bd_key: BitDecomposed, value: S, + breakdown_count: Option, ) -> Result, Error> where BK: GaloisField, @@ -677,16 +679,21 @@ where F: PrimeField + ExtendableField, { let mut step: usize = 1 << BK::BITS; - let mut row_contribution = vec![value; 1 << BK::BITS]; + let total_breakdowns = match breakdown_count { + Some(x) => x, + None => 1 << BK::BITS, + }; + assert!(total_breakdowns <= 1 << BK::BITS); + let mut row_contribution = vec![value; total_breakdowns]; for (tree_depth, bit_of_bdkey) in bd_key.iter().rev().enumerate() { let depth_c = ctx.narrow(&BinaryTreeDepthStep::from(tree_depth)); let span = step >> 1; - let mut futures = Vec::with_capacity((1 << BK::BITS) / step); - for i in (0..1 << BK::BITS).step_by(step) { + let mut futures = Vec::with_capacity(bd_key.len() / step); + for i in (0..total_breakdowns).step_by(step) { let bit_c = depth_c.narrow(&BitOpStep::from(i)); - if i + span < 1 << BK::BITS { + if i + span < total_breakdowns { futures.push(row_contribution[i].multiply(bit_of_bdkey, bit_c, record_id)); } } @@ -694,10 +701,11 @@ where for (index, bdbit_contribution) in contributions.into_iter().enumerate() { let left_index = index * step; - let right_index = left_index + span; - - row_contribution[left_index] -= &bdbit_contribution; - row_contribution[right_index] = bdbit_contribution; + if left_index + span < total_breakdowns { + let right_index = left_index + span; + row_contribution[left_index] -= &bdbit_contribution; + row_contribution[right_index] = bdbit_contribution; + } } step = span; } @@ -706,9 +714,11 @@ where #[cfg(all(test, unit_test))] pub mod tests { + use rand::thread_rng; + use super::{attribution_and_capping, CappedAttributionOutputs, PrfShardedIpaInputRow}; use crate::{ - ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf3Bit, Gf5Bit}, + ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf3Bit, Gf5Bit, Gf8Bit}, protocol::{ context::{Context, UpgradableContext, Validator}, prf_sharding::{ @@ -1026,26 +1036,68 @@ pub mod tests { fn semi_honest_move_value_to_single_bucket() { run(|| async move { let world = TestWorld::default(); + let mut rng: rand::rngs::ThreadRng = thread_rng(); + + let breakdown_key = rng.gen_range(0..32); + + let value = Fp32BitPrime::truncate_from(10_u128); + let mut expected = [Fp32BitPrime::truncate_from(0_u128); 32]; + expected[breakdown_key] = value; + + let result: Vec<_> = world + .semi_honest( + ( + get_bits::(breakdown_key.try_into().unwrap(), Gf5Bit::BITS), + value, + ), + |ctx, (breakdown_key_share, value_share)| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + move_single_value_to_bucket::( + ctx.set_total_records(1), + RecordId::from(0), + breakdown_key_share, + value_share, + None, + ) + .await + .unwrap() + }, + ) + .await + .reconstruct(); + assert_eq!(result, &expected); + }); + } + + #[test] + fn semi_honest_move_value_to_single_bucket_not_power_two() { + const BREAKDOWN_COUNT: usize = 50; + + run(|| async move { + let world = TestWorld::default(); + let mut rng: rand::rngs::ThreadRng = thread_rng(); - let breakdown_key = 5_u128; + let breakdown_key = rng.gen_range(0..BREAKDOWN_COUNT); let value = Fp32BitPrime::truncate_from(10_u128); - let mut expected = [0_u128; 8]; - expected[5] = 10; + let mut expected = [Fp32BitPrime::truncate_from(0_u128); BREAKDOWN_COUNT]; + expected[breakdown_key] = value; let result: Vec<_> = world .semi_honest( ( - get_bits::(breakdown_key.try_into().unwrap(), Gf3Bit::BITS), + get_bits::(breakdown_key.try_into().unwrap(), Gf8Bit::BITS), value, ), |ctx, (breakdown_key_share, value_share)| async move { let validator = ctx.validator(); let ctx = validator.context(); - move_single_value_to_bucket::( + move_single_value_to_bucket::( ctx.set_total_records(1), RecordId::from(0), breakdown_key_share, value_share, + Some(BREAKDOWN_COUNT), ) .await .unwrap() From de5f32c87e5c7d4ef5e89c6c8f256819e77f190c Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 6 Oct 2023 13:47:06 -0700 Subject: [PATCH 017/124] prf eval test passes --- src/protocol/prf_eval/mod.rs | 134 +++++++++++++++++++++++++++++++---- 1 file changed, 119 insertions(+), 15 deletions(-) diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index 193cb360c..e3d0cd6a4 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -3,7 +3,31 @@ use crate::{ ff::{ec_prime_field::Fp25519, curve_points::RP25519}, protocol::{context::Context, RecordId, prss::SharedRandomness, basics::Reveal, basics::SecureMul}, secret_sharing::{replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, + seq_join::seq_try_join_all, }; +use ipa_macros::Step; + +#[derive(Step)] +pub(crate) enum Step { + PRFKeyGen, + GenRandomMask, + MultMaskWithPRFInput, + RevealR, + Revealz, +} + +pub async fn compute_match_key_pseudonym( + sh_ctx: C, + prf_key: AdditiveShare, + input_match_keys: Vec>, +) -> Result, Error> + where + C: Context, +{ + let ctx =sh_ctx.set_total_records(input_match_keys.len()); + let futures=input_match_keys.iter().enumerate().map(|(i,x)|eval_dy_prf(ctx.clone(),i.into(),&prf_key,x)); + Ok(seq_try_join_all(sh_ctx.active_work(), futures).await?.iter().map(|&x|u64::from(x)).collect()) +} impl From> for AdditiveShare { @@ -17,7 +41,7 @@ pub fn gen_prf_key (ctx: C) -> AdditiveShare where C: Context, { - ctx.prss().generate_replicated(u128::MAX-100u128) + ctx.narrow(&Step::PRFKeyGen).prss().generate_replicated(u128::MAX-100u128) } @@ -26,7 +50,7 @@ pub fn gen_prf_key (ctx: C) -> AdditiveShare /// PRF key k is generated using keygen /// In 3IPA, x is the match key /// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output -pub async fn eval_dy_prf( +pub async fn eval_dy_prf( ctx: C, record_id: RecordId, k: &AdditiveShare, @@ -35,31 +59,111 @@ pub async fn eval_dy_prf( where C: Context, { - // generate random shares using shared randomness, use index max/2 to not clash with multiply - let sh_r: AdditiveShare = ctx.prss().generate_replicated(u128::MAX/2+u128::from(record_id)); + let sh_r: AdditiveShare = ctx.narrow(&Step::GenRandomMask).prss().generate_replicated(record_id); //compute (g^left, g^right) let sh_gr = AdditiveShare::::from(sh_r.clone()); //compute x+k - let y =x+k; + let mut y =x+k; //compute y <- r*y - //Daniel: probably shouldn't use ctx anymore? why doesn't y need to be mutable? - y.multiply(&sh_r, ctx.clone(), record_id).await?; + y = y.multiply(&sh_r, ctx.narrow(&Step::MultMaskWithPRFInput), record_id).await?; //reconstruct (z,R) - let mut gr: RP25519 = sh_gr.reveal(ctx, record_id).await?; - - //invert z - let mut z: Fp25519 = Fp25519::ONE; - z=z.invert(); - //compute R^z - gr=gr.s_mul(z); - Ok(u64::from(gr)) + let gr: RP25519 = sh_gr.reveal(ctx.narrow(&Step::RevealR), record_id).await?; + let z = y.reveal(ctx.narrow(&Step::Revealz), record_id).await?; + + + + //compute R^(1/z) to u64 + Ok(u64::from(gr.s_mul(z.invert()))) } #[cfg(all(test, unit_test))] mod test { + use rand::Rng; + use crate::{ + test_executor::run, + test_fixture::{TestWorld, Reconstruct, Runner}, + ff::{ec_prime_field::Fp25519, curve_points::RP25519}, + secret_sharing::{IntoShares,replicated::semi_honest::AdditiveShare}, + protocol::prf_eval::{compute_match_key_pseudonym}, + }; + + #[derive(Copy, Clone)] + struct ShuffledTestInput { + match_key: Fp25519, + } + + #[derive(Debug, PartialEq)] + struct TestOutput { + match_key_pseudonym: u64, + } + + fn test_input( mk: u64) -> ShuffledTestInput { + ShuffledTestInput { + match_key: Fp25519::from(mk), + } + } + + impl IntoShares> for ShuffledTestInput + { + fn share_with(self, rng: &mut R) -> [AdditiveShare; 3] { + self.match_key.share_with(rng) + } + } + + + impl Reconstruct for [&u64; 3] { + fn reconstruct(&self) -> TestOutput { + TestOutput{ + match_key_pseudonym: if *self[0]==*self[1] && *self[0]==*self[2] {*self[0]} else {0u64}, + } + } + } + + + #[test] + fn semi_honest() { + run(|| async move { + let world = TestWorld::default(); + + //first two need to be identical for test to succeed + let records: Vec = vec![ + test_input(3), + test_input(3), + test_input(23443524523), + test_input(56), + test_input(895764542), + test_input(456764576), + test_input(56), + test_input(3), + test_input(56), + test_input(23443524523), + ]; + + //PRF Key Gen + let u = 3216412445u64; + let k:Fp25519 = Fp25519::from(u); + + let expected: Vec=records.iter().map(|&x| TestOutput{match_key_pseudonym: (RP25519::from((x.match_key+k).invert())).into()} ).collect(); + + let result: Vec<_> = world + .semi_honest((records.into_iter(),k), |ctx, (input_match_keys,prf_key)| async move { + compute_match_key_pseudonym::<_>( + ctx, + prf_key, + input_match_keys, + ) + .await + .unwrap() + }) + .await + .reconstruct(); + assert_eq!(result, expected); + assert_eq!(result[0], result[1]); + }); + } } \ No newline at end of file From 7a4c052208dcb51abb1064ce6f99e831a6c65b2b Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Fri, 6 Oct 2023 20:19:53 -0700 Subject: [PATCH 018/124] Relax semi-honest reveal bounds It shouldn't require fields, semi-honest reveal works with shared values just fine. --- src/protocol/basics/reveal.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/protocol/basics/reveal.rs b/src/protocol/basics/reveal.rs index 28e54a904..672c8a473 100644 --- a/src/protocol/basics/reveal.rs +++ b/src/protocol/basics/reveal.rs @@ -18,7 +18,7 @@ use crate::{ malicious::{AdditiveShare as MaliciousReplicated, ExtendableField}, semi_honest::AdditiveShare as Replicated, }, - SecretSharing, + SecretSharing, SharedValue, }, }; @@ -47,10 +47,10 @@ pub trait Reveal: Sized { /// i.e. their own shares and received share. #[async_trait] #[embed_doc_image("reveal", "images/reveal.png")] -impl Reveal for Replicated { - type Output = F; +impl Reveal for Replicated { + type Output = V; - async fn reveal<'fut>(&self, ctx: C, record_id: RecordId) -> Result + async fn reveal<'fut>(&self, ctx: C, record_id: RecordId) -> Result where C: 'fut, { From c3c6d96c4d72074a205baf739ab022c7e8ded463 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Mon, 9 Oct 2023 14:30:17 +0800 Subject: [PATCH 019/124] It finally works now --- .../prf_sharding/feature_label_dot_product.rs | 133 +++++++----------- 1 file changed, 50 insertions(+), 83 deletions(-) diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index 32615d1cc..4f313e8e9 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -1,8 +1,8 @@ use futures::{stream::iter as stream_iter, TryStreamExt}; -use futures_util::{future::try_join, stream, StreamExt}; +use futures_util::{future::try_join, stream::unfold, StreamExt}; use ipa_macros::Step; -use std::iter::zip; +use std::{iter::zip, pin::pin}; use stream_flatten_iters::StreamExt as _; @@ -141,19 +141,6 @@ pub(crate) enum Step { ModulusConvertFeatureVectorBits, } -fn compute_histogram_of_users_with_row_count(rows_chunked_by_user: &[Vec]) -> Vec { - let mut output = vec![]; - for user_rows in rows_chunked_by_user { - for j in 0..user_rows.len() { - if j >= output.len() { - output.push(0); - } - output[j] += 1; - } - } - output -} - fn set_up_contexts(root_ctx: &C, histogram: &[usize]) -> Vec where C: UpgradedContext>, @@ -172,30 +159,6 @@ where context_per_row_depth } -fn chunk_rows_by_user( - input_rows: Vec>, -) -> Vec>> -where - FV: GaloisField, -{ - let mut rows_for_user: Vec> = vec![]; - - let mut rows_chunked_by_user = vec![]; - for row in input_rows { - if rows_for_user.is_empty() || row.prf_of_match_key == rows_for_user[0].prf_of_match_key { - rows_for_user.push(row); - } else { - rows_chunked_by_user.push(rows_for_user); - rows_for_user = vec![row]; - } - } - if !rows_for_user.is_empty() { - rows_chunked_by_user.push(rows_for_user); - } - - rows_chunked_by_user -} - /// Sub-protocol of the PRF-sharded IPA Protocol /// /// After the computation of the per-user PRF, addition of dummy records and shuffling, @@ -216,6 +179,7 @@ where pub async fn compute_feature_label_dot_product( sh_ctx: C, input_rows: Vec>, + histogram: &[usize], ) -> Result, Error> where C: UpgradableContext, @@ -227,42 +191,40 @@ where { assert!(FV::BITS > 0); - let mut num_outputs = input_rows.len(); - let rows_chunked_by_user = chunk_rows_by_user(input_rows); - num_outputs -= rows_chunked_by_user.len(); - let histogram = compute_histogram_of_users_with_row_count(&rows_chunked_by_user); + let num_outputs = input_rows.len() - histogram[0]; + let mut input_stream = stream_iter(input_rows); + let first_row = input_stream.next().await.unwrap(); + let rows_chunked_by_user = unfold(Some((input_stream, first_row)), |state| async move { + if state.is_none() { + return None; + } + let (mut s, last_row) = state.unwrap(); + let last_row_prf = last_row.prf_of_match_key; + let mut current_chunk = vec![last_row]; + while let Some(row) = s.next().await { + if row.prf_of_match_key == last_row_prf { + current_chunk.push(row); + } else { + return Some((current_chunk, Some((s, row)))); + } + } + Some((current_chunk, None)) + }); + let binary_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); let binary_m_ctx = binary_validator.context(); let mut num_users_who_encountered_row_depth = vec![0_u32; histogram.len()]; let ctx_for_row_number = set_up_contexts(&binary_m_ctx, &histogram); - let stream_of_per_user_circuits = stream::unfold( - ( - num_users_who_encountered_row_depth, - ctx_for_row_number, - stream_iter(rows_chunked_by_user), - ), - |state| async move { - let (mut count_by_row_depth, contexts, s) = state; - if let Some(rows_for_user) = s.next().await { - let num_user_rows = rows_for_user.len(); - let yielded = evaluate_per_user_attribution_circuit( - contexts[..num_user_rows - 1].to_owned(), - num_users_who_encountered_row_depth - .iter() - .take(num_user_rows) - .map(|x| RecordId(*x)) - .collect(), - rows_for_user, - ); - for i in 0..num_user_rows { - count_by_row_depth[i] += 1; - } - Some((yielded, (count_by_row_depth, contexts, s))) - } else { - None - } - }, - ); + let stream_of_per_user_circuits = pin!(rows_chunked_by_user.then(|rows_for_user| { + let num_user_rows = rows_for_user.len(); + let contexts = ctx_for_row_number[..num_user_rows - 1].to_owned(); + let record_ids = num_users_who_encountered_row_depth[..num_user_rows].to_owned(); + + for i in 0..rows_for_user.len() { + num_users_who_encountered_row_depth[i] += 1; + } + async move { evaluate_per_user_attribution_circuit(contexts, record_ids, rows_for_user) } + })); let flattenned_stream = seq_join(sh_ctx.active_work(), stream_of_per_user_circuits) .map(|x| x.unwrap().into_iter()) @@ -295,7 +257,7 @@ where async fn evaluate_per_user_attribution_circuit( ctx_for_row_number: Vec, - record_id_for_each_depth: Vec, + record_id_for_each_depth: Vec, rows_for_user: Vec>, ) -> Result>>, Error> where @@ -315,7 +277,7 @@ where for (i, (row, ctx)) in zip(rows_for_user.iter().skip(1), ctx_for_row_number.into_iter()).enumerate() { - let record_id_for_this_row_depth = record_id_for_each_depth[i + 1]; // skip row 0 + let record_id_for_this_row_depth = RecordId(record_id_for_each_depth[i + 1]); // skip row 0 let capped_attribution_outputs = prev_row_inputs .compute_row_with_previous(ctx, record_id_for_this_row_depth, row) @@ -468,16 +430,21 @@ pub mod tests { ]; expected.reverse(); // convert to little-endian order - let result: Vec<_> = world - .semi_honest(records.into_iter(), |ctx, input_rows| async move { - compute_feature_label_dot_product::< - _, - Gf32Bit, - Fp32BitPrime, - Replicated, - >(ctx, input_rows) - .await - .unwrap() + let histogram = vec![3, 3, 2, 2, 1, 1, 1, 1]; + + let result: Vec = world + .semi_honest(records.into_iter(), |ctx, input_rows| { + let h = histogram.as_slice(); + async move { + compute_feature_label_dot_product::< + _, + Gf32Bit, + Fp32BitPrime, + Replicated, + >(ctx, input_rows, h) + .await + .unwrap() + } }) .await .reconstruct(); From 9f6b0946b93d2782126680a5c88de96d8ef4a317 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Mon, 9 Oct 2023 15:16:30 +0800 Subject: [PATCH 020/124] Cleaned up and converted everything to streams --- .../prf_sharding/feature_label_dot_product.rs | 101 ++++++++++++------ 1 file changed, 71 insertions(+), 30 deletions(-) diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index 4f313e8e9..d35cc6951 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -1,5 +1,5 @@ use futures::{stream::iter as stream_iter, TryStreamExt}; -use futures_util::{future::try_join, stream::unfold, StreamExt}; +use futures_util::{future::try_join, stream::unfold, Stream, StreamExt}; use ipa_macros::Step; use std::{iter::zip, pin::pin}; @@ -159,6 +159,34 @@ where context_per_row_depth } +/// +/// Takes an input stream of `PrfShardedIpaInputRecordRow` which is assumed to have all records with a given PRF adjacent +/// and converts it into a stream of vectors of `PrfShardedIpaInputRecordRow` having the same PRF. +/// +fn chunk_rows_by_user( + input_stream: IS, + first_row: PrfShardedIpaInputRow, +) -> impl Stream>> +where + FV: GaloisField, + IS: Stream> + Unpin, +{ + unfold(Some((input_stream, first_row)), |state| async move { + state.as_ref()?; + let (mut s, last_row) = state.unwrap(); + let last_row_prf = last_row.prf_of_match_key; + let mut current_chunk = vec![last_row]; + while let Some(row) = s.next().await { + if row.prf_of_match_key == last_row_prf { + current_chunk.push(row); + } else { + return Some((current_chunk, Some((s, row)))); + } + } + Some((current_chunk, None)) + }) +} + /// Sub-protocol of the PRF-sharded IPA Protocol /// /// After the computation of the per-user PRF, addition of dummy records and shuffling, @@ -166,16 +194,35 @@ where /// device can be processed together. /// /// This circuit expects to receive records from multiple users, -/// but with all of the records from a given user adjacent to one another, and in time order. +/// but with all of the records from a given user adjacent to one another, and in reverse time order (most recent event comes first). /// /// This circuit will compute attribution, and per-user capping. /// -/// The output of this circuit is the input to the next stage: Aggregation. +/// After those steps, source events to which trigger events were attributed will contribute their feature vectors to an aggregate +/// +/// The aggregate is just the sum of all the feature vectors of source events which received attribution +/// +/// This is useful for performing logistic regression: `https://github.com/patcg-individual-drafts/ipa/blob/main/logistic_regression.md` +/// +/// Due to limitation in our infra, it's necessary to set the total number of records each channel will ever need to process. +/// The number of records each channel processes is a function of the distribution of number of records per user. +/// Rather than calculate this histogram within this function (challenging to do while streaming), at present the caller must pass this in. +/// +/// The count at a given index indicates the number of users having at least that many rows of data. +/// +/// Example: +/// If the input is from 3 users, +/// - the first having 2 records +/// - the second having 4 records +/// - the third having 6 records +/// Then the histogram that should be provided is: +/// - [3, 3, 2, 2, 1, 1] /// /// # Errors /// Propagates errors from multiplications /// # Panics /// Propagates errors from multiplications +#[allow(clippy::async_yields_async)] pub async fn compute_feature_label_dot_product( sh_ctx: C, input_rows: Vec>, @@ -191,49 +238,42 @@ where { assert!(FV::BITS > 0); + // Get the validator and context to use for Gf2 multiplication operations + let binary_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); + let binary_m_ctx = binary_validator.context(); + + // Get the validator and context to use for `Z_p` operations (modulus conversion) + let prime_field_validator = sh_ctx.narrow(&Step::PrimeFieldValidator).validator::(); + let prime_field_ctx = prime_field_validator.context(); + + // Tricky hacks to work around the limitations of our current infrastructure let num_outputs = input_rows.len() - histogram[0]; + let mut record_id_for_row_depth = vec![0_u32; histogram.len()]; + let ctx_for_row_number = set_up_contexts(&binary_m_ctx, histogram); + + // Chunk the incoming stream of records into stream of vectors of records with the same PRF let mut input_stream = stream_iter(input_rows); let first_row = input_stream.next().await.unwrap(); - let rows_chunked_by_user = unfold(Some((input_stream, first_row)), |state| async move { - if state.is_none() { - return None; - } - let (mut s, last_row) = state.unwrap(); - let last_row_prf = last_row.prf_of_match_key; - let mut current_chunk = vec![last_row]; - while let Some(row) = s.next().await { - if row.prf_of_match_key == last_row_prf { - current_chunk.push(row); - } else { - return Some((current_chunk, Some((s, row)))); - } - } - Some((current_chunk, None)) - }); + let rows_chunked_by_user = chunk_rows_by_user(input_stream, first_row); - let binary_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); - let binary_m_ctx = binary_validator.context(); - let mut num_users_who_encountered_row_depth = vec![0_u32; histogram.len()]; - let ctx_for_row_number = set_up_contexts(&binary_m_ctx, &histogram); + // Convert to a stream of async futures that represent the result of executing the per-user circuit let stream_of_per_user_circuits = pin!(rows_chunked_by_user.then(|rows_for_user| { let num_user_rows = rows_for_user.len(); let contexts = ctx_for_row_number[..num_user_rows - 1].to_owned(); - let record_ids = num_users_who_encountered_row_depth[..num_user_rows].to_owned(); + let record_ids = record_id_for_row_depth[..num_user_rows].to_owned(); - for i in 0..rows_for_user.len() { - num_users_who_encountered_row_depth[i] += 1; + for count in record_id_for_row_depth.iter_mut().take(rows_for_user.len()) { + *count += 1; } async move { evaluate_per_user_attribution_circuit(contexts, record_ids, rows_for_user) } })); + // Execute all of the async futures (sequentially), and flatten the result let flattenned_stream = seq_join(sh_ctx.active_work(), stream_of_per_user_circuits) .map(|x| x.unwrap().into_iter()) .flatten_iters(); - let prime_field_validator = sh_ctx.narrow(&Step::PrimeFieldValidator).validator::(); - let prime_field_ctx = prime_field_validator.context(); - - // modulus convert feature vector bits + // modulus convert feature vector bits from shares in `Z_2` to shares in `Z_p` let converted_feature_vector_bits = convert_bits( prime_field_ctx .narrow(&Step::ModulusConvertFeatureVectorBits) @@ -242,6 +282,7 @@ where 0..FV::BITS, ); + // Sum up all the vectors converted_feature_vector_bits .try_fold( vec![S::ZERO; usize::try_from(FV::BITS).unwrap()], From 3ccef7bf308a1bfdf6d47385c0190eca5a5f2422 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Mon, 9 Oct 2023 15:47:20 +0800 Subject: [PATCH 021/124] clippy imports thing --- src/protocol/prf_sharding/feature_label_dot_product.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index d35cc6951..a326c02ef 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -1,9 +1,8 @@ +use std::{iter::zip, pin::pin}; + use futures::{stream::iter as stream_iter, TryStreamExt}; use futures_util::{future::try_join, stream::unfold, Stream, StreamExt}; use ipa_macros::Step; - -use std::{iter::zip, pin::pin}; - use stream_flatten_iters::StreamExt as _; use crate::{ From 7fd070c0f45beb8a9b3325534d1f6ae0cc1e5144 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Mon, 9 Oct 2023 16:59:19 +0800 Subject: [PATCH 022/124] Config out the compact gate stuff --- src/protocol/prf_sharding/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index fdd3a6440..4c950251c 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -28,6 +28,7 @@ use crate::{ seq_join::{seq_join, seq_try_join_all}, }; +#[cfg(feature = "descriptive-gate")] pub mod feature_label_dot_product; pub struct PrfShardedIpaInputRow { From ea9debb35dc709806f26a05da97725f8d0c6695c Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Mon, 9 Oct 2023 16:13:38 +0800 Subject: [PATCH 023/124] Incorporate feedback --- src/protocol/prf_sharding/mod.rs | 145 ++++++++++++------------------- 1 file changed, 55 insertions(+), 90 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 8beb04cfa..eed74365b 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -631,8 +631,14 @@ where let record_id: RecordId = RecordId::from(i); let bd_key = bk_bits.unwrap(); async move { - move_single_value_to_bucket::(ctx, record_id, bd_key, value, None) - .await + move_single_value_to_bucket::( + ctx, + record_id, + bd_key, + value, + 1 << BK::BITS, + ) + .await } }); @@ -670,7 +676,7 @@ async fn move_single_value_to_bucket( record_id: RecordId, bd_key: BitDecomposed, value: S, - breakdown_count: Option, + breakdown_count: usize, ) -> Result, Error> where BK: GaloisField, @@ -679,21 +685,18 @@ where F: PrimeField + ExtendableField, { let mut step: usize = 1 << BK::BITS; - let total_breakdowns = match breakdown_count { - Some(x) => x, - None => 1 << BK::BITS, - }; - assert!(total_breakdowns <= 1 << BK::BITS); - let mut row_contribution = vec![value; total_breakdowns]; + + assert!(breakdown_count <= 1 << BK::BITS); + let mut row_contribution = vec![value; breakdown_count]; for (tree_depth, bit_of_bdkey) in bd_key.iter().rev().enumerate() { let depth_c = ctx.narrow(&BinaryTreeDepthStep::from(tree_depth)); let span = step >> 1; - let mut futures = Vec::with_capacity(bd_key.len() / step); - for i in (0..total_breakdowns).step_by(step) { + let mut futures = Vec::with_capacity(breakdown_count / step); + for i in (0..breakdown_count).step_by(step) { let bit_c = depth_c.narrow(&BitOpStep::from(i)); - if i + span < total_breakdowns { + if i + span < breakdown_count { futures.push(row_contribution[i].multiply(bit_of_bdkey, bit_c, record_id)); } } @@ -701,11 +704,9 @@ where for (index, bdbit_contribution) in contributions.into_iter().enumerate() { let left_index = index * step; - if left_index + span < total_breakdowns { - let right_index = left_index + span; - row_contribution[left_index] -= &bdbit_contribution; - row_contribution[right_index] = bdbit_contribution; - } + let right_index = left_index + span; + row_contribution[left_index] -= &bdbit_contribution; + row_contribution[right_index] = bdbit_contribution; } step = span; } @@ -1034,78 +1035,42 @@ pub mod tests { #[test] fn semi_honest_move_value_to_single_bucket() { - run(|| async move { - let world = TestWorld::default(); - let mut rng: rand::rngs::ThreadRng = thread_rng(); - - let breakdown_key = rng.gen_range(0..32); - - let value = Fp32BitPrime::truncate_from(10_u128); - let mut expected = [Fp32BitPrime::truncate_from(0_u128); 32]; - expected[breakdown_key] = value; - - let result: Vec<_> = world - .semi_honest( - ( - get_bits::(breakdown_key.try_into().unwrap(), Gf5Bit::BITS), - value, - ), - |ctx, (breakdown_key_share, value_share)| async move { - let validator = ctx.validator(); - let ctx = validator.context(); - move_single_value_to_bucket::( - ctx.set_total_records(1), - RecordId::from(0), - breakdown_key_share, - value_share, - None, - ) - .await - .unwrap() - }, - ) - .await - .reconstruct(); - assert_eq!(result, &expected); - }); - } - - #[test] - fn semi_honest_move_value_to_single_bucket_not_power_two() { - const BREAKDOWN_COUNT: usize = 50; - - run(|| async move { - let world = TestWorld::default(); - let mut rng: rand::rngs::ThreadRng = thread_rng(); - - let breakdown_key = rng.gen_range(0..BREAKDOWN_COUNT); - let value = Fp32BitPrime::truncate_from(10_u128); - let mut expected = [Fp32BitPrime::truncate_from(0_u128); BREAKDOWN_COUNT]; - expected[breakdown_key] = value; - - let result: Vec<_> = world - .semi_honest( - ( - get_bits::(breakdown_key.try_into().unwrap(), Gf8Bit::BITS), - value, - ), - |ctx, (breakdown_key_share, value_share)| async move { - let validator = ctx.validator(); - let ctx = validator.context(); - move_single_value_to_bucket::( - ctx.set_total_records(1), - RecordId::from(0), - breakdown_key_share, - value_share, - Some(BREAKDOWN_COUNT), - ) - .await - .unwrap() - }, - ) - .await - .reconstruct(); - assert_eq!(result, &expected); - }); + const MAX_BREAKDOWN_COUNT: usize = 63; + for _ in 1..10 { + run(|| async move { + let world = TestWorld::default(); + let mut rng: rand::rngs::ThreadRng = thread_rng(); + let count = rng.gen_range(1..MAX_BREAKDOWN_COUNT); + let breakdown_key = rng.gen_range(0..count); + + let value = Fp32BitPrime::truncate_from(10_u128); + let mut expected = vec![Fp32BitPrime::truncate_from(0_u128); count]; + expected[breakdown_key] = value; + + let breakdown_key_bits = + get_bits::(breakdown_key.try_into().unwrap(), Gf8Bit::BITS); + + let result: Vec<_> = world + .semi_honest( + (breakdown_key_bits, value), + |ctx, (breakdown_key_share, value_share)| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + move_single_value_to_bucket::( + ctx.set_total_records(1), + RecordId::from(0), + breakdown_key_share, + value_share, + count, + ) + .await + .unwrap() + }, + ) + .await + .reconstruct(); + assert_eq!(result, expected); + }); + } } } From 552cc6210e992ed45df895aa0dc5f25718632913 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 9 Oct 2023 15:03:41 -0700 Subject: [PATCH 024/124] fixed clippy complaints, panics when setting invalid curve points or multiply curve points --- src/error.rs | 2 ++ src/ff/curve_points.rs | 47 +++++++++++++++++++++--------------- src/ff/ec_prime_field.rs | 3 ++- src/protocol/prf_eval/mod.rs | 23 +++++++++++------- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/error.rs b/src/error.rs index c68b325a8..7d6456c7f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -58,6 +58,8 @@ pub enum Error { InvalidReport(#[from] InvalidReportError), #[error("unsupported: {0}")] Unsupported(String), + #[error("Decompressing invalid elliptic curve point")] + DecompressingInvalidCurvePoint, } impl Default for Error { diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index fad671dd0..d04eec535 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -5,7 +5,9 @@ use sha2::Sha256; use hkdf::Hkdf; + use crate::{ + error::Error, ff::{Serializable,ec_prime_field::Fp25519,Field}, secret_sharing::{Block, SharedValue}, }; @@ -91,12 +93,17 @@ fn sub_assign(&mut self, rhs: Self) { ///Scalar Multiplication -///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a +///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a impl RP25519 { pub const ONE: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); -pub fn s_mul(self, rhs: Fp25519) -> RP25519 { - RP25519((self.0.decompress().unwrap() * Scalar::from(rhs)).compress()) + /// # Errors + /// Propagates errors from decompressing invalid curve point +pub fn s_mul(self, rhs: Fp25519) -> Result { + self.0.decompress().map_or( + Err(Error::DecompressingInvalidCurvePoint), + |x| Ok((x * Scalar::from(rhs)).compress().into()) + ) } } @@ -191,7 +198,7 @@ impl Field for RP25519 { u128::from_le_bytes(okm) } - ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + ///PRSS uses `truncate_from function`, we need to expand the u128 using a PRG (Sha256) to a [u8;32] fn truncate_from>(v: T) -> Self { let hk = Hkdf::::new(None, &v.into().to_le_bytes()); let mut okm = [0u8; 32]; @@ -247,7 +254,7 @@ mod test { #[test] fn scalar_to_point() { let a = Scalar::ONE; - let b : RP25519 = a.clone().into(); + let b : RP25519 = a.into(); let d : Fp25519 = a.into(); let c : RP25519 = RP25519::from(d); assert_eq!(b,RP25519::ZERO); @@ -257,26 +264,26 @@ mod test { #[test] fn curve_arithmetics() { let mut rng = thread_rng(); - let a = rng.gen::(); - let b = rng.gen::(); - let c = a+b; - let d = RP25519::from(a)+RP25519::from(b); - assert_eq!(d, RP25519::from(c)); - assert_ne!(d, RP25519::ZERO); - let e = rng.gen::(); - let f=rng.gen::(); - let g =e*f; - let h = RP25519::from(e).s_mul(f); - assert_eq!(h,RP25519::from(g)); - assert_ne!(h, RP25519::ZERO); + let fp_a = rng.gen::(); + let fp_b = rng.gen::(); + let fp_c = fp_a+fp_b; + let fp_d = RP25519::from(fp_a)+RP25519::from(fp_b); + assert_eq!(fp_d, RP25519::from(fp_c)); + assert_ne!(fp_d, RP25519::ZERO); + let fp_e = rng.gen::(); + let fp_f=rng.gen::(); + let fp_g =fp_e*fp_f; + let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); + assert_eq!(fp_h,RP25519::from(fp_g)); + assert_ne!(fp_h, RP25519::ZERO); } #[test] fn curve_point_to_hash() { let mut rng = thread_rng(); - let a = rng.gen::(); - assert_ne!(0u64,u64::from(a)); - assert_ne!(0u32,u32::from(a)); + let fp_a = rng.gen::(); + assert_ne!(0u64,u64::from(fp_a)); + assert_ne!(0u32,u32::from(fp_a)); } } \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 2f88b07cc..624f91c64 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -22,6 +22,7 @@ impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); //must not use with ZERO + #[must_use] pub fn invert(&self) -> Fp25519 { Fp25519(self.0.invert()) } @@ -179,7 +180,7 @@ impl Field for Fp25519 { u128::from_le_bytes(okm) } - ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + ///PRSS uses `truncate_from function`, we need to expand the u128 using a PRG (Sha256) to a [u8;32] fn truncate_from>(v: T) -> Self { let hk = Hkdf::::new(None, &v.into().to_le_bytes()); let mut okm = [0u8; 32]; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index e3d0cd6a4..895ad524b 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -16,6 +16,8 @@ pub(crate) enum Step { Revealz, } +/// # Errors +/// Propagates errors from multiplications pub async fn compute_match_key_pseudonym( sh_ctx: C, prf_key: AdditiveShare, @@ -26,7 +28,7 @@ pub async fn compute_match_key_pseudonym( { let ctx =sh_ctx.set_total_records(input_match_keys.len()); let futures=input_match_keys.iter().enumerate().map(|(i,x)|eval_dy_prf(ctx.clone(),i.into(),&prf_key,x)); - Ok(seq_try_join_all(sh_ctx.active_work(), futures).await?.iter().map(|&x|u64::from(x)).collect()) + seq_try_join_all(sh_ctx.active_work(), futures).await } @@ -37,7 +39,7 @@ impl From> for AdditiveShare { } /// generates PRF key k as secret sharing over Fp25519 -pub fn gen_prf_key (ctx: C) -> AdditiveShare +pub fn gen_prf_key (ctx: &C) -> AdditiveShare where C: Context, { @@ -49,7 +51,10 @@ pub fn gen_prf_key (ctx: C) -> AdditiveShare /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 /// PRF key k is generated using keygen /// In 3IPA, x is the match key -/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output +/// outputs a u64 as specified in `protocol/prf_sharding/mod.rs`, all parties learn the output +/// # Errors +/// Propagates errors from multiplications, reveal and scalar multiplication + pub async fn eval_dy_prf( ctx: C, record_id: RecordId, @@ -77,7 +82,7 @@ pub async fn eval_dy_prf( //compute R^(1/z) to u64 - Ok(u64::from(gr.s_mul(z.invert()))) + Ok(u64::from(gr.s_mul(z.invert())?)) } #[cfg(all(test, unit_test))] @@ -133,18 +138,18 @@ mod test { let records: Vec = vec![ test_input(3), test_input(3), - test_input(23443524523), + test_input(23_443_524_523), test_input(56), - test_input(895764542), - test_input(456764576), + test_input(895_764_542), + test_input(456_764_576), test_input(56), test_input(3), test_input(56), - test_input(23443524523), + test_input(23_443_524_523), ]; //PRF Key Gen - let u = 3216412445u64; + let u = 3_216_412_445u64; let k:Fp25519 = Fp25519::from(u); let expected: Vec=records.iter().map(|&x| TestOutput{match_key_pseudonym: (RP25519::from((x.match_key+k).invert())).into()} ).collect(); From fe3f1308e625ee9f8971da8eb0a4aff03d0d68d0 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 9 Oct 2023 15:49:42 -0700 Subject: [PATCH 025/124] remove comment --- src/ff/curve_points.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index d04eec535..b8d6a2882 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -20,8 +20,6 @@ impl Block for CompressedRistretto { #[derive(Clone, Copy, PartialEq, Debug)] pub struct RP25519(::Storage); -//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? - /// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 impl SharedValue for RP25519 { type Storage = CompressedRistretto; From 159cd2cf37618f02b8d18d3c7905cc600f24c916 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 9 Oct 2023 16:29:30 -0700 Subject: [PATCH 026/124] adding some of Martins suggestions --- src/ff/curve_points.rs | 159 ++++++++++++++----------------- src/ff/ec_prime_field.rs | 176 ++++++++++++++++------------------- src/ff/mod.rs | 4 +- src/protocol/prf_eval/mod.rs | 111 ++++++++++++---------- 4 files changed, 217 insertions(+), 233 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index b8d6a2882..2b1f60f94 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,14 +1,16 @@ +use curve25519_dalek::{ + constants, + ristretto::{CompressedRistretto, RistrettoPoint}, + Scalar, +}; use generic_array::GenericArray; -use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; -use typenum::U32; -use sha2::Sha256; use hkdf::Hkdf; - - +use sha2::Sha256; +use typenum::U32; use crate::{ error::Error, - ff::{Serializable,ec_prime_field::Fp25519,Field}, + ff::{ec_prime_field::Fp25519, Field, Serializable}, secret_sharing::{Block, SharedValue}, }; @@ -31,7 +33,7 @@ impl Serializable for RP25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()] ; + let raw = &self.0.as_bytes()[..buf.len()]; buf.copy_from_slice(raw); } @@ -40,7 +42,6 @@ impl Serializable for RP25519 { } } - impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> RP25519 { //Fp25519(Scalar::random(rng: &mut R)) @@ -50,62 +51,59 @@ impl rand::distributions::Distribution for rand::distributions::Standar } } - impl std::ops::Add for RP25519 { -type Output = Self; + type Output = Self; -fn add(self, rhs: Self) -> Self::Output { - Self((self.0.decompress().unwrap()+rhs.0.decompress().unwrap()).compress()) -} + fn add(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap() + rhs.0.decompress().unwrap()).compress()) + } } impl std::ops::AddAssign for RP25519 { -#[allow(clippy::assign_op_pattern)] -fn add_assign(&mut self, rhs: Self) { - *self = *self + rhs; -} + #[allow(clippy::assign_op_pattern)] + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } } impl std::ops::Neg for RP25519 { -type Output = Self; + type Output = Self; -fn neg(self) -> Self::Output { - Self(self.0.decompress().unwrap().neg().compress()) -} + fn neg(self) -> Self::Output { + Self(self.0.decompress().unwrap().neg().compress()) + } } impl std::ops::Sub for RP25519 { -type Output = Self; + type Output = Self; -fn sub(self, rhs: Self) -> Self::Output { - Self((self.0.decompress().unwrap()-rhs.0.decompress().unwrap()).compress()) -} + fn sub(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap() - rhs.0.decompress().unwrap()).compress()) + } } impl std::ops::SubAssign for RP25519 { -#[allow(clippy::assign_op_pattern)] -fn sub_assign(&mut self, rhs: Self) { - *self = *self - rhs; -} + #[allow(clippy::assign_op_pattern)] + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } } - ///Scalar Multiplication ///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a impl RP25519 { - pub const ONE: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); /// # Errors /// Propagates errors from decompressing invalid curve point -pub fn s_mul(self, rhs: Fp25519) -> Result { - self.0.decompress().map_or( - Err(Error::DecompressingInvalidCurvePoint), - |x| Ok((x * Scalar::from(rhs)).compress().into()) - ) -} + pub fn s_mul(self, rhs: Fp25519) -> Result { + self.0 + .decompress() + .map_or(Err(Error::DecompressingInvalidCurvePoint), |x| { + Ok((x * Scalar::from(rhs)).compress().into()) + }) + } } - ///do not use impl std::ops::Mul for RP25519 { type Output = Self; @@ -117,8 +115,7 @@ impl std::ops::Mul for RP25519 { ///do not use impl std::ops::MulAssign for RP25519 { - - fn mul_assign(& mut self, _rhs: RP25519) { + fn mul_assign(&mut self, _rhs: RP25519) { panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); } } @@ -159,7 +156,7 @@ macro_rules! cp_hash_impl { } } - impl From<$u_type> for RP25519 { + impl From<$u_type> for RP25519 { fn from(s: $u_type) -> Self { let hk = Hkdf::::new(None, &s.to_le_bytes()); let mut okm = [0u8; 32]; @@ -168,23 +165,17 @@ macro_rules! cp_hash_impl { RP25519::deserialize(&okm.into()) } } - } + }; } -cp_hash_impl!( - u64, - 8 -); +cp_hash_impl!(u64, 8); -cp_hash_impl!( - u32, - 4 -); +cp_hash_impl!(u32, 4); /// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is /// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed impl Field for RP25519 { - const ONE: RP25519= RP25519::ONE; + const ONE: RP25519 = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 /// from or into u128. However it is sufficient to generate random elements in Fp25519 @@ -204,7 +195,6 @@ impl Field for RP25519 { hk.expand(&[], &mut okm).unwrap(); RP25519::deserialize(&okm.into()) } - } impl TryFrom for RP25519 { @@ -213,50 +203,46 @@ impl TryFrom for RP25519 { fn try_from(v: u128) -> Result { let mut bits = [0u8; 32]; bits[..].copy_from_slice(&v.to_le_bytes()); - let f: RP25519=RP25519::ONE; + let f: RP25519 = RP25519::ONE; f.serialize((&mut bits).into()); Ok(f) } } - - - #[cfg(all(test, unit_test))] mod test { - use generic_array::GenericArray; - use crate::ff::curve_points::RP25519; - use crate::ff::Serializable; - use typenum::U32; use curve25519_dalek::scalar::Scalar; + use generic_array::GenericArray; use rand::{thread_rng, Rng}; - use crate::ff::ec_prime_field::Fp25519; - use crate::secret_sharing::SharedValue; + use typenum::U32; + + use crate::{ + ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable, field::Field}, + }; #[test] fn serde_25519() { - let input:[u8;32] = [ - 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + let input: [u8; 32] = [ + 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0x00, 0x00, 0x00, ]; - let mut output: GenericArray = [0u8;32].into(); + let mut output: GenericArray = [0u8; 32].into(); let a = RP25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32],input); + assert_eq!(a.0.as_bytes()[..32], input); a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); - assert_eq!(input,output.as_slice()[..32]); + assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); + assert_eq!(input, output.as_slice()[..32]); } #[test] fn scalar_to_point() { let a = Scalar::ONE; - let b : RP25519 = a.into(); - let d : Fp25519 = a.into(); - let c : RP25519 = RP25519::from(d); - assert_eq!(b,RP25519::ZERO); - assert_eq!(c,RP25519::ZERO); + let b: RP25519 = a.into(); + let d: Fp25519 = a.into(); + let c: RP25519 = RP25519::from(d); + assert_eq!(b, RP25519::ONE); + assert_eq!(c, RP25519::ONE); } #[test] @@ -264,24 +250,23 @@ mod test { let mut rng = thread_rng(); let fp_a = rng.gen::(); let fp_b = rng.gen::(); - let fp_c = fp_a+fp_b; - let fp_d = RP25519::from(fp_a)+RP25519::from(fp_b); + let fp_c = fp_a + fp_b; + let fp_d = RP25519::from(fp_a) + RP25519::from(fp_b); assert_eq!(fp_d, RP25519::from(fp_c)); - assert_ne!(fp_d, RP25519::ZERO); + assert_ne!(fp_d, RP25519::ONE); let fp_e = rng.gen::(); - let fp_f=rng.gen::(); - let fp_g =fp_e*fp_f; + let fp_f = rng.gen::(); + let fp_g = fp_e * fp_f; let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); - assert_eq!(fp_h,RP25519::from(fp_g)); - assert_ne!(fp_h, RP25519::ZERO); + assert_eq!(fp_h, RP25519::from(fp_g)); + assert_ne!(fp_h, RP25519::ONE); } #[test] fn curve_point_to_hash() { let mut rng = thread_rng(); let fp_a = rng.gen::(); - assert_ne!(0u64,u64::from(fp_a)); - assert_ne!(0u32,u32::from(fp_a)); + assert_ne!(0u64, u64::from(fp_a)); + assert_ne!(0u32, u32::from(fp_a)); } - -} \ No newline at end of file +} diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 624f91c64..6613116f6 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -1,12 +1,12 @@ -use generic_array::GenericArray; use curve25519_dalek::scalar::Scalar; +use generic_array::GenericArray; +use hkdf::Hkdf; +use sha2::Sha256; //use rand_core::RngCore; use typenum::U32; -use sha2::Sha256; -use hkdf::Hkdf; use crate::{ - ff::{Serializable, Field}, + ff::{Field, Serializable}, secret_sharing::{Block, SharedValue}, }; @@ -17,7 +17,6 @@ impl Block for Scalar { #[derive(Clone, Copy, PartialEq, Debug)] pub struct Fp25519(::Storage); - impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); @@ -26,10 +25,8 @@ impl Fp25519 { pub fn invert(&self) -> Fp25519 { Fp25519(self.0.invert()) } - } - impl SharedValue for Fp25519 { type Storage = Scalar; const BITS: u32 = 256; @@ -46,7 +43,7 @@ impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()] ; + let raw = &self.0.as_bytes()[..buf.len()]; buf.copy_from_slice(raw); } @@ -58,7 +55,6 @@ impl Serializable for Fp25519 { } } - impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> Fp25519 { let mut scalar_bytes = [0u8; 32]; @@ -69,61 +65,59 @@ impl rand::distributions::Distribution for rand::distributions::Standar } } - impl std::ops::Add for Fp25519 { -type Output = Self; + type Output = Self; -fn add(self, rhs: Self) -> Self::Output { - Self(self.0+rhs.0) -} + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } } impl std::ops::AddAssign for Fp25519 { -#[allow(clippy::assign_op_pattern)] -fn add_assign(&mut self, rhs: Self) { - *self = *self + rhs; -} + #[allow(clippy::assign_op_pattern)] + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } } impl std::ops::Neg for Fp25519 { -type Output = Self; + type Output = Self; -fn neg(self) -> Self::Output { - Self(self.0.neg()) -} + fn neg(self) -> Self::Output { + Self(self.0.neg()) + } } impl std::ops::Sub for Fp25519 { -type Output = Self; + type Output = Self; -fn sub(self, rhs: Self) -> Self::Output { - Self(self.0- rhs.0) -} + fn sub(self, rhs: Self) -> Self::Output { + Self(self.0 - rhs.0) + } } impl std::ops::SubAssign for Fp25519 { -#[allow(clippy::assign_op_pattern)] -fn sub_assign(&mut self, rhs: Self) { - *self = *self - rhs; -} + #[allow(clippy::assign_op_pattern)] + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } } impl std::ops::Mul for Fp25519 { -type Output = Self; + type Output = Self; -fn mul(self, rhs: Self) -> Self::Output { - Self(self.0 * rhs.0) -} + fn mul(self, rhs: Self) -> Self::Output { + Self(self.0 * rhs.0) + } } impl std::ops::MulAssign for Fp25519 { -#[allow(clippy::assign_op_pattern)] -fn mul_assign(&mut self, rhs: Self) { - *self = *self * rhs; -} + #[allow(clippy::assign_op_pattern)] + fn mul_assign(&mut self, rhs: Self) { + *self = *self * rhs; + } } - impl From for Fp25519 { fn from(s: Scalar) -> Self { Fp25519(s) @@ -142,7 +136,7 @@ macro_rules! sc_hash_impl { } } - impl From<$u_type> for Fp25519 { + impl From<$u_type> for Fp25519 { fn from(s: $u_type) -> Self { let hk = Hkdf::::new(None, &s.to_le_bytes()); let mut okm = [0u8; 32]; @@ -151,24 +145,16 @@ macro_rules! sc_hash_impl { Fp25519::deserialize(&okm.into()) } } - } + }; } +sc_hash_impl!(u64, 8); -sc_hash_impl!( - u64, - 8 -); - -sc_hash_impl!( - u32, - 4 -); - +sc_hash_impl!(u32, 4); /// Daniel had to implement this since PRSS wants it, prefer not to impl Field for Fp25519 { - const ONE: Fp25519= Fp25519::ONE; + const ONE: Fp25519 = Fp25519::ONE; ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 /// from or into u128. However it is sufficient to generate random elements in Fp25519 @@ -188,7 +174,6 @@ impl Field for Fp25519 { hk.expand(&[], &mut okm).unwrap(); Fp25519::deserialize(&okm.into()) } - } impl TryFrom for Fp25519 { @@ -197,7 +182,7 @@ impl TryFrom for Fp25519 { fn try_from(v: u128) -> Result { let mut bits = [0u8; 32]; bits[..].copy_from_slice(&v.to_le_bytes()); - let f: Fp25519=Fp25519::ONE; + let f: Fp25519 = Fp25519::ONE; f.serialize((&mut bits).into()); Ok(f) } @@ -205,76 +190,73 @@ impl TryFrom for Fp25519 { #[cfg(all(test, unit_test))] mod test { - use generic_array::GenericArray; - use crate::ff::ec_prime_field::Fp25519; - use crate::ff::Serializable; - use typenum::U32; use curve25519_dalek::scalar::Scalar; + use generic_array::GenericArray; use rand::{thread_rng, Rng}; - use crate::secret_sharing::SharedValue; + use typenum::U32; + + use crate::{ + ff::{ec_prime_field::Fp25519, Serializable}, + secret_sharing::SharedValue, + }; #[test] fn serde_25519() { - let input:[u8;32] = [ - 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + let input: [u8; 32] = [ + 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0x00, 0x00, 0x00, ]; - let mut output: GenericArray = [0u8;32].into(); + let mut output: GenericArray = [0u8; 32].into(); let a = Fp25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32],input); + assert_eq!(a.0.as_bytes()[..32], input); a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); - assert_eq!(input,output.as_slice()[..32]); + assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); + assert_eq!(input, output.as_slice()[..32]); } // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek #[test] fn simple_arithmetics_25519() { let a = Fp25519(Scalar::from_bytes_mod_order([ - 0x02, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); let b = Fp25519(Scalar::from_bytes_mod_order([ - 0x03, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); let d = Fp25519(Scalar::from_bytes_mod_order([ - 0x05, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); let e = Fp25519(Scalar::from_bytes_mod_order([ - 0x06, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); - let cc = b-a; - let dc = a+b; - let ec = a*b; - assert_eq!(cc,Fp25519::ONE); - assert_eq!(dc,d); - assert_eq!(ec,e); + let cc = b - a; + let dc = a + b; + let ec = a * b; + assert_eq!(cc, Fp25519::ONE); + assert_eq!(dc, d); + assert_eq!(ec, e); } #[test] - fn simple_random_25519(){ + fn simple_random_25519() { let mut rng = thread_rng(); - assert_ne!(Fp25519::ZERO,rng.gen::()); + assert_ne!(Fp25519::ZERO, rng.gen::()); } #[test] - fn invert_25519(){ + fn invert_25519() { let mut rng = thread_rng(); - let a=rng.gen::(); - let ia=a.invert(); - assert_eq!(a*ia, Fp25519(Scalar::ONE)); + let a = rng.gen::(); + let ia = a.invert(); + assert_eq!(a * ia, Fp25519(Scalar::ONE)); } -} \ No newline at end of file +} diff --git a/src/ff/mod.rs b/src/ff/mod.rs index b26596086..c858aaf93 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -2,11 +2,11 @@ // // This is where we store arithmetic shared secret data models. +pub mod curve_points; +pub mod ec_prime_field; mod field; mod galois_field; mod prime_field; -pub mod ec_prime_field; -pub mod curve_points; use std::ops::{Add, AddAssign, Sub, SubAssign}; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index 895ad524b..ae43e55a1 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -1,11 +1,17 @@ +use ipa_macros::Step; + use crate::{ error::Error, - ff::{ec_prime_field::Fp25519, curve_points::RP25519}, - protocol::{context::Context, RecordId, prss::SharedRandomness, basics::Reveal, basics::SecureMul}, - secret_sharing::{replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, + ff::{curve_points::RP25519, ec_prime_field::Fp25519}, + protocol::{ + basics::{Reveal, SecureMul}, + context::Context, + prss::SharedRandomness, + RecordId, + }, + secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, seq_join::seq_try_join_all, }; -use ipa_macros::Step; #[derive(Step)] pub(crate) enum Step { @@ -23,30 +29,33 @@ pub async fn compute_match_key_pseudonym( prf_key: AdditiveShare, input_match_keys: Vec>, ) -> Result, Error> - where - C: Context, +where + C: Context, { - let ctx =sh_ctx.set_total_records(input_match_keys.len()); - let futures=input_match_keys.iter().enumerate().map(|(i,x)|eval_dy_prf(ctx.clone(),i.into(),&prf_key,x)); + let ctx = sh_ctx.set_total_records(input_match_keys.len()); + let futures = input_match_keys + .iter() + .enumerate() + .map(|(i, x)| eval_dy_prf(ctx.clone(), i.into(), &prf_key, x)); seq_try_join_all(sh_ctx.active_work(), futures).await } - impl From> for AdditiveShare { fn from(s: AdditiveShare) -> Self { - AdditiveShare::new(RP25519::from(s.left()),RP25519::from(s.right())) + AdditiveShare::new(RP25519::from(s.left()), RP25519::from(s.right())) } } /// generates PRF key k as secret sharing over Fp25519 -pub fn gen_prf_key (ctx: &C) -> AdditiveShare - where - C: Context, +pub fn gen_prf_key(ctx: &C) -> AdditiveShare +where + C: Context, { - ctx.narrow(&Step::PRFKeyGen).prss().generate_replicated(u128::MAX-100u128) + ctx.narrow(&Step::PRFKeyGen) + .prss() + .generate_replicated(u128::MAX - 100u128) } - /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 /// PRF key k is generated using keygen @@ -61,26 +70,29 @@ pub async fn eval_dy_prf( k: &AdditiveShare, x: &AdditiveShare, ) -> Result - where - C: Context, +where + C: Context, { - let sh_r: AdditiveShare = ctx.narrow(&Step::GenRandomMask).prss().generate_replicated(record_id); + let sh_r: AdditiveShare = ctx + .narrow(&Step::GenRandomMask) + .prss() + .generate_replicated(record_id); //compute (g^left, g^right) let sh_gr = AdditiveShare::::from(sh_r.clone()); //compute x+k - let mut y =x+k; + let mut y = x + k; //compute y <- r*y - y = y.multiply(&sh_r, ctx.narrow(&Step::MultMaskWithPRFInput), record_id).await?; + y = y + .multiply(&sh_r, ctx.narrow(&Step::MultMaskWithPRFInput), record_id) + .await?; //reconstruct (z,R) let gr: RP25519 = sh_gr.reveal(ctx.narrow(&Step::RevealR), record_id).await?; let z = y.reveal(ctx.narrow(&Step::Revealz), record_id).await?; - - //compute R^(1/z) to u64 Ok(u64::from(gr.s_mul(z.invert())?)) } @@ -88,12 +100,13 @@ pub async fn eval_dy_prf( #[cfg(all(test, unit_test))] mod test { use rand::Rng; + use crate::{ + ff::{curve_points::RP25519, ec_prime_field::Fp25519}, + protocol::prf_eval::compute_match_key_pseudonym, + secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_executor::run, - test_fixture::{TestWorld, Reconstruct, Runner}, - ff::{ec_prime_field::Fp25519, curve_points::RP25519}, - secret_sharing::{IntoShares,replicated::semi_honest::AdditiveShare}, - protocol::prf_eval::{compute_match_key_pseudonym}, + test_fixture::{Reconstruct, Runner, TestWorld}, }; #[derive(Copy, Clone)] @@ -106,29 +119,30 @@ mod test { match_key_pseudonym: u64, } - fn test_input( mk: u64) -> ShuffledTestInput { + fn test_input(mk: u64) -> ShuffledTestInput { ShuffledTestInput { match_key: Fp25519::from(mk), } } - impl IntoShares> for ShuffledTestInput - { + impl IntoShares> for ShuffledTestInput { fn share_with(self, rng: &mut R) -> [AdditiveShare; 3] { self.match_key.share_with(rng) } } - impl Reconstruct for [&u64; 3] { fn reconstruct(&self) -> TestOutput { - TestOutput{ - match_key_pseudonym: if *self[0]==*self[1] && *self[0]==*self[2] {*self[0]} else {0u64}, + TestOutput { + match_key_pseudonym: if *self[0] == *self[1] && *self[0] == *self[2] { + *self[0] + } else { + 0u64 + }, } } } - #[test] fn semi_honest() { run(|| async move { @@ -146,29 +160,32 @@ mod test { test_input(3), test_input(56), test_input(23_443_524_523), - ]; + ]; //PRF Key Gen let u = 3_216_412_445u64; - let k:Fp25519 = Fp25519::from(u); + let k: Fp25519 = Fp25519::from(u); - let expected: Vec=records.iter().map(|&x| TestOutput{match_key_pseudonym: (RP25519::from((x.match_key+k).invert())).into()} ).collect(); + let expected: Vec = records + .iter() + .map(|&x| TestOutput { + match_key_pseudonym: (RP25519::from((x.match_key + k).invert())).into(), + }) + .collect(); let result: Vec<_> = world - .semi_honest((records.into_iter(),k), |ctx, (input_match_keys,prf_key)| async move { - compute_match_key_pseudonym::<_>( - ctx, - prf_key, - input_match_keys, - ) - .await - .unwrap() - }) + .semi_honest( + (records.into_iter(), k), + |ctx, (input_match_keys, prf_key)| async move { + compute_match_key_pseudonym::<_>(ctx, prf_key, input_match_keys) + .await + .unwrap() + }, + ) .await .reconstruct(); assert_eq!(result, expected); assert_eq!(result[0], result[1]); }); } - -} \ No newline at end of file +} From 38fc761a545f4d154f077f1e85e84148766b3055 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 10 Oct 2023 12:25:46 +0800 Subject: [PATCH 027/124] move bucket can now handle upto breakdown = 127 --- src/protocol/prf_sharding/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index eed74365b..9b572280b 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -693,8 +693,8 @@ where let depth_c = ctx.narrow(&BinaryTreeDepthStep::from(tree_depth)); let span = step >> 1; let mut futures = Vec::with_capacity(breakdown_count / step); - for i in (0..breakdown_count).step_by(step) { - let bit_c = depth_c.narrow(&BitOpStep::from(i)); + for (step_no, i) in (0..breakdown_count).step_by(step).enumerate() { + let bit_c = depth_c.narrow(&BitOpStep::from(step_no)); if i + span < breakdown_count { futures.push(row_contribution[i].multiply(bit_of_bdkey, bit_c, record_id)); @@ -1035,7 +1035,7 @@ pub mod tests { #[test] fn semi_honest_move_value_to_single_bucket() { - const MAX_BREAKDOWN_COUNT: usize = 63; + const MAX_BREAKDOWN_COUNT: usize = 127; for _ in 1..10 { run(|| async move { let world = TestWorld::default(); From 7ca5813e69750419944466b2daf1004db6f48c52 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Tue, 10 Oct 2023 15:34:56 +0800 Subject: [PATCH 028/124] comments from Alex --- Cargo.toml | 1 - .../prf_sharding/feature_label_dot_product.rs | 24 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a72bfcfbf..e264d503f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,7 +95,6 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } typenum = "1.16" # hpke is pinned to it x25519-dalek = "2.0.0-pre.0" -stream-flatten-iters = "0.2.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.5.0" diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index a326c02ef..0dfc28f3a 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -3,7 +3,6 @@ use std::{iter::zip, pin::pin}; use futures::{stream::iter as stream_iter, TryStreamExt}; use futures_util::{future::try_join, stream::unfold, Stream, StreamExt}; use ipa_macros::Step; -use stream_flatten_iters::StreamExt as _; use crate::{ error::Error, @@ -171,8 +170,7 @@ where IS: Stream> + Unpin, { unfold(Some((input_stream, first_row)), |state| async move { - state.as_ref()?; - let (mut s, last_row) = state.unwrap(); + let (mut s, last_row) = state?; let last_row_prf = last_row.prf_of_match_key; let mut current_chunk = vec![last_row]; while let Some(row) = s.next().await { @@ -221,7 +219,6 @@ where /// Propagates errors from multiplications /// # Panics /// Propagates errors from multiplications -#[allow(clippy::async_yields_async)] pub async fn compute_feature_label_dot_product( sh_ctx: C, input_rows: Vec>, @@ -252,7 +249,11 @@ where // Chunk the incoming stream of records into stream of vectors of records with the same PRF let mut input_stream = stream_iter(input_rows); - let first_row = input_stream.next().await.unwrap(); + let first_row = input_stream.next().await; + if first_row.is_none() { + return Ok(vec![]); + } + let first_row = first_row.unwrap(); let rows_chunked_by_user = chunk_rows_by_user(input_stream, first_row); // Convert to a stream of async futures that represent the result of executing the per-user circuit @@ -261,16 +262,19 @@ where let contexts = ctx_for_row_number[..num_user_rows - 1].to_owned(); let record_ids = record_id_for_row_depth[..num_user_rows].to_owned(); - for count in record_id_for_row_depth.iter_mut().take(rows_for_user.len()) { + for count in &mut record_id_for_row_depth[..num_user_rows] { *count += 1; } - async move { evaluate_per_user_attribution_circuit(contexts, record_ids, rows_for_user) } + #[allow(clippy::async_yields_async)] + // this is ok, because seq join wants a stream of futures + async move { + evaluate_per_user_attribution_circuit(contexts, record_ids, rows_for_user) + } })); // Execute all of the async futures (sequentially), and flatten the result let flattenned_stream = seq_join(sh_ctx.active_work(), stream_of_per_user_circuits) - .map(|x| x.unwrap().into_iter()) - .flatten_iters(); + .flat_map(|x| stream_iter(x.unwrap())); // modulus convert feature vector bits from shares in `Z_2` to shares in `Z_p` let converted_feature_vector_bits = convert_bits( @@ -317,7 +321,7 @@ where for (i, (row, ctx)) in zip(rows_for_user.iter().skip(1), ctx_for_row_number.into_iter()).enumerate() { - let record_id_for_this_row_depth = RecordId(record_id_for_each_depth[i + 1]); // skip row 0 + let record_id_for_this_row_depth = RecordId::from(record_id_for_each_depth[i + 1]); // skip row 0 let capped_attribution_outputs = prev_row_inputs .compute_row_with_previous(ctx, record_id_for_this_row_depth, row) From 0751e18e3398865d689456450f32c3c011435942 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 10 Oct 2023 22:01:16 +0800 Subject: [PATCH 029/124] add robust + testcase for it --- src/protocol/prf_sharding/mod.rs | 232 +++++++++++++++++++++---------- 1 file changed, 157 insertions(+), 75 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 9b572280b..61fae9663 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -637,6 +637,7 @@ where bd_key, value, 1 << BK::BITS, + false, ) .await } @@ -671,12 +672,18 @@ where /// At each successive round, the next most significant bit is propagated from the leaf nodes of the tree into further leaf nodes: /// [`row_contribution`]_r+1,q,0 =[`row_contribution`]_r,q - [`bd_key`]_r+1.[`row_contribution`]_r,q and [`row_contribution`]_r+1,q,1 =[`bd_key`]_r+1.[`row_contribution`]_r,q. /// The work of each iteration therefore doubles relative to the one preceding. +/// +/// In case a malicious entity sends a out of range breakdown key (i.e. greater than the max count) to this function, we need to do some +/// extra processing to ensure contribution doesn't end up in a wrong bucket. However, this requires extra multiplications. +/// This would potentially not be needed in IPA (as aggregation is done after pre-processing which should be able to throw such input) but useful for PAM. +/// This can be by passing `robust_for_breakdown_key_gt_count as true async fn move_single_value_to_bucket( ctx: C, record_id: RecordId, bd_key: BitDecomposed, value: S, breakdown_count: usize, + robust_for_breakdown_key_gt_count: bool, ) -> Result, Error> where BK: GaloisField, @@ -693,11 +700,12 @@ where let depth_c = ctx.narrow(&BinaryTreeDepthStep::from(tree_depth)); let span = step >> 1; let mut futures = Vec::with_capacity(breakdown_count / step); - for (step_no, i) in (0..breakdown_count).step_by(step).enumerate() { - let bit_c = depth_c.narrow(&BitOpStep::from(step_no)); - if i + span < breakdown_count { - futures.push(row_contribution[i].multiply(bit_of_bdkey, bit_c, record_id)); + for (i, tree_index) in (0..breakdown_count).step_by(step).enumerate() { + let bit_c = depth_c.narrow(&BitOpStep::from(i)); + + if robust_for_breakdown_key_gt_count || tree_index + span < breakdown_count { + futures.push(row_contribution[tree_index].multiply(bit_of_bdkey, bit_c, record_id)); } } let contributions = ctx.parallel_join(futures).await?; @@ -705,8 +713,11 @@ where for (index, bdbit_contribution) in contributions.into_iter().enumerate() { let left_index = index * step; let right_index = left_index + span; + row_contribution[left_index] -= &bdbit_contribution; - row_contribution[right_index] = bdbit_contribution; + if right_index < breakdown_count { + row_contribution[right_index] = bdbit_contribution; + } } step = span; } @@ -744,7 +755,7 @@ pub mod tests { trigger_value: TV, } - fn test_input( + fn oprf_test_input( prf_of_match_key: u64, is_trigger: bool, breakdown_key: u8, @@ -760,15 +771,15 @@ pub mod tests { } } - fn test_output( + fn bitwise_bd_key_and_value( attributed_breakdown_key: u128, capped_attributed_trigger_value: u128, - ) -> PreAggregationTestOutput + ) -> PreAggregationTestInputInBits where BK: GaloisField, TV: GaloisField, { - PreAggregationTestOutput { + PreAggregationTestInputInBits { attributed_breakdown_key: get_bits::( attributed_breakdown_key.try_into().unwrap(), BK::BITS, @@ -780,8 +791,24 @@ pub mod tests { } } + fn decimal_bd_key_and_value( + attributed_breakdown_key: u128, + capped_attributed_trigger_value: u128, + ) -> PreAggregationTestOutputInDecimal { + PreAggregationTestOutputInDecimal { + attributed_breakdown_key, + capped_attributed_trigger_value, + } + } + + #[derive(Debug, PartialEq)] + struct PreAggregationTestOutputInDecimal { + attributed_breakdown_key: u128, + capped_attributed_trigger_value: u128, + } + #[derive(Debug, PartialEq)] - struct PreAggregationTestOutput { + struct PreAggregationTestInputInBits { attributed_breakdown_key: BitDecomposed, capped_attributed_trigger_value: BitDecomposed, } @@ -827,9 +854,9 @@ pub mod tests { } } - impl IntoShares for PreAggregationTestOutput { + impl IntoShares for PreAggregationTestInputInBits { fn share_with(self, rng: &mut R) -> [CappedAttributionOutputs; 3] { - let PreAggregationTestOutput { + let PreAggregationTestInputInBits { attributed_breakdown_key, capped_attributed_trigger_value, } = self; @@ -861,10 +888,10 @@ pub mod tests { ] } } - impl Reconstruct for [&CappedAttributionOutputs; 3] { - fn reconstruct(&self) -> PreAggregationTestOutput { - let [s0, s1, s2] = self; + impl Reconstruct for [&CappedAttributionOutputs; 3] { + fn reconstruct(&self) -> PreAggregationTestOutputInDecimal { + let [s0, s1, s2] = self; let attributed_breakdown_key_bits: BitDecomposed = BitDecomposed::new( s0.attributed_breakdown_key_bits .iter() @@ -872,7 +899,6 @@ pub mod tests { .zip(s2.attributed_breakdown_key_bits.iter()) .map(|((a, b), c)| [a, b, c].reconstruct()), ); - let capped_attributed_trigger_value_bits: BitDecomposed = BitDecomposed::new( s0.capped_attributed_trigger_value .iter() @@ -881,50 +907,58 @@ pub mod tests { .map(|((a, b), c)| [a, b, c].reconstruct()), ); - PreAggregationTestOutput { - attributed_breakdown_key: attributed_breakdown_key_bits, - capped_attributed_trigger_value: capped_attributed_trigger_value_bits, + PreAggregationTestOutputInDecimal { + attributed_breakdown_key: attributed_breakdown_key_bits + .iter() + .map(Field::as_u128) + .enumerate() + .fold(0_u128, |acc, (i, x)| acc + (x << i)), + capped_attributed_trigger_value: capped_attributed_trigger_value_bits + .iter() + .map(Field::as_u128) + .enumerate() + .fold(0_u128, |acc, (i, x)| acc + (x << i)), } } } #[test] - fn semi_honest() { + fn semi_honest_attribution_and_capping() { run(|| async move { let world = TestWorld::default(); let records: Vec> = vec![ /* First User */ - test_input(123, false, 17, 0), - test_input(123, true, 0, 7), - test_input(123, false, 20, 0), - test_input(123, true, 0, 3), + oprf_test_input(123, false, 17, 0), + oprf_test_input(123, true, 0, 7), + oprf_test_input(123, false, 20, 0), + oprf_test_input(123, true, 0, 3), /* Second User */ - test_input(234, false, 12, 0), - test_input(234, true, 0, 5), + oprf_test_input(234, false, 12, 0), + oprf_test_input(234, true, 0, 5), /* Third User */ - test_input(345, false, 20, 0), - test_input(345, true, 0, 7), - test_input(345, false, 18, 0), - test_input(345, false, 12, 0), - test_input(345, true, 0, 7), - test_input(345, true, 0, 7), - test_input(345, true, 0, 7), - test_input(345, true, 0, 7), + oprf_test_input(345, false, 20, 0), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, false, 18, 0), + oprf_test_input(345, false, 12, 0), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, true, 0, 7), ]; - let expected: [PreAggregationTestOutput; 11] = [ - test_output::(17, 7), - test_output::(20, 0), - test_output::(20, 3), - test_output::(12, 5), - test_output::(20, 7), - test_output::(18, 0), - test_output::(12, 0), - test_output::(12, 7), - test_output::(12, 7), - test_output::(12, 7), - test_output::(12, 4), + let expected: [PreAggregationTestOutputInDecimal; 11] = [ + decimal_bd_key_and_value(17, 7), + decimal_bd_key_and_value(20, 0), + decimal_bd_key_and_value(20, 3), + decimal_bd_key_and_value(12, 5), + decimal_bd_key_and_value(20, 7), + decimal_bd_key_and_value(18, 0), + decimal_bd_key_and_value(12, 0), + decimal_bd_key_and_value(12, 7), + decimal_bd_key_and_value(12, 7), + decimal_bd_key_and_value(12, 7), + decimal_bd_key_and_value(12, 4), ]; let num_saturating_bits: usize = 5; @@ -951,22 +985,22 @@ pub mod tests { let records: Vec> = vec![ /* First User */ - test_input(123, false, 17, 0), - test_input(123, true, 0, 7), - test_input(123, false, 20, 0), - test_input(123, true, 0, 3), + oprf_test_input(123, false, 17, 0), + oprf_test_input(123, true, 0, 7), + oprf_test_input(123, false, 20, 0), + oprf_test_input(123, true, 0, 3), /* Second User */ - test_input(234, false, 12, 0), - test_input(234, true, 0, 5), + oprf_test_input(234, false, 12, 0), + oprf_test_input(234, true, 0, 5), /* Third User */ - test_input(345, false, 20, 0), - test_input(345, true, 0, 7), - test_input(345, false, 18, 0), - test_input(345, false, 12, 0), - test_input(345, true, 0, 7), - test_input(345, true, 0, 7), - test_input(345, true, 0, 7), - test_input(345, true, 0, 7), + oprf_test_input(345, false, 20, 0), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, false, 18, 0), + oprf_test_input(345, false, 12, 0), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, true, 0, 7), + oprf_test_input(345, true, 0, 7), ]; let mut expected = [0_u128; 32]; @@ -1000,18 +1034,18 @@ pub mod tests { run(|| async move { let world = TestWorld::default(); - let records: Vec = vec![ - test_output::(17, 7), - test_output::(20, 0), - test_output::(20, 3), - test_output::(12, 5), - test_output::(20, 7), - test_output::(18, 0), - test_output::(12, 0), - test_output::(12, 7), - test_output::(12, 7), - test_output::(12, 7), - test_output::(12, 4), + let records: Vec = vec![ + bitwise_bd_key_and_value::(17, 7), + bitwise_bd_key_and_value::(20, 0), + bitwise_bd_key_and_value::(20, 3), + bitwise_bd_key_and_value::(12, 5), + bitwise_bd_key_and_value::(20, 7), + bitwise_bd_key_and_value::(18, 0), + bitwise_bd_key_and_value::(12, 0), + bitwise_bd_key_and_value::(12, 7), + bitwise_bd_key_and_value::(12, 7), + bitwise_bd_key_and_value::(12, 7), + bitwise_bd_key_and_value::(12, 4), ]; let mut expected = [0_u128; 32]; @@ -1034,16 +1068,16 @@ pub mod tests { } #[test] - fn semi_honest_move_value_to_single_bucket() { + fn semi_honest_move_value_to_single_bucket_in_range() { const MAX_BREAKDOWN_COUNT: usize = 127; for _ in 1..10 { run(|| async move { let world = TestWorld::default(); let mut rng: rand::rngs::ThreadRng = thread_rng(); let count = rng.gen_range(1..MAX_BREAKDOWN_COUNT); - let breakdown_key = rng.gen_range(0..count); + let breakdown_key = rng.gen_range(1..count); - let value = Fp32BitPrime::truncate_from(10_u128); + let value: Fp32BitPrime = Fp32BitPrime::truncate_from(10_u128); let mut expected = vec![Fp32BitPrime::truncate_from(0_u128); count]; expected[breakdown_key] = value; @@ -1062,6 +1096,7 @@ pub mod tests { breakdown_key_share, value_share, count, + false, ) .await .unwrap() @@ -1073,4 +1108,51 @@ pub mod tests { }); } } + + #[test] + fn semi_honest_move_value_to_single_bucket_out_of_range() { + const MAX_BREAKDOWN_COUNT: usize = 127; + for robust_for_breakdown_key_gt_count in [true, false] { + for _ in 1..10 { + run(move || async move { + let world = TestWorld::default(); + let mut rng: rand::rngs::ThreadRng = thread_rng(); + let count = rng.gen_range(1..MAX_BREAKDOWN_COUNT); + let breakdown_key = rng.gen_range(count..MAX_BREAKDOWN_COUNT); + + let value: Fp32BitPrime = Fp32BitPrime::truncate_from(10_u128); + let expected = vec![Fp32BitPrime::truncate_from(0_u128); count]; + + let breakdown_key_bits = + get_bits::(breakdown_key.try_into().unwrap(), Gf8Bit::BITS); + + let result: Vec<_> = world + .semi_honest( + (breakdown_key_bits, value), + |ctx, (breakdown_key_share, value_share)| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + move_single_value_to_bucket::( + ctx.set_total_records(1), + RecordId::from(0), + breakdown_key_share, + value_share, + count, + robust_for_breakdown_key_gt_count, + ) + .await + .unwrap() + }, + ) + .await + .reconstruct(); + if robust_for_breakdown_key_gt_count { + assert_eq!(result, expected); + } else { + assert_ne!(result, expected); + } + }); + } + } + } } From b6ef12aaa3506d351cb8105c0000b0fbb54fb3a6 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Wed, 11 Oct 2023 21:39:16 -0700 Subject: [PATCH 030/124] Fix some (all?) concurrency issues inside infrastructure (#792) * Add tests that fail or hang if there is a send/receive bug in channel creation * Add a failing test to `OrderingSender` suite to demonstrate concurrency bug * Fix the concurrency bug inside OrderingSender * Fix the concurrency bug inside UnorderedReceiver * Address feedback * Formatting --- src/helpers/buffers/ordering_sender.rs | 135 +++++++++++++++++++------ src/helpers/gateway/mod.rs | 113 ++++++++++++++++++++- src/helpers/gateway/receive.rs | 18 ++-- src/helpers/gateway/send.rs | 70 +++++++------ 4 files changed, 258 insertions(+), 78 deletions(-) diff --git a/src/helpers/buffers/ordering_sender.rs b/src/helpers/buffers/ordering_sender.rs index d74a3f11b..ea1459d62 100644 --- a/src/helpers/buffers/ordering_sender.rs +++ b/src/helpers/buffers/ordering_sender.rs @@ -1,13 +1,11 @@ -#![allow(dead_code)] // TODO remove - use std::{ borrow::Borrow, cmp::Ordering, collections::VecDeque, + fmt::Debug, mem::drop, num::NonZeroUsize, pin::Pin, - sync::Arc, task::{Context, Poll}, }; @@ -116,6 +114,7 @@ impl State { } /// An saved waker for a given index. +#[derive(Debug)] struct WakerItem { /// The index. i: usize, @@ -124,37 +123,59 @@ struct WakerItem { } /// A collection of saved wakers. -#[derive(Default)] +#[derive(Default, Debug)] struct WaitingShard { + /// The maximum index that was used to wake a task that belongs to this shard. + /// Updates to this shard will be rejected if the supplied index is less than this value. + /// See [`Add`] for more details. + /// + /// [`Add`]: WaitingShard::add + woken_at: usize, /// The saved wakers. These are sorted on insert (see `add`) and /// presumably removed constantly, so a circular buffer is used. wakers: VecDeque, } impl WaitingShard { - fn add(&mut self, i: usize, w: Waker) { + /// Add a waker that will be used to wake up a write to `i`. + /// + /// ## Errors + /// If `current` is behind the current position recorded in this shard. + fn add(&mut self, current: usize, i: usize, w: &Waker) -> Result<(), ()> { + if current < self.woken_at { + // this means this thread is out of sync and there was an update to channel's current + // position. Accepting a waker could mean it will never be awakened. Rejecting this operation + // will let the current thread to read the position again. + Err(())?; + } + // Each new addition will tend to have a larger index, so search backwards and // replace an equal index or insert after a smaller index. // TODO: consider a binary search if the item cannot be added to the end. - let item = WakerItem { i, w }; + let item = WakerItem { i, w: w.clone() }; for j in (0..self.wakers.len()).rev() { match self.wakers[j].i.cmp(&i) { Ordering::Greater => (), Ordering::Equal => { assert!(item.w.will_wake(&self.wakers[j].w)); self.wakers[j] = item; - return; + return Ok(()); } Ordering::Less => { self.wakers.insert(j + 1, item); - return; + return Ok(()); } } } self.wakers.insert(0, item); + Ok(()) } fn wake(&mut self, i: usize) { + // Waking thread may have lost the race and got the lock after the successful write + // to the next element. Moving `woken_at` back will introduce a concurrency bug. + self.woken_at = std::cmp::max(self.woken_at, i); + if let Some(idx) = self .wakers .iter() @@ -192,8 +213,12 @@ impl Waiting { self.shards[idx].lock().unwrap() } - fn add(&self, i: usize, w: Waker) { - self.shard(i).add(i, w); + /// Add a waker that will be used to wake up a write to `i`. + /// + /// ## Errors + /// If `current` is behind the current position recorded in this shard. + fn add(&self, current: usize, i: usize, w: &Waker) -> Result<(), ()> { + self.shard(i).add(current, i, w) } fn wake(&self, i: usize) { @@ -276,26 +301,43 @@ impl OrderingSender { { // This load here is on the hot path. // Don't acquire the state mutex unless this test passes. - match self.next.load(Acquire).cmp(&i) { - Ordering::Greater => { - panic!("attempt to write/close at index {i} twice"); - } - Ordering::Equal => { - // OK, now it is our turn, so we need to hold a lock. - // No one else should be incrementing this atomic, so - // there should be no contention on this lock except for - // any calls to `take()`, which is tolerable. - let res = f(&mut self.state.lock().unwrap()); - if res.is_ready() { - let curr = self.next.fetch_add(1, AcqRel); - debug_assert_eq!(i, curr, "we just checked this"); + loop { + let curr = self.next.load(Acquire); + match curr.cmp(&i) { + Ordering::Greater => { + panic!("attempt to write/close at index {i} twice"); + } + Ordering::Equal => { + // OK, now it is our turn, so we need to hold a lock. + // No one else should be incrementing this atomic, so + // there should be no contention on this lock except for + // any calls to `take()`, which is tolerable. + let res = f(&mut self.state.lock().unwrap()); + if res.is_ready() { + let curr = self.next.fetch_add(1, AcqRel); + debug_assert_eq!(i, curr, "we just checked this"); + } + break res; + } + Ordering::Less => { + // This is the hot path. Wait our turn. If our view of the world is obsolete + // we won't be able to add a waker and need to read the atomic again. + // + // Here is why it works: + // * The only thread updating the atomic is the one that is writing to `i`. + // * If the write to `i` is successful, it wakes up the thread waiting to write `i` + 1. + // * Adding a waker and waking it is within a critical section. + // + // There are two possible scenarios for two threads competing for `i` + 1 waker. + // * Waiting thread adds a waker before writer thread attempts to wake it. This is a normal case + // scenario and things work as expected + // * Waiting thread attempts to add a waker after writer tried to wake it. This attempt will + // be rejected because writer has moved the waiting shard position ahead and it won't match + // the value of `self.next` read by the waiting thread. + if self.waiting.add(curr, i, cx.waker()).is_ok() { + break Poll::Pending; + } } - res - } - Ordering::Less => { - // This is the hot path. Wait our turn. - self.waiting.add(i, cx.waker().clone()); - Poll::Pending } } } @@ -322,11 +364,15 @@ impl OrderingSender { /// The stream interface requires a mutable reference to the stream itself. /// That's not possible here as we create a ton of immutable references to this. /// This wrapper takes a trivial reference so that we can implement `Stream`. - pub(crate) fn as_stream(&self) -> OrderedStream<&Self> { + #[cfg(all(test, any(unit_test, feature = "shuttle")))] + fn as_stream(&self) -> OrderedStream<&Self> { OrderedStream { sender: self } } - pub(crate) fn as_rc_stream(self: Arc) -> OrderedStream> { + #[cfg(all(test, unit_test))] + pub(crate) fn as_rc_stream( + self: crate::sync::Arc, + ) -> OrderedStream> { OrderedStream { sender: self } } } @@ -403,8 +449,11 @@ mod test { stream::StreamExt, FutureExt, }; + use futures_util::future::try_join_all; use generic_array::GenericArray; use rand::Rng; + #[cfg(feature = "shuttle")] + use shuttle::future as tokio; use typenum::Unsigned; use super::OrderingSender; @@ -581,4 +630,28 @@ mod test { } }); } + + /// This test is supposed to eventually hang if there is a concurrency bug inside `OrderingSender`. + #[test] + fn parallel_send() { + const PARALLELISM: usize = 100; + + run(|| async { + let sender = Arc::new(OrderingSender::new( + NonZeroUsize::new(PARALLELISM * ::Size::USIZE).unwrap(), + NonZeroUsize::new(5).unwrap(), + )); + + try_join_all((0..PARALLELISM).map(|i| { + tokio::spawn({ + let sender = Arc::clone(&sender); + async move { + sender.send(i, Fp31::truncate_from(i as u128)).await; + } + }) + })) + .await + .unwrap(); + }); + } } diff --git a/src/helpers/gateway/mod.rs b/src/helpers/gateway/mod.rs index ead1f32dc..c2a37916c 100644 --- a/src/helpers/gateway/mod.rs +++ b/src/helpers/gateway/mod.rs @@ -149,12 +149,13 @@ impl GatewayConfig { #[cfg(all(test, unit_test))] mod tests { - use futures_util::future::{join, try_join}; + use std::iter::{repeat, zip}; + + use futures_util::future::{join, try_join, try_join_all}; - use super::*; use crate::{ ff::{Field, Fp31, Fp32BitPrime, Gf2}, - helpers::{Direction, GatewayConfig, SendingEnd}, + helpers::{Direction, GatewayConfig, Role, SendingEnd}, protocol::{context::Context, RecordId}, test_fixture::{Runner, TestWorld, TestWorldConfig}, }; @@ -237,4 +238,110 @@ mod tests { spawned.await.unwrap(); let _world = unsafe { Box::from_raw(world_ptr) }; } + + /// this test requires quite a few threads to simulate send contention and will panic if + /// there is more than one sender channel created per step. + #[tokio::test(flavor = "multi_thread", worker_threads = 20)] + pub async fn send_contention() { + let (world, world_ptr) = make_world(); + + try_join_all(world.contexts().map(|ctx| { + tokio::spawn(async move { + const TOTAL_RECORDS: usize = 10; + let ctx = ctx + .narrow("send_contention") + .set_total_records(TOTAL_RECORDS); + + let receive_handle = tokio::spawn({ + let ctx = ctx.clone(); + async move { + for record in 0..TOTAL_RECORDS { + let v = Fp31::truncate_from(u128::try_from(record).unwrap()); + let r = ctx + .recv_channel::(ctx.role().peer(Direction::Left)) + .receive(record.into()) + .await + .unwrap(); + + assert_eq!(v, r, "Bad value for record {record}"); + } + } + }); + + try_join_all(zip(0..TOTAL_RECORDS, repeat(ctx)).map(|(record, ctx)| { + tokio::spawn(async move { + let r = Fp31::truncate_from(u128::try_from(record).unwrap()); + ctx.send_channel(ctx.role().peer(Direction::Right)) + .send(RecordId::from(record), r) + .await + .unwrap(); + }) + })) + .await + .unwrap(); + + receive_handle.await.unwrap(); + }) + })) + .await + .unwrap(); + + let _world = unsafe { Box::from_raw(world_ptr) }; + } + + /// This test should hang if receiver channel is not created atomically. It may occasionally + /// pass, but it will not give false negatives. + #[tokio::test(flavor = "multi_thread", worker_threads = 20)] + pub async fn receive_contention() { + let (world, world_ptr) = make_world(); + let contexts = world.contexts(); + + try_join_all(contexts.map(|ctx| { + tokio::spawn(async move { + const TOTAL_RECORDS: u32 = 20; + let ctx = ctx + .narrow("receive_contention") + .set_total_records(usize::try_from(TOTAL_RECORDS).unwrap()); + + tokio::spawn({ + let ctx = ctx.clone(); + async move { + for record in 0..TOTAL_RECORDS { + ctx.send_channel(ctx.role().peer(Direction::Right)) + .send(RecordId::from(record), Fp31::truncate_from(record)) + .await + .unwrap(); + } + } + }); + + try_join_all((0..TOTAL_RECORDS).zip(repeat(ctx)).map(|(record, ctx)| { + tokio::spawn(async move { + let r = ctx + .recv_channel::(ctx.role().peer(Direction::Left)) + .receive(RecordId::from(record)) + .await + .unwrap(); + assert_eq!( + Fp31::truncate_from(record), + r, + "received bad value for record {record}" + ); + }) + })) + .await + .unwrap(); + }) + })) + .await + .unwrap(); + + let _world = unsafe { Box::from_raw(world_ptr) }; + } + + fn make_world() -> (&'static TestWorld, *mut TestWorld) { + let world = Box::leak(Box::::default()); + let world_ptr = world as *mut _; + (world, world_ptr) + } } diff --git a/src/helpers/gateway/receive.rs b/src/helpers/gateway/receive.rs index 4f50e7b48..282ff68e2 100644 --- a/src/helpers/gateway/receive.rs +++ b/src/helpers/gateway/receive.rs @@ -1,6 +1,6 @@ use std::marker::PhantomData; -use dashmap::DashMap; +use dashmap::{mapref::entry::Entry, DashMap}; use futures::Stream; use crate::{ @@ -65,13 +65,15 @@ impl Default for GatewayReceivers { impl GatewayReceivers { pub fn get_or_create UR>(&self, channel_id: &ChannelId, ctr: F) -> UR { - let receivers = &self.inner; - if let Some(recv) = receivers.get(channel_id) { - recv.clone() - } else { - let stream = ctr(); - receivers.insert(channel_id.clone(), stream.clone()); - stream + // TODO: raw entry API if it becomes available to avoid cloning the key + match self.inner.entry(channel_id.clone()) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + let stream = ctr(); + entry.insert(stream.clone()); + + stream + } } } } diff --git a/src/helpers/gateway/send.rs b/src/helpers/gateway/send.rs index c36875829..4eb876af0 100644 --- a/src/helpers/gateway/send.rs +++ b/src/helpers/gateway/send.rs @@ -5,7 +5,7 @@ use std::{ task::{Context, Poll}, }; -use dashmap::DashMap; +use dashmap::{mapref::entry::Entry, DashMap}; use futures::Stream; use typenum::Unsigned; @@ -127,42 +127,40 @@ impl GatewaySenders { total_records.is_specified(), "unspecified total records for {channel_id:?}" ); - let senders = &self.inner; - if let Some(sender) = senders.get(channel_id) { - (Arc::clone(&sender), None) - } else { - const SPARE: Option = NonZeroUsize::new(64); - // a little trick - if number of records is indeterminate, set the capacity to 1. - // Any send will wake the stream reader then, effectively disabling buffering. - // This mode is clearly inefficient, so avoid using this mode. - let write_size = if total_records.is_indeterminate() { - NonZeroUsize::new(1).unwrap() - } else { - // capacity is defined in terms of number of elements, while sender wants bytes - // so perform the conversion here - capacity - .checked_mul( - NonZeroUsize::new(M::Size::USIZE) - .expect("Message size should be greater than 0"), - ) - .expect("capacity should not overflow") - }; - - let sender = Arc::new(GatewaySender::new( - channel_id.clone(), - OrderingSender::new(write_size, SPARE.unwrap()), - total_records, - )); - if senders - .insert(channel_id.clone(), Arc::clone(&sender)) - .is_some() - { - panic!("TODO - make sender creation contention less dangerous"); + + // TODO: raw entry API would be nice to have here but it's not exposed yet + match self.inner.entry(channel_id.clone()) { + Entry::Occupied(entry) => (Arc::clone(entry.get()), None), + Entry::Vacant(entry) => { + const SPARE: Option = NonZeroUsize::new(64); + // a little trick - if number of records is indeterminate, set the capacity to 1. + // Any send will wake the stream reader then, effectively disabling buffering. + // This mode is clearly inefficient, so avoid using this mode. + let write_size = if total_records.is_indeterminate() { + NonZeroUsize::new(1).unwrap() + } else { + // capacity is defined in terms of number of elements, while sender wants bytes + // so perform the conversion here + capacity + .checked_mul( + NonZeroUsize::new(M::Size::USIZE) + .expect("Message size should be greater than 0"), + ) + .expect("capacity should not overflow") + }; + + let sender = Arc::new(GatewaySender::new( + channel_id.clone(), + OrderingSender::new(write_size, SPARE.unwrap()), + total_records, + )); + entry.insert(Arc::clone(&sender)); + + ( + Arc::clone(&sender), + Some(GatewaySendStream { inner: sender }), + ) } - let stream = GatewaySendStream { - inner: Arc::clone(&sender), - }; - (sender, Some(stream)) } } } From 64f6fc2f7d890a4a13e88b6b6a2d1716f83af4ee Mon Sep 17 00:00:00 2001 From: Artem Ignatyev Date: Fri, 13 Oct 2023 13:35:37 -0500 Subject: [PATCH 031/124] Simplify FromRequest::from_request for QueryConfigParams (#802) It seems that the code introduces a few unnecessary local structs to parse http query parameters into. But these structs are the same as the final QueryType-nested structs. So, we can parse the params into those structs directly and make the code simpler Co-authored-by: Artem Ignatyev --- src/helpers/transport/query.rs | 1 + src/net/http_serde.rs | 88 +++++++--------------------------- 2 files changed, 17 insertions(+), 72 deletions(-) diff --git a/src/helpers/transport/query.rs b/src/helpers/transport/query.rs index 8f77a3f9f..ab4e0761b 100644 --- a/src/helpers/transport/query.rs +++ b/src/helpers/transport/query.rs @@ -250,6 +250,7 @@ pub struct IpaQueryConfig { /// input report format in which all fields are secret-shared. This option is provided /// only for development and testing purposes and may be removed in the future. #[cfg_attr(feature = "clap", arg(long))] + #[serde(default)] pub plaintext_match_keys: bool, } diff --git a/src/net/http_serde.rs b/src/net/http_serde.rs index 25432654a..faa2ec13d 100644 --- a/src/net/http_serde.rs +++ b/src/net/http_serde.rs @@ -80,20 +80,14 @@ pub mod echo { } pub mod query { - use std::{ - fmt::{Display, Formatter}, - num::NonZeroU32, - }; + use std::fmt::{Display, Formatter}; use async_trait::async_trait; use axum::extract::{FromRequest, Query, RequestParts}; use crate::{ ff::FieldType, - helpers::query::{ - ContributionBits, IpaQueryConfig, QueryConfig, QuerySize, QueryType, - SparseAggregateQueryConfig, - }, + helpers::query::{QueryConfig, QuerySize, QueryType}, net::Error, }; @@ -129,71 +123,21 @@ pub mod query { let query_type = match query_type.as_str() { #[cfg(any(test, feature = "cli", feature = "test-fixture"))] QueryType::TEST_MULTIPLY_STR => Ok(QueryType::TestMultiply), - QueryType::SEMIHONEST_IPA_STR | QueryType::MALICIOUS_IPA_STR => { - #[derive(serde::Deserialize)] - struct IPAQueryConfigParam { - per_user_credit_cap: u32, - max_breakdown_key: u32, - attribution_window_seconds: Option, - num_multi_bits: u32, - #[serde(default)] - plaintext_match_keys: bool, - } - let Query(IPAQueryConfigParam { - per_user_credit_cap, - max_breakdown_key, - attribution_window_seconds, - num_multi_bits, - plaintext_match_keys, - }) = req.extract().await?; - - match query_type.as_str() { - QueryType::SEMIHONEST_IPA_STR => { - Ok(QueryType::SemiHonestIpa(IpaQueryConfig { - per_user_credit_cap, - max_breakdown_key, - attribution_window_seconds, - num_multi_bits, - plaintext_match_keys, - })) - } - QueryType::MALICIOUS_IPA_STR => { - Ok(QueryType::MaliciousIpa(IpaQueryConfig { - per_user_credit_cap, - max_breakdown_key, - attribution_window_seconds, - num_multi_bits, - plaintext_match_keys, - })) - } - &_ => unreachable!(), - } + QueryType::SEMIHONEST_IPA_STR => { + let Query(q) = req.extract().await?; + Ok(QueryType::SemiHonestIpa(q)) } - QueryType::SEMIHONEST_AGGREGATE_STR | QueryType::MALICIOUS_AGGREGATE_STR => { - #[derive(serde::Deserialize)] - struct AggregateQueryConfigParam { - contribution_bits: ContributionBits, - num_contributions: u32, - } - let Query(AggregateQueryConfigParam { - contribution_bits, - num_contributions, - }) = req.extract().await?; - match query_type.as_str() { - QueryType::SEMIHONEST_AGGREGATE_STR => Ok( - QueryType::SemiHonestSparseAggregate(SparseAggregateQueryConfig { - contribution_bits, - num_contributions, - }), - ), - QueryType::MALICIOUS_AGGREGATE_STR => Ok( - QueryType::MaliciousSparseAggregate(SparseAggregateQueryConfig { - contribution_bits, - num_contributions, - }), - ), - &_ => unreachable!(), - } + QueryType::MALICIOUS_IPA_STR => { + let Query(q) = req.extract().await?; + Ok(QueryType::MaliciousIpa(q)) + } + QueryType::SEMIHONEST_AGGREGATE_STR => { + let Query(q) = req.extract().await?; + Ok(QueryType::SemiHonestSparseAggregate(q)) + } + QueryType::MALICIOUS_AGGREGATE_STR => { + let Query(q) = req.extract().await?; + Ok(QueryType::MaliciousSparseAggregate(q)) } other => Err(Error::bad_query_value("query_type", other)), }?; From 16378e1e2c4fab70c2d8ca1bca7c926729e552f3 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Wed, 11 Oct 2023 21:12:15 -0700 Subject: [PATCH 032/124] Add sanitizer step --- .github/workflows/check.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5b63b1354..d13a5d2c1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -140,4 +140,19 @@ jobs: - name: Web Tests run: cargo test --no-default-features --features "aggregate-circuit cli web-app real-world-infra test-fixture descriptive-gate" + sanitize: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + sanitizer: [ address, leak, memory, thread ] + steps: + - uses: actions/checkout@v3 + # https://github.com/rust-lang/rust/issues/39699 + - uses: dtolnay/rust-toolchain@nightly + - name: Add rust source + run: rustup component add rust-src + - name: Run tests with sanitizer + run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test --target x86_64-unknown-linux-gnu From 872e32f6c2482c6710dfdfa4bb3a1ad298bdb157 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Wed, 11 Oct 2023 21:22:24 -0700 Subject: [PATCH 033/124] Manually schedule check workflow --- .github/workflows/check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d13a5d2c1..4f41a64dc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -14,6 +14,7 @@ on: - "src/**/*" - "benches/**/*" - "tests/**/*" + workflow_dispatch: env: CARGO_TERM_COLOR: always From 8755577c32c024ba28fc0825742e9eec54fb710b Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Thu, 12 Oct 2023 13:16:26 -0700 Subject: [PATCH 034/124] Fix memory leak in `TempDir` --- tests/common/mod.rs | 5 ++--- tests/common/tempdir.rs | 12 ++++++------ tests/helper_networks.rs | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 7990bccb6..774890b2f 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -188,8 +188,7 @@ pub fn test_multiply(config_dir: &Path, https: bool) { } pub fn test_network(https: bool) { - // set to true to always keep the temp dir after test finishes - let dir = TempDir::new(false); + let dir = TempDir::new_delete_on_drop(); let path = dir.path(); println!("generating configuration in {}", path.display()); @@ -206,7 +205,7 @@ pub fn test_ipa(mode: IpaSecurityModel, https: bool) { pub fn test_ipa_with_config(mode: IpaSecurityModel, https: bool, config: IpaQueryConfig) { const INPUT_SIZE: usize = 10; // set to true to always keep the temp dir after test finishes - let dir = TempDir::new(false); + let dir = TempDir::new_delete_on_drop(); let path = dir.path(); println!("generating configuration in {}", path.display()); diff --git a/tests/common/tempdir.rs b/tests/common/tempdir.rs index f1fd69465..3c056ab44 100644 --- a/tests/common/tempdir.rs +++ b/tests/common/tempdir.rs @@ -1,4 +1,4 @@ -use std::{mem, path::Path, thread}; +use std::{path::Path, thread}; use tempfile::tempdir; @@ -12,16 +12,16 @@ pub struct TempDir { } impl TempDir { - /// Creates a new temporary directory. If `delete` is set to `false`, then it won't be cleaned up - /// after drop. + /// Creates a new temporary directory that will be deleted when this instance is dropped. + /// There is an exception if thread is panicking, then it won't be. /// /// ## Panics /// Panics if a new temp dir cannot be created. #[must_use] - pub fn new(delete: bool) -> Self { + pub fn new_delete_on_drop() -> Self { Self { inner: Some(tempdir().expect("Can create temp directory")), - delete, + delete: true, } } @@ -37,7 +37,7 @@ impl Drop for TempDir { fn drop(&mut self) { if !self.delete || thread::panicking() { let td = self.inner.take().unwrap(); - mem::forget(td); + let _ = td.into_path(); } } } diff --git a/tests/helper_networks.rs b/tests/helper_networks.rs index d00340f75..28ffb6f35 100644 --- a/tests/helper_networks.rs +++ b/tests/helper_networks.rs @@ -39,7 +39,7 @@ fn https_semi_honest_ipa() { #[test] #[cfg(all(test, web_test))] fn keygen_confgen() { - let dir = TempDir::new(false); + let dir = TempDir::new_delete_on_drop(); let path = dir.path(); let sockets: [_; 3] = array::from_fn(|_| TcpListener::bind("127.0.0.1:0").unwrap()); From d8e195de8b9da91dda1305330e9daafd3e33372d Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Thu, 12 Oct 2023 17:12:11 -0700 Subject: [PATCH 035/124] Keep ASAN and Leak sanitizer only Others don't work --- .github/workflows/check.yml | 60 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4f41a64dc..a7eda1177 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -70,13 +70,11 @@ jobs: - name: Run compact gate tests run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate" - extra: - name: Additional Builds and Concurrency Tests - env: - RUSTFLAGS: -D warnings -C target-cpu=native - + release: + name: Release builds and tests runs-on: ubuntu-latest - + env: + RUSTFLAGS: -C target-cpu=native steps: - uses: actions/checkout@v3 @@ -97,34 +95,22 @@ jobs: - name: Release Build run: cargo build --release - - name: Build benchmarks - run: cargo build --benches --no-default-features --features "enable-benches descriptive-gate" - - name: Build concurrency tests run: cargo build --release --features shuttle - - name: Build concurrency tests (debug mode) - run: cargo build --features shuttle - - name: Run concurrency tests run: cargo test --release --features shuttle - - name: Run IPA bench - run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches descriptive-gate" - - - name: Run arithmetic bench - run: cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate" - - - name: Run compact gate tests - run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate" - - aggregate: - name: Build and test aggregate circuit + extra: + name: Additional Builds and Concurrency Tests + env: + RUSTFLAGS: -D warnings -C target-cpu=native runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable with: components: clippy,rustfmt @@ -139,21 +125,35 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} - - name: Web Tests - run: cargo test --no-default-features --features "aggregate-circuit cli web-app real-world-infra test-fixture descriptive-gate" + - name: Build benchmarks + run: cargo build --benches --no-default-features --features "enable-benches descriptive-gate" + + - name: Build concurrency tests (debug mode) + run: cargo build --features shuttle + + - name: Run IPA bench + run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches descriptive-gate" + + - name: Run arithmetic bench + run: cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate" + + - name: Run compact gate tests + run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate" + + # sanitizers currently require nightly https://github.com/rust-lang/rust/issues/39699 sanitize: runs-on: ubuntu-latest - timeout-minutes: 10 strategy: fail-fast: false matrix: - sanitizer: [ address, leak, memory, thread ] + sanitizer: [ address, leak ] + env: + TARGET: x86_64-unknown-linux-gnu steps: - uses: actions/checkout@v3 - # https://github.com/rust-lang/rust/issues/39699 - uses: dtolnay/rust-toolchain@nightly - - name: Add rust source + - name: Add Rust sources run: rustup component add rust-src - name: Run tests with sanitizer - run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test --target x86_64-unknown-linux-gnu + run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test -Z build-std --target $TARGET --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate" From 36b9c7dbe00907d40a27a2a219c923a3b038a1dd Mon Sep 17 00:00:00 2001 From: Martin Thomson Date: Mon, 16 Oct 2023 13:45:04 +1100 Subject: [PATCH 036/124] Restructure the bucketing code Move the function to a file on its own. Skip iterations when the number of buckets is less than half the number of bits and `robust` is false. Refactor tests. Add an overflow test and guard against having more than 128 buckets (which we can't support). Fix gen_range() invocations. Closes #803. --- Cargo.toml | 2 +- src/ff/prime_field.rs | 12 + src/protocol/prf_sharding/bucket.rs | 234 ++++++++++++++++++ .../prf_sharding/feature_label_dot_product.rs | 10 +- src/protocol/prf_sharding/mod.rs | 181 +------------- 5 files changed, 261 insertions(+), 178 deletions(-) create mode 100644 src/protocol/prf_sharding/bucket.rs diff --git a/Cargo.toml b/Cargo.toml index e264d503f..2ba6d8b67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,7 +106,7 @@ cfg_aliases = "0.1.1" command-fds = "0.2.2" hex = "0.4" permutation = "0.4.1" -proptest = "1.0.0" +proptest = "1.3" tempfile = "3" tokio-rustls = { version = "0.24.0", features = ["dangerous_configuration"] } diff --git a/src/ff/prime_field.rs b/src/ff/prime_field.rs index 49962a83d..56391a8a0 100644 --- a/src/ff/prime_field.rs +++ b/src/ff/prime_field.rs @@ -132,6 +132,18 @@ macro_rules! field_impl { } } + impl std::iter::Sum for $field { + fn sum>(iter: I) -> Self { + iter.fold(Self::ZERO, |a, b| a + b) + } + } + + impl<'a> std::iter::Sum<&'a $field> for $field { + fn sum>(iter: I) -> Self { + iter.fold(Self::ZERO, |a, b| a + *b) + } + } + impl TryFrom for $field { type Error = crate::error::Error; diff --git a/src/protocol/prf_sharding/bucket.rs b/src/protocol/prf_sharding/bucket.rs new file mode 100644 index 000000000..6349f91a6 --- /dev/null +++ b/src/protocol/prf_sharding/bucket.rs @@ -0,0 +1,234 @@ +use embed_doc_image::embed_doc_image; + +use crate::{ + error::Error, + ff::{GaloisField, PrimeField, Serializable}, + protocol::{ + basics::SecureMul, context::UpgradedContext, prf_sharding::BinaryTreeDepthStep, + step::BitOpStep, RecordId, + }, + secret_sharing::{ + replicated::malicious::ExtendableField, BitDecomposed, Linear as LinearSecretSharing, + }, +}; + +#[embed_doc_image("tree-aggregation", "images/tree_aggregation.png")] +/// This function moves a single value to a correct bucket using tree aggregation approach +/// +/// Here is how it works +/// The combined value, [`value`] forms the root of a binary tree as follows: +/// ![Tree propagation][tree-aggregation] +/// +/// This value is propagated through the tree, with each subsequent iteration doubling the number of multiplications. +/// In the first round, r=BK-1, multiply the most significant bit ,[`bd_key`]_r by the value to get [`bd_key`]_r.[`value`]. From that, +/// produce [`row_contribution`]_r,0 =[`value`]-[`bd_key`]_r.[`value`] and [`row_contribution`]_r,1=[`bd_key`]_r.[`value`]. +/// This takes the most significant bit of `bd_key` and places value in one of the two child nodes of the binary tree. +/// At each successive round, the next most significant bit is propagated from the leaf nodes of the tree into further leaf nodes: +/// [`row_contribution`]_r+1,q,0 =[`row_contribution`]_r,q - [`bd_key`]_r+1.[`row_contribution`]_r,q and [`row_contribution`]_r+1,q,1 =[`bd_key`]_r+1.[`row_contribution`]_r,q. +/// The work of each iteration therefore doubles relative to the one preceding. +/// +/// In case a malicious entity sends a out of range breakdown key (i.e. greater than the max count) to this function, we need to do some +/// extra processing to ensure contribution doesn't end up in a wrong bucket. However, this requires extra multiplications. +/// This would potentially not be needed in IPA (as the breakdown key is provided by the report collector, so a bad value only spoils their own result) but useful for PAM. +/// This can be by passing `robust` as true. +pub async fn move_single_value_to_bucket( + ctx: C, + record_id: RecordId, + bd_key: BitDecomposed, + value: S, + breakdown_count: usize, + robust: bool, +) -> Result, Error> +where + BK: GaloisField, + C: UpgradedContext, + S: LinearSecretSharing + Serializable + SecureMul, + F: PrimeField + ExtendableField, +{ + let mut step: usize = 1 << BK::BITS; + + assert!( + breakdown_count <= 1 << BK::BITS, + "Asking for more buckets ({breakdown_count}) than bits in the key ({}) allow", + BK::BITS + ); + assert!( + breakdown_count <= 128, + "Our step implementation (BitOpStep) cannot go past 64" + ); + let mut row_contribution = vec![value; breakdown_count]; + + for (tree_depth, bit_of_bdkey) in bd_key.iter().enumerate().rev() { + let span = step >> 1; + if !robust && span > breakdown_count { + step = span; + continue; + } + + let depth_c = ctx.narrow(&BinaryTreeDepthStep::from(tree_depth)); + let mut futures = Vec::with_capacity(breakdown_count / step); + + for (i, tree_index) in (0..breakdown_count).step_by(step).enumerate() { + let bit_c = depth_c.narrow(&BitOpStep::from(i)); + + if robust || tree_index + span < breakdown_count { + futures.push(row_contribution[tree_index].multiply(bit_of_bdkey, bit_c, record_id)); + } + } + let contributions = ctx.parallel_join(futures).await?; + + for (index, bdbit_contribution) in contributions.into_iter().enumerate() { + let left_index = index * step; + let right_index = left_index + span; + + row_contribution[left_index] -= &bdbit_contribution; + if right_index < breakdown_count { + row_contribution[right_index] = bdbit_contribution; + } + } + step = span; + } + Ok(row_contribution) +} + +#[cfg(all(test, unit_test))] +pub mod tests { + use rand::thread_rng; + + use crate::{ + ff::{Field, Fp32BitPrime, Gf5Bit, Gf8Bit}, + protocol::{ + context::{Context, UpgradableContext, Validator}, + prf_sharding::bucket::move_single_value_to_bucket, + RecordId, + }, + rand::Rng, + secret_sharing::SharedValue, + test_executor::run, + test_fixture::{get_bits, Reconstruct, Runner, TestWorld}, + }; + + const MAX_BREAKDOWN_COUNT: usize = 1 << Gf5Bit::BITS; + const VALUE: u32 = 10; + + async fn move_to_bucket(count: usize, breakdown_key: usize, robust: bool) -> Vec { + let breakdown_key_bits = + get_bits::(breakdown_key.try_into().unwrap(), Gf5Bit::BITS); + let value = Fp32BitPrime::truncate_from(VALUE); + + TestWorld::default() + .semi_honest( + (breakdown_key_bits, value), + |ctx, (breakdown_key_share, value_share)| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + move_single_value_to_bucket::( + ctx.set_total_records(1), + RecordId::from(0), + breakdown_key_share, + value_share, + count, + robust, + ) + .await + .unwrap() + }, + ) + .await + .reconstruct() + } + + #[test] + fn semi_honest_move_in_range() { + run(|| async move { + let mut rng = thread_rng(); + let count = rng.gen_range(1..MAX_BREAKDOWN_COUNT); + let breakdown_key = rng.gen_range(0..count); + let mut expected = vec![Fp32BitPrime::ZERO; count]; + expected[breakdown_key] = Fp32BitPrime::truncate_from(VALUE); + + let result = move_to_bucket(count, breakdown_key, false).await; + assert_eq!(result, expected, "expected value at index {breakdown_key}"); + }); + } + + #[test] + fn semi_honest_move_in_range_robust() { + run(|| async move { + let mut rng = thread_rng(); + let count = rng.gen_range(1..MAX_BREAKDOWN_COUNT); + let breakdown_key = rng.gen_range(0..count); + let mut expected = vec![Fp32BitPrime::ZERO; count]; + expected[breakdown_key] = Fp32BitPrime::truncate_from(VALUE); + + let result = move_to_bucket(count, breakdown_key, true).await; + assert_eq!(result, expected, "expected value at index {breakdown_key}"); + }); + } + + #[test] + fn semi_honest_move_out_of_range() { + run(move || async move { + let mut rng: rand::rngs::ThreadRng = thread_rng(); + let count = rng.gen_range(2..MAX_BREAKDOWN_COUNT - 1); + let breakdown_key = rng.gen_range(count..MAX_BREAKDOWN_COUNT); + + let result = move_to_bucket(count, breakdown_key, false).await; + assert_eq!(result.len(), count); + assert_eq!( + result.into_iter().sum::(), + Fp32BitPrime::truncate_from(VALUE) + ); + }); + } + + #[test] + fn semi_honest_move_out_of_range_robust() { + run(move || async move { + let mut rng: rand::rngs::ThreadRng = thread_rng(); + let count = rng.gen_range(2..MAX_BREAKDOWN_COUNT - 1); + let breakdown_key = rng.gen_range(count..MAX_BREAKDOWN_COUNT); + + let result = move_to_bucket(count, breakdown_key, true).await; + assert_eq!(result.len(), count); + assert!(result.into_iter().all(|x| x == Fp32BitPrime::ZERO)); + }); + } + + #[test] + #[should_panic] + fn move_out_of_range_too_many_buckets_type() { + run(move || async move { + _ = move_to_bucket(MAX_BREAKDOWN_COUNT + 1, 0, false).await; + }); + } + + #[test] + #[should_panic] + fn move_out_of_range_too_many_buckets_steps() { + run(move || async move { + let breakdown_key_bits = get_bits::(0, Gf8Bit::BITS); + let value = Fp32BitPrime::truncate_from(VALUE); + + _ = TestWorld::default() + .semi_honest( + (breakdown_key_bits, value), + |ctx, (breakdown_key_share, value_share)| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + move_single_value_to_bucket::( + ctx.set_total_records(1), + RecordId::from(0), + breakdown_key_share, + value_share, + 129, + false, + ) + .await + .unwrap() + }, + ) + .await; + }); + } +} diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index 0dfc28f3a..9ef0b0ad6 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -261,10 +261,10 @@ where let num_user_rows = rows_for_user.len(); let contexts = ctx_for_row_number[..num_user_rows - 1].to_owned(); let record_ids = record_id_for_row_depth[..num_user_rows].to_owned(); + record_id_for_row_depth[..num_user_rows] + .iter_mut() + .for_each(|count| *count += 1); - for count in &mut record_id_for_row_depth[..num_user_rows] { - *count += 1; - } #[allow(clippy::async_yields_async)] // this is ok, because seq join wants a stream of futures async move { @@ -273,7 +273,7 @@ where })); // Execute all of the async futures (sequentially), and flatten the result - let flattenned_stream = seq_join(sh_ctx.active_work(), stream_of_per_user_circuits) + let flattened_stream = seq_join(sh_ctx.active_work(), stream_of_per_user_circuits) .flat_map(|x| stream_iter(x.unwrap())); // modulus convert feature vector bits from shares in `Z_2` to shares in `Z_p` @@ -281,7 +281,7 @@ where prime_field_ctx .narrow(&Step::ModulusConvertFeatureVectorBits) .set_total_records(num_outputs), - flattenned_stream, + flattened_stream, 0..FV::BITS, ); diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 8b35b371f..cd589bc02 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -1,21 +1,18 @@ use std::iter::{repeat, zip}; -use embed_doc_image::embed_doc_image; use futures::{stream::iter as stream_iter, TryStreamExt}; use futures_util::{future::try_join, StreamExt}; use ipa_macros::Step; -use super::{ - basics::if_else, boolean::saturating_sum::SaturatingSum, modulus_conversion::convert_bits, - step::BitOpStep, -}; use crate::{ error::Error, ff::{Field, GaloisField, Gf2, PrimeField, Serializable}, protocol::{ - basics::{SecureMul, ShareKnownValue}, - boolean::or::or, + basics::{if_else, SecureMul, ShareKnownValue}, + boolean::{or::or, saturating_sum::SaturatingSum}, context::{UpgradableContext, UpgradedContext, Validator}, + modulus_conversion::convert_bits, + step::BitOpStep, RecordId, }, secret_sharing::{ @@ -28,6 +25,7 @@ use crate::{ seq_join::{seq_join, seq_try_join_all}, }; +pub mod bucket; #[cfg(feature = "descriptive-gate")] pub mod feature_label_dot_product; @@ -635,7 +633,7 @@ where let record_id: RecordId = RecordId::from(i); let bd_key = bk_bits.unwrap(); async move { - move_single_value_to_bucket::( + bucket::move_single_value_to_bucket::( ctx, record_id, bd_key, @@ -662,86 +660,14 @@ where .await } -#[embed_doc_image("tree-aggregation", "images/tree_aggregation.png")] -/// This function moves a single value to a correct bucket using tree aggregation approach -/// -/// Here is how it works -/// The combined value, [`value`] forms the root of a binary tree as follows: -/// ![Tree propagation][tree-aggregation] -/// -/// This value is propagated through the tree, with each subsequent iteration doubling the number of multiplications. -/// In the first round, r=BK-1, multiply the most significant bit ,[`bd_key`]_r by the value to get [`bd_key`]_r.[`value`]. From that, -/// produce [`row_contribution`]_r,0 =[`value`]-[`bd_key`]_r.[`value`] and [`row_contribution`]_r,1=[`bd_key`]_r.[`value`]. -/// This takes the most significant bit of `bd_key` and places value in one of the two child nodes of the binary tree. -/// At each successive round, the next most significant bit is propagated from the leaf nodes of the tree into further leaf nodes: -/// [`row_contribution`]_r+1,q,0 =[`row_contribution`]_r,q - [`bd_key`]_r+1.[`row_contribution`]_r,q and [`row_contribution`]_r+1,q,1 =[`bd_key`]_r+1.[`row_contribution`]_r,q. -/// The work of each iteration therefore doubles relative to the one preceding. -/// -/// In case a malicious entity sends a out of range breakdown key (i.e. greater than the max count) to this function, we need to do some -/// extra processing to ensure contribution doesn't end up in a wrong bucket. However, this requires extra multiplications. -/// This would potentially not be needed in IPA (as aggregation is done after pre-processing which should be able to throw such input) but useful for PAM. -/// This can be by passing `robust_for_breakdown_key_gt_count as true -async fn move_single_value_to_bucket( - ctx: C, - record_id: RecordId, - bd_key: BitDecomposed, - value: S, - breakdown_count: usize, - robust_for_breakdown_key_gt_count: bool, -) -> Result, Error> -where - BK: GaloisField, - C: UpgradedContext, - S: LinearSecretSharing + Serializable + SecureMul, - F: PrimeField + ExtendableField, -{ - let mut step: usize = 1 << BK::BITS; - - assert!(breakdown_count <= 1 << BK::BITS); - let mut row_contribution = vec![value; breakdown_count]; - - for (tree_depth, bit_of_bdkey) in bd_key.iter().rev().enumerate() { - let depth_c = ctx.narrow(&BinaryTreeDepthStep::from(tree_depth)); - let span = step >> 1; - let mut futures = Vec::with_capacity(breakdown_count / step); - - for (i, tree_index) in (0..breakdown_count).step_by(step).enumerate() { - let bit_c = depth_c.narrow(&BitOpStep::from(i)); - - if robust_for_breakdown_key_gt_count || tree_index + span < breakdown_count { - futures.push(row_contribution[tree_index].multiply(bit_of_bdkey, bit_c, record_id)); - } - } - let contributions = ctx.parallel_join(futures).await?; - - for (index, bdbit_contribution) in contributions.into_iter().enumerate() { - let left_index = index * step; - let right_index = left_index + span; - - row_contribution[left_index] -= &bdbit_contribution; - if right_index < breakdown_count { - row_contribution[right_index] = bdbit_contribution; - } - } - step = span; - } - Ok(row_contribution) -} - #[cfg(all(test, unit_test))] pub mod tests { - use rand::thread_rng; - use super::{attribution_and_capping, CappedAttributionOutputs, PrfShardedIpaInputRow}; use crate::{ - ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf3Bit, Gf5Bit, Gf8Bit}, + ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf3Bit, Gf5Bit}, protocol::{ - context::{Context, UpgradableContext, Validator}, - prf_sharding::{ - attribution_and_capping_and_aggregation, do_aggregation, - move_single_value_to_bucket, - }, - RecordId, + context::{UpgradableContext, Validator}, + prf_sharding::{attribution_and_capping_and_aggregation, do_aggregation}, }, rand::Rng, secret_sharing::{ @@ -1070,93 +996,4 @@ pub mod tests { assert_eq!(result, &expected); }); } - - #[test] - fn semi_honest_move_value_to_single_bucket_in_range() { - const MAX_BREAKDOWN_COUNT: usize = 127; - for _ in 1..10 { - run(|| async move { - let world = TestWorld::default(); - let mut rng: rand::rngs::ThreadRng = thread_rng(); - let count = rng.gen_range(1..MAX_BREAKDOWN_COUNT); - let breakdown_key = rng.gen_range(1..count); - - let value: Fp32BitPrime = Fp32BitPrime::truncate_from(10_u128); - let mut expected = vec![Fp32BitPrime::truncate_from(0_u128); count]; - expected[breakdown_key] = value; - - let breakdown_key_bits = - get_bits::(breakdown_key.try_into().unwrap(), Gf8Bit::BITS); - - let result: Vec<_> = world - .semi_honest( - (breakdown_key_bits, value), - |ctx, (breakdown_key_share, value_share)| async move { - let validator = ctx.validator(); - let ctx = validator.context(); - move_single_value_to_bucket::( - ctx.set_total_records(1), - RecordId::from(0), - breakdown_key_share, - value_share, - count, - false, - ) - .await - .unwrap() - }, - ) - .await - .reconstruct(); - assert_eq!(result, expected); - }); - } - } - - #[test] - fn semi_honest_move_value_to_single_bucket_out_of_range() { - const MAX_BREAKDOWN_COUNT: usize = 127; - for robust_for_breakdown_key_gt_count in [true, false] { - for _ in 1..10 { - run(move || async move { - let world = TestWorld::default(); - let mut rng: rand::rngs::ThreadRng = thread_rng(); - let count = rng.gen_range(1..MAX_BREAKDOWN_COUNT); - let breakdown_key = rng.gen_range(count..MAX_BREAKDOWN_COUNT); - - let value: Fp32BitPrime = Fp32BitPrime::truncate_from(10_u128); - let expected = vec![Fp32BitPrime::truncate_from(0_u128); count]; - - let breakdown_key_bits = - get_bits::(breakdown_key.try_into().unwrap(), Gf8Bit::BITS); - - let result: Vec<_> = world - .semi_honest( - (breakdown_key_bits, value), - |ctx, (breakdown_key_share, value_share)| async move { - let validator = ctx.validator(); - let ctx = validator.context(); - move_single_value_to_bucket::( - ctx.set_total_records(1), - RecordId::from(0), - breakdown_key_share, - value_share, - count, - robust_for_breakdown_key_gt_count, - ) - .await - .unwrap() - }, - ) - .await - .reconstruct(); - if robust_for_breakdown_key_gt_count { - assert_eq!(result, expected); - } else { - assert_ne!(result, expected); - } - }); - } - } - } } From 05626b1975743c2fab15d700d26365788d82cc5c Mon Sep 17 00:00:00 2001 From: Martin Thomson Date: Mon, 16 Oct 2023 14:17:01 +1100 Subject: [PATCH 037/124] Remove proptest bump --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2ba6d8b67..34b4f01d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,7 +106,7 @@ cfg_aliases = "0.1.1" command-fds = "0.2.2" hex = "0.4" permutation = "0.4.1" -proptest = "1.3" +proptest = "1" tempfile = "3" tokio-rustls = { version = "0.24.0", features = ["dangerous_configuration"] } From 301a6e56d5d80445fac47370a3c7393a4cc5a887 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 12:14:20 -0700 Subject: [PATCH 038/124] Alex suggestions, serialize, deserialize --- src/ff/curve_points.rs | 3 +-- src/ff/ec_prime_field.rs | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 2b1f60f94..38908dadf 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -33,8 +33,7 @@ impl Serializable for RP25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()]; - buf.copy_from_slice(raw); + *buf.as_mut() = self.0.to_bytes(); } fn deserialize(buf: &GenericArray) -> Self { diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 6613116f6..d7348ca1b 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -43,15 +43,11 @@ impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()]; - buf.copy_from_slice(raw); + *buf.as_mut() = self.0.to_bytes() } fn deserialize(buf: &GenericArray) -> Self { - let mut buf_to = [0u8; 32]; - buf_to[..buf.len()].copy_from_slice(buf); - - Fp25519(Scalar::from_bytes_mod_order(buf_to)) + Fp25519(Scalar::from_bytes_mod_order((*buf).into())) } } From 9a1fa7cbe1040d29e1472a51f96cbdb00fab84db Mon Sep 17 00:00:00 2001 From: danielmasny <46358615+danielmasny@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:19:22 -0700 Subject: [PATCH 039/124] Update src/ff/curve_points.rs Co-authored-by: Alex Koshelev --- src/ff/curve_points.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 38908dadf..bddc71d50 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -119,7 +119,7 @@ impl std::ops::MulAssign for RP25519 { } } -impl From for RP25519 { +impl From<&Scalar> for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) } From 8386d243f0e442c6489f0b5c7e3ce3ea4435f56d Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 12:29:06 -0700 Subject: [PATCH 040/124] Alex suggestions, simplify u32, u64 macros --- src/ff/curve_points.rs | 8 ++++---- src/ff/ec_prime_field.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 38908dadf..0890903a5 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -144,11 +144,11 @@ impl From for CompressedRistretto { } macro_rules! cp_hash_impl { - ( $u_type:ty, $byte_size:literal) => { + ( $u_type:ty) => { impl From for $u_type { fn from(s: RP25519) -> Self { let hk = Hkdf::::new(None, s.0.as_bytes()); - let mut okm = [0u8; $byte_size]; + let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large hk.expand(&[], &mut okm).unwrap(); <$u_type>::from_le_bytes(okm) @@ -167,9 +167,9 @@ macro_rules! cp_hash_impl { }; } -cp_hash_impl!(u64, 8); +cp_hash_impl!(u64); -cp_hash_impl!(u32, 4); +cp_hash_impl!(u32); /// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is /// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index d7348ca1b..9234cb346 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -121,11 +121,11 @@ impl From for Fp25519 { } macro_rules! sc_hash_impl { - ( $u_type:ty, $byte_size:literal) => { + ( $u_type:ty) => { impl From for $u_type { fn from(s: Fp25519) -> Self { let hk = Hkdf::::new(None, s.0.as_bytes()); - let mut okm = [0u8; $byte_size]; + let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large hk.expand(&[], &mut okm).unwrap(); <$u_type>::from_le_bytes(okm) @@ -144,9 +144,9 @@ macro_rules! sc_hash_impl { }; } -sc_hash_impl!(u64, 8); +sc_hash_impl!(u64); -sc_hash_impl!(u32, 4); +sc_hash_impl!(u32); /// Daniel had to implement this since PRSS wants it, prefer not to impl Field for Fp25519 { From 5dc07a070a25670ff332a5fd20907c286abd3eec Mon Sep 17 00:00:00 2001 From: danielmasny <46358615+danielmasny@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:06:29 -0700 Subject: [PATCH 041/124] Update src/ff/curve_points.rs Co-authored-by: Alex Koshelev --- src/ff/curve_points.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index bddc71d50..5ce8bf0bf 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -125,7 +125,7 @@ impl From<&Scalar> for RP25519 { } } -impl From for RP25519 { +impl From<&Fp25519> for RP25519 { fn from(s: Fp25519) -> Self { RP25519(RistrettoPoint::mul_base(&s.into()).compress()) } From 2bbfeec09a9faec07dbe9afdbc9f9dd873495d5a Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Tue, 17 Oct 2023 05:24:26 +0800 Subject: [PATCH 042/124] #[dynamic] attribute takes an integer literal to specify the max number of steps (#799) * #[dynamic] attribute takes an integer literal to specify the maximum number of steps we generate * change the for_each() to for loop --- ipa-macros/src/derive_step/mod.rs | 53 ++++++++++++++++--- src/protocol/attribution/mod.rs | 2 +- src/protocol/context/upgrade.rs | 2 +- .../modulus_conversion/convert_shares.rs | 2 +- .../prf_sharding/feature_label_dot_product.rs | 2 +- src/protocol/prf_sharding/mod.rs | 4 +- src/protocol/sort/apply_sort/shuffle.rs | 2 +- src/protocol/sort/mod.rs | 2 +- src/protocol/step/mod.rs | 4 +- 9 files changed, 57 insertions(+), 16 deletions(-) diff --git a/ipa-macros/src/derive_step/mod.rs b/ipa-macros/src/derive_step/mod.rs index 702c050b2..f2ffc006e 100644 --- a/ipa-macros/src/derive_step/mod.rs +++ b/ipa-macros/src/derive_step/mod.rs @@ -47,6 +47,8 @@ use crate::{ tree::Node, }; +const MAX_DYNAMIC_STEPS: usize = 1024; + trait CaseStyle { fn to_snake_case(&self) -> String; } @@ -113,19 +115,24 @@ fn impl_as_ref(ident: &syn::Ident, data: &syn::DataEnum) -> Result n, + Err(e) => return Err(e), + }; + + // create an array of `num_steps` strings and use the variant index as array index + let steps = (0..num_steps) .map(|i| format!("{}{}", ident_snake_case, i)) .collect::>(); let steps_array_ident = format_ident!("{}_DYNAMIC_STEP", ident_upper_case); const_arrays.extend(quote!( - const #steps_array_ident: [&str; 64] = [#(#steps),*]; + const #steps_array_ident: [&str; #num_steps] = [#(#steps),*]; )); arms.extend(quote!( Self::#ident(i) => #steps_array_ident[usize::try_from(*i).unwrap()], @@ -136,7 +143,7 @@ fn impl_as_ref(ident: &syn::Ident, data: &syn::DataEnum) -> Result #ident_snake_case, )); } - }); + } Ok(quote!( impl AsRef for #ident { @@ -212,7 +219,10 @@ fn get_meta_data_for( .iter() .flat_map(|v| { if is_dynamic_step(v) { - (0..64) + // using `unwrap()` here since we have already validated the + // format in `impl_as_ref()`. + let num_steps = get_dynamic_step_count(v).unwrap(); + (0..num_steps) .map(|i| format!("{}{}", v.ident.to_string().to_snake_case(), i)) .collect::>() } else { @@ -281,3 +291,34 @@ fn get_meta_data_for( fn is_dynamic_step(variant: &syn::Variant) -> bool { variant.attrs.iter().any(|x| x.path().is_ident("dynamic")) } + +/// Returns the number literal argument passed to #[dynamic(...)] attribute. +/// +/// # Errors +/// Returns an error if the argument format is invalid or the number of steps +/// exceeds `MAX_DYNAMIC_STEPS`. The error can be used to generate a compile +/// time error. The function assumes that `is_dynamic_step()` returns true for +/// the given variant. +fn get_dynamic_step_count(variant: &syn::Variant) -> Result { + let dynamic_attr = variant + .attrs + .iter() + .find(|x| x.path().is_ident("dynamic")) + .unwrap(); + let arg = dynamic_attr + .parse_args::() + .map(|x| x.base10_parse::().unwrap()) + .ok(); + match arg { + // guard against gigantic code generation + Some(n) if n <= MAX_DYNAMIC_STEPS => Ok(n), + _ => Err(syn::Error::new_spanned( + dynamic_attr, + format!( + "ipa_macros::step \"dynamic\" attribute expects a number of steps \ + (<= {}) in parentheses: #[dynamic(...)].", + MAX_DYNAMIC_STEPS, + ), + )), + } +} diff --git a/src/protocol/attribution/mod.rs b/src/protocol/attribution/mod.rs index bad651407..d3a117da5 100644 --- a/src/protocol/attribution/mod.rs +++ b/src/protocol/attribution/mod.rs @@ -438,7 +438,7 @@ pub(in crate::protocol) enum Step { #[derive(Step)] pub(crate) enum InteractionPatternStep { - #[dynamic] + #[dynamic(64)] Depth(usize), } diff --git a/src/protocol/context/upgrade.rs b/src/protocol/context/upgrade.rs index c75bd6d75..cb5365d13 100644 --- a/src/protocol/context/upgrade.rs +++ b/src/protocol/context/upgrade.rs @@ -165,7 +165,7 @@ where #[derive(Step)] pub(crate) enum Upgrade2DVectors { - #[dynamic] + #[dynamic(64)] Upgrade2d(usize), } diff --git a/src/protocol/modulus_conversion/convert_shares.rs b/src/protocol/modulus_conversion/convert_shares.rs index 82ad12e7a..406c05fdc 100644 --- a/src/protocol/modulus_conversion/convert_shares.rs +++ b/src/protocol/modulus_conversion/convert_shares.rs @@ -51,7 +51,7 @@ use crate::{ #[derive(Step)] pub(crate) enum ConvertSharesStep { - #[dynamic] + #[dynamic(64)] ConvertBit(u32), Upgrade, Xor1, diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/prf_sharding/feature_label_dot_product.rs index 0dfc28f3a..0043ddc16 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/prf_sharding/feature_label_dot_product.rs @@ -117,7 +117,7 @@ impl InputsRequiredFromPrevRow { #[derive(Step)] pub enum UserNthRowStep { - #[dynamic] + #[dynamic(64)] Row(usize), } diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 8b35b371f..6562e3e5b 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -183,7 +183,7 @@ pub struct CappedAttributionOutputs { #[derive(Step)] pub enum UserNthRowStep { - #[dynamic] + #[dynamic(64)] Row(usize), } @@ -195,7 +195,7 @@ impl From for UserNthRowStep { #[derive(Step)] pub enum BinaryTreeDepthStep { - #[dynamic] + #[dynamic(64)] Depth(usize), } diff --git a/src/protocol/sort/apply_sort/shuffle.rs b/src/protocol/sort/apply_sort/shuffle.rs index 6554db4f1..9268e40dd 100644 --- a/src/protocol/sort/apply_sort/shuffle.rs +++ b/src/protocol/sort/apply_sort/shuffle.rs @@ -18,7 +18,7 @@ use crate::{ #[derive(Step)] pub(crate) enum InnerVectorElementStep { - #[dynamic] + #[dynamic(64)] Elem(usize), } diff --git a/src/protocol/sort/mod.rs b/src/protocol/sort/mod.rs index 7caaf1c91..8427f0b03 100644 --- a/src/protocol/sort/mod.rs +++ b/src/protocol/sort/mod.rs @@ -25,7 +25,7 @@ pub(crate) enum SortStep { Compose, ShuffleRevealPermutation, SortKeys, - #[dynamic] + #[dynamic(64)] MultiApplyInv(u32), } diff --git a/src/protocol/step/mod.rs b/src/protocol/step/mod.rs index 495a6e0e8..fc5fca1ba 100644 --- a/src/protocol/step/mod.rs +++ b/src/protocol/step/mod.rs @@ -53,7 +53,7 @@ impl Step for str {} /// updated with a new step scheme. #[derive(Step)] pub enum BitOpStep { - #[dynamic] + #[dynamic(64)] Bit(usize), } @@ -79,6 +79,6 @@ impl From for BitOpStep { #[derive(Step)] pub(crate) enum IpaProtocolStep { /// Sort shares by the match key - #[dynamic] + #[dynamic(64)] Sort(usize), } From ea8da0d888a48032a0f517eecec7462215779db8 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 15:17:02 -0700 Subject: [PATCH 043/124] simplify serde test --- src/ff/curve_points.rs | 102 ++++++++++----------------------------- src/ff/ec_prime_field.rs | 23 ++++----- 2 files changed, 34 insertions(+), 91 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 76657874a..bf09072ef 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -10,7 +10,7 @@ use typenum::U32; use crate::{ error::Error, - ff::{ec_prime_field::Fp25519, Field, Serializable}, + ff::{ec_prime_field::Fp25519, Serializable}, secret_sharing::{Block, SharedValue}, }; @@ -37,16 +37,7 @@ impl Serializable for RP25519 { } fn deserialize(buf: &GenericArray) -> Self { - RP25519(CompressedRistretto::from_slice(buf).unwrap()) - } -} - -impl rand::distributions::Distribution for rand::distributions::Standard { - fn sample(&self, rng: &mut R) -> RP25519 { - //Fp25519(Scalar::random(rng: &mut R)) - let mut scalar_bytes = [0u8; 64]; - rng.fill_bytes(&mut scalar_bytes); - RP25519(RistrettoPoint::from_uniform_bytes(&scalar_bytes).compress()) + RP25519(CompressedRistretto((*buf).into())) } } @@ -91,7 +82,6 @@ impl std::ops::SubAssign for RP25519 { ///Scalar Multiplication ///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a impl RP25519 { - /// # Errors /// Propagates errors from decompressing invalid curve point pub fn s_mul(self, rhs: Fp25519) -> Result { @@ -119,13 +109,13 @@ impl std::ops::MulAssign for RP25519 { } } -impl From<&Scalar> for RP25519 { +impl From for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) } } -impl From<&Fp25519> for RP25519 { +impl From for RP25519 { fn from(s: Fp25519) -> Self { RP25519(RistrettoPoint::mul_base(&s.into()).compress()) } @@ -154,84 +144,42 @@ macro_rules! cp_hash_impl { <$u_type>::from_le_bytes(okm) } } - - impl From<$u_type> for RP25519 { - fn from(s: $u_type) -> Self { - let hk = Hkdf::::new(None, &s.to_le_bytes()); - let mut okm = [0u8; 32]; - //error invalid length from expand only happens when okm is very large - hk.expand(&[], &mut okm).unwrap(); - RP25519::deserialize(&okm.into()) - } - } }; } +cp_hash_impl!(u128); + cp_hash_impl!(u64); +#[cfg(test)] cp_hash_impl!(u32); -/// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is -/// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed -impl Field for RP25519 { - const ONE: RP25519 = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); - - ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 - /// from or into u128. However it is sufficient to generate random elements in Fp25519 - fn as_u128(&self) -> u128 { - let hk = Hkdf::::new(None, self.0.as_bytes()); - let mut okm = [0u8; 16]; - //error invalid length from expand only happens when okm is very large - hk.expand(&[], &mut okm).unwrap(); - u128::from_le_bytes(okm) - } - - ///PRSS uses `truncate_from function`, we need to expand the u128 using a PRG (Sha256) to a [u8;32] - fn truncate_from>(v: T) -> Self { - let hk = Hkdf::::new(None, &v.into().to_le_bytes()); - let mut okm = [0u8; 32]; - //error invalid length from expand only happens when okm is very large - hk.expand(&[], &mut okm).unwrap(); - RP25519::deserialize(&okm.into()) - } -} - -impl TryFrom for RP25519 { - type Error = crate::error::Error; - - fn try_from(v: u128) -> Result { - let mut bits = [0u8; 32]; - bits[..].copy_from_slice(&v.to_le_bytes()); - let f: RP25519 = RP25519::ONE; - f.serialize((&mut bits).into()); - Ok(f) +#[cfg(test)] +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> RP25519 { + let mut scalar_bytes = [0u8; 64]; + rng.fill_bytes(&mut scalar_bytes); + RP25519(RistrettoPoint::from_uniform_bytes(&scalar_bytes).compress()) } } #[cfg(all(test, unit_test))] mod test { - use curve25519_dalek::scalar::Scalar; + use curve25519_dalek::{constants, scalar::Scalar}; use generic_array::GenericArray; use rand::{thread_rng, Rng}; use typenum::U32; - use crate::{ - ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable, field::Field}, - }; + use crate::ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable}; #[test] fn serde_25519() { - let input: [u8; 32] = [ - 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0x00, 0x00, 0x00, - ]; - let mut output: GenericArray = [0u8; 32].into(); - let a = RP25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32], input); - a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); - assert_eq!(input, output.as_slice()[..32]); + let mut rng = thread_rng(); + let input = rng.gen::(); + let mut a: GenericArray = [0u8; 32].into(); + input.serialize(&mut a); + let output = RP25519::deserialize(&a); + assert_eq!(input, output); } #[test] @@ -240,8 +188,8 @@ mod test { let b: RP25519 = a.into(); let d: Fp25519 = a.into(); let c: RP25519 = RP25519::from(d); - assert_eq!(b, RP25519::ONE); - assert_eq!(c, RP25519::ONE); + assert_eq!(b, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); + assert_eq!(c, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } #[test] @@ -252,13 +200,13 @@ mod test { let fp_c = fp_a + fp_b; let fp_d = RP25519::from(fp_a) + RP25519::from(fp_b); assert_eq!(fp_d, RP25519::from(fp_c)); - assert_ne!(fp_d, RP25519::ONE); + assert_ne!(fp_d, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); let fp_e = rng.gen::(); let fp_f = rng.gen::(); let fp_g = fp_e * fp_f; let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); assert_eq!(fp_h, RP25519::from(fp_g)); - assert_ne!(fp_h, RP25519::ONE); + assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } #[test] diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 9234cb346..e67ffa235 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -2,7 +2,6 @@ use curve25519_dalek::scalar::Scalar; use generic_array::GenericArray; use hkdf::Hkdf; use sha2::Sha256; -//use rand_core::RngCore; use typenum::U32; use crate::{ @@ -23,6 +22,7 @@ impl Fp25519 { //must not use with ZERO #[must_use] pub fn invert(&self) -> Fp25519 { + assert_ne!(*self, Fp25519::ZERO); Fp25519(self.0.invert()) } } @@ -56,8 +56,6 @@ impl rand::distributions::Distribution for rand::distributions::Standar let mut scalar_bytes = [0u8; 32]; rng.fill_bytes(&mut scalar_bytes); Fp25519(Scalar::from_bytes_mod_order(scalar_bytes)) - //not needed since above has sufficiently small bias - //Fp25519(Scalar::from_bytes_mod_order(&scalar_bytes)) } } @@ -144,8 +142,10 @@ macro_rules! sc_hash_impl { }; } +#[cfg(test)] sc_hash_impl!(u64); +#[cfg(test)] sc_hash_impl!(u32); /// Daniel had to implement this since PRSS wants it, prefer not to @@ -198,17 +198,12 @@ mod test { #[test] fn serde_25519() { - let input: [u8; 32] = [ - 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0x00, 0x00, 0x00, - ]; - let mut output: GenericArray = [0u8; 32].into(); - let a = Fp25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32], input); - a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); - assert_eq!(input, output.as_slice()[..32]); + let mut rng = thread_rng(); + let input = rng.gen::(); + let mut a: GenericArray = [0u8; 32].into(); + input.serialize(&mut a); + let output = Fp25519::deserialize(&a); + assert_eq!(input, output); } // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek From bcea6c6fffa98562d02f434431be44b9d8b7fa0e Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 15:21:42 -0700 Subject: [PATCH 044/124] simplify serde test --- src/ff/ec_prime_field.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index e67ffa235..d0ff48b72 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -19,7 +19,8 @@ pub struct Fp25519(::Storage); impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); - //must not use with ZERO + ///# Panics + /// Panics when self is zero #[must_use] pub fn invert(&self) -> Fp25519 { assert_ne!(*self, Fp25519::ZERO); @@ -43,7 +44,7 @@ impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - *buf.as_mut() = self.0.to_bytes() + *buf.as_mut() = self.0.to_bytes(); } fn deserialize(buf: &GenericArray) -> Self { From 8bca6609c4813763f2d9412b79b63773f15bf0ef Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 17:57:38 -0700 Subject: [PATCH 045/124] error recursion limit --- src/ff/curve_points.rs | 8 ++-- src/secret_sharing/mod.rs | 47 ++++++++++++++++--- src/secret_sharing/replicated/mod.rs | 4 +- .../replicated/semi_honest/additive_share.rs | 43 ++++++++--------- src/secret_sharing/scheme.rs | 4 +- 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index bf09072ef..cf182fd69 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -11,7 +11,7 @@ use typenum::U32; use crate::{ error::Error, ff::{ec_prime_field::Fp25519, Serializable}, - secret_sharing::{Block, SharedValue}, + secret_sharing::{Block, WeakSharedValue}, }; impl Block for CompressedRistretto { @@ -20,17 +20,17 @@ impl Block for CompressedRistretto { ///ristretto point for curve 25519 #[derive(Clone, Copy, PartialEq, Debug)] -pub struct RP25519(::Storage); +pub struct RP25519(::Storage); /// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 -impl SharedValue for RP25519 { +impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; const ZERO: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); } impl Serializable for RP25519 { - type Size = <::Storage as Block>::Size; + type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { *buf.as_mut() = self.0.to_bytes(); diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 908a45feb..191ad5df7 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -23,22 +23,33 @@ pub use scheme::{Bitwise, Linear, LinearRefOps, SecretSharing}; use crate::ff::{AddSub, AddSubAssign, Serializable}; +/// Operations supported for weak shared values. +pub trait Additive: +AddSub ++ AddSubAssign ++ Neg +{ +} + +impl Additive for T where + T: AddSub + + AddSubAssign + + Neg +{ +} + /// Operations supported for shared values. pub trait Arithmetic: - AddSub - + AddSubAssign + Additive + Mul + MulAssign - + Neg { } impl Arithmetic for T where - T: AddSub - + AddSubAssign + T: Additive + Mul + MulAssign - + Neg { } @@ -48,6 +59,16 @@ pub trait Block: Sized + Copy + Debug { type Size: ArrayLength; } +pub trait WeakSharedValue: +Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static +{ + type Storage: Block; + + const BITS: u32; + + const ZERO: Self; +} + pub trait SharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Arithmetic + Serializable + 'static { @@ -58,10 +79,22 @@ pub trait SharedValue: const ZERO: Self; } +impl WeakSharedValue for T where + T: SharedValue, +{ + type Storage = T::Storage; + + const BITS: u32 = T::BITS; + + const ZERO: Self = T::ZERO; +} + + + #[cfg(any(test, feature = "test-fixture", feature = "cli"))] impl IntoShares> for V where - V: SharedValue, + V: WeakSharedValue, Standard: Distribution, { fn share_with(self, rng: &mut R) -> [AdditiveShare; 3] { diff --git a/src/secret_sharing/replicated/mod.rs b/src/secret_sharing/replicated/mod.rs index dcf51494e..80c8167b0 100644 --- a/src/secret_sharing/replicated/mod.rs +++ b/src/secret_sharing/replicated/mod.rs @@ -1,9 +1,9 @@ pub mod malicious; pub mod semi_honest; -use super::{SecretSharing, SharedValue}; +use super::{SecretSharing, SharedValue, WeakSharedValue}; -pub trait ReplicatedSecretSharing: SecretSharing { +pub trait ReplicatedSecretSharing: SecretSharing { fn new(a: V, b: V) -> Self; fn left(&self) -> V; fn right(&self) -> V; diff --git a/src/secret_sharing/replicated/semi_honest/additive_share.rs b/src/secret_sharing/replicated/semi_honest/additive_share.rs index cc48cf78b..03f35f647 100644 --- a/src/secret_sharing/replicated/semi_honest/additive_share.rs +++ b/src/secret_sharing/replicated/semi_honest/additive_share.rs @@ -11,31 +11,32 @@ use crate::{ secret_sharing::{ replicated::ReplicatedSecretSharing, Linear as LinearSecretSharing, SecretSharing, SharedValue, + WeakSharedValue, }, }; #[derive(Clone, PartialEq, Eq)] -pub struct AdditiveShare(V, V); +pub struct AdditiveShare(V, V); -impl SecretSharing for AdditiveShare { +impl SecretSharing for AdditiveShare { const ZERO: Self = AdditiveShare::ZERO; } impl LinearSecretSharing for AdditiveShare {} -impl Debug for AdditiveShare { +impl Debug for AdditiveShare { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "({:?}, {:?})", self.0, self.1) } } -impl Default for AdditiveShare { +impl Default for AdditiveShare { fn default() -> Self { AdditiveShare::new(V::ZERO, V::ZERO) } } -impl AdditiveShare { +impl AdditiveShare { /// Replicated secret share where both left and right values are `F::ZERO` pub const ZERO: Self = Self(V::ZERO, V::ZERO); @@ -44,7 +45,7 @@ impl AdditiveShare { } } -impl ReplicatedSecretSharing for AdditiveShare { +impl ReplicatedSecretSharing for AdditiveShare { fn new(a: V, b: V) -> Self { Self(a, b) } @@ -58,7 +59,7 @@ impl ReplicatedSecretSharing for AdditiveShare { } } -impl AdditiveShare +impl AdditiveShare where Self: Serializable, { @@ -73,7 +74,7 @@ where } } -impl<'a, 'b, V: SharedValue> Add<&'b AdditiveShare> for &'a AdditiveShare { +impl<'a, 'b, V: WeakSharedValue> Add<&'b AdditiveShare> for &'a AdditiveShare { type Output = AdditiveShare; fn add(self, rhs: &'b AdditiveShare) -> Self::Output { @@ -81,7 +82,7 @@ impl<'a, 'b, V: SharedValue> Add<&'b AdditiveShare> for &'a AdditiveShare } } -impl Add for AdditiveShare { +impl Add for AdditiveShare { type Output = Self; fn add(self, rhs: Self) -> Self::Output { @@ -89,7 +90,7 @@ impl Add for AdditiveShare { } } -impl Add> for &AdditiveShare { +impl Add> for &AdditiveShare { type Output = AdditiveShare; fn add(self, rhs: AdditiveShare) -> Self::Output { @@ -97,7 +98,7 @@ impl Add> for &AdditiveShare { } } -impl Add<&AdditiveShare> for AdditiveShare { +impl Add<&AdditiveShare> for AdditiveShare { type Output = Self; fn add(self, rhs: &Self) -> Self::Output { @@ -105,20 +106,20 @@ impl Add<&AdditiveShare> for AdditiveShare { } } -impl AddAssign<&Self> for AdditiveShare { +impl AddAssign<&Self> for AdditiveShare { fn add_assign(&mut self, rhs: &Self) { self.0 += rhs.0; self.1 += rhs.1; } } -impl AddAssign for AdditiveShare { +impl AddAssign for AdditiveShare { fn add_assign(&mut self, rhs: Self) { AddAssign::add_assign(self, &rhs); } } -impl Neg for &AdditiveShare { +impl Neg for &AdditiveShare { type Output = AdditiveShare; fn neg(self) -> Self::Output { @@ -126,7 +127,7 @@ impl Neg for &AdditiveShare { } } -impl Neg for AdditiveShare { +impl Neg for AdditiveShare { type Output = Self; fn neg(self) -> Self::Output { @@ -134,7 +135,7 @@ impl Neg for AdditiveShare { } } -impl Sub for &AdditiveShare { +impl Sub for &AdditiveShare { type Output = AdditiveShare; fn sub(self, rhs: Self) -> Self::Output { @@ -142,7 +143,7 @@ impl Sub for &AdditiveShare { } } -impl Sub for AdditiveShare { +impl Sub for AdditiveShare { type Output = Self; fn sub(self, rhs: Self) -> Self::Output { @@ -150,7 +151,7 @@ impl Sub for AdditiveShare { } } -impl Sub<&Self> for AdditiveShare { +impl Sub<&Self> for AdditiveShare { type Output = Self; fn sub(self, rhs: &Self) -> Self::Output { @@ -158,7 +159,7 @@ impl Sub<&Self> for AdditiveShare { } } -impl Sub> for &AdditiveShare { +impl Sub> for &AdditiveShare { type Output = AdditiveShare; fn sub(self, rhs: AdditiveShare) -> Self::Output { @@ -166,14 +167,14 @@ impl Sub> for &AdditiveShare { } } -impl SubAssign<&Self> for AdditiveShare { +impl SubAssign<&Self> for AdditiveShare { fn sub_assign(&mut self, rhs: &Self) { self.0 -= rhs.0; self.1 -= rhs.1; } } -impl SubAssign for AdditiveShare { +impl SubAssign for AdditiveShare { fn sub_assign(&mut self, rhs: Self) { SubAssign::sub_assign(self, &rhs); } diff --git a/src/secret_sharing/scheme.rs b/src/secret_sharing/scheme.rs index cd6556211..4d843f4b8 100644 --- a/src/secret_sharing/scheme.rs +++ b/src/secret_sharing/scheme.rs @@ -3,11 +3,11 @@ use std::{ ops::{Mul, Neg}, }; -use super::SharedValue; +use super::{SharedValue,WeakSharedValue}; use crate::ff::{AddSub, AddSubAssign, GaloisField}; /// Secret sharing scheme i.e. Replicated secret sharing -pub trait SecretSharing: Clone + Debug + Sized + Send + Sync { +pub trait SecretSharing: Clone + Debug + Sized + Send + Sync { const ZERO: Self; } From 64dac775d82ed0769451972c549da8c7475451a8 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Mon, 16 Oct 2023 22:38:49 -0700 Subject: [PATCH 046/124] Fix compiler recursion error --- src/ff/curve_points.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index cf182fd69..3fc51f896 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -13,6 +13,7 @@ use crate::{ ff::{ec_prime_field::Fp25519, Serializable}, secret_sharing::{Block, WeakSharedValue}, }; +use crate::secret_sharing::Additive; impl Block for CompressedRistretto { type Size = U32; @@ -20,7 +21,7 @@ impl Block for CompressedRistretto { ///ristretto point for curve 25519 #[derive(Clone, Copy, PartialEq, Debug)] -pub struct RP25519(::Storage); +pub struct RP25519(CompressedRistretto); /// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 impl WeakSharedValue for RP25519 { From 884aeec2e2136087ffa136bcf799a07dbcbd129c Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:04:59 -0700 Subject: [PATCH 047/124] define WeakSharedValue, adding ipa-prf feature --- Cargo.toml | 4 ++- src/ff/curve_points.rs | 48 +++++++++++++++++------------------ src/helpers/mod.rs | 4 +-- src/protocol/basics/reveal.rs | 4 +-- src/protocol/mod.rs | 1 + src/protocol/prf_eval/mod.rs | 2 +- 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 11e15fdc7..a1433ecea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,8 @@ default = [ "tracing/max_level_trace", "tracing/release_max_level_info", "descriptive-gate", - "aggregate-circuit" + "aggregate-circuit", + "ipa-prf" ] cli = ["comfy-table", "clap"] enable-serde = ["serde", "serde_json"] @@ -41,6 +42,7 @@ compact-gate = ["ipa-macros/compact-gate"] # Standalone aggregation protocol. We use IPA infra for communication # but it has nothing to do with IPA. aggregate-circuit = [] +ipa-prf = ["descriptive-gate"] [dependencies] ipa-macros = { version = "*", path = "./ipa-macros" } diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 3fc51f896..1094079d4 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -9,11 +9,10 @@ use sha2::Sha256; use typenum::U32; use crate::{ - error::Error, ff::{ec_prime_field::Fp25519, Serializable}, secret_sharing::{Block, WeakSharedValue}, }; -use crate::secret_sharing::Additive; + impl Block for CompressedRistretto { type Size = U32; @@ -23,7 +22,7 @@ impl Block for CompressedRistretto { #[derive(Clone, Copy, PartialEq, Debug)] pub struct RP25519(CompressedRistretto); -/// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 +/// using compressed ristretto point impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; @@ -42,6 +41,10 @@ impl Serializable for RP25519 { } } + +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve impl std::ops::Add for RP25519 { type Output = Self; @@ -57,6 +60,9 @@ impl std::ops::AddAssign for RP25519 { } } +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve impl std::ops::Neg for RP25519 { type Output = Self; @@ -65,6 +71,9 @@ impl std::ops::Neg for RP25519 { } } +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve impl std::ops::Sub for RP25519 { type Output = Self; @@ -82,34 +91,23 @@ impl std::ops::SubAssign for RP25519 { ///Scalar Multiplication ///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a -impl RP25519 { - /// # Errors - /// Propagates errors from decompressing invalid curve point - pub fn s_mul(self, rhs: Fp25519) -> Result { - self.0 - .decompress() - .map_or(Err(Error::DecompressingInvalidCurvePoint), |x| { - Ok((x * Scalar::from(rhs)).compress().into()) - }) - } -} - -///do not use -impl std::ops::Mul for RP25519 { +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve +impl std::ops::Mul< Fp25519> for RP25519 { type Output = Self; - fn mul(self, _rhs: RP25519) -> Self::Output { - panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); + fn mul(self, rhs: Fp25519) -> RP25519 { + (self.0.decompress().unwrap() * Scalar::from(rhs)).compress().into() } } -///do not use -impl std::ops::MulAssign for RP25519 { - fn mul_assign(&mut self, _rhs: RP25519) { - panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); - } +impl std::ops::MulAssign< Fp25519> for RP25519 { + #[allow(clippy::assign_op_pattern)] + fn mul_assign(&mut self, rhs: Fp25519) {*self = *self * rhs} } + impl From for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) @@ -205,7 +203,7 @@ mod test { let fp_e = rng.gen::(); let fp_f = rng.gen::(); let fp_g = fp_e * fp_f; - let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); + let fp_h = RP25519::from(fp_e) * fp_f; assert_eq!(fp_h, RP25519::from(fp_g)); assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 373070736..6e27bbf69 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -40,7 +40,7 @@ use crate::{ Role::{H1, H2, H3}, }, protocol::{step::Gate, RecordId}, - secret_sharing::SharedValue, + secret_sharing::WeakSharedValue, }; // TODO work with ArrayLength only @@ -409,7 +409,7 @@ impl Debug for ChannelId { pub trait Message: Debug + Send + Serializable + 'static + Sized {} /// Any shared value can be send as a message -impl Message for V {} +impl Message for V {} impl Serializable for PublicKey { type Size = typenum::U32; diff --git a/src/protocol/basics/reveal.rs b/src/protocol/basics/reveal.rs index 672c8a473..9fddb65b8 100644 --- a/src/protocol/basics/reveal.rs +++ b/src/protocol/basics/reveal.rs @@ -18,7 +18,7 @@ use crate::{ malicious::{AdditiveShare as MaliciousReplicated, ExtendableField}, semi_honest::AdditiveShare as Replicated, }, - SecretSharing, SharedValue, + SecretSharing, WeakSharedValue, }, }; @@ -47,7 +47,7 @@ pub trait Reveal: Sized { /// i.e. their own shares and received share. #[async_trait] #[embed_doc_image("reveal", "images/reveal.png")] -impl Reveal for Replicated { +impl Reveal for Replicated { type Output = V; async fn reveal<'fut>(&self, ctx: C, record_id: RecordId) -> Result diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 530c12f74..c50a86738 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -6,6 +6,7 @@ pub mod context; pub mod dp; pub mod ipa; pub mod modulus_conversion; +#[cfg(feature = "ipa-prf")] pub mod prf_eval; #[cfg(feature = "descriptive-gate")] pub mod prf_sharding; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index ae43e55a1..3a0a99a21 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -94,7 +94,7 @@ where let z = y.reveal(ctx.narrow(&Step::Revealz), record_id).await?; //compute R^(1/z) to u64 - Ok(u64::from(gr.s_mul(z.invert())?)) + Ok(u64::from(gr * (z.invert()))) } #[cfg(all(test, unit_test))] From a620e347f9de1d01de975e08bc4470caf5fa7348 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:15:12 -0700 Subject: [PATCH 048/124] fix zero --- src/ff/curve_points.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 1094079d4..1d60f934a 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,5 +1,4 @@ use curve25519_dalek::{ - constants, ristretto::{CompressedRistretto, RistrettoPoint}, Scalar, }; @@ -26,7 +25,7 @@ pub struct RP25519(CompressedRistretto); impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; - const ZERO: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); + const ZERO: Self = Self(CompressedRistretto([0_u8;32])); } impl Serializable for RP25519 { @@ -169,7 +168,10 @@ mod test { use rand::{thread_rng, Rng}; use typenum::U32; - use crate::ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable}; + use crate::{ + ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable}, + secret_sharing::WeakSharedValue, + }; #[test] fn serde_25519() { @@ -206,6 +208,7 @@ mod test { let fp_h = RP25519::from(fp_e) * fp_f; assert_eq!(fp_h, RP25519::from(fp_g)); assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); + assert_eq!(RP25519::ZERO,fp_h*Scalar::ZERO.into()); } #[test] From 55cef132834337db90817903017df0ac3c6d9f68 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:16:16 -0700 Subject: [PATCH 049/124] fmt --- src/ff/curve_points.rs | 21 +++++++++-------- src/secret_sharing/mod.rs | 23 ++++++------------- .../replicated/semi_honest/additive_share.rs | 3 +-- src/secret_sharing/scheme.rs | 2 +- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 1d60f934a..99aac7f18 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -12,7 +12,6 @@ use crate::{ secret_sharing::{Block, WeakSharedValue}, }; - impl Block for CompressedRistretto { type Size = U32; } @@ -25,7 +24,7 @@ pub struct RP25519(CompressedRistretto); impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; - const ZERO: Self = Self(CompressedRistretto([0_u8;32])); + const ZERO: Self = Self(CompressedRistretto([0_u8; 32])); } impl Serializable for RP25519 { @@ -40,7 +39,6 @@ impl Serializable for RP25519 { } } - ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve @@ -93,20 +91,23 @@ impl std::ops::SubAssign for RP25519 { ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve -impl std::ops::Mul< Fp25519> for RP25519 { +impl std::ops::Mul for RP25519 { type Output = Self; - fn mul(self, rhs: Fp25519) -> RP25519 { - (self.0.decompress().unwrap() * Scalar::from(rhs)).compress().into() + fn mul(self, rhs: Fp25519) -> RP25519 { + (self.0.decompress().unwrap() * Scalar::from(rhs)) + .compress() + .into() } } -impl std::ops::MulAssign< Fp25519> for RP25519 { +impl std::ops::MulAssign for RP25519 { #[allow(clippy::assign_op_pattern)] - fn mul_assign(&mut self, rhs: Fp25519) {*self = *self * rhs} + fn mul_assign(&mut self, rhs: Fp25519) { + *self = *self * rhs + } } - impl From for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) @@ -208,7 +209,7 @@ mod test { let fp_h = RP25519::from(fp_e) * fp_f; assert_eq!(fp_h, RP25519::from(fp_g)); assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); - assert_eq!(RP25519::ZERO,fp_h*Scalar::ZERO.into()); + assert_eq!(RP25519::ZERO, fp_h * Scalar::ZERO.into()); } #[test] diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 191ad5df7..1fadc42ef 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -25,31 +25,23 @@ use crate::ff::{AddSub, AddSubAssign, Serializable}; /// Operations supported for weak shared values. pub trait Additive: -AddSub -+ AddSubAssign -+ Neg + AddSub + AddSubAssign + Neg { } impl Additive for T where - T: AddSub - + AddSubAssign - + Neg + T: AddSub + AddSubAssign + Neg { } /// Operations supported for shared values. pub trait Arithmetic: - Additive - + Mul - + MulAssign + Additive + Mul + MulAssign { } impl Arithmetic for T where - T: Additive - + Mul - + MulAssign + T: Additive + Mul + MulAssign { } @@ -60,7 +52,7 @@ pub trait Block: Sized + Copy + Debug { } pub trait WeakSharedValue: -Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static + Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static { type Storage: Block; @@ -79,7 +71,8 @@ pub trait SharedValue: const ZERO: Self; } -impl WeakSharedValue for T where +impl WeakSharedValue for T +where T: SharedValue, { type Storage = T::Storage; @@ -89,8 +82,6 @@ impl WeakSharedValue for T where const ZERO: Self = T::ZERO; } - - #[cfg(any(test, feature = "test-fixture", feature = "cli"))] impl IntoShares> for V where diff --git a/src/secret_sharing/replicated/semi_honest/additive_share.rs b/src/secret_sharing/replicated/semi_honest/additive_share.rs index 03f35f647..0e3e57bd2 100644 --- a/src/secret_sharing/replicated/semi_honest/additive_share.rs +++ b/src/secret_sharing/replicated/semi_honest/additive_share.rs @@ -10,8 +10,7 @@ use crate::{ ff::Serializable, secret_sharing::{ replicated::ReplicatedSecretSharing, Linear as LinearSecretSharing, SecretSharing, - SharedValue, - WeakSharedValue, + SharedValue, WeakSharedValue, }, }; diff --git a/src/secret_sharing/scheme.rs b/src/secret_sharing/scheme.rs index 4d843f4b8..0d2131eeb 100644 --- a/src/secret_sharing/scheme.rs +++ b/src/secret_sharing/scheme.rs @@ -3,7 +3,7 @@ use std::{ ops::{Mul, Neg}, }; -use super::{SharedValue,WeakSharedValue}; +use super::{SharedValue, WeakSharedValue}; use crate::ff::{AddSub, AddSubAssign, GaloisField}; /// Secret sharing scheme i.e. Replicated secret sharing From 9736b7138cb68efd975a93e67474b7771a8a83cd Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:19:38 -0700 Subject: [PATCH 050/124] fix clippy --- src/ff/curve_points.rs | 2 +- src/ff/ec_prime_field.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 99aac7f18..a084b14a0 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -104,7 +104,7 @@ impl std::ops::Mul for RP25519 { impl std::ops::MulAssign for RP25519 { #[allow(clippy::assign_op_pattern)] fn mul_assign(&mut self, rhs: Fp25519) { - *self = *self * rhs + *self = *self * rhs; } } diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index d0ff48b72..15d5f33a7 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -119,6 +119,7 @@ impl From for Fp25519 { } } +#[cfg(test)] macro_rules! sc_hash_impl { ( $u_type:ty) => { impl From for $u_type { From b0ae43f37887ec908ec1819a45c0621cff25377b Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 18 Oct 2023 11:03:53 -0700 Subject: [PATCH 051/124] refactor prf_ipa in separate module --- src/protocol/ipa_prf/mod.rs | 2 ++ src/protocol/{ => ipa_prf}/prf_eval/mod.rs | 6 +++--- src/protocol/{ => ipa_prf}/prf_sharding/bucket.rs | 4 ++-- .../{ => ipa_prf}/prf_sharding/feature_label_dot_product.rs | 2 +- src/protocol/{ => ipa_prf}/prf_sharding/mod.rs | 2 +- src/protocol/mod.rs | 4 +--- 6 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 src/protocol/ipa_prf/mod.rs rename src/protocol/{ => ipa_prf}/prf_eval/mod.rs (95%) rename src/protocol/{ => ipa_prf}/prf_sharding/bucket.rs (98%) rename src/protocol/{ => ipa_prf}/prf_sharding/feature_label_dot_product.rs (99%) rename src/protocol/{ => ipa_prf}/prf_sharding/mod.rs (99%) diff --git a/src/protocol/ipa_prf/mod.rs b/src/protocol/ipa_prf/mod.rs new file mode 100644 index 000000000..755ef87a5 --- /dev/null +++ b/src/protocol/ipa_prf/mod.rs @@ -0,0 +1,2 @@ +pub mod prf_eval; +pub mod prf_sharding; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs similarity index 95% rename from src/protocol/prf_eval/mod.rs rename to src/protocol/ipa_prf/prf_eval/mod.rs index 3a0a99a21..6d95e40cd 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -9,7 +9,7 @@ use crate::{ prss::SharedRandomness, RecordId, }, - secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, + secret_sharing::replicated::{ReplicatedSecretSharing, semi_honest::AdditiveShare}, seq_join::seq_try_join_all, }; @@ -103,11 +103,11 @@ mod test { use crate::{ ff::{curve_points::RP25519, ec_prime_field::Fp25519}, - protocol::prf_eval::compute_match_key_pseudonym, - secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, + secret_sharing::{IntoShares, replicated::semi_honest::AdditiveShare}, test_executor::run, test_fixture::{Reconstruct, Runner, TestWorld}, }; + use crate::protocol::ipa_prf::prf_eval::compute_match_key_pseudonym; #[derive(Copy, Clone)] struct ShuffledTestInput { diff --git a/src/protocol/prf_sharding/bucket.rs b/src/protocol/ipa_prf/prf_sharding/bucket.rs similarity index 98% rename from src/protocol/prf_sharding/bucket.rs rename to src/protocol/ipa_prf/prf_sharding/bucket.rs index 6349f91a6..d3501e757 100644 --- a/src/protocol/prf_sharding/bucket.rs +++ b/src/protocol/ipa_prf/prf_sharding/bucket.rs @@ -4,7 +4,7 @@ use crate::{ error::Error, ff::{GaloisField, PrimeField, Serializable}, protocol::{ - basics::SecureMul, context::UpgradedContext, prf_sharding::BinaryTreeDepthStep, + basics::SecureMul, context::UpgradedContext, ipa_prf::prf_sharding::BinaryTreeDepthStep, step::BitOpStep, RecordId, }, secret_sharing::{ @@ -99,7 +99,7 @@ pub mod tests { ff::{Field, Fp32BitPrime, Gf5Bit, Gf8Bit}, protocol::{ context::{Context, UpgradableContext, Validator}, - prf_sharding::bucket::move_single_value_to_bucket, + ipa_prf::prf_sharding::bucket::move_single_value_to_bucket, RecordId, }, rand::Rng, diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/ipa_prf/prf_sharding/feature_label_dot_product.rs similarity index 99% rename from src/protocol/prf_sharding/feature_label_dot_product.rs rename to src/protocol/ipa_prf/prf_sharding/feature_label_dot_product.rs index f87f788d8..bc492b62c 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/ipa_prf/prf_sharding/feature_label_dot_product.rs @@ -371,7 +371,7 @@ where pub mod tests { use crate::{ ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf32Bit}, - protocol::prf_sharding::feature_label_dot_product::{ + protocol::ipa_prf::prf_sharding::feature_label_dot_product::{ compute_feature_label_dot_product, PrfShardedIpaInputRow, }, rand::Rng, diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/ipa_prf/prf_sharding/mod.rs similarity index 99% rename from src/protocol/prf_sharding/mod.rs rename to src/protocol/ipa_prf/prf_sharding/mod.rs index 110dc452a..8c1c6b7f2 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/ipa_prf/prf_sharding/mod.rs @@ -667,7 +667,7 @@ pub mod tests { ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf3Bit, Gf5Bit}, protocol::{ context::{UpgradableContext, Validator}, - prf_sharding::{attribution_and_capping_and_aggregation, do_aggregation}, + ipa_prf::prf_sharding::{attribution_and_capping_and_aggregation, do_aggregation}, }, rand::Rng, secret_sharing::{ diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index c50a86738..02c8d8786 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -7,9 +7,7 @@ pub mod dp; pub mod ipa; pub mod modulus_conversion; #[cfg(feature = "ipa-prf")] -pub mod prf_eval; -#[cfg(feature = "descriptive-gate")] -pub mod prf_sharding; +pub mod ipa_prf; pub mod prss; pub mod sort; pub mod step; From da13880cd5d72dfaac8ffdbfb81a4d0d22e37414 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 18 Oct 2023 11:05:41 -0700 Subject: [PATCH 052/124] fmt --- src/protocol/ipa_prf/prf_eval/mod.rs | 6 +++--- src/protocol/mod.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index 6d95e40cd..4da81759c 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -9,7 +9,7 @@ use crate::{ prss::SharedRandomness, RecordId, }, - secret_sharing::replicated::{ReplicatedSecretSharing, semi_honest::AdditiveShare}, + secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, seq_join::seq_try_join_all, }; @@ -103,11 +103,11 @@ mod test { use crate::{ ff::{curve_points::RP25519, ec_prime_field::Fp25519}, - secret_sharing::{IntoShares, replicated::semi_honest::AdditiveShare}, + protocol::ipa_prf::prf_eval::compute_match_key_pseudonym, + secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_executor::run, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use crate::protocol::ipa_prf::prf_eval::compute_match_key_pseudonym; #[derive(Copy, Clone)] struct ShuffledTestInput { diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 02c8d8786..f10df2cf8 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -5,9 +5,9 @@ pub mod boolean; pub mod context; pub mod dp; pub mod ipa; -pub mod modulus_conversion; #[cfg(feature = "ipa-prf")] pub mod ipa_prf; +pub mod modulus_conversion; pub mod prss; pub mod sort; pub mod step; From fc6aba7441472f9f9fb0420e2b9083ffc9d2b363 Mon Sep 17 00:00:00 2001 From: Andy Leiserson Date: Wed, 18 Oct 2023 16:36:56 -0700 Subject: [PATCH 053/124] Update aes crate and enable intrinsics on aarch64 (This used to require the nightly compiler, but no longer does.) --- .cargo/config.toml | 2 ++ Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..cc463d79d --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.aarch64-apple-darwin] +rustflags = ["--cfg", "aes_armv8"] diff --git a/Cargo.toml b/Cargo.toml index 34b4f01d9..9e4314b41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ aggregate-circuit = [] [dependencies] ipa-macros = { version = "*", path = "./ipa-macros" } -aes = "0.8" +aes = "0.8.3" async-trait = "0.1.68" axum = { version = "0.5.17", optional = true, features = ["http2"] } axum-server = { version = "0.5.1", optional = true, features = ["rustls", "rustls-pemfile", "tls-rustls"] } From ecd18856042ffeef6ac0e16a5b0c266e78c2ffbb Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Wed, 4 Oct 2023 10:02:53 +0800 Subject: [PATCH 054/124] Add attribution window to OPRF aggregation --- src/ff/galois_field.rs | 17 +- src/ff/mod.rs | 2 +- src/protocol/boolean/comparison.rs | 6 +- src/protocol/boolean/saturating_sum.rs | 6 +- src/protocol/prf_sharding/mod.rs | 350 ++++++++++++++++++++++--- src/secret_sharing/decomposed.rs | 97 ++++++- 6 files changed, 427 insertions(+), 51 deletions(-) diff --git a/src/ff/galois_field.rs b/src/ff/galois_field.rs index 18b6b58c8..f8a99c3f5 100644 --- a/src/ff/galois_field.rs +++ b/src/ff/galois_field.rs @@ -5,7 +5,7 @@ use std::{ use bitvec::prelude::{bitarr, BitArr, Lsb0}; use generic_array::GenericArray; -use typenum::{Unsigned, U1, U4, U5}; +use typenum::{Unsigned, U1, U3, U4, U5}; use crate::{ ff::{Field, Serializable}, @@ -25,6 +25,7 @@ pub trait GaloisField: // Bit store type definitions type U8_1 = BitArr!(for 8, in u8, Lsb0); +type U8_3 = BitArr!(for 24, in u8, Lsb0); type U8_4 = BitArr!(for 32, in u8, Lsb0); type U8_5 = BitArr!(for 40, in u8, Lsb0); @@ -32,6 +33,10 @@ impl Block for U8_1 { type Size = U1; } +impl Block for U8_3 { + type Size = U3; +} + impl Block for U8_4 { type Size = U4; } @@ -550,6 +555,16 @@ bit_array_impl!( 0b1_0000_0000_0000_0000_0000_0000_1000_1101_u128 ); +bit_array_impl!( + bit_array_20, + Gf20Bit, + U8_3, + 20, + bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + // x^20 + x^7 + x^3 + x^2 + 1 + 0b1000_0000_0000_1000_1101_u128 +); + bit_array_impl!( bit_array_8, Gf8Bit, diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 3a47dc5e0..1d5d3d9c4 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -9,7 +9,7 @@ mod prime_field; use std::ops::{Add, AddAssign, Sub, SubAssign}; pub use field::{Field, FieldType}; -pub use galois_field::{GaloisField, Gf2, Gf32Bit, Gf3Bit, Gf40Bit, Gf5Bit, Gf8Bit}; +pub use galois_field::{GaloisField, Gf2, Gf20Bit, Gf32Bit, Gf3Bit, Gf40Bit, Gf5Bit, Gf8Bit}; use generic_array::{ArrayLength, GenericArray}; #[cfg(any(test, feature = "weak-field"))] pub use prime_field::Fp31; diff --git a/src/protocol/boolean/comparison.rs b/src/protocol/boolean/comparison.rs index 148117854..e7ddbfb7a 100644 --- a/src/protocol/boolean/comparison.rs +++ b/src/protocol/boolean/comparison.rs @@ -3,7 +3,7 @@ use ipa_macros::Step; use super::or::or; use crate::{ error::Error, - ff::PrimeField, + ff::{Field, PrimeField}, protocol::{ boolean::random_bits_generator::RandomBitsGenerator, context::{Context, UpgradedContext}, @@ -205,7 +205,7 @@ pub async fn bitwise_less_than_constant( c: u128, ) -> Result where - F: PrimeField, + F: Field, C: Context, S: LinearSecretSharing + BasicProtocols, for<'a> &'a S: LinearRefOps<'a, S, F>, @@ -236,7 +236,7 @@ async fn first_differing_bit( b: u128, ) -> Result, Error> where - F: PrimeField, + F: Field, C: Context, S: LinearSecretSharing + BasicProtocols, for<'a> &'a S: LinearRefOps<'a, S, F>, diff --git a/src/protocol/boolean/saturating_sum.rs b/src/protocol/boolean/saturating_sum.rs index f57551d98..3ce2d9e4d 100644 --- a/src/protocol/boolean/saturating_sum.rs +++ b/src/protocol/boolean/saturating_sum.rs @@ -24,7 +24,7 @@ impl> SaturatingSum { /// If one of the multiplications errors /// /// # Panics - /// If something try to add a bit decomposed value larger than this `SaturatingSum` can accomodate + /// If something try to add a bit decomposed value larger than this `SaturatingSum` can accommodate pub async fn add( &self, ctx: C, @@ -175,7 +175,9 @@ where /// /// If `compute_carry_out` is set to `true`, then the mutable refernce to `carry_in` is mutated to take on the value of the `carry_out` bit /// -async fn one_bit_subtractor( +/// # Errors +/// If one of the multiplications errors +pub async fn one_bit_subtractor( ctx: C, record_id: RecordId, x: &SB, diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 110dc452a..39ab0c1ac 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -1,15 +1,19 @@ -use std::iter::{repeat, zip}; +use std::{ + iter::{repeat, zip}, + num::NonZeroU32, +}; use futures::{stream::iter as stream_iter, TryStreamExt}; use futures_util::{future::try_join, StreamExt}; use ipa_macros::Step; +use super::boolean::saturating_sum::SaturatingSum; use crate::{ error::Error, ff::{Field, GaloisField, Gf2, PrimeField, Serializable}, protocol::{ basics::{if_else, SecureMul, ShareKnownValue}, - boolean::{or::or, saturating_sum::SaturatingSum}, + boolean::{comparison::bitwise_less_than_constant, or::or}, context::{UpgradableContext, UpgradedContext, Validator}, modulus_conversion::convert_bits, step::BitOpStep, @@ -29,11 +33,32 @@ pub mod bucket; #[cfg(feature = "descriptive-gate")] pub mod feature_label_dot_product; -pub struct PrfShardedIpaInputRow { +pub struct PrfShardedIpaInputRow { prf_of_match_key: u64, is_trigger_bit: Replicated, breakdown_key: Replicated, trigger_value: Replicated, + timestamp: Replicated, +} + +impl PrfShardedIpaInputRow { + pub fn breakdown_key_bits(&self) -> BitDecomposed> { + BitDecomposed::decompose(BK::BITS, |i| { + self.breakdown_key.map(|v| Gf2::truncate_from(v[i])) + }) + } + + pub fn trigger_value_bits(&self) -> BitDecomposed> { + BitDecomposed::decompose(TV::BITS, |i| { + self.trigger_value.map(|v| Gf2::truncate_from(v[i])) + }) + } + + pub fn timestamp_bits(&self) -> BitDecomposed> { + BitDecomposed::decompose(TS::BITS, |i| { + self.timestamp.map(|v| Gf2::truncate_from(v[i])) + }) + } } struct InputsRequiredFromPrevRow { @@ -41,6 +66,7 @@ struct InputsRequiredFromPrevRow { attributed_breakdown_key_bits: BitDecomposed>, saturating_sum: SaturatingSum>, difference_to_cap: BitDecomposed>, + source_event_timestamp: BitDecomposed>, } impl InputsRequiredFromPrevRow { @@ -68,24 +94,27 @@ impl InputsRequiredFromPrevRow { /// - `did_trigger_get_attributed` - a secret-shared bit indicating if this row corresponds to a trigger event /// which was attributed. Might be able to reveal this (after a shuffle and the addition of dummies) to minimize /// the amount of processing work that must be done in the Aggregation stage. - pub async fn compute_row_with_previous( + #[allow(clippy::too_many_lines)] + pub async fn compute_row_with_previous( &mut self, ctx: C, record_id: RecordId, - input_row: &PrfShardedIpaInputRow, + input_row: &PrfShardedIpaInputRow, num_saturating_sum_bits: usize, + attribution_window_seconds: Option, ) -> Result where C: UpgradedContext>, BK: GaloisField, TV: GaloisField, + TS: GaloisField, { - let bd_key = BitDecomposed::decompose(BK::BITS, |i| { - input_row.breakdown_key.map(|v| Gf2::truncate_from(v[i])) - }); - let tv = BitDecomposed::decompose(TV::BITS, |i| { - input_row.trigger_value.map(|v| Gf2::truncate_from(v[i])) - }); + let (bd_key, tv, timestamp) = ( + input_row.breakdown_key_bits(), + input_row.trigger_value_bits(), + input_row.timestamp_bits(), + ); + assert_eq!(self.saturating_sum.sum.len(), num_saturating_sum_bits); let share_of_one = Replicated::share_known_value(&ctx, Gf2::ONE); @@ -108,6 +137,20 @@ impl InputsRequiredFromPrevRow { ) .await?; + let source_event_timestamp = match attribution_window_seconds { + None => self.source_event_timestamp.clone(), + Some(_) => { + timestamp_of_most_recent_source_event( + ctx.narrow(&Step::SourceEventTimestamp), + record_id, + &input_row.is_trigger_bit, + &self.source_event_timestamp, + ×tamp, + ) + .await? + } + }; + let did_trigger_get_attributed = input_row .is_trigger_bit .multiply( @@ -117,7 +160,7 @@ impl InputsRequiredFromPrevRow { ) .await?; - let attributed_trigger_value = zero_out_trigger_value_unless_attributed( + let mut attributed_trigger_value = zero_out_trigger_value_unless_attributed( ctx.narrow(&Step::AttributedTriggerValue), record_id, &did_trigger_get_attributed, @@ -125,6 +168,16 @@ impl InputsRequiredFromPrevRow { ) .await?; + attributed_trigger_value = zero_out_trigger_value_if_outside_attribution_window( + ctx.narrow(&Step::CheckAttributionWindow), + record_id, + &attributed_trigger_value, + attribution_window_seconds, + ×tamp, + &source_event_timestamp, + ) + .await?; + let updated_sum = self .saturating_sum .add( @@ -162,6 +215,7 @@ impl InputsRequiredFromPrevRow { self.attributed_breakdown_key_bits = attributed_breakdown_key_bits.clone(); self.saturating_sum = updated_sum; self.difference_to_cap = difference_to_cap; + self.source_event_timestamp = source_event_timestamp; let outputs_for_aggregation = CappedAttributionOutputs { did_trigger_get_attributed, @@ -210,6 +264,10 @@ pub(crate) enum Step { DidTriggerGetAttributed, AttributedBreakdownKey, AttributedTriggerValue, + CheckAttributionWindow, + ComputeTimeDelta, + ConvertTimeDeltaBits, + SourceEventTimestamp, ComputeSaturatingSum, IsSaturatedAndPrevRowNotSaturated, ComputeDifferenceToCap, @@ -251,14 +309,15 @@ where context_per_row_depth } -fn chunk_rows_by_user( - input_rows: Vec>, -) -> Vec>> +fn chunk_rows_by_user( + input_rows: Vec>, +) -> Vec>> where BK: GaloisField, TV: GaloisField, + TS: GaloisField, { - let mut rows_for_user: Vec> = vec![]; + let mut rows_for_user: Vec> = vec![]; let mut rows_chunked_by_user = vec![]; for row in input_rows { @@ -293,20 +352,23 @@ where /// Propagates errors from multiplications /// # Panics /// Propagates errors from multiplications -pub async fn attribution_and_capping( +pub async fn attribution_and_capping( sh_ctx: C, - input_rows: Vec>, + input_rows: Vec>, num_saturating_sum_bits: usize, + attribution_window_seconds: Option, ) -> Result, Error> where C: UpgradableContext, C::UpgradedContext: UpgradedContext>, BK: GaloisField, TV: GaloisField, + TS: GaloisField, { assert!(num_saturating_sum_bits > TV::BITS as usize); assert!(TV::BITS > 0); assert!(BK::BITS > 0); + assert!(TS::BITS > 0); let rows_chunked_by_user = chunk_rows_by_user(input_rows); let histogram = compute_histogram_of_users_with_row_count(&rows_chunked_by_user); @@ -332,6 +394,7 @@ where .collect(), rows_for_user, num_saturating_sum_bits, + attribution_window_seconds, )); } let outputs_chunked_by_user = seq_try_join_all(sh_ctx.active_work(), futures).await?; @@ -341,16 +404,18 @@ where .collect::>()) } -async fn evaluate_per_user_attribution_circuit( +async fn evaluate_per_user_attribution_circuit( ctx_for_row_number: &[C], record_id_for_each_depth: Vec, - rows_for_user: Vec>, + rows_for_user: Vec>, num_saturating_sum_bits: usize, + attribution_window_seconds: Option, ) -> Result, Error> where C: UpgradedContext>, BK: GaloisField, TV: GaloisField, + TS: GaloisField, { assert!(!rows_for_user.is_empty()); if rows_for_user.len() == 1 { @@ -374,6 +439,7 @@ where record_id_for_this_row_depth, row, num_saturating_sum_bits, + attribution_window_seconds, ) .await?; @@ -387,14 +453,15 @@ where /// Upon encountering the first row of data from a new user (as distinguished by a different OPRF of the match key) /// this function encapsulates the variables that must be initialized. No communication is required for this first row. /// -fn initialize_new_device_attribution_variables( +fn initialize_new_device_attribution_variables( share_of_one: Replicated, - input_row: &PrfShardedIpaInputRow, + input_row: &PrfShardedIpaInputRow, num_saturating_sum_bits: usize, ) -> InputsRequiredFromPrevRow where BK: GaloisField, TV: GaloisField, + TS: GaloisField, { InputsRequiredFromPrevRow { ever_encountered_a_source_event: share_of_one - &input_row.is_trigger_bit, @@ -408,6 +475,7 @@ where // This is incorrect in the case that the CAP is less than the maximum value of "trigger value" for a single row // Not a problem if you assume that's an invalid input difference_to_cap: BitDecomposed::new(vec![Replicated::ZERO; TV::BITS as usize]), + source_event_timestamp: BitDecomposed::new(vec![Replicated::ZERO; TS::BITS as usize]), } } @@ -415,10 +483,6 @@ where /// To support "Last Touch Attribution" we move the `breakdown_key` of the most recent source event /// down to all of trigger events that follow it. /// -/// The logic here is extremely simple. For each row: -/// (a) if it is a source event, take the breakdown key bits. -/// (b) if it is a trigger event, take the breakdown key bits from the preceding line -/// async fn breakdown_key_of_most_recent_source_event( ctx: C, record_id: RecordId, @@ -426,14 +490,60 @@ async fn breakdown_key_of_most_recent_source_event( prev_row_breakdown_key_bits: &BitDecomposed>, cur_row_breakdown_key_bits: &BitDecomposed>, ) -> Result>, Error> +where + C: UpgradedContext>, +{ + field_of_most_recent_source_event( + ctx, + record_id, + is_trigger_bit, + prev_row_breakdown_key_bits, + cur_row_breakdown_key_bits, + ) + .await +} + +/// Same as above but for timestamps +async fn timestamp_of_most_recent_source_event( + ctx: C, + record_id: RecordId, + is_trigger_bit: &Replicated, + prev_row_timestamp_bits: &BitDecomposed>, + cur_row_timestamp_bits: &BitDecomposed>, +) -> Result>, Error> +where + C: UpgradedContext>, +{ + field_of_most_recent_source_event( + ctx, + record_id, + is_trigger_bit, + prev_row_timestamp_bits, + cur_row_timestamp_bits, + ) + .await +} + +/// Move a field of the most recent source event down to all of trigger events that follow it. +/// +/// The logic here is extremely simple. For each row: +/// (a) if it is a source event, take the current field. +/// (b) if it is a trigger event, take the field from the preceding line +async fn field_of_most_recent_source_event( + ctx: C, + record_id: RecordId, + is_trigger_bit: &Replicated, + prev_row_field_bits: &BitDecomposed>, + cur_row_field_bits: &BitDecomposed>, +) -> Result>, Error> where C: UpgradedContext>, { Ok(BitDecomposed::new( ctx.parallel_join( - cur_row_breakdown_key_bits + cur_row_field_bits .iter() - .zip(prev_row_breakdown_key_bits.iter()) + .zip(prev_row_field_bits.iter()) .enumerate() .map(|(i, (cur_bit, prev_bit))| { let c = ctx.narrow(&BitOpStep::from(i)); @@ -480,6 +590,66 @@ where )) } +/// If the `attribution_window_seconds` is `None`, then all trigger events are +/// attributed to the most recent source event. Otherwise, we calculate the time +/// difference between the trigger event and the most recent source event, and +/// only attribute the trigger event if it is within the attribution window. +async fn zero_out_trigger_value_if_outside_attribution_window( + ctx: C, + record_id: RecordId, + attributed_trigger_value: &BitDecomposed>, + attribution_window_seconds: Option, + trigger_event_timestamp: &BitDecomposed>, + source_event_timestamp: &BitDecomposed>, +) -> Result>, Error> +where + C: UpgradedContext>, +{ + if attribution_window_seconds.is_none() { + return Ok(attributed_trigger_value.clone()); + } + + assert_eq!(trigger_event_timestamp.len(), source_event_timestamp.len()); + + let attribution_window_seconds = attribution_window_seconds.unwrap().get(); + let time_delta_bits = trigger_event_timestamp + .sub( + ctx.narrow(&Step::ComputeTimeDelta), + record_id, + source_event_timestamp, + ) + .await?; + + // The result is true if the time delta is `[0, attribution_window_seconds)` + // If we want to include the upper bound, we need to use `bitwise_greater_than_constant()` + // and negate the result. + let is_trigger_event_inside_attribution_window = bitwise_less_than_constant( + ctx.narrow(&Step::ConvertTimeDeltaBits), + record_id, + &time_delta_bits, + u128::from(attribution_window_seconds), + ) + .await?; + + Ok(BitDecomposed::new( + ctx.parallel_join( + attributed_trigger_value + .iter() + .zip(repeat(is_trigger_event_inside_attribution_window)) + .enumerate() + .map(|(i, (trigger_value_bit, is_within_attribution_window))| { + let c = ctx.narrow(&BitOpStep::from(i)); + async move { + trigger_value_bit + .multiply(&is_within_attribution_window, c, record_id) + .await + } + }), + ) + .await?, + )) +} + /// /// To provide a differential privacy guarantee, we need to bound the maximum contribution from any given user to some cap. /// @@ -549,10 +719,11 @@ where /// the results per breakdown key /// # Errors /// If there is an issue in multiplication, it will error -pub async fn attribution_and_capping_and_aggregation( +pub async fn attribution_and_capping_and_aggregation( sh_ctx: C, - input_rows: Vec>, + input_rows: Vec>, num_saturating_sum_bits: usize, + attribution_window_seconds: Option, ) -> Result, Error> where C: UpgradableContext, @@ -562,12 +733,18 @@ where F: PrimeField + ExtendableField, TV: GaloisField, BK: GaloisField, + TS: GaloisField, { let prime_field_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); let prime_field_m_ctx = prime_field_validator.context(); - let user_level_attributions: Vec = - attribution_and_capping(sh_ctx, input_rows, num_saturating_sum_bits).await?; + let user_level_attributions: Vec = attribution_and_capping( + sh_ctx, + input_rows, + num_saturating_sum_bits, + attribution_window_seconds, + ) + .await?; do_aggregation::<_, BK, TV, F, S>(prime_field_m_ctx, user_level_attributions).await } @@ -617,7 +794,7 @@ where 0..TV::BITS, ); - // tranform value bits to large field + // transform value bits to large field let large_field_values = converted_values .map(|val| BitDecomposed::to_additive_sharing_in_large_field_consuming(val.unwrap())); @@ -662,9 +839,11 @@ where #[cfg(all(test, unit_test))] pub mod tests { + use std::num::NonZeroU32; + use super::{attribution_and_capping, CappedAttributionOutputs, PrfShardedIpaInputRow}; use crate::{ - ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf3Bit, Gf5Bit}, + ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf20Bit, Gf3Bit, Gf5Bit}, protocol::{ context::{UpgradableContext, Validator}, prf_sharding::{attribution_and_capping_and_aggregation, do_aggregation}, @@ -678,11 +857,12 @@ pub mod tests { test_fixture::{get_bits, Reconstruct, Runner, TestWorld}, }; - struct PreShardedAndSortedOPRFTestInput { + struct PreShardedAndSortedOPRFTestInput { prf_of_match_key: u64, is_trigger_bit: Gf2, breakdown_key: BK, trigger_value: TV, + timestamp: TS, } fn oprf_test_input( @@ -690,7 +870,23 @@ pub mod tests { is_trigger: bool, breakdown_key: u8, trigger_value: u8, - ) -> PreShardedAndSortedOPRFTestInput { + ) -> PreShardedAndSortedOPRFTestInput { + oprf_test_input_with_timestamp( + prf_of_match_key, + is_trigger, + breakdown_key, + trigger_value, + 0, + ) + } + + fn oprf_test_input_with_timestamp( + prf_of_match_key: u64, + is_trigger: bool, + breakdown_key: u8, + trigger_value: u8, + timestamp: u32, + ) -> PreShardedAndSortedOPRFTestInput { let is_trigger_bit = if is_trigger { Gf2::ONE } else { Gf2::ZERO }; PreShardedAndSortedOPRFTestInput { @@ -698,6 +894,7 @@ pub mod tests { is_trigger_bit, breakdown_key: Gf5Bit::truncate_from(breakdown_key), trigger_value: Gf3Bit::truncate_from(trigger_value), + timestamp: Gf20Bit::truncate_from(timestamp), } } @@ -743,23 +940,27 @@ pub mod tests { capped_attributed_trigger_value: BitDecomposed, } - impl IntoShares> for PreShardedAndSortedOPRFTestInput + impl IntoShares> + for PreShardedAndSortedOPRFTestInput where BK: GaloisField + IntoShares>, TV: GaloisField + IntoShares>, + TS: GaloisField + IntoShares>, { - fn share_with(self, rng: &mut R) -> [PrfShardedIpaInputRow; 3] { + fn share_with(self, rng: &mut R) -> [PrfShardedIpaInputRow; 3] { let PreShardedAndSortedOPRFTestInput { prf_of_match_key, is_trigger_bit, breakdown_key, trigger_value, + timestamp, } = self; let [is_trigger_bit0, is_trigger_bit1, is_trigger_bit2] = is_trigger_bit.share_with(rng); let [breakdown_key0, breakdown_key1, breakdown_key2] = breakdown_key.share_with(rng); let [trigger_value0, trigger_value1, trigger_value2] = trigger_value.share_with(rng); + let [timestamp0, timestamp1, timestamp2] = timestamp.share_with(rng); [ PrfShardedIpaInputRow { @@ -767,18 +968,21 @@ pub mod tests { is_trigger_bit: is_trigger_bit0, breakdown_key: breakdown_key0, trigger_value: trigger_value0, + timestamp: timestamp0, }, PrfShardedIpaInputRow { prf_of_match_key, is_trigger_bit: is_trigger_bit1, breakdown_key: breakdown_key1, trigger_value: trigger_value1, + timestamp: timestamp1, }, PrfShardedIpaInputRow { prf_of_match_key, is_trigger_bit: is_trigger_bit2, breakdown_key: breakdown_key2, trigger_value: trigger_value2, + timestamp: timestamp2, }, ] } @@ -853,11 +1057,11 @@ pub mod tests { } #[test] - fn semi_honest_attribution_and_capping() { + fn semi_honest_attribution_and_capping_no_attribution_window() { run(|| async move { let world = TestWorld::default(); - let records: Vec> = vec![ + let records: Vec> = vec![ /* First User */ oprf_test_input(123, false, 17, 0), oprf_test_input(123, true, 0, 7), @@ -894,10 +1098,69 @@ pub mod tests { let result: Vec<_> = world .semi_honest(records.into_iter(), |ctx, input_rows| async move { - attribution_and_capping::<_, Gf5Bit, Gf3Bit>( + attribution_and_capping::<_, Gf5Bit, Gf3Bit, Gf20Bit>( + ctx, + input_rows, + num_saturating_bits, + None, + ) + .await + .unwrap() + }) + .await + .reconstruct(); + assert_eq!(result, &expected); + }); + } + + #[test] + fn semi_honest_with_attribution_window() { + run(|| async move { + let world = TestWorld::default(); + + let records: Vec> = vec![ + /* First User */ + oprf_test_input_with_timestamp(123, false, 17, 0, 0), + oprf_test_input_with_timestamp(123, true, 0, 7, 100), + oprf_test_input_with_timestamp(123, false, 20, 0, 200), + oprf_test_input_with_timestamp(123, true, 0, 3, 300), + /* Second User */ + oprf_test_input_with_timestamp(234, false, 12, 0, 0), + oprf_test_input_with_timestamp(234, true, 0, 5, 100), + /* Third User */ + oprf_test_input_with_timestamp(345, false, 20, 0, 0), + oprf_test_input_with_timestamp(345, true, 0, 7, 100), + oprf_test_input_with_timestamp(345, false, 18, 0, 200), + oprf_test_input_with_timestamp(345, false, 12, 0, 300), + oprf_test_input_with_timestamp(345, true, 0, 7, 400), + oprf_test_input_with_timestamp(345, true, 0, 7, 499), + // all the following events are ignored if the attribution window is <= 200s + oprf_test_input_with_timestamp(345, true, 0, 7, 600), + oprf_test_input_with_timestamp(345, true, 0, 7, 700), + ]; + + let expected: [PreAggregationTestOutputInDecimal; 11] = [ + decimal_bd_key_and_value(17, 7), + decimal_bd_key_and_value(20, 0), + decimal_bd_key_and_value(20, 3), + decimal_bd_key_and_value(12, 5), + decimal_bd_key_and_value(20, 7), + decimal_bd_key_and_value(18, 0), + decimal_bd_key_and_value(12, 0), + decimal_bd_key_and_value(12, 7), + decimal_bd_key_and_value(12, 7), + decimal_bd_key_and_value(12, 0), + decimal_bd_key_and_value(12, 0), + ]; + let num_saturating_bits: usize = 5; + + let result: Vec<_> = world + .semi_honest(records.into_iter(), |ctx, input_rows| async move { + attribution_and_capping::<_, Gf5Bit, Gf3Bit, Gf20Bit>( ctx, input_rows, num_saturating_bits, + Some(NonZeroU32::new(200).unwrap()), ) .await .unwrap() @@ -913,7 +1176,7 @@ pub mod tests { run(|| async move { let world = TestWorld::default(); - let records: Vec> = vec![ + let records: Vec> = vec![ /* First User */ oprf_test_input(123, false, 17, 0), oprf_test_input(123, true, 0, 7), @@ -946,10 +1209,11 @@ pub mod tests { _, Gf5Bit, Gf3Bit, + Gf20Bit, Fp32BitPrime, _, Replicated, - >(ctx, input_rows, num_saturating_bits) + >(ctx, input_rows, num_saturating_bits, None) .await .unwrap() }) diff --git a/src/secret_sharing/decomposed.rs b/src/secret_sharing/decomposed.rs index 6a3f7d28d..c3be6be81 100644 --- a/src/secret_sharing/decomposed.rs +++ b/src/secret_sharing/decomposed.rs @@ -2,7 +2,11 @@ use std::{fmt::Debug, ops::Deref}; use crate::{ error::Error, - ff::PrimeField, + ff::{Field, Gf2, PrimeField}, + protocol::{ + boolean::saturating_sum::one_bit_subtractor, context::Context, step::BitOpStep, + BasicProtocols, RecordId, + }, secret_sharing::{Linear as LinearSecretSharing, LinearRefOps}, }; @@ -79,6 +83,46 @@ impl BitDecomposed { acc + (b * F::truncate_from(1_u128 << i)) }) } + + /// Subtraction of two bit-wise secret shares, `self - rhs`, in two's complement. + /// Subtracting a value larger than `self` value will cause the result to overflow. + /// Be especially careful as the overflow is not checked and could lead to a privacy + /// violation (e.g., invalid capping). + /// + /// # Errors + /// If one of the multiplications errors + /// # Panics + /// If something try to add a bit decomposed value larger than this `SaturatingSum` can accommodate + pub async fn sub( + &self, + ctx: C, + record_id: RecordId, + rhs: &BitDecomposed, + ) -> Result, Error> + where + C: Context, + S: LinearSecretSharing + BasicProtocols, + for<'a> &'a S: LinearRefOps<'a, S, Gf2>, + { + assert!(self.len() >= rhs.len()); + + let mut output = vec![]; + let mut carry_in = S::share_known_value(&ctx, Gf2::ONE); + let zero = S::ZERO; + for i in 0..self.len() { + let c = ctx.narrow(&BitOpStep::from(i)); + let compute_carry_out = i < self.len() - 1; + let lfs = &self[i]; + let rhs = rhs.get(i).unwrap_or(&zero); + let difference_bit = + one_bit_subtractor(c, record_id, lfs, rhs, &mut carry_in, compute_carry_out) + .await?; + + output.push(difference_bit); + } + + Ok(BitDecomposed::new(output)) + } } impl TryFrom> for BitDecomposed { @@ -106,3 +150,54 @@ impl IntoIterator for BitDecomposed { self.bits.into_iter() } } + +#[cfg(all(test, unit_test))] +mod tests { + use crate::{ + ff::Gf2, + protocol::{context::Context, RecordId}, + secret_sharing::BitDecomposed, + test_fixture::{get_bits, Reconstruct, Runner, TestWorld}, + }; + + #[tokio::test] + pub async fn subtraction() { + // `lhs >= rhs` + assert_eq!(0, subtract(1, 2, 1, 2).await); + assert_eq!(1, subtract(2, 2, 1, 2).await); + assert_eq!(1, subtract(3, 2, 2, 2).await); + assert_eq!(2, subtract(3, 2, 1, 2).await); + assert_eq!(3, subtract(3, 2, 0, 2).await); + assert_eq!(2, subtract(3, 5, 1, 5).await); + assert_eq!(6, subtract(7, 5, 1, 2).await); + assert_eq!(6, subtract(7, 5, 1, 5).await); + + // `lhs < rhs` so the result is an unsigned integer in two's complement + // representation in whatever many bits of the `lhs` + assert_eq!(3, subtract(0, 2, 1, 2).await); + assert_eq!(31, subtract(1, 5, 2, 2).await); + assert_eq!(30, subtract(1, 5, 3, 2).await); + assert_eq!(26, subtract(1, 5, 7, 5).await); + } + + async fn subtract(a: u32, num_a_bits: u32, b: u32, num_b_bits: u32) -> u128 { + let world = TestWorld::default(); + + let a_bits = get_bits::(a, num_a_bits); + let b_bits = get_bits::(b, num_b_bits); + + let foo = world + .semi_honest( + (a_bits, b_bits), + |ctx, (a_bits, b_bits): (BitDecomposed<_>, BitDecomposed<_>)| async move { + a_bits + .sub(ctx.set_total_records(1), RecordId::from(0), &b_bits) + .await + .unwrap() + }, + ) + .await; + + foo.reconstruct() + } +} From 141a6b16898bc94edde8657834a36daab31c737e Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Thu, 19 Oct 2023 17:21:59 +0800 Subject: [PATCH 055/124] Optimize by removing redundant multiplications --- src/protocol/prf_sharding/mod.rs | 147 +++++++++++++++---------------- src/secret_sharing/decomposed.rs | 6 +- 2 files changed, 75 insertions(+), 78 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 39ab0c1ac..649c36a9e 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -4,7 +4,10 @@ use std::{ }; use futures::{stream::iter as stream_iter, TryStreamExt}; -use futures_util::{future::try_join, StreamExt}; +use futures_util::{ + future::{try_join, try_join3}, + StreamExt, +}; use ipa_macros::Step; use super::boolean::saturating_sum::SaturatingSum; @@ -120,7 +123,11 @@ impl InputsRequiredFromPrevRow { let share_of_one = Replicated::share_known_value(&ctx, Gf2::ONE); let is_source_event = &share_of_one - &input_row.is_trigger_bit; - let (ever_encountered_a_source_event, attributed_breakdown_key_bits) = try_join( + let ( + ever_encountered_a_source_event, + attributed_breakdown_key_bits, + source_event_timestamp, + ) = try_join3( or( ctx.narrow(&Step::EverEncounteredSourceEvent), record_id, @@ -134,50 +141,42 @@ impl InputsRequiredFromPrevRow { &self.attributed_breakdown_key_bits, &bd_key, ), + timestamp_of_most_recent_source_event( + ctx.narrow(&Step::SourceEventTimestamp), + record_id, + attribution_window_seconds, + &input_row.is_trigger_bit, + &self.source_event_timestamp, + ×tamp, + ), ) .await?; - let source_event_timestamp = match attribution_window_seconds { - None => self.source_event_timestamp.clone(), - Some(_) => { - timestamp_of_most_recent_source_event( - ctx.narrow(&Step::SourceEventTimestamp), - record_id, - &input_row.is_trigger_bit, - &self.source_event_timestamp, - ×tamp, - ) - .await? - } - }; - - let did_trigger_get_attributed = input_row - .is_trigger_bit - .multiply( + let (did_trigger_get_attributed, is_trigger_within_window) = try_join( + input_row.is_trigger_bit.multiply( &ever_encountered_a_source_event, ctx.narrow(&Step::DidTriggerGetAttributed), record_id, - ) - .await?; + ), + is_trigger_event_within_attribution_window( + ctx.narrow(&Step::CheckAttributionWindow), + record_id, + attribution_window_seconds, + ×tamp, + &source_event_timestamp, + ), + ) + .await?; - let mut attributed_trigger_value = zero_out_trigger_value_unless_attributed( + let attributed_trigger_value = zero_out_trigger_value_unless_attributed( ctx.narrow(&Step::AttributedTriggerValue), record_id, &did_trigger_get_attributed, + &is_trigger_within_window, &tv, ) .await?; - attributed_trigger_value = zero_out_trigger_value_if_outside_attribution_window( - ctx.narrow(&Step::CheckAttributionWindow), - record_id, - &attributed_trigger_value, - attribution_window_seconds, - ×tamp, - &source_event_timestamp, - ) - .await?; - let updated_sum = self .saturating_sum .add( @@ -264,9 +263,10 @@ pub(crate) enum Step { DidTriggerGetAttributed, AttributedBreakdownKey, AttributedTriggerValue, + AttributedEventCheckFlag, CheckAttributionWindow, ComputeTimeDelta, - ConvertTimeDeltaBits, + CompareTimeDeltaToAttributionWindow, SourceEventTimestamp, ComputeSaturatingSum, IsSaturatedAndPrevRowNotSaturated, @@ -503,10 +503,12 @@ where .await } -/// Same as above but for timestamps +/// Same as above but for timestamps. If `attribution_window_seconds` is `None`, just +/// return the previous row's timestamp. The bits aren't used but saves some multiplications. async fn timestamp_of_most_recent_source_event( ctx: C, record_id: RecordId, + attribution_window_seconds: Option, is_trigger_bit: &Replicated, prev_row_timestamp_bits: &BitDecomposed>, cur_row_timestamp_bits: &BitDecomposed>, @@ -514,14 +516,19 @@ async fn timestamp_of_most_recent_source_event( where C: UpgradedContext>, { - field_of_most_recent_source_event( - ctx, - record_id, - is_trigger_bit, - prev_row_timestamp_bits, - cur_row_timestamp_bits, - ) - .await + match attribution_window_seconds { + None => Ok(prev_row_timestamp_bits.clone()), + Some(_) => { + field_of_most_recent_source_event( + ctx, + record_id, + is_trigger_bit, + prev_row_timestamp_bits, + cur_row_timestamp_bits, + ) + .await + } + } } /// Move a field of the most recent source event down to all of trigger events that follow it. @@ -559,29 +566,36 @@ where /// (a) Attributed to a single `breakdown_key` /// (b) Not attributed, and thus zeroed out /// -/// The logic here is extremely simple. There is a secret-shared bit indicating if a given row is an "attributed trigger event" -/// The bits of the `trigger_value` are all multiplied by this bit in order to zero out contributions from unattributed trigger events +/// The logic here is extremely simple. There is a secret-shared bit indicating if a given row is an "attributed trigger event" and +/// another secret-shared bit indicating if a given row is within the attribution window. We multiply these two bits together and +/// multiply it with the bits of the `trigger_value` in order to zero out contributions from unattributed trigger events. /// async fn zero_out_trigger_value_unless_attributed( ctx: C, record_id: RecordId, did_trigger_get_attributed: &Replicated, + is_trigger_within_window: &Replicated, trigger_value: &BitDecomposed>, ) -> Result>, Error> where C: UpgradedContext>, { + let c = ctx.narrow(&Step::AttributedEventCheckFlag); + let zero_out_flag = &did_trigger_get_attributed + .multiply(is_trigger_within_window, c, record_id) + .await?; + Ok(BitDecomposed::new( ctx.parallel_join( trigger_value .iter() - .zip(repeat(did_trigger_get_attributed)) + .zip(repeat(zero_out_flag)) .enumerate() - .map(|(i, (trigger_value_bit, did_trigger_get_attributed))| { + .map(|(i, (trigger_value_bit, zero_out_flag))| { let c = ctx.narrow(&BitOpStep::from(i)); async move { trigger_value_bit - .multiply(did_trigger_get_attributed, c, record_id) + .multiply(zero_out_flag, c, record_id) .await } }), @@ -590,23 +604,24 @@ where )) } -/// If the `attribution_window_seconds` is `None`, then all trigger events are -/// attributed to the most recent source event. Otherwise, we calculate the time +/// If the `attribution_window_seconds` is not `None`, we calculate the time /// difference between the trigger event and the most recent source event, and -/// only attribute the trigger event if it is within the attribution window. -async fn zero_out_trigger_value_if_outside_attribution_window( +/// returns a secret-shared bit indicating if the trigger event is within the +/// attribution window. +async fn is_trigger_event_within_attribution_window( ctx: C, record_id: RecordId, - attributed_trigger_value: &BitDecomposed>, attribution_window_seconds: Option, trigger_event_timestamp: &BitDecomposed>, source_event_timestamp: &BitDecomposed>, -) -> Result>, Error> +) -> Result, Error> where C: UpgradedContext>, { + // if there is no attribution window, then all trigger events are attributed if attribution_window_seconds.is_none() { - return Ok(attributed_trigger_value.clone()); + let share_of_one = Replicated::share_known_value(&ctx, Gf2::ONE); + return Ok(share_of_one); } assert_eq!(trigger_event_timestamp.len(), source_event_timestamp.len()); @@ -623,31 +638,13 @@ where // The result is true if the time delta is `[0, attribution_window_seconds)` // If we want to include the upper bound, we need to use `bitwise_greater_than_constant()` // and negate the result. - let is_trigger_event_inside_attribution_window = bitwise_less_than_constant( - ctx.narrow(&Step::ConvertTimeDeltaBits), + bitwise_less_than_constant( + ctx.narrow(&Step::CompareTimeDeltaToAttributionWindow), record_id, &time_delta_bits, u128::from(attribution_window_seconds), ) - .await?; - - Ok(BitDecomposed::new( - ctx.parallel_join( - attributed_trigger_value - .iter() - .zip(repeat(is_trigger_event_inside_attribution_window)) - .enumerate() - .map(|(i, (trigger_value_bit, is_within_attribution_window))| { - let c = ctx.narrow(&BitOpStep::from(i)); - async move { - trigger_value_bit - .multiply(&is_within_attribution_window, c, record_id) - .await - } - }), - ) - .await?, - )) + .await } /// diff --git a/src/secret_sharing/decomposed.rs b/src/secret_sharing/decomposed.rs index c3be6be81..7e9025455 100644 --- a/src/secret_sharing/decomposed.rs +++ b/src/secret_sharing/decomposed.rs @@ -92,7 +92,7 @@ impl BitDecomposed { /// # Errors /// If one of the multiplications errors /// # Panics - /// If something try to add a bit decomposed value larger than this `SaturatingSum` can accommodate + /// If something try to subtract a bit decomposed value larger than this `BitDecomposed` can accommodate pub async fn sub( &self, ctx: C, @@ -112,10 +112,10 @@ impl BitDecomposed { for i in 0..self.len() { let c = ctx.narrow(&BitOpStep::from(i)); let compute_carry_out = i < self.len() - 1; - let lfs = &self[i]; + let lhs = &self[i]; let rhs = rhs.get(i).unwrap_or(&zero); let difference_bit = - one_bit_subtractor(c, record_id, lfs, rhs, &mut carry_in, compute_carry_out) + one_bit_subtractor(c, record_id, lhs, rhs, &mut carry_in, compute_carry_out) .await?; output.push(difference_bit); From 946a9ee211bed9a75d357046d3caa883ba0208ba Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Fri, 20 Oct 2023 12:46:57 +0800 Subject: [PATCH 056/124] save one more multiplication --- src/protocol/prf_sharding/mod.rs | 141 +++++++++++++++---------------- 1 file changed, 66 insertions(+), 75 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 649c36a9e..aec7bee85 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -174,6 +174,7 @@ impl InputsRequiredFromPrevRow { &did_trigger_get_attributed, &is_trigger_within_window, &tv, + attribution_window_seconds, ) .await?; @@ -483,6 +484,9 @@ where /// To support "Last Touch Attribution" we move the `breakdown_key` of the most recent source event /// down to all of trigger events that follow it. /// +/// The logic here is extremely simple. For each row: +/// (a) if it is a source event, take the current `breakdown_key`. +/// (b) if it is a trigger event, take the `breakdown_key` from the preceding line async fn breakdown_key_of_most_recent_source_event( ctx: C, record_id: RecordId, @@ -493,14 +497,19 @@ async fn breakdown_key_of_most_recent_source_event( where C: UpgradedContext>, { - field_of_most_recent_source_event( - ctx, - record_id, - is_trigger_bit, - prev_row_breakdown_key_bits, - cur_row_breakdown_key_bits, - ) - .await + Ok(BitDecomposed::new( + ctx.parallel_join( + cur_row_breakdown_key_bits + .iter() + .zip(prev_row_breakdown_key_bits.iter()) + .enumerate() + .map(|(i, (cur_bit, prev_bit))| { + let c = ctx.narrow(&BitOpStep::from(i)); + async move { if_else(c, record_id, is_trigger_bit, prev_bit, cur_bit).await } + }), + ) + .await?, + )) } /// Same as above but for timestamps. If `attribution_window_seconds` is `None`, just @@ -519,48 +528,26 @@ where match attribution_window_seconds { None => Ok(prev_row_timestamp_bits.clone()), Some(_) => { - field_of_most_recent_source_event( - ctx, - record_id, - is_trigger_bit, - prev_row_timestamp_bits, - cur_row_timestamp_bits, - ) - .await + Ok(BitDecomposed::new( + ctx + .parallel_join( + cur_row_timestamp_bits + .iter() + .zip(prev_row_timestamp_bits.iter()) + .enumerate() + .map(|(i, (cur_bit, prev_bit))| { + let c = ctx.narrow(&BitOpStep::from(i)); + async move { + if_else(c, record_id, is_trigger_bit, prev_bit, cur_bit).await + } + }), + ) + .await?, + )) } } } -/// Move a field of the most recent source event down to all of trigger events that follow it. -/// -/// The logic here is extremely simple. For each row: -/// (a) if it is a source event, take the current field. -/// (b) if it is a trigger event, take the field from the preceding line -async fn field_of_most_recent_source_event( - ctx: C, - record_id: RecordId, - is_trigger_bit: &Replicated, - prev_row_field_bits: &BitDecomposed>, - cur_row_field_bits: &BitDecomposed>, -) -> Result>, Error> -where - C: UpgradedContext>, -{ - Ok(BitDecomposed::new( - ctx.parallel_join( - cur_row_field_bits - .iter() - .zip(prev_row_field_bits.iter()) - .enumerate() - .map(|(i, (cur_bit, prev_bit))| { - let c = ctx.narrow(&BitOpStep::from(i)); - async move { if_else(c, record_id, is_trigger_bit, prev_bit, cur_bit).await } - }), - ) - .await?, - )) -} - /// /// In this simple "Last Touch Attribution" model, the `trigger_value` of a trigger event is either /// (a) Attributed to a single `breakdown_key` @@ -576,14 +563,20 @@ async fn zero_out_trigger_value_unless_attributed( did_trigger_get_attributed: &Replicated, is_trigger_within_window: &Replicated, trigger_value: &BitDecomposed>, + attribution_window_seconds: Option, ) -> Result>, Error> where C: UpgradedContext>, { - let c = ctx.narrow(&Step::AttributedEventCheckFlag); - let zero_out_flag = &did_trigger_get_attributed - .multiply(is_trigger_within_window, c, record_id) - .await?; + // save 1 multiplication if there is no attribution window + let zero_out_flag = if attribution_window_seconds.is_some() { + let c = ctx.narrow(&Step::AttributedEventCheckFlag); + did_trigger_get_attributed + .multiply(is_trigger_within_window, c, record_id) + .await? + } else { + did_trigger_get_attributed.clone() + }; Ok(BitDecomposed::new( ctx.parallel_join( @@ -595,7 +588,7 @@ where let c = ctx.narrow(&BitOpStep::from(i)); async move { trigger_value_bit - .multiply(zero_out_flag, c, record_id) + .multiply(&zero_out_flag, c, record_id) .await } }), @@ -618,33 +611,31 @@ async fn is_trigger_event_within_attribution_window( where C: UpgradedContext>, { - // if there is no attribution window, then all trigger events are attributed - if attribution_window_seconds.is_none() { - let share_of_one = Replicated::share_known_value(&ctx, Gf2::ONE); - return Ok(share_of_one); - } + if let Some(attribution_window_seconds) = attribution_window_seconds { + assert_eq!(trigger_event_timestamp.len(), source_event_timestamp.len()); - assert_eq!(trigger_event_timestamp.len(), source_event_timestamp.len()); + let time_delta_bits = trigger_event_timestamp + .sub( + ctx.narrow(&Step::ComputeTimeDelta), + record_id, + source_event_timestamp, + ) + .await?; - let attribution_window_seconds = attribution_window_seconds.unwrap().get(); - let time_delta_bits = trigger_event_timestamp - .sub( - ctx.narrow(&Step::ComputeTimeDelta), + // The result is true if the time delta is `[0, attribution_window_seconds)` + // If we want to include the upper bound, we need to use `bitwise_greater_than_constant()` + // and negate the result. + bitwise_less_than_constant( + ctx.narrow(&Step::CompareTimeDeltaToAttributionWindow), record_id, - source_event_timestamp, + &time_delta_bits, + u128::from(attribution_window_seconds.get()), ) - .await?; - - // The result is true if the time delta is `[0, attribution_window_seconds)` - // If we want to include the upper bound, we need to use `bitwise_greater_than_constant()` - // and negate the result. - bitwise_less_than_constant( - ctx.narrow(&Step::CompareTimeDeltaToAttributionWindow), - record_id, - &time_delta_bits, - u128::from(attribution_window_seconds), - ) - .await + .await + } else { + // if there is no attribution window, then all trigger events are attributed + Ok(Replicated::share_known_value(&ctx, Gf2::ONE)) + } } /// @@ -1157,7 +1148,7 @@ pub mod tests { ctx, input_rows, num_saturating_bits, - Some(NonZeroU32::new(200).unwrap()), + NonZeroU32::new(200), ) .await .unwrap() From a5d3587aa6ad4e72f3f5db73d5d55c749c91fac5 Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Fri, 20 Oct 2023 12:53:25 +0800 Subject: [PATCH 057/124] inline the variables --- src/secret_sharing/decomposed.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/secret_sharing/decomposed.rs b/src/secret_sharing/decomposed.rs index 7e9025455..1fc87d2c6 100644 --- a/src/secret_sharing/decomposed.rs +++ b/src/secret_sharing/decomposed.rs @@ -112,11 +112,15 @@ impl BitDecomposed { for i in 0..self.len() { let c = ctx.narrow(&BitOpStep::from(i)); let compute_carry_out = i < self.len() - 1; - let lhs = &self[i]; - let rhs = rhs.get(i).unwrap_or(&zero); - let difference_bit = - one_bit_subtractor(c, record_id, lhs, rhs, &mut carry_in, compute_carry_out) - .await?; + let difference_bit = one_bit_subtractor( + c, + record_id, + &self[i], + rhs.get(i).unwrap_or(&zero), + &mut carry_in, + compute_carry_out, + ) + .await?; output.push(difference_bit); } From 3ef3e7e392c3c159fc5546bbd16514d7b429b567 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Thu, 19 Oct 2023 23:54:53 -0700 Subject: [PATCH 058/124] Upgrade dependencies (#811) --- Cargo.toml | 32 +++++++++++-------- src/bin/test_mpc.rs | 2 +- src/cli/keygen.rs | 6 ++-- src/cli/playbook/multiply.rs | 2 +- src/config.rs | 2 +- src/ff/mod.rs | 2 +- src/helpers/transport/stream/input.rs | 10 +++--- src/hpke/mod.rs | 7 ++-- src/hpke/registry.rs | 8 ++--- src/net/client/mod.rs | 5 +-- src/net/server/mod.rs | 22 +++++-------- src/net/test.rs | 2 +- src/protocol/aggregation/input.rs | 2 +- src/protocol/ipa/mod.rs | 2 +- src/protocol/prss/crypto.rs | 2 +- src/report.rs | 2 +- src/secret_sharing/mod.rs | 2 +- .../replicated/malicious/additive_share.rs | 2 +- .../replicated/semi_honest/additive_share.rs | 2 +- 19 files changed, 56 insertions(+), 58 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9e4314b41..1767e8e24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ enable-serde = ["serde", "serde_json"] disable-metrics = [] # TODO move web-app to a separate crate. It adds a lot of build time to people who mostly write protocols # TODO Consider moving out benches as well -web-app = ["axum", "axum-server", "base64", "clap", "comfy-table", "enable-serde", "hyper", "hyper-rustls", "rcgen", "rustls-pemfile", "time", "tokio-rustls", "toml", "tower", "tower-http"] +web-app = ["axum", "axum-server", "base64", "clap", "comfy-table", "enable-serde", "hyper", "hyper-rustls", "rcgen", "rustls", "rustls-pemfile", "time", "tokio-rustls", "toml", "tower", "tower-http"] test-fixture = ["enable-serde", "weak-field"] shuttle = ["shuttle-crate", "test-fixture"] debug-trace = ["tracing/max_level_trace", "tracing/release_max_level_debug"] @@ -43,7 +43,6 @@ compact-gate = ["ipa-macros/compact-gate"] aggregate-circuit = [] [dependencies] -ipa-macros = { version = "*", path = "./ipa-macros" } aes = "0.8.3" async-trait = "0.1.68" axum = { version = "0.5.17", optional = true, features = ["http2"] } @@ -60,13 +59,14 @@ dhat = "0.3.2" embed-doc-image = "0.1.4" futures = "0.3.28" futures-util = "0.3.28" -generic-array = "0.14.7" +generic-array = "1.0.0" hex = { version = "0.4", features = ["serde"] } hkdf = "0.12.3" -hpke = { version = "0.10.0", default-features = false, features = ["std", "x25519-dalek"] } +hpke = { version = "0.11.0", default-features = false, features = ["std", "x25519"] } hyper = { version = "0.14.26", optional = true, features = ["client", "h2", "stream", "runtime"] } -hyper-rustls = { version = "0.24.0", optional = true, features = ["http2"] } +hyper-rustls = { version = "0.24.1", optional = true, features = ["http2"] } iai = { version = "0.1.1", optional = true } +ipa-macros = { version = "*", path = "./ipa-macros" } metrics = "0.21.0" metrics-tracing-context = "0.14.0" metrics-util = { version = "0.15.0" } @@ -74,27 +74,31 @@ once_cell = "1.18" pin-project = "1.0" rand = "0.8" rand_core = "0.6" -rcgen = { version = "0.10", optional = true } -rustls = "0.21.5" +rcgen = { version = "0.11.3", optional = true } +rustls = { version = "0.21", optional = true } rustls-pemfile = { version = "1", optional = true } +# TODO: https://rustsec.org/advisories/RUSTSEC-2023-0053. tokio-rustls and hyper-rustls need to be upgraded first, before +# we can remove pinning +rustls-webpki = "^0.101.4" # TODO consider using zerocopy or serde_bytes or in-house serialization serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } -sha2 = "0.10.6" +sha2 = "0.10" shuttle-crate = { package = "shuttle", version = "0.6.1", optional = true } thiserror = "1.0" time = { version = "0.3", optional = true } tokio = { version = "1.28", features = ["rt", "rt-multi-thread", "macros"] } -tokio-rustls = { version = "0.24.0", optional = true } +# TODO: axum-server holds onto 0.24 and we can't upgrade until they do. Or we move away from axum-server +tokio-rustls = { version = "0.24", optional = true } tokio-stream = "0.1.14" -toml = { version = "0.7", optional = true } +toml = { version = "0.8", optional = true } tower = { version = "0.4.13", optional = true } tower-http = { version = "0.4.0", optional = true, features = ["trace"] } -tracing = "0.1.37" -tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } typenum = "1.16" # hpke is pinned to it -x25519-dalek = "2.0.0-pre.0" +x25519-dalek = "2.0.0-rc.3" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.5.0" @@ -107,8 +111,8 @@ command-fds = "0.2.2" hex = "0.4" permutation = "0.4.1" proptest = "1" +rustls = { version = "0.21", features = ["dangerous_configuration"] } tempfile = "3" -tokio-rustls = { version = "0.24.0", features = ["dangerous_configuration"] } [profile.release] incremental = true diff --git a/src/bin/test_mpc.rs b/src/bin/test_mpc.rs index edee8d82d..ce4172df3 100644 --- a/src/bin/test_mpc.rs +++ b/src/bin/test_mpc.rs @@ -103,7 +103,7 @@ async fn multiply_in_field(args: &Args, helper_clients: &[MpcHelperCli where F: Field + IntoShares>, ::Size: Add<::Size>, - <::Size as Add<::Size>>::Output: ArrayLength, + <::Size as Add<::Size>>::Output: ArrayLength, { let input = InputSource::from(&args.input); let input_rows = input.iter::<(F, F)>().collect::>(); diff --git a/src/cli/keygen.rs b/src/cli/keygen.rs index a32c30ac3..7c8c68fa7 100644 --- a/src/cli/keygen.rs +++ b/src/cli/keygen.rs @@ -9,7 +9,7 @@ use rand::{thread_rng, Rng}; use rand_core::CryptoRng; use rcgen::{ Certificate, CertificateParams, DistinguishedName, ExtendedKeyUsagePurpose, IsCa, - KeyUsagePurpose, SanType, PKCS_ECDSA_P256_SHA256, + KeyUsagePurpose, SanType, SerialNumber, PKCS_ECDSA_P256_SHA256, }; use time::{Duration, OffsetDateTime}; @@ -75,7 +75,9 @@ pub fn keygen_tls(args: &KeygenArgs, rng: &mut R) -> Result< ]; params.not_before = OffsetDateTime::now_utc() - Duration::days(1); params.not_after = params.not_before + Duration::days(91); - params.serial_number = Some(rng.gen_range(0..=i64::MAX.try_into().unwrap())); + params.serial_number = Some(SerialNumber::from( + rng.gen_range(0..=i64::MAX.try_into().unwrap()), + )); let mut name = DistinguishedName::new(); name.push(rcgen::DnType::CommonName, args.name.clone()); diff --git a/src/cli/playbook/multiply.rs b/src/cli/playbook/multiply.rs index caadd8d7b..cc16e1c3c 100644 --- a/src/cli/playbook/multiply.rs +++ b/src/cli/playbook/multiply.rs @@ -27,7 +27,7 @@ pub async fn secure_mul( where F: Field + IntoShares>, ::Size: Add<::Size>, - <::Size as Add<::Size>>::Output: ArrayLength, + <::Size as Add<::Size>>::Output: ArrayLength, { // prepare inputs let inputs = input.into_iter().share().map(|vec| { diff --git a/src/config.rs b/src/config.rs index 24db8d149..7794666d2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,10 +9,10 @@ use std::{ }; use hyper::{client::Builder, http::uri::Scheme, Uri}; +use rustls::Certificate; use rustls_pemfile::Item; use serde::{Deserialize, Deserializer, Serialize}; use tokio::fs; -use tokio_rustls::rustls::Certificate; use crate::{ error::BoxError, diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 3a47dc5e0..5544f2378 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -38,7 +38,7 @@ impl AddSubAssign for T where T: AddAssign + SubAssign {} /// Trait for items that have fixed-byte length representation. pub trait Serializable: Sized { /// Required number of bytes to store this message on disk/network - type Size: ArrayLength; + type Size: ArrayLength; /// Serialize this message to a mutable slice. It is enforced at compile time or on the caller /// side that this slice is sized to fit this instance. Implementations do not need to check diff --git a/src/helpers/transport/stream/input.rs b/src/helpers/transport/stream/input.rs index 2ca6e4a22..9a1b995af 100644 --- a/src/helpers/transport/stream/input.rs +++ b/src/helpers/transport/stream/input.rs @@ -14,6 +14,7 @@ use futures::{ stream::{iter, once, Fuse, FusedStream, Iter, Map, Once}, Stream, StreamExt, }; +use generic_array::GenericArray; use pin_project::pin_project; use typenum::{Unsigned, U2}; @@ -97,7 +98,7 @@ impl BufDeque { self.read_bytes(count * T::Size::USIZE).map(|bytes| { bytes .chunks(T::Size::USIZE) - .map(|bytes| T::deserialize(bytes.into())) + .map(|bytes| T::deserialize(GenericArray::from_slice(bytes))) .collect() }) } @@ -108,7 +109,7 @@ impl BufDeque { /// Returns `None` if there is insufficient data available. fn read(&mut self) -> Option { self.read_bytes(T::Size::USIZE) - .map(|bytes| T::deserialize(bytes.as_ref().into())) + .map(|bytes| T::deserialize(GenericArray::from_slice(&bytes))) } /// Update the buffer with the result of polling a stream. @@ -517,8 +518,7 @@ mod test { let mut stream = RecordsStream::::from(chunks); assert_eq!(stream.buffer.len(), 0); for expected_chunk in vec.chunks(::Size::USIZE) { - let expected = - Fp32BitPrime::deserialize(<&GenericArray>::from(expected_chunk)); + let expected = Fp32BitPrime::deserialize(GenericArray::from_slice(expected_chunk)); let n = stream.next().await.unwrap().unwrap(); // `RecordsStream` outputs correct value assert_eq!(vec![expected], n); @@ -704,7 +704,7 @@ mod test { (data in arb_aligned_bytes(size_in_bytes, max_len), seed in any::()) -> (Vec, Vec>, u64) { let expected = data.chunks(::Size::USIZE) - .map(|chunk| Fp32BitPrime::deserialize(<&GenericArray>::from(chunk))) + .map(|chunk| Fp32BitPrime::deserialize(>::from_slice(chunk))) .collect(); (expected, random_chunks(&data, &mut StdRng::seed_from_u64(seed)), seed) } diff --git a/src/hpke/mod.rs b/src/hpke/mod.rs index 2e940f05a..5658183bd 100644 --- a/src/hpke/mod.rs +++ b/src/hpke/mod.rs @@ -31,12 +31,13 @@ type IpaKdf = hpke::kdf::HkdfSha256; pub type IpaPublicKey = ::PublicKey; pub type IpaPrivateKey = ::PrivateKey; +pub type IpaEncappedKey = ::EncappedKey; pub use hpke::{Deserializable, Serializable}; pub trait FieldShareCrypt: GaloisField + IpaSerializable { - type EncapKeySize: ArrayLength; - type CiphertextSize: ArrayLength; + type EncapKeySize: ArrayLength; + type CiphertextSize: ArrayLength; type SemiHonestShares: IpaSerializable + Clone + Debug + Eq; } @@ -50,7 +51,7 @@ where AdditiveShare: IpaSerializable + Clone + Debug + Eq, AeadTag: Serializable, as IpaSerializable>::Size: Add, - < as IpaSerializable>::Size as Add>::Output: ArrayLength, + < as IpaSerializable>::Size as Add>::Output: ArrayLength, { type EncapKeySize = <::EncappedKey as Serializable>::OutputSize; type CiphertextSize = < as IpaSerializable>::Size as Add>::Output; diff --git a/src/hpke/registry.rs b/src/hpke/registry.rs index 5a14905e2..338553a8d 100644 --- a/src/hpke/registry.rs +++ b/src/hpke/registry.rs @@ -123,12 +123,12 @@ impl PublicKeyRegistry for KeyRegistry { #[cfg(all(test, unit_test))] mod tests { - use hpke::{kem::EncappedKey, HpkeError, OpModeR, OpModeS}; + use hpke::{HpkeError, OpModeR, OpModeS}; use rand::rngs::StdRng; use rand_core::{CryptoRng, RngCore, SeedableRng}; use super::*; - use crate::hpke::{IpaAead, IpaKdf, IpaKem}; + use crate::hpke::{IpaAead, IpaEncappedKey, IpaKdf, IpaKem}; const INFO_STR: &[u8] = b"This is an INFO string."; const AAD: &[u8] = b"This is AAD."; @@ -137,7 +137,7 @@ mod tests { pk: &IpaPublicKey, pt: &[u8], r: &mut R, - ) -> (EncappedKey, Vec) { + ) -> (IpaEncappedKey, Vec) { let (encapsulated_key, mut encryption_context) = hpke::setup_sender::(&OpModeS::Base, pk, INFO_STR, r) .expect("Can setup the sender."); @@ -152,7 +152,7 @@ mod tests { fn decrypt>( sk: &IpaPrivateKey, - payload: &(EncappedKey, I), + payload: &(IpaEncappedKey, I), ) -> Result, HpkeError> { let (encap_key, ct) = payload; let mut decryption_context = hpke::setup_receiver::( diff --git a/src/net/client/mod.rs b/src/net/client/mod.rs index 725cc63ab..8f9e7b519 100644 --- a/src/net/client/mod.rs +++ b/src/net/client/mod.rs @@ -16,10 +16,7 @@ use hyper::{ }; use hyper_rustls::{ConfigBuilderExt, HttpsConnector, HttpsConnectorBuilder}; use pin_project::pin_project; -use tokio_rustls::{ - rustls, - rustls::{Certificate, PrivateKey, RootCertStore}, -}; +use rustls::{Certificate, PrivateKey, RootCertStore}; use tracing::error; use crate::{ diff --git a/src/net/server/mod.rs b/src/net/server/mod.rs index 8ddd474cb..2e1667c1f 100644 --- a/src/net/server/mod.rs +++ b/src/net/server/mod.rs @@ -30,16 +30,13 @@ use futures::{ }; use hyper::{header::HeaderName, server::conn::AddrStream, Request}; use metrics::increment_counter; +use rustls::{ + server::AllowAnyAnonymousOrAuthenticatedClient, Certificate, PrivateKey, RootCertStore, +}; use rustls_pemfile::Item; #[cfg(all(feature = "shuttle", test))] use shuttle::future as tokio; -use tokio_rustls::{ - rustls::{ - server::AllowAnyAnonymousOrAuthenticatedClient, Certificate, PrivateKey, RootCertStore, - ServerConfig as RustlsServerConfig, - }, - server::TlsStream, -}; +use tokio_rustls::server::TlsStream; use tower::{layer::layer_fn, Service}; use tower_http::trace::TraceLayer; use tracing::{error, Span}; @@ -306,7 +303,7 @@ async fn rustls_config( } let verifier = AllowAnyAnonymousOrAuthenticatedClient::new(trusted_certs); - let mut config = RustlsServerConfig::builder() + let mut config = rustls::ServerConfig::builder() .with_safe_defaults() .with_client_cert_verifier(verifier.boxed()) .with_single_cert(cert, key)?; @@ -488,12 +485,9 @@ mod e2e_tests { use hyper::{client::HttpConnector, http::uri, StatusCode, Version}; use hyper_rustls::HttpsConnector; use metrics_util::debugging::Snapshotter; - use tokio_rustls::{ - rustls, - rustls::{ - client::{ServerCertVerified, ServerCertVerifier}, - ServerName, - }, + use rustls::{ + client::{ServerCertVerified, ServerCertVerifier}, + ServerName, }; use tracing::Level; diff --git a/src/net/test.rs b/src/net/test.rs index 92b0a2be7..414e808f2 100644 --- a/src/net/test.rs +++ b/src/net/test.rs @@ -15,8 +15,8 @@ use std::{ }; use once_cell::sync::Lazy; +use rustls::Certificate; use tokio::task::JoinHandle; -use tokio_rustls::rustls::Certificate; use crate::{ config::{ diff --git a/src/protocol/aggregation/input.rs b/src/protocol/aggregation/input.rs index 7f8c6ee70..426ed0f99 100644 --- a/src/protocol/aggregation/input.rs +++ b/src/protocol/aggregation/input.rs @@ -21,7 +21,7 @@ where Replicated: Serializable, as Serializable>::Size: Add< as Serializable>::Size>, < as Serializable>::Size as Add< as Serializable>::Size>>::Output: - ArrayLength, + ArrayLength, { type Size = < as Serializable>::Size as Add< as Serializable>::Size, diff --git a/src/protocol/ipa/mod.rs b/src/protocol/ipa/mod.rs index eca798743..b8f516684 100644 --- a/src/protocol/ipa/mod.rs +++ b/src/protocol/ipa/mod.rs @@ -104,7 +104,7 @@ where >>::Output, >>::Output, >>::Output, - >>::Output: ArrayLength, + >>::Output: ArrayLength, { type Size = < as Serializable>::Size as Add< < as Serializable>::Size as Add< diff --git a/src/protocol/prss/crypto.rs b/src/protocol/prss/crypto.rs index e3eb415b7..a2fa2d558 100644 --- a/src/protocol/prss/crypto.rs +++ b/src/protocol/prss/crypto.rs @@ -101,7 +101,7 @@ pub struct KeyExchange { impl KeyExchange { pub fn new(r: &mut R) -> Self { Self { - sk: EphemeralSecret::new(r), + sk: EphemeralSecret::random_from_rng(r), } } diff --git a/src/report.rs b/src/report.rs index 88d26d995..094a31997 100644 --- a/src/report.rs +++ b/src/report.rs @@ -256,7 +256,7 @@ where .unwrap(); // validated on construction let mut ciphertext: GenericArray::CiphertextSize> = - GenericArray::clone_from_slice(self.match_key_ciphertext()); + *GenericArray::from_slice(self.match_key_ciphertext()); let plaintext = open_in_place(key_registry, self.encap_key(), &mut ciphertext, &info)?; Ok(Report { diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 908a45feb..f366c870c 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -45,7 +45,7 @@ impl Arithmetic for T where // Trait for primitive integer types used to represent the underlying type for shared values pub trait Block: Sized + Copy + Debug { /// Size of a block in bytes big enough to hold the shared value. `Size * 8 >= VALID_BIT_LENGTH`. - type Size: ArrayLength; + type Size: ArrayLength; } pub trait SharedValue: diff --git a/src/secret_sharing/replicated/malicious/additive_share.rs b/src/secret_sharing/replicated/malicious/additive_share.rs index 025590193..c2fe440c4 100644 --- a/src/secret_sharing/replicated/malicious/additive_share.rs +++ b/src/secret_sharing/replicated/malicious/additive_share.rs @@ -294,7 +294,7 @@ where Add< as Serializable>::Size>, < as Serializable>::Size as Add< as Serializable>::Size, - >>::Output: ArrayLength, + >>::Output: ArrayLength, { type Size = < as Serializable>::Size as Add< as Serializable>::Size, diff --git a/src/secret_sharing/replicated/semi_honest/additive_share.rs b/src/secret_sharing/replicated/semi_honest/additive_share.rs index cc48cf78b..10b8b2b39 100644 --- a/src/secret_sharing/replicated/semi_honest/additive_share.rs +++ b/src/secret_sharing/replicated/semi_honest/additive_share.rs @@ -220,7 +220,7 @@ impl From<(V, V)> for AdditiveShare { impl Serializable for AdditiveShare where V::Size: Add, - >::Output: ArrayLength, + >::Output: ArrayLength, { type Size = >::Output; From 160ba83fed3becc710f02d3a6926f361217a4caf Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 15:00:05 -0700 Subject: [PATCH 059/124] adressing Alex's comments --- src/error.rs | 4 ++-- src/ff/curve_points.rs | 10 +++++----- src/ff/ec_prime_field.rs | 10 +++++++--- src/protocol/ipa_prf/prf_eval/mod.rs | 5 ++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/error.rs b/src/error.rs index 7d6456c7f..de41eda4b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -58,8 +58,8 @@ pub enum Error { InvalidReport(#[from] InvalidReportError), #[error("unsupported: {0}")] Unsupported(String), - #[error("Decompressing invalid elliptic curve point")] - DecompressingInvalidCurvePoint, + #[error("Decompressing invalid elliptic curve point: {0}")] + DecompressingInvalidCurvePoint(String), } impl Default for Error { diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index a084b14a0..f4cb50bba 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -3,8 +3,6 @@ use curve25519_dalek::{ Scalar, }; use generic_array::GenericArray; -use hkdf::Hkdf; -use sha2::Sha256; use typenum::U32; use crate::{ @@ -35,6 +33,7 @@ impl Serializable for RP25519 { } fn deserialize(buf: &GenericArray) -> Self { + debug_assert!(CompressedRistretto((*buf).into()).decompress().is_some()); RP25519(CompressedRistretto((*buf).into())) } } @@ -87,7 +86,6 @@ impl std::ops::SubAssign for RP25519 { } ///Scalar Multiplication -///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve @@ -136,6 +134,8 @@ macro_rules! cp_hash_impl { ( $u_type:ty) => { impl From for $u_type { fn from(s: RP25519) -> Self { + use hkdf::Hkdf; + use sha2::Sha256; let hk = Hkdf::::new(None, s.0.as_bytes()); let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large @@ -150,8 +150,6 @@ cp_hash_impl!(u128); cp_hash_impl!(u64); -#[cfg(test)] -cp_hash_impl!(u32); #[cfg(test)] impl rand::distributions::Distribution for rand::distributions::Standard { @@ -174,6 +172,8 @@ mod test { secret_sharing::WeakSharedValue, }; + cp_hash_impl!(u32); + #[test] fn serde_25519() { let mut rng = thread_rng(); diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 15d5f33a7..02f0742df 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -124,6 +124,8 @@ macro_rules! sc_hash_impl { ( $u_type:ty) => { impl From for $u_type { fn from(s: Fp25519) -> Self { + use hkdf::Hkdf; + use sha2::Sha256; let hk = Hkdf::::new(None, s.0.as_bytes()); let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large @@ -134,6 +136,8 @@ macro_rules! sc_hash_impl { impl From<$u_type> for Fp25519 { fn from(s: $u_type) -> Self { + use hkdf::Hkdf; + use sha2::Sha256; let hk = Hkdf::::new(None, &s.to_le_bytes()); let mut okm = [0u8; 32]; //error invalid length from expand only happens when okm is very large @@ -147,10 +151,7 @@ macro_rules! sc_hash_impl { #[cfg(test)] sc_hash_impl!(u64); -#[cfg(test)] -sc_hash_impl!(u32); -/// Daniel had to implement this since PRSS wants it, prefer not to impl Field for Fp25519 { const ONE: Fp25519 = Fp25519::ONE; @@ -198,6 +199,9 @@ mod test { secret_sharing::SharedValue, }; + + sc_hash_impl!(u32); + #[test] fn serde_25519() { let mut rng = thread_rng(); diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index 4da81759c..fe9b2540e 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -10,7 +10,6 @@ use crate::{ RecordId, }, secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, - seq_join::seq_try_join_all, }; #[derive(Step)] @@ -37,7 +36,7 @@ where .iter() .enumerate() .map(|(i, x)| eval_dy_prf(ctx.clone(), i.into(), &prf_key, x)); - seq_try_join_all(sh_ctx.active_work(), futures).await + ctx.try_join(futures).await } impl From> for AdditiveShare { @@ -53,7 +52,7 @@ where { ctx.narrow(&Step::PRFKeyGen) .prss() - .generate_replicated(u128::MAX - 100u128) + .generate_replicated(RecordId(0)) } /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) From 8407938cab4e476ab375fb0fa14b25431d9582d5 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 15:01:26 -0700 Subject: [PATCH 060/124] fmt --- src/ff/curve_points.rs | 1 - src/ff/ec_prime_field.rs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index f4cb50bba..2b06f973e 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -150,7 +150,6 @@ cp_hash_impl!(u128); cp_hash_impl!(u64); - #[cfg(test)] impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> RP25519 { diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 02f0742df..ba753eb40 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -151,7 +151,6 @@ macro_rules! sc_hash_impl { #[cfg(test)] sc_hash_impl!(u64); - impl Field for Fp25519 { const ONE: Fp25519 = Fp25519::ONE; @@ -199,7 +198,6 @@ mod test { secret_sharing::SharedValue, }; - sc_hash_impl!(u32); #[test] From 43696a67ecac2e51fefac351492d33e1732c55d7 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 16:59:31 -0700 Subject: [PATCH 061/124] add comments --- src/ff/curve_points.rs | 22 ++++++++++++++++++++-- src/ff/ec_prime_field.rs | 14 +++++++++++++- src/protocol/ipa_prf/prf_eval/mod.rs | 12 ++++++++++-- src/secret_sharing/mod.rs | 4 ++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 2b06f973e..ec42eba76 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -14,11 +14,21 @@ impl Block for CompressedRistretto { type Size = U32; } -///ristretto point for curve 25519 +///ristretto point for curve 25519, +/// we store it in compressed format since it is 3 times smaller and we do a limited amount of +/// arithmetic operations on the curve points +/// +/// We use ristretto points such that we have a prime order elliptic curve, +/// This is needed for the Dodis Yampolski PRF +/// +/// decompressing invalid curve points will cause panics, +/// since we always generate curve points from scalars (elements in Fp25519) and +/// only deserialize previously serialized valid points, panics will not occur +/// However, we still added a debug assert to deserialize since values are sent by other servers #[derive(Clone, Copy, PartialEq, Debug)] pub struct RP25519(CompressedRistretto); -/// using compressed ristretto point +/// Implementing trait for secret sharing impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; @@ -86,6 +96,7 @@ impl std::ops::SubAssign for RP25519 { } ///Scalar Multiplication +/// allows to multiply curve points with scalars from Fp25519 ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve @@ -130,6 +141,7 @@ impl From for CompressedRistretto { } } +///allows to convert curve points into unsigned integers, preserving high entropy macro_rules! cp_hash_impl { ( $u_type:ty) => { impl From for $u_type { @@ -150,6 +162,8 @@ cp_hash_impl!(u128); cp_hash_impl!(u64); +/// implementing random curve point generation for testing purposes, +/// in the actual IPA protocol, we generate them from scalars, i.e. Fp25519 #[cfg(test)] impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> RP25519 { @@ -173,6 +187,7 @@ mod test { cp_hash_impl!(u32); + ///testing serialize and deserialize #[test] fn serde_25519() { let mut rng = thread_rng(); @@ -183,6 +198,7 @@ mod test { assert_eq!(input, output); } + ///testing conversion from scalar to Fp25519 and curve point, i.e. RP25519 #[test] fn scalar_to_point() { let a = Scalar::ONE; @@ -193,6 +209,7 @@ mod test { assert_eq!(c, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } + ///testing simple curve arithmetics to check that curve25519_dalek library is used correctly #[test] fn curve_arithmetics() { let mut rng = thread_rng(); @@ -211,6 +228,7 @@ mod test { assert_eq!(RP25519::ZERO, fp_h * Scalar::ZERO.into()); } + ///testing curve to unsigned integer conversion has entropy (!= 0) #[test] fn curve_point_to_hash() { let mut rng = thread_rng(); diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index ba753eb40..9bd31ddbb 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -13,12 +13,15 @@ impl Block for Scalar { type Size = U32; } +///implements the Scalar field for elliptic curve 25519 +/// we use elements in Fp25519 to generate curve points and operate on the curve #[derive(Clone, Copy, PartialEq, Debug)] pub struct Fp25519(::Storage); impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); + ///allow invert for scalars, i.e. computes 1/a mod p ///# Panics /// Panics when self is zero #[must_use] @@ -28,12 +31,14 @@ impl Fp25519 { } } +///trait for secret sharing impl SharedValue for Fp25519 { type Storage = Scalar; const BITS: u32 = 256; const ZERO: Self = Self(Scalar::ZERO); } +///conversion to Scalar struct of curve25519_dalek impl From for Scalar { fn from(s: Fp25519) -> Self { s.0 @@ -52,6 +57,7 @@ impl Serializable for Fp25519 { } } +///generate random elements in Fp25519 impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> Fp25519 { let mut scalar_bytes = [0u8; 32]; @@ -119,6 +125,7 @@ impl From for Fp25519 { } } +///conversion from and to unsigned integers, preserving entropy, for testing purposes only #[cfg(test)] macro_rules! sc_hash_impl { ( $u_type:ty) => { @@ -151,6 +158,7 @@ macro_rules! sc_hash_impl { #[cfg(test)] sc_hash_impl!(u64); +///implement Field because required by PRSS impl Field for Fp25519 { const ONE: Fp25519 = Fp25519::ONE; @@ -174,6 +182,7 @@ impl Field for Fp25519 { } } +///implement TryFrom since required by Field impl TryFrom for Fp25519 { type Error = crate::error::Error; @@ -200,6 +209,7 @@ mod test { sc_hash_impl!(u32); + ///test serialize and deserialize #[test] fn serde_25519() { let mut rng = thread_rng(); @@ -210,7 +220,7 @@ mod test { assert_eq!(input, output); } - // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek + ///test simple arithmetics to check that curve25519_dalek is used correctly #[test] fn simple_arithmetics_25519() { let a = Fp25519(Scalar::from_bytes_mod_order([ @@ -241,12 +251,14 @@ mod test { assert_eq!(ec, e); } + ///test random field element generation (!= 0) #[test] fn simple_random_25519() { let mut rng = thread_rng(); assert_ne!(Fp25519::ZERO, rng.gen::()); } + ///test inversion for field elements #[test] fn invert_25519() { let mut rng = thread_rng(); diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index fe9b2540e..7a65c0191 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -21,6 +21,10 @@ pub(crate) enum Step { Revealz, } +/// generates match key pseudonyms from match keys (in Fp25519 format) and PRF key +/// PRF key needs to be generated separately using gen_prf_key +/// +/// gen_prf_key is not included such that compute_match_key_pseudonym can be tested for correctness /// # Errors /// Propagates errors from multiplications pub async fn compute_match_key_pseudonym( @@ -57,8 +61,8 @@ where /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 -/// PRF key k is generated using keygen -/// In 3IPA, x is the match key +/// PRF key k needs to be generated using gen_prf_key +/// x is the match key in Fp25519 format /// outputs a u64 as specified in `protocol/prf_sharding/mod.rs`, all parties learn the output /// # Errors /// Propagates errors from multiplications, reveal and scalar multiplication @@ -108,11 +112,13 @@ mod test { test_fixture::{Reconstruct, Runner, TestWorld}, }; + ///defining test input struct #[derive(Copy, Clone)] struct ShuffledTestInput { match_key: Fp25519, } + ///defining test output struct #[derive(Debug, PartialEq)] struct TestOutput { match_key_pseudonym: u64, @@ -142,6 +148,8 @@ mod test { } } + ///testing correctness of DY PRF evaluation + /// by checking MPC generated pseudonym with pseudonym generated in the clear #[test] fn semi_honest() { run(|| async move { diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 8564bf4be..1a7dda8c2 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -51,6 +51,8 @@ pub trait Block: Sized + Copy + Debug { type Size: ArrayLength; } + +///allows basic secret sharing operations pub trait WeakSharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static { @@ -61,6 +63,7 @@ pub trait WeakSharedValue: const ZERO: Self; } +///allows advanced secret sharing operations, requires multiplication pub trait SharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Arithmetic + Serializable + 'static { @@ -71,6 +74,7 @@ pub trait SharedValue: const ZERO: Self; } +///any SharedValue is also a WeakSharedValue impl WeakSharedValue for T where T: SharedValue, From 4268a0590d8264fd75bc7ff870343433dac18df9 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 17:02:24 -0700 Subject: [PATCH 062/124] fix lint --- src/ff/curve_points.rs | 2 +- src/ff/ec_prime_field.rs | 6 +++--- src/protocol/ipa_prf/prf_eval/mod.rs | 6 +++--- src/secret_sharing/mod.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index ec42eba76..8eaa57623 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -209,7 +209,7 @@ mod test { assert_eq!(c, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } - ///testing simple curve arithmetics to check that curve25519_dalek library is used correctly + ///testing simple curve arithmetics to check that `curve25519_dalek` library is used correctly #[test] fn curve_arithmetics() { let mut rng = thread_rng(); diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 9bd31ddbb..3385f10e8 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -38,7 +38,7 @@ impl SharedValue for Fp25519 { const ZERO: Self = Self(Scalar::ZERO); } -///conversion to Scalar struct of curve25519_dalek +///conversion to Scalar struct of `curve25519_dalek` impl From for Scalar { fn from(s: Fp25519) -> Self { s.0 @@ -182,7 +182,7 @@ impl Field for Fp25519 { } } -///implement TryFrom since required by Field +///implement `TryFrom` since required by Field impl TryFrom for Fp25519 { type Error = crate::error::Error; @@ -220,7 +220,7 @@ mod test { assert_eq!(input, output); } - ///test simple arithmetics to check that curve25519_dalek is used correctly + ///test simple arithmetics to check that `curve25519_dalek` is used correctly #[test] fn simple_arithmetics_25519() { let a = Fp25519(Scalar::from_bytes_mod_order([ diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index 7a65c0191..ff37e30eb 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -22,9 +22,9 @@ pub(crate) enum Step { } /// generates match key pseudonyms from match keys (in Fp25519 format) and PRF key -/// PRF key needs to be generated separately using gen_prf_key +/// PRF key needs to be generated separately using `gen_prf_key` /// -/// gen_prf_key is not included such that compute_match_key_pseudonym can be tested for correctness +/// `gen_prf_key` is not included such that `compute_match_key_pseudonym` can be tested for correctness /// # Errors /// Propagates errors from multiplications pub async fn compute_match_key_pseudonym( @@ -61,7 +61,7 @@ where /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 -/// PRF key k needs to be generated using gen_prf_key +/// PRF key k needs to be generated using `gen_prf_key` /// x is the match key in Fp25519 format /// outputs a u64 as specified in `protocol/prf_sharding/mod.rs`, all parties learn the output /// # Errors diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 1a7dda8c2..155b1f609 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -74,7 +74,7 @@ pub trait SharedValue: const ZERO: Self; } -///any SharedValue is also a WeakSharedValue +///any `SharedValue` is also a `WeakSharedValue` impl WeakSharedValue for T where T: SharedValue, From 4208d1b91385953c21f5ed45b331a1d33d83cd69 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 17:03:06 -0700 Subject: [PATCH 063/124] fmt --- src/secret_sharing/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 155b1f609..441fe1f21 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -51,7 +51,6 @@ pub trait Block: Sized + Copy + Debug { type Size: ArrayLength; } - ///allows basic secret sharing operations pub trait WeakSharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static From 53d71f20b9aca3a850a6f506559deb04f748c692 Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Mon, 23 Oct 2023 16:05:30 +0800 Subject: [PATCH 064/124] remove did_trigger_get_attributed and refactor attribution flag calculations --- src/protocol/prf_sharding/mod.rs | 56 +++++++++++++++----------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index aec7bee85..74505c09a 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -97,7 +97,6 @@ impl InputsRequiredFromPrevRow { /// - `did_trigger_get_attributed` - a secret-shared bit indicating if this row corresponds to a trigger event /// which was attributed. Might be able to reveal this (after a shuffle and the addition of dummies) to minimize /// the amount of processing work that must be done in the Aggregation stage. - #[allow(clippy::too_many_lines)] pub async fn compute_row_with_previous( &mut self, ctx: C, @@ -152,29 +151,15 @@ impl InputsRequiredFromPrevRow { ) .await?; - let (did_trigger_get_attributed, is_trigger_within_window) = try_join( - input_row.is_trigger_bit.multiply( - &ever_encountered_a_source_event, - ctx.narrow(&Step::DidTriggerGetAttributed), - record_id, - ), - is_trigger_event_within_attribution_window( - ctx.narrow(&Step::CheckAttributionWindow), - record_id, - attribution_window_seconds, - ×tamp, - &source_event_timestamp, - ), - ) - .await?; - let attributed_trigger_value = zero_out_trigger_value_unless_attributed( ctx.narrow(&Step::AttributedTriggerValue), record_id, - &did_trigger_get_attributed, - &is_trigger_within_window, + &input_row.is_trigger_bit, + &ever_encountered_a_source_event, &tv, attribution_window_seconds, + ×tamp, + &source_event_timestamp, ) .await?; @@ -218,7 +203,6 @@ impl InputsRequiredFromPrevRow { self.source_event_timestamp = source_event_timestamp; let outputs_for_aggregation = CappedAttributionOutputs { - did_trigger_get_attributed, attributed_breakdown_key_bits, capped_attributed_trigger_value, }; @@ -228,7 +212,6 @@ impl InputsRequiredFromPrevRow { #[derive(Debug)] pub struct CappedAttributionOutputs { - pub did_trigger_get_attributed: Replicated, pub attributed_breakdown_key_bits: BitDecomposed>, pub capped_attributed_trigger_value: BitDecomposed>, } @@ -557,22 +540,41 @@ where /// another secret-shared bit indicating if a given row is within the attribution window. We multiply these two bits together and /// multiply it with the bits of the `trigger_value` in order to zero out contributions from unattributed trigger events. /// +#[allow(clippy::too_many_arguments)] async fn zero_out_trigger_value_unless_attributed( ctx: C, record_id: RecordId, - did_trigger_get_attributed: &Replicated, - is_trigger_within_window: &Replicated, + is_trigger_bit: &Replicated, + ever_encountered_a_source_event: &Replicated, trigger_value: &BitDecomposed>, attribution_window_seconds: Option, + trigger_event_timestamp: &BitDecomposed>, + source_event_timestamp: &BitDecomposed>, ) -> Result>, Error> where C: UpgradedContext>, { + let (did_trigger_get_attributed, is_trigger_within_window) = try_join( + is_trigger_bit.multiply( + ever_encountered_a_source_event, + ctx.narrow(&Step::DidTriggerGetAttributed), + record_id, + ), + is_trigger_event_within_attribution_window( + ctx.narrow(&Step::CheckAttributionWindow), + record_id, + attribution_window_seconds, + trigger_event_timestamp, + source_event_timestamp, + ), + ) + .await?; + // save 1 multiplication if there is no attribution window let zero_out_flag = if attribution_window_seconds.is_some() { let c = ctx.narrow(&Step::AttributedEventCheckFlag); did_trigger_get_attributed - .multiply(is_trigger_within_window, c, record_id) + .multiply(&is_trigger_within_window, c, record_id) .await? } else { did_trigger_get_attributed.clone() @@ -983,27 +985,21 @@ pub mod tests { capped_attributed_trigger_value, } = self; - let did_trigger_get_attributed = Gf2::ONE; let [attributed_breakdown_key0, attributed_breakdown_key1, attributed_breakdown_key2] = attributed_breakdown_key.share_with(rng); let [capped_attributed_trigger_value0, capped_attributed_trigger_value1, capped_attributed_trigger_value2] = capped_attributed_trigger_value.share_with(rng); - let [did_trigger_get_attributed0, did_trigger_get_attributed1, did_trigger_get_attributed2] = - did_trigger_get_attributed.share_with(rng); [ CappedAttributionOutputs { - did_trigger_get_attributed: did_trigger_get_attributed0, attributed_breakdown_key_bits: attributed_breakdown_key0, capped_attributed_trigger_value: capped_attributed_trigger_value0, }, CappedAttributionOutputs { - did_trigger_get_attributed: did_trigger_get_attributed1, attributed_breakdown_key_bits: attributed_breakdown_key1, capped_attributed_trigger_value: capped_attributed_trigger_value1, }, CappedAttributionOutputs { - did_trigger_get_attributed: did_trigger_get_attributed2, attributed_breakdown_key_bits: attributed_breakdown_key2, capped_attributed_trigger_value: capped_attributed_trigger_value2, }, From 1ce428103ef31f072071e1bd7330e28f084724fe Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 24 Oct 2023 10:49:21 +0800 Subject: [PATCH 065/124] 1. Increasing max number of breakdowns we can support with OPRF 2. In case no conversion is attributed to any bucket, do_aggregation would return vector with zeroes --- src/protocol/prf_sharding/bucket.rs | 38 +++++++++++++++++++++-------- src/protocol/prf_sharding/mod.rs | 7 ++++++ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/protocol/prf_sharding/bucket.rs b/src/protocol/prf_sharding/bucket.rs index 6349f91a6..d199557bd 100644 --- a/src/protocol/prf_sharding/bucket.rs +++ b/src/protocol/prf_sharding/bucket.rs @@ -1,17 +1,35 @@ use embed_doc_image::embed_doc_image; +use ipa_macros::Step; use crate::{ error::Error, ff::{GaloisField, PrimeField, Serializable}, protocol::{ - basics::SecureMul, context::UpgradedContext, prf_sharding::BinaryTreeDepthStep, - step::BitOpStep, RecordId, + basics::SecureMul, context::UpgradedContext, prf_sharding::BinaryTreeDepthStep, RecordId, }, secret_sharing::{ replicated::malicious::ExtendableField, BitDecomposed, Linear as LinearSecretSharing, }, }; +#[derive(Step)] +pub enum BucketStep { + #[dynamic(256)] + Bit(usize), +} + +impl From for BucketStep { + fn from(v: u32) -> Self { + Self::Bit(usize::try_from(v).unwrap()) + } +} + +impl From for BucketStep { + fn from(v: usize) -> Self { + Self::Bit(v) + } +} + #[embed_doc_image("tree-aggregation", "images/tree_aggregation.png")] /// This function moves a single value to a correct bucket using tree aggregation approach /// @@ -53,8 +71,8 @@ where BK::BITS ); assert!( - breakdown_count <= 128, - "Our step implementation (BitOpStep) cannot go past 64" + breakdown_count <= 512, + "Our step implementation (BucketStep) cannot go past 256" ); let mut row_contribution = vec![value; breakdown_count]; @@ -69,7 +87,7 @@ where let mut futures = Vec::with_capacity(breakdown_count / step); for (i, tree_index) in (0..breakdown_count).step_by(step).enumerate() { - let bit_c = depth_c.narrow(&BitOpStep::from(i)); + let bit_c = depth_c.narrow(&BucketStep::from(i)); if robust || tree_index + span < breakdown_count { futures.push(row_contribution[tree_index].multiply(bit_of_bdkey, bit_c, record_id)); @@ -96,7 +114,7 @@ pub mod tests { use rand::thread_rng; use crate::{ - ff::{Field, Fp32BitPrime, Gf5Bit, Gf8Bit}, + ff::{Field, Fp32BitPrime, Gf8Bit}, protocol::{ context::{Context, UpgradableContext, Validator}, prf_sharding::bucket::move_single_value_to_bucket, @@ -108,12 +126,12 @@ pub mod tests { test_fixture::{get_bits, Reconstruct, Runner, TestWorld}, }; - const MAX_BREAKDOWN_COUNT: usize = 1 << Gf5Bit::BITS; + const MAX_BREAKDOWN_COUNT: usize = 256; const VALUE: u32 = 10; async fn move_to_bucket(count: usize, breakdown_key: usize, robust: bool) -> Vec { let breakdown_key_bits = - get_bits::(breakdown_key.try_into().unwrap(), Gf5Bit::BITS); + get_bits::(breakdown_key.try_into().unwrap(), Gf8Bit::BITS); let value = Fp32BitPrime::truncate_from(VALUE); TestWorld::default() @@ -122,7 +140,7 @@ pub mod tests { |ctx, (breakdown_key_share, value_share)| async move { let validator = ctx.validator(); let ctx = validator.context(); - move_single_value_to_bucket::( + move_single_value_to_bucket::( ctx.set_total_records(1), RecordId::from(0), breakdown_key_share, @@ -221,7 +239,7 @@ pub mod tests { RecordId::from(0), breakdown_key_share, value_share, - 129, + 513, false, ) .await diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 110dc452a..4412d0e0a 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -592,6 +592,13 @@ where F: PrimeField + ExtendableField, { let num_records = user_level_attributions.len(); + + // in case no attributable conversion is found, return 0. + // as anyways the helpers know that no attributions resulted. + if num_records == 0 { + return Ok(vec![S::ZERO; 1 << BK::BITS]); + } + let (bk_vec, tv_vec): (Vec<_>, Vec<_>) = user_level_attributions .into_iter() .map(|row| { From 034f556e578790ce0def32d04bc2b05179d0a3eb Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 24 Oct 2023 13:49:02 +0800 Subject: [PATCH 066/124] Adding a oneshot for oprf It fails if user cap is small which we are investigating. However, this PR can still be reviewed and checked in Sample command to run cargo bench --bench oneshot_ipa --features="enable-benches descriptive-gate" --no-default-features -- --per-user-cap 64 --oprf --max-trigger-value 4 -n 100 --- benches/oneshot/ipa.rs | 24 +++++---- pre-commit | 2 +- src/ff/galois_field.rs | 17 ++++++- src/ff/mod.rs | 2 +- src/protocol/mod.rs | 4 +- src/protocol/prf_sharding/mod.rs | 8 +-- src/report.rs | 14 +++++ src/test_fixture/input/sharing.rs | 32 +++++++++++- src/test_fixture/ipa.rs | 85 ++++++++++++++++++++++++++++++- 9 files changed, 168 insertions(+), 20 deletions(-) diff --git a/benches/oneshot/ipa.rs b/benches/oneshot/ipa.rs index a7102f91b..ab6e723e2 100644 --- a/benches/oneshot/ipa.rs +++ b/benches/oneshot/ipa.rs @@ -9,7 +9,7 @@ use ipa::{ ff::Fp32BitPrime, helpers::{query::IpaQueryConfig, GatewayConfig}, test_fixture::{ - ipa::{ipa_in_the_clear, test_ipa, IpaSecurityModel}, + ipa::{ipa_in_the_clear, test_ipa, test_oprf_ipa, IpaSecurityModel}, EventGenerator, EventGeneratorConfig, TestWorld, TestWorldConfig, }, }; @@ -70,6 +70,8 @@ struct Args { /// Needed for benches. #[arg(long, hide = true)] bench: bool, + #[arg(short = 'o', long)] + oprf: bool, } impl Args { @@ -132,14 +134,18 @@ async fn run(args: Args) -> Result<(), Error> { tracing::trace!("Preparation complete in {:?}", _prep_time.elapsed()); let _protocol_time = Instant::now(); - test_ipa::( - &world, - &raw_data, - &expected_results, - args.config(), - args.mode, - ) - .await; + if args.oprf { + test_oprf_ipa::(&world, raw_data, &expected_results, args.config()).await; + } else { + test_ipa::( + &world, + &raw_data, + &expected_results, + args.config(), + args.mode, + ) + .await; + } tracing::trace!( "{m:?} IPA for {q} records took {t:?}", m = args.mode, diff --git a/pre-commit b/pre-commit index 12d63958b..6cf2be072 100755 --- a/pre-commit +++ b/pre-commit @@ -103,4 +103,4 @@ check "Arithmetic circuit benchmark" \ cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate" check "Sort benchmark" \ - cargo bench --bench oneshot_sort --no-default-features --features="enable-benches descriptive-gate" + cargo bench --bench oneshot_sort --no-default-features --features="enable-benches descriptive-gate" \ No newline at end of file diff --git a/src/ff/galois_field.rs b/src/ff/galois_field.rs index 18b6b58c8..cca6f3936 100644 --- a/src/ff/galois_field.rs +++ b/src/ff/galois_field.rs @@ -5,7 +5,7 @@ use std::{ use bitvec::prelude::{bitarr, BitArr, Lsb0}; use generic_array::GenericArray; -use typenum::{Unsigned, U1, U4, U5}; +use typenum::{Unsigned, U1, U3, U4, U5}; use crate::{ ff::{Field, Serializable}, @@ -25,6 +25,7 @@ pub trait GaloisField: // Bit store type definitions type U8_1 = BitArr!(for 8, in u8, Lsb0); +type U8_3 = BitArr!(for 24, in u8, Lsb0); type U8_4 = BitArr!(for 32, in u8, Lsb0); type U8_5 = BitArr!(for 40, in u8, Lsb0); @@ -32,6 +33,10 @@ impl Block for U8_1 { type Size = U1; } +impl Block for U8_3 { + type Size = U3; +} + impl Block for U8_4 { type Size = U4; } @@ -550,6 +555,16 @@ bit_array_impl!( 0b1_0000_0000_0000_0000_0000_0000_1000_1101_u128 ); +bit_array_impl!( + bit_array_20, + Gf20Bit, + U8_3, + 20, + bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + // x^20 + x^17 + x^15 + x^14 + x^11 + x^10 + x^9 + x^7 + x^6 + x^5 + 1 + 0b1_0010_1100_1110_1110_0001_u128 +); + bit_array_impl!( bit_array_8, Gf8Bit, diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 5544f2378..e324a4a72 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -9,7 +9,7 @@ mod prime_field; use std::ops::{Add, AddAssign, Sub, SubAssign}; pub use field::{Field, FieldType}; -pub use galois_field::{GaloisField, Gf2, Gf32Bit, Gf3Bit, Gf40Bit, Gf5Bit, Gf8Bit}; +pub use galois_field::{GaloisField, Gf2, Gf20Bit, Gf32Bit, Gf3Bit, Gf40Bit, Gf5Bit, Gf8Bit}; use generic_array::{ArrayLength, GenericArray}; #[cfg(any(test, feature = "weak-field"))] pub use prime_field::Fp31; diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 7931a1e37..03270a1e1 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -22,11 +22,13 @@ pub use basics::BasicProtocols; use crate::{ error::Error, - ff::{Gf40Bit, Gf8Bit}, + ff::{Gf20Bit, Gf3Bit, Gf40Bit, Gf8Bit}, }; pub type MatchKey = Gf40Bit; pub type BreakdownKey = Gf8Bit; +pub type TriggerValue = Gf3Bit; +pub type Timestamp = Gf20Bit; /// Unique identifier of the MPC query requested by report collectors /// TODO(615): Generating this unique id may be tricky as it may involve communication between helpers and diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 4412d0e0a..b6d334ee0 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -30,10 +30,10 @@ pub mod bucket; pub mod feature_label_dot_product; pub struct PrfShardedIpaInputRow { - prf_of_match_key: u64, - is_trigger_bit: Replicated, - breakdown_key: Replicated, - trigger_value: Replicated, + pub prf_of_match_key: u64, + pub is_trigger_bit: Replicated, + pub breakdown_key: Replicated, + pub trigger_value: Replicated, } struct InputsRequiredFromPrevRow { diff --git a/src/report.rs b/src/report.rs index 094a31997..2b9cf1039 100644 --- a/src/report.rs +++ b/src/report.rs @@ -386,6 +386,20 @@ where } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OprfReport +where + TS: GaloisField, + BK: GaloisField, + TV: GaloisField, +{ + pub timestamp: Replicated, + pub mk_oprf: u64, + pub event_type: EventType, + pub breakdown_key: Replicated, + pub trigger_value: Replicated, +} + #[cfg(all(test, unit_test))] mod test { use rand::{distributions::Alphanumeric, rngs::StdRng, Rng}; diff --git a/src/test_fixture/input/sharing.rs b/src/test_fixture/input/sharing.rs index f21dc4920..1287ad8e0 100644 --- a/src/test_fixture/input/sharing.rs +++ b/src/test_fixture/input/sharing.rs @@ -12,7 +12,7 @@ use crate::{ BreakdownKey, MatchKey, }, rand::Rng, - report::{EventType, Report}, + report::{EventType, OprfReport, Report}, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, IntoShares}, test_fixture::{ input::{GenericReportShare, GenericReportTestInput}, @@ -354,3 +354,33 @@ where } } } + +impl IntoShares> for TestRawDataRecord +where + TS: GaloisField + IntoShares>, + BK: GaloisField + IntoShares>, + TV: GaloisField + IntoShares>, +{ + fn share_with(self, rng: &mut R) -> [OprfReport; 3] { + let event_type = if self.is_trigger_report { + EventType::Trigger + } else { + EventType::Source + }; + let timestamp = TS::truncate_from(self.timestamp).share_with(rng); + let breakdown_key = BK::truncate_from(self.breakdown_key).share_with(rng); + let trigger_value = TV::truncate_from(self.trigger_value).share_with(rng); + + zip(zip(timestamp, breakdown_key), trigger_value) + .map(|((ts_share, bk_share), tv_share)| OprfReport { + timestamp: ts_share, + mk_oprf: self.user_id, + event_type, + breakdown_key: bk_share, + trigger_value: tv_share, + }) + .collect::>() + .try_into() + .unwrap() + } +} diff --git a/src/test_fixture/ipa.rs b/src/test_fixture/ipa.rs index 37e31b535..d279806c2 100644 --- a/src/test_fixture/ipa.rs +++ b/src/test_fixture/ipa.rs @@ -5,9 +5,13 @@ use crate::{ ff::{PrimeField, Serializable}, helpers::query::IpaQueryConfig, ipa_test_input, - protocol::{ipa::ipa, BreakdownKey, MatchKey}, + protocol::{ipa::ipa, BreakdownKey, MatchKey, Timestamp, TriggerValue}, + report::OprfReport, secret_sharing::{ - replicated::{malicious, malicious::ExtendableField, semi_honest}, + replicated::{ + malicious, malicious::ExtendableField, semi_honest, + semi_honest::AdditiveShare as Replicated, + }, IntoShares, }, test_fixture::{input::GenericReportTestInput, Reconstruct}, @@ -191,3 +195,80 @@ pub async fn test_ipa( .collect::>(); assert_eq!(result, expected_results); } + +/// # Panics +/// If any of the IPA protocol modules panic +#[cfg(feature = "in-memory-infra")] +pub async fn test_oprf_ipa( + world: &super::TestWorld, + mut records: Vec, + expected_results: &[u32], + config: IpaQueryConfig, +) where + F: PrimeField + ExtendableField + IntoShares>, + rand::distributions::Standard: rand::distributions::Distribution, + semi_honest::AdditiveShare: Serializable, +{ + use crate::{ + ff::{Field, Gf2}, + protocol::{ + basics::ShareKnownValue, + prf_sharding::{attribution_and_capping_and_aggregation, PrfShardedIpaInputRow}, + }, + report::EventType, + secret_sharing::SharedValue, + test_fixture::Runner, + }; + + let user_cap: i32 = config.per_user_credit_cap.try_into().unwrap(); + assert!( + user_cap & (user_cap - 1) == 0, + "This code only works for a user cap which is a power of 2" + ); + + //TODO(richaj) This manual sorting will be removed once we have the PRF sharding in place + records.sort_by(|a, b| b.user_id.cmp(&a.user_id)); + + let result: Vec = world + .semi_honest( + records.into_iter(), + |ctx, input_rows: Vec>| async move { + let sharded_input = input_rows + .into_iter() + .map(|single_row| { + let is_trigger_bit_share = if single_row.event_type == EventType::Trigger { + Replicated::share_known_value(&ctx, Gf2::ONE) + } else { + Replicated::share_known_value(&ctx, Gf2::ZERO) + }; + PrfShardedIpaInputRow { + prf_of_match_key: single_row.mk_oprf, + is_trigger_bit: is_trigger_bit_share, + breakdown_key: single_row.breakdown_key, + trigger_value: single_row.trigger_value, + } + }) + .collect::>(); + + attribution_and_capping_and_aggregation::< + _, + BreakdownKey, + TriggerValue, + F, + _, + Replicated, + >(ctx, sharded_input, user_cap.ilog2().try_into().unwrap()) + .await + .unwrap() + }, + ) + .await + .reconstruct(); + + let mut result = result + .into_iter() + .map(|v| u32::try_from(v.as_u128()).unwrap()) + .collect::>(); + let _ = result.split_off(expected_results.len()); + assert_eq!(result, expected_results); +} From 2692af8241a7b5ec13e120286a6fc91ad2167f0f Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 24 Oct 2023 15:28:13 +0800 Subject: [PATCH 067/124] Now oprf can run for compact-gate as well --- scripts/collect_steps.py | 2 +- src/protocol/mod.rs | 1 - src/protocol/prf_sharding/mod.rs | 1 - src/protocol/step/steps.txt | 427 +++++++++++++++++++++++++++++++ 4 files changed, 428 insertions(+), 3 deletions(-) diff --git a/scripts/collect_steps.py b/scripts/collect_steps.py index 3768fbe07..105cab428 100755 --- a/scripts/collect_steps.py +++ b/scripts/collect_steps.py @@ -175,4 +175,4 @@ def extract_intermediate_steps(steps): sorted_steps = sorted(full_steps) for step in sorted_steps: - print(step) + print(step) \ No newline at end of file diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 03270a1e1..cc0ae5ab8 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -6,7 +6,6 @@ pub mod context; pub mod dp; pub mod ipa; pub mod modulus_conversion; -#[cfg(feature = "descriptive-gate")] pub mod prf_sharding; pub mod prss; pub mod sort; diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index b6d334ee0..7feb01f9a 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -26,7 +26,6 @@ use crate::{ }; pub mod bucket; -#[cfg(feature = "descriptive-gate")] pub mod feature_label_dot_product; pub struct PrfShardedIpaInputRow { diff --git a/src/protocol/step/steps.txt b/src/protocol/step/steps.txt index d2bc41200..fc5726835 100644 --- a/src/protocol/step/steps.txt +++ b/src/protocol/step/steps.txt @@ -8812,3 +8812,430 @@ ipa::protocol::ipa::Step::gen_sort_permutation_from_match_keys/ipa::protocol::so ipa::protocol::ipa::Step::gen_sort_permutation_from_match_keys/ipa::protocol::sort::SortStep::sort_keys/ipa::protocol::step::IpaProtocolStep::sort9/ipa::protocol::sort::SortStep::shuffle_reveal_permutation/ipa::protocol::sort::ShuffleRevealPermutationStep::shuffle/ipa::protocol::sort::ShuffleStep::shuffle1 ipa::protocol::ipa::Step::gen_sort_permutation_from_match_keys/ipa::protocol::sort::SortStep::sort_keys/ipa::protocol::step::IpaProtocolStep::sort9/ipa::protocol::sort::SortStep::shuffle_reveal_permutation/ipa::protocol::sort::ShuffleRevealPermutationStep::shuffle/ipa::protocol::sort::ShuffleStep::shuffle2 ipa::protocol::ipa::Step::gen_sort_permutation_from_match_keys/ipa::protocol::sort::SortStep::sort_keys/ipa::protocol::step::IpaProtocolStep::sort9/ipa::protocol::sort::SortStep::shuffle_reveal_permutation/ipa::protocol::sort::ShuffleRevealPermutationStep::shuffle/ipa::protocol::sort::ShuffleStep::shuffle3 +ipa::protocol::prf_sharding::Step::binary_validator +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit4/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit5/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit6/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit7/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_breakdown_key_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit0/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit1/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::upgrade/ipa::protocol::context::semi_honest::UpgradeStep::upgrade_semi_honest/ipa::protocol::context::upgrade::UpgradeTripleStep::upgrade_bit_triple2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit2/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::xor2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::modulus_convert_conversion_value_bits/ipa::protocol::modulus_conversion::convert_shares::ConvertSharesStep::convert_bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit100 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit101 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit102 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit103 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit104 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit105 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit106 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit107 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit108 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit109 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit110 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit111 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit112 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit113 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit114 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit115 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit116 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit117 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit118 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit119 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit120 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit121 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit122 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit123 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit124 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit125 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit126 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit127 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit20 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit21 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit22 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit23 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit24 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit25 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit26 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit27 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit28 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit29 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit30 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit31 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit32 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit33 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit34 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit35 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit36 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit37 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit38 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit39 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit40 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit41 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit42 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit43 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit44 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit45 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit46 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit47 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit48 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit49 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit50 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit51 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit52 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit53 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit54 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit55 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit56 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit57 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit58 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit59 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit60 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit61 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit62 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit63 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit64 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit65 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit66 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit67 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit68 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit69 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit70 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit71 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit72 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit73 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit74 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit75 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit76 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit77 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit78 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit79 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit80 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit81 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit82 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit83 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit84 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit85 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit86 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit87 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit88 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit89 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit90 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit91 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit92 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit93 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit94 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit95 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit96 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit97 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit98 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth0/ipa::protocol::prf_sharding::bucket::BucketStep::bit99 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit20 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit21 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit22 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit23 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit24 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit25 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit26 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit27 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit28 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit29 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit30 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit31 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit32 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit33 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit34 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit35 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit36 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit37 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit38 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit39 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit40 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit41 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit42 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit43 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit44 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit45 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit46 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit47 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit48 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit49 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit50 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit51 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit52 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit53 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit54 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit55 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit56 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit57 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit58 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit59 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit60 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit61 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit62 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit63 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth1/ipa::protocol::prf_sharding::bucket::BucketStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit20 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit21 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit22 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit23 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit24 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit25 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit26 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit27 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit28 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit29 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit30 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit31 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth2/ipa::protocol::prf_sharding::bucket::BucketStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth3/ipa::protocol::prf_sharding::bucket::BucketStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth4/ipa::protocol::prf_sharding::bucket::BucketStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth5/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth5/ipa::protocol::prf_sharding::bucket::BucketStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth5/ipa::protocol::prf_sharding::bucket::BucketStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth5/ipa::protocol::prf_sharding::bucket::BucketStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth6/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth6/ipa::protocol::prf_sharding::bucket::BucketStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::Step::move_value_to_correct_breakdown/ipa::protocol::prf_sharding::BinaryTreeDepthStep::depth7/ipa::protocol::prf_sharding::bucket::BucketStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_difference_to_cap +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::did_trigger_get_attributed +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::ever_encountered_source_event +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::is_saturated_and_prev_row_not_saturated +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_difference_to_cap +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::did_trigger_get_attributed +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::ever_encountered_source_event +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::is_saturated_and_prev_row_not_saturated From 2a798a3c90a97bad639fcd9714e509d4a4fc9783 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 24 Oct 2023 15:56:11 +0800 Subject: [PATCH 068/124] Adding oprf handling in collect_steps.py --- scripts/collect_steps.py | 26 ++++++++++++++++++++++++++ src/protocol/prf_sharding/mod.rs | 1 + src/protocol/step/steps.txt | 6 ++++++ 3 files changed, 33 insertions(+) diff --git a/scripts/collect_steps.py b/scripts/collect_steps.py index 105cab428..bf47eefbe 100755 --- a/scripts/collect_steps.py +++ b/scripts/collect_steps.py @@ -31,6 +31,9 @@ SECURITY_MODEL = ["malicious", "semi-honest"] ROOT_STEP_PREFIX = "protocol/alloc::string::String::run-0" +OPRF_USER_CAP = [16, 64, 128] +OPRF_SECURITY_MODEL = ["semi-honest"] +OPRF_TRIGGER_VALUE = [4] # TODO(taikiy): #771 allows us to remove this synthetic step generation code # There are protocols in IPA that that will generate log(N) steps where N is the number @@ -171,6 +174,29 @@ def extract_intermediate_steps(steps): print(" ".join(args), file=sys.stderr) steps.update(collect_steps(args)) + for c in OPRF_USER_CAP: + for w in ATTRIBUTION_WINDOW: + for b in BREAKDOWN_KEYS: + for m in OPRF_SECURITY_MODEL: + for tv in OPRF_TRIGGER_VALUE: + args = ARGS + [ + "-n", + str(QUERY_SIZE), + "-c", + str(c), + "-w", + str(w), + "-b", + str(b), + "-m", + m, + "-t", + str(tv), + "-o" + ] + print(" ".join(args), file=sys.stderr) + steps.update(collect_steps(args)) + full_steps = extract_intermediate_steps(steps) sorted_steps = sorted(full_steps) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 7feb01f9a..b6d334ee0 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -26,6 +26,7 @@ use crate::{ }; pub mod bucket; +#[cfg(feature = "descriptive-gate")] pub mod feature_label_dot_product; pub struct PrfShardedIpaInputRow { diff --git a/src/protocol/step/steps.txt b/src/protocol/step/steps.txt index fc5726835..042e01c6f 100644 --- a/src/protocol/step/steps.txt +++ b/src/protocol/step/steps.txt @@ -9193,6 +9193,9 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit2 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit3 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit7 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 @@ -9228,6 +9231,9 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit2 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit3 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit7 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 From 8fa67457b91296d4263d780a691f30af91c55942 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 24 Oct 2023 16:08:17 +0800 Subject: [PATCH 069/124] refactoring collect_steps --- scripts/collect_steps.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/collect_steps.py b/scripts/collect_steps.py index bf47eefbe..9664a1c06 100755 --- a/scripts/collect_steps.py +++ b/scripts/collect_steps.py @@ -31,9 +31,6 @@ SECURITY_MODEL = ["malicious", "semi-honest"] ROOT_STEP_PREFIX = "protocol/alloc::string::String::run-0" -OPRF_USER_CAP = [16, 64, 128] -OPRF_SECURITY_MODEL = ["semi-honest"] -OPRF_TRIGGER_VALUE = [4] # TODO(taikiy): #771 allows us to remove this synthetic step generation code # There are protocols in IPA that that will generate log(N) steps where N is the number @@ -152,9 +149,7 @@ def extract_intermediate_steps(steps): return steps - -if __name__ == "__main__": - steps = set() +def ipa_steps(steps): for c in PER_USER_CAP: for w in ATTRIBUTION_WINDOW: for b in BREAKDOWN_KEYS: @@ -174,6 +169,11 @@ def extract_intermediate_steps(steps): print(" ".join(args), file=sys.stderr) steps.update(collect_steps(args)) +OPRF_USER_CAP = [16, 64, 128] +OPRF_SECURITY_MODEL = ["semi-honest"] +OPRF_TRIGGER_VALUE = [4] + +def oprf_steps(steps): for c in OPRF_USER_CAP: for w in ATTRIBUTION_WINDOW: for b in BREAKDOWN_KEYS: @@ -197,6 +197,11 @@ def extract_intermediate_steps(steps): print(" ".join(args), file=sys.stderr) steps.update(collect_steps(args)) +if __name__ == "__main__": + steps = set() + ipa_steps(steps) + oprf_steps(steps) + full_steps = extract_intermediate_steps(steps) sorted_steps = sorted(full_steps) From d9d9672e9c645fa4c4569d8dd9a5c1613747fd22 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 24 Oct 2023 16:58:43 +0800 Subject: [PATCH 070/124] Add test for no contribution + handle 512 breakdowns in test using Gf9Bit --- src/ff/galois_field.rs | 17 ++++++++++++++++- src/ff/mod.rs | 2 +- src/protocol/prf_sharding/bucket.rs | 19 +++++++++++++------ src/protocol/prf_sharding/mod.rs | 23 +++++++++++++++++++++++ 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/ff/galois_field.rs b/src/ff/galois_field.rs index 18b6b58c8..7432b54d8 100644 --- a/src/ff/galois_field.rs +++ b/src/ff/galois_field.rs @@ -5,7 +5,7 @@ use std::{ use bitvec::prelude::{bitarr, BitArr, Lsb0}; use generic_array::GenericArray; -use typenum::{Unsigned, U1, U4, U5}; +use typenum::{Unsigned, U1, U2, U4, U5}; use crate::{ ff::{Field, Serializable}, @@ -25,6 +25,7 @@ pub trait GaloisField: // Bit store type definitions type U8_1 = BitArr!(for 8, in u8, Lsb0); +type U8_2 = BitArr!(for 9, in u8, Lsb0); type U8_4 = BitArr!(for 32, in u8, Lsb0); type U8_5 = BitArr!(for 40, in u8, Lsb0); @@ -32,6 +33,10 @@ impl Block for U8_1 { type Size = U1; } +impl Block for U8_2 { + type Size = U2; +} + impl Block for U8_4 { type Size = U4; } @@ -560,6 +565,16 @@ bit_array_impl!( 0b1_0001_1011_u128 ); +bit_array_impl!( + bit_array_9, + Gf9Bit, + U8_2, + 9, + bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0), + // x^9 + x^4 + x^3 + x + 1 + 0b10_0001_1011_u128 +); + bit_array_impl!( bit_array_5, Gf5Bit, diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 5544f2378..1cb8c0c43 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -9,7 +9,7 @@ mod prime_field; use std::ops::{Add, AddAssign, Sub, SubAssign}; pub use field::{Field, FieldType}; -pub use galois_field::{GaloisField, Gf2, Gf32Bit, Gf3Bit, Gf40Bit, Gf5Bit, Gf8Bit}; +pub use galois_field::{GaloisField, Gf2, Gf32Bit, Gf3Bit, Gf40Bit, Gf5Bit, Gf8Bit, Gf9Bit}; use generic_array::{ArrayLength, GenericArray}; #[cfg(any(test, feature = "weak-field"))] pub use prime_field::Fp31; diff --git a/src/protocol/prf_sharding/bucket.rs b/src/protocol/prf_sharding/bucket.rs index d199557bd..d2ad77a11 100644 --- a/src/protocol/prf_sharding/bucket.rs +++ b/src/protocol/prf_sharding/bucket.rs @@ -18,9 +18,16 @@ pub enum BucketStep { Bit(usize), } -impl From for BucketStep { - fn from(v: u32) -> Self { - Self::Bit(usize::try_from(v).unwrap()) +impl TryFrom for BucketStep { + type Error = String; + + fn try_from(v: u32) -> Result { + let val = usize::try_from(v); + let val = match val { + Ok(val) => Self::Bit(val), + Err(error) => panic!("{error:?}"), + }; + Ok(val) } } @@ -114,7 +121,7 @@ pub mod tests { use rand::thread_rng; use crate::{ - ff::{Field, Fp32BitPrime, Gf8Bit}, + ff::{Field, Fp32BitPrime, Gf8Bit, Gf9Bit}, protocol::{ context::{Context, UpgradableContext, Validator}, prf_sharding::bucket::move_single_value_to_bucket, @@ -225,7 +232,7 @@ pub mod tests { #[should_panic] fn move_out_of_range_too_many_buckets_steps() { run(move || async move { - let breakdown_key_bits = get_bits::(0, Gf8Bit::BITS); + let breakdown_key_bits = get_bits::(0, Gf9Bit::BITS); let value = Fp32BitPrime::truncate_from(VALUE); _ = TestWorld::default() @@ -234,7 +241,7 @@ pub mod tests { |ctx, (breakdown_key_share, value_share)| async move { let validator = ctx.validator(); let ctx = validator.context(); - move_single_value_to_bucket::( + move_single_value_to_bucket::( ctx.set_total_records(1), RecordId::from(0), breakdown_key_share, diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 4412d0e0a..94250fdfa 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -1003,4 +1003,27 @@ pub mod tests { assert_eq!(result, &expected); }); } + + #[test] + fn semi_honest_aggregation_empty_input() { + run(|| async move { + let world = TestWorld::default(); + + let records: Vec = vec![]; + + let expected = [0_u128; 32]; + + let result: Vec<_> = world + .semi_honest(records.into_iter(), |ctx, input_rows| async move { + let validator = ctx.validator(); + let ctx = validator.context(); + do_aggregation::<_, Gf5Bit, Gf3Bit, Fp32BitPrime, _>(ctx, input_rows) + .await + .unwrap() + }) + .await + .reconstruct(); + assert_eq!(result, &expected); + }); + } } From 5ea37ca4b61a908039d6e18fd91be5f5345f9515 Mon Sep 17 00:00:00 2001 From: Ben Savage Date: Wed, 25 Oct 2023 14:54:23 +0800 Subject: [PATCH 071/124] Supporting the opposite capping logic --- src/test_fixture/ipa.rs | 52 ++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/test_fixture/ipa.rs b/src/test_fixture/ipa.rs index 37e31b535..ef7b32c8a 100644 --- a/src/test_fixture/ipa.rs +++ b/src/test_fixture/ipa.rs @@ -77,12 +77,18 @@ pub fn ipa_in_the_clear( &mut breakdowns, per_user_cap, attribution_window, + CappingOrder::CapOldestFirst, ); } breakdowns } +enum CappingOrder { + CapOldestFirst, + CapMostRecentFirst, +} + /// Assumes records all belong to the same user, and are in reverse chronological order /// Will give incorrect results if this is not true #[allow(clippy::missing_panics_doc)] @@ -91,6 +97,7 @@ fn update_expected_output_for_user<'a, I: IntoIterator, + order: CappingOrder, ) { let within_window = |value: u64| -> bool { if let Some(window) = attribution_window_seconds { @@ -102,17 +109,13 @@ fn update_expected_output_for_user<'a, I: IntoIterator= per_user_cap { - break; - } - if record.is_trigger_report { pending_trigger_reports.push(record); } else if !pending_trigger_reports.is_empty() { - for trigger_report in &pending_trigger_reports { + for trigger_report in pending_trigger_reports.into_iter() { let time_delta_to_source_report = trigger_report.timestamp - record.timestamp; // only count trigger reports that are within the attribution window @@ -121,16 +124,39 @@ fn update_expected_output_for_user<'a, I: IntoIterator update_breakdowns( + attributed_triggers.into_iter(), + expected_results, + per_user_cap, + ), + CappingOrder::CapMostRecentFirst => update_breakdowns( + attributed_triggers.into_iter().rev(), + expected_results, + per_user_cap, + ), + } +} + +fn update_breakdowns<'a, I>(attributed_triggers: I, expected_results: &mut [u32], per_user_cap: u32) +where + I: IntoIterator, +{ + let mut total_contribution = 0; + for (trigger_report, source_report) in attributed_triggers { + let delta_to_per_user_cap = per_user_cap - total_contribution; + let capped_contribution = + std::cmp::min(delta_to_per_user_cap, trigger_report.trigger_value); + let bk: usize = source_report.breakdown_key.try_into().unwrap(); + expected_results[bk] += capped_contribution; + total_contribution += capped_contribution; + } } /// # Panics From dd07816234150a9c49b74bdfece59ea14e6009e9 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Wed, 25 Oct 2023 15:32:31 +0800 Subject: [PATCH 072/124] Peer feedback --- benches/oneshot/ipa.rs | 8 +++++++- pre-commit | 6 +++++- scripts/collect_steps.py | 3 ++- src/bin/report_collector.rs | 1 + src/protocol/ipa/mod.rs | 3 ++- src/protocol/prf_sharding/mod.rs | 10 +++++----- src/test_fixture/ipa.rs | 28 ++++++++++++++++++---------- 7 files changed, 40 insertions(+), 19 deletions(-) diff --git a/benches/oneshot/ipa.rs b/benches/oneshot/ipa.rs index ab6e723e2..ae8826f6a 100644 --- a/benches/oneshot/ipa.rs +++ b/benches/oneshot/ipa.rs @@ -9,7 +9,7 @@ use ipa::{ ff::Fp32BitPrime, helpers::{query::IpaQueryConfig, GatewayConfig}, test_fixture::{ - ipa::{ipa_in_the_clear, test_ipa, test_oprf_ipa, IpaSecurityModel}, + ipa::{ipa_in_the_clear, test_ipa, test_oprf_ipa, CappingOrder, IpaSecurityModel}, EventGenerator, EventGeneratorConfig, TestWorld, TestWorldConfig, }, }; @@ -123,11 +123,17 @@ async fn run(args: Args) -> Result<(), Error> { .take(args.query_size) .collect::>(); + let order = if args.oprf { + CappingOrder::CapOldestFirst + } else { + CappingOrder::CapMostRecentFirst + }; let expected_results = ipa_in_the_clear( &raw_data, args.per_user_cap, args.attribution_window(), args.breakdown_keys, + &order, ); let world = TestWorld::new_with(config.clone()); diff --git a/pre-commit b/pre-commit index 6cf2be072..c60774f4d 100755 --- a/pre-commit +++ b/pre-commit @@ -99,8 +99,12 @@ check "Concurrency tests" \ check "IPA benchmark" \ cargo bench --bench oneshot_ipa --no-default-features --features="enable-benches descriptive-gate" -- -n 62 +check "IPA OPRF benchmark" \ + cargo bench --bench oneshot_ipa --no-default-features --features="enable-benches descriptive-gate" -- -n 62 --oprf + check "Arithmetic circuit benchmark" \ cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate" check "Sort benchmark" \ - cargo bench --bench oneshot_sort --no-default-features --features="enable-benches descriptive-gate" \ No newline at end of file + cargo bench --bench oneshot_sort --no-default-features --features="enable-benches descriptive-gate" + diff --git a/scripts/collect_steps.py b/scripts/collect_steps.py index 9664a1c06..3e87d3a2f 100755 --- a/scripts/collect_steps.py +++ b/scripts/collect_steps.py @@ -206,4 +206,5 @@ def oprf_steps(steps): sorted_steps = sorted(full_steps) for step in sorted_steps: - print(step) \ No newline at end of file + print(step) + \ No newline at end of file diff --git a/src/bin/report_collector.rs b/src/bin/report_collector.rs index bac36b0db..fb55cd2a1 100644 --- a/src/bin/report_collector.rs +++ b/src/bin/report_collector.rs @@ -247,6 +247,7 @@ async fn ipa( ipa_query_config.per_user_credit_cap, ipa_query_config.attribution_window_seconds, ipa_query_config.max_breakdown_key, + CappingOrder::CapOldestFirst, ); // pad the output vector to the max breakdown key, to make sure it is aligned with the MPC results diff --git a/src/protocol/ipa/mod.rs b/src/protocol/ipa/mod.rs index b8f516684..bac3d6e82 100644 --- a/src/protocol/ipa/mod.rs +++ b/src/protocol/ipa/mod.rs @@ -464,7 +464,7 @@ pub mod tests { test_executor::{run, run_with}, test_fixture::{ input::GenericReportTestInput, - ipa::{ipa_in_the_clear, test_ipa, IpaSecurityModel}, + ipa::{ipa_in_the_clear, test_ipa, CappingOrder, IpaSecurityModel}, logging, EventGenerator, EventGeneratorConfig, Reconstruct, Runner, TestWorld, TestWorldConfig, }, @@ -815,6 +815,7 @@ pub mod tests { per_user_cap, ATTRIBUTION_WINDOW_SECONDS, MAX_BREAKDOWN_KEY, + &CappingOrder::CapOldestFirst, ); let config = TestWorldConfig { diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index a0e7909fa..87832f797 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -37,11 +37,11 @@ pub mod bucket; pub mod feature_label_dot_product; pub struct PrfShardedIpaInputRow { - prf_of_match_key: u64, - is_trigger_bit: Replicated, - breakdown_key: Replicated, - trigger_value: Replicated, - timestamp: Replicated, + pub prf_of_match_key: u64, + pub is_trigger_bit: Replicated, + pub breakdown_key: Replicated, + pub trigger_value: Replicated, + pub timestamp: Replicated, } impl PrfShardedIpaInputRow { diff --git a/src/test_fixture/ipa.rs b/src/test_fixture/ipa.rs index 153ce77e3..a33b92e5b 100644 --- a/src/test_fixture/ipa.rs +++ b/src/test_fixture/ipa.rs @@ -48,6 +48,7 @@ pub fn ipa_in_the_clear( per_user_cap: u32, attribution_window: Option, max_breakdown: u32, + order: &CappingOrder, ) -> Vec { // build a view that is convenient for attribution. match key -> events sorted by timestamp in reverse // that is more memory intensive, but should be faster to compute. We can always opt-out and @@ -81,14 +82,14 @@ pub fn ipa_in_the_clear( &mut breakdowns, per_user_cap, attribution_window, - CappingOrder::CapOldestFirst, + order, ); } breakdowns } -enum CappingOrder { +pub enum CappingOrder { CapOldestFirst, CapMostRecentFirst, } @@ -101,7 +102,7 @@ fn update_expected_output_for_user<'a, I: IntoIterator, - order: CappingOrder, + order: &CappingOrder, ) { let within_window = |value: u64| -> bool { if let Some(window) = attribution_window_seconds { @@ -119,7 +120,7 @@ fn update_expected_output_for_user<'a, I: IntoIterator update_breakdowns( - attributed_triggers.into_iter(), - expected_results, - per_user_cap, - ), + CappingOrder::CapOldestFirst => { + update_breakdowns(attributed_triggers, expected_results, per_user_cap); + } CappingOrder::CapMostRecentFirst => update_breakdowns( attributed_triggers.into_iter().rev(), expected_results, @@ -272,6 +271,7 @@ pub async fn test_oprf_ipa( is_trigger_bit: is_trigger_bit_share, breakdown_key: single_row.breakdown_key, trigger_value: single_row.trigger_value, + timestamp: single_row.timestamp, } }) .collect::>(); @@ -280,10 +280,16 @@ pub async fn test_oprf_ipa( _, BreakdownKey, TriggerValue, + Timestamp, F, _, Replicated, - >(ctx, sharded_input, user_cap.ilog2().try_into().unwrap()) + >( + ctx, + sharded_input, + user_cap.ilog2().try_into().unwrap(), + config.attribution_window_seconds, + ) .await .unwrap() }, @@ -295,6 +301,8 @@ pub async fn test_oprf_ipa( .into_iter() .map(|v| u32::try_from(v.as_u128()).unwrap()) .collect::>(); + + //TODO(richaj): To be removed once the function supports non power of 2 breakdowns let _ = result.split_off(expected_results.len()); assert_eq!(result, expected_results); } From 51e05fe4c6b6682893b5db65f7c6d84384f538f0 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Wed, 25 Oct 2023 16:42:39 +0800 Subject: [PATCH 073/124] Some more fixes --- benches/oneshot/ipa.rs | 4 +- pre-commit | 1 - scripts/collect_steps.py | 6 +- src/bin/report_collector.rs | 4 +- src/protocol/step/steps.txt | 238 +++++++++++++++++++++++++++++- src/test_fixture/input/sharing.rs | 11 +- steps1.txt | 1 + 7 files changed, 251 insertions(+), 14 deletions(-) create mode 100644 steps1.txt diff --git a/benches/oneshot/ipa.rs b/benches/oneshot/ipa.rs index ae8826f6a..c597b7e29 100644 --- a/benches/oneshot/ipa.rs +++ b/benches/oneshot/ipa.rs @@ -124,9 +124,9 @@ async fn run(args: Args) -> Result<(), Error> { .collect::>(); let order = if args.oprf { - CappingOrder::CapOldestFirst - } else { CappingOrder::CapMostRecentFirst + } else { + CappingOrder::CapOldestFirst }; let expected_results = ipa_in_the_clear( &raw_data, diff --git a/pre-commit b/pre-commit index c60774f4d..40dcbf745 100755 --- a/pre-commit +++ b/pre-commit @@ -107,4 +107,3 @@ check "Arithmetic circuit benchmark" \ check "Sort benchmark" \ cargo bench --bench oneshot_sort --no-default-features --features="enable-benches descriptive-gate" - diff --git a/scripts/collect_steps.py b/scripts/collect_steps.py index 3e87d3a2f..3e1371ef2 100755 --- a/scripts/collect_steps.py +++ b/scripts/collect_steps.py @@ -169,14 +169,15 @@ def ipa_steps(steps): print(" ".join(args), file=sys.stderr) steps.update(collect_steps(args)) +OPRF_BREAKDOWN_KEYS = [256] OPRF_USER_CAP = [16, 64, 128] OPRF_SECURITY_MODEL = ["semi-honest"] -OPRF_TRIGGER_VALUE = [4] +OPRF_TRIGGER_VALUE = [7] def oprf_steps(steps): for c in OPRF_USER_CAP: for w in ATTRIBUTION_WINDOW: - for b in BREAKDOWN_KEYS: + for b in OPRF_BREAKDOWN_KEYS: for m in OPRF_SECURITY_MODEL: for tv in OPRF_TRIGGER_VALUE: args = ARGS + [ @@ -207,4 +208,3 @@ def oprf_steps(steps): for step in sorted_steps: print(step) - \ No newline at end of file diff --git a/src/bin/report_collector.rs b/src/bin/report_collector.rs index fb55cd2a1..a8ca7ba82 100644 --- a/src/bin/report_collector.rs +++ b/src/bin/report_collector.rs @@ -26,7 +26,7 @@ use ipa::{ protocol::{BreakdownKey, MatchKey}, report::{KeyIdentifier, DEFAULT_KEY_ID}, test_fixture::{ - ipa::{ipa_in_the_clear, IpaSecurityModel, TestRawDataRecord}, + ipa::{ipa_in_the_clear, CappingOrder, IpaSecurityModel, TestRawDataRecord}, EventGenerator, EventGeneratorConfig, }, }; @@ -247,7 +247,7 @@ async fn ipa( ipa_query_config.per_user_credit_cap, ipa_query_config.attribution_window_seconds, ipa_query_config.max_breakdown_key, - CappingOrder::CapOldestFirst, + &CappingOrder::CapOldestFirst, ); // pad the output vector to the max breakdown key, to make sure it is aligned with the MPC results diff --git a/src/protocol/step/steps.txt b/src/protocol/step/steps.txt index 042e01c6f..2079b4ee3 100644 --- a/src/protocol/step/steps.txt +++ b/src/protocol/step/steps.txt @@ -9180,6 +9180,52 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit6 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit7 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::attributed_event_check_flag +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::dot_product +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::did_trigger_get_attributed ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit1 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit2 @@ -9204,9 +9250,29 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit1 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit2 -ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::did_trigger_get_attributed ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::ever_encountered_source_event ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::is_saturated_and_prev_row_not_saturated +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row1/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit9 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit0 @@ -9218,6 +9284,52 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit6 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit7 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::attributed_event_check_flag +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::dot_product +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::did_trigger_get_attributed ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit1 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit2 @@ -9233,7 +9345,6 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit5 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit6 -ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit7 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 @@ -9242,6 +9353,127 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit1 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit2 -ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::did_trigger_get_attributed ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::ever_encountered_source_event ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::is_saturated_and_prev_row_not_saturated +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_breakdown_key/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::attributed_event_check_flag +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::dot_product +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compare_time_delta_to_attribution_window/ipa::protocol::boolean::comparison::Step::prefix_or/ipa::protocol::step::BitOpStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::check_attribution_window/ipa::protocol::prf_sharding::Step::compute_time_delta/ipa::protocol::step::BitOpStep::bit9 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::prf_sharding::Step::did_trigger_get_attributed +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::attributed_trigger_value/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_difference_to_cap +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_difference_to_cap/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_not_saturated_case/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::ever_encountered_source_event +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::is_saturated_and_prev_row_not_saturated +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit0 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit1 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit10 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit11 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit12 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit13 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit14 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit15 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit16 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit17 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit18 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit19 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit2 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit3 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit7 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit8 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::source_event_timestamp/ipa::protocol::step::BitOpStep::bit9 diff --git a/src/test_fixture/input/sharing.rs b/src/test_fixture/input/sharing.rs index 1287ad8e0..e5e306e7d 100644 --- a/src/test_fixture/input/sharing.rs +++ b/src/test_fixture/input/sharing.rs @@ -367,9 +367,14 @@ where } else { EventType::Source }; - let timestamp = TS::truncate_from(self.timestamp).share_with(rng); - let breakdown_key = BK::truncate_from(self.breakdown_key).share_with(rng); - let trigger_value = TV::truncate_from(self.trigger_value).share_with(rng); + let timestamp: [Replicated; 3] = + TS::try_from(self.timestamp.into()).unwrap().share_with(rng); + let breakdown_key = BK::try_from(self.breakdown_key.into()) + .unwrap() + .share_with(rng); + let trigger_value = TV::try_from(self.trigger_value.into()) + .unwrap() + .share_with(rng); zip(zip(timestamp, breakdown_key), trigger_value) .map(|((ts_share, bk_share), tv_share)| OprfReport { diff --git a/steps1.txt b/steps1.txt new file mode 100644 index 000000000..ff3d9f397 --- /dev/null +++ b/steps1.txt @@ -0,0 +1 @@ +No steps in the output From d8c059871721e8b61f19fa441c3951517b9f2ede Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Wed, 25 Oct 2023 23:45:20 +0800 Subject: [PATCH 074/124] Some code refactor based on Taiki's feedback --- 0001-CI-split-extra-step-into-two.patch | 88 ------------------------- pre-commit | 2 +- scripts/collect_steps.py | 58 ++++++++-------- src/protocol/step/steps.txt | 4 ++ steps1.txt | 1 - 5 files changed, 35 insertions(+), 118 deletions(-) delete mode 100644 0001-CI-split-extra-step-into-two.patch delete mode 100644 steps1.txt diff --git a/0001-CI-split-extra-step-into-two.patch b/0001-CI-split-extra-step-into-two.patch deleted file mode 100644 index 85d7146d2..000000000 --- a/0001-CI-split-extra-step-into-two.patch +++ /dev/null @@ -1,88 +0,0 @@ -From ba6c503fbe930a9298d83c04bab15f329faf13ae Mon Sep 17 00:00:00 2001 -From: Alex Koshelev -Date: Fri, 11 Aug 2023 14:25:03 -0700 -Subject: [PATCH] CI: split extra step into two - -This is an attempt to increase parallelism on CI. Those builds take ~4 mins and running tests take another 5-6 mins. Splitting this step should make CI run faster. ---- - .github/workflows/check.yml | 42 ++++++++++++++++++++++++++++--------- - 1 file changed, 32 insertions(+), 10 deletions(-) - -diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml -index 54f869ac..9a8e1886 100644 ---- a/.github/workflows/check.yml -+++ b/.github/workflows/check.yml -@@ -15,13 +15,15 @@ on: - - "benches/**/*" - - "tests/**/*" - -+env: -+ CARGO_TERM_COLOR: always -+ RUSTFLAGS: -D warnings -+ RUSTDOCFLAGS: -D warnings -+ - jobs: - basic: - name: Basic Checks - env: -- CARGO_TERM_COLOR: always -- RUSTFLAGS: -D warnings -- RUSTDOCFLAGS: -D warnings - CARGO_INCREMENTAL: 0 - - runs-on: ubuntu-latest -@@ -64,12 +66,13 @@ jobs: - - name: Run Web Tests - run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate" - -- extra: -- name: Additional Builds and Concurrency Tests -+ - name: Run compact gate tests -+ run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate" -+ -+ extra-builds: -+ name: Additional Builds - env: -- CARGO_TERM_COLOR: always - RUSTFLAGS: -D warnings -C target-cpu=native -- RUSTDOCFLAGS: -D warnings - - runs-on: ubuntu-latest - -@@ -102,11 +105,30 @@ jobs: - - name: Build concurrency tests (debug mode) - run: cargo build --features shuttle - -+ benches-and-fuzzy: -+ name: Run benchmarks and concurrency tests -+ -+ runs-on: ubuntu-latest -+ -+ steps: -+ - uses: actions/checkout@v3 -+ -+ - uses: dtolnay/rust-toolchain@stable -+ with: -+ components: clippy,rustfmt -+ -+ - uses: actions/cache@v3 -+ with: -+ path: | -+ ~/.cargo/bin/ -+ ~/.cargo/registry/index/ -+ ~/.cargo/registry/cache/ -+ ~/.cargo/git/db/ -+ target/ -+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} -+ - - name: Run concurrency tests - run: cargo test --release --features shuttle - - - name: Run IPA bench - run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches descriptive-gate" -- -- - name: Run compact gate tests -- run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate" --- -2.31.1 - diff --git a/pre-commit b/pre-commit index 40dcbf745..d60bece9e 100755 --- a/pre-commit +++ b/pre-commit @@ -100,7 +100,7 @@ check "IPA benchmark" \ cargo bench --bench oneshot_ipa --no-default-features --features="enable-benches descriptive-gate" -- -n 62 check "IPA OPRF benchmark" \ - cargo bench --bench oneshot_ipa --no-default-features --features="enable-benches descriptive-gate" -- -n 62 --oprf + cargo bench --bench oneshot_ipa --no-default-features --features="enable-benches descriptive-gate" -- -n 62 --oprf -c 16 check "Arithmetic circuit benchmark" \ cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate" diff --git a/scripts/collect_steps.py b/scripts/collect_steps.py index 3e1371ef2..23c8ffc5e 100755 --- a/scripts/collect_steps.py +++ b/scripts/collect_steps.py @@ -149,7 +149,8 @@ def extract_intermediate_steps(steps): return steps -def ipa_steps(steps): +def ipa_steps(): + output = set() for c in PER_USER_CAP: for w in ATTRIBUTION_WINDOW: for b in BREAKDOWN_KEYS: @@ -167,41 +168,42 @@ def ipa_steps(steps): m, ] print(" ".join(args), file=sys.stderr) - steps.update(collect_steps(args)) + output.update(collect_steps(args)) + return output -OPRF_BREAKDOWN_KEYS = [256] +OPRF_BREAKDOWN_KEY = 256 OPRF_USER_CAP = [16, 64, 128] -OPRF_SECURITY_MODEL = ["semi-honest"] -OPRF_TRIGGER_VALUE = [7] +OPRF_SECURITY_MODEL = "semi-honest" +OPRF_TRIGGER_VALUE = [6, 7] -def oprf_steps(steps): +def oprf_steps(): + output = set() for c in OPRF_USER_CAP: for w in ATTRIBUTION_WINDOW: - for b in OPRF_BREAKDOWN_KEYS: - for m in OPRF_SECURITY_MODEL: - for tv in OPRF_TRIGGER_VALUE: - args = ARGS + [ - "-n", - str(QUERY_SIZE), - "-c", - str(c), - "-w", - str(w), - "-b", - str(b), - "-m", - m, - "-t", - str(tv), - "-o" - ] - print(" ".join(args), file=sys.stderr) - steps.update(collect_steps(args)) + for tv in OPRF_TRIGGER_VALUE: + args = ARGS + [ + "-n", + str(QUERY_SIZE), + "-c", + str(c), + "-w", + str(w), + "-b", + str(OPRF_BREAKDOWN_KEY), + "-m", + OPRF_SECURITY_MODEL, + "-t", + str(tv), + "-o" + ] + print(" ".join(args), file=sys.stderr) + output.update(collect_steps(args)) + return output if __name__ == "__main__": steps = set() - ipa_steps(steps) - oprf_steps(steps) + steps.update(ipa_steps()) + steps.update(oprf_steps()) full_steps = extract_intermediate_steps(steps) sorted_steps = sorted(full_steps) diff --git a/src/protocol/step/steps.txt b/src/protocol/step/steps.txt index 2079b4ee3..c5bafcded 100644 --- a/src/protocol/step/steps.txt +++ b/src/protocol/step/steps.txt @@ -9345,6 +9345,7 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit5 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit7 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row2/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 @@ -9446,6 +9447,9 @@ ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding: ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit2 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit3 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit4 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit5 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit6 +ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::compute_saturating_sum/ipa::protocol::step::BitOpStep::bit7 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit0 ipa::protocol::prf_sharding::Step::binary_validator/ipa::protocol::prf_sharding::UserNthRowStep::row3/ipa::protocol::prf_sharding::Step::computed_capped_attributed_trigger_value_just_saturated_case/ipa::protocol::step::BitOpStep::bit1 diff --git a/steps1.txt b/steps1.txt deleted file mode 100644 index ff3d9f397..000000000 --- a/steps1.txt +++ /dev/null @@ -1 +0,0 @@ -No steps in the output From 31a155c6067ca4546f7073db0116548490e965a4 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Thu, 26 Oct 2023 14:27:51 +0800 Subject: [PATCH 075/124] OPRF test on real world infra --- src/bin/report_collector.rs | 67 ++++++++++++----- src/cli/playbook/ipa.rs | 95 +++++++++++++++++++++++- src/cli/playbook/mod.rs | 3 +- src/helpers/transport/query.rs | 3 + src/net/http_serde.rs | 8 ++- src/query/executor.rs | 28 ++++++++ src/query/runner/mod.rs | 4 +- src/query/runner/oprf_ipa.rs | 127 +++++++++++++++++++++++++++++++++ src/report.rs | 94 +++++++++++++++++++++++- 9 files changed, 404 insertions(+), 25 deletions(-) create mode 100644 src/query/runner/oprf_ipa.rs diff --git a/src/bin/report_collector.rs b/src/bin/report_collector.rs index a8ca7ba82..89b80bb0e 100644 --- a/src/bin/report_collector.rs +++ b/src/bin/report_collector.rs @@ -15,7 +15,7 @@ use hyper::http::uri::Scheme; use ipa::{ cli::{ noise::{apply, ApplyDpArgs}, - playbook::{make_clients, playbook_ipa, validate, InputSource}, + playbook::{make_clients, playbook_ipa, playbook_oprf_ipa, validate, InputSource}, CsvSerializer, IpaQueryResult, Verbosity, }, config::NetworkConfig, @@ -103,6 +103,8 @@ enum ReportCollectorCommand { }, /// Apply differential privacy noise to IPA inputs ApplyDpNoise(ApplyDpArgs), + /// Execute OPRF IPA in a semi-honest majority setting + OprfIpa(IpaQueryConfig), } #[derive(Debug, clap::Args)] @@ -134,6 +136,7 @@ async fn main() -> Result<(), Box> { IpaSecurityModel::SemiHonest, config, &clients, + false, ) .await? } @@ -144,6 +147,7 @@ async fn main() -> Result<(), Box> { IpaSecurityModel::Malicious, config, &clients, + false, ) .await? } @@ -153,6 +157,17 @@ async fn main() -> Result<(), Box> { gen_args, } => gen_inputs(count, seed, args.output_file, gen_args)?, ReportCollectorCommand::ApplyDpNoise(ref dp_args) => apply_dp_noise(&args, dp_args)?, + ReportCollectorCommand::OprfIpa(config) => { + ipa( + &args, + &network, + IpaSecurityModel::SemiHonest, + config, + &clients, + true, + ) + .await? + } }; Ok(()) @@ -221,16 +236,29 @@ async fn ipa( security_model: IpaSecurityModel, ipa_query_config: IpaQueryConfig, helper_clients: &[MpcHelperClient; 3], + oprf_algorithm: bool, ) -> Result<(), Box> { let input = InputSource::from(&args.input); let query_type: QueryType; - match security_model { - IpaSecurityModel::SemiHonest => { + match (security_model, oprf_algorithm) { + (IpaSecurityModel::SemiHonest, false) => { query_type = QueryType::SemiHonestIpa(ipa_query_config.clone()); } - IpaSecurityModel::Malicious => { + (IpaSecurityModel::Malicious, false) => { query_type = QueryType::MaliciousIpa(ipa_query_config.clone()) } + (IpaSecurityModel::SemiHonest, true) => { + query_type = QueryType::OprfIpa(ipa_query_config.clone()); + } + (IpaSecurityModel::Malicious, true) => { + panic!("OPRF for malicious is not implemented as yet") + } + }; + + let order = if oprf_algorithm { + CappingOrder::CapMostRecentFirst + } else { + CappingOrder::CapOldestFirst }; let input_rows = input.iter::().collect::>(); @@ -247,7 +275,7 @@ async fn ipa( ipa_query_config.per_user_credit_cap, ipa_query_config.attribution_window_seconds, ipa_query_config.max_breakdown_key, - &CappingOrder::CapOldestFirst, + &order, ); // pad the output vector to the max breakdown key, to make sure it is aligned with the MPC results @@ -260,18 +288,19 @@ async fn ipa( }; let mut key_registries = KeyRegistries::default(); - let actual = playbook_ipa::( - &input_rows, - &helper_clients, - query_id, - ipa_query_config, - key_registries.init_from(network), - ) - .await; - - tracing::info!("{m:?}", m = ipa_query_config); - - validate(&expected, &actual.breakdowns); + let actual = if oprf_algorithm { + playbook_oprf_ipa::(input_rows, &helper_clients, query_id, ipa_query_config) + .await + } else { + playbook_ipa::( + &input_rows, + &helper_clients, + query_id, + ipa_query_config, + key_registries.init_from(network), + ) + .await + }; if let Some(ref path) = args.output_file { // it will be sad to lose the results if file already exists. @@ -308,6 +337,10 @@ async fn ipa( write!(file, "{}", serde_json::to_string_pretty(&actual)?)?; } + tracing::info!("{m:?}", m = ipa_query_config); + + validate(&expected, &actual.breakdowns); + Ok(()) } diff --git a/src/cli/playbook/ipa.rs b/src/cli/playbook/ipa.rs index 9047baeb3..c509cb400 100644 --- a/src/cli/playbook/ipa.rs +++ b/src/cli/playbook/ipa.rs @@ -22,9 +22,9 @@ use crate::{ hpke::PublicKeyRegistry, ipa_test_input, net::MpcHelperClient, - protocol::{ipa::IPAInputRow, BreakdownKey, MatchKey, QueryId}, + protocol::{ipa::IPAInputRow, BreakdownKey, MatchKey, QueryId, Timestamp, TriggerValue}, query::QueryStatus, - report::{KeyIdentifier, Report}, + report::{KeyIdentifier, OprfReport, Report}, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::{input::GenericReportTestInput, ipa::TestRawDataRecord, Reconstruct}, }; @@ -158,3 +158,94 @@ where breakdowns, } } + +pub async fn playbook_oprf_ipa( + mut records: Vec, + clients: &[MpcHelperClient; 3], + query_id: QueryId, + query_config: IpaQueryConfig, +) -> IpaQueryResult +where + F: PrimeField, + AdditiveShare: Serializable, +{ + let mut buffers: [_; 3] = std::array::from_fn(|_| Vec::new()); + let query_size = records.len(); + + let sz = as Serializable>::Size::USIZE; + for buffer in &mut buffers { + buffer.resize(query_size * sz, 0u8); + } + + //TODO(richaj) This manual sorting will be removed once we have the PRF sharding in place + records.sort_by(|a, b| b.user_id.cmp(&a.user_id)); + + let shares: [Vec>; 3] = + records.iter().cloned().share(); + zip(&mut buffers, shares).for_each(|(buf, shares)| { + for (share, chunk) in zip(shares, buf.chunks_mut(sz)) { + share.serialize(GenericArray::from_mut_slice(chunk)); + } + }); + + let inputs = buffers.map(BodyStream::from); + tracing::info!("Starting query for OPRF"); + let mpc_time = Instant::now(); + try_join_all( + inputs + .into_iter() + .zip(clients) + .map(|(input_stream, client)| { + client.query_input(QueryInput { + query_id, + input_stream, + }) + }), + ) + .await + .unwrap(); + + let mut delay = Duration::from_millis(125); + loop { + if try_join_all(clients.iter().map(|client| client.query_status(query_id))) + .await + .unwrap() + .into_iter() + .all(|status| status == QueryStatus::Completed) + { + break; + } + + sleep(delay).await; + delay = min(Duration::from_secs(5), delay * 2); + // TODO: Add a timeout of some sort. Possibly, add some sort of progress indicator to + // the status API so we can check whether the query is making progress. + } + + // wait until helpers have processed the query and get the results from them + let results: [_; 3] = try_join_all(clients.iter().map(|client| client.query_results(query_id))) + .await + .unwrap() + .try_into() + .unwrap(); + + let mut results: Vec = results + .map(|bytes| AdditiveShare::::from_byte_slice(&bytes).collect::>()) + .reconstruct(); + + let _ = results.split_off(query_config.max_breakdown_key.try_into().unwrap()); + + let lat = mpc_time.elapsed(); + tracing::info!("Running IPA for {query_size:?} records took {t:?}", t = lat); + let mut breakdowns = vec![0; usize::try_from(query_config.max_breakdown_key).unwrap()]; + for (breakdown_key, trigger_value) in results.into_iter().enumerate() { + breakdowns[breakdown_key] += u32::try_from(trigger_value.as_u128()).unwrap(); + } + + IpaQueryResult { + input_size: QuerySize::try_from(query_size).unwrap(), + config: query_config, + latency: lat, + breakdowns, + } +} diff --git a/src/cli/playbook/mod.rs b/src/cli/playbook/mod.rs index cdc47be00..842c0a1a0 100644 --- a/src/cli/playbook/mod.rs +++ b/src/cli/playbook/mod.rs @@ -11,7 +11,8 @@ pub use input::InputSource; pub use multiply::secure_mul; use tokio::time::sleep; -pub use self::ipa::playbook_ipa; +pub use self::ipa::{playbook_ipa, playbook_oprf_ipa}; + use crate::{ config::{ClientConfig, NetworkConfig, PeerConfig}, net::{ClientIdentity, MpcHelperClient}, diff --git a/src/helpers/transport/query.rs b/src/helpers/transport/query.rs index ab4e0761b..6bb7a9ed9 100644 --- a/src/helpers/transport/query.rs +++ b/src/helpers/transport/query.rs @@ -206,6 +206,7 @@ pub enum QueryType { MaliciousIpa(IpaQueryConfig), SemiHonestSparseAggregate(SparseAggregateQueryConfig), MaliciousSparseAggregate(SparseAggregateQueryConfig), + OprfIpa(IpaQueryConfig), } impl QueryType { @@ -214,6 +215,7 @@ impl QueryType { pub const MALICIOUS_IPA_STR: &'static str = "malicious-ipa"; pub const SEMIHONEST_AGGREGATE_STR: &'static str = "semihonest-sparse-aggregate"; pub const MALICIOUS_AGGREGATE_STR: &'static str = "malicious-sparse-aggregate"; + pub const OPRF_IPA_STR: &'static str = "oprf_ipa"; } /// TODO: should this `AsRef` impl (used for `Substep`) take into account config of IPA? @@ -226,6 +228,7 @@ impl AsRef for QueryType { QueryType::MaliciousIpa(_) => Self::MALICIOUS_IPA_STR, QueryType::SemiHonestSparseAggregate(_) => Self::SEMIHONEST_AGGREGATE_STR, QueryType::MaliciousSparseAggregate(_) => Self::MALICIOUS_AGGREGATE_STR, + QueryType::OprfIpa(_) => Self::OPRF_IPA_STR, } } } diff --git a/src/net/http_serde.rs b/src/net/http_serde.rs index faa2ec13d..406ec9a48 100644 --- a/src/net/http_serde.rs +++ b/src/net/http_serde.rs @@ -139,6 +139,10 @@ pub mod query { let Query(q) = req.extract().await?; Ok(QueryType::MaliciousSparseAggregate(q)) } + QueryType::OPRF_IPA_STR => { + let Query(q) = req.extract().await?; + Ok(QueryType::OprfIpa(q)) + } other => Err(Error::bad_query_value("query_type", other)), }?; Ok(QueryConfigQueryParams(QueryConfig { @@ -161,7 +165,9 @@ pub mod query { match self.query_type { #[cfg(any(test, feature = "test-fixture", feature = "cli"))] QueryType::TestMultiply => Ok(()), - QueryType::SemiHonestIpa(config) | QueryType::MaliciousIpa(config) => { + QueryType::SemiHonestIpa(config) + | QueryType::MaliciousIpa(config) + | QueryType::OprfIpa(config) => { write!( f, "&per_user_credit_cap={}&max_breakdown_key={}&num_multi_bits={}", diff --git a/src/query/executor.rs b/src/query/executor.rs index d99e6df4f..210bf2a61 100644 --- a/src/query/executor.rs +++ b/src/query/executor.rs @@ -5,6 +5,7 @@ use std::{ sync::Arc, }; +use super::runner::OprfIpaQuery; use ::tokio::sync::oneshot; use futures::FutureExt; use generic_array::GenericArray; @@ -202,6 +203,33 @@ pub fn execute( }, ) } + (QueryType::OprfIpa(ipa_config), FieldType::Fp32BitPrime) => do_query( + config, + gateway, + input, + move |prss, gateway, config, input| { + let ctx = SemiHonestContext::new(prss, gateway); + Box::pin( + OprfIpaQuery::<_, Fp32BitPrime>::new(ipa_config) + .execute(ctx, config.size, input) + .then(|res| ready(res.map(|out| Box::new(out) as Box))), + ) + }, + ), + #[cfg(any(test, feature = "weak-field"))] + (QueryType::OprfIpa(ipa_config), FieldType::Fp31) => do_query( + config, + gateway, + input, + move |prss, gateway, config, input| { + let ctx = SemiHonestContext::new(prss, gateway); + Box::pin( + OprfIpaQuery::<_, Fp32BitPrime>::new(ipa_config) + .execute(ctx, config.size, input) + .then(|res| ready(res.map(|out| Box::new(out) as Box))), + ) + }, + ), } } diff --git a/src/query/runner/mod.rs b/src/query/runner/mod.rs index bcba34275..d9eb28f8f 100644 --- a/src/query/runner/mod.rs +++ b/src/query/runner/mod.rs @@ -1,12 +1,14 @@ mod aggregate; mod ipa; +mod oprf_ipa; + #[cfg(any(test, feature = "cli", feature = "test-fixture"))] mod test_multiply; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] pub(super) use test_multiply::execute_test_multiply; -pub(super) use self::{aggregate::SparseAggregateQuery, ipa::IpaQuery}; +pub(super) use self::{aggregate::SparseAggregateQuery, ipa::IpaQuery, oprf_ipa::OprfIpaQuery}; use crate::{error::Error, query::ProtocolResult}; pub(super) type QueryResult = Result, Error>; diff --git a/src/query/runner/oprf_ipa.rs b/src/query/runner/oprf_ipa.rs new file mode 100644 index 000000000..94d7ff0ec --- /dev/null +++ b/src/query/runner/oprf_ipa.rs @@ -0,0 +1,127 @@ +use std::marker::PhantomData; + +use futures::{Stream, TryStreamExt}; + +use crate::{ + error::Error, + ff::{Field, Gf2, PrimeField, Serializable}, + helpers::{ + query::{IpaQueryConfig, QuerySize}, + BodyStream, RecordsStream, + }, + protocol::{ + basics::ShareKnownValue, + context::{UpgradableContext, UpgradedContext}, + prf_sharding::{attribution_and_capping_and_aggregation, PrfShardedIpaInputRow}, + BreakdownKey, Timestamp, TriggerValue, + }, + report::{EventType, OprfReport}, + secret_sharing::{ + replicated::{malicious::ExtendableField, semi_honest::AdditiveShare as Replicated}, + SharedValue, + }, +}; + +pub struct OprfIpaQuery { + config: IpaQueryConfig, + phantom_data: PhantomData<(C, F)>, +} + +impl OprfIpaQuery { + pub fn new(config: IpaQueryConfig) -> Self { + Self { + config, + phantom_data: PhantomData, + } + } +} + +impl OprfIpaQuery +where + C: UpgradableContext, + C::UpgradedContext: UpgradedContext>, + C::UpgradedContext: UpgradedContext>, + F: PrimeField + ExtendableField, + Replicated: Serializable + ShareKnownValue, + Replicated: Serializable + ShareKnownValue, +{ + #[tracing::instrument("oprf_ipa_query", skip_all, fields(sz=%query_size))] + pub async fn execute<'a>( + self, + ctx: C, + query_size: QuerySize, + input_stream: BodyStream, + ) -> Result>, Error> { + let Self { + config, + phantom_data: _, + } = self; + tracing::info!("New query: {config:?}"); + let sz = usize::from(query_size); + + let input = if config.plaintext_match_keys { + let mut v = assert_stream_send(RecordsStream::< + OprfReport, + _, + >::new(input_stream)) + .try_concat() + .await?; + v.truncate(sz); + v + } else { + panic!(); + }; + + // TODO: Compute OPRFs and shuffle and add dummies and stuff (Daniel's code will be called here) + let sharded_input = input + .into_iter() + .map(|single_row| { + let is_trigger_bit_share = if single_row.event_type == EventType::Trigger { + Replicated::share_known_value(&ctx, Gf2::ONE) + } else { + Replicated::share_known_value(&ctx, Gf2::ZERO) + }; + PrfShardedIpaInputRow { + prf_of_match_key: single_row.mk_oprf, + is_trigger_bit: is_trigger_bit_share, + breakdown_key: single_row.breakdown_key, + trigger_value: single_row.trigger_value, + timestamp: single_row.timestamp, + } + }) + .collect::>(); + // Until then, we convert the output to something next function is happy about. + + let user_cap: i32 = config.per_user_credit_cap.try_into().unwrap(); + assert!( + user_cap & (user_cap - 1) == 0, + "This code only works for a user cap which is a power of 2" + ); + + attribution_and_capping_and_aggregation::< + C, + BreakdownKey, + TriggerValue, + Timestamp, + F, + _, + Replicated, + >( + ctx, + sharded_input, + user_cap.ilog2().try_into().unwrap(), + config.attribution_window_seconds, + ) + .await + } +} + +/// Helps to convince the compiler that things are `Send`. Like `seq_join::assert_send`, but for +/// streams. +/// +/// +pub fn assert_stream_send<'a, T>( + st: impl Stream + Send + 'a, +) -> impl Stream + Send + 'a { + st +} diff --git a/src/report.rs b/src/report.rs index 2b9cf1039..fc1686c70 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,14 +1,15 @@ use std::{ fmt::{Display, Formatter}, marker::PhantomData, - ops::Deref, + mem::size_of, + ops::{Add, Deref}, }; use bytes::{BufMut, Bytes}; -use generic_array::GenericArray; +use generic_array::{ArrayLength, GenericArray}; use hpke::Serializable as _; use rand_core::{CryptoRng, RngCore}; -use typenum::Unsigned; +use typenum::{Unsigned, U8}; use crate::{ ff::{GaloisField, Gf40Bit, Gf8Bit, PrimeField, Serializable}, @@ -400,6 +401,93 @@ where pub trigger_value: Replicated, } +impl Serializable for u64 { + type Size = U8; + + fn serialize(&self, buf: &mut GenericArray) { + let raw = &self.to_le_bytes()[..buf.len()]; + buf.copy_from_slice(raw); + } + + fn deserialize(buf: &GenericArray) -> Self { + let mut buf_to = [0u8; 8]; + buf_to[..buf.len()].copy_from_slice(buf); + u64::from_le_bytes(buf_to) + } +} + +impl Serializable for OprfReport +where + Replicated: Serializable, + Replicated: Serializable, + Replicated: Serializable, + as Serializable>::Size: Add, + as Serializable>::Size: + Add<< as Serializable>::Size as Add>::Output>, + as Serializable>::Size: Add< + < as Serializable>::Size as Add< + < as Serializable>::Size as Add>::Output, + >>::Output, + >, + < as Serializable>::Size as Add< + < as Serializable>::Size as Add< + < as Serializable>::Size as Add>::Output, + >>::Output, + >>::Output: ArrayLength, +{ + type Size = < as Serializable>::Size as Add< + < as Serializable>::Size as Add< + < as Serializable>::Size as Add>::Output, + >>::Output, + >>::Output; + fn serialize(&self, buf: &mut GenericArray) { + let sizeof_u64 = size_of::(); + let ts_sz = as Serializable>::Size::USIZE; + let bk_sz = as Serializable>::Size::USIZE; + let tv_sz = as Serializable>::Size::USIZE; + + self.mk_oprf + .serialize(GenericArray::from_mut_slice(&mut buf[..sizeof_u64])); + + self.timestamp.serialize(GenericArray::from_mut_slice( + &mut buf[sizeof_u64..sizeof_u64 + ts_sz], + )); + + self.breakdown_key.serialize(GenericArray::from_mut_slice( + &mut buf[sizeof_u64 + ts_sz..sizeof_u64 + ts_sz + bk_sz], + )); + + self.trigger_value.serialize(GenericArray::from_mut_slice( + &mut buf[sizeof_u64 + ts_sz + bk_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz], + )); + } + + fn deserialize(buf: &GenericArray) -> Self { + let sizeof_u64 = size_of::(); + let ts_sz = as Serializable>::Size::USIZE; + let bk_sz = as Serializable>::Size::USIZE; + let tv_sz = as Serializable>::Size::USIZE; + + let mk_oprf = u64::deserialize(GenericArray::from_slice(&buf[..sizeof_u64])); + let timestamp = Replicated::::deserialize(GenericArray::from_slice( + &buf[sizeof_u64..sizeof_u64 + ts_sz], + )); + let breakdown_key = Replicated::::deserialize(GenericArray::from_slice( + &buf[sizeof_u64 + ts_sz..sizeof_u64 + ts_sz + bk_sz], + )); + let trigger_value = Replicated::::deserialize(GenericArray::from_slice( + &buf[sizeof_u64 + ts_sz + bk_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz], + )); + Self { + timestamp, + breakdown_key, + trigger_value, + event_type: EventType::Source, + mk_oprf, + } + } +} + #[cfg(all(test, unit_test))] mod test { use rand::{distributions::Alphanumeric, rngs::StdRng, Rng}; From 73468dc85448b589cd7c7477d3d51c4703ec9f08 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Fri, 27 Oct 2023 17:45:31 +0800 Subject: [PATCH 076/124] Adding EventType in serialization and deserialization --- src/protocol/prf_sharding/mod.rs | 2 ++ src/report.rs | 50 +++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 87832f797..7f8fa56bb 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -728,6 +728,7 @@ where let prime_field_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); let prime_field_m_ctx = prime_field_validator.context(); + println!("attribution_and_capping starting"); let user_level_attributions: Vec = attribution_and_capping( sh_ctx, input_rows, @@ -736,6 +737,7 @@ where ) .await?; + println!("do_aggregation starting"); do_aggregation::<_, BK, TV, F, S>(prime_field_m_ctx, user_level_attributions).await } diff --git a/src/report.rs b/src/report.rs index fc1686c70..8af5fb41a 100644 --- a/src/report.rs +++ b/src/report.rs @@ -9,7 +9,7 @@ use bytes::{BufMut, Bytes}; use generic_array::{ArrayLength, GenericArray}; use hpke::Serializable as _; use rand_core::{CryptoRng, RngCore}; -use typenum::{Unsigned, U8}; +use typenum::{Unsigned, U8, U9, U1}; use crate::{ ff::{GaloisField, Gf40Bit, Gf8Bit, PrimeField, Serializable}, @@ -48,6 +48,31 @@ pub enum EventType { Source, } +impl Serializable for EventType { + type Size = U1; + + fn serialize(&self, buf: &mut GenericArray) { + + let raw: &[u8] = match self { + EventType::Trigger => &[0], + EventType::Source => &[1], + }; + buf.copy_from_slice(raw); + } + + fn deserialize(buf: &GenericArray) -> Self { + + let mut buf_to = [0u8; 1]; + buf_to[..buf.len()].copy_from_slice(buf); + + match buf[0] { + 0 => EventType::Trigger, + 1 => EventType::Source, + 2_u8..=u8::MAX => panic!("Unreachable code"), + } + } +} + #[derive(Debug, PartialEq, Eq)] pub struct ParseEventTypeError(u8); @@ -421,27 +446,29 @@ where Replicated: Serializable, Replicated: Serializable, Replicated: Serializable, - as Serializable>::Size: Add, + as Serializable>::Size: Add, as Serializable>::Size: - Add<< as Serializable>::Size as Add>::Output>, + Add<< as Serializable>::Size as Add>::Output>, as Serializable>::Size: Add< < as Serializable>::Size as Add< - < as Serializable>::Size as Add>::Output, + < as Serializable>::Size as Add>::Output, >>::Output, >, < as Serializable>::Size as Add< < as Serializable>::Size as Add< - < as Serializable>::Size as Add>::Output, + < as Serializable>::Size as Add>::Output, >>::Output, >>::Output: ArrayLength, { type Size = < as Serializable>::Size as Add< < as Serializable>::Size as Add< - < as Serializable>::Size as Add>::Output, + < as Serializable>::Size as Add>::Output, >>::Output, >>::Output; + fn serialize(&self, buf: &mut GenericArray) { let sizeof_u64 = size_of::(); + let sizeof_eventtype = size_of::(); let ts_sz = as Serializable>::Size::USIZE; let bk_sz = as Serializable>::Size::USIZE; let tv_sz = as Serializable>::Size::USIZE; @@ -460,10 +487,16 @@ where self.trigger_value.serialize(GenericArray::from_mut_slice( &mut buf[sizeof_u64 + ts_sz + bk_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz], )); + + self.event_type.serialize(GenericArray::from_mut_slice( + &mut buf[sizeof_u64 + ts_sz + bk_sz + tv_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz + sizeof_eventtype], + )); } fn deserialize(buf: &GenericArray) -> Self { let sizeof_u64 = size_of::(); + let sizeof_eventtype = size_of::(); + let ts_sz = as Serializable>::Size::USIZE; let bk_sz = as Serializable>::Size::USIZE; let tv_sz = as Serializable>::Size::USIZE; @@ -478,11 +511,14 @@ where let trigger_value = Replicated::::deserialize(GenericArray::from_slice( &buf[sizeof_u64 + ts_sz + bk_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz], )); + let event_type = EventType::deserialize(GenericArray::from_slice( + &buf[sizeof_u64 + ts_sz + bk_sz + tv_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz + sizeof_eventtype], + )); Self { timestamp, breakdown_key, trigger_value, - event_type: EventType::Source, + event_type, mk_oprf, } } From f77a8b44c6facdc336d0ba0d8b0b55fc533092b0 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Mon, 30 Oct 2023 17:11:16 +0800 Subject: [PATCH 077/124] Incorporate feedback --- src/bin/report_collector.rs | 61 ++++++++++++++++++++----------------- src/report.rs | 12 ++++---- src/test_fixture/ipa.rs | 5 +++ 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/bin/report_collector.rs b/src/bin/report_collector.rs index 89b80bb0e..f8ba75d7e 100644 --- a/src/bin/report_collector.rs +++ b/src/bin/report_collector.rs @@ -26,7 +26,7 @@ use ipa::{ protocol::{BreakdownKey, MatchKey}, report::{KeyIdentifier, DEFAULT_KEY_ID}, test_fixture::{ - ipa::{ipa_in_the_clear, CappingOrder, IpaSecurityModel, TestRawDataRecord}, + ipa::{ipa_in_the_clear, CappingOrder, IpaQueryStyle, IpaSecurityModel, TestRawDataRecord}, EventGenerator, EventGeneratorConfig, }, }; @@ -136,7 +136,7 @@ async fn main() -> Result<(), Box> { IpaSecurityModel::SemiHonest, config, &clients, - false, + IpaQueryStyle::SortInMpc, ) .await? } @@ -147,7 +147,7 @@ async fn main() -> Result<(), Box> { IpaSecurityModel::Malicious, config, &clients, - false, + IpaQueryStyle::SortInMpc, ) .await? } @@ -164,7 +164,7 @@ async fn main() -> Result<(), Box> { IpaSecurityModel::SemiHonest, config, &clients, - true, + IpaQueryStyle::Oprf, ) .await? } @@ -236,31 +236,25 @@ async fn ipa( security_model: IpaSecurityModel, ipa_query_config: IpaQueryConfig, helper_clients: &[MpcHelperClient; 3], - oprf_algorithm: bool, + query_style: IpaQueryStyle, ) -> Result<(), Box> { let input = InputSource::from(&args.input); let query_type: QueryType; - match (security_model, oprf_algorithm) { - (IpaSecurityModel::SemiHonest, false) => { + match (security_model, &query_style) { + (IpaSecurityModel::SemiHonest, IpaQueryStyle::SortInMpc) => { query_type = QueryType::SemiHonestIpa(ipa_query_config.clone()); } - (IpaSecurityModel::Malicious, false) => { + (IpaSecurityModel::Malicious, IpaQueryStyle::SortInMpc) => { query_type = QueryType::MaliciousIpa(ipa_query_config.clone()) } - (IpaSecurityModel::SemiHonest, true) => { + (IpaSecurityModel::SemiHonest, IpaQueryStyle::Oprf) => { query_type = QueryType::OprfIpa(ipa_query_config.clone()); } - (IpaSecurityModel::Malicious, true) => { + (IpaSecurityModel::Malicious, IpaQueryStyle::Oprf) => { panic!("OPRF for malicious is not implemented as yet") } }; - let order = if oprf_algorithm { - CappingOrder::CapMostRecentFirst - } else { - CappingOrder::CapOldestFirst - }; - let input_rows = input.iter::().collect::>(); let query_config = QueryConfig { size: QuerySize::try_from(input_rows.len()).unwrap(), @@ -275,7 +269,10 @@ async fn ipa( ipa_query_config.per_user_credit_cap, ipa_query_config.attribution_window_seconds, ipa_query_config.max_breakdown_key, - &order, + &(match query_style { + IpaQueryStyle::Oprf => CappingOrder::CapMostRecentFirst, + IpaQueryStyle::SortInMpc => CappingOrder::CapOldestFirst, + }), ); // pad the output vector to the max breakdown key, to make sure it is aligned with the MPC results @@ -288,18 +285,26 @@ async fn ipa( }; let mut key_registries = KeyRegistries::default(); - let actual = if oprf_algorithm { - playbook_oprf_ipa::(input_rows, &helper_clients, query_id, ipa_query_config) + let actual = match query_style { + IpaQueryStyle::Oprf => { + playbook_oprf_ipa::( + input_rows, + &helper_clients, + query_id, + ipa_query_config, + ) .await - } else { - playbook_ipa::( - &input_rows, - &helper_clients, - query_id, - ipa_query_config, - key_registries.init_from(network), - ) - .await + } + IpaQueryStyle::SortInMpc => { + playbook_ipa::( + &input_rows, + &helper_clients, + query_id, + ipa_query_config, + key_registries.init_from(network), + ) + .await + } }; if let Some(ref path) = args.output_file { diff --git a/src/report.rs b/src/report.rs index 8af5fb41a..01d28deff 100644 --- a/src/report.rs +++ b/src/report.rs @@ -9,7 +9,7 @@ use bytes::{BufMut, Bytes}; use generic_array::{ArrayLength, GenericArray}; use hpke::Serializable as _; use rand_core::{CryptoRng, RngCore}; -use typenum::{Unsigned, U8, U9, U1}; +use typenum::{Unsigned, U1, U8, U9}; use crate::{ ff::{GaloisField, Gf40Bit, Gf8Bit, PrimeField, Serializable}, @@ -52,7 +52,6 @@ impl Serializable for EventType { type Size = U1; fn serialize(&self, buf: &mut GenericArray) { - let raw: &[u8] = match self { EventType::Trigger => &[0], EventType::Source => &[1], @@ -61,10 +60,9 @@ impl Serializable for EventType { } fn deserialize(buf: &GenericArray) -> Self { - let mut buf_to = [0u8; 1]; buf_to[..buf.len()].copy_from_slice(buf); - + match buf[0] { 0 => EventType::Trigger, 1 => EventType::Source, @@ -489,7 +487,8 @@ where )); self.event_type.serialize(GenericArray::from_mut_slice( - &mut buf[sizeof_u64 + ts_sz + bk_sz + tv_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz + sizeof_eventtype], + &mut buf[sizeof_u64 + ts_sz + bk_sz + tv_sz + ..sizeof_u64 + ts_sz + bk_sz + tv_sz + sizeof_eventtype], )); } @@ -512,7 +511,8 @@ where &buf[sizeof_u64 + ts_sz + bk_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz], )); let event_type = EventType::deserialize(GenericArray::from_slice( - &buf[sizeof_u64 + ts_sz + bk_sz + tv_sz..sizeof_u64 + ts_sz + bk_sz + tv_sz + sizeof_eventtype], + &buf[sizeof_u64 + ts_sz + bk_sz + tv_sz + ..sizeof_u64 + ts_sz + bk_sz + tv_sz + sizeof_eventtype], )); Self { timestamp, diff --git a/src/test_fixture/ipa.rs b/src/test_fixture/ipa.rs index a33b92e5b..e6f899b79 100644 --- a/src/test_fixture/ipa.rs +++ b/src/test_fixture/ipa.rs @@ -24,6 +24,11 @@ pub enum IpaSecurityModel { Malicious, } +pub enum IpaQueryStyle { + SortInMpc, + Oprf, +} + #[derive(Debug, Clone)] pub struct TestRawDataRecord { pub timestamp: u64, From 4d2469e78fa6d8596ab529daf3f93ac467e36244 Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Mon, 30 Oct 2023 17:14:49 +0800 Subject: [PATCH 078/124] Make clippy happy --- output.txt | 21 ++++++++ src/cli/playbook/ipa.rs | 92 +++++++++++--------------------- src/cli/playbook/mod.rs | 1 - src/protocol/prf_sharding/mod.rs | 2 - src/query/executor.rs | 2 +- src/query/runner/aggregate.rs | 8 ++- src/query/runner/ipa.rs | 29 +++------- src/query/runner/oprf_ipa.rs | 21 ++------ src/report.rs | 4 +- 9 files changed, 70 insertions(+), 110 deletions(-) create mode 100644 output.txt diff --git a/output.txt b/output.txt new file mode 100644 index 000000000..77fb36b55 --- /dev/null +++ b/output.txt @@ -0,0 +1,21 @@ +{ + "input_size": 3, + "config": { + "per_user_credit_cap": 16, + "max_breakdown_key": 8, + "attribution_window_seconds": null, + "num_multi_bits": 3, + "plaintext_match_keys": true + }, + "latency": 1.8909392980000002, + "breakdowns": [ + 48, + 67, + 54, + 64, + 54, + 78, + 74, + 61 + ] +} \ No newline at end of file diff --git a/src/cli/playbook/ipa.rs b/src/cli/playbook/ipa.rs index c509cb400..1f672d75c 100644 --- a/src/cli/playbook/ipa.rs +++ b/src/cli/playbook/ipa.rs @@ -99,64 +99,8 @@ where let inputs = buffers.map(BodyStream::from); tracing::info!("Starting query after finishing encryption"); - let mpc_time = Instant::now(); - try_join_all( - inputs - .into_iter() - .zip(clients) - .map(|(input_stream, client)| { - client.query_input(QueryInput { - query_id, - input_stream, - }) - }), - ) - .await - .unwrap(); - let mut delay = Duration::from_millis(125); - loop { - if try_join_all(clients.iter().map(|client| client.query_status(query_id))) - .await - .unwrap() - .into_iter() - .all(|status| status == QueryStatus::Completed) - { - break; - } - - sleep(delay).await; - delay = min(Duration::from_secs(5), delay * 2); - // TODO: Add a timeout of some sort. Possibly, add some sort of progress indicator to - // the status API so we can check whether the query is making progress. - } - - // wait until helpers have processed the query and get the results from them - let results: [_; 3] = try_join_all(clients.iter().map(|client| client.query_results(query_id))) - .await - .unwrap() - .try_into() - .unwrap(); - - let results: Vec = results - .map(|bytes| AdditiveShare::::from_byte_slice(&bytes).collect::>()) - .reconstruct(); - - let lat = mpc_time.elapsed(); - tracing::info!("Running IPA for {query_size:?} records took {t:?}", t = lat); - let mut breakdowns = vec![0; usize::try_from(query_config.max_breakdown_key).unwrap()]; - for (breakdown_key, trigger_value) in results.into_iter().enumerate() { - // TODO: make the data type used consistent with `ipa_in_the_clear` - // I think using u32 is wrong, we should move to u128 - breakdowns[breakdown_key] += u32::try_from(trigger_value.as_u128()).unwrap(); - } - - IpaQueryResult { - input_size: QuerySize::try_from(query_size).unwrap(), - config: query_config, - latency: lat, - breakdowns, - } + do_processing::(inputs, clients, query_id, query_config).await } pub async fn playbook_oprf_ipa( @@ -177,7 +121,8 @@ where buffer.resize(query_size * sz, 0u8); } - //TODO(richaj) This manual sorting will be removed once we have the PRF sharding in place + //TODO(richaj) This manual sorting will be removed once we have the PRF sharding in place. + //This does a stable sort. It also expects the inputs to be sorted by timestamp records.sort_by(|a, b| b.user_id.cmp(&a.user_id)); let shares: [Vec>; 3] = @@ -190,6 +135,21 @@ where let inputs = buffers.map(BodyStream::from); tracing::info!("Starting query for OPRF"); + + do_processing::(inputs, clients, query_id, query_config).await +} + +pub async fn do_processing( + inputs: [BodyStream; 3], + clients: &[MpcHelperClient; 3], + query_id: QueryId, + query_config: IpaQueryConfig, +) -> IpaQueryResult +where + F: PrimeField, + AdditiveShare: Serializable, +{ + let query_size = inputs.len(); let mpc_time = Instant::now(); try_join_all( inputs @@ -229,17 +189,25 @@ where .try_into() .unwrap(); - let mut results: Vec = results + let results: Vec = results .map(|bytes| AdditiveShare::::from_byte_slice(&bytes).collect::>()) .reconstruct(); - let _ = results.split_off(query_config.max_breakdown_key.try_into().unwrap()); - let lat = mpc_time.elapsed(); + tracing::info!("Running IPA for {query_size:?} records took {t:?}", t = lat); let mut breakdowns = vec![0; usize::try_from(query_config.max_breakdown_key).unwrap()]; for (breakdown_key, trigger_value) in results.into_iter().enumerate() { - breakdowns[breakdown_key] += u32::try_from(trigger_value.as_u128()).unwrap(); + // TODO: make the data type used consistent with `ipa_in_the_clear` + // I think using u32 is wrong, we should move to u128 + assert!( + breakdown_key < query_config.max_breakdown_key.try_into().unwrap() + || trigger_value == F::ZERO, + "trigger values were attributed to buckets more than max breakdown key" + ); + if breakdown_key < query_config.max_breakdown_key.try_into().unwrap() { + breakdowns[breakdown_key] += u32::try_from(trigger_value.as_u128()).unwrap(); + } } IpaQueryResult { diff --git a/src/cli/playbook/mod.rs b/src/cli/playbook/mod.rs index 842c0a1a0..4fe0ac6ca 100644 --- a/src/cli/playbook/mod.rs +++ b/src/cli/playbook/mod.rs @@ -12,7 +12,6 @@ pub use multiply::secure_mul; use tokio::time::sleep; pub use self::ipa::{playbook_ipa, playbook_oprf_ipa}; - use crate::{ config::{ClientConfig, NetworkConfig, PeerConfig}, net::{ClientIdentity, MpcHelperClient}, diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 7f8fa56bb..87832f797 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -728,7 +728,6 @@ where let prime_field_validator = sh_ctx.narrow(&Step::BinaryValidator).validator::(); let prime_field_m_ctx = prime_field_validator.context(); - println!("attribution_and_capping starting"); let user_level_attributions: Vec = attribution_and_capping( sh_ctx, input_rows, @@ -737,7 +736,6 @@ where ) .await?; - println!("do_aggregation starting"); do_aggregation::<_, BK, TV, F, S>(prime_field_m_ctx, user_level_attributions).await } diff --git a/src/query/executor.rs b/src/query/executor.rs index 210bf2a61..4da5f7a6c 100644 --- a/src/query/executor.rs +++ b/src/query/executor.rs @@ -5,7 +5,6 @@ use std::{ sync::Arc, }; -use super::runner::OprfIpaQuery; use ::tokio::sync::oneshot; use futures::FutureExt; use generic_array::GenericArray; @@ -15,6 +14,7 @@ use rand_core::SeedableRng; use shuttle::future as tokio; use typenum::Unsigned; +use super::runner::OprfIpaQuery; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] use crate::query::runner::execute_test_multiply; use crate::{ diff --git a/src/query/runner/aggregate.rs b/src/query/runner/aggregate.rs index e0dee3550..23cd6e7ba 100644 --- a/src/query/runner/aggregate.rs +++ b/src/query/runner/aggregate.rs @@ -2,7 +2,6 @@ use std::marker::PhantomData; use futures_util::TryStreamExt; -use super::ipa::assert_stream_send; use crate::{ error::Error, ff::{Gf2, Gf8Bit, PrimeField, Serializable}, @@ -83,10 +82,9 @@ where let input = { //TODO: Replace `Gf8Bit` with an appropriate type specified by the config `contribution_bits` - let mut v = assert_stream_send(RecordsStream::< - SparseAggregateInputRow, - _, - >::new(input_stream)) + let mut v = RecordsStream::, _>::new( + input_stream, + ) .try_concat() .await?; v.truncate(sz); diff --git a/src/query/runner/ipa.rs b/src/query/runner/ipa.rs index 2b19ead23..ed4a3da43 100644 --- a/src/query/runner/ipa.rs +++ b/src/query/runner/ipa.rs @@ -2,7 +2,7 @@ use std::marker::PhantomData; use futures::{ stream::{iter, repeat}, - Stream, StreamExt, TryStreamExt, + StreamExt, TryStreamExt, }; use crate::{ @@ -89,19 +89,16 @@ where let sz = usize::from(query_size); let input = if config.plaintext_match_keys { - let mut v = assert_stream_send(RecordsStream::< - IPAInputRow, - _, - >::new(input_stream)) - .try_concat() - .await?; + let mut v = + RecordsStream::, _>::new(input_stream) + .try_concat() + .await?; v.truncate(sz); v } else { - assert_stream_send(LengthDelimitedStream::< - EncryptedReport, - _, - >::new(input_stream)) + LengthDelimitedStream::, _>::new( + input_stream, + ) .map_err(Into::::into) .map_ok(|enc_reports| { iter(enc_reports.into_iter().map(|enc_report| { @@ -147,16 +144,6 @@ where } } -/// Helps to convince the compiler that things are `Send`. Like `seq_join::assert_send`, but for -/// streams. -/// -/// -pub fn assert_stream_send<'a, T>( - st: impl Stream + Send + 'a, -) -> impl Stream + Send + 'a { - st -} - /// no dependency on `weak-field` feature because it is enabled in tests by default #[cfg(all(test, unit_test))] mod tests { diff --git a/src/query/runner/oprf_ipa.rs b/src/query/runner/oprf_ipa.rs index 94d7ff0ec..e77a0300e 100644 --- a/src/query/runner/oprf_ipa.rs +++ b/src/query/runner/oprf_ipa.rs @@ -1,6 +1,6 @@ use std::marker::PhantomData; -use futures::{Stream, TryStreamExt}; +use futures::TryStreamExt; use crate::{ error::Error, @@ -60,16 +60,15 @@ where let sz = usize::from(query_size); let input = if config.plaintext_match_keys { - let mut v = assert_stream_send(RecordsStream::< - OprfReport, - _, - >::new(input_stream)) + let mut v = RecordsStream::, _>::new( + input_stream, + ) .try_concat() .await?; v.truncate(sz); v } else { - panic!(); + panic!("Encrypted match key handling is not handled for OPRF flow as yet"); }; // TODO: Compute OPRFs and shuffle and add dummies and stuff (Daniel's code will be called here) @@ -115,13 +114,3 @@ where .await } } - -/// Helps to convince the compiler that things are `Send`. Like `seq_join::assert_send`, but for -/// streams. -/// -/// -pub fn assert_stream_send<'a, T>( - st: impl Stream + Send + 'a, -) -> impl Stream + Send + 'a { - st -} diff --git a/src/report.rs b/src/report.rs index 01d28deff..5c8ada0ae 100644 --- a/src/report.rs +++ b/src/report.rs @@ -516,10 +516,10 @@ where )); Self { timestamp, + mk_oprf, + event_type, breakdown_key, trigger_value, - event_type, - mk_oprf, } } } From 0e9f41b43a6568dd6e6a72d1908bffc77895918f Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 31 Oct 2023 11:18:51 +0800 Subject: [PATCH 079/124] More fixes --- src/cli/playbook/ipa.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/playbook/ipa.rs b/src/cli/playbook/ipa.rs index 1f672d75c..3c758a6a9 100644 --- a/src/cli/playbook/ipa.rs +++ b/src/cli/playbook/ipa.rs @@ -100,7 +100,7 @@ where let inputs = buffers.map(BodyStream::from); tracing::info!("Starting query after finishing encryption"); - do_processing::(inputs, clients, query_id, query_config).await + do_processing::(inputs, query_size, clients, query_id, query_config).await } pub async fn playbook_oprf_ipa( @@ -136,11 +136,12 @@ where let inputs = buffers.map(BodyStream::from); tracing::info!("Starting query for OPRF"); - do_processing::(inputs, clients, query_id, query_config).await + do_processing::(inputs, query_size, clients, query_id, query_config).await } pub async fn do_processing( inputs: [BodyStream; 3], + query_size: usize, clients: &[MpcHelperClient; 3], query_id: QueryId, query_config: IpaQueryConfig, @@ -149,7 +150,6 @@ where F: PrimeField, AdditiveShare: Serializable, { - let query_size = inputs.len(); let mpc_time = Instant::now(); try_join_all( inputs From 91905cddc5269661247a61b2e8a0cf8d315aa13b Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 31 Oct 2023 11:30:39 +0800 Subject: [PATCH 080/124] Remove output.txt --- output.txt | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 output.txt diff --git a/output.txt b/output.txt deleted file mode 100644 index 77fb36b55..000000000 --- a/output.txt +++ /dev/null @@ -1,21 +0,0 @@ -{ - "input_size": 3, - "config": { - "per_user_credit_cap": 16, - "max_breakdown_key": 8, - "attribution_window_seconds": null, - "num_multi_bits": 3, - "plaintext_match_keys": true - }, - "latency": 1.8909392980000002, - "breakdowns": [ - 48, - 67, - 54, - 64, - 54, - 78, - 74, - 61 - ] -} \ No newline at end of file From 0747a6f773b392309fcc8cccb84e2c46fecf07ba Mon Sep 17 00:00:00 2001 From: Richa Jain Date: Tue, 31 Oct 2023 11:33:51 +0800 Subject: [PATCH 081/124] Rename function --- src/cli/playbook/ipa.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/playbook/ipa.rs b/src/cli/playbook/ipa.rs index 3c758a6a9..6c258f858 100644 --- a/src/cli/playbook/ipa.rs +++ b/src/cli/playbook/ipa.rs @@ -100,7 +100,7 @@ where let inputs = buffers.map(BodyStream::from); tracing::info!("Starting query after finishing encryption"); - do_processing::(inputs, query_size, clients, query_id, query_config).await + run_query_and_validate::(inputs, query_size, clients, query_id, query_config).await } pub async fn playbook_oprf_ipa( @@ -136,10 +136,10 @@ where let inputs = buffers.map(BodyStream::from); tracing::info!("Starting query for OPRF"); - do_processing::(inputs, query_size, clients, query_id, query_config).await + run_query_and_validate::(inputs, query_size, clients, query_id, query_config).await } -pub async fn do_processing( +pub async fn run_query_and_validate( inputs: [BodyStream; 3], query_size: usize, clients: &[MpcHelperClient; 3], From 2b50c45f5775931baec20c3fb440fd1376167c5a Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Tue, 31 Oct 2023 15:16:23 -0700 Subject: [PATCH 082/124] Fix OPRF attribution and capping circuit It was using sequential join that limits number of futures driven concurrently to completion. This circuit requires all user-level circuits to be polled, so `parallel_join` is more appropriate There may be other places where parallel join must be used, I did not check those, this got me to 10k rows. --- src/protocol/prf_sharding/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/prf_sharding/mod.rs index 87832f797..fcca9ab97 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/prf_sharding/mod.rs @@ -29,7 +29,7 @@ use crate::{ }, BitDecomposed, Linear as LinearSecretSharing, SharedValue, }, - seq_join::{seq_join, seq_try_join_all}, + seq_join::seq_join, }; pub mod bucket; @@ -381,7 +381,7 @@ where attribution_window_seconds, )); } - let outputs_chunked_by_user = seq_try_join_all(sh_ctx.active_work(), futures).await?; + let outputs_chunked_by_user = sh_ctx.parallel_join(futures).await?; Ok(outputs_chunked_by_user .into_iter() .flatten() From afeb72c05fce1279dc26e6b18c7fd7e03a70e4e9 Mon Sep 17 00:00:00 2001 From: Martin Thomson Date: Wed, 1 Nov 2023 16:25:13 +1100 Subject: [PATCH 083/124] modulus conversion with retention If you only want to convert some fields of a struct, this would allow you to do that. This is incomplete in the sense that it produces tuples. You might want to have a type that can be reconstructed from the converted and retained fields. Maybe that could be part of this functionality also, but I'm leaving that for a future refactoring. --- src/ff/galois_field.rs | 29 +- src/protocol/boolean/generate_random_bits.rs | 25 +- .../modulus_conversion/convert_shares.rs | 256 ++++++++++++++++-- src/protocol/modulus_conversion/mod.rs | 3 +- src/protocol/sort/generate_permutation.rs | 2 +- src/protocol/sort/generate_permutation_opt.rs | 2 +- 6 files changed, 270 insertions(+), 47 deletions(-) diff --git a/src/ff/galois_field.rs b/src/ff/galois_field.rs index 0c3814d27..633e1fd72 100644 --- a/src/ff/galois_field.rs +++ b/src/ff/galois_field.rs @@ -131,7 +131,7 @@ fn clmul(a: GF, b: GF) -> u128 { } macro_rules! bit_array_impl { - ( $modname:ident, $name:ident, $store:ty, $bits:expr, $one:expr, $polynomial:expr ) => { + ( $modname:ident, $name:ident, $store:ty, $bits:expr, $one:expr, $polynomial:expr, $({$($extra:item)*})? ) => { #[allow(clippy::suspicious_arithmetic_impl)] #[allow(clippy::suspicious_op_assign_impl)] mod $modname { @@ -534,6 +534,8 @@ macro_rules! bit_array_impl { assert_eq!(a, $name::deserialize(&buf)); } } + + $( $( $extra )* )? } pub use $modname::$name; @@ -547,7 +549,7 @@ bit_array_impl!( 40, bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), // x^40 + x^5 + x^3 + x^2 + 1 - 0b1_0000_0000_0000_0000_0000_0000_0000_0000_0010_1101_u128 + 0b1_0000_0000_0000_0000_0000_0000_0000_0000_0010_1101_u128, ); bit_array_impl!( @@ -557,7 +559,7 @@ bit_array_impl!( 32, bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), // x^32 + x^7 + x^3 + x^2 + 1 - 0b1_0000_0000_0000_0000_0000_0000_1000_1101_u128 + 0b1_0000_0000_0000_0000_0000_0000_1000_1101_u128, ); bit_array_impl!( @@ -567,7 +569,7 @@ bit_array_impl!( 20, bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), // x^20 + x^7 + x^3 + x^2 + 1 - 0b1000_0000_0000_1000_1101_u128 + 0b1000_0000_0000_1000_1101_u128, ); bit_array_impl!( @@ -577,7 +579,7 @@ bit_array_impl!( 8, bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0), // x^8 + x^4 + x^3 + x + 1 - 0b1_0001_1011_u128 + 0b1_0001_1011_u128, ); bit_array_impl!( @@ -587,7 +589,7 @@ bit_array_impl!( 9, bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0), // x^9 + x^4 + x^3 + x + 1 - 0b10_0001_1011_u128 + 0b10_0001_1011_u128, ); bit_array_impl!( @@ -597,7 +599,7 @@ bit_array_impl!( 5, bitarr!(const u8, Lsb0; 1, 0, 0, 0, 0), // x^5 + x^4 + x^3 + x^2 + x + 1 - 0b111_111_u128 + 0b111_111_u128, ); bit_array_impl!( @@ -607,7 +609,7 @@ bit_array_impl!( 3, bitarr!(const u8, Lsb0; 1, 0, 0), // x^3 + x + 1 - 0b1_011_u128 + 0b1_011_u128, ); bit_array_impl!( @@ -617,5 +619,14 @@ bit_array_impl!( 1, bitarr!(const u8, Lsb0; 1), // x - 0b10_u128 + 0b10_u128, + { + impl From for Gf2 { + fn from(value: bool) -> Self { + let mut v = Gf2::ZERO; + v.0.set(0, value); + v + } + } + } ); diff --git a/src/protocol/boolean/generate_random_bits.rs b/src/protocol/boolean/generate_random_bits.rs index f8e2d2e87..184bf7d80 100644 --- a/src/protocol/boolean/generate_random_bits.rs +++ b/src/protocol/boolean/generate_random_bits.rs @@ -5,6 +5,7 @@ use futures::stream::{iter as stream_iter, Stream, StreamExt}; use crate::{ error::Error, ff::PrimeField, + helpers::Role, protocol::{ basics::SecureMul, context::{prss::InstrumentedIndexedSharedRandomness, Context, UpgradedContext}, @@ -47,16 +48,14 @@ impl RawRandomBits { } impl ToBitConversionTriples for RawRandomBits { + type Residual = (); + // TODO const for this in place of the function fn bits(&self) -> u32 { self.count } - fn triple( - &self, - role: crate::helpers::Role, - i: u32, - ) -> BitConversionTriple> { + fn triple(&self, role: Role, i: u32) -> BitConversionTriple> { debug_assert!(u128::BITS - F::PRIME.into().leading_zeros() >= self.count); assert!(i < self.count); BitConversionTriple::new( @@ -65,6 +64,21 @@ impl ToBitConversionTriples for RawRandomBits { ((self.right >> i) & 1) == 1, ) } + + fn into_triples( + self, + role: Role, + indices: I, + ) -> ( + BitDecomposed>>, + Self::Residual, + ) + where + F: PrimeField, + I: IntoIterator, + { + (self.triple_range(role, indices), ()) + } } struct RawRandomBitIter { @@ -135,4 +149,5 @@ where .next() .await .unwrap() + .map(|(v, ())| v) } diff --git a/src/protocol/modulus_conversion/convert_shares.rs b/src/protocol/modulus_conversion/convert_shares.rs index 406c05fdc..87df09abf 100644 --- a/src/protocol/modulus_conversion/convert_shares.rs +++ b/src/protocol/modulus_conversion/convert_shares.rs @@ -96,11 +96,13 @@ impl BitConversionTriple> { } pub trait ToBitConversionTriples { + /// The type of a collection of fields that need to be carried in the stream without conversion. + type Residual: Send; // TODO: associated type defaults would be nice here. + /// Get the maximum number of bits that can be produced for this type. /// /// Note that this should be an associated constant, but one of the implementations would then need /// const generics to be more fully available in the language, so this is a method instead. For now. - fn bits(&self) -> u32; /// Produce a `BitConversionTriple` for the given role and bit index. fn triple(&self, role: Role, i: u32) -> BitConversionTriple>; @@ -116,9 +118,23 @@ pub trait ToBitConversionTriples { { BitDecomposed::new(indices.into_iter().map(|i| self.triple(role, i))) } + + fn into_triples( + self, + role: Role, + indices: I, + ) -> ( + BitDecomposed>>, + Self::Residual, + ) + where + F: PrimeField, + I: IntoIterator; } impl ToBitConversionTriples for Replicated { + type Residual = (); + fn bits(&self) -> u32 { B::BITS } @@ -126,9 +142,26 @@ impl ToBitConversionTriples for Replicated { fn triple(&self, role: Role, i: u32) -> BitConversionTriple> { BitConversionTriple::new(role, self.left()[i], self.right()[i]) } + + fn into_triples( + self, + role: Role, + indices: I, + ) -> ( + BitDecomposed>>, + Self::Residual, + ) + where + F: PrimeField, + I: IntoIterator, + { + (self.triple_range(role, indices), ()) + } } impl ToBitConversionTriples for BitDecomposed> { + type Residual = (); + fn bits(&self) -> u32 { u32::try_from(self.len()).unwrap() } @@ -138,13 +171,28 @@ impl ToBitConversionTriples for BitDecomposed> { let i = usize::try_from(i).unwrap(); BitConversionTriple::new(role, self[i].left()[BIT0], self[i].right()[BIT0]) } + + fn into_triples( + self, + role: Role, + indices: I, + ) -> ( + BitDecomposed>>, + Self::Residual, + ) + where + F: PrimeField, + I: IntoIterator, + { + (self.triple_range(role, indices), ()) + } } #[pin_project] -pub struct LocalBitConverter +pub struct LocalBitConverter where F: PrimeField, - V: ToBitConversionTriples, + V: ToBitConversionTriples, S: Stream + Send, { role: Role, @@ -154,10 +202,10 @@ where _f: PhantomData, } -impl LocalBitConverter +impl LocalBitConverter where F: PrimeField, - V: ToBitConversionTriples, + V: ToBitConversionTriples, S: Stream + Send, { pub fn new(role: Role, input: S, bits: Range) -> Self { @@ -170,19 +218,19 @@ where } } -impl Stream for LocalBitConverter +impl Stream for LocalBitConverter where F: PrimeField, - V: ToBitConversionTriples, + V: ToBitConversionTriples, S: Stream + Send, { - type Item = BitDecomposed>>; + type Item = (BitDecomposed>>, R); fn poll_next(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { let mut this = self.project(); match this.input.as_mut().poll_next(cx) { Poll::Ready(Some(input)) => { - Poll::Ready(Some(input.triple_range(*this.role, this.bits.clone()))) + Poll::Ready(Some(input.into_triples(*this.role, this.bits.clone()))) } Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, @@ -194,10 +242,10 @@ where } } -impl ExactSizeStream for LocalBitConverter +impl ExactSizeStream for LocalBitConverter where F: PrimeField, - V: ToBitConversionTriples, + V: ToBitConversionTriples, S: Stream + Send, { } @@ -251,7 +299,28 @@ pub fn convert_bits( ) -> impl Stream, Error>> where F: PrimeField, - V: ToBitConversionTriples, + V: ToBitConversionTriples, + C: UpgradedContext, + S: LinearSecretSharing + SecureMul, + VS: Stream + Unpin + Send, + for<'u> UpgradeContext<'u, C, F, RecordId>: + UpgradeToMalicious<'u, BitConversionTriple>, BitConversionTriple>, +{ + convert_some_bits(ctx, binary_shares, RecordId::FIRST, bit_range).map(|v| v.map(|(v, ())| v)) +} + +/// A version of `convert_bits` that allows for the retention of unconverted fields in the input. +/// Note that unconverted fields are not upgraded, so they might need to be upgraded either before or +/// after invoking this function. +#[tracing::instrument(name = "modulus_conversion", skip_all, fields(bits = ?bit_range, gate = %ctx.gate().as_ref()))] +pub fn convert_selected_bits( + ctx: C, + binary_shares: VS, + bit_range: Range, +) -> impl Stream, R), Error>> +where + F: PrimeField, + V: ToBitConversionTriples, C: UpgradedContext, S: LinearSecretSharing + SecureMul, VS: Stream + Unpin + Send, @@ -261,15 +330,15 @@ where convert_some_bits(ctx, binary_shares, RecordId::FIRST, bit_range) } -pub(crate) fn convert_some_bits( +pub(crate) fn convert_some_bits( ctx: C, binary_shares: VS, first_record: RecordId, bit_range: Range, -) -> impl Stream, Error>> +) -> impl Stream, R), Error>> where F: PrimeField, - V: ToBitConversionTriples, + V: ToBitConversionTriples, C: UpgradedContext, S: LinearSecretSharing + SecureMul, VS: Stream + Unpin + Send, @@ -288,7 +357,7 @@ where let stream = unfold( (ctx, locally_converted, first_record), |(ctx, mut locally_converted, record_id)| async move { - let Some(triple) = locally_converted.next().await else { + let Some((triple, residual)) = locally_converted.next().await else { return None; }; let bit_contexts = (0..).map(|i| ctx.narrow(&ConvertSharesStep::ConvertBit(i))); @@ -300,10 +369,15 @@ where .await?; convert_bit(ctx, record_id, &upgraded).await })); - Some((converted, (ctx, locally_converted, record_id + 1))) + Some(( + (converted, residual), + (ctx, locally_converted, record_id + 1), + )) }, ) - .map(|res| async move { res.await.map(|bits| BitDecomposed::new(bits)) }); + .map(|(row, residual)| async move { + row.await.map(|bits| (BitDecomposed::new(bits), residual)) + }); seq_join(active, stream) } @@ -315,16 +389,20 @@ mod tests { use crate::{ error::Error, - ff::{Field, Fp31, Fp32BitPrime}, + ff::{Field, Fp31, Fp32BitPrime, Gf2, PrimeField}, helpers::{Direction, Role}, protocol::{ context::{Context, UpgradableContext, UpgradedContext, Validator}, - modulus_conversion::{convert_bits, BitConversionTriple, LocalBitConverter}, + modulus_conversion::{ + convert_bits, convert_selected_bits, BitConversionTriple, LocalBitConverter, + ToBitConversionTriples, + }, MatchKey, RecordId, }, rand::{thread_rng, Rng}, - secret_sharing::replicated::{ - semi_honest::AdditiveShare as Replicated, ReplicatedSecretSharing, + secret_sharing::{ + replicated::{semi_honest::AdditiveShare as Replicated, ReplicatedSecretSharing}, + IntoShares, }, test_fixture::{Reconstruct, Runner, TestWorld}, }; @@ -356,6 +434,123 @@ mod tests { assert_eq!(Fp31::truncate_from(match_key[BITNUM]), result.reconstruct()); } + struct TwoBits { + convert: Replicated, + keep: Replicated, + } + + impl ToBitConversionTriples for TwoBits { + type Residual = Replicated; + + fn bits(&self) -> u32 { + 1 + } + + fn triple(&self, role: Role, i: u32) -> BitConversionTriple> { + assert_eq!(i, 0, "there is only one convertible bit in TwoBits"); + BitConversionTriple::new( + role, + self.convert.left() == Gf2::ONE, + self.convert.right() == Gf2::ONE, + ) + } + + fn into_triples( + self, + role: Role, + indices: I, + ) -> ( + crate::secret_sharing::BitDecomposed>>, + Self::Residual, + ) + where + F: PrimeField, + I: IntoIterator, + { + (self.triple_range(role, indices), self.keep) + } + } + + #[derive(Clone, Copy)] + struct TwoBitsRaw { + convert: bool, + keep: bool, + } + + impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> TwoBitsRaw { + let r = rng.next_u32(); + TwoBitsRaw { + convert: r & 1 == 1, + keep: r & 2 == 2, + } + } + } + + impl IntoShares for TwoBitsRaw { + fn share_with(self, rng: &mut R) -> [TwoBits; 3] { + let r = rng.next_u32(); + let mut offset = 0; + let mut next_bit = || { + let b = (r >> offset) & 1 == 1; + offset += 1; + Gf2::from(b) + }; + + let c0 = next_bit(); + let c1 = next_bit(); + let k0 = next_bit(); + let k1 = next_bit(); + let c2 = c0 + c1 + Gf2::from(self.convert); + let k2 = k0 + k1 + Gf2::from(self.keep); + [ + TwoBits { + convert: Replicated::new(c0, c1), + keep: Replicated::new(k0, k1), + }, + TwoBits { + convert: Replicated::new(c1, c2), + keep: Replicated::new(k1, k2), + }, + TwoBits { + convert: Replicated::new(c2, c0), + keep: Replicated::new(k2, k0), + }, + ] + } + } + + #[tokio::test] + pub async fn retain() { + let mut rng = thread_rng(); + let world = TestWorld::default(); + let two_bits = rng.gen::(); + let result: [(Replicated, Replicated); 3] = world + .semi_honest(two_bits, |ctx, bits_share| async move { + let v = ctx.validator(); + let result = convert_selected_bits( + v.context().set_total_records(1), + once(ready(bits_share)), + 0..1, + ) + .try_collect::>() + .await + .unwrap(); + assert_eq!(result.len(), 1); + let (converted, kept) = result.into_iter().next().unwrap(); + assert_eq!(converted.len(), 1); + (converted.into_iter().next().unwrap(), kept) + }) + .await; + assert_eq!( + ( + Fp31::truncate_from(two_bits.convert), + Gf2::from(two_bits.keep) + ), + result.reconstruct() + ); + } + #[tokio::test] pub async fn one_bit_malicious() { const BITNUM: u32 = 4; @@ -426,14 +621,15 @@ mod tests { let match_key = rng.gen::(); world .malicious(match_key, |ctx, mk_share| async move { - let triples = LocalBitConverter::, _>::new( - ctx.role(), - once(ready(mk_share)), - 0..1, - ) - .collect::>() - .await; - let tweaked = tweak.flip_bit(ctx.role(), triples[0][0].clone()); + let triples = + LocalBitConverter::, _, ()>::new( + ctx.role(), + once(ready(mk_share)), + 0..1, + ) + .collect::>() + .await; + let tweaked = tweak.flip_bit(ctx.role(), triples[0].0[0].clone()); let v = ctx.validator(); let m_triples = v.context().upgrade([tweaked]).await.unwrap(); diff --git a/src/protocol/modulus_conversion/mod.rs b/src/protocol/modulus_conversion/mod.rs index 9e4d53716..67a5e8ef9 100644 --- a/src/protocol/modulus_conversion/mod.rs +++ b/src/protocol/modulus_conversion/mod.rs @@ -3,5 +3,6 @@ pub mod convert_shares; // TODO: wean usage off convert_some_bits. pub(crate) use convert_shares::convert_some_bits; pub use convert_shares::{ - convert_bits, BitConversionTriple, LocalBitConverter, ToBitConversionTriples, + convert_bits, convert_selected_bits, BitConversionTriple, LocalBitConverter, + ToBitConversionTriples, }; diff --git a/src/protocol/sort/generate_permutation.rs b/src/protocol/sort/generate_permutation.rs index 11b02ea3b..9a7cb969d 100644 --- a/src/protocol/sort/generate_permutation.rs +++ b/src/protocol/sort/generate_permutation.rs @@ -116,7 +116,7 @@ where S: LinearSecretSharing + BasicProtocols, F> + 'static, ShuffledPermutationWrapper>: DowngradeMalicious>, I: Stream, - I::Item: ToBitConversionTriples + Clone + Send + Sync, + I::Item: ToBitConversionTriples + Clone + Send + Sync, for<'u> UpgradeContext<'u, C::UpgradedContext, F, RecordId>: UpgradeToMalicious<'u, BitConversionTriple>, BitConversionTriple>, { diff --git a/src/protocol/sort/generate_permutation_opt.rs b/src/protocol/sort/generate_permutation_opt.rs index 6be3ce437..3d1db3dff 100644 --- a/src/protocol/sort/generate_permutation_opt.rs +++ b/src/protocol/sort/generate_permutation_opt.rs @@ -96,7 +96,7 @@ where C::UpgradedContext: UpgradedContext, S: LinearSecretSharing + BasicProtocols, F> + 'static, I: Stream, - I::Item: ToBitConversionTriples + Clone + Send + Sync, + I::Item: ToBitConversionTriples + Clone + Send + Sync, ShuffledPermutationWrapper>: DowngradeMalicious>, for<'u> UpgradeContext<'u, C::UpgradedContext, F, RecordId>: UpgradeToMalicious<'u, BitConversionTriple>, BitConversionTriple>, From da2ee44e3397ed6258f2ce1f15bcef8bef9f56bd Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Tue, 31 Oct 2023 13:17:12 -0700 Subject: [PATCH 084/124] Configure logging from env as well Currently, it is not possible to have module-level granularity configuration for logging. Setting up `-vvv` makes IPA to log all trace events, including those coming from third-party. This change enables this functionality by optionally reading logging configuration from `RUST_LOG` --- src/cli/verbosity.rs | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/cli/verbosity.rs b/src/cli/verbosity.rs index 74346f558..53a2bee39 100644 --- a/src/cli/verbosity.rs +++ b/src/cli/verbosity.rs @@ -4,7 +4,7 @@ use clap::Parser; use metrics_tracing_context::MetricsLayer; use tracing::{info, metadata::LevelFilter, Level}; use tracing_subscriber::{ - fmt, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, + fmt, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, }; use crate::{ @@ -31,14 +31,14 @@ pub struct LoggingHandle { impl Verbosity { #[must_use] pub fn setup_logging(&self) -> LoggingHandle { - let filter_layer = self.level_filter(); + let filter_layer = self.log_filter(); let fmt_layer = fmt::layer() .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) .with_ansi(std::io::stderr().is_terminal()) .with_writer(stderr); tracing_subscriber::registry() - .with(self.level_filter()) + .with(self.log_filter()) .with(fmt_layer) .with(MetricsLayer::new()) .init(); @@ -53,15 +53,20 @@ impl Verbosity { handle } - fn level_filter(&self) -> LevelFilter { - if self.quiet { - LevelFilter::OFF - } else { - LevelFilter::from_level(match self.verbose { - 0 => Level::INFO, - 1 => Level::DEBUG, - _ => Level::TRACE, - }) - } + fn log_filter(&self) -> EnvFilter { + EnvFilter::builder() + .with_default_directive( + if self.quiet { + LevelFilter::OFF + } else { + LevelFilter::from_level(match self.verbose { + 0 => Level::INFO, + 1 => Level::DEBUG, + _ => Level::TRACE, + }) + } + .into(), + ) + .from_env_lossy() } } From b4d10d665c0c58b81d340ea659a769ecc27cf62a Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Tue, 31 Oct 2023 13:44:23 -0700 Subject: [PATCH 085/124] Add send and receive trace spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` 2023-10-31T20:35:01.600364Z TRACE oprf_ipa_query{sz=10000}:send{i=914 total=1658 to=H2 gate="protocol/binary_validator/row1/attributed_breakdown_key/bit7"}: ipa::helpers::gateway::send: new 2023-10-31T20:35:01.600382Z TRACE oprf_ipa_query{sz=10000}:send{i=914 total=1658 to=H2 gate="protocol/binary_validator/row1/attributed_breakdown_key/bit7"}: ipa::helpers::gateway::send: close time.busy=2.42µs time.idle=15.7µs ``` it is a bit verbose, but logging enter/exit allows to see if send was completed (because it may be blocked if it is out of bounds) and the timing. --- src/helpers/gateway/receive.rs | 1 + src/helpers/gateway/send.rs | 1 + src/helpers/mod.rs | 12 +++++++++++- src/protocol/mod.rs | 6 ++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/helpers/gateway/receive.rs b/src/helpers/gateway/receive.rs index 282ff68e2..b039c8653 100644 --- a/src/helpers/gateway/receive.rs +++ b/src/helpers/gateway/receive.rs @@ -43,6 +43,7 @@ impl ReceivingEnd { /// ## Panics /// This will panic if message size does not fit into 8 bytes and it somehow got serialized /// and sent to this helper. + #[tracing::instrument(level = "trace", "receive", skip_all, fields(i = %record_id, from = ?self.channel_id.role, gate = ?self.channel_id.gate.as_ref()))] pub async fn receive(&self, record_id: RecordId) -> Result { self.unordered_rx .recv::(record_id) diff --git a/src/helpers/gateway/send.rs b/src/helpers/gateway/send.rs index 4eb876af0..0d32de973 100644 --- a/src/helpers/gateway/send.rs +++ b/src/helpers/gateway/send.rs @@ -98,6 +98,7 @@ impl SendingEnd { /// call. /// /// [`set_total_records`]: crate::protocol::context::Context::set_total_records + #[tracing::instrument(level = "trace", "send", skip_all, fields(i = %record_id, total = %self.inner.total_records, to = ?self.channel_id.role, gate = ?self.channel_id.gate.as_ref()))] pub async fn send(&self, record_id: RecordId, msg: M) -> Result<(), Error> { let r = self.inner.send(record_id, msg).await; metrics::increment_counter!(RECORDS_SENT, diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 373070736..359bf305f 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -1,5 +1,5 @@ use std::{ - fmt::{Debug, Formatter}, + fmt::{Debug, Display, Formatter}, num::NonZeroUsize, }; @@ -441,6 +441,16 @@ pub enum TotalRecords { Indeterminate, } +impl Display for TotalRecords { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + TotalRecords::Unspecified => write!(f, "unspecified"), + TotalRecords::Specified(v) => write!(f, "{v}"), + TotalRecords::Indeterminate => write!(f, "∞"), + } + } +} + impl TotalRecords { #[must_use] pub fn is_specified(&self) -> bool { diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index cc0ae5ab8..8c5e72c7d 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -86,6 +86,12 @@ impl TryFrom<&str> for QueryId { #[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct RecordId(u32); +impl Display for RecordId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + impl From for RecordId { fn from(v: u32) -> Self { RecordId(v) From 9b7649bffb434033ac9dac43fa6c911f532068ee Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 09:44:30 -0700 Subject: [PATCH 086/124] added 25519 prime field --- Cargo.toml | 1 + src/ff/ec_prime_field.rs | 194 +++++++++++++++++++++++++++++++++++++++ src/ff/mod.rs | 1 + 3 files changed, 196 insertions(+) create mode 100644 src/ff/ec_prime_field.rs diff --git a/Cargo.toml b/Cargo.toml index 1767e8e24..c3161a216 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ clap = { version = "4.3.2", optional = true, features = ["derive"] } comfy-table = { version = "7.0", optional = true } config = "0.13.2" criterion = { version = "0.5.1", optional = true, default-features = false, features = ["async_tokio", "plotters", "html_reports"] } +curve25519-dalek = "4.1.1" dashmap = "5.4" dhat = "0.3.2" embed-doc-image = "0.1.4" diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs new file mode 100644 index 000000000..c87dbfa94 --- /dev/null +++ b/src/ff/ec_prime_field.rs @@ -0,0 +1,194 @@ +use generic_array::GenericArray; +use curve25519_dalek::scalar::Scalar; +use rand_core::RngCore; +use typenum::U32; + +use crate::{ + ff::Serializable, + secret_sharing::{Block, SharedValue}, +}; + +impl Block for Scalar { + type Size = U32; +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct Fp25519(::Storage); + +//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? + + +impl SharedValue for Fp25519 { + type Storage = Scalar; + const BITS: u32 = 256; + const ZERO: Self = Self(Scalar::ZERO); +} + +impl Serializable for Fp25519 { + type Size = <::Storage as Block>::Size; + + fn serialize(&self, buf: &mut GenericArray) { + let raw = &self.0.as_bytes()[..buf.len()] ; + buf.copy_from_slice(raw); + } + + fn deserialize(buf: &GenericArray) -> Self { + let mut buf_to = [0u8; 32]; + buf_to[..buf.len()].copy_from_slice(buf); + + Fp25519(Scalar::from_bytes_mod_order(buf_to)) + } +} + + +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> Fp25519 { + //Fp25519(Scalar::random(rng: &mut R)) + let mut scalar_bytes = [0u8; 64]; + rng.fill_bytes(&mut scalar_bytes); + Fp25519(Scalar::from_bytes_mod_order_wide(&scalar_bytes)) + } +} + + +impl std::ops::Add for Fp25519 { +type Output = Self; + +fn add(self, rhs: Self) -> Self::Output { + Self(self.0+rhs.0) +} +} + +impl std::ops::AddAssign for Fp25519 { +#[allow(clippy::assign_op_pattern)] +fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; +} +} + +impl std::ops::Neg for Fp25519 { +type Output = Self; + +fn neg(self) -> Self::Output { + Self(self.0.neg()) +} +} + +impl std::ops::Sub for Fp25519 { +type Output = Self; + +fn sub(self, rhs: Self) -> Self::Output { + Self(self.0- rhs.0) +} +} + +impl std::ops::SubAssign for Fp25519 { +#[allow(clippy::assign_op_pattern)] +fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; +} +} + +impl std::ops::Mul for Fp25519 { +type Output = Self; + +fn mul(self, rhs: Self) -> Self::Output { + Self(self.0 * rhs.0) +} +} + +impl std::ops::MulAssign for Fp25519 { +#[allow(clippy::assign_op_pattern)] +fn mul_assign(&mut self, rhs: Self) { + *self = *self * rhs; +} +} + +//must not use with ZERO +impl Fp25519 { + pub fn invert(&self) -> Fp25519 { + Fp25519(self.0.invert()) + } +} + +#[cfg(all(test, unit_test))] +mod test { + use generic_array::GenericArray; + use crate::ff::ec_prime_field::Fp25519; + use crate::ff::Serializable; + use typenum::U32; + use curve25519_dalek::scalar::Scalar; + use rand::{thread_rng, Rng}; + use crate::secret_sharing::SharedValue; + + #[test] + fn serde_25519() { + let input:[u8;32] = [ + 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + ]; + let mut output: GenericArray = [0u8;32].into(); + let a = Fp25519::deserialize(&input.into()); + assert_eq!(a.0.as_bytes()[..32],input); + a.serialize(&mut output); + assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); + assert_eq!(input,output.as_slice()[..32]); + } + + // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek + #[test] + fn simple_arithmetics_25519() { + let a = Fp25519(Scalar::from_bytes_mod_order([ + 0x02, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let b = Fp25519(Scalar::from_bytes_mod_order([ + 0x03, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let c = Fp25519(Scalar::from_bytes_mod_order([ + 0x01, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let d = Fp25519(Scalar::from_bytes_mod_order([ + 0x05, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let e = Fp25519(Scalar::from_bytes_mod_order([ + 0x06, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, + 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + ])); + let cc = b-a; + let dc = a+b; + let ec = a*b; + assert_eq!(cc,c); + assert_eq!(dc,d); + assert_eq!(ec,e); + } + + #[test] + fn simple_random_25519(){ + let mut rng = thread_rng(); + assert_ne!(Fp25519::ZERO,rng.gen::()); + } + + #[test] + fn invert_25519(){ + let mut rng = thread_rng(); + let a=rng.gen::(); + let ia=a.invert(); + assert_eq!(a*ia, Fp25519(Scalar::ONE)); + } +} \ No newline at end of file diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 5f18196c7..27e312663 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -5,6 +5,7 @@ mod field; mod galois_field; mod prime_field; +mod ec_prime_field; use std::ops::{Add, AddAssign, Sub, SubAssign}; From 786d7c85084c00f8d4651d3ec1e1492a1aed43c2 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 11:49:49 -0700 Subject: [PATCH 087/124] add files, make ec pub --- src/ff/mod.rs | 2 +- src/protocol/mod.rs | 2 ++ src/protocol/prf_eval/eval.rs | 0 src/protocol/prf_eval/keygen.rs | 0 src/protocol/prf_eval/mod.rs | 0 5 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 src/protocol/prf_eval/eval.rs create mode 100644 src/protocol/prf_eval/keygen.rs create mode 100644 src/protocol/prf_eval/mod.rs diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 27e312663..a438bee3c 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -5,7 +5,7 @@ mod field; mod galois_field; mod prime_field; -mod ec_prime_field; +pub mod ec_prime_field; use std::ops::{Add, AddAssign, Sub, SubAssign}; diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 8c5e72c7d..d9d1ea382 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -6,6 +6,8 @@ pub mod context; pub mod dp; pub mod ipa; pub mod modulus_conversion; +pub mod prf_eval; +#[cfg(feature = "descriptive-gate")] pub mod prf_sharding; pub mod prss; pub mod sort; diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/protocol/prf_eval/keygen.rs b/src/protocol/prf_eval/keygen.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs new file mode 100644 index 000000000..e69de29bb From c9cb5ee0e6859bb95e44fff3df3796be3db08f1c Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 14:20:19 -0700 Subject: [PATCH 088/124] last commit before working on shared curve points --- src/protocol/prf_eval/eval.rs | 28 ++++++++++++++++++++++++++++ src/protocol/prf_eval/mod.rs | 2 ++ 2 files changed, 30 insertions(+) diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs index e69de29bb..30c5c1d38 100644 --- a/src/protocol/prf_eval/eval.rs +++ b/src/protocol/prf_eval/eval.rs @@ -0,0 +1,28 @@ +use crate::{ + error::Error, + helpers::Direction, + ff::ec_prime_field::Fp25519, + protocol::{context::Context, step::BitOpStep, BasicProtocols, RecordId}, + secret_sharing::{Linear as LinearSecretSharing, LinearRefOps}, +}; + +/// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) +/// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 +/// PRF key k is generated using keygen +/// In 3IPA, x is the match key +/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output +pub async fn eval_dy( + ctx: C, + record_id: RecordId, + k: &[S], + x: &[S], +) -> Result + where + C: Context, + S: LinearSecretSharing + BasicProtocols, + for<'a> &'a S: LinearRefOps<'a, S, Fp25519>, +{ + let role = ctx.role(); + + Ok(5u64) +} \ No newline at end of file diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index e69de29bb..6183868b8 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -0,0 +1,2 @@ +mod eval; +mod keygen; \ No newline at end of file From 31cbde8465d81e08f9be052789ad3ba44fa9630e Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 2 Oct 2023 16:48:31 -0700 Subject: [PATCH 089/124] adding curve points + conversion --- src/ff/curve_points.rs | 170 +++++++++++++++++++++++++++++++++++++++ src/ff/ec_prime_field.rs | 6 ++ src/ff/mod.rs | 1 + 3 files changed, 177 insertions(+) create mode 100644 src/ff/curve_points.rs diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs new file mode 100644 index 000000000..3351efc51 --- /dev/null +++ b/src/ff/curve_points.rs @@ -0,0 +1,170 @@ +use std::ops::Mul; +use generic_array::GenericArray; +use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; +use rand_core::RngCore; +use typenum::U32; + +use crate::{ + ff::{Serializable,ec_prime_field::Fp25519}, + secret_sharing::{Block, SharedValue}, +}; + +impl Block for CompressedRistretto { + type Size = U32; +} + +///ristretto point for curve 25519 +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct RP25519(::Storage); + +//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? + +/// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 +impl SharedValue for RP25519 { + type Storage = CompressedRistretto; + const BITS: u32 = 256; + const ZERO: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); +} + +impl Serializable for RP25519 { + type Size = <::Storage as Block>::Size; + + fn serialize(&self, buf: &mut GenericArray) { + let raw = &self.0.as_bytes()[..buf.len()] ; + buf.copy_from_slice(raw); + } + + fn deserialize(buf: &GenericArray) -> Self { + RP25519(CompressedRistretto::from_slice(buf).unwrap()) + } +} + + +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> RP25519 { + //Fp25519(Scalar::random(rng: &mut R)) + let mut scalar_bytes = [0u8; 64]; + rng.fill_bytes(&mut scalar_bytes); + RP25519(RistrettoPoint::from_uniform_bytes(&scalar_bytes).compress()) + } +} + + +impl std::ops::Add for RP25519 { +type Output = Self; + +fn add(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap()+rhs.0.decompress().unwrap()).compress()) +} +} + +impl std::ops::AddAssign for RP25519 { +#[allow(clippy::assign_op_pattern)] +fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; +} +} + +impl std::ops::Neg for RP25519 { +type Output = Self; + +fn neg(self) -> Self::Output { + Self(self.0.decompress().unwrap().neg().compress()) +} +} + +impl std::ops::Sub for RP25519 { +type Output = Self; + +fn sub(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap()-rhs.0.decompress().unwrap()).compress()) +} +} + +impl std::ops::SubAssign for RP25519 { +#[allow(clippy::assign_op_pattern)] +fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; +} +} + + +///Scalar Multiplication +///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a +impl RP25519 { + +fn smul(self, rhs: Fp25519) -> RP25519 { + RP25519((self.0.decompress().unwrap() * >::into(rhs)).compress()) +} +} + +///<'a> std::ops::MulAssign<&'a Fp25519> for +impl RP25519 { +#[allow(clippy::assign_op_pattern)] +fn smul_assign(&mut self, rhs: Fp25519) { + *self = self.smul(rhs); +} +} + +///do not use +impl std::ops::Mul for RP25519 { + type Output = Self; + + fn mul(self, rhs: RP25519) -> Self::Output { + Self::ZERO + } +} + +///do not use +impl std::ops::MulAssign for RP25519 { + + fn mul_assign(& mut self, rhs: RP25519) { + *self=Self::ZERO; + } +} + +impl From for RP25519 { + fn from(s: Scalar) -> Self { + RP25519(RistrettoPoint::mul_base(&s).compress()) + } +} + +impl From for RP25519 { + fn from(s: Fp25519) -> Self { + RP25519(RistrettoPoint::mul_base(&s.into()).compress()) + } +} + +// impl Into for Fp25519 { +// fn into(self) -> RP25519 { +// RP25519(RistrettoPoint::mul_base(self.into()).compress()) +// } +// } + +#[cfg(all(test, unit_test))] +mod test { + use generic_array::GenericArray; + use crate::ff::curve_points::RP25519; + use crate::ff::Serializable; + use typenum::U32; + use curve25519_dalek::scalar::Scalar; + use rand::{thread_rng, Rng}; + use crate::secret_sharing::SharedValue; + + #[test] + fn serde_25519() { + let input:[u8;32] = [ + 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, + 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + ]; + let mut output: GenericArray = [0u8;32].into(); + let a = RP25519::deserialize(&input.into()); + assert_eq!(a.0.as_bytes()[..32],input); + a.serialize(&mut output); + assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); + assert_eq!(input,output.as_slice()[..32]); + } + +} \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index c87dbfa94..5e2d13fa5 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -24,6 +24,12 @@ impl SharedValue for Fp25519 { const ZERO: Self = Self(Scalar::ZERO); } +impl Into for Fp25519 { + fn into(self) -> Scalar { + self.0 + } +} + impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; diff --git a/src/ff/mod.rs b/src/ff/mod.rs index a438bee3c..e617359d1 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -6,6 +6,7 @@ mod field; mod galois_field; mod prime_field; pub mod ec_prime_field; +pub mod curve_points; use std::ops::{Add, AddAssign, Sub, SubAssign}; From 6219b4cc2f44d47511b4e7d6fa084f903b4dc32d Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 3 Oct 2023 10:17:02 -0700 Subject: [PATCH 090/124] from, into scalar for RP25519 + test --- src/ff/curve_points.rs | 22 +++++++++++++++++++--- src/ff/ec_prime_field.rs | 6 ++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 3351efc51..c0236c04a 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -93,7 +93,7 @@ fn sub_assign(&mut self, rhs: Self) { ///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a impl RP25519 { -fn smul(self, rhs: Fp25519) -> RP25519 { +fn s_mul(self, rhs: Fp25519) -> RP25519 { RP25519((self.0.decompress().unwrap() * >::into(rhs)).compress()) } } @@ -101,8 +101,8 @@ fn smul(self, rhs: Fp25519) -> RP25519 { ///<'a> std::ops::MulAssign<&'a Fp25519> for impl RP25519 { #[allow(clippy::assign_op_pattern)] -fn smul_assign(&mut self, rhs: Fp25519) { - *self = self.smul(rhs); +fn s_mul_assign(&mut self, rhs: Fp25519) { + *self = self.s_mul(rhs); } } @@ -149,6 +149,7 @@ mod test { use typenum::U32; use curve25519_dalek::scalar::Scalar; use rand::{thread_rng, Rng}; + use crate::ff::ec_prime_field::Fp25519; use crate::secret_sharing::SharedValue; #[test] @@ -167,4 +168,19 @@ mod test { assert_eq!(input,output.as_slice()[..32]); } + #[test] + fn scalar_to_point() { + let a = Scalar::ONE; + let b : RP25519 = a.clone().into(); + let d : Fp25519 = a.into(); + let c : RP25519 = RP25519::from(d); + assert_eq!(b,RP25519::ZERO); + assert_eq!(c,RP25519::ZERO); + } + + #[test] + fn curve_arithmetics() { + + } + } \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 5e2d13fa5..30ee3955e 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -117,6 +117,12 @@ impl Fp25519 { } } +impl From for Fp25519 { + fn from(s: Scalar) -> Self { + Fp25519(s) + } +} + #[cfg(all(test, unit_test))] mod test { use generic_array::GenericArray; From c65a93f27de80fa1ebb82a43c557b7c301f2eee6 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 3 Oct 2023 11:03:32 -0700 Subject: [PATCH 091/124] test aritmetics for RP25519 --- src/ff/curve_points.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index c0236c04a..1678210d7 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -180,7 +180,19 @@ mod test { #[test] fn curve_arithmetics() { - + let mut rng = rand::thread_rng(); + let a = rng.gen::(); + let b = rng.gen::(); + let c = a+b; + let d = RP25519::from(a)+RP25519::from(b); + assert_eq!(d, RP25519::from(c)); + assert_ne!(d, RP25519::ZERO); + let e = rng.gen::(); + let f=rng.gen::(); + let g =e*f; + let h = RP25519::from(e).s_mul(f); + assert_eq!(h,RP25519::from(g)); + assert_ne!(h, RP25519::ZERO); } } \ No newline at end of file From 1acb8696706a13206902fd71d78a3b7600a73e31 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 4 Oct 2023 11:29:56 -0700 Subject: [PATCH 092/124] upgrade share to share --- src/ff/curve_points.rs | 3 +-- src/ff/ec_prime_field.rs | 7 ++++--- src/protocol/prf_eval/eval.rs | 13 +++++++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 1678210d7..084b1a48d 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,4 +1,3 @@ -use std::ops::Mul; use generic_array::GenericArray; use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; use rand_core::RngCore; @@ -180,7 +179,7 @@ mod test { #[test] fn curve_arithmetics() { - let mut rng = rand::thread_rng(); + let mut rng = thread_rng(); let a = rng.gen::(); let b = rng.gen::(); let c = a+b; diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 30ee3955e..313cb9753 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -49,10 +49,11 @@ impl Serializable for Fp25519 { impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> Fp25519 { - //Fp25519(Scalar::random(rng: &mut R)) - let mut scalar_bytes = [0u8; 64]; + let mut scalar_bytes = [0u8; 32]; rng.fill_bytes(&mut scalar_bytes); - Fp25519(Scalar::from_bytes_mod_order_wide(&scalar_bytes)) + Fp25519(Scalar::from_bytes_mod_order(scalar_bytes)) + //not needed since above has sufficiently small bias + //Fp25519(Scalar::from_bytes_mod_order(&scalar_bytes)) } } diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs index 30c5c1d38..0307f42b8 100644 --- a/src/protocol/prf_eval/eval.rs +++ b/src/protocol/prf_eval/eval.rs @@ -1,11 +1,20 @@ use crate::{ error::Error, helpers::Direction, - ff::ec_prime_field::Fp25519, + ff::{ec_prime_field::Fp25519, curve_points::RP25519}, protocol::{context::Context, step::BitOpStep, BasicProtocols, RecordId}, - secret_sharing::{Linear as LinearSecretSharing, LinearRefOps}, + secret_sharing::{Linear as LinearSecretSharing, LinearRefOps, SharedValue, replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, }; + +impl Into> for AdditiveShare { + fn into(self) -> AdditiveShare + { + AdditiveShare::new(RP25519::from(self.left()),RP25519::from(self.right())) + } +} + + /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 /// PRF key k is generated using keygen From 644a6fc59afab5ca1e58f15116f82b0b658f0ae5 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 4 Oct 2023 11:57:10 -0700 Subject: [PATCH 093/124] remove warning --- src/ff/ec_prime_field.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 313cb9753..cedad411a 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -1,6 +1,6 @@ use generic_array::GenericArray; use curve25519_dalek::scalar::Scalar; -use rand_core::RngCore; +//use rand_core::RngCore; use typenum::U32; use crate::{ From 851f624e826e72593df78af684eabe13565ff151 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 4 Oct 2023 15:47:45 -0700 Subject: [PATCH 094/124] add hash curve points --- src/ff/curve_points.rs | 65 +++++++++++++++++++++++++++++----------- src/ff/ec_prime_field.rs | 31 +++++++++---------- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 084b1a48d..eed23e5ec 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,7 +1,9 @@ use generic_array::GenericArray; use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; -use rand_core::RngCore; use typenum::U32; +use sha2::Sha256; +use hkdf::Hkdf; + use crate::{ ff::{Serializable,ec_prime_field::Fp25519}, @@ -93,32 +95,25 @@ fn sub_assign(&mut self, rhs: Self) { impl RP25519 { fn s_mul(self, rhs: Fp25519) -> RP25519 { - RP25519((self.0.decompress().unwrap() * >::into(rhs)).compress()) + RP25519((self.0.decompress().unwrap() * Scalar::from(rhs)).compress()) } } -///<'a> std::ops::MulAssign<&'a Fp25519> for -impl RP25519 { -#[allow(clippy::assign_op_pattern)] -fn s_mul_assign(&mut self, rhs: Fp25519) { - *self = self.s_mul(rhs); -} -} ///do not use impl std::ops::Mul for RP25519 { type Output = Self; - fn mul(self, rhs: RP25519) -> Self::Output { - Self::ZERO + fn mul(self, _rhs: RP25519) -> Self::Output { + panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); } } ///do not use impl std::ops::MulAssign for RP25519 { - fn mul_assign(& mut self, rhs: RP25519) { - *self=Self::ZERO; + fn mul_assign(& mut self, _rhs: RP25519) { + panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); } } @@ -134,11 +129,36 @@ impl From for RP25519 { } } -// impl Into for Fp25519 { -// fn into(self) -> RP25519 { -// RP25519(RistrettoPoint::mul_base(self.into()).compress()) -// } -// } +macro_rules! cp_hash_impl { + ( $u_type:ty, $byte_size:literal) => { + impl From for $u_type { + fn from(s: RP25519) -> Self { + let hk = Hkdf::::new(None, s.0.as_bytes()); + let mut okm = [0u8; $byte_size]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + <$u_type>::from_le_bytes(okm) + } + } + } +} + +cp_hash_impl!( + u128, + 16 +); + +cp_hash_impl!( + u64, + 8 +); + +cp_hash_impl!( + u32, + 4 +); + + #[cfg(all(test, unit_test))] mod test { @@ -194,4 +214,13 @@ mod test { assert_ne!(h, RP25519::ZERO); } + #[test] + fn curve_point_to_hash() { + let mut rng = thread_rng(); + let a = rng.gen::(); + assert_ne!(0u128,u128::from(a)); + assert_ne!(0u64,u64::from(a)); + assert_ne!(0u32,u32::from(a)); + } + } \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index cedad411a..3e4f762d8 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -15,7 +15,16 @@ impl Block for Scalar { #[derive(Clone, Copy, PartialEq, Debug)] pub struct Fp25519(::Storage); -//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? + +impl Fp25519 { + const ONE: Self = Self(Scalar::ONE); + + //must not use with ZERO + pub fn invert(&self) -> Fp25519 { + Fp25519(self.0.invert()) + } + +} impl SharedValue for Fp25519 { @@ -24,9 +33,9 @@ impl SharedValue for Fp25519 { const ZERO: Self = Self(Scalar::ZERO); } -impl Into for Fp25519 { - fn into(self) -> Scalar { - self.0 +impl From for Scalar { + fn from(s: Fp25519) -> Self { + s.0 } } @@ -111,12 +120,6 @@ fn mul_assign(&mut self, rhs: Self) { } } -//must not use with ZERO -impl Fp25519 { - pub fn invert(&self) -> Fp25519 { - Fp25519(self.0.invert()) - } -} impl From for Fp25519 { fn from(s: Scalar) -> Self { @@ -165,12 +168,6 @@ mod test { 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 ])); - let c = Fp25519(Scalar::from_bytes_mod_order([ - 0x01, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 - ])); let d = Fp25519(Scalar::from_bytes_mod_order([ 0x05, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, @@ -186,7 +183,7 @@ mod test { let cc = b-a; let dc = a+b; let ec = a*b; - assert_eq!(cc,c); + assert_eq!(cc,Fp25519::ONE); assert_eq!(dc,d); assert_eq!(ec,e); } From 45b2a46122c0f6e4799ffacb9bf0fe037cd6d6e6 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Thu, 5 Oct 2023 13:46:24 -0700 Subject: [PATCH 095/124] implementation of PRF eval protocol --- src/ff/curve_points.rs | 72 ++++++++++++++++++++++++++---- src/ff/ec_prime_field.rs | 79 ++++++++++++++++++++++++++++++++- src/protocol/prf_eval/eval.rs | 37 --------------- src/protocol/prf_eval/keygen.rs | 0 src/protocol/prf_eval/mod.rs | 67 +++++++++++++++++++++++++++- 5 files changed, 206 insertions(+), 49 deletions(-) delete mode 100644 src/protocol/prf_eval/eval.rs delete mode 100644 src/protocol/prf_eval/keygen.rs diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index eed23e5ec..fad671dd0 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -6,7 +6,7 @@ use hkdf::Hkdf; use crate::{ - ff::{Serializable,ec_prime_field::Fp25519}, + ff::{Serializable,ec_prime_field::Fp25519,Field}, secret_sharing::{Block, SharedValue}, }; @@ -93,8 +93,9 @@ fn sub_assign(&mut self, rhs: Self) { ///Scalar Multiplication ///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a impl RP25519 { + pub const ONE: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); -fn s_mul(self, rhs: Fp25519) -> RP25519 { +pub fn s_mul(self, rhs: Fp25519) -> RP25519 { RP25519((self.0.decompress().unwrap() * Scalar::from(rhs)).compress()) } } @@ -129,6 +130,18 @@ impl From for RP25519 { } } +impl From for RP25519 { + fn from(s: CompressedRistretto) -> Self { + RP25519(s) + } +} + +impl From for CompressedRistretto { + fn from(s: RP25519) -> Self { + s.0 + } +} + macro_rules! cp_hash_impl { ( $u_type:ty, $byte_size:literal) => { impl From for $u_type { @@ -140,14 +153,19 @@ macro_rules! cp_hash_impl { <$u_type>::from_le_bytes(okm) } } + + impl From<$u_type> for RP25519 { + fn from(s: $u_type) -> Self { + let hk = Hkdf::::new(None, &s.to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + RP25519::deserialize(&okm.into()) + } + } } } -cp_hash_impl!( - u128, - 16 -); - cp_hash_impl!( u64, 8 @@ -158,6 +176,45 @@ cp_hash_impl!( 4 ); +/// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is +/// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed +impl Field for RP25519 { + const ONE: RP25519= RP25519::ONE; + + ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 + /// from or into u128. However it is sufficient to generate random elements in Fp25519 + fn as_u128(&self) -> u128 { + let hk = Hkdf::::new(None, self.0.as_bytes()); + let mut okm = [0u8; 16]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + u128::from_le_bytes(okm) + } + + ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + fn truncate_from>(v: T) -> Self { + let hk = Hkdf::::new(None, &v.into().to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + RP25519::deserialize(&okm.into()) + } + +} + +impl TryFrom for RP25519 { + type Error = crate::error::Error; + + fn try_from(v: u128) -> Result { + let mut bits = [0u8; 32]; + bits[..].copy_from_slice(&v.to_le_bytes()); + let f: RP25519=RP25519::ONE; + f.serialize((&mut bits).into()); + Ok(f) + } +} + + #[cfg(all(test, unit_test))] @@ -218,7 +275,6 @@ mod test { fn curve_point_to_hash() { let mut rng = thread_rng(); let a = rng.gen::(); - assert_ne!(0u128,u128::from(a)); assert_ne!(0u64,u64::from(a)); assert_ne!(0u32,u32::from(a)); } diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 3e4f762d8..2f88b07cc 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -2,9 +2,11 @@ use generic_array::GenericArray; use curve25519_dalek::scalar::Scalar; //use rand_core::RngCore; use typenum::U32; +use sha2::Sha256; +use hkdf::Hkdf; use crate::{ - ff::Serializable, + ff::{Serializable, Field}, secret_sharing::{Block, SharedValue}, }; @@ -17,7 +19,7 @@ pub struct Fp25519(::Storage); impl Fp25519 { - const ONE: Self = Self(Scalar::ONE); + pub const ONE: Self = Self(Scalar::ONE); //must not use with ZERO pub fn invert(&self) -> Fp25519 { @@ -127,6 +129,79 @@ impl From for Fp25519 { } } +macro_rules! sc_hash_impl { + ( $u_type:ty, $byte_size:literal) => { + impl From for $u_type { + fn from(s: Fp25519) -> Self { + let hk = Hkdf::::new(None, s.0.as_bytes()); + let mut okm = [0u8; $byte_size]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + <$u_type>::from_le_bytes(okm) + } + } + + impl From<$u_type> for Fp25519 { + fn from(s: $u_type) -> Self { + let hk = Hkdf::::new(None, &s.to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + Fp25519::deserialize(&okm.into()) + } + } + } +} + + +sc_hash_impl!( + u64, + 8 +); + +sc_hash_impl!( + u32, + 4 +); + + +/// Daniel had to implement this since PRSS wants it, prefer not to +impl Field for Fp25519 { + const ONE: Fp25519= Fp25519::ONE; + + ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 + /// from or into u128. However it is sufficient to generate random elements in Fp25519 + fn as_u128(&self) -> u128 { + let hk = Hkdf::::new(None, self.0.as_bytes()); + let mut okm = [0u8; 16]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + u128::from_le_bytes(okm) + } + + ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + fn truncate_from>(v: T) -> Self { + let hk = Hkdf::::new(None, &v.into().to_le_bytes()); + let mut okm = [0u8; 32]; + //error invalid length from expand only happens when okm is very large + hk.expand(&[], &mut okm).unwrap(); + Fp25519::deserialize(&okm.into()) + } + +} + +impl TryFrom for Fp25519 { + type Error = crate::error::Error; + + fn try_from(v: u128) -> Result { + let mut bits = [0u8; 32]; + bits[..].copy_from_slice(&v.to_le_bytes()); + let f: Fp25519=Fp25519::ONE; + f.serialize((&mut bits).into()); + Ok(f) + } +} + #[cfg(all(test, unit_test))] mod test { use generic_array::GenericArray; diff --git a/src/protocol/prf_eval/eval.rs b/src/protocol/prf_eval/eval.rs deleted file mode 100644 index 0307f42b8..000000000 --- a/src/protocol/prf_eval/eval.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::{ - error::Error, - helpers::Direction, - ff::{ec_prime_field::Fp25519, curve_points::RP25519}, - protocol::{context::Context, step::BitOpStep, BasicProtocols, RecordId}, - secret_sharing::{Linear as LinearSecretSharing, LinearRefOps, SharedValue, replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, -}; - - -impl Into> for AdditiveShare { - fn into(self) -> AdditiveShare - { - AdditiveShare::new(RP25519::from(self.left()),RP25519::from(self.right())) - } -} - - -/// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) -/// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 -/// PRF key k is generated using keygen -/// In 3IPA, x is the match key -/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output -pub async fn eval_dy( - ctx: C, - record_id: RecordId, - k: &[S], - x: &[S], -) -> Result - where - C: Context, - S: LinearSecretSharing + BasicProtocols, - for<'a> &'a S: LinearRefOps<'a, S, Fp25519>, -{ - let role = ctx.role(); - - Ok(5u64) -} \ No newline at end of file diff --git a/src/protocol/prf_eval/keygen.rs b/src/protocol/prf_eval/keygen.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index 6183868b8..193cb360c 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -1,2 +1,65 @@ -mod eval; -mod keygen; \ No newline at end of file +use crate::{ + error::Error, + ff::{ec_prime_field::Fp25519, curve_points::RP25519}, + protocol::{context::Context, RecordId, prss::SharedRandomness, basics::Reveal, basics::SecureMul}, + secret_sharing::{replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, +}; + + +impl From> for AdditiveShare { + fn from(s: AdditiveShare) -> Self { + AdditiveShare::new(RP25519::from(s.left()),RP25519::from(s.right())) + } +} + +/// generates PRF key k as secret sharing over Fp25519 +pub fn gen_prf_key (ctx: C) -> AdditiveShare + where + C: Context, +{ + ctx.prss().generate_replicated(u128::MAX-100u128) +} + + +/// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) +/// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 +/// PRF key k is generated using keygen +/// In 3IPA, x is the match key +/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output +pub async fn eval_dy_prf( + ctx: C, + record_id: RecordId, + k: &AdditiveShare, + x: &AdditiveShare, +) -> Result + where + C: Context, +{ + // generate random shares using shared randomness, use index max/2 to not clash with multiply + let sh_r: AdditiveShare = ctx.prss().generate_replicated(u128::MAX/2+u128::from(record_id)); + + //compute (g^left, g^right) + let sh_gr = AdditiveShare::::from(sh_r.clone()); + + //compute x+k + let y =x+k; + + //compute y <- r*y + //Daniel: probably shouldn't use ctx anymore? why doesn't y need to be mutable? + y.multiply(&sh_r, ctx.clone(), record_id).await?; + + //reconstruct (z,R) + let mut gr: RP25519 = sh_gr.reveal(ctx, record_id).await?; + + //invert z + let mut z: Fp25519 = Fp25519::ONE; + z=z.invert(); + //compute R^z + gr=gr.s_mul(z); + Ok(u64::from(gr)) +} + +#[cfg(all(test, unit_test))] +mod test { + +} \ No newline at end of file From dd07d7ac48c13377e3d5254b85d617e4f2ad0de3 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 6 Oct 2023 13:47:06 -0700 Subject: [PATCH 096/124] prf eval test passes --- src/protocol/prf_eval/mod.rs | 134 +++++++++++++++++++++++++++++++---- 1 file changed, 119 insertions(+), 15 deletions(-) diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index 193cb360c..e3d0cd6a4 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -3,7 +3,31 @@ use crate::{ ff::{ec_prime_field::Fp25519, curve_points::RP25519}, protocol::{context::Context, RecordId, prss::SharedRandomness, basics::Reveal, basics::SecureMul}, secret_sharing::{replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, + seq_join::seq_try_join_all, }; +use ipa_macros::Step; + +#[derive(Step)] +pub(crate) enum Step { + PRFKeyGen, + GenRandomMask, + MultMaskWithPRFInput, + RevealR, + Revealz, +} + +pub async fn compute_match_key_pseudonym( + sh_ctx: C, + prf_key: AdditiveShare, + input_match_keys: Vec>, +) -> Result, Error> + where + C: Context, +{ + let ctx =sh_ctx.set_total_records(input_match_keys.len()); + let futures=input_match_keys.iter().enumerate().map(|(i,x)|eval_dy_prf(ctx.clone(),i.into(),&prf_key,x)); + Ok(seq_try_join_all(sh_ctx.active_work(), futures).await?.iter().map(|&x|u64::from(x)).collect()) +} impl From> for AdditiveShare { @@ -17,7 +41,7 @@ pub fn gen_prf_key (ctx: C) -> AdditiveShare where C: Context, { - ctx.prss().generate_replicated(u128::MAX-100u128) + ctx.narrow(&Step::PRFKeyGen).prss().generate_replicated(u128::MAX-100u128) } @@ -26,7 +50,7 @@ pub fn gen_prf_key (ctx: C) -> AdditiveShare /// PRF key k is generated using keygen /// In 3IPA, x is the match key /// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output -pub async fn eval_dy_prf( +pub async fn eval_dy_prf( ctx: C, record_id: RecordId, k: &AdditiveShare, @@ -35,31 +59,111 @@ pub async fn eval_dy_prf( where C: Context, { - // generate random shares using shared randomness, use index max/2 to not clash with multiply - let sh_r: AdditiveShare = ctx.prss().generate_replicated(u128::MAX/2+u128::from(record_id)); + let sh_r: AdditiveShare = ctx.narrow(&Step::GenRandomMask).prss().generate_replicated(record_id); //compute (g^left, g^right) let sh_gr = AdditiveShare::::from(sh_r.clone()); //compute x+k - let y =x+k; + let mut y =x+k; //compute y <- r*y - //Daniel: probably shouldn't use ctx anymore? why doesn't y need to be mutable? - y.multiply(&sh_r, ctx.clone(), record_id).await?; + y = y.multiply(&sh_r, ctx.narrow(&Step::MultMaskWithPRFInput), record_id).await?; //reconstruct (z,R) - let mut gr: RP25519 = sh_gr.reveal(ctx, record_id).await?; - - //invert z - let mut z: Fp25519 = Fp25519::ONE; - z=z.invert(); - //compute R^z - gr=gr.s_mul(z); - Ok(u64::from(gr)) + let gr: RP25519 = sh_gr.reveal(ctx.narrow(&Step::RevealR), record_id).await?; + let z = y.reveal(ctx.narrow(&Step::Revealz), record_id).await?; + + + + //compute R^(1/z) to u64 + Ok(u64::from(gr.s_mul(z.invert()))) } #[cfg(all(test, unit_test))] mod test { + use rand::Rng; + use crate::{ + test_executor::run, + test_fixture::{TestWorld, Reconstruct, Runner}, + ff::{ec_prime_field::Fp25519, curve_points::RP25519}, + secret_sharing::{IntoShares,replicated::semi_honest::AdditiveShare}, + protocol::prf_eval::{compute_match_key_pseudonym}, + }; + + #[derive(Copy, Clone)] + struct ShuffledTestInput { + match_key: Fp25519, + } + + #[derive(Debug, PartialEq)] + struct TestOutput { + match_key_pseudonym: u64, + } + + fn test_input( mk: u64) -> ShuffledTestInput { + ShuffledTestInput { + match_key: Fp25519::from(mk), + } + } + + impl IntoShares> for ShuffledTestInput + { + fn share_with(self, rng: &mut R) -> [AdditiveShare; 3] { + self.match_key.share_with(rng) + } + } + + + impl Reconstruct for [&u64; 3] { + fn reconstruct(&self) -> TestOutput { + TestOutput{ + match_key_pseudonym: if *self[0]==*self[1] && *self[0]==*self[2] {*self[0]} else {0u64}, + } + } + } + + + #[test] + fn semi_honest() { + run(|| async move { + let world = TestWorld::default(); + + //first two need to be identical for test to succeed + let records: Vec = vec![ + test_input(3), + test_input(3), + test_input(23443524523), + test_input(56), + test_input(895764542), + test_input(456764576), + test_input(56), + test_input(3), + test_input(56), + test_input(23443524523), + ]; + + //PRF Key Gen + let u = 3216412445u64; + let k:Fp25519 = Fp25519::from(u); + + let expected: Vec=records.iter().map(|&x| TestOutput{match_key_pseudonym: (RP25519::from((x.match_key+k).invert())).into()} ).collect(); + + let result: Vec<_> = world + .semi_honest((records.into_iter(),k), |ctx, (input_match_keys,prf_key)| async move { + compute_match_key_pseudonym::<_>( + ctx, + prf_key, + input_match_keys, + ) + .await + .unwrap() + }) + .await + .reconstruct(); + assert_eq!(result, expected); + assert_eq!(result[0], result[1]); + }); + } } \ No newline at end of file From e1cb4b73428c48fd22b4a41117e424ddc12de1c5 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 9 Oct 2023 15:03:41 -0700 Subject: [PATCH 097/124] fixed clippy complaints, panics when setting invalid curve points or multiply curve points --- src/error.rs | 2 ++ src/ff/curve_points.rs | 47 +++++++++++++++++++++--------------- src/ff/ec_prime_field.rs | 3 ++- src/protocol/prf_eval/mod.rs | 23 +++++++++++------- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/error.rs b/src/error.rs index c68b325a8..7d6456c7f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -58,6 +58,8 @@ pub enum Error { InvalidReport(#[from] InvalidReportError), #[error("unsupported: {0}")] Unsupported(String), + #[error("Decompressing invalid elliptic curve point")] + DecompressingInvalidCurvePoint, } impl Default for Error { diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index fad671dd0..d04eec535 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -5,7 +5,9 @@ use sha2::Sha256; use hkdf::Hkdf; + use crate::{ + error::Error, ff::{Serializable,ec_prime_field::Fp25519,Field}, secret_sharing::{Block, SharedValue}, }; @@ -91,12 +93,17 @@ fn sub_assign(&mut self, rhs: Self) { ///Scalar Multiplication -///<'a, 'b> std::ops::Mul<&'b Fp25519> for &'a +///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a impl RP25519 { pub const ONE: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); -pub fn s_mul(self, rhs: Fp25519) -> RP25519 { - RP25519((self.0.decompress().unwrap() * Scalar::from(rhs)).compress()) + /// # Errors + /// Propagates errors from decompressing invalid curve point +pub fn s_mul(self, rhs: Fp25519) -> Result { + self.0.decompress().map_or( + Err(Error::DecompressingInvalidCurvePoint), + |x| Ok((x * Scalar::from(rhs)).compress().into()) + ) } } @@ -191,7 +198,7 @@ impl Field for RP25519 { u128::from_le_bytes(okm) } - ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + ///PRSS uses `truncate_from function`, we need to expand the u128 using a PRG (Sha256) to a [u8;32] fn truncate_from>(v: T) -> Self { let hk = Hkdf::::new(None, &v.into().to_le_bytes()); let mut okm = [0u8; 32]; @@ -247,7 +254,7 @@ mod test { #[test] fn scalar_to_point() { let a = Scalar::ONE; - let b : RP25519 = a.clone().into(); + let b : RP25519 = a.into(); let d : Fp25519 = a.into(); let c : RP25519 = RP25519::from(d); assert_eq!(b,RP25519::ZERO); @@ -257,26 +264,26 @@ mod test { #[test] fn curve_arithmetics() { let mut rng = thread_rng(); - let a = rng.gen::(); - let b = rng.gen::(); - let c = a+b; - let d = RP25519::from(a)+RP25519::from(b); - assert_eq!(d, RP25519::from(c)); - assert_ne!(d, RP25519::ZERO); - let e = rng.gen::(); - let f=rng.gen::(); - let g =e*f; - let h = RP25519::from(e).s_mul(f); - assert_eq!(h,RP25519::from(g)); - assert_ne!(h, RP25519::ZERO); + let fp_a = rng.gen::(); + let fp_b = rng.gen::(); + let fp_c = fp_a+fp_b; + let fp_d = RP25519::from(fp_a)+RP25519::from(fp_b); + assert_eq!(fp_d, RP25519::from(fp_c)); + assert_ne!(fp_d, RP25519::ZERO); + let fp_e = rng.gen::(); + let fp_f=rng.gen::(); + let fp_g =fp_e*fp_f; + let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); + assert_eq!(fp_h,RP25519::from(fp_g)); + assert_ne!(fp_h, RP25519::ZERO); } #[test] fn curve_point_to_hash() { let mut rng = thread_rng(); - let a = rng.gen::(); - assert_ne!(0u64,u64::from(a)); - assert_ne!(0u32,u32::from(a)); + let fp_a = rng.gen::(); + assert_ne!(0u64,u64::from(fp_a)); + assert_ne!(0u32,u32::from(fp_a)); } } \ No newline at end of file diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 2f88b07cc..624f91c64 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -22,6 +22,7 @@ impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); //must not use with ZERO + #[must_use] pub fn invert(&self) -> Fp25519 { Fp25519(self.0.invert()) } @@ -179,7 +180,7 @@ impl Field for Fp25519 { u128::from_le_bytes(okm) } - ///PRSS uses truncate_from function, we need to expand the u128 using a PRG (Sha256) to a [u8;32] + ///PRSS uses `truncate_from function`, we need to expand the u128 using a PRG (Sha256) to a [u8;32] fn truncate_from>(v: T) -> Self { let hk = Hkdf::::new(None, &v.into().to_le_bytes()); let mut okm = [0u8; 32]; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index e3d0cd6a4..895ad524b 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -16,6 +16,8 @@ pub(crate) enum Step { Revealz, } +/// # Errors +/// Propagates errors from multiplications pub async fn compute_match_key_pseudonym( sh_ctx: C, prf_key: AdditiveShare, @@ -26,7 +28,7 @@ pub async fn compute_match_key_pseudonym( { let ctx =sh_ctx.set_total_records(input_match_keys.len()); let futures=input_match_keys.iter().enumerate().map(|(i,x)|eval_dy_prf(ctx.clone(),i.into(),&prf_key,x)); - Ok(seq_try_join_all(sh_ctx.active_work(), futures).await?.iter().map(|&x|u64::from(x)).collect()) + seq_try_join_all(sh_ctx.active_work(), futures).await } @@ -37,7 +39,7 @@ impl From> for AdditiveShare { } /// generates PRF key k as secret sharing over Fp25519 -pub fn gen_prf_key (ctx: C) -> AdditiveShare +pub fn gen_prf_key (ctx: &C) -> AdditiveShare where C: Context, { @@ -49,7 +51,10 @@ pub fn gen_prf_key (ctx: C) -> AdditiveShare /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 /// PRF key k is generated using keygen /// In 3IPA, x is the match key -/// eval_DY outputs a u64 as specified in protocol/prf_sharding/mod.rs, all parties learn the output +/// outputs a u64 as specified in `protocol/prf_sharding/mod.rs`, all parties learn the output +/// # Errors +/// Propagates errors from multiplications, reveal and scalar multiplication + pub async fn eval_dy_prf( ctx: C, record_id: RecordId, @@ -77,7 +82,7 @@ pub async fn eval_dy_prf( //compute R^(1/z) to u64 - Ok(u64::from(gr.s_mul(z.invert()))) + Ok(u64::from(gr.s_mul(z.invert())?)) } #[cfg(all(test, unit_test))] @@ -133,18 +138,18 @@ mod test { let records: Vec = vec![ test_input(3), test_input(3), - test_input(23443524523), + test_input(23_443_524_523), test_input(56), - test_input(895764542), - test_input(456764576), + test_input(895_764_542), + test_input(456_764_576), test_input(56), test_input(3), test_input(56), - test_input(23443524523), + test_input(23_443_524_523), ]; //PRF Key Gen - let u = 3216412445u64; + let u = 3_216_412_445u64; let k:Fp25519 = Fp25519::from(u); let expected: Vec=records.iter().map(|&x| TestOutput{match_key_pseudonym: (RP25519::from((x.match_key+k).invert())).into()} ).collect(); From b2f5c7836c838488a7c959560e10b0f2b9753f6c Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 9 Oct 2023 15:49:42 -0700 Subject: [PATCH 098/124] remove comment --- src/ff/curve_points.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index d04eec535..b8d6a2882 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -20,8 +20,6 @@ impl Block for CompressedRistretto { #[derive(Clone, Copy, PartialEq, Debug)] pub struct RP25519(::Storage); -//how to add const ONE: Self = Self(Scalar::ONE); within the struct, do I need to do it via a trait? - /// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 impl SharedValue for RP25519 { type Storage = CompressedRistretto; From 8b0ab7743f1409ed9270b75fc1cae2d04fad6221 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 9 Oct 2023 16:29:30 -0700 Subject: [PATCH 099/124] adding some of Martins suggestions --- src/ff/curve_points.rs | 159 ++++++++++++++----------------- src/ff/ec_prime_field.rs | 176 ++++++++++++++++------------------- src/ff/mod.rs | 4 +- src/protocol/prf_eval/mod.rs | 111 ++++++++++++---------- 4 files changed, 217 insertions(+), 233 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index b8d6a2882..2b1f60f94 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,14 +1,16 @@ +use curve25519_dalek::{ + constants, + ristretto::{CompressedRistretto, RistrettoPoint}, + Scalar, +}; use generic_array::GenericArray; -use curve25519_dalek::{ristretto::{CompressedRistretto, RistrettoPoint},constants, Scalar}; -use typenum::U32; -use sha2::Sha256; use hkdf::Hkdf; - - +use sha2::Sha256; +use typenum::U32; use crate::{ error::Error, - ff::{Serializable,ec_prime_field::Fp25519,Field}, + ff::{ec_prime_field::Fp25519, Field, Serializable}, secret_sharing::{Block, SharedValue}, }; @@ -31,7 +33,7 @@ impl Serializable for RP25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()] ; + let raw = &self.0.as_bytes()[..buf.len()]; buf.copy_from_slice(raw); } @@ -40,7 +42,6 @@ impl Serializable for RP25519 { } } - impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> RP25519 { //Fp25519(Scalar::random(rng: &mut R)) @@ -50,62 +51,59 @@ impl rand::distributions::Distribution for rand::distributions::Standar } } - impl std::ops::Add for RP25519 { -type Output = Self; + type Output = Self; -fn add(self, rhs: Self) -> Self::Output { - Self((self.0.decompress().unwrap()+rhs.0.decompress().unwrap()).compress()) -} + fn add(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap() + rhs.0.decompress().unwrap()).compress()) + } } impl std::ops::AddAssign for RP25519 { -#[allow(clippy::assign_op_pattern)] -fn add_assign(&mut self, rhs: Self) { - *self = *self + rhs; -} + #[allow(clippy::assign_op_pattern)] + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } } impl std::ops::Neg for RP25519 { -type Output = Self; + type Output = Self; -fn neg(self) -> Self::Output { - Self(self.0.decompress().unwrap().neg().compress()) -} + fn neg(self) -> Self::Output { + Self(self.0.decompress().unwrap().neg().compress()) + } } impl std::ops::Sub for RP25519 { -type Output = Self; + type Output = Self; -fn sub(self, rhs: Self) -> Self::Output { - Self((self.0.decompress().unwrap()-rhs.0.decompress().unwrap()).compress()) -} + fn sub(self, rhs: Self) -> Self::Output { + Self((self.0.decompress().unwrap() - rhs.0.decompress().unwrap()).compress()) + } } impl std::ops::SubAssign for RP25519 { -#[allow(clippy::assign_op_pattern)] -fn sub_assign(&mut self, rhs: Self) { - *self = *self - rhs; -} + #[allow(clippy::assign_op_pattern)] + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } } - ///Scalar Multiplication ///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a impl RP25519 { - pub const ONE: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); /// # Errors /// Propagates errors from decompressing invalid curve point -pub fn s_mul(self, rhs: Fp25519) -> Result { - self.0.decompress().map_or( - Err(Error::DecompressingInvalidCurvePoint), - |x| Ok((x * Scalar::from(rhs)).compress().into()) - ) -} + pub fn s_mul(self, rhs: Fp25519) -> Result { + self.0 + .decompress() + .map_or(Err(Error::DecompressingInvalidCurvePoint), |x| { + Ok((x * Scalar::from(rhs)).compress().into()) + }) + } } - ///do not use impl std::ops::Mul for RP25519 { type Output = Self; @@ -117,8 +115,7 @@ impl std::ops::Mul for RP25519 { ///do not use impl std::ops::MulAssign for RP25519 { - - fn mul_assign(& mut self, _rhs: RP25519) { + fn mul_assign(&mut self, _rhs: RP25519) { panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); } } @@ -159,7 +156,7 @@ macro_rules! cp_hash_impl { } } - impl From<$u_type> for RP25519 { + impl From<$u_type> for RP25519 { fn from(s: $u_type) -> Self { let hk = Hkdf::::new(None, &s.to_le_bytes()); let mut okm = [0u8; 32]; @@ -168,23 +165,17 @@ macro_rules! cp_hash_impl { RP25519::deserialize(&okm.into()) } } - } + }; } -cp_hash_impl!( - u64, - 8 -); +cp_hash_impl!(u64, 8); -cp_hash_impl!( - u32, - 4 -); +cp_hash_impl!(u32, 4); /// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is /// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed impl Field for RP25519 { - const ONE: RP25519= RP25519::ONE; + const ONE: RP25519 = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 /// from or into u128. However it is sufficient to generate random elements in Fp25519 @@ -204,7 +195,6 @@ impl Field for RP25519 { hk.expand(&[], &mut okm).unwrap(); RP25519::deserialize(&okm.into()) } - } impl TryFrom for RP25519 { @@ -213,50 +203,46 @@ impl TryFrom for RP25519 { fn try_from(v: u128) -> Result { let mut bits = [0u8; 32]; bits[..].copy_from_slice(&v.to_le_bytes()); - let f: RP25519=RP25519::ONE; + let f: RP25519 = RP25519::ONE; f.serialize((&mut bits).into()); Ok(f) } } - - - #[cfg(all(test, unit_test))] mod test { - use generic_array::GenericArray; - use crate::ff::curve_points::RP25519; - use crate::ff::Serializable; - use typenum::U32; use curve25519_dalek::scalar::Scalar; + use generic_array::GenericArray; use rand::{thread_rng, Rng}; - use crate::ff::ec_prime_field::Fp25519; - use crate::secret_sharing::SharedValue; + use typenum::U32; + + use crate::{ + ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable, field::Field}, + }; #[test] fn serde_25519() { - let input:[u8;32] = [ - 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + let input: [u8; 32] = [ + 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0x00, 0x00, 0x00, ]; - let mut output: GenericArray = [0u8;32].into(); + let mut output: GenericArray = [0u8; 32].into(); let a = RP25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32],input); + assert_eq!(a.0.as_bytes()[..32], input); a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); - assert_eq!(input,output.as_slice()[..32]); + assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); + assert_eq!(input, output.as_slice()[..32]); } #[test] fn scalar_to_point() { let a = Scalar::ONE; - let b : RP25519 = a.into(); - let d : Fp25519 = a.into(); - let c : RP25519 = RP25519::from(d); - assert_eq!(b,RP25519::ZERO); - assert_eq!(c,RP25519::ZERO); + let b: RP25519 = a.into(); + let d: Fp25519 = a.into(); + let c: RP25519 = RP25519::from(d); + assert_eq!(b, RP25519::ONE); + assert_eq!(c, RP25519::ONE); } #[test] @@ -264,24 +250,23 @@ mod test { let mut rng = thread_rng(); let fp_a = rng.gen::(); let fp_b = rng.gen::(); - let fp_c = fp_a+fp_b; - let fp_d = RP25519::from(fp_a)+RP25519::from(fp_b); + let fp_c = fp_a + fp_b; + let fp_d = RP25519::from(fp_a) + RP25519::from(fp_b); assert_eq!(fp_d, RP25519::from(fp_c)); - assert_ne!(fp_d, RP25519::ZERO); + assert_ne!(fp_d, RP25519::ONE); let fp_e = rng.gen::(); - let fp_f=rng.gen::(); - let fp_g =fp_e*fp_f; + let fp_f = rng.gen::(); + let fp_g = fp_e * fp_f; let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); - assert_eq!(fp_h,RP25519::from(fp_g)); - assert_ne!(fp_h, RP25519::ZERO); + assert_eq!(fp_h, RP25519::from(fp_g)); + assert_ne!(fp_h, RP25519::ONE); } #[test] fn curve_point_to_hash() { let mut rng = thread_rng(); let fp_a = rng.gen::(); - assert_ne!(0u64,u64::from(fp_a)); - assert_ne!(0u32,u32::from(fp_a)); + assert_ne!(0u64, u64::from(fp_a)); + assert_ne!(0u32, u32::from(fp_a)); } - -} \ No newline at end of file +} diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 624f91c64..6613116f6 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -1,12 +1,12 @@ -use generic_array::GenericArray; use curve25519_dalek::scalar::Scalar; +use generic_array::GenericArray; +use hkdf::Hkdf; +use sha2::Sha256; //use rand_core::RngCore; use typenum::U32; -use sha2::Sha256; -use hkdf::Hkdf; use crate::{ - ff::{Serializable, Field}, + ff::{Field, Serializable}, secret_sharing::{Block, SharedValue}, }; @@ -17,7 +17,6 @@ impl Block for Scalar { #[derive(Clone, Copy, PartialEq, Debug)] pub struct Fp25519(::Storage); - impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); @@ -26,10 +25,8 @@ impl Fp25519 { pub fn invert(&self) -> Fp25519 { Fp25519(self.0.invert()) } - } - impl SharedValue for Fp25519 { type Storage = Scalar; const BITS: u32 = 256; @@ -46,7 +43,7 @@ impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()] ; + let raw = &self.0.as_bytes()[..buf.len()]; buf.copy_from_slice(raw); } @@ -58,7 +55,6 @@ impl Serializable for Fp25519 { } } - impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> Fp25519 { let mut scalar_bytes = [0u8; 32]; @@ -69,61 +65,59 @@ impl rand::distributions::Distribution for rand::distributions::Standar } } - impl std::ops::Add for Fp25519 { -type Output = Self; + type Output = Self; -fn add(self, rhs: Self) -> Self::Output { - Self(self.0+rhs.0) -} + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } } impl std::ops::AddAssign for Fp25519 { -#[allow(clippy::assign_op_pattern)] -fn add_assign(&mut self, rhs: Self) { - *self = *self + rhs; -} + #[allow(clippy::assign_op_pattern)] + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } } impl std::ops::Neg for Fp25519 { -type Output = Self; + type Output = Self; -fn neg(self) -> Self::Output { - Self(self.0.neg()) -} + fn neg(self) -> Self::Output { + Self(self.0.neg()) + } } impl std::ops::Sub for Fp25519 { -type Output = Self; + type Output = Self; -fn sub(self, rhs: Self) -> Self::Output { - Self(self.0- rhs.0) -} + fn sub(self, rhs: Self) -> Self::Output { + Self(self.0 - rhs.0) + } } impl std::ops::SubAssign for Fp25519 { -#[allow(clippy::assign_op_pattern)] -fn sub_assign(&mut self, rhs: Self) { - *self = *self - rhs; -} + #[allow(clippy::assign_op_pattern)] + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } } impl std::ops::Mul for Fp25519 { -type Output = Self; + type Output = Self; -fn mul(self, rhs: Self) -> Self::Output { - Self(self.0 * rhs.0) -} + fn mul(self, rhs: Self) -> Self::Output { + Self(self.0 * rhs.0) + } } impl std::ops::MulAssign for Fp25519 { -#[allow(clippy::assign_op_pattern)] -fn mul_assign(&mut self, rhs: Self) { - *self = *self * rhs; -} + #[allow(clippy::assign_op_pattern)] + fn mul_assign(&mut self, rhs: Self) { + *self = *self * rhs; + } } - impl From for Fp25519 { fn from(s: Scalar) -> Self { Fp25519(s) @@ -142,7 +136,7 @@ macro_rules! sc_hash_impl { } } - impl From<$u_type> for Fp25519 { + impl From<$u_type> for Fp25519 { fn from(s: $u_type) -> Self { let hk = Hkdf::::new(None, &s.to_le_bytes()); let mut okm = [0u8; 32]; @@ -151,24 +145,16 @@ macro_rules! sc_hash_impl { Fp25519::deserialize(&okm.into()) } } - } + }; } +sc_hash_impl!(u64, 8); -sc_hash_impl!( - u64, - 8 -); - -sc_hash_impl!( - u32, - 4 -); - +sc_hash_impl!(u32, 4); /// Daniel had to implement this since PRSS wants it, prefer not to impl Field for Fp25519 { - const ONE: Fp25519= Fp25519::ONE; + const ONE: Fp25519 = Fp25519::ONE; ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 /// from or into u128. However it is sufficient to generate random elements in Fp25519 @@ -188,7 +174,6 @@ impl Field for Fp25519 { hk.expand(&[], &mut okm).unwrap(); Fp25519::deserialize(&okm.into()) } - } impl TryFrom for Fp25519 { @@ -197,7 +182,7 @@ impl TryFrom for Fp25519 { fn try_from(v: u128) -> Result { let mut bits = [0u8; 32]; bits[..].copy_from_slice(&v.to_le_bytes()); - let f: Fp25519=Fp25519::ONE; + let f: Fp25519 = Fp25519::ONE; f.serialize((&mut bits).into()); Ok(f) } @@ -205,76 +190,73 @@ impl TryFrom for Fp25519 { #[cfg(all(test, unit_test))] mod test { - use generic_array::GenericArray; - use crate::ff::ec_prime_field::Fp25519; - use crate::ff::Serializable; - use typenum::U32; use curve25519_dalek::scalar::Scalar; + use generic_array::GenericArray; use rand::{thread_rng, Rng}; - use crate::secret_sharing::SharedValue; + use typenum::U32; + + use crate::{ + ff::{ec_prime_field::Fp25519, Serializable}, + secret_sharing::SharedValue, + }; #[test] fn serde_25519() { - let input:[u8;32] = [ - 0x01, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0xff,0x00, 0xff, - 0x00, 0xff,0x00, 0xff,0x00, 0x00,0x00, 0x00 + let input: [u8; 32] = [ + 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0x00, 0x00, 0x00, ]; - let mut output: GenericArray = [0u8;32].into(); + let mut output: GenericArray = [0u8; 32].into(); let a = Fp25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32],input); + assert_eq!(a.0.as_bytes()[..32], input); a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32],output.as_slice()[..32]); - assert_eq!(input,output.as_slice()[..32]); + assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); + assert_eq!(input, output.as_slice()[..32]); } // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek #[test] fn simple_arithmetics_25519() { let a = Fp25519(Scalar::from_bytes_mod_order([ - 0x02, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); let b = Fp25519(Scalar::from_bytes_mod_order([ - 0x03, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); let d = Fp25519(Scalar::from_bytes_mod_order([ - 0x05, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); let e = Fp25519(Scalar::from_bytes_mod_order([ - 0x06, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00, - 0x00, 0x00,0x00, 0x00,0x00, 0x00,0x00, 0x00 + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, ])); - let cc = b-a; - let dc = a+b; - let ec = a*b; - assert_eq!(cc,Fp25519::ONE); - assert_eq!(dc,d); - assert_eq!(ec,e); + let cc = b - a; + let dc = a + b; + let ec = a * b; + assert_eq!(cc, Fp25519::ONE); + assert_eq!(dc, d); + assert_eq!(ec, e); } #[test] - fn simple_random_25519(){ + fn simple_random_25519() { let mut rng = thread_rng(); - assert_ne!(Fp25519::ZERO,rng.gen::()); + assert_ne!(Fp25519::ZERO, rng.gen::()); } #[test] - fn invert_25519(){ + fn invert_25519() { let mut rng = thread_rng(); - let a=rng.gen::(); - let ia=a.invert(); - assert_eq!(a*ia, Fp25519(Scalar::ONE)); + let a = rng.gen::(); + let ia = a.invert(); + assert_eq!(a * ia, Fp25519(Scalar::ONE)); } -} \ No newline at end of file +} diff --git a/src/ff/mod.rs b/src/ff/mod.rs index e617359d1..1ced6caf0 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -2,11 +2,11 @@ // // This is where we store arithmetic shared secret data models. +pub mod curve_points; +pub mod ec_prime_field; mod field; mod galois_field; mod prime_field; -pub mod ec_prime_field; -pub mod curve_points; use std::ops::{Add, AddAssign, Sub, SubAssign}; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index 895ad524b..ae43e55a1 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -1,11 +1,17 @@ +use ipa_macros::Step; + use crate::{ error::Error, - ff::{ec_prime_field::Fp25519, curve_points::RP25519}, - protocol::{context::Context, RecordId, prss::SharedRandomness, basics::Reveal, basics::SecureMul}, - secret_sharing::{replicated::{semi_honest::AdditiveShare,ReplicatedSecretSharing}}, + ff::{curve_points::RP25519, ec_prime_field::Fp25519}, + protocol::{ + basics::{Reveal, SecureMul}, + context::Context, + prss::SharedRandomness, + RecordId, + }, + secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, seq_join::seq_try_join_all, }; -use ipa_macros::Step; #[derive(Step)] pub(crate) enum Step { @@ -23,30 +29,33 @@ pub async fn compute_match_key_pseudonym( prf_key: AdditiveShare, input_match_keys: Vec>, ) -> Result, Error> - where - C: Context, +where + C: Context, { - let ctx =sh_ctx.set_total_records(input_match_keys.len()); - let futures=input_match_keys.iter().enumerate().map(|(i,x)|eval_dy_prf(ctx.clone(),i.into(),&prf_key,x)); + let ctx = sh_ctx.set_total_records(input_match_keys.len()); + let futures = input_match_keys + .iter() + .enumerate() + .map(|(i, x)| eval_dy_prf(ctx.clone(), i.into(), &prf_key, x)); seq_try_join_all(sh_ctx.active_work(), futures).await } - impl From> for AdditiveShare { fn from(s: AdditiveShare) -> Self { - AdditiveShare::new(RP25519::from(s.left()),RP25519::from(s.right())) + AdditiveShare::new(RP25519::from(s.left()), RP25519::from(s.right())) } } /// generates PRF key k as secret sharing over Fp25519 -pub fn gen_prf_key (ctx: &C) -> AdditiveShare - where - C: Context, +pub fn gen_prf_key(ctx: &C) -> AdditiveShare +where + C: Context, { - ctx.narrow(&Step::PRFKeyGen).prss().generate_replicated(u128::MAX-100u128) + ctx.narrow(&Step::PRFKeyGen) + .prss() + .generate_replicated(u128::MAX - 100u128) } - /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 /// PRF key k is generated using keygen @@ -61,26 +70,29 @@ pub async fn eval_dy_prf( k: &AdditiveShare, x: &AdditiveShare, ) -> Result - where - C: Context, +where + C: Context, { - let sh_r: AdditiveShare = ctx.narrow(&Step::GenRandomMask).prss().generate_replicated(record_id); + let sh_r: AdditiveShare = ctx + .narrow(&Step::GenRandomMask) + .prss() + .generate_replicated(record_id); //compute (g^left, g^right) let sh_gr = AdditiveShare::::from(sh_r.clone()); //compute x+k - let mut y =x+k; + let mut y = x + k; //compute y <- r*y - y = y.multiply(&sh_r, ctx.narrow(&Step::MultMaskWithPRFInput), record_id).await?; + y = y + .multiply(&sh_r, ctx.narrow(&Step::MultMaskWithPRFInput), record_id) + .await?; //reconstruct (z,R) let gr: RP25519 = sh_gr.reveal(ctx.narrow(&Step::RevealR), record_id).await?; let z = y.reveal(ctx.narrow(&Step::Revealz), record_id).await?; - - //compute R^(1/z) to u64 Ok(u64::from(gr.s_mul(z.invert())?)) } @@ -88,12 +100,13 @@ pub async fn eval_dy_prf( #[cfg(all(test, unit_test))] mod test { use rand::Rng; + use crate::{ + ff::{curve_points::RP25519, ec_prime_field::Fp25519}, + protocol::prf_eval::compute_match_key_pseudonym, + secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_executor::run, - test_fixture::{TestWorld, Reconstruct, Runner}, - ff::{ec_prime_field::Fp25519, curve_points::RP25519}, - secret_sharing::{IntoShares,replicated::semi_honest::AdditiveShare}, - protocol::prf_eval::{compute_match_key_pseudonym}, + test_fixture::{Reconstruct, Runner, TestWorld}, }; #[derive(Copy, Clone)] @@ -106,29 +119,30 @@ mod test { match_key_pseudonym: u64, } - fn test_input( mk: u64) -> ShuffledTestInput { + fn test_input(mk: u64) -> ShuffledTestInput { ShuffledTestInput { match_key: Fp25519::from(mk), } } - impl IntoShares> for ShuffledTestInput - { + impl IntoShares> for ShuffledTestInput { fn share_with(self, rng: &mut R) -> [AdditiveShare; 3] { self.match_key.share_with(rng) } } - impl Reconstruct for [&u64; 3] { fn reconstruct(&self) -> TestOutput { - TestOutput{ - match_key_pseudonym: if *self[0]==*self[1] && *self[0]==*self[2] {*self[0]} else {0u64}, + TestOutput { + match_key_pseudonym: if *self[0] == *self[1] && *self[0] == *self[2] { + *self[0] + } else { + 0u64 + }, } } } - #[test] fn semi_honest() { run(|| async move { @@ -146,29 +160,32 @@ mod test { test_input(3), test_input(56), test_input(23_443_524_523), - ]; + ]; //PRF Key Gen let u = 3_216_412_445u64; - let k:Fp25519 = Fp25519::from(u); + let k: Fp25519 = Fp25519::from(u); - let expected: Vec=records.iter().map(|&x| TestOutput{match_key_pseudonym: (RP25519::from((x.match_key+k).invert())).into()} ).collect(); + let expected: Vec = records + .iter() + .map(|&x| TestOutput { + match_key_pseudonym: (RP25519::from((x.match_key + k).invert())).into(), + }) + .collect(); let result: Vec<_> = world - .semi_honest((records.into_iter(),k), |ctx, (input_match_keys,prf_key)| async move { - compute_match_key_pseudonym::<_>( - ctx, - prf_key, - input_match_keys, - ) - .await - .unwrap() - }) + .semi_honest( + (records.into_iter(), k), + |ctx, (input_match_keys, prf_key)| async move { + compute_match_key_pseudonym::<_>(ctx, prf_key, input_match_keys) + .await + .unwrap() + }, + ) .await .reconstruct(); assert_eq!(result, expected); assert_eq!(result[0], result[1]); }); } - -} \ No newline at end of file +} From 4c175bd80accc0401f86c8d8dd5c2ae4300b7ead Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 12:14:20 -0700 Subject: [PATCH 100/124] Alex suggestions, serialize, deserialize --- src/ff/curve_points.rs | 3 +-- src/ff/ec_prime_field.rs | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 2b1f60f94..38908dadf 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -33,8 +33,7 @@ impl Serializable for RP25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()]; - buf.copy_from_slice(raw); + *buf.as_mut() = self.0.to_bytes(); } fn deserialize(buf: &GenericArray) -> Self { diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 6613116f6..d7348ca1b 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -43,15 +43,11 @@ impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - let raw = &self.0.as_bytes()[..buf.len()]; - buf.copy_from_slice(raw); + *buf.as_mut() = self.0.to_bytes() } fn deserialize(buf: &GenericArray) -> Self { - let mut buf_to = [0u8; 32]; - buf_to[..buf.len()].copy_from_slice(buf); - - Fp25519(Scalar::from_bytes_mod_order(buf_to)) + Fp25519(Scalar::from_bytes_mod_order((*buf).into())) } } From 537e7af1f6515e78b6a9496d8478bd27b29a2d50 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 12:29:06 -0700 Subject: [PATCH 101/124] Alex suggestions, simplify u32, u64 macros --- src/ff/curve_points.rs | 8 ++++---- src/ff/ec_prime_field.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 38908dadf..0890903a5 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -144,11 +144,11 @@ impl From for CompressedRistretto { } macro_rules! cp_hash_impl { - ( $u_type:ty, $byte_size:literal) => { + ( $u_type:ty) => { impl From for $u_type { fn from(s: RP25519) -> Self { let hk = Hkdf::::new(None, s.0.as_bytes()); - let mut okm = [0u8; $byte_size]; + let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large hk.expand(&[], &mut okm).unwrap(); <$u_type>::from_le_bytes(okm) @@ -167,9 +167,9 @@ macro_rules! cp_hash_impl { }; } -cp_hash_impl!(u64, 8); +cp_hash_impl!(u64); -cp_hash_impl!(u32, 4); +cp_hash_impl!(u32); /// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is /// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index d7348ca1b..9234cb346 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -121,11 +121,11 @@ impl From for Fp25519 { } macro_rules! sc_hash_impl { - ( $u_type:ty, $byte_size:literal) => { + ( $u_type:ty) => { impl From for $u_type { fn from(s: Fp25519) -> Self { let hk = Hkdf::::new(None, s.0.as_bytes()); - let mut okm = [0u8; $byte_size]; + let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large hk.expand(&[], &mut okm).unwrap(); <$u_type>::from_le_bytes(okm) @@ -144,9 +144,9 @@ macro_rules! sc_hash_impl { }; } -sc_hash_impl!(u64, 8); +sc_hash_impl!(u64); -sc_hash_impl!(u32, 4); +sc_hash_impl!(u32); /// Daniel had to implement this since PRSS wants it, prefer not to impl Field for Fp25519 { From 5325f6dbcc2db7f17305786921a87654bc5e117d Mon Sep 17 00:00:00 2001 From: danielmasny <46358615+danielmasny@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:19:22 -0700 Subject: [PATCH 102/124] Update src/ff/curve_points.rs Co-authored-by: Alex Koshelev --- src/ff/curve_points.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 0890903a5..fb605205e 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -119,7 +119,7 @@ impl std::ops::MulAssign for RP25519 { } } -impl From for RP25519 { +impl From<&Scalar> for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) } From dd0aac091698520ea37fa28798592dc2c6f639cf Mon Sep 17 00:00:00 2001 From: danielmasny <46358615+danielmasny@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:06:29 -0700 Subject: [PATCH 103/124] Update src/ff/curve_points.rs Co-authored-by: Alex Koshelev --- src/ff/curve_points.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index fb605205e..76657874a 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -125,7 +125,7 @@ impl From<&Scalar> for RP25519 { } } -impl From for RP25519 { +impl From<&Fp25519> for RP25519 { fn from(s: Fp25519) -> Self { RP25519(RistrettoPoint::mul_base(&s.into()).compress()) } From 5edee6fdc58f60546c55fe709e9fbb5b5de8d0f2 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 15:17:02 -0700 Subject: [PATCH 104/124] simplify serde test --- src/ff/curve_points.rs | 102 ++++++++++----------------------------- src/ff/ec_prime_field.rs | 23 ++++----- 2 files changed, 34 insertions(+), 91 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 76657874a..bf09072ef 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -10,7 +10,7 @@ use typenum::U32; use crate::{ error::Error, - ff::{ec_prime_field::Fp25519, Field, Serializable}, + ff::{ec_prime_field::Fp25519, Serializable}, secret_sharing::{Block, SharedValue}, }; @@ -37,16 +37,7 @@ impl Serializable for RP25519 { } fn deserialize(buf: &GenericArray) -> Self { - RP25519(CompressedRistretto::from_slice(buf).unwrap()) - } -} - -impl rand::distributions::Distribution for rand::distributions::Standard { - fn sample(&self, rng: &mut R) -> RP25519 { - //Fp25519(Scalar::random(rng: &mut R)) - let mut scalar_bytes = [0u8; 64]; - rng.fill_bytes(&mut scalar_bytes); - RP25519(RistrettoPoint::from_uniform_bytes(&scalar_bytes).compress()) + RP25519(CompressedRistretto((*buf).into())) } } @@ -91,7 +82,6 @@ impl std::ops::SubAssign for RP25519 { ///Scalar Multiplication ///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a impl RP25519 { - /// # Errors /// Propagates errors from decompressing invalid curve point pub fn s_mul(self, rhs: Fp25519) -> Result { @@ -119,13 +109,13 @@ impl std::ops::MulAssign for RP25519 { } } -impl From<&Scalar> for RP25519 { +impl From for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) } } -impl From<&Fp25519> for RP25519 { +impl From for RP25519 { fn from(s: Fp25519) -> Self { RP25519(RistrettoPoint::mul_base(&s.into()).compress()) } @@ -154,84 +144,42 @@ macro_rules! cp_hash_impl { <$u_type>::from_le_bytes(okm) } } - - impl From<$u_type> for RP25519 { - fn from(s: $u_type) -> Self { - let hk = Hkdf::::new(None, &s.to_le_bytes()); - let mut okm = [0u8; 32]; - //error invalid length from expand only happens when okm is very large - hk.expand(&[], &mut okm).unwrap(); - RP25519::deserialize(&okm.into()) - } - } }; } +cp_hash_impl!(u128); + cp_hash_impl!(u64); +#[cfg(test)] cp_hash_impl!(u32); -/// Daniel had to implement this since Reveal wants it, prefer not to, I dont understand why it is -/// actually needed there, maybe to upgrade it to malicious? but it still shouldn't be needed -impl Field for RP25519 { - const ONE: RP25519 = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); - - ///both following methods are based on hashing and do not allow to actually convert elements in Fp25519 - /// from or into u128. However it is sufficient to generate random elements in Fp25519 - fn as_u128(&self) -> u128 { - let hk = Hkdf::::new(None, self.0.as_bytes()); - let mut okm = [0u8; 16]; - //error invalid length from expand only happens when okm is very large - hk.expand(&[], &mut okm).unwrap(); - u128::from_le_bytes(okm) - } - - ///PRSS uses `truncate_from function`, we need to expand the u128 using a PRG (Sha256) to a [u8;32] - fn truncate_from>(v: T) -> Self { - let hk = Hkdf::::new(None, &v.into().to_le_bytes()); - let mut okm = [0u8; 32]; - //error invalid length from expand only happens when okm is very large - hk.expand(&[], &mut okm).unwrap(); - RP25519::deserialize(&okm.into()) - } -} - -impl TryFrom for RP25519 { - type Error = crate::error::Error; - - fn try_from(v: u128) -> Result { - let mut bits = [0u8; 32]; - bits[..].copy_from_slice(&v.to_le_bytes()); - let f: RP25519 = RP25519::ONE; - f.serialize((&mut bits).into()); - Ok(f) +#[cfg(test)] +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> RP25519 { + let mut scalar_bytes = [0u8; 64]; + rng.fill_bytes(&mut scalar_bytes); + RP25519(RistrettoPoint::from_uniform_bytes(&scalar_bytes).compress()) } } #[cfg(all(test, unit_test))] mod test { - use curve25519_dalek::scalar::Scalar; + use curve25519_dalek::{constants, scalar::Scalar}; use generic_array::GenericArray; use rand::{thread_rng, Rng}; use typenum::U32; - use crate::{ - ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable, field::Field}, - }; + use crate::ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable}; #[test] fn serde_25519() { - let input: [u8; 32] = [ - 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0x00, 0x00, 0x00, - ]; - let mut output: GenericArray = [0u8; 32].into(); - let a = RP25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32], input); - a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); - assert_eq!(input, output.as_slice()[..32]); + let mut rng = thread_rng(); + let input = rng.gen::(); + let mut a: GenericArray = [0u8; 32].into(); + input.serialize(&mut a); + let output = RP25519::deserialize(&a); + assert_eq!(input, output); } #[test] @@ -240,8 +188,8 @@ mod test { let b: RP25519 = a.into(); let d: Fp25519 = a.into(); let c: RP25519 = RP25519::from(d); - assert_eq!(b, RP25519::ONE); - assert_eq!(c, RP25519::ONE); + assert_eq!(b, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); + assert_eq!(c, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } #[test] @@ -252,13 +200,13 @@ mod test { let fp_c = fp_a + fp_b; let fp_d = RP25519::from(fp_a) + RP25519::from(fp_b); assert_eq!(fp_d, RP25519::from(fp_c)); - assert_ne!(fp_d, RP25519::ONE); + assert_ne!(fp_d, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); let fp_e = rng.gen::(); let fp_f = rng.gen::(); let fp_g = fp_e * fp_f; let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); assert_eq!(fp_h, RP25519::from(fp_g)); - assert_ne!(fp_h, RP25519::ONE); + assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } #[test] diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 9234cb346..e67ffa235 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -2,7 +2,6 @@ use curve25519_dalek::scalar::Scalar; use generic_array::GenericArray; use hkdf::Hkdf; use sha2::Sha256; -//use rand_core::RngCore; use typenum::U32; use crate::{ @@ -23,6 +22,7 @@ impl Fp25519 { //must not use with ZERO #[must_use] pub fn invert(&self) -> Fp25519 { + assert_ne!(*self, Fp25519::ZERO); Fp25519(self.0.invert()) } } @@ -56,8 +56,6 @@ impl rand::distributions::Distribution for rand::distributions::Standar let mut scalar_bytes = [0u8; 32]; rng.fill_bytes(&mut scalar_bytes); Fp25519(Scalar::from_bytes_mod_order(scalar_bytes)) - //not needed since above has sufficiently small bias - //Fp25519(Scalar::from_bytes_mod_order(&scalar_bytes)) } } @@ -144,8 +142,10 @@ macro_rules! sc_hash_impl { }; } +#[cfg(test)] sc_hash_impl!(u64); +#[cfg(test)] sc_hash_impl!(u32); /// Daniel had to implement this since PRSS wants it, prefer not to @@ -198,17 +198,12 @@ mod test { #[test] fn serde_25519() { - let input: [u8; 32] = [ - 0x01, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, - 0x00, 0x00, 0x00, 0x00, - ]; - let mut output: GenericArray = [0u8; 32].into(); - let a = Fp25519::deserialize(&input.into()); - assert_eq!(a.0.as_bytes()[..32], input); - a.serialize(&mut output); - assert_eq!(a.0.as_bytes()[..32], output.as_slice()[..32]); - assert_eq!(input, output.as_slice()[..32]); + let mut rng = thread_rng(); + let input = rng.gen::(); + let mut a: GenericArray = [0u8; 32].into(); + input.serialize(&mut a); + let output = Fp25519::deserialize(&a); + assert_eq!(input, output); } // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek From 3fdd719e7de2bc993e7b3c8bb32a1e0522040327 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 15:21:42 -0700 Subject: [PATCH 105/124] simplify serde test --- src/ff/ec_prime_field.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index e67ffa235..d0ff48b72 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -19,7 +19,8 @@ pub struct Fp25519(::Storage); impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); - //must not use with ZERO + ///# Panics + /// Panics when self is zero #[must_use] pub fn invert(&self) -> Fp25519 { assert_ne!(*self, Fp25519::ZERO); @@ -43,7 +44,7 @@ impl Serializable for Fp25519 { type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { - *buf.as_mut() = self.0.to_bytes() + *buf.as_mut() = self.0.to_bytes(); } fn deserialize(buf: &GenericArray) -> Self { From 5e65826533f91dc203c94d8843a6adba0fedab3a Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Mon, 16 Oct 2023 17:57:38 -0700 Subject: [PATCH 106/124] error recursion limit --- src/ff/curve_points.rs | 8 ++-- src/secret_sharing/mod.rs | 47 ++++++++++++++++--- src/secret_sharing/replicated/mod.rs | 4 +- .../replicated/semi_honest/additive_share.rs | 43 ++++++++--------- src/secret_sharing/scheme.rs | 4 +- 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index bf09072ef..cf182fd69 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -11,7 +11,7 @@ use typenum::U32; use crate::{ error::Error, ff::{ec_prime_field::Fp25519, Serializable}, - secret_sharing::{Block, SharedValue}, + secret_sharing::{Block, WeakSharedValue}, }; impl Block for CompressedRistretto { @@ -20,17 +20,17 @@ impl Block for CompressedRistretto { ///ristretto point for curve 25519 #[derive(Clone, Copy, PartialEq, Debug)] -pub struct RP25519(::Storage); +pub struct RP25519(::Storage); /// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 -impl SharedValue for RP25519 { +impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; const ZERO: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); } impl Serializable for RP25519 { - type Size = <::Storage as Block>::Size; + type Size = <::Storage as Block>::Size; fn serialize(&self, buf: &mut GenericArray) { *buf.as_mut() = self.0.to_bytes(); diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index f366c870c..110bf74f5 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -23,22 +23,33 @@ pub use scheme::{Bitwise, Linear, LinearRefOps, SecretSharing}; use crate::ff::{AddSub, AddSubAssign, Serializable}; +/// Operations supported for weak shared values. +pub trait Additive: +AddSub ++ AddSubAssign ++ Neg +{ +} + +impl Additive for T where + T: AddSub + + AddSubAssign + + Neg +{ +} + /// Operations supported for shared values. pub trait Arithmetic: - AddSub - + AddSubAssign + Additive + Mul + MulAssign - + Neg { } impl Arithmetic for T where - T: AddSub - + AddSubAssign + T: Additive + Mul + MulAssign - + Neg { } @@ -48,6 +59,16 @@ pub trait Block: Sized + Copy + Debug { type Size: ArrayLength; } +pub trait WeakSharedValue: +Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static +{ + type Storage: Block; + + const BITS: u32; + + const ZERO: Self; +} + pub trait SharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Arithmetic + Serializable + 'static { @@ -58,10 +79,22 @@ pub trait SharedValue: const ZERO: Self; } +impl WeakSharedValue for T where + T: SharedValue, +{ + type Storage = T::Storage; + + const BITS: u32 = T::BITS; + + const ZERO: Self = T::ZERO; +} + + + #[cfg(any(test, feature = "test-fixture", feature = "cli"))] impl IntoShares> for V where - V: SharedValue, + V: WeakSharedValue, Standard: Distribution, { fn share_with(self, rng: &mut R) -> [AdditiveShare; 3] { diff --git a/src/secret_sharing/replicated/mod.rs b/src/secret_sharing/replicated/mod.rs index dcf51494e..80c8167b0 100644 --- a/src/secret_sharing/replicated/mod.rs +++ b/src/secret_sharing/replicated/mod.rs @@ -1,9 +1,9 @@ pub mod malicious; pub mod semi_honest; -use super::{SecretSharing, SharedValue}; +use super::{SecretSharing, SharedValue, WeakSharedValue}; -pub trait ReplicatedSecretSharing: SecretSharing { +pub trait ReplicatedSecretSharing: SecretSharing { fn new(a: V, b: V) -> Self; fn left(&self) -> V; fn right(&self) -> V; diff --git a/src/secret_sharing/replicated/semi_honest/additive_share.rs b/src/secret_sharing/replicated/semi_honest/additive_share.rs index 10b8b2b39..fb7438c2f 100644 --- a/src/secret_sharing/replicated/semi_honest/additive_share.rs +++ b/src/secret_sharing/replicated/semi_honest/additive_share.rs @@ -11,31 +11,32 @@ use crate::{ secret_sharing::{ replicated::ReplicatedSecretSharing, Linear as LinearSecretSharing, SecretSharing, SharedValue, + WeakSharedValue, }, }; #[derive(Clone, PartialEq, Eq)] -pub struct AdditiveShare(V, V); +pub struct AdditiveShare(V, V); -impl SecretSharing for AdditiveShare { +impl SecretSharing for AdditiveShare { const ZERO: Self = AdditiveShare::ZERO; } impl LinearSecretSharing for AdditiveShare {} -impl Debug for AdditiveShare { +impl Debug for AdditiveShare { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "({:?}, {:?})", self.0, self.1) } } -impl Default for AdditiveShare { +impl Default for AdditiveShare { fn default() -> Self { AdditiveShare::new(V::ZERO, V::ZERO) } } -impl AdditiveShare { +impl AdditiveShare { /// Replicated secret share where both left and right values are `F::ZERO` pub const ZERO: Self = Self(V::ZERO, V::ZERO); @@ -44,7 +45,7 @@ impl AdditiveShare { } } -impl ReplicatedSecretSharing for AdditiveShare { +impl ReplicatedSecretSharing for AdditiveShare { fn new(a: V, b: V) -> Self { Self(a, b) } @@ -58,7 +59,7 @@ impl ReplicatedSecretSharing for AdditiveShare { } } -impl AdditiveShare +impl AdditiveShare where Self: Serializable, { @@ -73,7 +74,7 @@ where } } -impl<'a, 'b, V: SharedValue> Add<&'b AdditiveShare> for &'a AdditiveShare { +impl<'a, 'b, V: WeakSharedValue> Add<&'b AdditiveShare> for &'a AdditiveShare { type Output = AdditiveShare; fn add(self, rhs: &'b AdditiveShare) -> Self::Output { @@ -81,7 +82,7 @@ impl<'a, 'b, V: SharedValue> Add<&'b AdditiveShare> for &'a AdditiveShare } } -impl Add for AdditiveShare { +impl Add for AdditiveShare { type Output = Self; fn add(self, rhs: Self) -> Self::Output { @@ -89,7 +90,7 @@ impl Add for AdditiveShare { } } -impl Add> for &AdditiveShare { +impl Add> for &AdditiveShare { type Output = AdditiveShare; fn add(self, rhs: AdditiveShare) -> Self::Output { @@ -97,7 +98,7 @@ impl Add> for &AdditiveShare { } } -impl Add<&AdditiveShare> for AdditiveShare { +impl Add<&AdditiveShare> for AdditiveShare { type Output = Self; fn add(self, rhs: &Self) -> Self::Output { @@ -105,20 +106,20 @@ impl Add<&AdditiveShare> for AdditiveShare { } } -impl AddAssign<&Self> for AdditiveShare { +impl AddAssign<&Self> for AdditiveShare { fn add_assign(&mut self, rhs: &Self) { self.0 += rhs.0; self.1 += rhs.1; } } -impl AddAssign for AdditiveShare { +impl AddAssign for AdditiveShare { fn add_assign(&mut self, rhs: Self) { AddAssign::add_assign(self, &rhs); } } -impl Neg for &AdditiveShare { +impl Neg for &AdditiveShare { type Output = AdditiveShare; fn neg(self) -> Self::Output { @@ -126,7 +127,7 @@ impl Neg for &AdditiveShare { } } -impl Neg for AdditiveShare { +impl Neg for AdditiveShare { type Output = Self; fn neg(self) -> Self::Output { @@ -134,7 +135,7 @@ impl Neg for AdditiveShare { } } -impl Sub for &AdditiveShare { +impl Sub for &AdditiveShare { type Output = AdditiveShare; fn sub(self, rhs: Self) -> Self::Output { @@ -142,7 +143,7 @@ impl Sub for &AdditiveShare { } } -impl Sub for AdditiveShare { +impl Sub for AdditiveShare { type Output = Self; fn sub(self, rhs: Self) -> Self::Output { @@ -150,7 +151,7 @@ impl Sub for AdditiveShare { } } -impl Sub<&Self> for AdditiveShare { +impl Sub<&Self> for AdditiveShare { type Output = Self; fn sub(self, rhs: &Self) -> Self::Output { @@ -158,7 +159,7 @@ impl Sub<&Self> for AdditiveShare { } } -impl Sub> for &AdditiveShare { +impl Sub> for &AdditiveShare { type Output = AdditiveShare; fn sub(self, rhs: AdditiveShare) -> Self::Output { @@ -166,14 +167,14 @@ impl Sub> for &AdditiveShare { } } -impl SubAssign<&Self> for AdditiveShare { +impl SubAssign<&Self> for AdditiveShare { fn sub_assign(&mut self, rhs: &Self) { self.0 -= rhs.0; self.1 -= rhs.1; } } -impl SubAssign for AdditiveShare { +impl SubAssign for AdditiveShare { fn sub_assign(&mut self, rhs: Self) { SubAssign::sub_assign(self, &rhs); } diff --git a/src/secret_sharing/scheme.rs b/src/secret_sharing/scheme.rs index cd6556211..4d843f4b8 100644 --- a/src/secret_sharing/scheme.rs +++ b/src/secret_sharing/scheme.rs @@ -3,11 +3,11 @@ use std::{ ops::{Mul, Neg}, }; -use super::SharedValue; +use super::{SharedValue,WeakSharedValue}; use crate::ff::{AddSub, AddSubAssign, GaloisField}; /// Secret sharing scheme i.e. Replicated secret sharing -pub trait SecretSharing: Clone + Debug + Sized + Send + Sync { +pub trait SecretSharing: Clone + Debug + Sized + Send + Sync { const ZERO: Self; } From b89d15a50724357955b0c9695543391ed789c05b Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Mon, 16 Oct 2023 22:38:49 -0700 Subject: [PATCH 107/124] Fix compiler recursion error --- src/ff/curve_points.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index cf182fd69..3fc51f896 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -13,6 +13,7 @@ use crate::{ ff::{ec_prime_field::Fp25519, Serializable}, secret_sharing::{Block, WeakSharedValue}, }; +use crate::secret_sharing::Additive; impl Block for CompressedRistretto { type Size = U32; @@ -20,7 +21,7 @@ impl Block for CompressedRistretto { ///ristretto point for curve 25519 #[derive(Clone, Copy, PartialEq, Debug)] -pub struct RP25519(::Storage); +pub struct RP25519(CompressedRistretto); /// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 impl WeakSharedValue for RP25519 { From b293fb585c595c235d29b2d1264ffd7d55973e6c Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:04:59 -0700 Subject: [PATCH 108/124] define WeakSharedValue, adding ipa-prf feature --- Cargo.toml | 4 ++- src/ff/curve_points.rs | 48 +++++++++++++++++------------------ src/helpers/mod.rs | 4 +-- src/protocol/basics/reveal.rs | 4 +-- src/protocol/mod.rs | 1 + src/protocol/prf_eval/mod.rs | 2 +- 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c3161a216..ce7d91446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,8 @@ default = [ "tracing/max_level_trace", "tracing/release_max_level_info", "descriptive-gate", - "aggregate-circuit" + "aggregate-circuit", + "ipa-prf" ] cli = ["comfy-table", "clap"] enable-serde = ["serde", "serde_json"] @@ -41,6 +42,7 @@ compact-gate = ["ipa-macros/compact-gate"] # Standalone aggregation protocol. We use IPA infra for communication # but it has nothing to do with IPA. aggregate-circuit = [] +ipa-prf = ["descriptive-gate"] [dependencies] aes = "0.8.3" diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 3fc51f896..1094079d4 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -9,11 +9,10 @@ use sha2::Sha256; use typenum::U32; use crate::{ - error::Error, ff::{ec_prime_field::Fp25519, Serializable}, secret_sharing::{Block, WeakSharedValue}, }; -use crate::secret_sharing::Additive; + impl Block for CompressedRistretto { type Size = U32; @@ -23,7 +22,7 @@ impl Block for CompressedRistretto { #[derive(Clone, Copy, PartialEq, Debug)] pub struct RP25519(CompressedRistretto); -/// using compressed ristretto point, Zero is generator of the curve, i.e. g^0 +/// using compressed ristretto point impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; @@ -42,6 +41,10 @@ impl Serializable for RP25519 { } } + +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve impl std::ops::Add for RP25519 { type Output = Self; @@ -57,6 +60,9 @@ impl std::ops::AddAssign for RP25519 { } } +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve impl std::ops::Neg for RP25519 { type Output = Self; @@ -65,6 +71,9 @@ impl std::ops::Neg for RP25519 { } } +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve impl std::ops::Sub for RP25519 { type Output = Self; @@ -82,34 +91,23 @@ impl std::ops::SubAssign for RP25519 { ///Scalar Multiplication ///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a -impl RP25519 { - /// # Errors - /// Propagates errors from decompressing invalid curve point - pub fn s_mul(self, rhs: Fp25519) -> Result { - self.0 - .decompress() - .map_or(Err(Error::DecompressingInvalidCurvePoint), |x| { - Ok((x * Scalar::from(rhs)).compress().into()) - }) - } -} - -///do not use -impl std::ops::Mul for RP25519 { +///## Panics +/// Panics when decompressing invalid curve point. This can happen when deserialize curve point +/// from bit array that does not have a valid representation on the curve +impl std::ops::Mul< Fp25519> for RP25519 { type Output = Self; - fn mul(self, _rhs: RP25519) -> Self::Output { - panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); + fn mul(self, rhs: Fp25519) -> RP25519 { + (self.0.decompress().unwrap() * Scalar::from(rhs)).compress().into() } } -///do not use -impl std::ops::MulAssign for RP25519 { - fn mul_assign(&mut self, _rhs: RP25519) { - panic!("Two curve points cannot be multiplied! Do not use *, *= for RP25519 or secret shares of RP25519"); - } +impl std::ops::MulAssign< Fp25519> for RP25519 { + #[allow(clippy::assign_op_pattern)] + fn mul_assign(&mut self, rhs: Fp25519) {*self = *self * rhs} } + impl From for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) @@ -205,7 +203,7 @@ mod test { let fp_e = rng.gen::(); let fp_f = rng.gen::(); let fp_g = fp_e * fp_f; - let fp_h = RP25519::from(fp_e).s_mul(fp_f).unwrap(); + let fp_h = RP25519::from(fp_e) * fp_f; assert_eq!(fp_h, RP25519::from(fp_g)); assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 359bf305f..1c510485e 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -40,7 +40,7 @@ use crate::{ Role::{H1, H2, H3}, }, protocol::{step::Gate, RecordId}, - secret_sharing::SharedValue, + secret_sharing::WeakSharedValue, }; // TODO work with ArrayLength only @@ -409,7 +409,7 @@ impl Debug for ChannelId { pub trait Message: Debug + Send + Serializable + 'static + Sized {} /// Any shared value can be send as a message -impl Message for V {} +impl Message for V {} impl Serializable for PublicKey { type Size = typenum::U32; diff --git a/src/protocol/basics/reveal.rs b/src/protocol/basics/reveal.rs index 672c8a473..9fddb65b8 100644 --- a/src/protocol/basics/reveal.rs +++ b/src/protocol/basics/reveal.rs @@ -18,7 +18,7 @@ use crate::{ malicious::{AdditiveShare as MaliciousReplicated, ExtendableField}, semi_honest::AdditiveShare as Replicated, }, - SecretSharing, SharedValue, + SecretSharing, WeakSharedValue, }, }; @@ -47,7 +47,7 @@ pub trait Reveal: Sized { /// i.e. their own shares and received share. #[async_trait] #[embed_doc_image("reveal", "images/reveal.png")] -impl Reveal for Replicated { +impl Reveal for Replicated { type Output = V; async fn reveal<'fut>(&self, ctx: C, record_id: RecordId) -> Result diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index d9d1ea382..660fa8eaa 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -6,6 +6,7 @@ pub mod context; pub mod dp; pub mod ipa; pub mod modulus_conversion; +#[cfg(feature = "ipa-prf")] pub mod prf_eval; #[cfg(feature = "descriptive-gate")] pub mod prf_sharding; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/prf_eval/mod.rs index ae43e55a1..3a0a99a21 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/prf_eval/mod.rs @@ -94,7 +94,7 @@ where let z = y.reveal(ctx.narrow(&Step::Revealz), record_id).await?; //compute R^(1/z) to u64 - Ok(u64::from(gr.s_mul(z.invert())?)) + Ok(u64::from(gr * (z.invert()))) } #[cfg(all(test, unit_test))] From deeca7d8c72f6afad4d1d284387879dc1cae5eec Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:15:12 -0700 Subject: [PATCH 109/124] fix zero --- src/ff/curve_points.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 1094079d4..1d60f934a 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -1,5 +1,4 @@ use curve25519_dalek::{ - constants, ristretto::{CompressedRistretto, RistrettoPoint}, Scalar, }; @@ -26,7 +25,7 @@ pub struct RP25519(CompressedRistretto); impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; - const ZERO: Self = Self(constants::RISTRETTO_BASEPOINT_COMPRESSED); + const ZERO: Self = Self(CompressedRistretto([0_u8;32])); } impl Serializable for RP25519 { @@ -169,7 +168,10 @@ mod test { use rand::{thread_rng, Rng}; use typenum::U32; - use crate::ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable}; + use crate::{ + ff::{curve_points::RP25519, ec_prime_field::Fp25519, Serializable}, + secret_sharing::WeakSharedValue, + }; #[test] fn serde_25519() { @@ -206,6 +208,7 @@ mod test { let fp_h = RP25519::from(fp_e) * fp_f; assert_eq!(fp_h, RP25519::from(fp_g)); assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); + assert_eq!(RP25519::ZERO,fp_h*Scalar::ZERO.into()); } #[test] From 729c87f70bd809868b93da2cbc20eae881a5c0ee Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:16:16 -0700 Subject: [PATCH 110/124] fmt --- src/ff/curve_points.rs | 21 +++++++++-------- src/secret_sharing/mod.rs | 23 ++++++------------- .../replicated/semi_honest/additive_share.rs | 3 +-- src/secret_sharing/scheme.rs | 2 +- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 1d60f934a..99aac7f18 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -12,7 +12,6 @@ use crate::{ secret_sharing::{Block, WeakSharedValue}, }; - impl Block for CompressedRistretto { type Size = U32; } @@ -25,7 +24,7 @@ pub struct RP25519(CompressedRistretto); impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; - const ZERO: Self = Self(CompressedRistretto([0_u8;32])); + const ZERO: Self = Self(CompressedRistretto([0_u8; 32])); } impl Serializable for RP25519 { @@ -40,7 +39,6 @@ impl Serializable for RP25519 { } } - ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve @@ -93,20 +91,23 @@ impl std::ops::SubAssign for RP25519 { ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve -impl std::ops::Mul< Fp25519> for RP25519 { +impl std::ops::Mul for RP25519 { type Output = Self; - fn mul(self, rhs: Fp25519) -> RP25519 { - (self.0.decompress().unwrap() * Scalar::from(rhs)).compress().into() + fn mul(self, rhs: Fp25519) -> RP25519 { + (self.0.decompress().unwrap() * Scalar::from(rhs)) + .compress() + .into() } } -impl std::ops::MulAssign< Fp25519> for RP25519 { +impl std::ops::MulAssign for RP25519 { #[allow(clippy::assign_op_pattern)] - fn mul_assign(&mut self, rhs: Fp25519) {*self = *self * rhs} + fn mul_assign(&mut self, rhs: Fp25519) { + *self = *self * rhs + } } - impl From for RP25519 { fn from(s: Scalar) -> Self { RP25519(RistrettoPoint::mul_base(&s).compress()) @@ -208,7 +209,7 @@ mod test { let fp_h = RP25519::from(fp_e) * fp_f; assert_eq!(fp_h, RP25519::from(fp_g)); assert_ne!(fp_h, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); - assert_eq!(RP25519::ZERO,fp_h*Scalar::ZERO.into()); + assert_eq!(RP25519::ZERO, fp_h * Scalar::ZERO.into()); } #[test] diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 110bf74f5..8564bf4be 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -25,31 +25,23 @@ use crate::ff::{AddSub, AddSubAssign, Serializable}; /// Operations supported for weak shared values. pub trait Additive: -AddSub -+ AddSubAssign -+ Neg + AddSub + AddSubAssign + Neg { } impl Additive for T where - T: AddSub - + AddSubAssign - + Neg + T: AddSub + AddSubAssign + Neg { } /// Operations supported for shared values. pub trait Arithmetic: - Additive - + Mul - + MulAssign + Additive + Mul + MulAssign { } impl Arithmetic for T where - T: Additive - + Mul - + MulAssign + T: Additive + Mul + MulAssign { } @@ -60,7 +52,7 @@ pub trait Block: Sized + Copy + Debug { } pub trait WeakSharedValue: -Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static + Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static { type Storage: Block; @@ -79,7 +71,8 @@ pub trait SharedValue: const ZERO: Self; } -impl WeakSharedValue for T where +impl WeakSharedValue for T +where T: SharedValue, { type Storage = T::Storage; @@ -89,8 +82,6 @@ impl WeakSharedValue for T where const ZERO: Self = T::ZERO; } - - #[cfg(any(test, feature = "test-fixture", feature = "cli"))] impl IntoShares> for V where diff --git a/src/secret_sharing/replicated/semi_honest/additive_share.rs b/src/secret_sharing/replicated/semi_honest/additive_share.rs index fb7438c2f..0ae455ea9 100644 --- a/src/secret_sharing/replicated/semi_honest/additive_share.rs +++ b/src/secret_sharing/replicated/semi_honest/additive_share.rs @@ -10,8 +10,7 @@ use crate::{ ff::Serializable, secret_sharing::{ replicated::ReplicatedSecretSharing, Linear as LinearSecretSharing, SecretSharing, - SharedValue, - WeakSharedValue, + SharedValue, WeakSharedValue, }, }; diff --git a/src/secret_sharing/scheme.rs b/src/secret_sharing/scheme.rs index 4d843f4b8..0d2131eeb 100644 --- a/src/secret_sharing/scheme.rs +++ b/src/secret_sharing/scheme.rs @@ -3,7 +3,7 @@ use std::{ ops::{Mul, Neg}, }; -use super::{SharedValue,WeakSharedValue}; +use super::{SharedValue, WeakSharedValue}; use crate::ff::{AddSub, AddSubAssign, GaloisField}; /// Secret sharing scheme i.e. Replicated secret sharing From c9dc93193636ae83a6707d645cf3072cfb4e2988 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Tue, 17 Oct 2023 15:19:38 -0700 Subject: [PATCH 111/124] fix clippy --- src/ff/curve_points.rs | 2 +- src/ff/ec_prime_field.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 99aac7f18..a084b14a0 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -104,7 +104,7 @@ impl std::ops::Mul for RP25519 { impl std::ops::MulAssign for RP25519 { #[allow(clippy::assign_op_pattern)] fn mul_assign(&mut self, rhs: Fp25519) { - *self = *self * rhs + *self = *self * rhs; } } diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index d0ff48b72..15d5f33a7 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -119,6 +119,7 @@ impl From for Fp25519 { } } +#[cfg(test)] macro_rules! sc_hash_impl { ( $u_type:ty) => { impl From for $u_type { From 3aaeba40695e3c36703226d75f80afb72b807bdb Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 18 Oct 2023 11:03:53 -0700 Subject: [PATCH 112/124] refactor prf_ipa in separate module --- src/protocol/ipa_prf/mod.rs | 2 ++ src/protocol/{ => ipa_prf}/prf_eval/mod.rs | 6 +++--- src/protocol/{ => ipa_prf}/prf_sharding/bucket.rs | 2 +- .../{ => ipa_prf}/prf_sharding/feature_label_dot_product.rs | 2 +- src/protocol/{ => ipa_prf}/prf_sharding/mod.rs | 2 +- src/protocol/mod.rs | 4 +--- 6 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 src/protocol/ipa_prf/mod.rs rename src/protocol/{ => ipa_prf}/prf_eval/mod.rs (95%) rename src/protocol/{ => ipa_prf}/prf_sharding/bucket.rs (99%) rename src/protocol/{ => ipa_prf}/prf_sharding/feature_label_dot_product.rs (99%) rename src/protocol/{ => ipa_prf}/prf_sharding/mod.rs (99%) diff --git a/src/protocol/ipa_prf/mod.rs b/src/protocol/ipa_prf/mod.rs new file mode 100644 index 000000000..755ef87a5 --- /dev/null +++ b/src/protocol/ipa_prf/mod.rs @@ -0,0 +1,2 @@ +pub mod prf_eval; +pub mod prf_sharding; diff --git a/src/protocol/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs similarity index 95% rename from src/protocol/prf_eval/mod.rs rename to src/protocol/ipa_prf/prf_eval/mod.rs index 3a0a99a21..6d95e40cd 100644 --- a/src/protocol/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -9,7 +9,7 @@ use crate::{ prss::SharedRandomness, RecordId, }, - secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, + secret_sharing::replicated::{ReplicatedSecretSharing, semi_honest::AdditiveShare}, seq_join::seq_try_join_all, }; @@ -103,11 +103,11 @@ mod test { use crate::{ ff::{curve_points::RP25519, ec_prime_field::Fp25519}, - protocol::prf_eval::compute_match_key_pseudonym, - secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, + secret_sharing::{IntoShares, replicated::semi_honest::AdditiveShare}, test_executor::run, test_fixture::{Reconstruct, Runner, TestWorld}, }; + use crate::protocol::ipa_prf::prf_eval::compute_match_key_pseudonym; #[derive(Copy, Clone)] struct ShuffledTestInput { diff --git a/src/protocol/prf_sharding/bucket.rs b/src/protocol/ipa_prf/prf_sharding/bucket.rs similarity index 99% rename from src/protocol/prf_sharding/bucket.rs rename to src/protocol/ipa_prf/prf_sharding/bucket.rs index d2ad77a11..450ffafa0 100644 --- a/src/protocol/prf_sharding/bucket.rs +++ b/src/protocol/ipa_prf/prf_sharding/bucket.rs @@ -124,7 +124,7 @@ pub mod tests { ff::{Field, Fp32BitPrime, Gf8Bit, Gf9Bit}, protocol::{ context::{Context, UpgradableContext, Validator}, - prf_sharding::bucket::move_single_value_to_bucket, + ipa_prf::prf_sharding::bucket::move_single_value_to_bucket, RecordId, }, rand::Rng, diff --git a/src/protocol/prf_sharding/feature_label_dot_product.rs b/src/protocol/ipa_prf/prf_sharding/feature_label_dot_product.rs similarity index 99% rename from src/protocol/prf_sharding/feature_label_dot_product.rs rename to src/protocol/ipa_prf/prf_sharding/feature_label_dot_product.rs index f87f788d8..bc492b62c 100644 --- a/src/protocol/prf_sharding/feature_label_dot_product.rs +++ b/src/protocol/ipa_prf/prf_sharding/feature_label_dot_product.rs @@ -371,7 +371,7 @@ where pub mod tests { use crate::{ ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf32Bit}, - protocol::prf_sharding::feature_label_dot_product::{ + protocol::ipa_prf::prf_sharding::feature_label_dot_product::{ compute_feature_label_dot_product, PrfShardedIpaInputRow, }, rand::Rng, diff --git a/src/protocol/prf_sharding/mod.rs b/src/protocol/ipa_prf/prf_sharding/mod.rs similarity index 99% rename from src/protocol/prf_sharding/mod.rs rename to src/protocol/ipa_prf/prf_sharding/mod.rs index fcca9ab97..06c75fdc4 100644 --- a/src/protocol/prf_sharding/mod.rs +++ b/src/protocol/ipa_prf/prf_sharding/mod.rs @@ -843,7 +843,7 @@ pub mod tests { ff::{Field, Fp32BitPrime, GaloisField, Gf2, Gf20Bit, Gf3Bit, Gf5Bit}, protocol::{ context::{UpgradableContext, Validator}, - prf_sharding::{attribution_and_capping_and_aggregation, do_aggregation}, + ipa_prf::prf_sharding::{attribution_and_capping_and_aggregation, do_aggregation}, }, rand::Rng, secret_sharing::{ diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 660fa8eaa..fec8d66a4 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -7,9 +7,7 @@ pub mod dp; pub mod ipa; pub mod modulus_conversion; #[cfg(feature = "ipa-prf")] -pub mod prf_eval; -#[cfg(feature = "descriptive-gate")] -pub mod prf_sharding; +pub mod ipa_prf; pub mod prss; pub mod sort; pub mod step; From 04fe4d3e686900235dbe75b4f1e1218de54c3e0f Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 18 Oct 2023 11:05:41 -0700 Subject: [PATCH 113/124] fmt --- src/protocol/ipa_prf/prf_eval/mod.rs | 6 +++--- src/protocol/mod.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index 6d95e40cd..4da81759c 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -9,7 +9,7 @@ use crate::{ prss::SharedRandomness, RecordId, }, - secret_sharing::replicated::{ReplicatedSecretSharing, semi_honest::AdditiveShare}, + secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, seq_join::seq_try_join_all, }; @@ -103,11 +103,11 @@ mod test { use crate::{ ff::{curve_points::RP25519, ec_prime_field::Fp25519}, - secret_sharing::{IntoShares, replicated::semi_honest::AdditiveShare}, + protocol::ipa_prf::prf_eval::compute_match_key_pseudonym, + secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_executor::run, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use crate::protocol::ipa_prf::prf_eval::compute_match_key_pseudonym; #[derive(Copy, Clone)] struct ShuffledTestInput { diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index fec8d66a4..3dac77c62 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -5,9 +5,9 @@ pub mod boolean; pub mod context; pub mod dp; pub mod ipa; -pub mod modulus_conversion; #[cfg(feature = "ipa-prf")] pub mod ipa_prf; +pub mod modulus_conversion; pub mod prss; pub mod sort; pub mod step; From 7db9376853588232a3fc6693d6e866a995e1ffb5 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 15:00:05 -0700 Subject: [PATCH 114/124] adressing Alex's comments --- src/error.rs | 4 ++-- src/ff/curve_points.rs | 10 +++++----- src/ff/ec_prime_field.rs | 10 +++++++--- src/protocol/ipa_prf/prf_eval/mod.rs | 5 ++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/error.rs b/src/error.rs index 7d6456c7f..de41eda4b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -58,8 +58,8 @@ pub enum Error { InvalidReport(#[from] InvalidReportError), #[error("unsupported: {0}")] Unsupported(String), - #[error("Decompressing invalid elliptic curve point")] - DecompressingInvalidCurvePoint, + #[error("Decompressing invalid elliptic curve point: {0}")] + DecompressingInvalidCurvePoint(String), } impl Default for Error { diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index a084b14a0..f4cb50bba 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -3,8 +3,6 @@ use curve25519_dalek::{ Scalar, }; use generic_array::GenericArray; -use hkdf::Hkdf; -use sha2::Sha256; use typenum::U32; use crate::{ @@ -35,6 +33,7 @@ impl Serializable for RP25519 { } fn deserialize(buf: &GenericArray) -> Self { + debug_assert!(CompressedRistretto((*buf).into()).decompress().is_some()); RP25519(CompressedRistretto((*buf).into())) } } @@ -87,7 +86,6 @@ impl std::ops::SubAssign for RP25519 { } ///Scalar Multiplication -///<'a, 'b> `std::ops::Mul<&'b"` Fp25519 for &'a ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve @@ -136,6 +134,8 @@ macro_rules! cp_hash_impl { ( $u_type:ty) => { impl From for $u_type { fn from(s: RP25519) -> Self { + use hkdf::Hkdf; + use sha2::Sha256; let hk = Hkdf::::new(None, s.0.as_bytes()); let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large @@ -150,8 +150,6 @@ cp_hash_impl!(u128); cp_hash_impl!(u64); -#[cfg(test)] -cp_hash_impl!(u32); #[cfg(test)] impl rand::distributions::Distribution for rand::distributions::Standard { @@ -174,6 +172,8 @@ mod test { secret_sharing::WeakSharedValue, }; + cp_hash_impl!(u32); + #[test] fn serde_25519() { let mut rng = thread_rng(); diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 15d5f33a7..02f0742df 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -124,6 +124,8 @@ macro_rules! sc_hash_impl { ( $u_type:ty) => { impl From for $u_type { fn from(s: Fp25519) -> Self { + use hkdf::Hkdf; + use sha2::Sha256; let hk = Hkdf::::new(None, s.0.as_bytes()); let mut okm = <$u_type>::MIN.to_le_bytes(); //error invalid length from expand only happens when okm is very large @@ -134,6 +136,8 @@ macro_rules! sc_hash_impl { impl From<$u_type> for Fp25519 { fn from(s: $u_type) -> Self { + use hkdf::Hkdf; + use sha2::Sha256; let hk = Hkdf::::new(None, &s.to_le_bytes()); let mut okm = [0u8; 32]; //error invalid length from expand only happens when okm is very large @@ -147,10 +151,7 @@ macro_rules! sc_hash_impl { #[cfg(test)] sc_hash_impl!(u64); -#[cfg(test)] -sc_hash_impl!(u32); -/// Daniel had to implement this since PRSS wants it, prefer not to impl Field for Fp25519 { const ONE: Fp25519 = Fp25519::ONE; @@ -198,6 +199,9 @@ mod test { secret_sharing::SharedValue, }; + + sc_hash_impl!(u32); + #[test] fn serde_25519() { let mut rng = thread_rng(); diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index 4da81759c..fe9b2540e 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -10,7 +10,6 @@ use crate::{ RecordId, }, secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, - seq_join::seq_try_join_all, }; #[derive(Step)] @@ -37,7 +36,7 @@ where .iter() .enumerate() .map(|(i, x)| eval_dy_prf(ctx.clone(), i.into(), &prf_key, x)); - seq_try_join_all(sh_ctx.active_work(), futures).await + ctx.try_join(futures).await } impl From> for AdditiveShare { @@ -53,7 +52,7 @@ where { ctx.narrow(&Step::PRFKeyGen) .prss() - .generate_replicated(u128::MAX - 100u128) + .generate_replicated(RecordId(0)) } /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) From 7c211b7d0dec9b3e10bc64a6a25958a7953db0a3 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 15:01:26 -0700 Subject: [PATCH 115/124] fmt --- src/ff/curve_points.rs | 1 - src/ff/ec_prime_field.rs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index f4cb50bba..2b06f973e 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -150,7 +150,6 @@ cp_hash_impl!(u128); cp_hash_impl!(u64); - #[cfg(test)] impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> RP25519 { diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 02f0742df..ba753eb40 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -151,7 +151,6 @@ macro_rules! sc_hash_impl { #[cfg(test)] sc_hash_impl!(u64); - impl Field for Fp25519 { const ONE: Fp25519 = Fp25519::ONE; @@ -199,7 +198,6 @@ mod test { secret_sharing::SharedValue, }; - sc_hash_impl!(u32); #[test] From 4d5b1050543be557c606229cf3e7848f6ba40c94 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 16:59:31 -0700 Subject: [PATCH 116/124] add comments --- src/ff/curve_points.rs | 22 ++++++++++++++++++++-- src/ff/ec_prime_field.rs | 14 +++++++++++++- src/protocol/ipa_prf/prf_eval/mod.rs | 12 ++++++++++-- src/secret_sharing/mod.rs | 4 ++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index 2b06f973e..ec42eba76 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -14,11 +14,21 @@ impl Block for CompressedRistretto { type Size = U32; } -///ristretto point for curve 25519 +///ristretto point for curve 25519, +/// we store it in compressed format since it is 3 times smaller and we do a limited amount of +/// arithmetic operations on the curve points +/// +/// We use ristretto points such that we have a prime order elliptic curve, +/// This is needed for the Dodis Yampolski PRF +/// +/// decompressing invalid curve points will cause panics, +/// since we always generate curve points from scalars (elements in Fp25519) and +/// only deserialize previously serialized valid points, panics will not occur +/// However, we still added a debug assert to deserialize since values are sent by other servers #[derive(Clone, Copy, PartialEq, Debug)] pub struct RP25519(CompressedRistretto); -/// using compressed ristretto point +/// Implementing trait for secret sharing impl WeakSharedValue for RP25519 { type Storage = CompressedRistretto; const BITS: u32 = 256; @@ -86,6 +96,7 @@ impl std::ops::SubAssign for RP25519 { } ///Scalar Multiplication +/// allows to multiply curve points with scalars from Fp25519 ///## Panics /// Panics when decompressing invalid curve point. This can happen when deserialize curve point /// from bit array that does not have a valid representation on the curve @@ -130,6 +141,7 @@ impl From for CompressedRistretto { } } +///allows to convert curve points into unsigned integers, preserving high entropy macro_rules! cp_hash_impl { ( $u_type:ty) => { impl From for $u_type { @@ -150,6 +162,8 @@ cp_hash_impl!(u128); cp_hash_impl!(u64); +/// implementing random curve point generation for testing purposes, +/// in the actual IPA protocol, we generate them from scalars, i.e. Fp25519 #[cfg(test)] impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> RP25519 { @@ -173,6 +187,7 @@ mod test { cp_hash_impl!(u32); + ///testing serialize and deserialize #[test] fn serde_25519() { let mut rng = thread_rng(); @@ -183,6 +198,7 @@ mod test { assert_eq!(input, output); } + ///testing conversion from scalar to Fp25519 and curve point, i.e. RP25519 #[test] fn scalar_to_point() { let a = Scalar::ONE; @@ -193,6 +209,7 @@ mod test { assert_eq!(c, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } + ///testing simple curve arithmetics to check that curve25519_dalek library is used correctly #[test] fn curve_arithmetics() { let mut rng = thread_rng(); @@ -211,6 +228,7 @@ mod test { assert_eq!(RP25519::ZERO, fp_h * Scalar::ZERO.into()); } + ///testing curve to unsigned integer conversion has entropy (!= 0) #[test] fn curve_point_to_hash() { let mut rng = thread_rng(); diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index ba753eb40..9bd31ddbb 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -13,12 +13,15 @@ impl Block for Scalar { type Size = U32; } +///implements the Scalar field for elliptic curve 25519 +/// we use elements in Fp25519 to generate curve points and operate on the curve #[derive(Clone, Copy, PartialEq, Debug)] pub struct Fp25519(::Storage); impl Fp25519 { pub const ONE: Self = Self(Scalar::ONE); + ///allow invert for scalars, i.e. computes 1/a mod p ///# Panics /// Panics when self is zero #[must_use] @@ -28,12 +31,14 @@ impl Fp25519 { } } +///trait for secret sharing impl SharedValue for Fp25519 { type Storage = Scalar; const BITS: u32 = 256; const ZERO: Self = Self(Scalar::ZERO); } +///conversion to Scalar struct of curve25519_dalek impl From for Scalar { fn from(s: Fp25519) -> Self { s.0 @@ -52,6 +57,7 @@ impl Serializable for Fp25519 { } } +///generate random elements in Fp25519 impl rand::distributions::Distribution for rand::distributions::Standard { fn sample(&self, rng: &mut R) -> Fp25519 { let mut scalar_bytes = [0u8; 32]; @@ -119,6 +125,7 @@ impl From for Fp25519 { } } +///conversion from and to unsigned integers, preserving entropy, for testing purposes only #[cfg(test)] macro_rules! sc_hash_impl { ( $u_type:ty) => { @@ -151,6 +158,7 @@ macro_rules! sc_hash_impl { #[cfg(test)] sc_hash_impl!(u64); +///implement Field because required by PRSS impl Field for Fp25519 { const ONE: Fp25519 = Fp25519::ONE; @@ -174,6 +182,7 @@ impl Field for Fp25519 { } } +///implement TryFrom since required by Field impl TryFrom for Fp25519 { type Error = crate::error::Error; @@ -200,6 +209,7 @@ mod test { sc_hash_impl!(u32); + ///test serialize and deserialize #[test] fn serde_25519() { let mut rng = thread_rng(); @@ -210,7 +220,7 @@ mod test { assert_eq!(input, output); } - // These are just simple arithmetic tests since arithmetics are checked by curve25519_dalek + ///test simple arithmetics to check that curve25519_dalek is used correctly #[test] fn simple_arithmetics_25519() { let a = Fp25519(Scalar::from_bytes_mod_order([ @@ -241,12 +251,14 @@ mod test { assert_eq!(ec, e); } + ///test random field element generation (!= 0) #[test] fn simple_random_25519() { let mut rng = thread_rng(); assert_ne!(Fp25519::ZERO, rng.gen::()); } + ///test inversion for field elements #[test] fn invert_25519() { let mut rng = thread_rng(); diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index fe9b2540e..7a65c0191 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -21,6 +21,10 @@ pub(crate) enum Step { Revealz, } +/// generates match key pseudonyms from match keys (in Fp25519 format) and PRF key +/// PRF key needs to be generated separately using gen_prf_key +/// +/// gen_prf_key is not included such that compute_match_key_pseudonym can be tested for correctness /// # Errors /// Propagates errors from multiplications pub async fn compute_match_key_pseudonym( @@ -57,8 +61,8 @@ where /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 -/// PRF key k is generated using keygen -/// In 3IPA, x is the match key +/// PRF key k needs to be generated using gen_prf_key +/// x is the match key in Fp25519 format /// outputs a u64 as specified in `protocol/prf_sharding/mod.rs`, all parties learn the output /// # Errors /// Propagates errors from multiplications, reveal and scalar multiplication @@ -108,11 +112,13 @@ mod test { test_fixture::{Reconstruct, Runner, TestWorld}, }; + ///defining test input struct #[derive(Copy, Clone)] struct ShuffledTestInput { match_key: Fp25519, } + ///defining test output struct #[derive(Debug, PartialEq)] struct TestOutput { match_key_pseudonym: u64, @@ -142,6 +148,8 @@ mod test { } } + ///testing correctness of DY PRF evaluation + /// by checking MPC generated pseudonym with pseudonym generated in the clear #[test] fn semi_honest() { run(|| async move { diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 8564bf4be..1a7dda8c2 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -51,6 +51,8 @@ pub trait Block: Sized + Copy + Debug { type Size: ArrayLength; } + +///allows basic secret sharing operations pub trait WeakSharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static { @@ -61,6 +63,7 @@ pub trait WeakSharedValue: const ZERO: Self; } +///allows advanced secret sharing operations, requires multiplication pub trait SharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Arithmetic + Serializable + 'static { @@ -71,6 +74,7 @@ pub trait SharedValue: const ZERO: Self; } +///any SharedValue is also a WeakSharedValue impl WeakSharedValue for T where T: SharedValue, From 39c7ced4902fb59eac7d3c9d4bdc29a55c33b8f6 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 17:02:24 -0700 Subject: [PATCH 117/124] fix lint --- src/ff/curve_points.rs | 2 +- src/ff/ec_prime_field.rs | 6 +++--- src/protocol/ipa_prf/prf_eval/mod.rs | 6 +++--- src/secret_sharing/mod.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ff/curve_points.rs b/src/ff/curve_points.rs index ec42eba76..8eaa57623 100644 --- a/src/ff/curve_points.rs +++ b/src/ff/curve_points.rs @@ -209,7 +209,7 @@ mod test { assert_eq!(c, RP25519(constants::RISTRETTO_BASEPOINT_COMPRESSED)); } - ///testing simple curve arithmetics to check that curve25519_dalek library is used correctly + ///testing simple curve arithmetics to check that `curve25519_dalek` library is used correctly #[test] fn curve_arithmetics() { let mut rng = thread_rng(); diff --git a/src/ff/ec_prime_field.rs b/src/ff/ec_prime_field.rs index 9bd31ddbb..3385f10e8 100644 --- a/src/ff/ec_prime_field.rs +++ b/src/ff/ec_prime_field.rs @@ -38,7 +38,7 @@ impl SharedValue for Fp25519 { const ZERO: Self = Self(Scalar::ZERO); } -///conversion to Scalar struct of curve25519_dalek +///conversion to Scalar struct of `curve25519_dalek` impl From for Scalar { fn from(s: Fp25519) -> Self { s.0 @@ -182,7 +182,7 @@ impl Field for Fp25519 { } } -///implement TryFrom since required by Field +///implement `TryFrom` since required by Field impl TryFrom for Fp25519 { type Error = crate::error::Error; @@ -220,7 +220,7 @@ mod test { assert_eq!(input, output); } - ///test simple arithmetics to check that curve25519_dalek is used correctly + ///test simple arithmetics to check that `curve25519_dalek` is used correctly #[test] fn simple_arithmetics_25519() { let a = Fp25519(Scalar::from_bytes_mod_order([ diff --git a/src/protocol/ipa_prf/prf_eval/mod.rs b/src/protocol/ipa_prf/prf_eval/mod.rs index 7a65c0191..ff37e30eb 100644 --- a/src/protocol/ipa_prf/prf_eval/mod.rs +++ b/src/protocol/ipa_prf/prf_eval/mod.rs @@ -22,9 +22,9 @@ pub(crate) enum Step { } /// generates match key pseudonyms from match keys (in Fp25519 format) and PRF key -/// PRF key needs to be generated separately using gen_prf_key +/// PRF key needs to be generated separately using `gen_prf_key` /// -/// gen_prf_key is not included such that compute_match_key_pseudonym can be tested for correctness +/// `gen_prf_key` is not included such that `compute_match_key_pseudonym` can be tested for correctness /// # Errors /// Propagates errors from multiplications pub async fn compute_match_key_pseudonym( @@ -61,7 +61,7 @@ where /// evaluates the Dodis-Yampolski PRF g^(1/(k+x)) /// the input x and k are secret shared over finite field Fp25519, i.e. the scalar field of curve 25519 -/// PRF key k needs to be generated using gen_prf_key +/// PRF key k needs to be generated using `gen_prf_key` /// x is the match key in Fp25519 format /// outputs a u64 as specified in `protocol/prf_sharding/mod.rs`, all parties learn the output /// # Errors diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 1a7dda8c2..155b1f609 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -74,7 +74,7 @@ pub trait SharedValue: const ZERO: Self; } -///any SharedValue is also a WeakSharedValue +///any `SharedValue` is also a `WeakSharedValue` impl WeakSharedValue for T where T: SharedValue, From e5db3971ee8e67a403519f1a853279b5c057bb95 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Fri, 20 Oct 2023 17:03:06 -0700 Subject: [PATCH 118/124] fmt --- src/secret_sharing/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index 155b1f609..441fe1f21 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -51,7 +51,6 @@ pub trait Block: Sized + Copy + Debug { type Size: ArrayLength; } - ///allows basic secret sharing operations pub trait WeakSharedValue: Clone + Copy + PartialEq + Debug + Send + Sync + Sized + Additive + Serializable + 'static From 0efd323824820afa3d886a737b5dd60334af2cf9 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 1 Nov 2023 17:21:47 -0700 Subject: [PATCH 119/124] fix errors after merge --- src/protocol/ipa_prf/prf_sharding/bucket.rs | 5 ++--- src/protocol/ipa_prf/prf_sharding/mod.rs | 3 +-- src/query/runner/oprf_ipa.rs | 2 +- src/test_fixture/ipa.rs | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/protocol/ipa_prf/prf_sharding/bucket.rs b/src/protocol/ipa_prf/prf_sharding/bucket.rs index c45c375dc..32cfd8b56 100644 --- a/src/protocol/ipa_prf/prf_sharding/bucket.rs +++ b/src/protocol/ipa_prf/prf_sharding/bucket.rs @@ -1,12 +1,11 @@ use embed_doc_image::embed_doc_image; +use ipa_macros::Step; use crate::{ error::Error, ff::{GaloisField, PrimeField, Serializable}, protocol::{ - basics::SecureMul, context::UpgradedContext, ipa_prf::prf_sharding::BinaryTreeDepthStep, - step::BitOpStep, RecordId, - basics::SecureMul, context::UpgradedContext, prf_sharding::BinaryTreeDepthStep, RecordId, + basics::SecureMul, context::UpgradedContext, ipa_prf::prf_sharding::BinaryTreeDepthStep, RecordId, }, secret_sharing::{ replicated::malicious::ExtendableField, BitDecomposed, Linear as LinearSecretSharing, diff --git a/src/protocol/ipa_prf/prf_sharding/mod.rs b/src/protocol/ipa_prf/prf_sharding/mod.rs index 06c75fdc4..0f6aa8ed2 100644 --- a/src/protocol/ipa_prf/prf_sharding/mod.rs +++ b/src/protocol/ipa_prf/prf_sharding/mod.rs @@ -10,13 +10,12 @@ use futures_util::{ }; use ipa_macros::Step; -use super::boolean::saturating_sum::SaturatingSum; use crate::{ error::Error, ff::{Field, GaloisField, Gf2, PrimeField, Serializable}, protocol::{ basics::{if_else, SecureMul, ShareKnownValue}, - boolean::{comparison::bitwise_less_than_constant, or::or}, + boolean::{comparison::bitwise_less_than_constant, or::or, saturating_sum::SaturatingSum}, context::{UpgradableContext, UpgradedContext, Validator}, modulus_conversion::convert_bits, step::BitOpStep, diff --git a/src/query/runner/oprf_ipa.rs b/src/query/runner/oprf_ipa.rs index e77a0300e..d2951f0f1 100644 --- a/src/query/runner/oprf_ipa.rs +++ b/src/query/runner/oprf_ipa.rs @@ -12,7 +12,7 @@ use crate::{ protocol::{ basics::ShareKnownValue, context::{UpgradableContext, UpgradedContext}, - prf_sharding::{attribution_and_capping_and_aggregation, PrfShardedIpaInputRow}, + ipa_prf::prf_sharding::{attribution_and_capping_and_aggregation, PrfShardedIpaInputRow}, BreakdownKey, Timestamp, TriggerValue, }, report::{EventType, OprfReport}, diff --git a/src/test_fixture/ipa.rs b/src/test_fixture/ipa.rs index e6f899b79..530084029 100644 --- a/src/test_fixture/ipa.rs +++ b/src/test_fixture/ipa.rs @@ -243,7 +243,7 @@ pub async fn test_oprf_ipa( ff::{Field, Gf2}, protocol::{ basics::ShareKnownValue, - prf_sharding::{attribution_and_capping_and_aggregation, PrfShardedIpaInputRow}, + ipa_prf::prf_sharding::{attribution_and_capping_and_aggregation, PrfShardedIpaInputRow}, }, report::EventType, secret_sharing::SharedValue, From 60d8ab6c65d62a9200427108a2c02f97cd9a34ee Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 1 Nov 2023 17:22:14 -0700 Subject: [PATCH 120/124] fmt --- src/protocol/ipa_prf/prf_sharding/bucket.rs | 3 ++- src/test_fixture/ipa.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/protocol/ipa_prf/prf_sharding/bucket.rs b/src/protocol/ipa_prf/prf_sharding/bucket.rs index 32cfd8b56..a1e62d1a2 100644 --- a/src/protocol/ipa_prf/prf_sharding/bucket.rs +++ b/src/protocol/ipa_prf/prf_sharding/bucket.rs @@ -5,7 +5,8 @@ use crate::{ error::Error, ff::{GaloisField, PrimeField, Serializable}, protocol::{ - basics::SecureMul, context::UpgradedContext, ipa_prf::prf_sharding::BinaryTreeDepthStep, RecordId, + basics::SecureMul, context::UpgradedContext, ipa_prf::prf_sharding::BinaryTreeDepthStep, + RecordId, }, secret_sharing::{ replicated::malicious::ExtendableField, BitDecomposed, Linear as LinearSecretSharing, diff --git a/src/test_fixture/ipa.rs b/src/test_fixture/ipa.rs index 530084029..7668881c8 100644 --- a/src/test_fixture/ipa.rs +++ b/src/test_fixture/ipa.rs @@ -243,7 +243,9 @@ pub async fn test_oprf_ipa( ff::{Field, Gf2}, protocol::{ basics::ShareKnownValue, - ipa_prf::prf_sharding::{attribution_and_capping_and_aggregation, PrfShardedIpaInputRow}, + ipa_prf::prf_sharding::{ + attribution_and_capping_and_aggregation, PrfShardedIpaInputRow, + }, }, report::EventType, secret_sharing::SharedValue, From a17eaa9dbf72814fd0a9235616cb504daef634f3 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 1 Nov 2023 17:28:46 -0700 Subject: [PATCH 121/124] add flag --- src/query/runner/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/runner/mod.rs b/src/query/runner/mod.rs index d9eb28f8f..9875c7ced 100644 --- a/src/query/runner/mod.rs +++ b/src/query/runner/mod.rs @@ -1,5 +1,6 @@ mod aggregate; mod ipa; +#[cfg(feature = "ipa-prf")] mod oprf_ipa; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] From 83d74cb4656e94b1eefdd182b4b457c72a7059d1 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Wed, 1 Nov 2023 17:30:10 -0700 Subject: [PATCH 122/124] remove flag --- src/query/runner/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/runner/mod.rs b/src/query/runner/mod.rs index 9875c7ced..d9eb28f8f 100644 --- a/src/query/runner/mod.rs +++ b/src/query/runner/mod.rs @@ -1,6 +1,5 @@ mod aggregate; mod ipa; -#[cfg(feature = "ipa-prf")] mod oprf_ipa; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] From f8d26c958488e2dfc4ed0536e69839d02eddef90 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Thu, 2 Nov 2023 12:52:25 -0700 Subject: [PATCH 123/124] add flag --- src/query/runner/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/runner/mod.rs b/src/query/runner/mod.rs index d9eb28f8f..9875c7ced 100644 --- a/src/query/runner/mod.rs +++ b/src/query/runner/mod.rs @@ -1,5 +1,6 @@ mod aggregate; mod ipa; +#[cfg(feature = "ipa-prf")] mod oprf_ipa; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] From 6c715ffd0eccda6c28cc201a6a0031783dd30869 Mon Sep 17 00:00:00 2001 From: Daniel Masny Date: Thu, 2 Nov 2023 12:55:48 -0700 Subject: [PATCH 124/124] add flag descriptive gate --- src/protocol/mod.rs | 2 +- src/query/runner/mod.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 3dac77c62..cd46abdda 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -5,7 +5,7 @@ pub mod boolean; pub mod context; pub mod dp; pub mod ipa; -#[cfg(feature = "ipa-prf")] +#[cfg(feature = "descriptive-gate")] pub mod ipa_prf; pub mod modulus_conversion; pub mod prss; diff --git a/src/query/runner/mod.rs b/src/query/runner/mod.rs index 9875c7ced..d9eb28f8f 100644 --- a/src/query/runner/mod.rs +++ b/src/query/runner/mod.rs @@ -1,6 +1,5 @@ mod aggregate; mod ipa; -#[cfg(feature = "ipa-prf")] mod oprf_ipa; #[cfg(any(test, feature = "cli", feature = "test-fixture"))]