From 399c6e9076e3d618a535c17ecfd78bc3b28f5d82 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Wed, 23 Oct 2024 17:30:03 -0700 Subject: [PATCH 01/14] Initial --- ipa-core/src/report/hybrid.rs | 351 +++++++++++++++++++++++++++-- ipa-core/src/report/hybrid_info.rs | 81 ++++++- 2 files changed, 407 insertions(+), 25 deletions(-) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index 0956495ae..3bc7419dd 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -20,7 +20,7 @@ use crate::{ PublicKeyRegistry, TagSize, }, report::{ - hybrid_info::HybridImpressionInfo, EncryptedOprfReport, EventType, InvalidReportError, + hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, EncryptedOprfReport, EventType, InvalidReportError, KeyIdentifier, }, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, SharedValue}, @@ -115,10 +115,11 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, + info: &HybridImpressionInfo, rng: &mut R, ) -> Result, InvalidHybridReportError> { let mut out = Vec::with_capacity(usize::from(self.encrypted_len())); - self.encrypt_to(key_id, key_registry, rng, &mut out)?; + self.encrypt_to(key_id, key_registry, info, rng, &mut out)?; debug_assert_eq!(out.len(), usize::from(self.encrypted_len())); Ok(out) } @@ -129,11 +130,10 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, + info: &HybridImpressionInfo, rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { - let info = HybridImpressionInfo::new(key_id, HELPER_ORIGIN)?; - let mut plaintext_mk = GenericArray::default(); self.match_key.serialize(&mut plaintext_mk); @@ -178,6 +178,119 @@ where value: Replicated, } +impl Serializable for HybridConversionReport +where + V: SharedValue, + Replicated: Serializable, + as Serializable>::Size: Add, + < as Serializable>::Size as Add< as Serializable>::Size>>:: Output: ArrayLength, +{ + type Size = < as Serializable>::Size as Add< as Serializable>::Size>>:: Output; + type DeserializationError = InvalidHybridReportError; + + fn serialize(&self, buf: &mut GenericArray) { + let mk_sz = as Serializable>::Size::USIZE; + let v_sz = as Serializable>::Size::USIZE; + + self.match_key + .serialize(GenericArray::from_mut_slice(&mut buf[..mk_sz])); + + self.value + .serialize(GenericArray::from_mut_slice(&mut buf[mk_sz..mk_sz + v_sz])); + } + fn deserialize(buf: &GenericArray) -> Result { + let mk_sz = as Serializable>::Size::USIZE; + let v_sz = as Serializable>::Size::USIZE; + let match_key = + Replicated::::deserialize(GenericArray::from_slice(&buf[..mk_sz])) + .map_err(|e| InvalidHybridReportError::DeserializationError("match_key", e.into()))?; + let value = + Replicated::::deserialize(GenericArray::from_slice(&buf[mk_sz..mk_sz + v_sz])) + .map_err(|e| InvalidHybridReportError::DeserializationError("breakdown_key", e.into()))?; + Ok(Self { match_key, value }) + } +} + +impl HybridConversionReport +where + V: SharedValue, + Replicated: Serializable, + as Serializable>::Size: Add, + < as Serializable>::Size as Add< as Serializable>::Size>>:: Output: ArrayLength, +{ + const BTT_END: usize = as Serializable>::Size::USIZE; + + /// # Panics + /// If report length does not fit in `u16`. + pub fn encrypted_len(&self) -> u16 { + let len = EncryptedHybridConversionReport::::SITE_DOMAIN_OFFSET; + len.try_into().unwrap() + } + /// # Errors + /// If there is a problem encrypting the report. + pub fn encrypt( + &self, + key_id: KeyIdentifier, + key_registry: &impl PublicKeyRegistry, + info: &HybridConversionInfo, + rng: &mut R, + ) -> Result, InvalidHybridReportError> { + let mut out = Vec::with_capacity(usize::from(self.encrypted_len())); + self.encrypt_to(key_id, key_registry, info, rng, &mut out)?; + debug_assert_eq!(out.len(), usize::from(self.encrypted_len())); + Ok(out) + } + + /// # Errors + /// If there is a problem encrypting the report. + pub fn encrypt_to( + &self, + key_id: KeyIdentifier, + key_registry: &impl PublicKeyRegistry, + info: &HybridConversionInfo, + rng: &mut R, + out: &mut B, + ) -> Result<(), InvalidHybridReportError> { + //let info = HybridConversionInfo::new(key_id, HELPER_ORIGIN)?; + + //let mut plaintext_mk: GenericArray as Serializable>::Size> = GenericArray::default(); + //self.value.serialize(&mut plaintext_mk); + + let mut plaintext_mk = GenericArray::default(); + self.match_key.serialize(&mut plaintext_mk); + + let mut plaintext_btt = vec![0u8; Self::BTT_END]; + self.value + .serialize(GenericArray::from_mut_slice(&mut plaintext_btt[..])); + + let pk = key_registry.public_key(key_id).ok_or(CryptError::NoSuchKey(key_id))?; + + let (encap_key_mk, ciphertext_mk, tag_mk) = seal_in_place( + pk, + plaintext_mk.as_mut(), + &info.to_bytes(), + rng, + )?; + + let (encap_key_btt, ciphertext_btt, tag_btt) = seal_in_place( + pk, + plaintext_btt.as_mut(), + &info.to_bytes(), + rng, + )?; + + out.put_slice(&encap_key_mk.to_bytes()); + out.put_slice(ciphertext_mk); + out.put_slice(&tag_mk.to_bytes()); + out.put_slice(&encap_key_btt.to_bytes()); + out.put_slice(ciphertext_btt); + out.put_slice(&tag_btt.to_bytes()); + out.put_slice(&[key_id]); + + Ok(()) + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub enum HybridReport where @@ -227,12 +340,14 @@ where const CIPHERTEXT_MK_OFFSET: usize = Self::ENCAP_KEY_MK_OFFSET + EncapsulationSize::USIZE; const ENCAP_KEY_BTT_OFFSET: usize = (Self::CIPHERTEXT_MK_OFFSET + TagSize::USIZE - + as Serializable>::Size::USIZE); + //+ as Serializable>::Size::USIZE); + + Replicated::::size()); const CIPHERTEXT_BTT_OFFSET: usize = Self::ENCAP_KEY_BTT_OFFSET + EncapsulationSize::USIZE; const KEY_IDENTIFIER_OFFSET: usize = (Self::CIPHERTEXT_BTT_OFFSET + TagSize::USIZE - + as Serializable>::Size::USIZE); + //+ as Serializable>::Size::USIZE); + + Replicated::::size()); const SITE_DOMAIN_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; pub fn encap_key_mk(&self) -> &[u8] { @@ -279,12 +394,11 @@ where pub fn decrypt( &self, key_registry: &P, + info: &HybridImpressionInfo, ) -> Result, InvalidHybridReportError> { type CTMKLength = Sum< as Serializable>::Size, TagSize>; type CTBTTLength = < as Serializable>::Size as Add>::Output; - let info = HybridImpressionInfo::new(self.key_id(), HELPER_ORIGIN).unwrap(); // validated on construction - let mut ct_mk: GenericArray = *GenericArray::from_slice(self.mk_ciphertext()); let sk = key_registry @@ -308,6 +422,127 @@ where } } +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct EncryptedHybridConversionReport +where + B: Deref, + V: SharedValue, +{ + data: B, + phantom_data: PhantomData, +} + +impl EncryptedHybridConversionReport +where + B: Deref, + V: SharedValue, + Replicated: Serializable, + as Serializable>::Size: Add, + < as Serializable>::Size as Add>::Output: ArrayLength, +{ + const ENCAP_KEY_MK_OFFSET: usize = 0; + const CIPHERTEXT_MK_OFFSET: usize = Self::ENCAP_KEY_MK_OFFSET + EncapsulationSize::USIZE; + const ENCAP_KEY_BTT_OFFSET: usize = (Self::CIPHERTEXT_MK_OFFSET + + TagSize::USIZE + + Replicated::::size()); + const CIPHERTEXT_BTT_OFFSET: usize = Self::ENCAP_KEY_BTT_OFFSET + EncapsulationSize::USIZE; + + const KEY_IDENTIFIER_OFFSET: usize = (Self::CIPHERTEXT_BTT_OFFSET + + TagSize::USIZE + + Replicated::::size()); + const SITE_DOMAIN_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; + + pub fn encap_key_mk(&self) -> &[u8] { + &self.data[Self::ENCAP_KEY_MK_OFFSET..Self::CIPHERTEXT_MK_OFFSET] + } + + pub fn mk_ciphertext(&self) -> &[u8] { + &self.data[Self::CIPHERTEXT_MK_OFFSET..Self::ENCAP_KEY_BTT_OFFSET] + } + + pub fn encap_key_btt(&self) -> &[u8] { + &self.data[Self::ENCAP_KEY_BTT_OFFSET..Self::CIPHERTEXT_BTT_OFFSET] + } + + pub fn btt_ciphertext(&self) -> &[u8] { + &self.data[Self::CIPHERTEXT_BTT_OFFSET..Self::KEY_IDENTIFIER_OFFSET] + } + + pub fn key_id(&self) -> KeyIdentifier { + self.data[Self::KEY_IDENTIFIER_OFFSET] + } + + /// ## Errors + /// If the report contents are invalid. + pub fn from_bytes(bytes: B) -> Result { + if bytes.len() < Self::SITE_DOMAIN_OFFSET { + return Err(InvalidHybridReportError::Length( + bytes.len(), + Self::SITE_DOMAIN_OFFSET, + )); + } + Ok(Self { + data: bytes, + phantom_data: PhantomData, + }) + } + + /// ## Errors + /// If the match key shares in the report cannot be decrypted (e.g. due to a + /// failure of the authenticated encryption). + /// ## Panics + /// Should not panic. Only panics if a `Report` constructor failed to validate the + /// contents properly, which would be a bug. + pub fn decrypt( + &self, + key_registry: &P, + info: &HybridConversionInfo, + ) -> Result, InvalidHybridReportError> { + type CTMKLength = Sum< as Serializable>::Size, TagSize>; + type CTBTTLength = < as Serializable>::Size as Add>::Output; + + /*let info = HybridImpressionInfo::new(self.key_id(), HELPER_ORIGIN).unwrap(); // validated on construction + + let info = HybridConversionInfo::new( + self.key_id(), + HELPER_ORIGIN, + self.converion_site_domain(), + self.timestamp(), + self.epsilon(), + self.sensitivity(), + ) + .unwrap(); // validated on construction*/ + + // ConversionInfo fields: + /*pub key_id: KeyIdentifier, + pub helper_origin: &'a str, + pub converion_site_domain: &'a str, + pub timestamp: u64, + pub epsilon: f64, + pub sensitivity: f64,*/ + let mut ct_mk: GenericArray = + *GenericArray::from_slice(self.mk_ciphertext()); + let sk = key_registry + .private_key(self.key_id()) + .ok_or(CryptError::NoSuchKey(self.key_id()))?; + let plaintext_mk = open_in_place(sk, self.encap_key_mk(), &mut ct_mk, &info.to_bytes())?; + let mut ct_btt: GenericArray> = + GenericArray::from_slice(self.btt_ciphertext()).clone(); + + let plaintext_btt = open_in_place(sk, self.encap_key_btt(), &mut ct_btt, &info.to_bytes())?; + + Ok(HybridConversionReport:: { + match_key: Replicated::::deserialize_infallible(GenericArray::from_slice( + plaintext_mk, + )), + value: Replicated::::deserialize(GenericArray::from_slice(plaintext_btt)) + .map_err(|e| { + InvalidHybridReportError::DeserializationError("trigger_value", e.into()) + })?, + }) + } +} + #[derive(Clone)] pub struct EncryptedHybridReport { bytes: Bytes, @@ -497,7 +732,7 @@ mod test { use super::{ EncryptedHybridImpressionReport, EncryptedHybridReport, GenericArray, HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, - UniqueTagValidator, + UniqueTagValidator, HELPER_ORIGIN, }; use crate::{ error::Error, @@ -507,8 +742,8 @@ mod test { }, hpke::{KeyPair, KeyRegistry}, report::{ - hybrid::{NonAsciiStringError, BA64}, - hybrid_info::HybridImpressionInfo, + hybrid::{EncryptedHybridConversionReport, NonAsciiStringError, BA64}, + hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, EventType, OprfReport, }, secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, @@ -638,7 +873,29 @@ mod test { } #[test] - fn deserialzation_from_constant() { + fn serialization_hybrid_conversion() { + let mut rng = thread_rng(); + let b = EventType::Source; + let oprf_report = build_oprf_report(b, &mut rng); + + let hybrid_conversion_report = HybridConversionReport:: { + match_key: oprf_report.match_key.clone(), + value: oprf_report.trigger_value.clone(), + }; + let mut hybrid_conversion_report_bytes = + [0u8; as Serializable>::Size::USIZE]; + hybrid_conversion_report.serialize(GenericArray::from_mut_slice( + &mut hybrid_conversion_report_bytes[..], + )); + let hybrid_conversion_report2 = HybridConversionReport::::deserialize( + GenericArray::from_mut_slice(&mut hybrid_conversion_report_bytes[..]), + ) + .unwrap(); + assert_eq!(hybrid_conversion_report, hybrid_conversion_report2); + } + + #[test] + fn constant_serialization_hybrid_impression() { let hybrid_report = HybridImpressionReport::::deserialize(GenericArray::from_slice( &hex::decode("4123a6e38ef1d6d9785c948797cb744d38f4").unwrap(), )) @@ -673,6 +930,42 @@ mod test { ); } + #[test] + fn constant_serialization_hybrid_conversion() { + let hybrid_report = HybridConversionReport::::deserialize(GenericArray::from_slice( + &hex::decode("4123a6e38ef1d6d9785c948797cb744d0203").unwrap(), + )) + .unwrap(); + + let match_key = AdditiveShare::::deserialize(GenericArray::from_slice( + &hex::decode("4123a6e38ef1d6d9785c948797cb744d").unwrap(), + )) + .unwrap(); + let value = AdditiveShare::::deserialize(GenericArray::from_slice( + &hex::decode("0203").unwrap(), + )) + .unwrap(); + + assert_eq!( + hybrid_report, + HybridConversionReport:: { + match_key, + value + } + ); + + let mut hybrid_conversion_report_bytes = + [0u8; as Serializable>::Size::USIZE]; + hybrid_report.serialize(GenericArray::from_mut_slice( + &mut hybrid_conversion_report_bytes[..], + )); + + assert_eq!( + hybrid_conversion_report_bytes.to_vec(), + hex::decode("4123a6e38ef1d6d9785c948797cb744d0203").unwrap() + ); + } + #[test] fn enc_dec_roundtrip_hybrid_impression() { let mut rng = thread_rng(); @@ -687,18 +980,48 @@ mod test { let key_registry = KeyRegistry::::random(1, &mut rng); let key_id = 0; + let info = HybridImpressionInfo::new(key_id, HELPER_ORIGIN).unwrap(); + let enc_report_bytes = hybrid_impression_report - .encrypt(key_id, &key_registry, &mut rng) + .encrypt(key_id, &key_registry, &info, &mut rng) .unwrap(); let enc_report = EncryptedHybridImpressionReport::::from_bytes(enc_report_bytes.as_slice()) .unwrap(); - let dec_report: HybridImpressionReport = enc_report.decrypt(&key_registry).unwrap(); + let dec_report: HybridImpressionReport = enc_report.decrypt(&key_registry, &info).unwrap(); assert_eq!(dec_report, hybrid_impression_report); } + #[test] + fn enc_dec_roundtrip_hybrid_conversion() { + let mut rng = thread_rng(); + let b = EventType::Source; + let oprf_report = build_oprf_report(b, &mut rng); + + let hybrid_conversion_report = HybridConversionReport:: { + match_key: oprf_report.match_key.clone(), + value: oprf_report.trigger_value.clone(), + }; + + let key_registry = KeyRegistry::::random(1, &mut rng); + let key_id = 0; + + let info = HybridConversionInfo::new(key_id, HELPER_ORIGIN, "meta.com", 1729707432, 5.0, 1.1).unwrap(); + + let enc_report_bytes = hybrid_conversion_report + .encrypt(key_id, &key_registry, &info, &mut rng) + .unwrap(); + + let enc_report = + EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.as_slice()) + .unwrap(); + let dec_report: HybridConversionReport = enc_report.decrypt(&key_registry, &info).unwrap(); + + assert_eq!(dec_report, hybrid_conversion_report); + } + #[test] fn non_ascii_string() { let non_ascii_string = "☃️☃️☃️"; diff --git a/ipa-core/src/report/hybrid_info.rs b/ipa-core/src/report/hybrid_info.rs index c41849121..6ff6cddff 100644 --- a/ipa-core/src/report/hybrid_info.rs +++ b/ipa-core/src/report/hybrid_info.rs @@ -8,37 +8,81 @@ pub struct HybridImpressionInfo<'a> { pub helper_origin: &'a str, } -#[allow(dead_code)] +impl<'a> HybridImpressionInfo<'a> { + /// Creates a new instance. + /// + /// ## Errors + /// if helper or site origin is not a valid ASCII string. + pub fn new(key_id: KeyIdentifier, helper_origin: &'a str) -> Result { + // If the types of errors returned from this function change, then the validation in + // `EncryptedReport::from_bytes` may need to change as well. + if !helper_origin.is_ascii() { + return Err(helper_origin.into()); + } + + Ok(Self { + key_id, + helper_origin, + }) + } + + // Converts this instance into an owned byte slice that can further be used to create HPKE + // sender or receiver context. + pub(super) fn to_bytes(&self) -> Box<[u8]> { + let info_len = DOMAIN.len() + + self.helper_origin.len() + + 2 // delimiters(?) + + std::mem::size_of_val(&self.key_id); + let mut r = Vec::with_capacity(info_len); + + r.extend_from_slice(DOMAIN.as_bytes()); + r.push(0); + r.extend_from_slice(self.helper_origin.as_bytes()); + r.push(0); + + r.push(self.key_id); + + debug_assert_eq!(r.len(), info_len, "HPKE Info length estimation is incorrect and leads to extra allocation or wasted memory"); + + r.into_boxed_slice() + } +} + + +#[derive(Debug)] pub struct HybridConversionInfo<'a> { pub key_id: KeyIdentifier, pub helper_origin: &'a str, - pub converion_site_domain: &'a str, + pub conversion_site_domain: &'a str, pub timestamp: u64, pub epsilon: f64, pub sensitivity: f64, } -#[allow(dead_code)] -pub enum HybridInfo<'a> { - Impression(HybridImpressionInfo<'a>), - Conversion(HybridConversionInfo<'a>), -} -impl<'a> HybridImpressionInfo<'a> { +impl<'a> HybridConversionInfo<'a> { /// Creates a new instance. /// /// ## Errors /// if helper or site origin is not a valid ASCII string. - pub fn new(key_id: KeyIdentifier, helper_origin: &'a str) -> Result { + pub fn new(key_id: KeyIdentifier, helper_origin: &'a str, conversion_site_domain: &'a str, timestamp: u64, epsilon: f64, sensitivity: f64) -> Result { // If the types of errors returned from this function change, then the validation in // `EncryptedReport::from_bytes` may need to change as well. if !helper_origin.is_ascii() { return Err(helper_origin.into()); } + if !conversion_site_domain.is_ascii() { + return Err(conversion_site_domain.into()); + } + Ok(Self { key_id, helper_origin, + conversion_site_domain, + timestamp, + epsilon, + sensitivity, }) } @@ -47,19 +91,34 @@ impl<'a> HybridImpressionInfo<'a> { pub(super) fn to_bytes(&self) -> Box<[u8]> { let info_len = DOMAIN.len() + self.helper_origin.len() - + 2 // delimiters(?) - + std::mem::size_of_val(&self.key_id); + + self.conversion_site_domain.len() + + 3 // delimiters + + std::mem::size_of_val(&self.key_id) + + std::mem::size_of_val(&self.timestamp) + + std::mem::size_of_val(&self.epsilon) + + std::mem::size_of_val(&self.sensitivity); let mut r = Vec::with_capacity(info_len); r.extend_from_slice(DOMAIN.as_bytes()); r.push(0); r.extend_from_slice(self.helper_origin.as_bytes()); r.push(0); + r.extend_from_slice(self.conversion_site_domain.as_bytes()); + r.push(0); r.push(self.key_id); + r.extend_from_slice(&self.timestamp.to_be_bytes()); + r.extend_from_slice(&self.epsilon.to_be_bytes()); + r.extend_from_slice(&self.sensitivity.to_be_bytes()); debug_assert_eq!(r.len(), info_len, "HPKE Info length estimation is incorrect and leads to extra allocation or wasted memory"); r.into_boxed_slice() } } + +#[allow(dead_code)] +pub enum HybridInfo<'a> { + Impression(HybridImpressionInfo<'a>), + Conversion(HybridConversionInfo<'a>), +} From d1c97b5fe33643182f4b184792139357a069418a Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Wed, 23 Oct 2024 17:38:42 -0700 Subject: [PATCH 02/14] fix pre-commit --- ipa-core/src/report/hybrid.rs | 39 +++++++++++++++--------------- ipa-core/src/report/hybrid_info.rs | 11 ++++++--- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index 3bc7419dd..b98f2b33b 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -20,14 +20,15 @@ use crate::{ PublicKeyRegistry, TagSize, }, report::{ - hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, EncryptedOprfReport, EventType, InvalidReportError, - KeyIdentifier, + hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, + EncryptedOprfReport, EventType, InvalidReportError, KeyIdentifier, }, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, SharedValue}, sharding::ShardIndex, }; // TODO(679): This needs to come from configuration. +#[allow(dead_code)] static HELPER_ORIGIN: &str = "github.com/private-attribution"; #[derive(Debug, thiserror::Error)] @@ -442,14 +443,12 @@ where { const ENCAP_KEY_MK_OFFSET: usize = 0; const CIPHERTEXT_MK_OFFSET: usize = Self::ENCAP_KEY_MK_OFFSET + EncapsulationSize::USIZE; - const ENCAP_KEY_BTT_OFFSET: usize = (Self::CIPHERTEXT_MK_OFFSET - + TagSize::USIZE - + Replicated::::size()); + const ENCAP_KEY_BTT_OFFSET: usize = + (Self::CIPHERTEXT_MK_OFFSET + TagSize::USIZE + Replicated::::size()); const CIPHERTEXT_BTT_OFFSET: usize = Self::ENCAP_KEY_BTT_OFFSET + EncapsulationSize::USIZE; - const KEY_IDENTIFIER_OFFSET: usize = (Self::CIPHERTEXT_BTT_OFFSET - + TagSize::USIZE - + Replicated::::size()); + const KEY_IDENTIFIER_OFFSET: usize = + (Self::CIPHERTEXT_BTT_OFFSET + TagSize::USIZE + Replicated::::size()); const SITE_DOMAIN_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; pub fn encap_key_mk(&self) -> &[u8] { @@ -535,10 +534,9 @@ where match_key: Replicated::::deserialize_infallible(GenericArray::from_slice( plaintext_mk, )), - value: Replicated::::deserialize(GenericArray::from_slice(plaintext_btt)) - .map_err(|e| { - InvalidHybridReportError::DeserializationError("trigger_value", e.into()) - })?, + value: Replicated::::deserialize(GenericArray::from_slice(plaintext_btt)).map_err( + |e| InvalidHybridReportError::DeserializationError("trigger_value", e.into()), + )?, }) } } @@ -884,7 +882,7 @@ mod test { }; let mut hybrid_conversion_report_bytes = [0u8; as Serializable>::Size::USIZE]; - hybrid_conversion_report.serialize(GenericArray::from_mut_slice( + hybrid_conversion_report.serialize(GenericArray::from_mut_slice( &mut hybrid_conversion_report_bytes[..], )); let hybrid_conversion_report2 = HybridConversionReport::::deserialize( @@ -948,10 +946,7 @@ mod test { assert_eq!( hybrid_report, - HybridConversionReport:: { - match_key, - value - } + HybridConversionReport:: { match_key, value } ); let mut hybrid_conversion_report_bytes = @@ -989,7 +984,8 @@ mod test { let enc_report = EncryptedHybridImpressionReport::::from_bytes(enc_report_bytes.as_slice()) .unwrap(); - let dec_report: HybridImpressionReport = enc_report.decrypt(&key_registry, &info).unwrap(); + let dec_report: HybridImpressionReport = + enc_report.decrypt(&key_registry, &info).unwrap(); assert_eq!(dec_report, hybrid_impression_report); } @@ -1008,7 +1004,9 @@ mod test { let key_registry = KeyRegistry::::random(1, &mut rng); let key_id = 0; - let info = HybridConversionInfo::new(key_id, HELPER_ORIGIN, "meta.com", 1729707432, 5.0, 1.1).unwrap(); + let info = + HybridConversionInfo::new(key_id, HELPER_ORIGIN, "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(); let enc_report_bytes = hybrid_conversion_report .encrypt(key_id, &key_registry, &info, &mut rng) @@ -1017,7 +1015,8 @@ mod test { let enc_report = EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.as_slice()) .unwrap(); - let dec_report: HybridConversionReport = enc_report.decrypt(&key_registry, &info).unwrap(); + let dec_report: HybridConversionReport = + enc_report.decrypt(&key_registry, &info).unwrap(); assert_eq!(dec_report, hybrid_conversion_report); } diff --git a/ipa-core/src/report/hybrid_info.rs b/ipa-core/src/report/hybrid_info.rs index 6ff6cddff..ceeff618e 100644 --- a/ipa-core/src/report/hybrid_info.rs +++ b/ipa-core/src/report/hybrid_info.rs @@ -48,7 +48,6 @@ impl<'a> HybridImpressionInfo<'a> { } } - #[derive(Debug)] pub struct HybridConversionInfo<'a> { pub key_id: KeyIdentifier, @@ -59,13 +58,19 @@ pub struct HybridConversionInfo<'a> { pub sensitivity: f64, } - impl<'a> HybridConversionInfo<'a> { /// Creates a new instance. /// /// ## Errors /// if helper or site origin is not a valid ASCII string. - pub fn new(key_id: KeyIdentifier, helper_origin: &'a str, conversion_site_domain: &'a str, timestamp: u64, epsilon: f64, sensitivity: f64) -> Result { + pub fn new( + key_id: KeyIdentifier, + helper_origin: &'a str, + conversion_site_domain: &'a str, + timestamp: u64, + epsilon: f64, + sensitivity: f64, + ) -> Result { // If the types of errors returned from this function change, then the validation in // `EncryptedReport::from_bytes` may need to change as well. if !helper_origin.is_ascii() { From 53437af8dfcc0cf9b7b1a9727310ea3d0c5ef101 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Fri, 25 Oct 2024 00:40:51 -0700 Subject: [PATCH 03/14] how do I fix this... --- ipa-core/src/report/hybrid.rs | 218 +++++++++++++++++++++++++++++----- 1 file changed, 186 insertions(+), 32 deletions(-) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index b98f2b33b..d403ac115 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -51,8 +51,35 @@ pub enum InvalidHybridReportError { DeserializationError(&'static str, #[source] BoxError), #[error("report is too short: {0}, expected length at least: {1}")] Length(usize, usize), + #[error("unknown event type: {0}. Only 0 and 1 are allowed")] + UnknownEventType(u8), } +/*#[derive(thiserror::Error, Debug)] +#[error("{0} is not a valid event type, only 0 and 1 are allowed.")] +pub struct UnknownEventType(u8); + +impl Serializable for EventType { + type Size = U1; + type DeserializationError = UnknownEventType; + + fn serialize(&self, buf: &mut GenericArray) { + let raw: &[u8] = match self { + EventType::Trigger => &[1], + EventType::Source => &[0], + }; + buf.copy_from_slice(raw); + } + + fn deserialize(buf: &GenericArray) -> Result { + match buf[0] { + 1 => Ok(EventType::Trigger), + 0 => Ok(EventType::Source), + _ => Err(UnknownEventType(buf[0])), + } + } +}*/ + #[derive(Clone, Debug, Eq, PartialEq)] pub struct HybridImpressionReport where @@ -86,8 +113,8 @@ where let mk_sz = as Serializable>::Size::USIZE; let bk_sz = as Serializable>::Size::USIZE; let match_key = - Replicated::::deserialize(GenericArray::from_slice(&buf[..mk_sz])) - .map_err(|e| InvalidHybridReportError::DeserializationError("match_key", e.into()))?; + Replicated::::deserialize_infallible(GenericArray::from_slice(&buf[..mk_sz])); + //.map_err(|e| InvalidHybridReportError::DeserializationError("match_key", e.into()))?; let breakdown_key = Replicated::::deserialize(GenericArray::from_slice(&buf[mk_sz..mk_sz + bk_sz])) .map_err(|e| InvalidHybridReportError::DeserializationError("breakdown_key", e.into()))?; @@ -252,10 +279,6 @@ where rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { - //let info = HybridConversionInfo::new(key_id, HELPER_ORIGIN)?; - - //let mut plaintext_mk: GenericArray as Serializable>::Size> = GenericArray::default(); - //self.value.serialize(&mut plaintext_mk); let mut plaintext_mk = GenericArray::default(); self.match_key.serialize(&mut plaintext_mk); @@ -322,7 +345,7 @@ where #[derive(Copy, Clone, Eq, PartialEq)] pub struct EncryptedHybridImpressionReport where - B: Deref, + B: Deref + std::fmt::Debug, BK: SharedValue, { data: B, @@ -331,7 +354,7 @@ where impl EncryptedHybridImpressionReport where - B: Deref, + B: Deref + std::fmt::Debug, BK: SharedValue, Replicated: Serializable, as Serializable>::Size: Add, @@ -426,7 +449,7 @@ where #[derive(Copy, Clone, Eq, PartialEq)] pub struct EncryptedHybridConversionReport where - B: Deref, + B: Deref + std::fmt::Debug, V: SharedValue, { data: B, @@ -435,7 +458,7 @@ where impl EncryptedHybridConversionReport where - B: Deref, + B: Deref + std::fmt::Debug, V: SharedValue, Replicated: Serializable, as Serializable>::Size: Add, @@ -500,25 +523,8 @@ where type CTMKLength = Sum< as Serializable>::Size, TagSize>; type CTBTTLength = < as Serializable>::Size as Add>::Output; - /*let info = HybridImpressionInfo::new(self.key_id(), HELPER_ORIGIN).unwrap(); // validated on construction + println!("data: {:?}", self.data); - let info = HybridConversionInfo::new( - self.key_id(), - HELPER_ORIGIN, - self.converion_site_domain(), - self.timestamp(), - self.epsilon(), - self.sensitivity(), - ) - .unwrap(); // validated on construction*/ - - // ConversionInfo fields: - /*pub key_id: KeyIdentifier, - pub helper_origin: &'a str, - pub converion_site_domain: &'a str, - pub timestamp: u64, - pub epsilon: f64, - pub sensitivity: f64,*/ let mut ct_mk: GenericArray = *GenericArray::from_slice(self.mk_ciphertext()); let sk = key_registry @@ -541,6 +547,112 @@ where } } +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum EncryptedHybridGeneralReport +where + B: Deref + std::fmt::Debug, + BK: SharedValue, + V: SharedValue, +{ + Impression(EncryptedHybridImpressionReport), + Conversion(EncryptedHybridConversionReport), +} + +impl EncryptedHybridGeneralReport +where + B: Deref + std::fmt::Debug, + V: SharedValue, + BK: SharedValue, + Replicated: Serializable, + Replicated: Serializable, + as Serializable>::Size: Add, + < as Serializable>::Size as Add>::Output: ArrayLength, + as Serializable>::Size: Add, + < as Serializable>::Size as Add>::Output: ArrayLength, +{ + /*pub fn encap_key_mk(&self) -> &[u8] { + &self.data[Self::ENCAP_KEY_MK_OFFSET..Self::CIPHERTEXT_MK_OFFSET] + }*/ + + pub fn mk_ciphertext(&self) -> &[u8] { + match self { + EncryptedHybridGeneralReport::Impression(impression_report) => { + impression_report.mk_ciphertext() + } + EncryptedHybridGeneralReport::Conversion(conversion_report) => { + conversion_report.mk_ciphertext() + } + } + } + + /*pub fn encap_key_btt(&self) -> &[u8] { + &self.data[Self::ENCAP_KEY_BTT_OFFSET..Self::CIPHERTEXT_BTT_OFFSET] + } + + pub fn btt_ciphertext(&self) -> &[u8] { + &self.data[Self::CIPHERTEXT_BTT_OFFSET..Self::KEY_IDENTIFIER_OFFSET] + } + + pub fn key_id(&self) -> KeyIdentifier { + self.data[Self::KEY_IDENTIFIER_OFFSET] + }*/ + + /// ## Errors + /// If the report contents are invalid. + pub fn from_bytes(bytes: B) -> Result { + //let mut iter = bytes.iter(); + //let first_byte = iter.next().ok_or(InvalidHybridReportError::Length(0,1))?; + match bytes[0] { + 1 => { + //let impression_report = EncryptedHybridImpressionReport::::from_bytes(&bytes[1..])?; + let impression_report = EncryptedHybridImpressionReport::::from_bytes(&bytes[1..])?; + Ok(EncryptedHybridGeneralReport::Impression(impression_report)) + }, + 0 => { + //let conversion_report = EncryptedHybridConversionReport::::from_bytes(&bytes[1..])?; + let conversion_report = EncryptedHybridConversionReport::::from_bytes(&bytes[1..])?; + Ok(EncryptedHybridGeneralReport::Conversion(conversion_report)) + }, + _ => Err(InvalidHybridReportError::UnknownEventType(bytes[0])), + } + } + + /*/// ## Errors + /// If the match key shares in the report cannot be decrypted (e.g. due to a + /// failure of the authenticated encryption). + /// ## Panics + /// Should not panic. Only panics if a `Report` constructor failed to validate the + /// contents properly, which would be a bug. + pub fn decrypt( + &self, + key_registry: &P, + info: &HybridConversionInfo, + ) -> Result, InvalidHybridReportError> { + type CTMKLength = Sum< as Serializable>::Size, TagSize>; + type CTBTTLength = < as Serializable>::Size as Add>::Output; + + let mut ct_mk: GenericArray = + *GenericArray::from_slice(self.mk_ciphertext()); + let sk = key_registry + .private_key(self.key_id()) + .ok_or(CryptError::NoSuchKey(self.key_id()))?; + let plaintext_mk = open_in_place(sk, self.encap_key_mk(), &mut ct_mk, &info.to_bytes())?; + let mut ct_btt: GenericArray> = + GenericArray::from_slice(self.btt_ciphertext()).clone(); + + let plaintext_btt = open_in_place(sk, self.encap_key_btt(), &mut ct_btt, &info.to_bytes())?; + + Ok(HybridConversionReport:: { + match_key: Replicated::::deserialize_infallible(GenericArray::from_slice( + plaintext_mk, + )), + value: Replicated::::deserialize(GenericArray::from_slice(plaintext_btt)).map_err( + |e| InvalidHybridReportError::DeserializationError("trigger_value", e.into()), + )?, + }) + }*/ +} + #[derive(Clone)] pub struct EncryptedHybridReport { bytes: Bytes, @@ -728,9 +840,7 @@ mod test { use typenum::Unsigned; use super::{ - EncryptedHybridImpressionReport, EncryptedHybridReport, GenericArray, - HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, - UniqueTagValidator, HELPER_ORIGIN, + EncryptedHybridGeneralReport, EncryptedHybridImpressionReport, EncryptedHybridReport, GenericArray, HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, UniqueTagValidator, HELPER_ORIGIN }; use crate::{ error::Error, @@ -993,7 +1103,7 @@ mod test { #[test] fn enc_dec_roundtrip_hybrid_conversion() { let mut rng = thread_rng(); - let b = EventType::Source; + let b = EventType::Trigger; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_conversion_report = HybridConversionReport:: { @@ -1021,6 +1131,50 @@ mod test { assert_eq!(dec_report, hybrid_conversion_report); } + #[test] + fn enc_report_serialization() { + let mut rng = thread_rng(); + let b = EventType::Trigger; + let oprf_report = build_oprf_report(b, &mut rng); + + let hybrid_conversion_report = HybridConversionReport:: { + match_key: oprf_report.match_key.clone(), + value: oprf_report.trigger_value.clone(), + }; + + let key_registry = KeyRegistry::::random(1, &mut rng); + let key_id = 0; + + let info = + HybridConversionInfo::new(key_id, HELPER_ORIGIN, "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(); + + let mut enc_report_bytes = hybrid_conversion_report + .encrypt(key_id, &key_registry, &info, &mut rng) + .unwrap(); + + let enc_report = + EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.as_slice()) + .unwrap(); + enc_report_bytes.splice(0..0, [0]); + print!("{:?}", enc_report_bytes); + + let enc_report2 = EncryptedHybridGeneralReport::::from_bytes(enc_report_bytes.as_slice()).unwrap(); + match enc_report2 { + EncryptedHybridGeneralReport::Impression(_) => panic!("Expected conversion report"), + EncryptedHybridGeneralReport::Conversion(enc_report) => { + let dec_report: HybridConversionReport = + enc_report.decrypt(&key_registry, &info).unwrap(); + assert_eq!(dec_report, hybrid_conversion_report); + } + } + + /*let dec_report: HybridConversionReport = + enc_report.decrypt(&key_registry, &info).unwrap(); + + assert_eq!(dec_report, hybrid_conversion_report);*/ + } + #[test] fn non_ascii_string() { let non_ascii_string = "☃️☃️☃️"; From 041c48feb85cf5368751714c86b3c8b90d6f4194 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Mon, 28 Oct 2024 14:30:01 -0700 Subject: [PATCH 04/14] almost done --- ipa-core/src/report/hybrid.rs | 216 ++++++++++++++++------------- ipa-core/src/report/hybrid_info.rs | 1 - 2 files changed, 122 insertions(+), 95 deletions(-) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index d403ac115..2b5eef163 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -1,12 +1,7 @@ -use std::{ - collections::HashSet, - convert::Infallible, - marker::PhantomData, - ops::{Add, Deref}, -}; +use std::{collections::HashSet, convert::Infallible, marker::PhantomData, ops::Add}; use assertions::const_assert; -use bytes::{BufMut, Bytes}; +use bytes::{Buf, BufMut, Bytes}; use generic_array::{ArrayLength, GenericArray}; use hpke::Serializable as _; use rand_core::{CryptoRng, RngCore}; @@ -20,7 +15,7 @@ use crate::{ PublicKeyRegistry, TagSize, }, report::{ - hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, + hybrid_info::{HybridConversionInfo, HybridImpressionInfo, HybridInfo}, EncryptedOprfReport, EventType, InvalidReportError, KeyIdentifier, }, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, SharedValue}, @@ -53,6 +48,8 @@ pub enum InvalidHybridReportError { Length(usize, usize), #[error("unknown event type: {0}. Only 0 and 1 are allowed")] UnknownEventType(u8), + #[error("Incorrect hybrid info type: Expected {0}")] + WrongInfoType(&'static str), } /*#[derive(thiserror::Error, Debug)] @@ -134,7 +131,7 @@ where /// # Panics /// If report length does not fit in `u16`. pub fn encrypted_len(&self) -> u16 { - let len = EncryptedHybridImpressionReport::::SITE_DOMAIN_OFFSET; + let len = EncryptedHybridImpressionReport::::SITE_DOMAIN_OFFSET; len.try_into().unwrap() } /// # Errors @@ -251,7 +248,7 @@ where /// # Panics /// If report length does not fit in `u16`. pub fn encrypted_len(&self) -> u16 { - let len = EncryptedHybridConversionReport::::SITE_DOMAIN_OFFSET; + let len = EncryptedHybridConversionReport::::SITE_DOMAIN_OFFSET; len.try_into().unwrap() } /// # Errors @@ -342,19 +339,17 @@ where } } -#[derive(Copy, Clone, Eq, PartialEq)] -pub struct EncryptedHybridImpressionReport +#[derive(Clone, Eq, PartialEq)] +pub struct EncryptedHybridImpressionReport where - B: Deref + std::fmt::Debug, BK: SharedValue, { - data: B, + data: Bytes, phantom_data: PhantomData, } -impl EncryptedHybridImpressionReport +impl EncryptedHybridImpressionReport where - B: Deref + std::fmt::Debug, BK: SharedValue, Replicated: Serializable, as Serializable>::Size: Add, @@ -396,7 +391,7 @@ where /// ## Errors /// If the report contents are invalid. - pub fn from_bytes(bytes: B) -> Result { + pub fn from_bytes(bytes: Bytes) -> Result { if bytes.len() < Self::SITE_DOMAIN_OFFSET { return Err(InvalidHybridReportError::Length( bytes.len(), @@ -446,19 +441,17 @@ where } } -#[derive(Copy, Clone, Eq, PartialEq)] -pub struct EncryptedHybridConversionReport +#[derive(Clone, Eq, PartialEq)] +pub struct EncryptedHybridConversionReport where - B: Deref + std::fmt::Debug, V: SharedValue, { - data: B, + data: Bytes, phantom_data: PhantomData, } -impl EncryptedHybridConversionReport +impl EncryptedHybridConversionReport where - B: Deref + std::fmt::Debug, V: SharedValue, Replicated: Serializable, as Serializable>::Size: Add, @@ -496,7 +489,7 @@ where /// ## Errors /// If the report contents are invalid. - pub fn from_bytes(bytes: B) -> Result { + pub fn from_bytes(bytes: Bytes) -> Result { if bytes.len() < Self::SITE_DOMAIN_OFFSET { return Err(InvalidHybridReportError::Length( bytes.len(), @@ -547,20 +540,18 @@ where } } -#[derive(Copy, Clone, Eq, PartialEq)] -pub enum EncryptedHybridGeneralReport +#[derive(Clone, Eq, PartialEq)] +pub enum EncryptedHybridGeneralReport where - B: Deref + std::fmt::Debug, BK: SharedValue, V: SharedValue, { - Impression(EncryptedHybridImpressionReport), - Conversion(EncryptedHybridConversionReport), + Impression(EncryptedHybridImpressionReport), + Conversion(EncryptedHybridConversionReport), } -impl EncryptedHybridGeneralReport +impl EncryptedHybridGeneralReport where - B: Deref + std::fmt::Debug, V: SharedValue, BK: SharedValue, Replicated: Serializable, @@ -570,9 +561,16 @@ where as Serializable>::Size: Add, < as Serializable>::Size as Add>::Output: ArrayLength, { - /*pub fn encap_key_mk(&self) -> &[u8] { - &self.data[Self::ENCAP_KEY_MK_OFFSET..Self::CIPHERTEXT_MK_OFFSET] - }*/ + pub fn encap_key_mk(&self) -> &[u8] { + match self { + EncryptedHybridGeneralReport::Impression(impression_report) => { + impression_report.encap_key_mk() + } + EncryptedHybridGeneralReport::Conversion(conversion_report) => { + conversion_report.encap_key_mk() + } + } + } pub fn mk_ciphertext(&self) -> &[u8] { match self { @@ -585,39 +583,62 @@ where } } - /*pub fn encap_key_btt(&self) -> &[u8] { - &self.data[Self::ENCAP_KEY_BTT_OFFSET..Self::CIPHERTEXT_BTT_OFFSET] + pub fn encap_key_btt(&self) -> &[u8] { + match self { + EncryptedHybridGeneralReport::Impression(impression_report) => { + impression_report.encap_key_btt() + } + EncryptedHybridGeneralReport::Conversion(conversion_report) => { + conversion_report.encap_key_btt() + } + } } pub fn btt_ciphertext(&self) -> &[u8] { - &self.data[Self::CIPHERTEXT_BTT_OFFSET..Self::KEY_IDENTIFIER_OFFSET] + match self { + EncryptedHybridGeneralReport::Impression(impression_report) => { + impression_report.btt_ciphertext() + } + EncryptedHybridGeneralReport::Conversion(conversion_report) => { + conversion_report.btt_ciphertext() + } + } } - pub fn key_id(&self) -> KeyIdentifier { - self.data[Self::KEY_IDENTIFIER_OFFSET] - }*/ + pub fn key_id(&self) -> u8 { + match self { + EncryptedHybridGeneralReport::Impression(impression_report) => { + impression_report.key_id() + } + EncryptedHybridGeneralReport::Conversion(conversion_report) => { + conversion_report.key_id() + } + } + } /// ## Errors /// If the report contents are invalid. - pub fn from_bytes(bytes: B) -> Result { - //let mut iter = bytes.iter(); - //let first_byte = iter.next().ok_or(InvalidHybridReportError::Length(0,1))?; + pub fn from_bytes(mut bytes: Bytes) -> Result { + //let first_byte = bytes.next().ok_or(InvalidHybridReportError::Length(0,1))?; + //let the_rest = bytes[1..]; match bytes[0] { 1 => { //let impression_report = EncryptedHybridImpressionReport::::from_bytes(&bytes[1..])?; - let impression_report = EncryptedHybridImpressionReport::::from_bytes(&bytes[1..])?; + bytes.advance(1); + let impression_report = EncryptedHybridImpressionReport::::from_bytes(bytes)?; Ok(EncryptedHybridGeneralReport::Impression(impression_report)) - }, + } 0 => { //let conversion_report = EncryptedHybridConversionReport::::from_bytes(&bytes[1..])?; - let conversion_report = EncryptedHybridConversionReport::::from_bytes(&bytes[1..])?; + bytes.advance(1); + let conversion_report = EncryptedHybridConversionReport::::from_bytes(bytes)?; Ok(EncryptedHybridGeneralReport::Conversion(conversion_report)) - }, + } _ => Err(InvalidHybridReportError::UnknownEventType(bytes[0])), } } - /*/// ## Errors + /// ## Errors /// If the match key shares in the report cannot be decrypted (e.g. due to a /// failure of the authenticated encryption). /// ## Panics @@ -626,31 +647,27 @@ where pub fn decrypt( &self, key_registry: &P, - info: &HybridConversionInfo, - ) -> Result, InvalidHybridReportError> { - type CTMKLength = Sum< as Serializable>::Size, TagSize>; - type CTBTTLength = < as Serializable>::Size as Add>::Output; - - let mut ct_mk: GenericArray = - *GenericArray::from_slice(self.mk_ciphertext()); - let sk = key_registry - .private_key(self.key_id()) - .ok_or(CryptError::NoSuchKey(self.key_id()))?; - let plaintext_mk = open_in_place(sk, self.encap_key_mk(), &mut ct_mk, &info.to_bytes())?; - let mut ct_btt: GenericArray> = - GenericArray::from_slice(self.btt_ciphertext()).clone(); - - let plaintext_btt = open_in_place(sk, self.encap_key_btt(), &mut ct_btt, &info.to_bytes())?; - - Ok(HybridConversionReport:: { - match_key: Replicated::::deserialize_infallible(GenericArray::from_slice( - plaintext_mk, - )), - value: Replicated::::deserialize(GenericArray::from_slice(plaintext_btt)).map_err( - |e| InvalidHybridReportError::DeserializationError("trigger_value", e.into()), - )?, - }) - }*/ + info: &HybridInfo, + ) -> Result, InvalidHybridReportError> { + match self { + EncryptedHybridGeneralReport::Impression(impression_report) => match info { + HybridInfo::Impression(impression_info) => Ok(HybridReport::Impression( + impression_report.decrypt(key_registry, impression_info)?, + )), + HybridInfo::Conversion(_) => { + Err(InvalidHybridReportError::WrongInfoType("Impression")) + } + }, + EncryptedHybridGeneralReport::Conversion(conversion_report) => match info { + HybridInfo::Conversion(conversion_info) => Ok(HybridReport::Conversion( + conversion_report.decrypt(key_registry, conversion_info)?, + )), + HybridInfo::Impression(_) => { + Err(InvalidHybridReportError::WrongInfoType("Conversion")) + } + }, + } + } } #[derive(Clone)] @@ -840,7 +857,9 @@ mod test { use typenum::Unsigned; use super::{ - EncryptedHybridGeneralReport, EncryptedHybridImpressionReport, EncryptedHybridReport, GenericArray, HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, UniqueTagValidator, HELPER_ORIGIN + EncryptedHybridGeneralReport, EncryptedHybridImpressionReport, EncryptedHybridReport, + GenericArray, HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, + UniqueTagValidator, HELPER_ORIGIN, }; use crate::{ error::Error, @@ -851,7 +870,7 @@ mod test { hpke::{KeyPair, KeyRegistry}, report::{ hybrid::{EncryptedHybridConversionReport, NonAsciiStringError, BA64}, - hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, + hybrid_info::{HybridConversionInfo, HybridImpressionInfo, HybridInfo}, EventType, OprfReport, }, secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, @@ -1092,8 +1111,7 @@ mod test { .unwrap(); let enc_report = - EncryptedHybridImpressionReport::::from_bytes(enc_report_bytes.as_slice()) - .unwrap(); + EncryptedHybridImpressionReport::::from_bytes(enc_report_bytes.into()).unwrap(); let dec_report: HybridImpressionReport = enc_report.decrypt(&key_registry, &info).unwrap(); @@ -1123,8 +1141,7 @@ mod test { .unwrap(); let enc_report = - EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.as_slice()) - .unwrap(); + EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.into()).unwrap(); let dec_report: HybridConversionReport = enc_report.decrypt(&key_registry, &info).unwrap(); @@ -1149,30 +1166,41 @@ mod test { HybridConversionInfo::new(key_id, HELPER_ORIGIN, "meta.com", 1_729_707_432, 5.0, 1.1) .unwrap(); - let mut enc_report_bytes = hybrid_conversion_report + let enc_report_bytes = hybrid_conversion_report .encrypt(key_id, &key_registry, &info, &mut rng) .unwrap(); + let mut enc_report_bytes2 = enc_report_bytes.clone(); + let enc_report = - EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.as_slice()) - .unwrap(); - enc_report_bytes.splice(0..0, [0]); - print!("{:?}", enc_report_bytes); + EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.into()).unwrap(); + let dec_report: HybridConversionReport = + enc_report.decrypt(&key_registry, &info).unwrap(); + assert_eq!(dec_report, hybrid_conversion_report); - let enc_report2 = EncryptedHybridGeneralReport::::from_bytes(enc_report_bytes.as_slice()).unwrap(); + // Prepend a 0 byte to the ciphertext to mark it as a ConversionReport + enc_report_bytes2.splice(0..0, [0]); + + let enc_report2 = + EncryptedHybridGeneralReport::::from_bytes(enc_report_bytes2.into()).unwrap(); + let enc_report3 = enc_report2.clone(); + // Match first, then decrypt match enc_report2 { EncryptedHybridGeneralReport::Impression(_) => panic!("Expected conversion report"), - EncryptedHybridGeneralReport::Conversion(enc_report) => { - let dec_report: HybridConversionReport = - enc_report.decrypt(&key_registry, &info).unwrap(); - assert_eq!(dec_report, hybrid_conversion_report); + EncryptedHybridGeneralReport::Conversion(enc_report_conv) => { + let dec_report2: HybridConversionReport = + enc_report_conv.decrypt(&key_registry, &info).unwrap(); + assert_eq!(dec_report2, hybrid_conversion_report); } } - - /*let dec_report: HybridConversionReport = - enc_report.decrypt(&key_registry, &info).unwrap(); - - assert_eq!(dec_report, hybrid_conversion_report);*/ + // Decrypt directly + let dec_report3 = enc_report3 + .decrypt(&key_registry, &HybridInfo::Conversion(info)) + .unwrap(); + assert_eq!( + dec_report3, + HybridReport::Conversion(hybrid_conversion_report) + ); } #[test] diff --git a/ipa-core/src/report/hybrid_info.rs b/ipa-core/src/report/hybrid_info.rs index ceeff618e..24adad56d 100644 --- a/ipa-core/src/report/hybrid_info.rs +++ b/ipa-core/src/report/hybrid_info.rs @@ -122,7 +122,6 @@ impl<'a> HybridConversionInfo<'a> { } } -#[allow(dead_code)] pub enum HybridInfo<'a> { Impression(HybridImpressionInfo<'a>), Conversion(HybridConversionInfo<'a>), From ed9652aa7c2faa9a7e1a2e7b0e5dfbd8a5994ac9 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Mon, 28 Oct 2024 16:11:36 -0700 Subject: [PATCH 05/14] ready for review --- ipa-core/src/report/hybrid.rs | 92 ++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 12 deletions(-) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index 2b5eef163..4b44c2f44 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -638,6 +638,53 @@ where } } + /// ## Errors + /// If decryption of the provided oprf report fails. + pub fn decrypt_from_oprf_report_bytes( + bytes: Bytes, + key_registry: &P, + ) -> Result, InvalidHybridReportError> + where + P: PrivateKeyRegistry, + TS: SharedValue, + Replicated: Serializable, + as Serializable>::Size: Add< as Serializable>::Size>, + Sum< as Serializable>::Size, as Serializable>::Size>: + Add< as Serializable>::Size>, + Sum< + Sum< as Serializable>::Size, as Serializable>::Size>, + as Serializable>::Size, + >: Add, + Sum< + Sum< + Sum< as Serializable>::Size, as Serializable>::Size>, + as Serializable>::Size, + >, + U16, + >: ArrayLength, + { + let encrypted_oprf_report = EncryptedOprfReport::::try_from(bytes) + .map_err(|e| { + InvalidHybridReportError::DeserializationError("EncryptedOprfReport", e.into()) + })?; + let oprf_report = encrypted_oprf_report.decrypt(key_registry).map_err(|e| { + InvalidHybridReportError::DeserializationError( + "EncryptedOprfReport Decryption Failure", + e.into(), + ) + })?; + match oprf_report.event_type { + EventType::Source => Ok(HybridReport::Impression(HybridImpressionReport { + match_key: oprf_report.match_key, + breakdown_key: oprf_report.breakdown_key, + })), + EventType::Trigger => Ok(HybridReport::Conversion(HybridConversionReport { + match_key: oprf_report.match_key, + value: oprf_report.trigger_value, + })), + } + } + /// ## Errors /// If the match key shares in the report cannot be decrypted (e.g. due to a /// failure of the authenticated encryption). @@ -757,6 +804,27 @@ impl UniqueBytes for UniqueTag { } } +impl UniqueBytes for EncryptedHybridGeneralReport +where + V: SharedValue, + BK: SharedValue, + Replicated: Serializable, + Replicated: Serializable, + as Serializable>::Size: Add, + < as Serializable>::Size as Add>::Output: ArrayLength, + as Serializable>::Size: Add, + < as Serializable>::Size as Add>::Output: ArrayLength, +{ + /// We use the `TagSize` (the first 16 bytes of the ciphertext) for collision-detection + /// See [analysis here for uniqueness](https://eprint.iacr.org/2019/624) + fn unique_bytes(&self) -> [u8; TAG_SIZE] { + let slice = &self.mk_ciphertext()[0..TAG_SIZE]; + let mut array = [0u8; TAG_SIZE]; + array.copy_from_slice(slice); + array + } +} + impl UniqueBytes for EncryptedHybridReport { /// We use the `TagSize` (the first 16 bytes of the ciphertext) for collision-detection /// See [analysis here for uniqueness](https://eprint.iacr.org/2019/624) @@ -857,8 +925,8 @@ mod test { use typenum::Unsigned; use super::{ - EncryptedHybridGeneralReport, EncryptedHybridImpressionReport, EncryptedHybridReport, - GenericArray, HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, + EncryptedHybridGeneralReport, EncryptedHybridImpressionReport, GenericArray, + HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, UniqueTagValidator, HELPER_ORIGIN, }; use crate::{ @@ -917,12 +985,12 @@ mod test { let enc_report_bytes = oprf_report .encrypt(key_id, &key_registry, &mut rng) .unwrap(); - let enc_report = EncryptedHybridReport { - bytes: enc_report_bytes.into(), - }; - let hybrid_report2 = enc_report - .decrypt::<_, BA8, BA3, BA20>(&key_registry) + let hybrid_report2 = + EncryptedHybridGeneralReport::::decrypt_from_oprf_report_bytes::<_, BA20>( + enc_report_bytes.into(), + &key_registry, + ) .unwrap(); assert_eq!(hybrid_report, hybrid_report2); @@ -946,11 +1014,11 @@ mod test { let enc_report_bytes = oprf_report .encrypt(key_id, &key_registry, &mut rng) .unwrap(); - let enc_report = EncryptedHybridReport { - bytes: enc_report_bytes.into(), - }; - let hybrid_report2 = enc_report - .decrypt::<_, BA8, BA3, BA20>(&key_registry) + let hybrid_report2 = + EncryptedHybridGeneralReport::::decrypt_from_oprf_report_bytes::<_, BA20>( + enc_report_bytes.into(), + &key_registry, + ) .unwrap(); assert_eq!(hybrid_report, hybrid_report2); From fe9062dc56a003b34906b385f986a8b5de426536 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Wed, 30 Oct 2024 00:55:24 -0700 Subject: [PATCH 06/14] try, and not quite succeed, to get get all the runner stuff working --- ipa-core/src/error.rs | 4 +- ipa-core/src/query/runner/hybrid.rs | 87 +++++++--- ipa-core/src/report/hybrid.rs | 188 +++++++++++++++++---- ipa-core/src/report/hybrid_info.rs | 5 +- ipa-core/src/test_fixture/input/sharing.rs | 46 ++++- 5 files changed, 271 insertions(+), 59 deletions(-) diff --git a/ipa-core/src/error.rs b/ipa-core/src/error.rs index 168827c8e..24fdd6851 100644 --- a/ipa-core/src/error.rs +++ b/ipa-core/src/error.rs @@ -9,7 +9,7 @@ use thiserror::Error; use crate::{ helpers::{Role, ZeroRecordsError}, protocol::RecordId, - report::InvalidReportError, + report::{hybrid::InvalidHybridReportError, InvalidReportError}, sharding::ShardIndex, task::JoinError, }; @@ -69,6 +69,8 @@ pub enum Error { InvalidQueryParameter(BoxError), #[error("invalid report: {0}")] InvalidReport(#[from] InvalidReportError), + #[error("invalid hybrid report: {0}")] + InvalidHybridReport(#[from] InvalidHybridReportError), #[error("unsupported: {0}")] Unsupported(String), #[error("Decompressing invalid elliptic curve point: {0}")] diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index cc3b861c7..8addf9976 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -4,7 +4,7 @@ use futures::{stream::iter, StreamExt, TryStreamExt}; use crate::{ error::Error, - ff::boolean_array::{BA20, BA3, BA8}, + ff::boolean_array::{BA3, BA8}, helpers::{ query::{HybridQueryParams, QuerySize}, BodyStream, LengthDelimitedStream, @@ -15,25 +15,34 @@ use crate::{ hybrid::step::HybridStep, step::ProtocolStep::Hybrid, }, - report::hybrid::{EncryptedHybridReport, HybridReport, UniqueTag, UniqueTagValidator}, + report::{ + hybrid::{EncryptedHybridGeneralReport, HybridReport, UniqueTag, UniqueTagValidator}, + hybrid_info::HybridInfo, + }, secret_sharing::{replicated::semi_honest::AdditiveShare as ReplicatedShare, SharedValue}, }; #[allow(dead_code)] -pub struct Query { +pub struct Query<'a, C, HV, R: PrivateKeyRegistry> { config: HybridQueryParams, key_registry: Arc, + hybrid_info: HybridInfo<'a>, phantom_data: PhantomData<(C, HV)>, } -impl Query +impl<'a, C, HV: SharedValue, R: PrivateKeyRegistry> Query<'a, C, HV, R> where C: ShardedContext, { - pub fn new(query_params: HybridQueryParams, key_registry: Arc) -> Self { + pub fn new( + query_params: HybridQueryParams, + key_registry: Arc, + hybrid_info: HybridInfo<'a>, + ) -> Self { Self { config: query_params, key_registry, + hybrid_info, phantom_data: PhantomData, } } @@ -48,6 +57,7 @@ where let Self { config, key_registry, + hybrid_info, phantom_data: _, } = self; @@ -62,16 +72,20 @@ where } let (_decrypted_reports, tags): (Vec>, Vec) = - LengthDelimitedStream::::new(input_stream) + //LengthDelimitedStream::, _>::new(input_stream) + LengthDelimitedStream::, _>::new(input_stream) .map_err(Into::::into) .map_ok(|enc_reports| { iter(enc_reports.into_iter().map({ |enc_report| { - let dec_report = enc_report - .decrypt::(key_registry.as_ref()) - .map_err(Into::::into); + let dec_report = enc_report.decrypt(key_registry.as_ref(), &hybrid_info).map_err(Into::::into); + //let dec_report = EncryptedHybridGeneralReport::::decrypt_from_oprf_report_bytes::(enc_report.serialize(), key_registry.as_ref()).unwrap();//.map_err(Into::::into); + /*let dec_report = enc_report + .decrypt_from_::(key_registry.as_ref()) + .map_err(Into::::into);*/ let unique_tag = UniqueTag::from_unique_bytes(&enc_report); dec_report.map(|dec_report1| (dec_report1, unique_tag)) + //(dec_report, unique_tag) } })) }) @@ -114,7 +128,7 @@ mod tests { use crate::{ ff::{ - boolean_array::{BA16, BA20, BA3, BA8}, + boolean_array::{BA16, BA3, BA8}, U128Conversions, }, helpers::{ @@ -123,7 +137,11 @@ mod tests { }, hpke::{KeyPair, KeyRegistry}, query::runner::hybrid::Query as HybridQuery, - report::{OprfReport, DEFAULT_KEY_ID}, + report::{ + hybrid::HybridReport, + hybrid_info::{HybridConversionInfo, HybridInfo}, + DEFAULT_KEY_ID, + }, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::{ flatten3v, ipa::TestRawDataRecord, Reconstruct, RoundRobinInputDistribution, TestWorld, @@ -137,7 +155,7 @@ mod tests { // TODO: When Encryption/Decryption exists for HybridReports // update these to use that, rather than generating OprfReports vec![ - TestRawDataRecord { + /*TestRawDataRecord { timestamp: 0, user_id: 12345, is_trigger_report: false, @@ -150,7 +168,7 @@ mod tests { is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - }, + },*/ TestRawDataRecord { timestamp: 10, user_id: 12345, @@ -165,13 +183,13 @@ mod tests { breakdown_key: 0, trigger_value: 2, }, - TestRawDataRecord { + /*TestRawDataRecord { timestamp: 20, user_id: 68362, is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - }, + },*/ TestRawDataRecord { timestamp: 30, user_id: 68362, @@ -188,17 +206,28 @@ mod tests { query_sizes: Vec, } - fn build_buffers_from_records(records: &[TestRawDataRecord], s: usize) -> BufferAndKeyRegistry { + fn build_buffers_from_records( + records: &[TestRawDataRecord], + s: usize, + info: &HybridInfo, + ) -> BufferAndKeyRegistry { let mut rng = StdRng::seed_from_u64(42); let key_id = DEFAULT_KEY_ID; let key_registry = Arc::new(KeyRegistry::::random(1, &mut rng)); let mut buffers: [_; 3] = std::array::from_fn(|_| vec![Vec::new(); s]); - let shares: [Vec>; 3] = records.iter().cloned().share(); + //let shares: [Vec>; 3] = records.iter().cloned().share(); + let shares: [Vec>; 3] = records.iter().cloned().share(); for (buf, shares) in zip(&mut buffers, shares) { for (i, share) in shares.into_iter().enumerate() { share - .delimited_encrypt_to(key_id, key_registry.as_ref(), &mut rng, &mut buf[i % s]) + .delimited_encrypt_to( + key_id, + key_registry.as_ref(), + info, + &mut rng, + &mut buf[i % s], + ) .unwrap(); } } @@ -236,12 +265,16 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); + let hybrid_info = HybridInfo::Conversion( + HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(), + ); let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS); + } = build_buffers_from_records(&records, SHARDS, &hybrid_info); let world: TestWorld> = TestWorld::with_shards(TestWorldConfig::default()); @@ -267,6 +300,7 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), + hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -300,12 +334,16 @@ mod tests { async fn duplicate_encrypted_hybrid_reports() { const SHARDS: usize = 2; let records = build_records(); + let hybrid_info = HybridInfo::Conversion( + HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(), + ); let BufferAndKeyRegistry { mut buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS); + } = build_buffers_from_records(&records, SHARDS, &hybrid_info); // this is double, since we duplicate the data below let query_sizes = query_sizes @@ -353,6 +391,7 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), + hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -372,11 +411,16 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); + let hybrid_info = HybridInfo::Conversion( + HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(), + ); + let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS); + } = build_buffers_from_records(&records, SHARDS, &hybrid_info); let world: TestWorld> = TestWorld::with_shards(TestWorldConfig::default()); @@ -402,6 +446,7 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), + hybrid_info.clone(), ) .execute(ctx, query_size, input) }) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index 4b44c2f44..6c281f4be 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -52,38 +52,13 @@ pub enum InvalidHybridReportError { WrongInfoType(&'static str), } -/*#[derive(thiserror::Error, Debug)] -#[error("{0} is not a valid event type, only 0 and 1 are allowed.")] -pub struct UnknownEventType(u8); - -impl Serializable for EventType { - type Size = U1; - type DeserializationError = UnknownEventType; - - fn serialize(&self, buf: &mut GenericArray) { - let raw: &[u8] = match self { - EventType::Trigger => &[1], - EventType::Source => &[0], - }; - buf.copy_from_slice(raw); - } - - fn deserialize(buf: &GenericArray) -> Result { - match buf[0] { - 1 => Ok(EventType::Trigger), - 0 => Ok(EventType::Source), - _ => Err(UnknownEventType(buf[0])), - } - } -}*/ - #[derive(Clone, Debug, Eq, PartialEq)] pub struct HybridImpressionReport where BK: SharedValue, { - match_key: Replicated, - breakdown_key: Replicated, + pub match_key: Replicated, + pub breakdown_key: Replicated, } impl Serializable for HybridImpressionReport @@ -134,6 +109,21 @@ where let len = EncryptedHybridImpressionReport::::SITE_DOMAIN_OFFSET; len.try_into().unwrap() } + + /// # Errors + /// If there is a problem encrypting the report. + pub fn delimited_encrypt_to( + &self, + key_id: KeyIdentifier, + key_registry: &impl PublicKeyRegistry, + info: &HybridImpressionInfo, + rng: &mut R, + out: &mut B, + ) -> Result<(), InvalidHybridReportError> { + out.put_u16_le(self.encrypted_len()); + self.encrypt_to(key_id, key_registry, info, rng, out) + } + /// # Errors /// If there is a problem encrypting the report. pub fn encrypt( @@ -199,8 +189,8 @@ pub struct HybridConversionReport where V: SharedValue, { - match_key: Replicated, - value: Replicated, + pub match_key: Replicated, + pub value: Replicated, } impl Serializable for HybridConversionReport @@ -251,6 +241,21 @@ where let len = EncryptedHybridConversionReport::::SITE_DOMAIN_OFFSET; len.try_into().unwrap() } + + /// # Errors + /// If there is a problem encrypting the report. + pub fn delimited_encrypt_to( + &self, + key_id: KeyIdentifier, + key_registry: &impl PublicKeyRegistry, + info: &HybridConversionInfo, + rng: &mut R, + out: &mut B, + ) -> Result<(), InvalidHybridReportError> { + out.put_u16_le(self.encrypted_len()); + self.encrypt_to(key_id, key_registry, info, rng, out) + } + /// # Errors /// If there is a problem encrypting the report. pub fn encrypt( @@ -326,16 +331,113 @@ impl HybridReport where BK: SharedValue, V: SharedValue, + Replicated: Serializable, + Replicated: Serializable, + as Serializable>::Size: Add, + as Serializable>::Size: Add, + < as Serializable>::Size as Add< as Serializable>::Size>>:: Output: ArrayLength, + < as Serializable>::Size as Add< as Serializable>::Size>>:: Output: ArrayLength, { + /// # Panics + /// If report length does not fit in `u16`. + pub fn encrypted_len(&self) -> u16 { + match self { + HybridReport::Impression(impression_report) => { + impression_report.encrypted_len() + } + HybridReport::Conversion(conversion_report) => { + conversion_report.encrypted_len() + } + } + } + + /// # Errors + /// If there is a problem encrypting the report. + pub fn delimited_encrypt_to( + &self, + key_id: KeyIdentifier, + key_registry: &impl PublicKeyRegistry, + info: &HybridInfo, + rng: &mut R, + out: &mut B, + ) -> Result<(), InvalidHybridReportError> { + match self { + HybridReport::Impression(impression_report) => match info { + HybridInfo::Impression(impression_info) =>{ + out.put_u8(1u8); + impression_report.delimited_encrypt_to(key_id, key_registry, impression_info, rng, out)}, + HybridInfo::Conversion(_) => { + Err(InvalidHybridReportError::WrongInfoType("Impression")) + } + }, + HybridReport::Conversion(conversion_report) => match info { + HybridInfo::Conversion(conversion_info) =>{ + out.put_u8(0u8); + conversion_report.delimited_encrypt_to(key_id, key_registry, conversion_info, rng, out)}, + HybridInfo::Impression(_) => { + Err(InvalidHybridReportError::WrongInfoType("Conversion")) + } + }, + } + } + /// # Errors /// If there is a problem encrypting the report. pub fn encrypt( &self, - _key_id: KeyIdentifier, - _key_registry: &impl PublicKeyRegistry, - _rng: &mut R, - ) -> Result, InvalidReportError> { - unimplemented!() + key_id: KeyIdentifier, + key_registry: &impl PublicKeyRegistry, + info: &HybridInfo, + rng: &mut R, + ) -> Result, InvalidHybridReportError> { + match self { + HybridReport::Impression(impression_report) => match info { + HybridInfo::Impression(impression_info) => + // Prepend a 1u8 byte to indicate this is an impression report + impression_report.encrypt(key_id, key_registry, impression_info, rng).map(|v| vec![1u8].into_iter().chain(v).collect()), + HybridInfo::Conversion(_) => { + Err(InvalidHybridReportError::WrongInfoType("Impression")) + } + }, + HybridReport::Conversion(conversion_report) => match info { + HybridInfo::Conversion(conversion_info) => + // Prepend a 0u8 byte to indicate this is a conversion report + conversion_report.encrypt(key_id, key_registry, conversion_info, rng).map(|v| vec![0u8].into_iter().chain(v).collect()), + HybridInfo::Impression(_) => { + Err(InvalidHybridReportError::WrongInfoType("Conversion")) + } + }, + } + } + + /// # Errors + /// If there is a problem encrypting the report. + pub fn encrypt_to( + &self, + key_id: KeyIdentifier, + key_registry: &impl PublicKeyRegistry, + info: &HybridInfo, + rng: &mut R, + out: &mut B, + ) -> Result<(), InvalidHybridReportError> { + match self { + HybridReport::Impression(impression_report) => match info { + HybridInfo::Impression(impression_info) =>{ + out.put_u8(1u8); + impression_report.encrypt_to(key_id, key_registry, impression_info, rng, out)}, + HybridInfo::Conversion(_) => { + Err(InvalidHybridReportError::WrongInfoType("Impression")) + } + }, + HybridReport::Conversion(conversion_report) => match info { + HybridInfo::Conversion(conversion_info) =>{ + out.put_u8(0u8); + conversion_report.encrypt_to(key_id, key_registry, conversion_info, rng, out)}, + HybridInfo::Impression(_) => { + Err(InvalidHybridReportError::WrongInfoType("Conversion")) + } + }, + } } } @@ -717,6 +819,24 @@ where } } +impl TryFrom for EncryptedHybridGeneralReport +where + V: SharedValue, + BK: SharedValue, + Replicated: Serializable, + Replicated: Serializable, + as Serializable>::Size: Add, + < as Serializable>::Size as Add>::Output: ArrayLength, + as Serializable>::Size: Add, + < as Serializable>::Size as Add>::Output: ArrayLength, +{ + type Error = InvalidHybridReportError; + + fn try_from(bytes: Bytes) -> Result { + Self::from_bytes(bytes) + } +} + #[derive(Clone)] pub struct EncryptedHybridReport { bytes: Bytes, diff --git a/ipa-core/src/report/hybrid_info.rs b/ipa-core/src/report/hybrid_info.rs index 24adad56d..30b1b04c4 100644 --- a/ipa-core/src/report/hybrid_info.rs +++ b/ipa-core/src/report/hybrid_info.rs @@ -2,7 +2,7 @@ use crate::report::{hybrid::NonAsciiStringError, KeyIdentifier}; const DOMAIN: &str = "private-attribution"; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct HybridImpressionInfo<'a> { pub key_id: KeyIdentifier, pub helper_origin: &'a str, @@ -48,7 +48,7 @@ impl<'a> HybridImpressionInfo<'a> { } } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct HybridConversionInfo<'a> { pub key_id: KeyIdentifier, pub helper_origin: &'a str, @@ -122,6 +122,7 @@ impl<'a> HybridConversionInfo<'a> { } } +#[derive(Clone, Debug)] pub enum HybridInfo<'a> { Impression(HybridImpressionInfo<'a>), Conversion(HybridConversionInfo<'a>), diff --git a/ipa-core/src/test_fixture/input/sharing.rs b/ipa-core/src/test_fixture/input/sharing.rs index 14dddd0db..8041a362e 100644 --- a/ipa-core/src/test_fixture/input/sharing.rs +++ b/ipa-core/src/test_fixture/input/sharing.rs @@ -8,7 +8,10 @@ use crate::{ }, protocol::ipa_prf::OPRFIPAInputRow, rand::Rng, - report::{EventType, OprfReport}, + report::{ + hybrid::{HybridConversionReport, HybridImpressionReport, HybridReport}, + EventType, OprfReport, + }, secret_sharing::{ replicated::{semi_honest::AdditiveShare as Replicated, ReplicatedSecretSharing}, IntoShares, @@ -69,6 +72,47 @@ where } } +impl IntoShares> for TestRawDataRecord +where + BK: BooleanArray + U128Conversions + IntoShares>, + V: BooleanArray + U128Conversions + IntoShares>, +{ + fn share_with(self, rng: &mut R) -> [HybridReport; 3] { + let match_key = BA64::try_from(u128::from(self.user_id)) + .unwrap() + .share_with(rng); + let breakdown_key = BK::try_from(self.breakdown_key.into()) + .unwrap() + .share_with(rng); + let trigger_value = V::try_from(self.trigger_value.into()) + .unwrap() + .share_with(rng); + if self.is_trigger_report { + zip(match_key, trigger_value) + .map(|(match_key_share, trigger_value_share)| { + HybridReport::Conversion::(HybridConversionReport { + match_key: match_key_share, + value: trigger_value_share, + }) + }) + .collect::>() + .try_into() + .unwrap() + } else { + zip(match_key, breakdown_key) + .map(|(match_key_share, breakdown_key_share)| { + HybridReport::Impression::(HybridImpressionReport { + match_key: match_key_share, + breakdown_key: breakdown_key_share, + }) + }) + .collect::>() + .try_into() + .unwrap() + } + } +} + impl IntoShares> for TestRawDataRecord where BK: BooleanArray + U128Conversions + IntoShares>, From 8457a3bcc8877599edd8fa7902ad1711eea842b9 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Wed, 30 Oct 2024 23:54:06 -0700 Subject: [PATCH 07/14] just give up --- ipa-core/src/query/runner/hybrid.rs | 183 +++++++++++++--------------- 1 file changed, 83 insertions(+), 100 deletions(-) diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index 8addf9976..06cc2da4a 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -1,63 +1,65 @@ -use std::{marker::PhantomData, sync::Arc}; +use std::{convert::Into, marker::PhantomData, sync::Arc}; use futures::{stream::iter, StreamExt, TryStreamExt}; use crate::{ error::Error, - ff::boolean_array::{BA3, BA8}, + ff::{ + boolean_array::{BooleanArray, BA20, BA3, BA8}, + U128Conversions, + }, helpers::{ - query::{HybridQueryParams, QuerySize}, + query::{DpMechanism, HybridQueryParams, QuerySize}, BodyStream, LengthDelimitedStream, }, hpke::PrivateKeyRegistry, protocol::{ - context::{reshard_iter, ShardedContext}, - hybrid::step::HybridStep, + context::{ShardedContext, UpgradableContext}, + hybrid::{hybrid_protocol, step::HybridStep}, + ipa_prf::{oprf_padding::PaddingParameters, shuffle::Shuffle}, step::ProtocolStep::Hybrid, }, - report::{ - hybrid::{EncryptedHybridGeneralReport, HybridReport, UniqueTag, UniqueTagValidator}, - hybrid_info::HybridInfo, + query::runner::reshard_tag::reshard_aad, + report::hybrid::{ + EncryptedHybridReport, IndistinguishableHybridReport, UniqueTag, UniqueTagValidator, }, - secret_sharing::{replicated::semi_honest::AdditiveShare as ReplicatedShare, SharedValue}, + secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, }; #[allow(dead_code)] -pub struct Query<'a, C, HV, R: PrivateKeyRegistry> { +pub struct Query { config: HybridQueryParams, key_registry: Arc, - hybrid_info: HybridInfo<'a>, phantom_data: PhantomData<(C, HV)>, } -impl<'a, C, HV: SharedValue, R: PrivateKeyRegistry> Query<'a, C, HV, R> -where - C: ShardedContext, -{ - pub fn new( - query_params: HybridQueryParams, - key_registry: Arc, - hybrid_info: HybridInfo<'a>, - ) -> Self { +#[allow(dead_code)] +impl Query { + pub fn new(query_params: HybridQueryParams, key_registry: Arc) -> Self { Self { config: query_params, key_registry, - hybrid_info, phantom_data: PhantomData, } } +} +impl Query +where + C: UpgradableContext + Shuffle + ShardedContext, + HV: BooleanArray + U128Conversions, + R: PrivateKeyRegistry, +{ #[tracing::instrument("hybrid_query", skip_all, fields(sz=%query_size))] pub async fn execute( self, ctx: C, query_size: QuerySize, input_stream: BodyStream, - ) -> Result>, Error> { + ) -> Result>, Error> { let Self { config, key_registry, - hybrid_info, phantom_data: _, } = self; @@ -71,39 +73,24 @@ where )); } - let (_decrypted_reports, tags): (Vec>, Vec) = - //LengthDelimitedStream::, _>::new(input_stream) - LengthDelimitedStream::, _>::new(input_stream) - .map_err(Into::::into) - .map_ok(|enc_reports| { - iter(enc_reports.into_iter().map({ - |enc_report| { - let dec_report = enc_report.decrypt(key_registry.as_ref(), &hybrid_info).map_err(Into::::into); - //let dec_report = EncryptedHybridGeneralReport::::decrypt_from_oprf_report_bytes::(enc_report.serialize(), key_registry.as_ref()).unwrap();//.map_err(Into::::into); - /*let dec_report = enc_report - .decrypt_from_::(key_registry.as_ref()) - .map_err(Into::::into);*/ - let unique_tag = UniqueTag::from_unique_bytes(&enc_report); - dec_report.map(|dec_report1| (dec_report1, unique_tag)) - //(dec_report, unique_tag) - } - })) - }) - .try_flatten() - .take(sz) - .try_fold( - (Vec::with_capacity(sz), Vec::with_capacity(sz)), - |mut acc, result| async move { - acc.0.push(result.0); - acc.1.push(result.1); - Ok(acc) - }, - ) - .await?; - - let resharded_tags = reshard_iter( + let stream = LengthDelimitedStream::::new(input_stream) + .map_err(Into::::into) + .map_ok(|enc_reports| { + iter(enc_reports.into_iter().map({ + |enc_report| { + let dec_report = enc_report + .decrypt::(key_registry.as_ref()) + .map_err(Into::::into); + let unique_tag = UniqueTag::from_unique_bytes(&enc_report); + dec_report.map(|dec_report1| (dec_report1, unique_tag)) + } + })) + }) + .try_flatten() + .take(sz); + let (decrypted_reports, resharded_tags) = reshard_aad( ctx.narrow(&HybridStep::ReshardByTag), - tags, + stream, |ctx, _, tag| tag.shard_picker(ctx.shard_count()), ) .await?; @@ -115,7 +102,34 @@ where .check_duplicates(&resharded_tags) .unwrap(); - unimplemented!("query::runnner::HybridQuery.execute is not fully implemented") + let indistinguishable_reports: Vec> = + decrypted_reports.into_iter().map(Into::into).collect(); + + let dp_params: DpMechanism = match config.with_dp { + 0 => DpMechanism::NoDp, + _ => DpMechanism::DiscreteLaplace { + epsilon: config.epsilon, + }, + }; + + #[cfg(feature = "relaxed-dp")] + let padding_params = PaddingParameters::relaxed(); + #[cfg(not(feature = "relaxed-dp"))] + let padding_params = PaddingParameters::default(); + + match config.per_user_credit_cap { + 1 => hybrid_protocol::<_, BA8, BA3, HV, 1, 256>(ctx, indistinguishable_reports, dp_params, padding_params).await, + 2 | 4 => hybrid_protocol::<_, BA8, BA3, HV, 2, 256>(ctx, indistinguishable_reports, dp_params, padding_params).await, + 8 => hybrid_protocol::<_, BA8, BA3, HV, 3, 256>(ctx, indistinguishable_reports, dp_params, padding_params).await, + 16 => hybrid_protocol::<_, BA8, BA3, HV, 4, 256>(ctx, indistinguishable_reports, dp_params, padding_params).await, + 32 => hybrid_protocol::<_, BA8, BA3, HV, 5, 256>(ctx, indistinguishable_reports, dp_params, padding_params).await, + 64 => hybrid_protocol::<_, BA8, BA3, HV, 6, 256>(ctx, indistinguishable_reports, dp_params, padding_params).await, + 128 => hybrid_protocol::<_, BA8, BA3, HV, 7, 256>(ctx, indistinguishable_reports, dp_params, padding_params).await, + _ => panic!( + "Invalid value specified for per-user cap: {:?}. Must be one of 1, 2, 4, 8, 16, 32, 64, or 128.", + config.per_user_credit_cap + ), + } } } @@ -128,7 +142,7 @@ mod tests { use crate::{ ff::{ - boolean_array::{BA16, BA3, BA8}, + boolean_array::{BA16, BA20, BA3, BA8}, U128Conversions, }, helpers::{ @@ -137,11 +151,7 @@ mod tests { }, hpke::{KeyPair, KeyRegistry}, query::runner::hybrid::Query as HybridQuery, - report::{ - hybrid::HybridReport, - hybrid_info::{HybridConversionInfo, HybridInfo}, - DEFAULT_KEY_ID, - }, + report::{OprfReport, DEFAULT_KEY_ID}, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::{ flatten3v, ipa::TestRawDataRecord, Reconstruct, RoundRobinInputDistribution, TestWorld, @@ -155,7 +165,7 @@ mod tests { // TODO: When Encryption/Decryption exists for HybridReports // update these to use that, rather than generating OprfReports vec![ - /*TestRawDataRecord { + TestRawDataRecord { timestamp: 0, user_id: 12345, is_trigger_report: false, @@ -168,7 +178,7 @@ mod tests { is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - },*/ + }, TestRawDataRecord { timestamp: 10, user_id: 12345, @@ -183,13 +193,13 @@ mod tests { breakdown_key: 0, trigger_value: 2, }, - /*TestRawDataRecord { + TestRawDataRecord { timestamp: 20, user_id: 68362, is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - },*/ + }, TestRawDataRecord { timestamp: 30, user_id: 68362, @@ -206,28 +216,17 @@ mod tests { query_sizes: Vec, } - fn build_buffers_from_records( - records: &[TestRawDataRecord], - s: usize, - info: &HybridInfo, - ) -> BufferAndKeyRegistry { + fn build_buffers_from_records(records: &[TestRawDataRecord], s: usize) -> BufferAndKeyRegistry { let mut rng = StdRng::seed_from_u64(42); let key_id = DEFAULT_KEY_ID; let key_registry = Arc::new(KeyRegistry::::random(1, &mut rng)); let mut buffers: [_; 3] = std::array::from_fn(|_| vec![Vec::new(); s]); - //let shares: [Vec>; 3] = records.iter().cloned().share(); - let shares: [Vec>; 3] = records.iter().cloned().share(); + let shares: [Vec>; 3] = records.iter().cloned().share(); for (buf, shares) in zip(&mut buffers, shares) { for (i, share) in shares.into_iter().enumerate() { share - .delimited_encrypt_to( - key_id, - key_registry.as_ref(), - info, - &mut rng, - &mut buf[i % s], - ) + .delimited_encrypt_to(key_id, key_registry.as_ref(), &mut rng, &mut buf[i % s]) .unwrap(); } } @@ -257,7 +256,7 @@ mod tests { // placeholder until the protocol is complete. can be updated to make sure we // get to the unimplemented() call #[should_panic( - expected = "not implemented: query::runnner::HybridQuery.execute is not fully implemented" + expected = "not implemented: protocol::hybrid::hybrid_protocol is not fully implemented" )] async fn encrypted_hybrid_reports() { // While this test currently checks for an unimplemented panic it is @@ -265,16 +264,12 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); - let hybrid_info = HybridInfo::Conversion( - HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(), - ); let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS, &hybrid_info); + } = build_buffers_from_records(&records, SHARDS); let world: TestWorld> = TestWorld::with_shards(TestWorldConfig::default()); @@ -300,7 +295,6 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), - hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -334,16 +328,12 @@ mod tests { async fn duplicate_encrypted_hybrid_reports() { const SHARDS: usize = 2; let records = build_records(); - let hybrid_info = HybridInfo::Conversion( - HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(), - ); let BufferAndKeyRegistry { mut buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS, &hybrid_info); + } = build_buffers_from_records(&records, SHARDS); // this is double, since we duplicate the data below let query_sizes = query_sizes @@ -391,7 +381,6 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), - hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -411,16 +400,11 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); - let hybrid_info = HybridInfo::Conversion( - HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(), - ); - let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS, &hybrid_info); + } = build_buffers_from_records(&records, SHARDS); let world: TestWorld> = TestWorld::with_shards(TestWorldConfig::default()); @@ -446,7 +430,6 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), - hybrid_info.clone(), ) .execute(ctx, query_size, input) }) From 8cda0d1f889fe1986b3b5f37705581363115179c Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Thu, 31 Oct 2024 00:28:30 -0700 Subject: [PATCH 08/14] fix horrific merge --- ipa-core/src/report/hybrid.rs | 163 +++++++++++++++++++++------------- 1 file changed, 99 insertions(+), 64 deletions(-) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index 860d13766..90124b667 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -1,7 +1,3 @@ -use std::{collections::HashSet, convert::Infallible, marker::PhantomData, ops::Add}; - -use assertions::const_assert; -use bytes::{Buf, BufMut, Bytes}; //! Provides report types which are aggregated by the Hybrid protocol //! //! The `IndistinguishableHybridReport` is the primary data type which each helpers uses @@ -38,7 +34,7 @@ use std::{ ops::{Add, Deref}, }; -use bytes::{BufMut, Bytes}; +use bytes::{Buf, BufMut, Bytes}; use generic_array::{ArrayLength, GenericArray}; use hpke::Serializable as _; use rand_core::{CryptoRng, RngCore}; @@ -485,7 +481,7 @@ where /// `HybridImpressionReport`s are encrypted when they arrive to the helpers, /// which is represented here. A `EncryptedHybridImpressionReport` decrypts /// into a `HybridImpressionReport`. -#[derive(Copy, Clone, Eq, PartialEq)] +#[derive(Clone, Eq, PartialEq)] pub struct EncryptedHybridImpressionReport where BK: SharedValue, @@ -687,9 +683,6 @@ where } } -#[derive(Clone, Eq, PartialEq)] -pub enum EncryptedHybridGeneralReport -======= /// This struct is designed to fit both `HybridConversionReport`s /// and `HybridImpressionReport`s so that they can be made indistingushable. /// Note: these need to be shuffled (and secret shares need to be rerandomized) @@ -718,6 +711,103 @@ where } impl From> for IndistinguishableHybridReport +where + BK: SharedValue, + V: SharedValue, +{ + fn from(match_key: Replicated) -> Self { + Self { + match_key, + value: Replicated::::ZERO, + breakdown_key: Replicated::::ZERO, + } + } +} + +impl From> for IndistinguishableHybridReport +where + BK: SharedValue, + V: SharedValue, +{ + fn from(report: HybridReport) -> Self { + match report { + HybridReport::Impression(r) => r.into(), + HybridReport::Conversion(r) => r.into(), + } + } +} + +impl From> for IndistinguishableHybridReport +where + BK: SharedValue, + V: SharedValue, +{ + fn from(impression_report: HybridImpressionReport) -> Self { + Self { + match_key: impression_report.match_key, + value: Replicated::ZERO, + breakdown_key: impression_report.breakdown_key, + } + } +} + +impl From> for IndistinguishableHybridReport +where + BK: SharedValue, + V: SharedValue, +{ + fn from(conversion_report: HybridConversionReport) -> Self { + Self { + match_key: conversion_report.match_key, + value: conversion_report.value, + breakdown_key: Replicated::ZERO, + } + } +} + +/*impl From> for IndistinguishableHybridReport +where + BK: SharedValue, + V: SharedValue, +{ + fn from(report: HybridReport) -> Self { + match report { + HybridReport::Impression(r) => r.into(), + HybridReport::Conversion(r) => r.into(), + } + } +}*/ + +/*impl From> for IndistinguishableHybridReport +where + BK: SharedValue, + V: SharedValue, +{ + fn from(impression_report: HybridImpressionReport) -> Self { + Self { + match_key: impression_report.match_key, + value: Replicated::ZERO, + breakdown_key: impression_report.breakdown_key, + } + } +}*/ + +/*impl From> for IndistinguishableHybridReport +where + BK: SharedValue, + V: SharedValue, +{ + fn from(conversion_report: HybridConversionReport) -> Self { + Self { + match_key: conversion_report.match_key, + value: conversion_report.value, + breakdown_key: Replicated::ZERO, + } + } +}*/ + +#[derive(Clone, Eq, PartialEq)] +pub enum EncryptedHybridGeneralReport where BK: SharedValue, V: SharedValue, @@ -725,7 +815,6 @@ where Impression(EncryptedHybridImpressionReport), Conversion(EncryptedHybridConversionReport), } - impl EncryptedHybridGeneralReport where V: SharedValue, @@ -747,7 +836,6 @@ where } } } - pub fn mk_ciphertext(&self) -> &[u8] { match self { EncryptedHybridGeneralReport::Impression(impression_report) => { @@ -758,7 +846,6 @@ where } } } - pub fn encap_key_btt(&self) -> &[u8] { match self { EncryptedHybridGeneralReport::Impression(impression_report) => { @@ -769,7 +856,6 @@ where } } } - pub fn btt_ciphertext(&self) -> &[u8] { match self { EncryptedHybridGeneralReport::Impression(impression_report) => { @@ -780,7 +866,6 @@ where } } } - pub fn key_id(&self) -> u8 { match self { EncryptedHybridGeneralReport::Impression(impression_report) => { @@ -791,7 +876,6 @@ where } } } - /// ## Errors /// If the report contents are invalid. pub fn from_bytes(mut bytes: Bytes) -> Result { @@ -813,7 +897,6 @@ where _ => Err(InvalidHybridReportError::UnknownEventType(bytes[0])), } } - /// ## Errors /// If decryption of the provided oprf report fails. pub fn decrypt_from_oprf_report_bytes( @@ -860,7 +943,6 @@ where })), } } - /// ## Errors /// If the match key shares in the report cannot be decrypted (e.g. due to a /// failure of the authenticated encryption). @@ -908,53 +990,6 @@ where fn try_from(bytes: Bytes) -> Result { Self::from_bytes(bytes) - fn from(match_key: Replicated) -> Self { - Self { - match_key, - value: Replicated::::ZERO, - breakdown_key: Replicated::::ZERO, - } - } -} - -impl From> for IndistinguishableHybridReport -where - BK: SharedValue, - V: SharedValue, -{ - fn from(report: HybridReport) -> Self { - match report { - HybridReport::Impression(r) => r.into(), - HybridReport::Conversion(r) => r.into(), - } - } -} - -impl From> for IndistinguishableHybridReport -where - BK: SharedValue, - V: SharedValue, -{ - fn from(impression_report: HybridImpressionReport) -> Self { - Self { - match_key: impression_report.match_key, - value: Replicated::ZERO, - breakdown_key: impression_report.breakdown_key, - } - } -} - -impl From> for IndistinguishableHybridReport -where - BK: SharedValue, - V: SharedValue, -{ - fn from(conversion_report: HybridConversionReport) -> Self { - Self { - match_key: conversion_report.match_key, - value: conversion_report.value, - breakdown_key: Replicated::ZERO, - } } } From 9a6c491a0023e2b4e2704d2dd057f21022e4f7eb Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Thu, 31 Oct 2024 01:35:44 -0700 Subject: [PATCH 09/14] finally found the issue --- ipa-core/src/query/runner/hybrid.rs | 84 ++++++++--- ipa-core/src/report/hybrid.rs | 209 ++++++---------------------- 2 files changed, 107 insertions(+), 186 deletions(-) diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index 06cc2da4a..a3b27eda3 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -5,7 +5,7 @@ use futures::{stream::iter, StreamExt, TryStreamExt}; use crate::{ error::Error, ff::{ - boolean_array::{BooleanArray, BA20, BA3, BA8}, + boolean_array::{BooleanArray, BA3, BA8}, U128Conversions, }, helpers::{ @@ -20,31 +20,40 @@ use crate::{ step::ProtocolStep::Hybrid, }, query::runner::reshard_tag::reshard_aad, - report::hybrid::{ - EncryptedHybridReport, IndistinguishableHybridReport, UniqueTag, UniqueTagValidator, + report::{ + hybrid::{ + EncryptedHybridReport, IndistinguishableHybridReport, UniqueTag, UniqueTagValidator, + }, + hybrid_info::HybridInfo, }, secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, }; #[allow(dead_code)] -pub struct Query { +pub struct Query<'a, C, HV, R: PrivateKeyRegistry> { config: HybridQueryParams, key_registry: Arc, + hybrid_info: HybridInfo<'a>, phantom_data: PhantomData<(C, HV)>, } #[allow(dead_code)] -impl Query { - pub fn new(query_params: HybridQueryParams, key_registry: Arc) -> Self { +impl<'a, C, HV, R: PrivateKeyRegistry> Query<'a, C, HV, R> { + pub fn new( + query_params: HybridQueryParams, + key_registry: Arc, + hybrid_info: HybridInfo<'a>, + ) -> Self { Self { config: query_params, key_registry, + hybrid_info, phantom_data: PhantomData, } } } -impl Query +impl<'a, C, HV, R> Query<'a, C, HV, R> where C: UpgradableContext + Shuffle + ShardedContext, HV: BooleanArray + U128Conversions, @@ -60,6 +69,7 @@ where let Self { config, key_registry, + hybrid_info, phantom_data: _, } = self; @@ -73,13 +83,13 @@ where )); } - let stream = LengthDelimitedStream::::new(input_stream) + let stream = LengthDelimitedStream::, _>::new(input_stream) .map_err(Into::::into) .map_ok(|enc_reports| { iter(enc_reports.into_iter().map({ |enc_report| { let dec_report = enc_report - .decrypt::(key_registry.as_ref()) + .decrypt(key_registry.as_ref(), &hybrid_info) .map_err(Into::::into); let unique_tag = UniqueTag::from_unique_bytes(&enc_report); dec_report.map(|dec_report1| (dec_report1, unique_tag)) @@ -142,7 +152,7 @@ mod tests { use crate::{ ff::{ - boolean_array::{BA16, BA20, BA3, BA8}, + boolean_array::{BA16, BA3, BA8}, U128Conversions, }, helpers::{ @@ -151,7 +161,11 @@ mod tests { }, hpke::{KeyPair, KeyRegistry}, query::runner::hybrid::Query as HybridQuery, - report::{OprfReport, DEFAULT_KEY_ID}, + report::{ + hybrid::HybridReport, + hybrid_info::{HybridConversionInfo, HybridInfo}, + DEFAULT_KEY_ID, + }, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::{ flatten3v, ipa::TestRawDataRecord, Reconstruct, RoundRobinInputDistribution, TestWorld, @@ -165,7 +179,7 @@ mod tests { // TODO: When Encryption/Decryption exists for HybridReports // update these to use that, rather than generating OprfReports vec![ - TestRawDataRecord { + /*TestRawDataRecord { timestamp: 0, user_id: 12345, is_trigger_report: false, @@ -178,7 +192,7 @@ mod tests { is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - }, + },*/ TestRawDataRecord { timestamp: 10, user_id: 12345, @@ -193,13 +207,13 @@ mod tests { breakdown_key: 0, trigger_value: 2, }, - TestRawDataRecord { + /*TestRawDataRecord { timestamp: 20, user_id: 68362, is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - }, + },*/ TestRawDataRecord { timestamp: 30, user_id: 68362, @@ -216,17 +230,27 @@ mod tests { query_sizes: Vec, } - fn build_buffers_from_records(records: &[TestRawDataRecord], s: usize) -> BufferAndKeyRegistry { + fn build_buffers_from_records( + records: &[TestRawDataRecord], + s: usize, + info: &HybridInfo, + ) -> BufferAndKeyRegistry { let mut rng = StdRng::seed_from_u64(42); let key_id = DEFAULT_KEY_ID; let key_registry = Arc::new(KeyRegistry::::random(1, &mut rng)); let mut buffers: [_; 3] = std::array::from_fn(|_| vec![Vec::new(); s]); - let shares: [Vec>; 3] = records.iter().cloned().share(); + let shares: [Vec>; 3] = records.iter().cloned().share(); for (buf, shares) in zip(&mut buffers, shares) { for (i, share) in shares.into_iter().enumerate() { share - .delimited_encrypt_to(key_id, key_registry.as_ref(), &mut rng, &mut buf[i % s]) + .delimited_encrypt_to( + key_id, + key_registry.as_ref(), + info, + &mut rng, + &mut buf[i % s], + ) .unwrap(); } } @@ -265,11 +289,16 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); + let hybrid_info = HybridInfo::Conversion( + HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(), + ); + let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS); + } = build_buffers_from_records(&records, SHARDS, &hybrid_info); let world: TestWorld> = TestWorld::with_shards(TestWorldConfig::default()); @@ -295,6 +324,7 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), + hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -329,11 +359,16 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); + let hybrid_info = HybridInfo::Conversion( + HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(), + ); + let BufferAndKeyRegistry { mut buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS); + } = build_buffers_from_records(&records, SHARDS, &hybrid_info); // this is double, since we duplicate the data below let query_sizes = query_sizes @@ -381,6 +416,7 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), + hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -400,11 +436,16 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); + let hybrid_info = HybridInfo::Conversion( + HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) + .unwrap(), + ); + let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&records, SHARDS); + } = build_buffers_from_records(&records, SHARDS, &hybrid_info); let world: TestWorld> = TestWorld::with_shards(TestWorldConfig::default()); @@ -430,6 +471,7 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), + hybrid_info.clone(), ) .execute(ctx, query_size, input) }) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index 90124b667..cf1fe461f 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -27,12 +27,7 @@ //! all secret sharings (including the sharings of zero), making the collection of reports //! cryptographically indistinguishable. -use std::{ - collections::HashSet, - convert::Infallible, - marker::PhantomData, - ops::{Add, Deref}, -}; +use std::{collections::HashSet, convert::Infallible, marker::PhantomData, ops::Add}; use bytes::{Buf, BufMut, Bytes}; use generic_array::{ArrayLength, GenericArray}; @@ -50,7 +45,7 @@ use crate::{ }, report::{ hybrid_info::{HybridConversionInfo, HybridImpressionInfo, HybridInfo}, - EncryptedOprfReport, EventType, InvalidReportError, KeyIdentifier, + EncryptedOprfReport, EventType, KeyIdentifier, }, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, SharedValue}, sharding::ShardIndex, @@ -380,10 +375,10 @@ where pub fn encrypted_len(&self) -> u16 { match self { HybridReport::Impression(impression_report) => { - impression_report.encrypted_len() + impression_report.encrypted_len() +1 } HybridReport::Conversion(conversion_report) => { - conversion_report.encrypted_len() + conversion_report.encrypted_len() +1 } } } @@ -401,16 +396,18 @@ where match self { HybridReport::Impression(impression_report) => match info { HybridInfo::Impression(impression_info) =>{ + out.put_u16_le(self.encrypted_len()); out.put_u8(1u8); - impression_report.delimited_encrypt_to(key_id, key_registry, impression_info, rng, out)}, + impression_report.encrypt_to(key_id, key_registry, impression_info, rng, out)}, HybridInfo::Conversion(_) => { Err(InvalidHybridReportError::WrongInfoType("Impression")) } }, HybridReport::Conversion(conversion_report) => match info { HybridInfo::Conversion(conversion_info) =>{ + out.put_u16_le(self.encrypted_len()); out.put_u8(0u8); - conversion_report.delimited_encrypt_to(key_id, key_registry, conversion_info, rng, out)}, + conversion_report.encrypt_to(key_id, key_registry, conversion_info, rng, out)}, HybridInfo::Impression(_) => { Err(InvalidHybridReportError::WrongInfoType("Conversion")) } @@ -583,7 +580,6 @@ where } } - #[derive(Clone, Eq, PartialEq)] pub struct EncryptedHybridConversionReport where @@ -765,49 +761,8 @@ where } } -/*impl From> for IndistinguishableHybridReport -where - BK: SharedValue, - V: SharedValue, -{ - fn from(report: HybridReport) -> Self { - match report { - HybridReport::Impression(r) => r.into(), - HybridReport::Conversion(r) => r.into(), - } - } -}*/ - -/*impl From> for IndistinguishableHybridReport -where - BK: SharedValue, - V: SharedValue, -{ - fn from(impression_report: HybridImpressionReport) -> Self { - Self { - match_key: impression_report.match_key, - value: Replicated::ZERO, - breakdown_key: impression_report.breakdown_key, - } - } -}*/ - -/*impl From> for IndistinguishableHybridReport -where - BK: SharedValue, - V: SharedValue, -{ - fn from(conversion_report: HybridConversionReport) -> Self { - Self { - match_key: conversion_report.match_key, - value: conversion_report.value, - breakdown_key: Replicated::ZERO, - } - } -}*/ - #[derive(Clone, Eq, PartialEq)] -pub enum EncryptedHybridGeneralReport +pub enum EncryptedHybridReport where BK: SharedValue, V: SharedValue, @@ -815,7 +770,7 @@ where Impression(EncryptedHybridImpressionReport), Conversion(EncryptedHybridConversionReport), } -impl EncryptedHybridGeneralReport +impl EncryptedHybridReport where V: SharedValue, BK: SharedValue, @@ -828,52 +783,48 @@ where { pub fn encap_key_mk(&self) -> &[u8] { match self { - EncryptedHybridGeneralReport::Impression(impression_report) => { + EncryptedHybridReport::Impression(impression_report) => { impression_report.encap_key_mk() } - EncryptedHybridGeneralReport::Conversion(conversion_report) => { + EncryptedHybridReport::Conversion(conversion_report) => { conversion_report.encap_key_mk() } } } pub fn mk_ciphertext(&self) -> &[u8] { match self { - EncryptedHybridGeneralReport::Impression(impression_report) => { + EncryptedHybridReport::Impression(impression_report) => { impression_report.mk_ciphertext() } - EncryptedHybridGeneralReport::Conversion(conversion_report) => { + EncryptedHybridReport::Conversion(conversion_report) => { conversion_report.mk_ciphertext() } } } pub fn encap_key_btt(&self) -> &[u8] { match self { - EncryptedHybridGeneralReport::Impression(impression_report) => { + EncryptedHybridReport::Impression(impression_report) => { impression_report.encap_key_btt() } - EncryptedHybridGeneralReport::Conversion(conversion_report) => { + EncryptedHybridReport::Conversion(conversion_report) => { conversion_report.encap_key_btt() } } } pub fn btt_ciphertext(&self) -> &[u8] { match self { - EncryptedHybridGeneralReport::Impression(impression_report) => { + EncryptedHybridReport::Impression(impression_report) => { impression_report.btt_ciphertext() } - EncryptedHybridGeneralReport::Conversion(conversion_report) => { + EncryptedHybridReport::Conversion(conversion_report) => { conversion_report.btt_ciphertext() } } } pub fn key_id(&self) -> u8 { match self { - EncryptedHybridGeneralReport::Impression(impression_report) => { - impression_report.key_id() - } - EncryptedHybridGeneralReport::Conversion(conversion_report) => { - conversion_report.key_id() - } + EncryptedHybridReport::Impression(impression_report) => impression_report.key_id(), + EncryptedHybridReport::Conversion(conversion_report) => conversion_report.key_id(), } } /// ## Errors @@ -886,13 +837,13 @@ where //let impression_report = EncryptedHybridImpressionReport::::from_bytes(&bytes[1..])?; bytes.advance(1); let impression_report = EncryptedHybridImpressionReport::::from_bytes(bytes)?; - Ok(EncryptedHybridGeneralReport::Impression(impression_report)) + Ok(EncryptedHybridReport::Impression(impression_report)) } 0 => { //let conversion_report = EncryptedHybridConversionReport::::from_bytes(&bytes[1..])?; bytes.advance(1); let conversion_report = EncryptedHybridConversionReport::::from_bytes(bytes)?; - Ok(EncryptedHybridGeneralReport::Conversion(conversion_report)) + Ok(EncryptedHybridReport::Conversion(conversion_report)) } _ => Err(InvalidHybridReportError::UnknownEventType(bytes[0])), } @@ -955,7 +906,7 @@ where info: &HybridInfo, ) -> Result, InvalidHybridReportError> { match self { - EncryptedHybridGeneralReport::Impression(impression_report) => match info { + EncryptedHybridReport::Impression(impression_report) => match info { HybridInfo::Impression(impression_info) => Ok(HybridReport::Impression( impression_report.decrypt(key_registry, impression_info)?, )), @@ -963,7 +914,7 @@ where Err(InvalidHybridReportError::WrongInfoType("Impression")) } }, - EncryptedHybridGeneralReport::Conversion(conversion_report) => match info { + EncryptedHybridReport::Conversion(conversion_report) => match info { HybridInfo::Conversion(conversion_info) => Ok(HybridReport::Conversion( conversion_report.decrypt(key_registry, conversion_info)?, )), @@ -975,7 +926,7 @@ where } } -impl TryFrom for EncryptedHybridGeneralReport +impl TryFrom for EncryptedHybridReport where V: SharedValue, BK: SharedValue, @@ -993,76 +944,6 @@ where } } -#[derive(Clone)] -pub struct EncryptedHybridReport { - bytes: Bytes, -} - -impl EncryptedHybridReport { - /// ## Errors - /// If the report fails to decrypt - pub fn decrypt( - &self, - key_registry: &P, - ) -> Result, InvalidReportError> - where - P: PrivateKeyRegistry, - BK: SharedValue, - V: SharedValue, - TS: SharedValue, - Replicated: Serializable, - Replicated: Serializable, - Replicated: Serializable, - as Serializable>::Size: Add< as Serializable>::Size>, - Sum< as Serializable>::Size, as Serializable>::Size>: - Add< as Serializable>::Size>, - Sum< - Sum< as Serializable>::Size, as Serializable>::Size>, - as Serializable>::Size, - >: Add, - Sum< - Sum< - Sum< as Serializable>::Size, as Serializable>::Size>, - as Serializable>::Size, - >, - U16, - >: ArrayLength, - { - let encrypted_oprf_report = - EncryptedOprfReport::::try_from(self.bytes.clone())?; - let oprf_report = encrypted_oprf_report.decrypt(key_registry)?; - match oprf_report.event_type { - EventType::Source => Ok(HybridReport::Impression(HybridImpressionReport { - match_key: oprf_report.match_key, - breakdown_key: oprf_report.breakdown_key, - })), - EventType::Trigger => Ok(HybridReport::Conversion(HybridConversionReport { - match_key: oprf_report.match_key, - value: oprf_report.trigger_value, - })), - } - } - - /// TODO: update these when we produce a proper encapsulation of - /// `EncryptedHybridReport`, rather than pigggybacking on `EncryptedOprfReport` - pub fn mk_ciphertext(&self) -> &[u8] { - let encap_key_mk_offset: usize = 0; - let ciphertext_mk_offset: usize = encap_key_mk_offset + EncapsulationSize::USIZE; - let encap_key_btt_offset: usize = - ciphertext_mk_offset + TagSize::USIZE + as Serializable>::Size::USIZE; - - &self.bytes[ciphertext_mk_offset..encap_key_btt_offset] - } -} - -impl TryFrom for EncryptedHybridReport { - type Error = InvalidReportError; - - fn try_from(bytes: Bytes) -> Result { - Ok(EncryptedHybridReport { bytes }) - } -} - const TAG_SIZE: usize = TagSize::USIZE; #[derive(Clone, Debug)] @@ -1080,7 +961,7 @@ impl UniqueBytes for UniqueTag { } } -impl UniqueBytes for EncryptedHybridGeneralReport +impl UniqueBytes for EncryptedHybridReport where V: SharedValue, BK: SharedValue, @@ -1101,7 +982,7 @@ where } } -impl UniqueBytes for EncryptedHybridReport { +/*impl UniqueBytes for EncryptedHybridReport { /// We use the `TagSize` (the first 16 bytes of the ciphertext) for collision-detection /// See [analysis here for uniqueness](https://eprint.iacr.org/2019/624) fn unique_bytes(&self) -> [u8; TAG_SIZE] { @@ -1110,7 +991,7 @@ impl UniqueBytes for EncryptedHybridReport { array.copy_from_slice(slice); array } -} +}*/ impl UniqueTag { // Function to attempt to create a UniqueTag from a UniqueBytes implementor @@ -1196,9 +1077,9 @@ mod test { use typenum::Unsigned; use super::{ - EncryptedHybridGeneralReport, EncryptedHybridImpressionReport, GenericArray, - HybridConversionReport, HybridImpressionReport, HybridReport, UniqueTag, - UniqueTagValidator, HELPER_ORIGIN, IndistinguishableHybridReport + EncryptedHybridImpressionReport, EncryptedHybridReport, GenericArray, + HybridConversionReport, HybridImpressionReport, HybridReport, + IndistinguishableHybridReport, UniqueTag, UniqueTagValidator, HELPER_ORIGIN, }; use crate::{ error::Error, @@ -1257,12 +1138,11 @@ mod test { .encrypt(key_id, &key_registry, &mut rng) .unwrap(); - let hybrid_report2 = - EncryptedHybridGeneralReport::::decrypt_from_oprf_report_bytes::<_, BA20>( - enc_report_bytes.into(), - &key_registry, - ) - .unwrap(); + let hybrid_report2 = EncryptedHybridReport::::decrypt_from_oprf_report_bytes::< + _, + BA20, + >(enc_report_bytes.into(), &key_registry) + .unwrap(); assert_eq!(hybrid_report, hybrid_report2); } @@ -1285,12 +1165,11 @@ mod test { let enc_report_bytes = oprf_report .encrypt(key_id, &key_registry, &mut rng) .unwrap(); - let hybrid_report2 = - EncryptedHybridGeneralReport::::decrypt_from_oprf_report_bytes::<_, BA20>( - enc_report_bytes.into(), - &key_registry, - ) - .unwrap(); + let hybrid_report2 = EncryptedHybridReport::::decrypt_from_oprf_report_bytes::< + _, + BA20, + >(enc_report_bytes.into(), &key_registry) + .unwrap(); assert_eq!(hybrid_report, hybrid_report2); } @@ -1582,12 +1461,12 @@ mod test { enc_report_bytes2.splice(0..0, [0]); let enc_report2 = - EncryptedHybridGeneralReport::::from_bytes(enc_report_bytes2.into()).unwrap(); + EncryptedHybridReport::::from_bytes(enc_report_bytes2.into()).unwrap(); let enc_report3 = enc_report2.clone(); // Match first, then decrypt match enc_report2 { - EncryptedHybridGeneralReport::Impression(_) => panic!("Expected conversion report"), - EncryptedHybridGeneralReport::Conversion(enc_report_conv) => { + EncryptedHybridReport::Impression(_) => panic!("Expected conversion report"), + EncryptedHybridReport::Conversion(enc_report_conv) => { let dec_report2: HybridConversionReport = enc_report_conv.decrypt(&key_registry, &info).unwrap(); assert_eq!(dec_report2, hybrid_conversion_report); From c718db711eea7af5e5778c733d95c969e0497bdc Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Thu, 31 Oct 2024 02:47:24 -0700 Subject: [PATCH 10/14] cleanup --- ipa-core/src/report/hybrid.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index cf1fe461f..f1c6f8788 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -982,17 +982,6 @@ where } } -/*impl UniqueBytes for EncryptedHybridReport { - /// We use the `TagSize` (the first 16 bytes of the ciphertext) for collision-detection - /// See [analysis here for uniqueness](https://eprint.iacr.org/2019/624) - fn unique_bytes(&self) -> [u8; TAG_SIZE] { - let slice = &self.mk_ciphertext()[0..TAG_SIZE]; - let mut array = [0u8; TAG_SIZE]; - array.copy_from_slice(slice); - array - } -}*/ - impl UniqueTag { // Function to attempt to create a UniqueTag from a UniqueBytes implementor pub fn from_unique_bytes(item: &T) -> Self { @@ -1463,7 +1452,8 @@ mod test { let enc_report2 = EncryptedHybridReport::::from_bytes(enc_report_bytes2.into()).unwrap(); let enc_report3 = enc_report2.clone(); - // Match first, then decrypt + + // Case 1: Match first, then decrypt match enc_report2 { EncryptedHybridReport::Impression(_) => panic!("Expected conversion report"), EncryptedHybridReport::Conversion(enc_report_conv) => { @@ -1472,7 +1462,7 @@ mod test { assert_eq!(dec_report2, hybrid_conversion_report); } } - // Decrypt directly + // Case 2: Decrypt directly let dec_report3 = enc_report3 .decrypt(&key_registry, &HybridInfo::Conversion(info)) .unwrap(); From 12552d799a7a142c32e05f9b37ea9819b4d9bc30 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Fri, 1 Nov 2024 02:09:22 -0700 Subject: [PATCH 11/14] address PR feedback --- ipa-core/src/query/runner/hybrid.rs | 2 - ipa-core/src/report/hybrid.rs | 124 ++++++++++++++-------------- ipa-core/src/report/hybrid_info.rs | 17 ++-- 3 files changed, 74 insertions(+), 69 deletions(-) diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index a3b27eda3..a85461c01 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -176,8 +176,6 @@ mod tests { const EXPECTED: &[u128] = &[0, 8, 5]; fn build_records() -> Vec { - // TODO: When Encryption/Decryption exists for HybridReports - // update these to use that, rather than generating OprfReports vec![ /*TestRawDataRecord { timestamp: 0, diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index f1c6f8788..e78c25145 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -27,7 +27,7 @@ //! all secret sharings (including the sharings of zero), making the collection of reports //! cryptographically indistinguishable. -use std::{collections::HashSet, convert::Infallible, marker::PhantomData, ops::Add}; +use std::{collections::HashSet, convert::Infallible, iter::once, marker::PhantomData, ops::Add}; use bytes::{Buf, BufMut, Bytes}; use generic_array::{ArrayLength, GenericArray}; @@ -45,7 +45,7 @@ use crate::{ }, report::{ hybrid_info::{HybridConversionInfo, HybridImpressionInfo, HybridInfo}, - EncryptedOprfReport, EventType, KeyIdentifier, + EncryptedOprfReport, EventType as OprfEventType, KeyIdentifier, }, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, SharedValue}, sharding::ShardIndex, @@ -81,6 +81,29 @@ pub enum InvalidHybridReportError { WrongInfoType(&'static str), } +/// Event type as described [`ipa-issue`] +/// Initially we will just support trigger vs source event types but could extend to others in +/// the future. +/// +/// ['ipa-issue']: https://github.com/patcg-individual-drafts/ipa/issues/38 +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[repr(u8)] +pub enum HybridEventType { + Impression, + Conversion, +} + +impl TryFrom for HybridEventType { + type Error = InvalidHybridReportError; + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Impression), + 1 => Ok(Self::Conversion), + _ => Err(InvalidHybridReportError::UnknownEventType(value)), + } + } +} + /// Reports for impression events are represented here. #[derive(Clone, Debug, Eq, PartialEq)] pub struct HybridImpressionReport @@ -116,7 +139,6 @@ where let bk_sz = as Serializable>::Size::USIZE; let match_key = Replicated::::deserialize_infallible(GenericArray::from_slice(&buf[..mk_sz])); - //.map_err(|e| InvalidHybridReportError::DeserializationError("match_key", e.into()))?; let breakdown_key = Replicated::::deserialize(GenericArray::from_slice(&buf[mk_sz..mk_sz + bk_sz])) .map_err(|e| InvalidHybridReportError::DeserializationError("breakdown_key", e.into()))?; @@ -393,25 +415,17 @@ where rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { - match self { - HybridReport::Impression(impression_report) => match info { - HybridInfo::Impression(impression_info) =>{ - out.put_u16_le(self.encrypted_len()); - out.put_u8(1u8); - impression_report.encrypt_to(key_id, key_registry, impression_info, rng, out)}, - HybridInfo::Conversion(_) => { - Err(InvalidHybridReportError::WrongInfoType("Impression")) - } - }, - HybridReport::Conversion(conversion_report) => match info { - HybridInfo::Conversion(conversion_info) =>{ - out.put_u16_le(self.encrypted_len()); - out.put_u8(0u8); - conversion_report.encrypt_to(key_id, key_registry, conversion_info, rng, out)}, - HybridInfo::Impression(_) => { - Err(InvalidHybridReportError::WrongInfoType("Conversion")) - } - }, + match (self, info) { + (HybridReport::Impression(impression_report), HybridInfo::Impression(impression_info)) => { + out.put_u16_le(self.encrypted_len()); + out.put_u8(HybridEventType::Impression as u8); + impression_report.encrypt_to(key_id, key_registry, impression_info, rng, out)}, + (HybridReport::Conversion(conversion_report), HybridInfo::Conversion(conversion_info)) => { + out.put_u16_le(self.encrypted_len()); + out.put_u8(HybridEventType::Conversion as u8); + conversion_report.encrypt_to(key_id, key_registry, conversion_info, rng, out)}, + (HybridReport::Impression(_), _) => Err(InvalidHybridReportError::WrongInfoType("Impression")), + (HybridReport::Conversion(_), _) => Err(InvalidHybridReportError::WrongInfoType("Conversion")), } } @@ -427,16 +441,14 @@ where match self { HybridReport::Impression(impression_report) => match info { HybridInfo::Impression(impression_info) => - // Prepend a 1u8 byte to indicate this is an impression report - impression_report.encrypt(key_id, key_registry, impression_info, rng).map(|v| vec![1u8].into_iter().chain(v).collect()), + impression_report.encrypt(key_id, key_registry, impression_info, rng).map(|v| once(HybridEventType::Impression as u8).chain(v).collect()), HybridInfo::Conversion(_) => { Err(InvalidHybridReportError::WrongInfoType("Impression")) } }, HybridReport::Conversion(conversion_report) => match info { HybridInfo::Conversion(conversion_info) => - // Prepend a 0u8 byte to indicate this is a conversion report - conversion_report.encrypt(key_id, key_registry, conversion_info, rng).map(|v| vec![0u8].into_iter().chain(v).collect()), + conversion_report.encrypt(key_id, key_registry, conversion_info, rng).map(|v| once(HybridEventType::Conversion as u8).chain(v).collect()), HybridInfo::Impression(_) => { Err(InvalidHybridReportError::WrongInfoType("Conversion")) } @@ -457,7 +469,7 @@ where match self { HybridReport::Impression(impression_report) => match info { HybridInfo::Impression(impression_info) =>{ - out.put_u8(1u8); + out.put_u8(HybridEventType::Impression as u8); impression_report.encrypt_to(key_id, key_registry, impression_info, rng, out)}, HybridInfo::Conversion(_) => { Err(InvalidHybridReportError::WrongInfoType("Impression")) @@ -465,7 +477,7 @@ where }, HybridReport::Conversion(conversion_report) => match info { HybridInfo::Conversion(conversion_info) =>{ - out.put_u8(0u8); + out.put_u8(HybridEventType::Conversion as u8); conversion_report.encrypt_to(key_id, key_registry, conversion_info, rng, out)}, HybridInfo::Impression(_) => { Err(InvalidHybridReportError::WrongInfoType("Conversion")) @@ -496,16 +508,12 @@ where { const ENCAP_KEY_MK_OFFSET: usize = 0; const CIPHERTEXT_MK_OFFSET: usize = Self::ENCAP_KEY_MK_OFFSET + EncapsulationSize::USIZE; - const ENCAP_KEY_BTT_OFFSET: usize = (Self::CIPHERTEXT_MK_OFFSET - + TagSize::USIZE - //+ as Serializable>::Size::USIZE); - + Replicated::::size()); + const ENCAP_KEY_BTT_OFFSET: usize = + (Self::CIPHERTEXT_MK_OFFSET + TagSize::USIZE + Replicated::::size()); const CIPHERTEXT_BTT_OFFSET: usize = Self::ENCAP_KEY_BTT_OFFSET + EncapsulationSize::USIZE; - const KEY_IDENTIFIER_OFFSET: usize = (Self::CIPHERTEXT_BTT_OFFSET - + TagSize::USIZE - //+ as Serializable>::Size::USIZE); - + Replicated::::size()); + const KEY_IDENTIFIER_OFFSET: usize = + (Self::CIPHERTEXT_BTT_OFFSET + TagSize::USIZE + Replicated::::size()); const SITE_DOMAIN_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; pub fn encap_key_mk(&self) -> &[u8] { @@ -655,8 +663,6 @@ where type CTMKLength = Sum< as Serializable>::Size, TagSize>; type CTBTTLength = < as Serializable>::Size as Add>::Output; - println!("data: {:?}", self.data); - let mut ct_mk: GenericArray = *GenericArray::from_slice(self.mk_ciphertext()); let sk = key_registry @@ -830,22 +836,17 @@ where /// ## Errors /// If the report contents are invalid. pub fn from_bytes(mut bytes: Bytes) -> Result { - //let first_byte = bytes.next().ok_or(InvalidHybridReportError::Length(0,1))?; - //let the_rest = bytes[1..]; - match bytes[0] { - 1 => { - //let impression_report = EncryptedHybridImpressionReport::::from_bytes(&bytes[1..])?; + match HybridEventType::try_from(bytes[0])? { + HybridEventType::Impression => { bytes.advance(1); let impression_report = EncryptedHybridImpressionReport::::from_bytes(bytes)?; Ok(EncryptedHybridReport::Impression(impression_report)) } - 0 => { - //let conversion_report = EncryptedHybridConversionReport::::from_bytes(&bytes[1..])?; + HybridEventType::Conversion => { bytes.advance(1); let conversion_report = EncryptedHybridConversionReport::::from_bytes(bytes)?; Ok(EncryptedHybridReport::Conversion(conversion_report)) } - _ => Err(InvalidHybridReportError::UnknownEventType(bytes[0])), } } /// ## Errors @@ -884,11 +885,11 @@ where ) })?; match oprf_report.event_type { - EventType::Source => Ok(HybridReport::Impression(HybridImpressionReport { + OprfEventType::Source => Ok(HybridReport::Impression(HybridImpressionReport { match_key: oprf_report.match_key, breakdown_key: oprf_report.breakdown_key, })), - EventType::Trigger => Ok(HybridReport::Conversion(HybridConversionReport { + OprfEventType::Trigger => Ok(HybridReport::Conversion(HybridConversionReport { match_key: oprf_report.match_key, value: oprf_report.trigger_value, })), @@ -1078,14 +1079,17 @@ mod test { }, hpke::{KeyPair, KeyRegistry}, report::{ - hybrid::{EncryptedHybridConversionReport, NonAsciiStringError, BA64}, + hybrid::{EncryptedHybridConversionReport, HybridEventType, NonAsciiStringError, BA64}, hybrid_info::{HybridConversionInfo, HybridImpressionInfo, HybridInfo}, - EventType, OprfReport, + EventType as OprfEventType, OprfReport, }, secret_sharing::replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}, }; - fn build_oprf_report(event_type: EventType, rng: &mut ThreadRng) -> OprfReport { + fn build_oprf_report( + event_type: OprfEventType, + rng: &mut ThreadRng, + ) -> OprfReport { OprfReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), timestamp: AdditiveShare::new(rng.gen(), rng.gen()), @@ -1112,7 +1116,7 @@ mod test { fn convert_to_hybrid_impression_report() { let mut rng = thread_rng(); - let b = EventType::Source; + let b = OprfEventType::Source; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_report = HybridReport::Impression::(HybridImpressionReport:: { @@ -1140,7 +1144,7 @@ mod test { fn convert_to_hybrid_conversion_report() { let mut rng = thread_rng(); - let b = EventType::Trigger; + let b = OprfEventType::Trigger; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_report = HybridReport::Conversion::(HybridConversionReport:: { @@ -1248,7 +1252,7 @@ mod test { #[test] fn serialization_hybrid_impression() { let mut rng = thread_rng(); - let b = EventType::Source; + let b = OprfEventType::Source; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_impression_report = HybridImpressionReport:: { @@ -1270,7 +1274,7 @@ mod test { #[test] fn serialization_hybrid_conversion() { let mut rng = thread_rng(); - let b = EventType::Source; + let b = OprfEventType::Source; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_conversion_report = HybridConversionReport:: { @@ -1361,7 +1365,7 @@ mod test { #[test] fn enc_dec_roundtrip_hybrid_impression() { let mut rng = thread_rng(); - let b = EventType::Source; + let b = OprfEventType::Source; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_impression_report = HybridImpressionReport:: { @@ -1389,7 +1393,7 @@ mod test { #[test] fn enc_dec_roundtrip_hybrid_conversion() { let mut rng = thread_rng(); - let b = EventType::Trigger; + let b = OprfEventType::Trigger; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_conversion_report = HybridConversionReport:: { @@ -1419,7 +1423,7 @@ mod test { #[test] fn enc_report_serialization() { let mut rng = thread_rng(); - let b = EventType::Trigger; + let b = OprfEventType::Trigger; let oprf_report = build_oprf_report(b, &mut rng); let hybrid_conversion_report = HybridConversionReport:: { @@ -1446,8 +1450,8 @@ mod test { enc_report.decrypt(&key_registry, &info).unwrap(); assert_eq!(dec_report, hybrid_conversion_report); - // Prepend a 0 byte to the ciphertext to mark it as a ConversionReport - enc_report_bytes2.splice(0..0, [0]); + // Prepend a byte to the ciphertext to mark it as a ConversionReport + enc_report_bytes2.splice(0..0, [HybridEventType::Conversion as u8]); let enc_report2 = EncryptedHybridReport::::from_bytes(enc_report_bytes2.into()).unwrap(); diff --git a/ipa-core/src/report/hybrid_info.rs b/ipa-core/src/report/hybrid_info.rs index 30b1b04c4..785a5bc01 100644 --- a/ipa-core/src/report/hybrid_info.rs +++ b/ipa-core/src/report/hybrid_info.rs @@ -3,17 +3,20 @@ use crate::report::{hybrid::NonAsciiStringError, KeyIdentifier}; const DOMAIN: &str = "private-attribution"; #[derive(Clone, Debug)] -pub struct HybridImpressionInfo<'a> { +pub struct HybridImpressionInfo { pub key_id: KeyIdentifier, - pub helper_origin: &'a str, + pub helper_origin: &'static str, } -impl<'a> HybridImpressionInfo<'a> { +impl HybridImpressionInfo { /// Creates a new instance. /// /// ## Errors /// if helper or site origin is not a valid ASCII string. - pub fn new(key_id: KeyIdentifier, helper_origin: &'a str) -> Result { + pub fn new( + key_id: KeyIdentifier, + helper_origin: &'static str, + ) -> Result { // If the types of errors returned from this function change, then the validation in // `EncryptedReport::from_bytes` may need to change as well. if !helper_origin.is_ascii() { @@ -51,7 +54,7 @@ impl<'a> HybridImpressionInfo<'a> { #[derive(Clone, Debug)] pub struct HybridConversionInfo<'a> { pub key_id: KeyIdentifier, - pub helper_origin: &'a str, + pub helper_origin: &'static str, pub conversion_site_domain: &'a str, pub timestamp: u64, pub epsilon: f64, @@ -65,7 +68,7 @@ impl<'a> HybridConversionInfo<'a> { /// if helper or site origin is not a valid ASCII string. pub fn new( key_id: KeyIdentifier, - helper_origin: &'a str, + helper_origin: &'static str, conversion_site_domain: &'a str, timestamp: u64, epsilon: f64, @@ -124,6 +127,6 @@ impl<'a> HybridConversionInfo<'a> { #[derive(Clone, Debug)] pub enum HybridInfo<'a> { - Impression(HybridImpressionInfo<'a>), + Impression(HybridImpressionInfo), Conversion(HybridConversionInfo<'a>), } From 94c6f7537abeb7f90d0483eaab51f48a858f6490 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Fri, 1 Nov 2024 10:51:21 -0700 Subject: [PATCH 12/14] Change HybridInfo from enum to struct that has both types --- ipa-core/src/query/runner/hybrid.rs | 32 ++++------- ipa-core/src/report/hybrid.rs | 84 ++++++++++------------------- ipa-core/src/report/hybrid_info.rs | 34 ++++++++++-- 3 files changed, 70 insertions(+), 80 deletions(-) diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index a85461c01..503bbb04f 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -161,11 +161,7 @@ mod tests { }, hpke::{KeyPair, KeyRegistry}, query::runner::hybrid::Query as HybridQuery, - report::{ - hybrid::HybridReport, - hybrid_info::{HybridConversionInfo, HybridInfo}, - DEFAULT_KEY_ID, - }, + report::{hybrid::HybridReport, hybrid_info::HybridInfo, DEFAULT_KEY_ID}, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::{ flatten3v, ipa::TestRawDataRecord, Reconstruct, RoundRobinInputDistribution, TestWorld, @@ -177,7 +173,7 @@ mod tests { fn build_records() -> Vec { vec![ - /*TestRawDataRecord { + TestRawDataRecord { timestamp: 0, user_id: 12345, is_trigger_report: false, @@ -190,7 +186,7 @@ mod tests { is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - },*/ + }, TestRawDataRecord { timestamp: 10, user_id: 12345, @@ -205,13 +201,13 @@ mod tests { breakdown_key: 0, trigger_value: 2, }, - /*TestRawDataRecord { + TestRawDataRecord { timestamp: 20, user_id: 68362, is_trigger_report: false, breakdown_key: 1, trigger_value: 0, - },*/ + }, TestRawDataRecord { timestamp: 30, user_id: 68362, @@ -287,10 +283,8 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); - let hybrid_info = HybridInfo::Conversion( - HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(), - ); + let hybrid_info = + HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); let BufferAndKeyRegistry { buffers, @@ -357,10 +351,8 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); - let hybrid_info = HybridInfo::Conversion( - HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(), - ); + let hybrid_info = + HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); let BufferAndKeyRegistry { mut buffers, @@ -434,10 +426,8 @@ mod tests { const SHARDS: usize = 2; let records = build_records(); - let hybrid_info = HybridInfo::Conversion( - HybridConversionInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(), - ); + let hybrid_info = + HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); let BufferAndKeyRegistry { buffers, diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index e78c25145..d8179a75d 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -415,17 +415,17 @@ where rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { - match (self, info) { - (HybridReport::Impression(impression_report), HybridInfo::Impression(impression_info)) => { + match self { + HybridReport::Impression(impression_report) => { out.put_u16_le(self.encrypted_len()); out.put_u8(HybridEventType::Impression as u8); - impression_report.encrypt_to(key_id, key_registry, impression_info, rng, out)}, - (HybridReport::Conversion(conversion_report), HybridInfo::Conversion(conversion_info)) => { + impression_report.encrypt_to(key_id, key_registry, &info.impression, rng, out) + }, + HybridReport::Conversion(conversion_report) => { out.put_u16_le(self.encrypted_len()); out.put_u8(HybridEventType::Conversion as u8); - conversion_report.encrypt_to(key_id, key_registry, conversion_info, rng, out)}, - (HybridReport::Impression(_), _) => Err(InvalidHybridReportError::WrongInfoType("Impression")), - (HybridReport::Conversion(_), _) => Err(InvalidHybridReportError::WrongInfoType("Conversion")), + conversion_report.encrypt_to(key_id, key_registry, &info.conversion, rng, out) + }, } } @@ -439,19 +439,11 @@ where rng: &mut R, ) -> Result, InvalidHybridReportError> { match self { - HybridReport::Impression(impression_report) => match info { - HybridInfo::Impression(impression_info) => - impression_report.encrypt(key_id, key_registry, impression_info, rng).map(|v| once(HybridEventType::Impression as u8).chain(v).collect()), - HybridInfo::Conversion(_) => { - Err(InvalidHybridReportError::WrongInfoType("Impression")) - } + HybridReport::Impression(impression_report) => { + impression_report.encrypt(key_id, key_registry, &info.impression, rng).map(|v| once(HybridEventType::Impression as u8).chain(v).collect()) }, - HybridReport::Conversion(conversion_report) => match info { - HybridInfo::Conversion(conversion_info) => - conversion_report.encrypt(key_id, key_registry, conversion_info, rng).map(|v| once(HybridEventType::Conversion as u8).chain(v).collect()), - HybridInfo::Impression(_) => { - Err(InvalidHybridReportError::WrongInfoType("Conversion")) - } + HybridReport::Conversion(conversion_report) => { + conversion_report.encrypt(key_id, key_registry, &info.conversion, rng).map(|v| once(HybridEventType::Conversion as u8).chain(v).collect()) }, } } @@ -467,21 +459,13 @@ where out: &mut B, ) -> Result<(), InvalidHybridReportError> { match self { - HybridReport::Impression(impression_report) => match info { - HybridInfo::Impression(impression_info) =>{ + HybridReport::Impression(impression_report) =>{ out.put_u8(HybridEventType::Impression as u8); - impression_report.encrypt_to(key_id, key_registry, impression_info, rng, out)}, - HybridInfo::Conversion(_) => { - Err(InvalidHybridReportError::WrongInfoType("Impression")) - } + impression_report.encrypt_to(key_id, key_registry, &info.impression, rng, out) }, - HybridReport::Conversion(conversion_report) => match info { - HybridInfo::Conversion(conversion_info) =>{ + HybridReport::Conversion(conversion_report) => { out.put_u8(HybridEventType::Conversion as u8); - conversion_report.encrypt_to(key_id, key_registry, conversion_info, rng, out)}, - HybridInfo::Impression(_) => { - Err(InvalidHybridReportError::WrongInfoType("Conversion")) - } + conversion_report.encrypt_to(key_id, key_registry, &info.conversion, rng, out) }, } } @@ -907,22 +891,12 @@ where info: &HybridInfo, ) -> Result, InvalidHybridReportError> { match self { - EncryptedHybridReport::Impression(impression_report) => match info { - HybridInfo::Impression(impression_info) => Ok(HybridReport::Impression( - impression_report.decrypt(key_registry, impression_info)?, - )), - HybridInfo::Conversion(_) => { - Err(InvalidHybridReportError::WrongInfoType("Impression")) - } - }, - EncryptedHybridReport::Conversion(conversion_report) => match info { - HybridInfo::Conversion(conversion_info) => Ok(HybridReport::Conversion( - conversion_report.decrypt(key_registry, conversion_info)?, - )), - HybridInfo::Impression(_) => { - Err(InvalidHybridReportError::WrongInfoType("Conversion")) - } - }, + EncryptedHybridReport::Impression(impression_report) => Ok(HybridReport::Impression( + impression_report.decrypt(key_registry, &info.impression)?, + )), + EncryptedHybridReport::Conversion(conversion_report) => Ok(HybridReport::Conversion( + conversion_report.decrypt(key_registry, &info.conversion)?, + )), } } } @@ -1435,11 +1409,10 @@ mod test { let key_id = 0; let info = - HybridConversionInfo::new(key_id, HELPER_ORIGIN, "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(); + HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); let enc_report_bytes = hybrid_conversion_report - .encrypt(key_id, &key_registry, &info, &mut rng) + .encrypt(key_id, &key_registry, &info.conversion, &mut rng) .unwrap(); let mut enc_report_bytes2 = enc_report_bytes.clone(); @@ -1447,7 +1420,7 @@ mod test { let enc_report = EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.into()).unwrap(); let dec_report: HybridConversionReport = - enc_report.decrypt(&key_registry, &info).unwrap(); + enc_report.decrypt(&key_registry, &info.conversion).unwrap(); assert_eq!(dec_report, hybrid_conversion_report); // Prepend a byte to the ciphertext to mark it as a ConversionReport @@ -1461,15 +1434,14 @@ mod test { match enc_report2 { EncryptedHybridReport::Impression(_) => panic!("Expected conversion report"), EncryptedHybridReport::Conversion(enc_report_conv) => { - let dec_report2: HybridConversionReport = - enc_report_conv.decrypt(&key_registry, &info).unwrap(); + let dec_report2: HybridConversionReport = enc_report_conv + .decrypt(&key_registry, &info.conversion) + .unwrap(); assert_eq!(dec_report2, hybrid_conversion_report); } } // Case 2: Decrypt directly - let dec_report3 = enc_report3 - .decrypt(&key_registry, &HybridInfo::Conversion(info)) - .unwrap(); + let dec_report3 = enc_report3.decrypt(&key_registry, &info).unwrap(); assert_eq!( dec_report3, HybridReport::Conversion(hybrid_conversion_report) diff --git a/ipa-core/src/report/hybrid_info.rs b/ipa-core/src/report/hybrid_info.rs index 785a5bc01..5dc2950f2 100644 --- a/ipa-core/src/report/hybrid_info.rs +++ b/ipa-core/src/report/hybrid_info.rs @@ -126,7 +126,35 @@ impl<'a> HybridConversionInfo<'a> { } #[derive(Clone, Debug)] -pub enum HybridInfo<'a> { - Impression(HybridImpressionInfo), - Conversion(HybridConversionInfo<'a>), +pub struct HybridInfo<'a> { + pub impression: HybridImpressionInfo, + pub conversion: HybridConversionInfo<'a>, +} + +impl HybridInfo<'_> { + /// Creates a new instance. + /// ## Errors + /// if helper or site origin is not a valid ASCII string. + pub fn new( + key_id: KeyIdentifier, + helper_origin: &'static str, + conversion_site_domain: &'static str, + timestamp: u64, + epsilon: f64, + sensitivity: f64, + ) -> Result { + let impression = HybridImpressionInfo::new(key_id, helper_origin)?; + let conversion = HybridConversionInfo::new( + key_id, + helper_origin, + conversion_site_domain, + timestamp, + epsilon, + sensitivity, + )?; + Ok(Self { + impression, + conversion, + }) + } } From 4cc78c78f079a20f77b2a0cb38431f6aa9c54297 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Fri, 1 Nov 2024 12:05:52 -0700 Subject: [PATCH 13/14] switch to TestHybridRecord --- ipa-core/src/query/runner/hybrid.rs | 56 +++++++++---------------- ipa-core/src/test_fixture/hybrid.rs | 63 +++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index 503bbb04f..3cfdcf4ff 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -164,56 +164,38 @@ mod tests { report::{hybrid::HybridReport, hybrid_info::HybridInfo, DEFAULT_KEY_ID}, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::{ - flatten3v, ipa::TestRawDataRecord, Reconstruct, RoundRobinInputDistribution, TestWorld, - TestWorldConfig, WithShards, + flatten3v, hybrid::TestHybridRecord, Reconstruct, RoundRobinInputDistribution, + TestWorld, TestWorldConfig, WithShards, }, }; const EXPECTED: &[u128] = &[0, 8, 5]; - fn build_records() -> Vec { + fn build_records() -> Vec { vec![ - TestRawDataRecord { - timestamp: 0, - user_id: 12345, - is_trigger_report: false, + TestHybridRecord::TestImpression { + match_key: 12345, breakdown_key: 2, - trigger_value: 0, }, - TestRawDataRecord { - timestamp: 4, - user_id: 68362, - is_trigger_report: false, + TestHybridRecord::TestImpression { + match_key: 68362, breakdown_key: 1, - trigger_value: 0, }, - TestRawDataRecord { - timestamp: 10, - user_id: 12345, - is_trigger_report: true, - breakdown_key: 0, - trigger_value: 5, + TestHybridRecord::TestConversion { + match_key: 12345, + value: 5, }, - TestRawDataRecord { - timestamp: 12, - user_id: 68362, - is_trigger_report: true, - breakdown_key: 0, - trigger_value: 2, + TestHybridRecord::TestConversion { + match_key: 68362, + value: 2, }, - TestRawDataRecord { - timestamp: 20, - user_id: 68362, - is_trigger_report: false, + TestHybridRecord::TestImpression { + match_key: 68362, breakdown_key: 1, - trigger_value: 0, }, - TestRawDataRecord { - timestamp: 30, - user_id: 68362, - is_trigger_report: true, - breakdown_key: 1, - trigger_value: 7, + TestHybridRecord::TestConversion { + match_key: 68362, + value: 7, }, ] } @@ -225,7 +207,7 @@ mod tests { } fn build_buffers_from_records( - records: &[TestRawDataRecord], + records: &[TestHybridRecord], s: usize, info: &HybridInfo, ) -> BufferAndKeyRegistry { diff --git a/ipa-core/src/test_fixture/hybrid.rs b/ipa-core/src/test_fixture/hybrid.rs index 3b8cc2460..a28fa7232 100644 --- a/ipa-core/src/test_fixture/hybrid.rs +++ b/ipa-core/src/test_fixture/hybrid.rs @@ -1,8 +1,17 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + iter::zip, +}; use crate::{ - ff::{boolean_array::BooleanArray, U128Conversions}, - report::hybrid::IndistinguishableHybridReport, + ff::{ + boolean_array::{BooleanArray, BA64}, + U128Conversions, + }, + rand::Rng, + report::hybrid::{ + HybridConversionReport, HybridImpressionReport, HybridReport, IndistinguishableHybridReport, + }, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, IntoShares}, test_fixture::sharing::Reconstruct, }; @@ -51,6 +60,54 @@ where } } +impl IntoShares> for TestHybridRecord +where + BK: BooleanArray + U128Conversions + IntoShares>, + V: BooleanArray + U128Conversions + IntoShares>, +{ + fn share_with(self, rng: &mut R) -> [HybridReport; 3] { + match self { + TestHybridRecord::TestImpression { + match_key, + breakdown_key, + } => { + let ba_match_key = BA64::try_from(u128::from(match_key)) + .unwrap() + .share_with(rng); + let ba_breakdown_key = BK::try_from(u128::from(breakdown_key)) + .unwrap() + .share_with(rng); + zip(ba_match_key, ba_breakdown_key) + .map(|(match_key_share, breakdown_key_share)| { + HybridReport::Impression::(HybridImpressionReport { + match_key: match_key_share, + breakdown_key: breakdown_key_share, + }) + }) + .collect::>() + .try_into() + .unwrap() + } + TestHybridRecord::TestConversion { match_key, value } => { + let ba_match_key = BA64::try_from(u128::from(match_key)) + .unwrap() + .share_with(rng); + let ba_value = V::try_from(u128::from(value)).unwrap().share_with(rng); + zip(ba_match_key, ba_value) + .map(|(match_key_share, value_share)| { + HybridReport::Conversion::(HybridConversionReport { + match_key: match_key_share, + value: value_share, + }) + }) + .collect::>() + .try_into() + .unwrap() + } + } + } +} + struct HashmapEntry { breakdown_key: u32, total_value: u32, From b597f3922f3a2285274a37d7d53496f947cb5f59 Mon Sep 17 00:00:00 2001 From: Thomas James Yurek Date: Fri, 1 Nov 2024 12:21:45 -0700 Subject: [PATCH 14/14] remove HybridReport sharing for TestRawDataRecord --- ipa-core/src/test_fixture/input/sharing.rs | 46 +--------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/ipa-core/src/test_fixture/input/sharing.rs b/ipa-core/src/test_fixture/input/sharing.rs index 8041a362e..14dddd0db 100644 --- a/ipa-core/src/test_fixture/input/sharing.rs +++ b/ipa-core/src/test_fixture/input/sharing.rs @@ -8,10 +8,7 @@ use crate::{ }, protocol::ipa_prf::OPRFIPAInputRow, rand::Rng, - report::{ - hybrid::{HybridConversionReport, HybridImpressionReport, HybridReport}, - EventType, OprfReport, - }, + report::{EventType, OprfReport}, secret_sharing::{ replicated::{semi_honest::AdditiveShare as Replicated, ReplicatedSecretSharing}, IntoShares, @@ -72,47 +69,6 @@ where } } -impl IntoShares> for TestRawDataRecord -where - BK: BooleanArray + U128Conversions + IntoShares>, - V: BooleanArray + U128Conversions + IntoShares>, -{ - fn share_with(self, rng: &mut R) -> [HybridReport; 3] { - let match_key = BA64::try_from(u128::from(self.user_id)) - .unwrap() - .share_with(rng); - let breakdown_key = BK::try_from(self.breakdown_key.into()) - .unwrap() - .share_with(rng); - let trigger_value = V::try_from(self.trigger_value.into()) - .unwrap() - .share_with(rng); - if self.is_trigger_report { - zip(match_key, trigger_value) - .map(|(match_key_share, trigger_value_share)| { - HybridReport::Conversion::(HybridConversionReport { - match_key: match_key_share, - value: trigger_value_share, - }) - }) - .collect::>() - .try_into() - .unwrap() - } else { - zip(match_key, breakdown_key) - .map(|(match_key_share, breakdown_key_share)| { - HybridReport::Impression::(HybridImpressionReport { - match_key: match_key_share, - breakdown_key: breakdown_key_share, - }) - }) - .collect::>() - .try_into() - .unwrap() - } - } -} - impl IntoShares> for TestRawDataRecord where BK: BooleanArray + U128Conversions + IntoShares>,