Merge branch 'prf_eval_two' of https://github.com/danielmasny/ipa int…

…o prf_eval_two
private-attribution · Nov 2, 2023 · 117ac61 · 117ac61
2 parents f640088 + 6c715ff
commit 117ac61
Show file tree

Hide file tree

Showing 75 changed files with 4,291 additions and 720 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -0,0 +1,2 @@
+[target.aarch64-apple-darwin]
+rustflags = ["--cfg", "aes_armv8"]
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -14,6 +14,7 @@ on:
       - "src/**/*"
       - "benches/**/*"
       - "tests/**/*"
+  workflow_dispatch:
 
 env:
   CARGO_TERM_COLOR: always
@@ -69,13 +70,11 @@ jobs:
       - name: Run compact gate tests
         run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
 
-  extra:
-    name: Additional Builds and Concurrency Tests
-    env:
-      RUSTFLAGS: -D warnings -C target-cpu=native
-
+  release:
+    name: Release builds and tests
     runs-on: ubuntu-latest
-
+    env:
+      RUSTFLAGS: -C target-cpu=native
     steps:
       - uses: actions/checkout@v3
 
@@ -96,34 +95,22 @@ jobs:
       - name: Release Build
         run: cargo build --release
 
-      - name: Build benchmarks
-        run: cargo build --benches --no-default-features --features "enable-benches descriptive-gate"
-
       - name: Build concurrency tests
         run: cargo build --release --features shuttle
 
-      - name: Build concurrency tests (debug mode)
-        run: cargo build --features shuttle
-
       - name: Run concurrency tests
         run: cargo test --release --features shuttle
 
-      - name: Run IPA bench
-        run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches descriptive-gate"
-
-      - name: Run arithmetic bench
-        run: cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate"
-
-      - name: Run compact gate tests
-        run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
-
-  aggregate:
-    name: Build and test aggregate circuit
+  extra:
+    name: Additional Builds and Concurrency Tests
+    env:
+      RUSTFLAGS: -D warnings -C target-cpu=native
 
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
+
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy,rustfmt
@@ -138,6 +125,35 @@ jobs:
             target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
 
-      - name: Web Tests
-        run: cargo test --no-default-features --features "aggregate-circuit cli web-app real-world-infra test-fixture descriptive-gate"
+      - name: Build benchmarks
+        run: cargo build --benches --no-default-features --features "enable-benches descriptive-gate"
+
+      - name: Build concurrency tests (debug mode)
+        run: cargo build --features shuttle
+
+      - name: Run IPA bench
+        run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches descriptive-gate"
+
+      - name: Run arithmetic bench
+        run: cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate"
+
+      - name: Run compact gate tests
+        run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+
+  # sanitizers currently require nightly https://github.com/rust-lang/rust/issues/39699
+  sanitize:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [ address, leak ]
+    env:
+      TARGET: x86_64-unknown-linux-gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@nightly
+      - name: Add Rust sources
+        run: rustup component add rust-src
+      - name: Run tests with sanitizer
+        run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test -Z build-std --target $TARGET --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate"
 
diff --git a/0001-CI-split-extra-step-into-two.patch b/0001-CI-split-extra-step-into-two.patch
diff --git a/Cargo.toml b/Cargo.toml
@@ -12,14 +12,15 @@ default = [
     "tracing/max_level_trace",
     "tracing/release_max_level_info",
     "descriptive-gate",
-    "aggregate-circuit"
+    "aggregate-circuit",
+    "ipa-prf"
 ]
 cli = ["comfy-table", "clap"]
 enable-serde = ["serde", "serde_json"]
 disable-metrics = []
 # TODO move web-app to a separate crate. It adds a lot of build time to people who mostly write protocols
 # TODO Consider moving out benches as well
-web-app = ["axum", "axum-server", "base64", "clap", "comfy-table", "enable-serde", "hyper", "hyper-rustls", "rcgen", "rustls-pemfile", "time", "tokio-rustls", "toml", "tower", "tower-http"]
+web-app = ["axum", "axum-server", "base64", "clap", "comfy-table", "enable-serde", "hyper", "hyper-rustls", "rcgen", "rustls", "rustls-pemfile", "time", "tokio-rustls", "toml", "tower", "tower-http"]
 test-fixture = ["enable-serde", "weak-field"]
 shuttle = ["shuttle-crate", "test-fixture"]
 debug-trace = ["tracing/max_level_trace", "tracing/release_max_level_debug"]
@@ -41,10 +42,10 @@ compact-gate = ["ipa-macros/compact-gate"]
 # Standalone aggregation protocol. We use IPA infra for communication
 # but it has nothing to do with IPA.
 aggregate-circuit = []
+ipa-prf = ["descriptive-gate"]
 
 [dependencies]
-ipa-macros = { version = "*", path = "./ipa-macros" }
-aes = "0.8"
+aes = "0.8.3"
 async-trait = "0.1.68"
 axum = { version = "0.5.17", optional = true, features = ["http2"] }
 axum-server = { version = "0.5.1", optional = true, features = ["rustls", "rustls-pemfile", "tls-rustls"] }
@@ -55,46 +56,52 @@ clap = { version = "4.3.2", optional = true, features = ["derive"] }
 comfy-table = { version = "7.0", optional = true }
 config = "0.13.2"
 criterion = { version = "0.5.1", optional = true, default-features = false, features = ["async_tokio", "plotters", "html_reports"] }
+curve25519-dalek = "4.1.1"
 dashmap = "5.4"
 dhat = "0.3.2"
 embed-doc-image = "0.1.4"
 futures = "0.3.28"
 futures-util = "0.3.28"
-generic-array = "0.14.7"
+generic-array = "1.0.0"
 hex = { version = "0.4", features = ["serde"] }
 hkdf = "0.12.3"
-hpke = { version = "0.10.0", default-features = false, features = ["std", "x25519-dalek"] }
+hpke = { version = "0.11.0", default-features = false, features = ["std", "x25519"] }
 hyper = { version = "0.14.26", optional = true, features = ["client", "h2", "stream", "runtime"] }
-hyper-rustls = { version = "0.24.0", optional = true, features = ["http2"] }
+hyper-rustls = { version = "0.24.1", optional = true, features = ["http2"] }
 iai = { version = "0.1.1", optional = true }
+ipa-macros = { version = "*", path = "./ipa-macros" }
 metrics = "0.21.0"
 metrics-tracing-context = "0.14.0"
 metrics-util = { version = "0.15.0" }
 once_cell = "1.18"
 pin-project = "1.0"
 rand = "0.8"
 rand_core = "0.6"
-rcgen = { version = "0.10", optional = true }
-rustls = "0.21.5"
+rcgen = { version = "0.11.3", optional = true }
+rustls = { version = "0.21", optional = true }
 rustls-pemfile = { version = "1", optional = true }
+# TODO: https://rustsec.org/advisories/RUSTSEC-2023-0053. tokio-rustls and hyper-rustls need to be upgraded first, before
+# we can remove pinning
+rustls-webpki = "^0.101.4"
 # TODO consider using zerocopy or serde_bytes or in-house serialization
 serde = { version = "1.0", optional = true, features = ["derive"] }
 serde_json = { version = "1.0", optional = true }
-sha2 = "0.10.6"
+sha2 = "0.10"
 shuttle-crate = { package = "shuttle", version = "0.6.1", optional = true }
 thiserror = "1.0"
 time = { version = "0.3", optional = true }
 tokio = { version = "1.28", features = ["rt", "rt-multi-thread", "macros"] }
-tokio-rustls = { version = "0.24.0", optional = true }
+# TODO: axum-server holds onto 0.24 and we can't upgrade until they do. Or we move away from axum-server
+tokio-rustls = { version = "0.24", optional = true }
 tokio-stream = "0.1.14"
-toml = { version = "0.7", optional = true }
+toml = { version = "0.8", optional = true }
 tower = { version = "0.4.13", optional = true }
 tower-http = { version = "0.4.0", optional = true, features = ["trace"] }
-tracing = "0.1.37"
-tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 typenum = "1.16"
 # hpke is pinned to it
-x25519-dalek = "2.0.0-pre.0"
+x25519-dalek = "2.0.0-rc.3"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = "0.5.0"
@@ -106,9 +113,9 @@ cfg_aliases = "0.1.1"
 command-fds = "0.2.2"
 hex = "0.4"
 permutation = "0.4.1"
-proptest = "1.0.0"
+proptest = "1"
+rustls = { version = "0.21", features = ["dangerous_configuration"] }
 tempfile = "3"
-tokio-rustls = { version = "0.24.0", features = ["dangerous_configuration"] }
 
 [profile.release]
 incremental = true

diff --git a/benches/oneshot/ipa.rs b/benches/oneshot/ipa.rs
@@ -9,7 +9,7 @@ use ipa::{
     ff::Fp32BitPrime,
     helpers::{query::IpaQueryConfig, GatewayConfig},
     test_fixture::{
-        ipa::{ipa_in_the_clear, test_ipa, IpaSecurityModel},
+        ipa::{ipa_in_the_clear, test_ipa, test_oprf_ipa, CappingOrder, IpaSecurityModel},
         EventGenerator, EventGeneratorConfig, TestWorld, TestWorldConfig,
     },
 };
@@ -70,6 +70,8 @@ struct Args {
     /// Needed for benches.
     #[arg(long, hide = true)]
     bench: bool,
+    #[arg(short = 'o', long)]
+    oprf: bool,
 }
 
 impl Args {
@@ -121,25 +123,35 @@ async fn run(args: Args) -> Result<(), Error> {
     .take(args.query_size)
     .collect::<Vec<_>>();
 
+    let order = if args.oprf {
+        CappingOrder::CapMostRecentFirst
+    } else {
+        CappingOrder::CapOldestFirst
+    };
     let expected_results = ipa_in_the_clear(
         &raw_data,
         args.per_user_cap,
         args.attribution_window(),
         args.breakdown_keys,
+        &order,
     );
 
     let world = TestWorld::new_with(config.clone());
     tracing::trace!("Preparation complete in {:?}", _prep_time.elapsed());
 
     let _protocol_time = Instant::now();
-    test_ipa::<BenchField>(
-        &world,
-        &raw_data,
-        &expected_results,
-        args.config(),
-        args.mode,
-    )
-    .await;
+    if args.oprf {
+        test_oprf_ipa::<BenchField>(&world, raw_data, &expected_results, args.config()).await;
+    } else {
+        test_ipa::<BenchField>(
+            &world,
+            &raw_data,
+            &expected_results,
+            args.config(),
+            args.mode,
+        )
+        .await;
+    }
     tracing::trace!(
         "{m:?} IPA for {q} records took {t:?}",
         m = args.mode,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		[target.aarch64-apple-darwin]
		rustflags = ["--cfg", "aes_armv8"]