Merge branch 'main' into decrypt

.clippy.toml

@@ -7,3 +7,5 @@ disallowed-methods = [
   { path = "std::mem::ManuallyDrop::new", reason = "Not running the destructors on futures created inside seq_join module will cause UB in IPA. Make sure you don't leak any of those." },
   { path = "std::vec::Vec::leak", reason = "Not running the destructors on futures created inside seq_join module will cause UB in IPA. Make sure you don't leak any of those." },
 ]
+
+future-size-threshold = 10240

.github/workflows/check.yml

@@ -53,7 +53,7 @@ jobs:
 
       - name: Clippy
         if: ${{ success() || failure() }}
-        run: cargo clippy --tests
+        run: cargo clippy --features "cli test-fixture" --tests
 
       - name: Clippy concurrency tests
         if: ${{ success() || failure() }}
@@ -68,17 +68,14 @@ jobs:
         run: cargo build --tests
 
       - name: Run tests
-        run: cargo test
+        run: cargo test --features "cli test-fixture relaxed-dp"
 
       - name: Run tests with multithreading feature enabled
         run: cargo test --features "multi-threading"
 
       - name: Run Web Tests
         run: cargo test -p ipa-core --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
 
-      - name: Run Integration Tests
-        run: cargo test --test encrypted_input --features "cli test-fixture web-app in-memory-infra"
-
   release:
     name: Release builds and tests
     runs-on: ubuntu-latest
@@ -148,8 +145,11 @@ jobs:
       - name: Run arithmetic bench
         run: cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches compact-gate"
 
-      - name: Run compact gate tests
+      - name: Run compact gate tests for HTTP stack
         run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+
+      - name: Run in-memory compact gate tests
+        run: cargo test --features "compact-gate"
   slow:
     name: Slow tests
     env:
@@ -172,8 +172,17 @@ jobs:
             target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
 
-      - name: End-to-end tests
-        run: cargo test --release --test "*" --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+      - name: Integration Tests - Compact Gate
+        run: cargo test --release --test "compact_gate" --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+
+      - name: Integration Tests - Helper Networks
+        run: cargo test --release --test "helper_networks" --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+
+      - name: Integration Tests - Hybrid
+        run: cargo test --release --test "hybrid" --features "cli test-fixture"
+
+      - name: Integration Tests - IPA with Relaxed DP
+        run: cargo test --release --test "ipa_with_relaxed_dp" --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate relaxed-dp"
 
   # sanitizers currently require nightly https://github.com/rust-lang/rust/issues/39699
   sanitize:
@@ -192,7 +201,7 @@ jobs:
       - name: Add Rust sources
         run: rustup component add rust-src
       - name: Run tests with sanitizer
-        run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test -Z build-std --target $TARGET --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate ${{ matrix.features }}"
+        run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test -Z build-std -p ipa-core --target $TARGET --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate ${{ matrix.features }}"
 
   miri:
     runs-on: ubuntu-latest
@@ -236,4 +245,3 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ipa.cov
           fail_ci_if_error: false
-

.github/workflows/docker.yml

@@ -34,7 +34,9 @@ jobs:
           type=sha
 
     - name: "Setup Docker Buildx"
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
+      with:
+        platforms: linux/amd64
 
     - name: "Login to GitHub Container Registry"
       uses: docker/login-action@v2
@@ -44,10 +46,10 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: "Build and Publish Helper Image"
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v6
       with:
         context: .
-        file: ./docker/ci/helper.Dockerfile
+        file: ./docker/helper.Dockerfile
         push: true
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -11,3 +11,4 @@
 /in-market-test/hpke/bin
 /in-market-test/hpke/lib
 /in-market-test/hpke/pyvenv.cfg
+input-data-*.txt

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["ipa-core", "ipa-step", "ipa-step-derive", "ipa-step-test"]
+members = ["ipa-core", "ipa-step", "ipa-step-derive", "ipa-step-test", "ipa-metrics", "ipa-metrics-tracing"]
 
 [profile.release]
 incremental = true

docker/helper.Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 ARG SOURCES_DIR=/usr/src/ipa
-FROM rust:bullseye AS builder
+FROM rust:bookworm AS builder
 ARG SOURCES_DIR
 
 # Prepare helper binaries
@@ -10,7 +10,7 @@ RUN set -eux; \
     cargo build --bin helper --release --no-default-features --features "web-app real-world-infra compact-gate"
 
 # Copy them to the final image
-FROM debian:bullseye-slim
+FROM rust:slim-bookworm
 ENV HELPER_BIN_PATH=/usr/local/bin/ipa-helper
 ENV CONF_DIR=/etc/ipa
 ARG SOURCES_DIR

ipa-core/Cargo.toml

@@ -1,7 +1,12 @@
 [package]
 name = "ipa-core"
 version = "0.1.0"
-rust-version = "1.80.0"
+# When updating the rust version:
+# 1. Check at https://hub.docker.com/_/rust that the relevant version of the
+#    rust:slim-bullseye docker image is available.
+# 2. Update the rust version used for draft in
+#    https://github.com/private-attribution/draft/blob/main/sidecar/ansible/provision.yaml.
+rust-version = "1.82.0"
 edition = "2021"
 build = "build.rs"
 
@@ -16,11 +21,14 @@ default = [
     "stall-detection",
     "aggregate-circuit",
     "ipa-prf",
-    "ipa-step/string-step",
+    "descriptive-gate",
 ]
 cli = ["comfy-table", "clap"]
-# Enabling compact gates disables any tests that rely on descriptive gates.
-compact-gate = ["ipa-step/string-step"]
+# Enable compact gate optimization
+compact-gate = []
+# mutually exclusive with compact-gate and disables compact gate optimization.
+# It is enabled by default
+descriptive-gate = ["ipa-step/string-step"]
 disable-metrics = []
 # TODO move web-app to a separate crate. It adds a lot of build time to people who mostly write protocols
 # TODO Consider moving out benches as well
@@ -74,6 +82,8 @@ reveal-aggregation = []
 aggregate-circuit = []
 # IPA protocol based on OPRF
 ipa-prf = []
+# relaxed DP, off by default
+relaxed-dp = []
 
 [dependencies]
 ipa-step = { version = "*", path = "../ipa-step" }
@@ -85,16 +95,14 @@ async-scoped = { version = "0.9.0", features = ["use-tokio"], optional = true }
 axum = { version = "0.7.5", optional = true, features = ["http2", "macros"] }
 # The following is a temporary version until we can stabilize the build on a higher version
 # of axum, rustls and the http stack.
-axum-server = { git = "https://github.com/cberkhoff/axum-server/", branch = "0.6.1", version = "0.6.1", optional = true, features = [
-    "tls-rustls",
-] }
+axum-server = { version = "0.7.1", optional = true, features = ["tls-rustls"] }
 base64 = { version = "0.21.2", optional = true }
 bitvec = "1.0"
 bytes = "1.4"
 clap = { version = "4.3.2", optional = true, features = ["derive"] }
 comfy-table = { version = "7.0", optional = true }
 config = "0.14"
-console-subscriber = {  version = "0.2", optional = true }
+console-subscriber = {  version = "0.4", optional = true }
 criterion = { version = "0.5.1", optional = true, default-features = false, features = [
     "async_tokio",
     "plotters",
@@ -150,7 +158,7 @@ typenum = { version = "1.17", features = ["i128"] }
 # hpke is pinned to it
 x25519-dalek = "2.0.0-rc.3"
 
-[target.'cfg(not(target_env = "msvc"))'.dependencies]
+[target.'cfg(all(not(target_env = "msvc"), not(target_os = "macos")))'.dependencies]
 tikv-jemallocator = "0.5.0"
 
 [build-dependencies]
@@ -196,7 +204,12 @@ bench = false
 
 [[bin]]
 name = "crypto_util"
-required-features = ["cli", "test-fixture", "web-app", "in-memory-infra"]
+required-features = ["cli", "test-fixture", "web-app"]
+bench = false
+
+[[bin]]
+name = "in_the_clear"
+required-features = ["cli", "test-fixture", "web-app"]
 bench = false
 
 [[bench]]
@@ -252,3 +265,21 @@ required-features = [
     "real-world-infra",
     "test-fixture",
 ]
+
+[[test]]
+name = "ipa_with_relaxed_dp"
+required-features = [
+    "cli",
+    "compact-gate",
+    "web-app",
+    "real-world-infra",
+    "test-fixture",
+    "relaxed-dp",
+]
+
+[[test]]
+name = "hybrid"
+required-features = [
+    "test-fixture",
+    "cli",
+]

ipa-core/benches/oneshot/ipa.rs

@@ -19,7 +19,11 @@ use ipa_step::StepNarrow;
 use rand::{random, rngs::StdRng, SeedableRng};
 use tokio::runtime::Builder;
 
-#[cfg(all(not(target_env = "msvc"), not(feature = "dhat-heap")))]
+#[cfg(all(
+    not(target_env = "msvc"),
+    not(feature = "dhat-heap"),
+    not(target_os = "macos")
+))]
 #[global_allocator]
 static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
@@ -82,6 +86,7 @@ impl Args {
         self.active_work
             .map(NonZeroUsize::get)
             .unwrap_or_else(|| self.query_size.clamp(16, 1024))
+            .next_power_of_two()
     }
 
     fn attribution_window(&self) -> Option<NonZeroU32> {