Merge pull request #961 from martinthomson/compact-step-reprise

.github/workflows/check.yml

@@ -61,7 +61,7 @@ jobs:
 
       - name: Clippy web
         if: ${{ success() || failure() }}
-        run: cargo clippy --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate"
+        run: cargo clippy --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
 
       - name: Build
         if: ${{ success() || failure() }}
@@ -74,7 +74,7 @@ jobs:
         run: cargo test --features "multi-threading"
 
       - name: Run Web Tests
-        run: cargo test -p ipa-core --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate"
+        run: cargo test -p ipa-core --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
 
   release:
     name: Release builds and tests
@@ -134,20 +134,19 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
 
       - name: Build benchmarks
-        run: cargo build --benches --no-default-features --features "enable-benches descriptive-gate"
+        run: cargo build --benches --no-default-features --features "enable-benches compact-gate"
 
       - name: Build concurrency tests (debug mode)
         run: cargo build --features shuttle
 
       - name: Run IPA bench
-        run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches descriptive-gate"
+        run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches compact-gate"
 
       - name: Run arithmetic bench
-        run: cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches descriptive-gate"
+        run: cargo bench --bench oneshot_arithmetic --no-default-features --features "enable-benches compact-gate"
 
-# disabled until 961 is merged
-#      - name: Run compact gate tests
-#        run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+      - name: Run compact gate tests
+        run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
 
   # sanitizers currently require nightly https://github.com/rust-lang/rust/issues/39699
   sanitize:
@@ -166,7 +165,7 @@ jobs:
       - name: Add Rust sources
         run: rustup component add rust-src
       - name: Run tests with sanitizer
-        run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test -Z build-std --target $TARGET --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate ${{ matrix.features }}"
+        run: RUSTFLAGS="-Z sanitizer=${{ matrix.sanitizer }} -Z sanitizer-memory-track-origins" cargo test -Z build-std --target $TARGET --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate ${{ matrix.features }}"
 
   miri:
     runs-on: ubuntu-latest

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["ipa-core", "ipa-macros"]
+members = ["ipa-core", "ipa-step", "ipa-step-derive", "ipa-step-test"]
 
 [profile.release]
 incremental = true

README.md

@@ -75,28 +75,28 @@ cargo test
 There are a handful of benchmarks which can be run, but `oneshot_ipa` will run the whole protocol locally. On a M1 Macbook Pro, this takes a couple minutes.
 
 ```
-cargo bench --bench oneshot_ipa --no-default-features --features="enable-benches descriptive-gate"
+cargo bench --bench oneshot_ipa --no-default-features --features="enable-benches compact-gate"
 ```
 
 Other benchmarks you can run:
 
 **Sorting**:
 ```
-cargo bench --bench oneshot_sort --no-default-features --features="enable-benches descriptive-gate"
+cargo bench --bench oneshot_sort --no-default-features --features="enable-benches compact-gate"
 ```
 
 **Arithmetic gates**:
 ```
-cargo bench --bench oneshot_arithmetic --no-default-features --features="enable-benches descriptive-gate" -- --width 1 --depth 1
+cargo bench --bench oneshot_arithmetic --no-default-features --features="enable-benches compact-gate" -- --width 1 --depth 1
 ```
 You can adjust the width and depth of the gates at the expense of a longer benchmarking run.
 
 **Other**:
 ```
-cargo bench --bench criterion_arithmetic --no-default-features --features="enable-benches descriptive-gate"
+cargo bench --bench criterion_arithmetic --no-default-features --features="enable-benches compact-gate"
 ```
 
 ```
-cargo bench --bench iai_arithmetic --no-default-features --features="enable-benches descriptive-gate"
+cargo bench --bench iai_arithmetic --no-default-features --features="enable-benches compact-gate"
 ```
 (NOTE: This benchmark only works on Linux. If you are on macOS, an error is expected.)

ipa-core/Cargo.toml

@@ -3,6 +3,7 @@ name = "ipa-core"
 version = "0.1.0"
 rust-version = "1.77.0"
 edition = "2021"
+build = "build.rs"
 
 [features]
 default = [
@@ -11,13 +12,15 @@ default = [
     # by default remove all TRACE, DEBUG spans from release builds
     "tracing/max_level_trace",
     "tracing/release_max_level_info",
-    "descriptive-gate",
     "aggregate-circuit",
     "stall-detection",
     "aggregate-circuit",
     "ipa-prf",
+    "ipa-step/string-step",
 ]
 cli = ["comfy-table", "clap"]
+# Enabling compact gates disables any tests that rely on descriptive gates.
+compact-gate = ["ipa-step/string-step"]
 disable-metrics = []
 # TODO move web-app to a separate crate. It adds a lot of build time to people who mostly write protocols
 # TODO Consider moving out benches as well
@@ -54,11 +57,7 @@ real-world-infra = []
 dhat-heap = ["cli", "test-fixture"]
 # Enable this feature to enable our colossally weak Fp31.
 weak-field = []
-step-trace = ["descriptive-gate"]
-# The following two features are mutually exclusive. Descriptive should be enabled by deafult as the vast majority
-# of unit tests use it. Compact uses memory-efficient gates and is suitable for production.
-descriptive-gate = []
-compact-gate = ["ipa-macros/compact-gate"]
+step-trace = ["ipa-step/trace"]
 # Enable using more than one thread for protocol execution. Most of the parallelism occurs at parallel/seq_join operations
 multi-threading = ["async-scoped"]
 # Enable tokio task profiling. Requires tokio_unstable flag to be passed to the compiler.
@@ -73,7 +72,8 @@ aggregate-circuit = []
 ipa-prf = []
 
 [dependencies]
-ipa-macros = { version = "*", path = "../ipa-macros" }
+ipa-step = { version = "*", path = "../ipa-step" }
+ipa-step-derive = { version = "*", path = "../ipa-step-derive" }
 
 aes = "0.8.3"
 async-trait = "0.1.79"
@@ -133,7 +133,7 @@ rustls-pemfile = { version = "2.1.2", optional = true }
 rustls-pki-types = "1.4.1"
 # TODO consider using zerocopy or serde_bytes or in-house serialization
 serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0" }
+serde_json = "1.0"
 sha2 = "0.10"
 shuttle-crate = { package = "shuttle", version = "0.6.1", optional = true }
 thiserror = "1.0"
@@ -155,15 +155,20 @@ tikv-jemallocator = "0.5.0"
 
 [build-dependencies]
 cfg_aliases = "0.1.1"
+ipa-step = { version = "*", path = "../ipa-step", features = ["build"] }
+ipa-step-derive = { version = "*", path = "../ipa-step-derive", features = ["build"] }
+serde = { version = "1.0", features = ["derive"] }
 
 [dev-dependencies]
 command-fds = "0.2.2"
 hex = "0.4"
+ipa-step = { version = "*", path = "../ipa-step", features = ["build", "string-step"] }
 permutation = "0.4.1"
 proptest = "1.4"
 rustls = { version = "0.23" }
 tempfile = "3"
 
+
 [lib]
 path = "src/lib.rs"
 bench = false
@@ -193,25 +198,25 @@ bench = false
 name = "criterion_arithmetic"
 path = "benches/ct/arithmetic_circuit.rs"
 harness = false
-required-features = ["enable-benches", "descriptive-gate"]
+required-features = ["enable-benches"]
 
 [[bench]]
 name = "iai_arithmetic"
 path = "benches/iai/arithmetic_circuit.rs"
 harness = false
-required-features = ["enable-benches", "descriptive-gate"]
+required-features = ["enable-benches"]
 
 [[bench]]
 name = "oneshot_arithmetic"
 path = "benches/oneshot/arithmetic_circuit.rs"
 harness = false
-required-features = ["enable-benches", "descriptive-gate"]
+required-features = ["enable-benches"]
 
 [[bench]]
 name = "oneshot_ipa"
 path = "benches/oneshot/ipa.rs"
 harness = false
-required-features = ["enable-benches", "descriptive-gate"]
+required-features = ["enable-benches"]
 
 [[bench]]
 name = "transpose"
@@ -225,15 +230,14 @@ required-features = [
     "web-app",
     "real-world-infra",
     "test-fixture",
-    "descriptive-gate",
 ]
 
 [[test]]
 name = "compact_gate"
 required-features = [
     "cli",
+    "compact-gate",
     "web-app",
     "real-world-infra",
     "test-fixture",
-    "compact-gate",
 ]

ipa-core/benches/README.md

@@ -34,7 +34,7 @@ Execute the following command to enable step-level metrics. It is possible to us
 `TestWorld` to set up the environment.
 
 ```bash
-RUST_LOG=ipa=DEBUG cargo bench --bench oneshot_sort --no-default-features --features="enable-benches debug-trace descriptive-gate"
+RUST_LOG=ipa=DEBUG cargo bench --bench oneshot_sort --no-default-features --features="enable-benches debug-trace compact-gate"
 ```
 
 The output would look similar to this:

ipa-core/benches/oneshot/ipa.rs

@@ -9,11 +9,13 @@ use ipa_core::{
     error::Error,
     ff::Fp32BitPrime,
     helpers::{query::IpaQueryConfig, GatewayConfig},
+    protocol::{step::ProtocolStep::IpaPrf, Gate},
     test_fixture::{
         ipa::{ipa_in_the_clear, test_oprf_ipa, CappingOrder, IpaSecurityModel},
         EventGenerator, EventGeneratorConfig, TestWorld, TestWorldConfig,
     },
 };
+use ipa_step::StepNarrow;
 use rand::{random, rngs::StdRng, SeedableRng};
 use tokio::runtime::Builder;
 
@@ -101,6 +103,7 @@ async fn run(args: Args) -> Result<(), Error> {
     let _prep_time = Instant::now();
     let config = TestWorldConfig {
         gateway_config: GatewayConfig::new(args.active()),
+        initial_gate: Some(Gate::default().narrow(&IpaPrf)),
         ..TestWorldConfig::default()
     };
     // Construct TestWorld early to initialize logging.

ipa-core/build.rs

@@ -1,14 +1,53 @@
+#![deny(clippy::pedantic, clippy::clone_on_ref_ptr)]
+#![allow(clippy::module_name_repetitions)]
+
 use cfg_aliases::cfg_aliases;
+use ipa_step::build_gate;
+use ipa_step_derive::track_steps;
+
+track_steps!(
+    setup_steps:
+    protocol::{
+        basics::{
+            mul::step,
+            step,
+        },
+        boolean::{
+            step,
+        },
+        context::step,
+        ipa_prf::{
+            boolean_ops::step,
+            prf_sharding::step,
+            shuffle::step,
+            aggregation::step,
+            step,
+        },
+        step,
+    },
+    test_fixture::step
+);
 
 fn main() {
+    if cfg!(feature = "compact-gate") {
+        // include protocol_gate.rs. This slows down the build significantly, so avoid doing that
+        // in dev builds.
+        setup_steps();
+        build_gate::<protocol::step::ProtocolStep>();
+    }
+
     // test is not supported because cfg_aliases is based on
     // https://docs.rs/tectonic_cfg_support macro and that only supports features, target_os, family
     // env, etc.
     // https://docs.rs/tectonic_cfg_support/latest/tectonic_cfg_support/struct.TargetConfiguration.html
     cfg_aliases! {
-        unit_test: { all(not(feature = "shuttle"), not(feature = "compact-gate"), feature = "in-memory-infra") },
-        web_test: { all(not(feature = "shuttle"), not(feature = "compact-gate"), feature = "real-world-infra") },
+        compact_gate: { feature = "compact-gate" },
+        descriptive_gate: { not(compact_gate) },
+        unit_test: { all(not(feature = "shuttle"), feature = "in-memory-infra", descriptive_gate) },
+        web_test: { all(not(feature = "shuttle"), feature = "real-world-infra") },
     }
+    println!("cargo::rustc-check-cfg=cfg(descriptive_gate)");
+    println!("cargo::rustc-check-cfg=cfg(compact_gate)");
     println!("cargo::rustc-check-cfg=cfg(unit_test)");
     println!("cargo::rustc-check-cfg=cfg(web_test)");
     println!("cargo::rustc-check-cfg=cfg(coverage)");

ipa-core/src/helpers/gateway/mod.rs

@@ -465,7 +465,7 @@ mod tests {
 
     #[test]
     #[should_panic(
-        expected = "Shard receiver channel[ShardIndex(1),\"protocol/iter0\"] can only be created once"
+        expected = "Shard receiver channel[ShardIndex(1),\"protocol/iter000\"] can only be created once"
     )]
     fn shards_receive_twice() {
         run(|| async move {

ipa-core/src/helpers/gateway/transport.rs

@@ -6,7 +6,7 @@ use crate::{
         transport::routing::RouteId, MpcTransportImpl, NoResourceIdentifier, QueryIdBinding, Role,
         RoleAssignment, RouteParams, StepBinding, Transport,
     },
-    protocol::{step::Gate, QueryId},
+    protocol::{Gate, QueryId},
     sharding::ShardIndex,
 };
 

ipa-core/src/helpers/mod.rs

@@ -82,7 +82,7 @@ use crate::{
         Direction::{Left, Right},
         Role::{H1, H2, H3},
     },
-    protocol::{step::Gate, RecordId},
+    protocol::{Gate, RecordId},
     secret_sharing::Sendable,
     sharding::ShardIndex,
 };

ipa-core/src/helpers/prss_protocol.rs

@@ -4,37 +4,22 @@ use x25519_dalek::PublicKey;
 
 use crate::{
     helpers::{ChannelId, Direction, Error, Gateway, Role, TotalRecords},
-    protocol::{
-        prss,
-        step::{Gate, Step, StepNarrow},
-        RecordId,
-    },
+    protocol::{prss, Gate, RecordId},
 };
 
-pub struct PrssExchangeStep;
-
-impl AsRef<str> for PrssExchangeStep {
-    fn as_ref(&self) -> &str {
-        "prss_exchange"
-    }
-}
-
-impl Step for PrssExchangeStep {}
-
-/// establish the prss endpoint by exchanging public keys with the other helpers
+/// Establish the prss endpoint by exchanging public keys with the other helpers.
 /// # Errors
 /// if communication with other helpers fails
 pub async fn negotiate<R: RngCore + CryptoRng>(
     gateway: &Gateway,
     gate: &Gate,
     rng: &mut R,
 ) -> Result<prss::Endpoint, Error<Role>> {
-    // setup protocol to exchange prss public keys. This protocol sends one message per peer.
+    // setup protocol to exchange PRSS public keys. This protocol sends one message per peer.
     // Each message contains this helper's public key. At the end of this protocol, all helpers
     // have completed key exchange and each of them have established a shared secret with each peer.
-    let step = gate.narrow(&PrssExchangeStep);
-    let left_channel = ChannelId::new(gateway.role().peer(Direction::Left), step.clone());
-    let right_channel = ChannelId::new(gateway.role().peer(Direction::Right), step.clone());
+    let left_channel = ChannelId::new(gateway.role().peer(Direction::Left), gate.clone());
+    let right_channel = ChannelId::new(gateway.role().peer(Direction::Right), gate.clone());
     let total_records = TotalRecords::from(1);
 
     let left_sender = gateway.get_mpc_sender::<PublicKey>(&left_channel, total_records);

ipa-core/src/helpers/transport/in_memory/sharding.rs

@@ -86,7 +86,7 @@ mod tests {
             transport::{in_memory::InMemoryShardNetwork, routing::RouteId},
             HelperIdentity, Transport,
         },
-        protocol::{step::Gate, QueryId},
+        protocol::{Gate, QueryId},
         sharding::ShardIndex,
         test_executor::run,
         test_fixture::logging,