From cc479381da2e9ca8c0298671b274ade977eef2d8 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Tue, 3 Sep 2024 15:04:35 -0600 Subject: [PATCH] feat: Rust CLI improvements and HTTP lockfile example (#72) * feat: combined CLI * refactor and add example http lock * http lock example * include `search/witness.json` * fix tests * clean up basic lock file --- .gitignore | 3 +- Cargo.lock | 22 +-- Cargo.toml | 8 +- .../test/json/extractor/extractor.test.ts | 4 +- .../test/codegen => extractor}/two_keys.json | 0 .../value_array_nested.json | 0 .../value_array_number.json | 0 .../value_array_object.json | 0 .../value_array_string.json | 0 .../codegen => extractor}/value_number.json | 0 .../codegen => extractor}/value_object.json | 0 .../codegen => extractor}/value_string.json | 0 examples/lockfile/test.lock.json | 27 ++++ notes.md | 51 ------- src/bin/witness.rs | 130 ------------------ src/{bin/codegen.rs => extractor.rs} | 39 ++---- src/http_lock.rs | 34 +++++ src/main.rs | 75 ++++++++++ src/witness.rs | 59 ++++++++ 19 files changed, 227 insertions(+), 225 deletions(-) rename examples/{json/test/codegen => extractor}/two_keys.json (100%) rename examples/{json/test/codegen => extractor}/value_array_nested.json (100%) rename examples/{json/test/codegen => extractor}/value_array_number.json (100%) rename examples/{json/test/codegen => extractor}/value_array_object.json (100%) rename examples/{json/test/codegen => extractor}/value_array_string.json (100%) rename examples/{json/test/codegen => extractor}/value_number.json (100%) rename examples/{json/test/codegen => extractor}/value_object.json (100%) rename examples/{json/test/codegen => extractor}/value_string.json (100%) create mode 100644 examples/lockfile/test.lock.json delete mode 100644 notes.md delete mode 100644 src/bin/witness.rs rename src/{bin/codegen.rs => extractor.rs} (97%) create mode 100644 src/http_lock.rs create mode 100644 src/main.rs create mode 100644 src/witness.rs diff --git a/.gitignore b/.gitignore index bbed9c8..faa5f81 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ circuits/test/*.circom circuits/main/* # Rust generated -inputs/**/*.json \ No newline at end of file +inputs/**/*.json +!inputs/search/witness.json \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 2479f30..f7d2691 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,9 +53,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.14" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c937d4061031a6d0c8da4b9a4f98a172fc2976dfb1c19213a9cf7d0d3c837e36" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -63,9 +63,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.14" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85379ba512b21a328adf887e85f7742d12e96eb31f3ef077df4ffc26b506ffed" +checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" dependencies = [ "anstream", "anstyle", @@ -147,18 +147,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.205" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.205" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.122" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" +checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" dependencies = [ "itoa", "memchr", @@ -280,7 +280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "witness" +name = "wpbuild" version = "0.0.0" dependencies = [ "clap", diff --git a/Cargo.toml b/Cargo.toml index 25d2171..dbfe087 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "witness" +name = "wpbuild" edition = "2021" [dependencies] -serde = { version = "1.0.204", features = ["derive"] } -serde_json = "1.0.120" -clap = { version = "4.5.14", features = ["derive"] } +serde = { version = "1.0.209", features = ["derive"] } +serde_json = "1.0.127" +clap = { version = "4.5.16", features = ["derive"] } diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index bc13620..4d47ea7 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -5,9 +5,9 @@ import { spawn } from "child_process"; function executeCodegen(inputFilename: string, outputFilename: string) { return new Promise((resolve, reject) => { - const inputPath = join(__dirname, "..", "..", "..", "..", "examples", "json", "test", "codegen", inputFilename); + const inputPath = join(__dirname, "..", "..", "..", "..", "examples", "extractor", inputFilename); - const codegen = spawn("cargo", ["run", "--bin", "codegen", "--", "--json-file", inputPath, "--output-filename", outputFilename]); + const codegen = spawn("cargo", ["run", "extractor", "--template", inputPath, "--output-filename", outputFilename]); codegen.stdout.on('data', (data) => { console.log(`stdout: ${data}`); diff --git a/examples/json/test/codegen/two_keys.json b/examples/extractor/two_keys.json similarity index 100% rename from examples/json/test/codegen/two_keys.json rename to examples/extractor/two_keys.json diff --git a/examples/json/test/codegen/value_array_nested.json b/examples/extractor/value_array_nested.json similarity index 100% rename from examples/json/test/codegen/value_array_nested.json rename to examples/extractor/value_array_nested.json diff --git a/examples/json/test/codegen/value_array_number.json b/examples/extractor/value_array_number.json similarity index 100% rename from examples/json/test/codegen/value_array_number.json rename to examples/extractor/value_array_number.json diff --git a/examples/json/test/codegen/value_array_object.json b/examples/extractor/value_array_object.json similarity index 100% rename from examples/json/test/codegen/value_array_object.json rename to examples/extractor/value_array_object.json diff --git a/examples/json/test/codegen/value_array_string.json b/examples/extractor/value_array_string.json similarity index 100% rename from examples/json/test/codegen/value_array_string.json rename to examples/extractor/value_array_string.json diff --git a/examples/json/test/codegen/value_number.json b/examples/extractor/value_number.json similarity index 100% rename from examples/json/test/codegen/value_number.json rename to examples/extractor/value_number.json diff --git a/examples/json/test/codegen/value_object.json b/examples/extractor/value_object.json similarity index 100% rename from examples/json/test/codegen/value_object.json rename to examples/extractor/value_object.json diff --git a/examples/json/test/codegen/value_string.json b/examples/extractor/value_string.json similarity index 100% rename from examples/json/test/codegen/value_string.json rename to examples/extractor/value_string.json diff --git a/examples/lockfile/test.lock.json b/examples/lockfile/test.lock.json new file mode 100644 index 0000000..a95f84e --- /dev/null +++ b/examples/lockfile/test.lock.json @@ -0,0 +1,27 @@ +{ + "request": { + "method": "GET", + "target": "/api", + "version": "HTTP/1.1", + "headers": [ + [ + "Host", + "localhost" + ], + [ + "Accept", + "application/json" + ] + ] + }, + "response": { + "version": "HTTP/1.1", + "status": "200", + "headers": [ + [ + "Content-Type", + "application/json" + ] + ] + } +} \ No newline at end of file diff --git a/notes.md b/notes.md deleted file mode 100644 index ba1c261..0000000 --- a/notes.md +++ /dev/null @@ -1,51 +0,0 @@ -# Notes - -## TODOs -### JSON Types -- [x] Object -- [x] String -- [x] Array -- [x] Number -- [ ] Boolean -- [ ] Null - -Parsing null and bool need to do some kind of look ahead parsing. To handle numbers properly we also probably need that actually since we need to look ahead to where we get white space. -Need to look ahead for `true` and `false` for example to ensure we get a full match, or we fail or something. Lookaehad might be overkill, but yeah. - -#### Numbers -Numbers can have `e` and decimal `.` in them. Riperoni. - -### string escape -shouldn't be too hard, just add one more state variable `escaping` that is only enabled when parsing a string and can only be toggled -- next state will always have to set back to 0. - -This could also allow for parsing unicode - -### Other thoughts - - Pointer may not actually be necessary, because it just points to the first unallocated position in the stac - - How do we know how tall to make the stack? Open braces `{`, open brackets `[` and colons `:` all push onto the stack. - - We might not actually need to push the stack higher with a colon, instead we could push state into the second slot of the stack like we do with commas inside of arrays. - -## Expected Output -> This is old at this point, but we should update it. -``` -Notes: for `test.json` -xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - POINTER | Read In: | STATE -------------------------------------------------- -State[1] | { | PARSING TO KEY -------------------------------------------------- -State[7] | " | INSIDE KEY -------------------------------------------------- -State[12]| " | NOT INSIDE KEY -------------------------------------------------- -State[13]| : | PARSING TO VALUE -------------------------------------------------- -State[15]| " | INSIDE VALUE -------------------------------------------------- -State[19]| " | NOT INSIDE VALUE -------------------------------------------------- -State[20]| " | COMPLETE WITH KV PARSING -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -State[20].next_tree_depth == 0 | VALID JSON -xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -``` \ No newline at end of file diff --git a/src/bin/witness.rs b/src/bin/witness.rs deleted file mode 100644 index bacabe0..0000000 --- a/src/bin/witness.rs +++ /dev/null @@ -1,130 +0,0 @@ -use clap::{Parser, Subcommand}; -use serde_json::Value; -use std::io::Write; -use std::path::PathBuf; - -#[derive(Parser, Debug)] -#[command(name = "witness")] -struct Args { - #[command(subcommand)] - command: Command, - - /// Output directory (will be created if it doesn't exist) - #[arg(global = true, long, default_value = ".")] - output_dir: PathBuf, - - /// Output filename (will be created if it doesn't exist) - #[arg(global = true, long, default_value = "output.json")] - output_filename: String, -} - -#[derive(Subcommand, Debug)] -enum Command { - Json { - /// Path to the JSON file - #[arg(short, long)] - input_file: PathBuf, - - /// Keys to extract (can be specified multiple times) - #[arg(short, long)] - keys: Vec, - }, - Http { - /// Path to the HTTP request file - #[arg(short, long)] - input_file: PathBuf, - }, -} - -#[derive(serde::Serialize)] -pub struct Witness { - #[serde(flatten)] - keys: serde_json::Map, - data: Vec, -} - -pub fn main() -> Result<(), Box> { - let args = Args::parse(); - - let (data, keys_map) = match &args.command { - Command::Json { input_file, keys } => { - let data = std::fs::read(input_file)?; - let mut keys_map = serde_json::Map::new(); - for (index, key) in keys.iter().enumerate() { - keys_map.insert( - format!("key{}", index + 1), - Value::Array( - key.as_bytes() - .iter() - .map(|x| serde_json::json!(x)) - .collect(), - ), - ); - } - (data, keys_map) - } - Command::Http { input_file } => { - let mut data = std::fs::read(input_file)?; - let mut i = 0; - while i < data.len() { - if data[i] == 10 && (i == 0 || data[i - 1] != 13) { - data.insert(i, 13); - i += 2; - } else { - i += 1; - } - } - let keys_map = serde_json::Map::new(); - (data, keys_map) - } - }; - - let witness = Witness { - keys: keys_map, - data: data.clone(), - }; - - if !args.output_dir.exists() { - std::fs::create_dir_all(&args.output_dir)?; - } - - let output_file = args.output_dir.join(args.output_filename); - let mut file = std::fs::File::create(output_file)?; - file.write_all(serde_json::to_string_pretty(&witness)?.as_bytes())?; - - // Prepare lines to print - let mut lines = Vec::new(); - match &args.command { - Command::Json { keys, .. } => { - lines.push(String::from("Key lengths:")); - for (index, key) in keys.iter().enumerate() { - lines.push(format!("key{} length: {}", index + 1, key.len())); - } - } - Command::Http { .. } => { - lines.push(String::from("HTTP request processed")); - } - } - lines.push(format!("Data length: {}", data.len())); - - // Print the output inside a nicely formatted box - print_boxed_output(lines); - - Ok(()) -} - -fn print_boxed_output(lines: Vec) { - // Determine the maximum length of the lines - let max_length = lines.iter().map(|line| line.len()).max().unwrap_or(0); - - // Characters for the box - let top_border = format!("┌{}┐", "─".repeat(max_length + 2)); - let bottom_border = format!("└{}┘", "─".repeat(max_length + 2)); - - // Print the box with content - println!("{}", top_border); - for line in lines { - println!("│ {:, value_type: ValueType, } -const PRAGMA: &str = "pragma circom 2.1.9;\n\n"; - fn extract_string(data: Data, circuit_buffer: &mut String) { *circuit_buffer += "template ExtractStringValue(DATA_BYTES, MAX_STACK_HEIGHT, "; for (i, key) in data.keys.iter().enumerate() { @@ -500,10 +489,8 @@ fn parse_json_request( Ok(()) } -pub fn main() -> Result<(), Box> { - let args = Args::parse(); - - let data = std::fs::read(&args.json_file)?; +pub fn extractor(args: ExtractorArgs) -> Result<(), Box> { + let data = std::fs::read(&args.template)?; let json_data: Data = serde_json::from_slice(&data)?; parse_json_request(json_data, args.output_filename)?; diff --git a/src/http_lock.rs b/src/http_lock.rs new file mode 100644 index 0000000..7bf2c24 --- /dev/null +++ b/src/http_lock.rs @@ -0,0 +1,34 @@ +use super::*; + +#[derive(Debug, Serialize, Deserialize)] +struct HttpData { + request: Request, + response: Response, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Request { + method: String, + target: String, + version: String, + headers: Vec<(String, String)>, + #[serde(rename = "Host")] + host: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Response { + version: String, + status: String, + headers: Vec<(String, serde_json::Value)>, +} + +// TODO: This needs to codegen a circuit now. +pub fn http_lock(args: HttpLockArgs) -> Result<(), Box> { + let data = std::fs::read(&args.lockfile)?; + let http_data: HttpData = serde_json::from_slice(&data)?; + + dbg!(http_data); + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..2312c5e --- /dev/null +++ b/src/main.rs @@ -0,0 +1,75 @@ +use clap::{Parser, Subcommand}; +use serde::{Deserialize, Serialize}; +use std::{error::Error, path::PathBuf}; + +pub mod extractor; +pub mod http_lock; +pub mod witness; + +#[derive(Parser, Debug)] +#[command(name = "wpbuild")] +pub struct Args { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand, Debug)] +pub enum Command { + Witness(WitnessArgs), + Extractor(ExtractorArgs), + HttpLock(HttpLockArgs), +} + +#[derive(Parser, Debug)] +pub struct WitnessArgs { + #[command(subcommand)] + subcommand: WitnessSubcommand, + + /// Path to the JSON file + #[arg(global = true, long)] + input_file: PathBuf, + + /// Output directory (will be created if it doesn't exist) + #[arg(global = true, long, default_value = ".")] + output_dir: PathBuf, + + /// Output filename (will be created if it doesn't exist) + #[arg(global = true, long, default_value = "output.json")] + output_filename: String, +} + +#[derive(Subcommand, Debug)] +pub enum WitnessSubcommand { + Json, + Http, +} + +#[derive(Parser, Debug)] +pub struct ExtractorArgs { + /// Path to the JSON file + #[arg(long)] + template: PathBuf, + + /// Output circuit file name + #[arg(long, default_value = "extractor")] + output_filename: String, +} + +#[derive(Parser, Debug)] +pub struct HttpLockArgs { + /// Path to the JSON file + #[arg(long)] + lockfile: PathBuf, + + /// Output circuit file name + #[arg(long, default_value = "extractor")] + output_filename: String, +} + +pub fn main() -> Result<(), Box> { + match Args::parse().command { + Command::Extractor(args) => extractor::extractor(args), + Command::Witness(args) => witness::witness(args), + Command::HttpLock(args) => http_lock::http_lock(args), + } +} diff --git a/src/witness.rs b/src/witness.rs new file mode 100644 index 0000000..8375627 --- /dev/null +++ b/src/witness.rs @@ -0,0 +1,59 @@ +use super::*; +use std::io::Write; + +#[derive(serde::Serialize)] +pub struct Witness(Vec); + +pub fn witness(args: WitnessArgs) -> Result<(), Box> { + let data = match &args.subcommand { + WitnessSubcommand::Json => std::fs::read(args.input_file)?, + WitnessSubcommand::Http => { + let mut data = std::fs::read(args.input_file)?; + let mut i = 0; + while i < data.len() { + if data[i] == 10 && (i == 0 || data[i - 1] != 13) { + data.insert(i, 13); + i += 2; + } else { + i += 1; + } + } + data + } + }; + + let witness = Witness(data.clone()); + + if !args.output_dir.exists() { + std::fs::create_dir_all(&args.output_dir)?; + } + + let output_file = args.output_dir.join(args.output_filename); + let mut file = std::fs::File::create(output_file)?; + file.write_all(serde_json::to_string_pretty(&witness)?.as_bytes())?; + + // Prepare lines to print + let mut lines = Vec::new(); + lines.push(format!("Data length: {}", data.len())); + + // Print the output inside a nicely formatted box + print_boxed_output(lines); + + Ok(()) +} + +fn print_boxed_output(lines: Vec) { + // Determine the maximum length of the lines + let max_length = lines.iter().map(|line| line.len()).max().unwrap_or(0); + + // Characters for the box + let top_border = format!("┌{}┐", "─".repeat(max_length + 2)); + let bottom_border = format!("└{}┘", "─".repeat(max_length + 2)); + + // Print the box with content + println!("{}", top_border); + for line in lines { + println!("│ {: