diff --git a/README.md b/README.md index 35382e0..3d00468 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,25 @@ npx circomkit clean extract All of the above should be ran from repository root. +## Rust Example Witness JSON Creation +To generate example input JSON files for the Circom circuits, you can +``` +cargo install --path . +``` +to install the `witness` binary. +To get the basic idea, run `witness --help`. +It can process and generate JSON files to be used for the circuits. +For example, if we have a given JSON file we want to parse such as `examples/json/test/example.json` for the `extract` circuit (see `circuits.json`), then we can: +``` +witness json --input-file examples/json/test/example.json --output-dir inputs/extract --output-filename input.json +``` + +For an HTTP request/response, you can generate a JSON input via: +``` +witness http --input-file examples/http/get_request.http --output-dir inputs/get_request --output-filename input.json +``` +Afterwards, you can run `circomkit compile get_request` then `circomkit witness get_request input`. + ## Testing To test, you can just run ``` diff --git a/circuits.json b/circuits.json index 59c7f4f..bd295ac 100644 --- a/circuits.json +++ b/circuits.json @@ -1,63 +1,63 @@ { "extract": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 157, 13 ] }, "value_string": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 12, 1 ] }, "value_number": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 12, 2 ] }, "value_array": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 18, 2 ] }, "value_array_nested": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 24, 4 ] }, "value_array_object": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 25, 4 ] }, "value_array_object_array": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 31, 5 ] }, "value_object": { - "file": "extract", - "template": "Extract", + "file": "parser_json/parser", + "template": "Parser", "params": [ 21, 3 @@ -70,5 +70,10 @@ 787, 10 ] + }, + "get_request": { + "file": "parser_http_request/parser", + "template": "Parser", + "params": [158] } } \ No newline at end of file diff --git a/circuits/parser_http_request/language.circom b/circuits/parser_http_request/language.circom new file mode 100644 index 0000000..a001d86 --- /dev/null +++ b/circuits/parser_http_request/language.circom @@ -0,0 +1,49 @@ +pragma circom 2.1.9; + +// All the possible request methods: https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods + +template Syntax() { + //-Delimeters---------------------------------------------------------------------------------// + // - ASCII char `:` + signal output COLON <== 58; + // - ASCII char `;` + signal output SEMICOLON <== 59; + // - ASCII char `,` + signal output COMMA <== 44; + // - ASCII char `"` + signal output QUOTE <== 34; + //-White_space--------------------------------------------------------------------------------// + // - ASCII pair: `\r\n` + signal output CLRF <== [13, 10]; // https://www.rfc-editor.org/rfc/rfc2616#section-2.2 + // https://www.rfc-editor.org/rfc/rfc7230#section-3.5 + // - ASCII char: ` ` + signal output SPACE <== 32; + //-Escape-------------------------------------------------------------------------------------// + // - ASCII char: `\` + signal output ESCAPE <== 92; +} + +template RequestMethod() { + signal output GET[3] <== [71, 69, 84]; + // signal output HEAD[4] <== [72, 69, 65, 68]; + signal output POST[4] <== [80, 79, 83, 84]; + // signal output PUT <== 3; + // signal output DELETE <== 4; + // signal output CONNECT <== 5; + // signal output OPTIONS <== 6; + // signal output TRACE <== 7; + // signal output PATCH <== 8; +} + +// NOTE: Starting at 1 to avoid a false positive with a 0. +template RequestMethodTag() { + signal output GET <== 1; + // signal output HEAD <== 2; + signal output POST <== 3; + // signal output PUT <== 4; + // signal output DELETE <== 5; + // signal output CONNECT <== 6; + // signal output OPTIONS <== 7; + // signal output TRACE <== 8; + // signal output PATCH <== 9; +} \ No newline at end of file diff --git a/circuits/parser_http_request/machine.circom b/circuits/parser_http_request/machine.circom new file mode 100644 index 0000000..ee0cff1 --- /dev/null +++ b/circuits/parser_http_request/machine.circom @@ -0,0 +1,28 @@ +pragma circom 2.1.9; + +include "language.circom"; +include "../utils/array.circom"; + +template ParseMethod() { + signal input bytes[7]; + signal output MethodTag; + + component RequestMethod = RequestMethod(); + component RequestMethodTag = RequestMethodTag(); + + component IsGet = IsEqualArray(3); + for(var byte_idx = 0; byte_idx < 3; byte_idx++) { + IsGet.in[0][byte_idx] <== bytes[byte_idx]; + IsGet.in[1][byte_idx] <== RequestMethod.GET[byte_idx]; + } + signal TagGet <== IsGet.out * RequestMethodTag.GET; + + component IsPost = IsEqualArray(4); + for(var byte_idx = 0; byte_idx < 4; byte_idx++) { + IsPost.in[0][byte_idx] <== bytes[byte_idx]; + IsPost.in[1][byte_idx] <== RequestMethod.POST[byte_idx]; + } + signal TagPost <== IsPost.out * RequestMethodTag.POST; + + MethodTag <== TagGet + TagPost; +} \ No newline at end of file diff --git a/circuits/parser_http_request/parser.circom b/circuits/parser_http_request/parser.circom new file mode 100644 index 0000000..9fd9524 --- /dev/null +++ b/circuits/parser_http_request/parser.circom @@ -0,0 +1,24 @@ +pragma circom 2.1.9; + +include "../utils/bytes.circom"; +include "machine.circom"; + + +template Parser(DATA_BYTES) { + signal input data[DATA_BYTES]; + + signal output Method; + + //--------------------------------------------------------------------------------------------// + //-CONSTRAINTS--------------------------------------------------------------------------------// + //--------------------------------------------------------------------------------------------// + component dataASCII = ASCII(DATA_BYTES); + dataASCII.in <== data; + //--------------------------------------------------------------------------------------------// + + component ParseMethod = ParseMethod(); + for(var byte_idx = 0; byte_idx < 7; byte_idx++) { + ParseMethod.bytes[byte_idx] <== data[byte_idx]; + } + log("MethodTag: ", ParseMethod.MethodTag); +} \ No newline at end of file diff --git a/circuits/language.circom b/circuits/parser_json/language.circom similarity index 100% rename from circuits/language.circom rename to circuits/parser_json/language.circom diff --git a/circuits/parser.circom b/circuits/parser_json/machine.circom similarity index 99% rename from circuits/parser.circom rename to circuits/parser_json/machine.circom index 0f63120..9fda500 100644 --- a/circuits/parser.circom +++ b/circuits/parser_json/machine.circom @@ -1,5 +1,5 @@ /* -# `parser` +# `machine` This module consists of the core parsing components for generating proofs of selective disclosure in JSON. ## Layout @@ -23,9 +23,9 @@ Tests for this module are located in the files: `circuits/test/parser/*.test.ts pragma circom 2.1.9; -include "./utils/array.circom"; -include "./utils/bytes.circom"; -include "./utils/operators.circom"; +include "../utils/array.circom"; +include "../utils/bytes.circom"; +include "../utils/operators.circom"; include "language.circom"; /* diff --git a/circuits/extract.circom b/circuits/parser_json/parser.circom similarity index 56% rename from circuits/extract.circom rename to circuits/parser_json/parser.circom index 8fa4bdb..f3e4b51 100644 --- a/circuits/extract.circom +++ b/circuits/parser_json/parser.circom @@ -1,9 +1,9 @@ pragma circom 2.1.9; -include "./utils/bytes.circom"; -include "parser.circom"; +include "../utils/bytes.circom"; +include "machine.circom"; -template Extract(DATA_BYTES, MAX_STACK_HEIGHT) { +template Parser(DATA_BYTES, MAX_STACK_HEIGHT) { signal input data[DATA_BYTES]; // TODO: Add assertions on the inputs here! @@ -31,23 +31,23 @@ template Extract(DATA_BYTES, MAX_STACK_HEIGHT) { State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string; State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number; - // Debugging - for(var i = 0; i { }); } - function generateFailCase(input: any, desc: string) { - const description = generateDescription(input); - - it(`(valid) witness: ${description}\n${desc}`, async () => { - await circuit.expectFail(input); - }); - } - before(async () => { circuit = await circomkit.WitnessTester(`StateUpdate`, { - file: "circuits/parser", + file: "circuits/parser_json/machine", template: "StateUpdate", params: [4], }); diff --git a/circuits/test/parser/stack.test.ts b/circuits/test/parser_json/stack.test.ts similarity index 99% rename from circuits/test/parser/stack.test.ts rename to circuits/test/parser_json/stack.test.ts index 69e0b1a..ff5e97a 100644 --- a/circuits/test/parser/stack.test.ts +++ b/circuits/test/parser_json/stack.test.ts @@ -5,7 +5,7 @@ describe("GetTopOfStack", () => { let circuit: WitnessTester<["stack"], ["value", "pointer"]>; before(async () => { circuit = await circomkit.WitnessTester(`GetTopOfStack`, { - file: "circuits/parser", + file: "circuits/parser_json/machine", template: "GetTopOfStack", params: [4], }); @@ -44,7 +44,7 @@ describe("StateUpdate :: RewriteStack", () => { >; before(async () => { circuit = await circomkit.WitnessTester(`GetTopOfStack`, { - file: "circuits/parser", + file: "circuits/parser_json/machine", template: "StateUpdate", params: [4], }); diff --git a/circuits/test/parser/values.test.ts b/circuits/test/parser_json/values.test.ts similarity index 99% rename from circuits/test/parser/values.test.ts rename to circuits/test/parser_json/values.test.ts index 7b6bd9f..d96e757 100644 --- a/circuits/test/parser/values.test.ts +++ b/circuits/test/parser_json/values.test.ts @@ -8,7 +8,7 @@ describe("StateUpdate :: Values", () => { >; before(async () => { circuit = await circomkit.WitnessTester(`GetTopOfStack`, { - file: "circuits/parser", + file: "circuits/parser_json/machine", template: "StateUpdate", params: [4], }); diff --git a/examples/http/get_request.http b/examples/http/get_request.http new file mode 100644 index 0000000..88eb48d --- /dev/null +++ b/examples/http/get_request.http @@ -0,0 +1,5 @@ +GET /objectserver/restapi/alerts/status HTTP/1.1 +Accept: application/json +Authorization: Basic dGVzdHVzZXIwMTpuZXRjb29s +Host: localhost +Connection: keep-alive \ No newline at end of file diff --git a/examples/http/get_response.http b/examples/http/get_response.http new file mode 100644 index 0000000..ac11942 --- /dev/null +++ b/examples/http/get_response.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Content-Type: application/json +Content-Length: 19 + +{"success":"true"} \ No newline at end of file diff --git a/json_examples/response/reddit.json b/examples/json/response/reddit.json similarity index 100% rename from json_examples/response/reddit.json rename to examples/json/response/reddit.json diff --git a/json_examples/response/spotify.json b/examples/json/response/spotify.json similarity index 100% rename from json_examples/response/spotify.json rename to examples/json/response/spotify.json diff --git a/json_examples/response/venmo.json b/examples/json/response/venmo.json similarity index 100% rename from json_examples/response/venmo.json rename to examples/json/response/venmo.json diff --git a/json_examples/test/example.json b/examples/json/test/example.json similarity index 100% rename from json_examples/test/example.json rename to examples/json/test/example.json diff --git a/json_examples/test/string_escape.json b/examples/json/test/string_escape.json similarity index 100% rename from json_examples/test/string_escape.json rename to examples/json/test/string_escape.json diff --git a/json_examples/test/two_keys.json b/examples/json/test/two_keys.json similarity index 100% rename from json_examples/test/two_keys.json rename to examples/json/test/two_keys.json diff --git a/json_examples/test/value_array.json b/examples/json/test/value_array.json similarity index 100% rename from json_examples/test/value_array.json rename to examples/json/test/value_array.json diff --git a/json_examples/test/value_array_nested.json b/examples/json/test/value_array_nested.json similarity index 100% rename from json_examples/test/value_array_nested.json rename to examples/json/test/value_array_nested.json diff --git a/json_examples/test/value_array_object.json b/examples/json/test/value_array_object.json similarity index 100% rename from json_examples/test/value_array_object.json rename to examples/json/test/value_array_object.json diff --git a/json_examples/test/value_array_object_array.json b/examples/json/test/value_array_object_array.json similarity index 100% rename from json_examples/test/value_array_object_array.json rename to examples/json/test/value_array_object_array.json diff --git a/json_examples/test/value_number.json b/examples/json/test/value_number.json similarity index 100% rename from json_examples/test/value_number.json rename to examples/json/test/value_number.json diff --git a/json_examples/test/value_object.json b/examples/json/test/value_object.json similarity index 100% rename from json_examples/test/value_object.json rename to examples/json/test/value_object.json diff --git a/json_examples/test/value_string.json b/examples/json/test/value_string.json similarity index 100% rename from json_examples/test/value_string.json rename to examples/json/test/value_string.json diff --git a/json_examples/reddit_response.json b/json_examples/reddit_response.json deleted file mode 100644 index 163e6b4..0000000 --- a/json_examples/reddit_response.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "data": { - "redditorInfoByName": { - "id": "t2_bepsb", - "karma": { - "fromAwardsGiven": 0, - "fromAwardsReceived": 470, - "fromComments": 9583, - "fromPosts": 13228, - "total": 23281 - } - } - } -} \ No newline at end of file diff --git a/json_examples/sambhav_example.json b/json_examples/sambhav_example.json deleted file mode 100644 index ab9c57e..0000000 --- a/json_examples/sambhav_example.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extract": { - "file": "extract", - "template": "Extract", - "params": "" - } -} \ No newline at end of file diff --git a/json_examples/test/example_json.md b/json_examples/test/example_json.md deleted file mode 100644 index a398ff8..0000000 --- a/json_examples/test/example_json.md +++ /dev/null @@ -1,25 +0,0 @@ -{ "a": // 7 -{ "b": "c", // 19 -"d": { // 25 -"e": "f", // 35 -"g": { // 42 -"h": { // 48 -"i": "j", -"k": "l", -"m": "n", -"o": "p", -"q": "r", -"s": { -"t": "u", -"v": [ -"w", -"x" -] -}, // 138 -"y": "z" // 147 -} // 149 -} // 151 -} // 153 -} // 155 -} // 157 - diff --git a/json_examples/test_depth.json b/json_examples/test_depth.json deleted file mode 100644 index ad2f816..0000000 --- a/json_examples/test_depth.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "key1": "abc", - "key2": { - "key3": "def" - } -} \ No newline at end of file diff --git a/json_examples/test_two_key.json b/json_examples/test_two_key.json deleted file mode 100644 index 437a6c5..0000000 --- a/json_examples/test_two_key.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "key1": "abc", - "key2": "def" -} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 28ce12c..f7ac343 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,12 @@ { "name": "extractor", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { + "name": "extractor", + "version": "0.1.0", "license": "Apache-2.0", "dependencies": { "circomkit": "^0.2.1", diff --git a/src/main.rs b/src/main.rs index 8f16d6c..d3c324f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{Parser, Subcommand}; use serde_json::Value; use std::io::Write; use std::path::PathBuf; @@ -6,21 +6,34 @@ use std::path::PathBuf; #[derive(Parser, Debug)] #[command(name = "witness")] struct Args { - /// Path to the JSON file - #[arg(short, long)] - json_file: PathBuf, - - /// Keys to extract (can be specified multiple times) - #[arg(short, long)] - keys: Vec, + #[command(subcommand)] + command: Command, /// Output directory (will be created if it doesn't exist) - #[arg(short, long, default_value = ".")] + #[arg(global = true, short, long, default_value = ".")] output_dir: PathBuf, /// Output filename (will be created if it doesn't exist) - #[arg(short, long, default_value = "output.json")] - filename: String, + #[arg(global = true, short, long, default_value = "output.json")] + output_filename: String, +} + +#[derive(Subcommand, Debug)] +enum Command { + Json { + /// Path to the JSON file + #[arg(short, long)] + input_file: PathBuf, + + /// Keys to extract (can be specified multiple times) + #[arg(short, long)] + keys: Vec, + }, + Http { + /// Path to the HTTP request file + #[arg(short, long)] + input_file: PathBuf, + }, } #[derive(serde::Serialize)] @@ -33,24 +46,30 @@ pub struct Witness { pub fn main() -> Result<(), Box> { let args = Args::parse(); - // Read the JSON file - let data = std::fs::read(&args.json_file)?; - - // Create a map to store keys - let mut keys_map = serde_json::Map::new(); - for (index, key) in args.keys.iter().enumerate() { - keys_map.insert( - format!("key{}", index + 1), - Value::Array( - key.as_bytes() - .iter() - .map(|x| serde_json::json!(x)) - .collect(), - ), - ); - } + let (data, keys_map) = match &args.command { + Command::Json { input_file, keys } => { + let data = std::fs::read(input_file)?; + let mut keys_map = serde_json::Map::new(); + for (index, key) in keys.iter().enumerate() { + keys_map.insert( + format!("key{}", index + 1), + Value::Array( + key.as_bytes() + .iter() + .map(|x| serde_json::json!(x)) + .collect(), + ), + ); + } + (data, keys_map) + } + Command::Http { input_file } => { + let data = std::fs::read(input_file)?; + let keys_map = serde_json::Map::new(); + (data, keys_map) + } + }; - // Create a witness file as `input.json` let witness = Witness { keys: keys_map, data: data.clone(), @@ -60,15 +79,22 @@ pub fn main() -> Result<(), Box> { std::fs::create_dir_all(&args.output_dir)?; } - let output_file = args.output_dir.join(args.filename); + let output_file = args.output_dir.join(args.output_filename); let mut file = std::fs::File::create(output_file)?; file.write_all(serde_json::to_string_pretty(&witness)?.as_bytes())?; // Prepare lines to print let mut lines = Vec::new(); - lines.push(String::from("Key lengths:")); - for (index, key) in args.keys.iter().enumerate() { - lines.push(format!("key{} length: {}", index + 1, key.len())); + match &args.command { + Command::Json { keys, .. } => { + lines.push(String::from("Key lengths:")); + for (index, key) in keys.iter().enumerate() { + lines.push(format!("key{} length: {}", index + 1, key.len())); + } + } + Command::Http { .. } => { + lines.push(String::from("HTTP request processed")); + } } lines.push(format!("Data length: {}", data.len()));