From d524cce625fd2ee335ec6919fe4e41e8e7810fc4 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Tue, 8 Oct 2024 00:58:56 +0530 Subject: [PATCH 1/7] init example --- circuits.json | 229 +++++++++++++++++++++-- circuits/json/parser/parser.circom | 28 +-- circuits/test/json/parser/parser.test.ts | 21 +++ examples/json/test/array_only.json | 12 ++ 4 files changed, 261 insertions(+), 29 deletions(-) create mode 100644 circuits/test/json/parser/parser.test.ts create mode 100644 examples/json/test/array_only.json diff --git a/circuits.json b/circuits.json index 09f0948..699d488 100644 --- a/circuits.json +++ b/circuits.json @@ -1,17 +1,216 @@ { - "json-parser": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 157, - 13 - ] - }, - "http-parser": { - "file": "http/parser/parser", - "template": "Parser", - "params": [ - 60 - ] - } + "two_keys_test": { + "file": "main/json_two_keys_test", + "template": "ExtractStringValue", + "params": [ + 40, + 1, + 4, + 0, + 3 + ] + }, + "value_array_number_test": { + "file": "main/json_value_array_number_test", + "template": "ExtractNumValue", + "params": [ + 73, + 2, + 1, + 0, + 2, + 1, + 4 + ] + }, + "value_number_test": { + "file": "main/json_value_number_test", + "template": "ExtractNumValue", + "params": [ + 12, + 1, + 1, + 0, + 2 + ] + }, + "http-parser": { + "file": "http/parser/parser", + "template": "Parser", + "params": [ + 60 + ] + }, + "get_request_test": { + "file": "main/http_get_request_test", + "template": "LockHTTPRequest", + "params": [ + 60, + 3, + 4, + 8, + 6, + 16, + 4, + 9 + ] + }, + "value_array_string_test": { + "file": "main/json_value_array_string_test", + "template": "ExtractStringValue", + "params": [ + 73, + 2, + 1, + 0, + 1, + 1, + 2 + ] + }, + "value_array_object_test": { + "file": "main/json_value_array_object_test", + "template": "ExtractNumValue", + "params": [ + 29, + 4, + 1, + 0, + 0, + 1, + 1, + 2, + 0, + 3, + 1 + ] + }, + "spotify_top_artists_test": { + "file": "main/http_spotify_top_artists_test", + "template": "LockHTTPResponse", + "params": [ + 203, + 85, + 8, + 3, + 2, + 12, + 31 + ] + }, + "spotify_test": { + "file": "main/json_spotify_test", + "template": "ExtractStringValue", + "params": [ + 85, + 5, + 4, + 0, + 5, + 1, + 0, + 2, + 7, + 3, + 4, + 4, + 12 + ] + }, + "value_string": { + "file": "main/json_value_string", + "template": "ExtractStringValue", + "params": [ + 12, + 1, + 1, + 0, + 1 + ] + }, + "get_response_test": { + "file": "main/http_get_response_test", + "template": "LockHTTPResponse", + "params": [ + 89, + 18, + 8, + 3, + 2, + 12, + 16 + ] + }, + "value_array_nested_test": { + "file": "main/json_value_array_nested_test", + "template": "ExtractNumValue", + "params": [ + 24, + 3, + 1, + 0, + 0, + 1, + 0, + 2, + 1 + ] + }, + "value_string_test": { + "file": "main/json_value_string_test", + "template": "ExtractStringValue", + "params": [ + 12, + 1, + 1, + 0, + 1 + ] + }, + "spotify_top_artists": { + "file": "main/extended_spotify_top_artists", + "template": "HttpJson", + "params": [ + 203, + 85, + 8, + 3, + 2, + 12, + 31, + 5, + 4, + 0, + 5, + 1, + 0, + 2, + 7, + 3, + 4, + 4, + 12 + ] + }, + "json-parser": { + "file": "json/parser/parser", + "template": "Parser", + "params": [ + 157, + 13 + ] + }, + "value_object_test": { + "file": "main/json_value_object_test", + "template": "ExtractStringValue", + "params": [ + 88, + 3, + 1, + 0, + 1, + 1, + 1 + ] + } } \ No newline at end of file diff --git a/circuits/json/parser/parser.circom b/circuits/json/parser/parser.circom index c3cc7e8..8a435ca 100644 --- a/circuits/json/parser/parser.circom +++ b/circuits/json/parser/parser.circom @@ -31,23 +31,23 @@ template Parser(DATA_BYTES, MAX_STACK_HEIGHT) { State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string; State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number; - // // Debugging - // for(var i = 0; i { + let circuit: WitnessTester<["data"]>; + + it(`array only input`, async () => { + let filename = "array_only"; + let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, [0]); + + circuit = await circomkit.WitnessTester(`StateUpdate`, { + file: "json/parser/machine", + template: "StateUpdate", + params: [input.length, 2], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + await circuit.expectPass({ + data: input + }); + }); +}) \ No newline at end of file diff --git a/examples/json/test/array_only.json b/examples/json/test/array_only.json new file mode 100644 index 0000000..5e787c3 --- /dev/null +++ b/examples/json/test/array_only.json @@ -0,0 +1,12 @@ +[ + 0, + { + "a": "b" + }, + 1, + [ + 0, + 1 + ], + "b" +] \ No newline at end of file From 9dbc27a4e901f9e9091a0f492c78202020cdf668 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Tue, 8 Oct 2024 22:58:31 +0530 Subject: [PATCH 2/7] feat: add single value extractor --- circuits/json/extractor.circom | 161 ++++++++++++++++++ circuits/json/interpreter.circom | 67 +++++++- .../test/json/extractor/extractor.test.ts | 155 ++++++++++++++++- circuits/test/json/parser/parser.test.ts | 22 ++- 4 files changed, 395 insertions(+), 10 deletions(-) create mode 100644 circuits/json/extractor.circom diff --git a/circuits/json/extractor.circom b/circuits/json/extractor.circom new file mode 100644 index 0000000..7f5fafd --- /dev/null +++ b/circuits/json/extractor.circom @@ -0,0 +1,161 @@ +pragma circom 2.1.9; + +include "interpreter.circom"; + +template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, maxValueLen) { + // Declaration of signals. + signal input data[DATA_BYTES]; + signal input key[keyLen]; + + signal output value[maxValueLen]; + + // Constraints. + signal value_starting_index[DATA_BYTES]; + // flag determining whether this byte is matched value + signal is_value_match[DATA_BYTES]; + // final mask + signal mask[DATA_BYTES]; + + component State[DATA_BYTES]; + State[0] = StateUpdate(MAX_STACK_HEIGHT); + State[0].byte <== data[0]; + for(var i = 0; i < MAX_STACK_HEIGHT; i++) { + State[0].stack[i] <== [0,0]; + } + State[0].parsing_string <== 0; + State[0].parsing_number <== 0; + + signal parsing_key[DATA_BYTES]; + signal parsing_value[DATA_BYTES]; + signal parsing_object_value[DATA_BYTES]; + signal is_key_match[DATA_BYTES]; + signal is_key_match_for_value[DATA_BYTES+1]; + is_key_match_for_value[0] <== 0; + signal is_next_pair_at_depth[DATA_BYTES]; + signal or[DATA_BYTES]; + + // initialise first iteration + + // check inside key or value + parsing_key[0] <== InsideKey(MAX_STACK_HEIGHT)(State[0].next_stack, State[0].next_parsing_string, State[0].next_parsing_number); + parsing_value[0] <== InsideValueObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number); + + is_key_match[0] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, 0)(data, key, 0, parsing_key[0], State[0].next_stack); + is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[0].next_stack, data[0]); + is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); + is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; + + mask[0] <== data[0] * is_value_match[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + State[data_idx] = StateUpdate(MAX_STACK_HEIGHT); + State[data_idx].byte <== data[data_idx]; + State[data_idx].stack <== State[data_idx - 1].next_stack; + State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string; + State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number; + + // - parsing key + // - parsing value (different for string/numbers and array) + // - key match (key 1, key 2) + // - is next pair + // - is key match for value + // - value_mask + // - mask + + // check if inside key or not + parsing_key[data_idx] <== InsideKey(MAX_STACK_HEIGHT)(State[data_idx].next_stack, State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); + // check if inside value + parsing_value[data_idx] <== InsideValueObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); + + // to get correct value, check: + // - key matches at current index and depth of key is as specified + // - whether next KV pair starts + // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) + is_key_match[data_idx] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, 0)(data, key, data_idx, parsing_key[data_idx], State[data_idx].next_stack); + is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[data_idx].next_stack, data[data_idx]); + is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); + is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; + + or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx - 1]); + + // mask = currently parsing value and all subsequent keys matched + mask[data_idx] <== data[data_idx] * or[data_idx]; + } + + // find starting index of value in data by matching mask + signal is_zero_mask[DATA_BYTES]; + signal is_prev_starting_index[DATA_BYTES]; + value_starting_index[0] <== 0; + is_prev_starting_index[0] <== 0; + is_zero_mask[0] <== IsZero()(mask[0]); + for (var i=1 ; i= 2); + + signal input data[DATA_BYTES]; + signal input index; + + signal output value[maxValueLen]; + + // value starting index in `data` + signal value_starting_index[DATA_BYTES]; + // final mask + signal mask[DATA_BYTES]; + + component State[DATA_BYTES]; + State[0] = StateUpdate(MAX_STACK_HEIGHT); + State[0].byte <== data[0]; + for(var i = 0; i < MAX_STACK_HEIGHT; i++) { + State[0].stack[i] <== [0,0]; + } + State[0].parsing_string <== 0; + State[0].parsing_number <== 0; + + signal parsing_array[DATA_BYTES]; + signal or[DATA_BYTES]; + + parsing_array[0] <== InsideArrayIndexObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number, index); + mask[0] <== data[0] * parsing_array[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + State[data_idx] = StateUpdate(MAX_STACK_HEIGHT); + State[data_idx].byte <== data[data_idx]; + State[data_idx].stack <== State[data_idx - 1].next_stack; + State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string; + State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number; + + parsing_array[data_idx] <== InsideArrayIndexObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number, index); + + or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]); + mask[data_idx] <== data[data_idx] * or[data_idx]; + } + + signal is_zero_mask[DATA_BYTES]; + signal is_prev_starting_index[DATA_BYTES]; + value_starting_index[0] <== 0; + is_prev_starting_index[0] <== 0; + is_zero_mask[0] <== IsZero()(mask[0]); + for (var i=1 ; i { await json_circuit.expectPass({ data: inputJson, key1: key[0], key2: key[1], key4: key[3], key5: key[4] }, { value: output }); }); +}); + +describe("array-only", async () => { + let circuit: WitnessTester<["data", "index"], ["value"]>; + let jsonFilename = "array_only"; + let inputJson: number[] = []; + let maxValueLen = 30; + + before(async () => { + let [jsonFile, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + 0 + ] + ); + inputJson = jsonFile; + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ArrayIndexExtractor", + params: [inputJson.length, 2, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + }); + + it("response-matcher index: 0", async () => { + let outputs = [52, 50, 44]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 0 }, { value: outputs }); + }); + + it("response-matcher index: 1", async () => { + let outputs = [123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 97, 34, 58, 32, 34, 98, 34, 10, 32, 32, 32, 32, 125]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 1 }, { value: outputs }); + }); + + it("response-matcher index: 2", async () => { + /* + [ + 0, + 1 + ] + */ + let outputs = [91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 48, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 49, 10, 32, 32, 32, 32, 93]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 2 }, { value: outputs }); + }); + + it("response-matcher index: 3", async () => { + // "foobar" + let outputs = [34, 102, 111, 111, 98, 97, 114, 34]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 3 }, { value: outputs }); + }); +}); + +describe("object-extractor", async () => { + let circuit: WitnessTester<["data", "key"], ["value"]>; + let jsonFilename = "value_object"; + let maxValueLen = 30; + + it("key: \"a\", value: \"{ \"d\" : \"e\", \"e\": \"c\" }\"", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "a" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 1, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // { "d" : "e", "e": "c" } + let outputs = [123, 32, 34, 100, 34, 32, 58, 32, 34, 101, 34, 44, 32, 34, 101, 34, 58, 32, 34, 99, 34, 32, 125]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); + + it("key: \"g\", value: \"{ \"h\": { \"a\": \"c\" }}\"", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "g" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 1, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // { "h": { "a": "c" }} + let outputs = [123, 32, 34, 104, 34, 58, 32, 123, 32, 34, 97, 34, 58, 32, 34, 99, 34, 32, 125, 125]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); + + it("key: \"ab\", value: \"foobar\"", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "ab" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 2, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // "foobar" + let outputs = [34, 102, 111, 111, 98, 97, 114, 34]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); + + it("key: \"ab\", value: 42", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "bc" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 2, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // "foobar" + let outputs = [52, 50]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); }); \ No newline at end of file diff --git a/circuits/test/json/parser/parser.test.ts b/circuits/test/json/parser/parser.test.ts index f266144..eccbc99 100644 --- a/circuits/test/json/parser/parser.test.ts +++ b/circuits/test/json/parser/parser.test.ts @@ -7,9 +7,9 @@ describe("json-parser", () => { let filename = "array_only"; let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, [0]); - circuit = await circomkit.WitnessTester(`StateUpdate`, { - file: "json/parser/machine", - template: "StateUpdate", + circuit = await circomkit.WitnessTester(`Parser`, { + file: "json/parser/parser", + template: "Parser", params: [input.length, 2], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -18,4 +18,20 @@ describe("json-parser", () => { data: input }); }); + + it(`object input`, async () => { + let filename = "value_object"; + let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a"]); + + circuit = await circomkit.WitnessTester(`Parser`, { + file: "json/parser/parser", + template: "Parser", + params: [input.length, 3], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + await circuit.expectPass({ + data: input + }); + }); }) \ No newline at end of file From cbf5595b815832da45c28aa93136c4d38c36f28c Mon Sep 17 00:00:00 2001 From: lonerapier Date: Tue, 8 Oct 2024 22:58:58 +0530 Subject: [PATCH 3/7] test: add example --- circuits/test/common/index.ts | 3 ++- examples/json/test/array_only.json | 5 ++--- examples/json/test/value_object.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/circuits/test/common/index.ts b/circuits/test/common/index.ts index eda2717..ecc66f5 100644 --- a/circuits/test/common/index.ts +++ b/circuits/test/common/index.ts @@ -52,7 +52,8 @@ export function readJSONInputFile(filename: string, key: any[]): [number[], numb input = byteArray; let jsonFile = JSON.parse(data); - let value: string = key.reduce((acc, key) => acc && acc[key], jsonFile).toString(); + let value = key.reduce((acc, key) => acc && acc[key], jsonFile); + value = value.toString(); for (let i = 0; i < value.length; i++) { output.push(value.charCodeAt(i)); } diff --git a/examples/json/test/array_only.json b/examples/json/test/array_only.json index 5e787c3..23f1146 100644 --- a/examples/json/test/array_only.json +++ b/examples/json/test/array_only.json @@ -1,12 +1,11 @@ [ - 0, + 42, { "a": "b" }, - 1, [ 0, 1 ], - "b" + "foobar" ] \ No newline at end of file diff --git a/examples/json/test/value_object.json b/examples/json/test/value_object.json index fdb08e1..bc60bcf 100644 --- a/examples/json/test/value_object.json +++ b/examples/json/test/value_object.json @@ -1 +1 @@ -{ "a": { "d" : "e", "e": "c" }, "e": { "f": "a", "e": "2" }, "g": { "h": { "a": "c" }} } \ No newline at end of file +{ "a": { "d" : "e", "e": "c" }, "e": { "f": "a", "e": "2" }, "g": { "h": { "a": "c" }}, "ab": "foobar", "bc": 42 } \ No newline at end of file From 6984bc9738229df7eda8da3c73fdb4140b463e65 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Wed, 9 Oct 2024 01:33:05 +0530 Subject: [PATCH 4/7] feat: remove keyLen and add maxKeyLen --- circuits/json/extractor.circom | 11 ++- circuits/json/interpreter.circom | 57 +++++++++++ .../test/json/extractor/extractor.test.ts | 95 ++++++------------- .../test/json/extractor/interpreter.test.ts | 56 +++++++++++ circuits/test/utils/search.test.ts | 42 ++++++++ circuits/utils/search.circom | 16 ++++ examples/json/test/value_object.json | 2 +- 7 files changed, 207 insertions(+), 72 deletions(-) diff --git a/circuits/json/extractor.circom b/circuits/json/extractor.circom index 7f5fafd..3a37942 100644 --- a/circuits/json/extractor.circom +++ b/circuits/json/extractor.circom @@ -2,10 +2,13 @@ pragma circom 2.1.9; include "interpreter.circom"; -template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, maxValueLen) { +template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { + assert(MAX_STACK_HEIGHT >= 2); + // Declaration of signals. signal input data[DATA_BYTES]; - signal input key[keyLen]; + signal input key[maxKeyLen]; + signal input keyLen; signal output value[maxValueLen]; @@ -40,7 +43,7 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, maxValueLen) { parsing_key[0] <== InsideKey(MAX_STACK_HEIGHT)(State[0].next_stack, State[0].next_parsing_string, State[0].next_parsing_number); parsing_value[0] <== InsideValueObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number); - is_key_match[0] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, 0)(data, key, 0, parsing_key[0], State[0].next_stack); + is_key_match[0] <== KeyMatchAtDepthWithIndex(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, 0)(data, key, keyLen, 0, parsing_key[0], State[0].next_stack); is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[0].next_stack, data[0]); is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; @@ -71,7 +74,7 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, maxValueLen) { // - key matches at current index and depth of key is as specified // - whether next KV pair starts // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) - is_key_match[data_idx] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, 0)(data, key, data_idx, parsing_key[data_idx], State[data_idx].next_stack); + is_key_match[data_idx] <== KeyMatchAtDepthWithIndex(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, 0)(data, key, keyLen, data_idx, parsing_key[data_idx], State[data_idx].next_stack); is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[data_idx].next_stack, data[data_idx]); is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; diff --git a/circuits/json/interpreter.circom b/circuits/json/interpreter.circom index 4d6a8a9..82c06ec 100644 --- a/circuits/json/interpreter.circom +++ b/circuits/json/interpreter.circom @@ -371,5 +371,62 @@ template KeyMatchAtDepth(dataLen, n, keyLen, depth) { signal is_parsing_correct_key_at_depth <== is_parsing_correct_key * is_key_at_depth; + signal output out <== substring_match * is_parsing_correct_key_at_depth; +} + +/// Matches a JSON key at an `index` using Substring Matching at specified depth +/// +/// # Arguments +/// - `dataLen`: parsed data length +/// - `n`: maximum stack height +/// - `keyLen`: key length +/// - `depth`: depth of key to be matched +/// +/// # Inputs +/// - `data`: data bytes +/// - `key`: key bytes +/// - `r`: random number for substring matching. **Need to be chosen carefully.** +/// - `index`: data index to match from +/// - `parsing_key`: if current byte is inside a key +/// - `stack`: parser stack output +/// +/// # Output +/// - `out`: Returns `1` if `key` matches `data` at `index` +template KeyMatchAtDepthWithIndex(dataLen, n, maxKeyLen, depth) { + signal input data[dataLen]; + signal input key[maxKeyLen]; + signal input keyLen; + signal input index; + signal input parsing_key; + signal input stack[n][2]; + + component topOfStack = GetTopOfStack(n); + topOfStack.stack <== stack; + signal pointer <== topOfStack.pointer; + _ <== topOfStack.value; + + // `"` -> 34 + + // end of key equals `"` + signal end_of_key <== IndexSelector(dataLen)(data, index + keyLen); + signal is_end_of_key_equal_to_quote <== IsEqual()([end_of_key, 34]); + + // start of key equals `"` + signal start_of_key <== IndexSelector(dataLen)(data, index - 1); + signal is_start_of_key_equal_to_quote <== IsEqual()([start_of_key, 34]); + + // key matches + signal substring_match <== SubstringMatchWithIndexx(dataLen, maxKeyLen)(data, key, keyLen, index); + + // key should be a string + signal is_key_between_quotes <== is_start_of_key_equal_to_quote * is_end_of_key_equal_to_quote; + + // is the index given correct? + signal is_parsing_correct_key <== is_key_between_quotes * parsing_key; + // is the key given by index at correct depth? + signal is_key_at_depth <== IsEqual()([pointer-1, depth]); + + signal is_parsing_correct_key_at_depth <== is_parsing_correct_key * is_key_at_depth; + signal output out <== substring_match * is_parsing_correct_key_at_depth; } \ No newline at end of file diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index fb7466b..74576fa 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -291,95 +291,56 @@ describe("array-only", async () => { }); describe("object-extractor", async () => { - let circuit: WitnessTester<["data", "key"], ["value"]>; + let circuit: WitnessTester<["data", "key", "keyLen"], ["value"]>; let jsonFilename = "value_object"; + let jsonFile: number[] = []; + let maxKeyLen = 10; let maxValueLen = 30; - it("key: \"a\", value: \"{ \"d\" : \"e\", \"e\": \"c\" }\"", async () => { + before(async () => { let [inputJson, key, output] = readJSONInputFile( `${jsonFilename}.json`, [ "a" ] ); + jsonFile = inputJson; circuit = await circomkit.WitnessTester(`Extract`, { file: `json/extractor`, template: "ObjectExtractor", - params: [inputJson.length, 3, 1, maxValueLen], + params: [inputJson.length, 3, maxKeyLen, maxValueLen], }); console.log("#constraints:", await circuit.getConstraintCount()); - - // { "d" : "e", "e": "c" } - let outputs = [123, 32, 34, 100, 34, 32, 58, 32, 34, 101, 34, 44, 32, 34, 101, 34, 58, 32, 34, 99, 34, 32, 125]; - outputs.fill(0, outputs.length, maxValueLen); - - await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); }); - it("key: \"g\", value: \"{ \"h\": { \"a\": \"c\" }}\"", async () => { - let [inputJson, key, output] = readJSONInputFile( - `${jsonFilename}.json`, - [ - "g" - ] - ); + function generatePassCase(key: number[], output: number[]) { + output = output.concat(Array(maxValueLen - output.length).fill(0)); + let padded_key = key.concat(Array(maxKeyLen - key.length).fill(0)); - circuit = await circomkit.WitnessTester(`Extract`, { - file: `json/extractor`, - template: "ObjectExtractor", - params: [inputJson.length, 3, 1, maxValueLen], + it(`key: ${key}, output: ${output}`, async () => { + await circuit.expectPass({ data: jsonFile, key: padded_key, keyLen: key.length }, { value: output }); }); - console.log("#constraints:", await circuit.getConstraintCount()); + } - // { "h": { "a": "c" }} - let outputs = [123, 32, 34, 104, 34, 58, 32, 123, 32, 34, 97, 34, 58, 32, 34, 99, 34, 32, 125, 125]; - outputs.fill(0, outputs.length, maxValueLen); + // { "d" : "e", "e": "c" } + let output1 = [123, 32, 34, 100, 34, 32, 58, 32, 34, 101, 34, 44, 32, 34, 101, 34, 58, 32, 34, 99, 34, 32, 125]; + generatePassCase(toByte("a"), output1); - await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); - }); + // { "h": { "a": "c" }} + let output2 = [123, 32, 34, 104, 34, 58, 32, 123, 32, 34, 97, 34, 58, 32, 34, 99, 34, 32, 125, 125]; + generatePassCase(toByte("g"), output2); - it("key: \"ab\", value: \"foobar\"", async () => { - let [inputJson, key, output] = readJSONInputFile( - `${jsonFilename}.json`, - [ - "ab" - ] - ); + // "foobar" + let output3 = [34, 102, 111, 111, 98, 97, 114, 34]; + generatePassCase(toByte("ab"), output3); - circuit = await circomkit.WitnessTester(`Extract`, { - file: `json/extractor`, - template: "ObjectExtractor", - params: [inputJson.length, 3, 2, maxValueLen], - }); - console.log("#constraints:", await circuit.getConstraintCount()); - - // "foobar" - let outputs = [34, 102, 111, 111, 98, 97, 114, 34]; - outputs.fill(0, outputs.length, maxValueLen); - - await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); - }); + // "42" + // TODO: currently number gives an extra byte. Fix this. + let output4 = [52, 50, 44]; + generatePassCase(toByte("bc"), output4); - it("key: \"ab\", value: 42", async () => { - let [inputJson, key, output] = readJSONInputFile( - `${jsonFilename}.json`, - [ - "bc" - ] - ); - - circuit = await circomkit.WitnessTester(`Extract`, { - file: `json/extractor`, - template: "ObjectExtractor", - params: [inputJson.length, 3, 2, maxValueLen], - }); - console.log("#constraints:", await circuit.getConstraintCount()); - - // "foobar" - let outputs = [52, 50]; - outputs.fill(0, outputs.length, maxValueLen); - - await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); - }); + // [ 0, 1, "a"] + let output5 = [91, 32, 48, 44, 32, 49, 44, 32, 34, 97, 34, 93]; + generatePassCase(toByte("dc"), output5); }); \ No newline at end of file diff --git a/circuits/test/json/extractor/interpreter.test.ts b/circuits/test/json/extractor/interpreter.test.ts index bf6a220..44318c8 100644 --- a/circuits/test/json/extractor/interpreter.test.ts +++ b/circuits/test/json/extractor/interpreter.test.ts @@ -354,4 +354,60 @@ describe("Interpreter", async () => { let input7 = { data: input[0], key: input[1][0], index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; generatePassCase(input6, { out: 0 }, 1, "wrong depth"); }); + + describe("KeyMatchAtDepthWithIndex", async () => { + let circuit: WitnessTester<["data", "key", "keyLen", "index", "parsing_key", "stack"], ["out"]>; + let maxKeyLen = 10; + + function generatePassCase(input: any, expected: any, depth: number, desc: string) { + const description = generateDescription(input); + + it(`(valid) witness: ${description} ${desc}`, async () => { + // pad key with 0's + let padded_key = input.key.concat(Array(maxKeyLen - input.key.length).fill(0)); + input.key = padded_key; + + circuit = await circomkit.WitnessTester(`KeyMatchAtDepthWithIndex`, { + file: "json/interpreter", + template: "KeyMatchAtDepthWithIndex", + params: [input.data.length, 4, maxKeyLen, depth], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + await circuit.expectPass(input, expected); + }); + } + + let input = readJSONInputFile("value_array_object.json", ["a", 0, "b", 0]); + + let output = { out: 1 }; + + let key1 = input[1][0]; + let input1 = { data: input[0], key: key1, keyLen: key1.length, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; + generatePassCase(input1, output, 0, ""); + + let key2 = input[1][2]; + let input2 = { data: input[0], key: key2, keyLen: key2.length, index: 8, parsing_key: 1, stack: [[1, 1], [2, 0], [1, 0], [0, 0]] }; + generatePassCase(input2, output, 2, ""); + + let input3 = { data: input[0], key: [99], keyLen: 1, index: 20, parsing_key: 1, stack: [[1, 1], [2, 1], [1, 1], [0, 0]] }; + generatePassCase(input3, output, 2, "wrong stack"); + + // fail cases + + let key4 = input[1][1]; + let input4 = { data: input[0], key: key4, keyLen: key4.length, index: 3, parsing_key: 1, stack: [[1, 0], [2, 0], [1, 0], [0, 0]] }; + generatePassCase(input4, { out: 0 }, 2, "wrong key"); + + let input5 = { data: input[0], key: [97], keyLen: 1, index: 12, parsing_key: 0, stack: [[1, 1], [2, 0], [1, 1], [0, 0]] }; + generatePassCase(input5, { out: 0 }, 3, "not parsing key"); + + let input6Data = input[0].slice(0); + input6Data.splice(1, 1, 35); + let input6 = { data: input6Data, key: input[1][0], keyLen: input[1][0].length, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; + generatePassCase(input6, { out: 0 }, 0, "invalid key (not surrounded by quotes)"); + + let input7 = { data: input[0], key: input[1][0], keyLen: input[1][0].length, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; + generatePassCase(input6, { out: 0 }, 1, "wrong depth"); + }); }); \ No newline at end of file diff --git a/circuits/test/utils/search.test.ts b/circuits/test/utils/search.test.ts index 91f2ded..98b8bd8 100644 --- a/circuits/test/utils/search.test.ts +++ b/circuits/test/utils/search.test.ts @@ -152,6 +152,48 @@ describe("search", () => { }); }); + describe("SubstringMatchWithIndexx", () => { + let circuit: WitnessTester<["data", "key", "keyLen", "start"], ["out"]>; + let maxKeyLen = 30; + + before(async () => { + circuit = await circomkit.WitnessTester(`SubstringSearch`, { + file: "utils/search", + template: "SubstringMatchWithIndexx", + params: [787, maxKeyLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + }); + + it("data = witness.json:data, key = witness.json:key, r = hash(key+data)", async () => { + let key = witness["key"]; + let pad_key = key.concat(Array(maxKeyLen - key.length).fill(0)); + await circuit.expectPass( + { + data: witness["data"], + key: pad_key, + keyLen: witness["key"].length, + start: 6 + }, + { out: 1 }, + ); + }); + + it("data = witness.json:data, key = witness.json:key, r = hash(key+data), output false", async () => { + let key = witness["key"]; + let pad_key = key.concat(Array(maxKeyLen - key.length).fill(0)); + await circuit.expectPass( + { + data: witness["data"], + key: pad_key, + keyLen: witness["key"].length, + start: 98 + }, + { out: 0 } + ); + }); + }); + describe("SubstringMatch", () => { let circuit: WitnessTester<["data", "key"], ["position"]>; diff --git a/circuits/utils/search.circom b/circuits/utils/search.circom index b839fd8..821051d 100644 --- a/circuits/utils/search.circom +++ b/circuits/utils/search.circom @@ -232,6 +232,22 @@ template SubstringMatchWithIndex(dataLen, keyLen) { signal output out <== isStartLessThanMaxLength * isSubarrayMatch; } +template SubstringMatchWithIndexx(dataLen, maxKeyLen) { + signal input data[dataLen]; + signal input key[maxKeyLen]; + signal input keyLen; + signal input start; + + var logDataLen = log2Ceil(dataLen + maxKeyLen + 1); + + signal isStartLessThanMaxLength <== LessThan(logDataLen)([start, dataLen]); + signal index <== start * isStartLessThanMaxLength; + + signal subarray[maxKeyLen] <== SelectSubArray(dataLen, maxKeyLen)(data, index, keyLen); + signal isSubarrayMatch <== IsEqualArray(maxKeyLen)([key, subarray]); + signal output out <== isStartLessThanMaxLength * isSubarrayMatch; +} + /* SubstringMatch: Matches a substring with an input string and returns the position diff --git a/examples/json/test/value_object.json b/examples/json/test/value_object.json index bc60bcf..8437b4d 100644 --- a/examples/json/test/value_object.json +++ b/examples/json/test/value_object.json @@ -1 +1 @@ -{ "a": { "d" : "e", "e": "c" }, "e": { "f": "a", "e": "2" }, "g": { "h": { "a": "c" }}, "ab": "foobar", "bc": 42 } \ No newline at end of file +{ "a": { "d" : "e", "e": "c" }, "e": { "f": "a", "e": "2" }, "g": { "h": { "a": "c" }}, "ab": "foobar", "bc": 42, "dc": [ 0, 1, "a"] } \ No newline at end of file From b9feeeb240ddf867da85198de7d59e73cba4b008 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Thu, 10 Oct 2024 17:43:02 +0530 Subject: [PATCH 5/7] add optimisations from web-prover --- circuits/json/extractor.circom | 38 +++--- circuits/json/interpreter.circom | 111 ++++++++++++------ .../test/json/extractor/extractor.test.ts | 7 +- .../test/json/extractor/interpreter.test.ts | 81 +++++++++---- circuits/utils/array.circom | 23 ++++ src/codegen/json.rs | 4 +- 6 files changed, 177 insertions(+), 87 deletions(-) diff --git a/circuits/json/extractor.circom b/circuits/json/extractor.circom index 3a37942..0b75ab1 100644 --- a/circuits/json/extractor.circom +++ b/circuits/json/extractor.circom @@ -13,13 +13,13 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { signal output value[maxValueLen]; // Constraints. - signal value_starting_index[DATA_BYTES]; + signal value_starting_index[DATA_BYTES - maxKeyLen]; // flag determining whether this byte is matched value - signal is_value_match[DATA_BYTES]; + signal is_value_match[DATA_BYTES - maxKeyLen]; // final mask - signal mask[DATA_BYTES]; + signal mask[DATA_BYTES - maxKeyLen]; - component State[DATA_BYTES]; + component State[DATA_BYTES - maxKeyLen]; State[0] = StateUpdate(MAX_STACK_HEIGHT); State[0].byte <== data[0]; for(var i = 0; i < MAX_STACK_HEIGHT; i++) { @@ -28,29 +28,29 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { State[0].parsing_string <== 0; State[0].parsing_number <== 0; - signal parsing_key[DATA_BYTES]; - signal parsing_value[DATA_BYTES]; - signal parsing_object_value[DATA_BYTES]; - signal is_key_match[DATA_BYTES]; - signal is_key_match_for_value[DATA_BYTES+1]; + signal parsing_key[DATA_BYTES - maxKeyLen]; + signal parsing_value[DATA_BYTES - maxKeyLen]; + signal parsing_object_value[DATA_BYTES - maxKeyLen]; + signal is_key_match[DATA_BYTES - maxKeyLen]; + signal is_key_match_for_value[DATA_BYTES+1 - maxKeyLen]; is_key_match_for_value[0] <== 0; - signal is_next_pair_at_depth[DATA_BYTES]; - signal or[DATA_BYTES]; + signal is_next_pair_at_depth[DATA_BYTES - maxKeyLen]; + signal or[DATA_BYTES - maxKeyLen]; // initialise first iteration // check inside key or value - parsing_key[0] <== InsideKey(MAX_STACK_HEIGHT)(State[0].next_stack, State[0].next_parsing_string, State[0].next_parsing_number); + parsing_key[0] <== InsideKey()(State[0].next_stack[0], State[0].next_parsing_string, State[0].next_parsing_number); parsing_value[0] <== InsideValueObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number); - is_key_match[0] <== KeyMatchAtDepthWithIndex(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, 0)(data, key, keyLen, 0, parsing_key[0], State[0].next_stack); + is_key_match[0] <== 0; is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[0].next_stack, data[0]); is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; mask[0] <== data[0] * is_value_match[0]; - for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + for(var data_idx = 1; data_idx < DATA_BYTES - maxKeyLen; data_idx++) { State[data_idx] = StateUpdate(MAX_STACK_HEIGHT); State[data_idx].byte <== data[data_idx]; State[data_idx].stack <== State[data_idx - 1].next_stack; @@ -66,7 +66,7 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { // - mask // check if inside key or not - parsing_key[data_idx] <== InsideKey(MAX_STACK_HEIGHT)(State[data_idx].next_stack, State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); + parsing_key[data_idx] <== InsideKey()(State[data_idx].next_stack[0], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); // check if inside value parsing_value[data_idx] <== InsideValueObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); @@ -74,7 +74,7 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { // - key matches at current index and depth of key is as specified // - whether next KV pair starts // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) - is_key_match[data_idx] <== KeyMatchAtDepthWithIndex(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, 0)(data, key, keyLen, data_idx, parsing_key[data_idx], State[data_idx].next_stack); + is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, maxKeyLen, data_idx)(data, key, keyLen, parsing_key[data_idx]); is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[data_idx].next_stack, data[data_idx]); is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; @@ -91,14 +91,14 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { value_starting_index[0] <== 0; is_prev_starting_index[0] <== 0; is_zero_mask[0] <== IsZero()(mask[0]); - for (var i=1 ; i 34 - // end of key equals `"` - signal end_of_key <== IndexSelector(dataLen)(data, index + keyLen); - signal is_end_of_key_equal_to_quote <== IsEqual()([end_of_key, 34]); - - // start of key equals `"` - signal start_of_key <== IndexSelector(dataLen)(data, index - 1); - signal is_start_of_key_equal_to_quote <== IsEqual()([start_of_key, 34]); + // start of key equal to quote + signal startOfKeyEqualToQuote <== IsEqual()([data[index - 1], 34]); + signal isParsingCorrectKey <== parsing_key * startOfKeyEqualToQuote; // key matches - signal substring_match <== SubstringMatchWithIndexx(dataLen, maxKeyLen)(data, key, keyLen, index); - - // key should be a string - signal is_key_between_quotes <== is_start_of_key_equal_to_quote * is_end_of_key_equal_to_quote; - - // is the index given correct? - signal is_parsing_correct_key <== is_key_between_quotes * parsing_key; - // is the key given by index at correct depth? - signal is_key_at_depth <== IsEqual()([pointer-1, depth]); - - signal is_parsing_correct_key_at_depth <== is_parsing_correct_key * is_key_at_depth; - - signal output out <== substring_match * is_parsing_correct_key_at_depth; + component isSubstringMatch = MatchPaddedKey(maxKeyLen); + isSubstringMatch.in[0] <== key; + isSubstringMatch.keyLen <== keyLen; + for(var matcher_idx = 0; matcher_idx < maxKeyLen; matcher_idx++) { + isSubstringMatch.in[1][matcher_idx] <== data[index + matcher_idx]; + } + + signal output out <== isSubstringMatch.out * isParsingCorrectKey; } \ No newline at end of file diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index 74576fa..30738e4 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -294,7 +294,8 @@ describe("object-extractor", async () => { let circuit: WitnessTester<["data", "key", "keyLen"], ["value"]>; let jsonFilename = "value_object"; let jsonFile: number[] = []; - let maxKeyLen = 10; + let maxDataLen = 200; + let maxKeyLen = 3; let maxValueLen = 30; before(async () => { @@ -304,12 +305,12 @@ describe("object-extractor", async () => { "a" ] ); - jsonFile = inputJson; + jsonFile = inputJson.concat(Array(maxDataLen - inputJson.length).fill(0)); circuit = await circomkit.WitnessTester(`Extract`, { file: `json/extractor`, template: "ObjectExtractor", - params: [inputJson.length, 3, maxKeyLen, maxValueLen], + params: [maxDataLen, 3, maxKeyLen, maxValueLen], }); console.log("#constraints:", await circuit.getConstraintCount()); }); diff --git a/circuits/test/json/extractor/interpreter.test.ts b/circuits/test/json/extractor/interpreter.test.ts index 44318c8..219beac 100644 --- a/circuits/test/json/extractor/interpreter.test.ts +++ b/circuits/test/json/extractor/interpreter.test.ts @@ -2,13 +2,13 @@ import { circomkit, WitnessTester, generateDescription, readJSONInputFile } from import { PoseidonModular } from "../../common/poseidon"; describe("Interpreter", async () => { - describe("InsideKey", async () => { + describe("InsideKeyAtTop", async () => { let circuit: WitnessTester<["stack", "parsing_string", "parsing_number"], ["out"]>; before(async () => { - circuit = await circomkit.WitnessTester(`InsideKey`, { + circuit = await circomkit.WitnessTester(`InsideKeyAtTop`, { file: "json/interpreter", - template: "InsideKey", + template: "InsideKeyAtTop", params: [4], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -41,6 +41,38 @@ describe("Interpreter", async () => { generatePassCase(input5, { out: 0 }, "parsing number as a key"); }); + describe("InsideKey", async () => { + let circuit: WitnessTester<["stack", "parsing_string", "parsing_number"], ["out"]>; + + before(async () => { + circuit = await circomkit.WitnessTester(`InsideKey`, { + file: "json/interpreter", + template: "InsideKey", + }); + console.log("#constraints:", await circuit.getConstraintCount()); + }); + + function generatePassCase(input: any, expected: any, desc: string) { + const description = generateDescription(input); + + it(`(valid) witness: ${description} ${desc}`, async () => { + await circuit.expectPass(input, expected); + }); + } + + let input1 = { stack: [1, 0], parsing_string: 1, parsing_number: 0 }; + let output = { out: 1 }; + generatePassCase(input1, output, ""); + + // fail cases + + let input2 = { stack: [1, 1], parsing_string: 1, parsing_number: 0 }; + generatePassCase(input2, { out: 0 }, "invalid stack"); + + let input3 = { stack: [1, 0], parsing_string: 1, parsing_number: 1 }; + generatePassCase(input3, { out: 0 }, "parsing number as a key"); + }); + describe("InsideValueAtTop", async () => { let circuit: WitnessTester<["stack", "parsing_string", "parsing_number"], ["out"]>; @@ -355,11 +387,11 @@ describe("Interpreter", async () => { generatePassCase(input6, { out: 0 }, 1, "wrong depth"); }); - describe("KeyMatchAtDepthWithIndex", async () => { - let circuit: WitnessTester<["data", "key", "keyLen", "index", "parsing_key", "stack"], ["out"]>; - let maxKeyLen = 10; + describe("KeyMatchAtIndex", async () => { + let circuit: WitnessTester<["data", "key", "keyLen", "parsing_key"], ["out"]>; + let maxKeyLen = 3; - function generatePassCase(input: any, expected: any, depth: number, desc: string) { + function generatePassCase(input: any, expected: any, index: number, desc: string) { const description = generateDescription(input); it(`(valid) witness: ${description} ${desc}`, async () => { @@ -367,10 +399,10 @@ describe("Interpreter", async () => { let padded_key = input.key.concat(Array(maxKeyLen - input.key.length).fill(0)); input.key = padded_key; - circuit = await circomkit.WitnessTester(`KeyMatchAtDepthWithIndex`, { + circuit = await circomkit.WitnessTester(`KeyMatchAtIndex`, { file: "json/interpreter", - template: "KeyMatchAtDepthWithIndex", - params: [input.data.length, 4, maxKeyLen, depth], + template: "KeyMatchAtIndex", + params: [input.data.length, maxKeyLen, index], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -383,31 +415,32 @@ describe("Interpreter", async () => { let output = { out: 1 }; let key1 = input[1][0]; - let input1 = { data: input[0], key: key1, keyLen: key1.length, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; - generatePassCase(input1, output, 0, ""); + let input1 = { data: input[0], key: key1, keyLen: key1.length, parsing_key: 1 }; + generatePassCase(input1, output, 2, ""); let key2 = input[1][2]; - let input2 = { data: input[0], key: key2, keyLen: key2.length, index: 8, parsing_key: 1, stack: [[1, 1], [2, 0], [1, 0], [0, 0]] }; - generatePassCase(input2, output, 2, ""); + let input2 = { data: input[0], key: key2, keyLen: key2.length, parsing_key: 1 }; + generatePassCase(input2, output, 8, ""); - let input3 = { data: input[0], key: [99], keyLen: 1, index: 20, parsing_key: 1, stack: [[1, 1], [2, 1], [1, 1], [0, 0]] }; - generatePassCase(input3, output, 2, "wrong stack"); + let input3 = { data: input[0], key: [99], keyLen: 1, parsing_key: 1 }; + generatePassCase(input3, output, 20, "wrong stack"); // fail cases + let failOutput = { out: 0 }; let key4 = input[1][1]; - let input4 = { data: input[0], key: key4, keyLen: key4.length, index: 3, parsing_key: 1, stack: [[1, 0], [2, 0], [1, 0], [0, 0]] }; - generatePassCase(input4, { out: 0 }, 2, "wrong key"); + let input4 = { data: input[0], key: key4, keyLen: key4.length, parsing_key: 1 }; + generatePassCase(input4, failOutput, 3, "wrong key"); - let input5 = { data: input[0], key: [97], keyLen: 1, index: 12, parsing_key: 0, stack: [[1, 1], [2, 0], [1, 1], [0, 0]] }; - generatePassCase(input5, { out: 0 }, 3, "not parsing key"); + let input5 = { data: input[0], key: [97], keyLen: 1, parsing_key: 0 }; + generatePassCase(input5, failOutput, 12, "not parsing key"); let input6Data = input[0].slice(0); input6Data.splice(1, 1, 35); - let input6 = { data: input6Data, key: input[1][0], keyLen: input[1][0].length, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; - generatePassCase(input6, { out: 0 }, 0, "invalid key (not surrounded by quotes)"); + let input6 = { data: input6Data, key: input[1][0], keyLen: input[1][0].length, parsing_key: 1 }; + generatePassCase(input6, failOutput, 2, "invalid key (not surrounded by quotes)"); - let input7 = { data: input[0], key: input[1][0], keyLen: input[1][0].length, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; - generatePassCase(input6, { out: 0 }, 1, "wrong depth"); + let input7 = { data: input[0], key: input[1][0], keyLen: input[1][0].length, parsing_key: 1 }; + generatePassCase(input6, failOutput, 2, "wrong depth"); }); }); \ No newline at end of file diff --git a/circuits/utils/array.circom b/circuits/utils/array.circom index ff45809..d19b4d6 100644 --- a/circuits/utils/array.circom +++ b/circuits/utils/array.circom @@ -36,6 +36,29 @@ template IsEqualArray(n) { out <== totalEqual.out; } +template IsEqualArrayPaddedLHS(n) { + signal input in[2][n]; + signal output out; + + var accum = 0; + component equalComponent[n]; + component isPaddedElement[n]; + + for(var i = 0; i < n; i++) { + isPaddedElement[i] = IsZero(); + isPaddedElement[i].in <== in[0][i]; + equalComponent[i] = IsEqual(); + equalComponent[i].in[0] <== in[0][i]; + equalComponent[i].in[1] <== in[1][i] * (1-isPaddedElement[i].out); + accum += equalComponent[i].out; + } + + component totalEqual = IsEqual(); + totalEqual.in[0] <== n; + totalEqual.in[1] <== accum; + out <== totalEqual.out; +} + // TODO: There should be a way to have the below assertion come from the field itself. /* This template is an indicator for if an array contains an element. diff --git a/src/codegen/json.rs b/src/codegen/json.rs index a90d545..4fc726f 100644 --- a/src/codegen/json.rs +++ b/src/codegen/json.rs @@ -383,7 +383,7 @@ fn build_json_circuit( // parsing_key and parsing_object{i}_value circuit_buffer += r#" // initialise first iteration - parsing_key[0] <== InsideKey(MAX_STACK_HEIGHT)(State[0].next_stack, State[0].next_parsing_string, State[0].next_parsing_number); + parsing_key[0] <== InsideKeyAtTop(MAX_STACK_HEIGHT)(State[0].next_stack, State[0].next_parsing_string, State[0].next_parsing_number); "#; @@ -487,7 +487,7 @@ fn build_json_circuit( // - mask // check if inside key or not - parsing_key[data_idx] <== InsideKey(MAX_STACK_HEIGHT)(State[data_idx].next_stack, State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); + parsing_key[data_idx] <== InsideKeyAtTop(MAX_STACK_HEIGHT)(State[data_idx].next_stack, State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); "#; From 18e65fa84fc536d1724f3651abe4cd8a2a99edf7 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Thu, 17 Oct 2024 16:30:37 -0700 Subject: [PATCH 6/7] `depth` as signal in `NextKVPairAtDepth` --- circuits/json/interpreter.circom | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circuits/json/interpreter.circom b/circuits/json/interpreter.circom index 9015af7..ce0e34c 100644 --- a/circuits/json/interpreter.circom +++ b/circuits/json/interpreter.circom @@ -275,9 +275,10 @@ template NextKVPair(n) { /// /// # Output /// - `out`: Returns `1` for next key-value pair at specified depth. -template NextKVPairAtDepth(n, depth) { +template NextKVPairAtDepth(n) { signal input stack[n][2]; signal input currByte; + signal input depth; signal output out; var logMaxDepth = log2Ceil(n+1); From 624ea55792414c8a1bf1ee1c5a55c77bfa793e7a Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Thu, 17 Oct 2024 16:47:11 -0700 Subject: [PATCH 7/7] fix broken tests --- circuits.json | 190 +++++++++--------- circuits/json/extractor.circom | 4 +- .../test/json/extractor/interpreter.test.ts | 22 +- src/codegen/json.rs | 4 +- 4 files changed, 110 insertions(+), 110 deletions(-) diff --git a/circuits.json b/circuits.json index 699d488..3ae4177 100644 --- a/circuits.json +++ b/circuits.json @@ -1,37 +1,51 @@ { - "two_keys_test": { - "file": "main/json_two_keys_test", - "template": "ExtractStringValue", + "value_number_test": { + "file": "main/json_value_number_test", + "template": "ExtractNumValue", "params": [ - 40, + 12, + 1, 1, - 4, 0, - 3 + 2 ] }, - "value_array_number_test": { - "file": "main/json_value_array_number_test", - "template": "ExtractNumValue", + "json-parser": { + "file": "json/parser/parser", + "template": "Parser", "params": [ - 73, - 2, - 1, - 0, - 2, - 1, - 4 + 157, + 13 ] }, - "value_number_test": { - "file": "main/json_value_number_test", - "template": "ExtractNumValue", + "value_string_test": { + "file": "main/json_value_string_test", + "template": "ExtractStringValue", "params": [ 12, 1, 1, 0, - 2 + 1 + ] + }, + "spotify_test": { + "file": "main/json_spotify_test", + "template": "ExtractStringValue", + "params": [ + 85, + 5, + 4, + 0, + 5, + 1, + 0, + 2, + 7, + 3, + 4, + 4, + 12 ] }, "http-parser": { @@ -41,31 +55,30 @@ 60 ] }, - "get_request_test": { - "file": "main/http_get_request_test", - "template": "LockHTTPRequest", + "spotify_top_artists_test": { + "file": "main/http_spotify_top_artists_test", + "template": "LockHTTPResponse", "params": [ - 60, - 3, - 4, + 203, + 85, 8, - 6, - 16, - 4, - 9 + 3, + 2, + 12, + 31 ] }, - "value_array_string_test": { - "file": "main/json_value_array_string_test", - "template": "ExtractStringValue", + "value_array_number_test": { + "file": "main/json_value_array_number_test", + "template": "ExtractNumValue", "params": [ 73, 2, 1, 0, + 2, 1, - 1, - 2 + 4 ] }, "value_array_object_test": { @@ -85,60 +98,40 @@ 1 ] }, - "spotify_top_artists_test": { - "file": "main/http_spotify_top_artists_test", - "template": "LockHTTPResponse", - "params": [ - 203, - 85, - 8, - 3, - 2, - 12, - 31 - ] - }, - "spotify_test": { - "file": "main/json_spotify_test", + "value_string": { + "file": "main/json_value_string", "template": "ExtractStringValue", "params": [ - 85, - 5, - 4, - 0, - 5, + 12, + 1, 1, 0, - 2, - 7, - 3, - 4, - 4, - 12 + 1 ] }, - "value_string": { - "file": "main/json_value_string", + "two_keys_test": { + "file": "main/json_two_keys_test", "template": "ExtractStringValue", "params": [ - 12, - 1, + 40, 1, + 4, 0, - 1 + 3 ] }, - "get_response_test": { - "file": "main/http_get_response_test", - "template": "LockHTTPResponse", + "get_request_test": { + "file": "main/http_get_request_test", + "template": "LockHTTPRequest", "params": [ - 89, - 18, - 8, + 60, 3, - 2, - 12, - 16 + 4, + 8, + 6, + 16, + 4, + 9 ] }, "value_array_nested_test": { @@ -156,17 +149,6 @@ 1 ] }, - "value_string_test": { - "file": "main/json_value_string_test", - "template": "ExtractStringValue", - "params": [ - 12, - 1, - 1, - 0, - 1 - ] - }, "spotify_top_artists": { "file": "main/extended_spotify_top_artists", "template": "HttpJson", @@ -192,19 +174,11 @@ 12 ] }, - "json-parser": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 157, - 13 - ] - }, "value_object_test": { "file": "main/json_value_object_test", "template": "ExtractStringValue", "params": [ - 88, + 134, 3, 1, 0, @@ -212,5 +186,31 @@ 1, 1 ] + }, + "value_array_string_test": { + "file": "main/json_value_array_string_test", + "template": "ExtractStringValue", + "params": [ + 73, + 2, + 1, + 0, + 1, + 1, + 2 + ] + }, + "get_response_test": { + "file": "main/http_get_response_test", + "template": "LockHTTPResponse", + "params": [ + 89, + 18, + 8, + 3, + 2, + 12, + 16 + ] } } \ No newline at end of file diff --git a/circuits/json/extractor.circom b/circuits/json/extractor.circom index 0b75ab1..b58da5c 100644 --- a/circuits/json/extractor.circom +++ b/circuits/json/extractor.circom @@ -44,7 +44,7 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { parsing_value[0] <== InsideValueObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number); is_key_match[0] <== 0; - is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[0].next_stack, data[0]); + is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT)(State[0].next_stack, data[0], 0); is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; @@ -75,7 +75,7 @@ template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) { // - whether next KV pair starts // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, maxKeyLen, data_idx)(data, key, keyLen, parsing_key[data_idx]); - is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[data_idx].next_stack, data[data_idx]); + is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT)(State[data_idx].next_stack, data[data_idx], 0); is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; diff --git a/circuits/test/json/extractor/interpreter.test.ts b/circuits/test/json/extractor/interpreter.test.ts index 219beac..82c3a21 100644 --- a/circuits/test/json/extractor/interpreter.test.ts +++ b/circuits/test/json/extractor/interpreter.test.ts @@ -267,16 +267,16 @@ describe("Interpreter", async () => { }); describe("NextKVPairAtDepth", async () => { - let circuit: WitnessTester<["stack", "currByte"], ["out"]>; + let circuit: WitnessTester<["stack", "currByte", "depth"], ["out"]>; - function generatePassCase(input: any, expected: any, depth: number, desc: string) { + function generatePassCase(input: any, expected: any, desc: string) { const description = generateDescription(input); it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`NextKVPairAtDepth`, { file: "json/interpreter", template: "NextKVPairAtDepth", - params: [4, depth], + params: [4], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -284,20 +284,20 @@ describe("Interpreter", async () => { }); } - let input1 = { stack: [[1, 0], [2, 0], [3, 1], [1, 0]], currByte: 44 }; + let input1 = { stack: [[1, 0], [2, 0], [3, 1], [1, 0]], currByte: 44, depth: 3 }; // output = 1 represents correct execution let output = { out: 1 }; - generatePassCase(input1, output, 3, ""); + generatePassCase(input1, output, ""); // key depth is 2, and even if new-kv pair starts at depth greater than 2, it returns 0. - let input2 = { stack: [[1, 0], [2, 0], [1, 1], [1, 0]], currByte: 44 }; - generatePassCase(input2, { out: 0 }, 2, ""); + let input2 = { stack: [[1, 0], [2, 0], [1, 1], [1, 0]], currByte: 44, depth: 2 }; + generatePassCase(input2, { out: 0 }, ""); - let input3 = { stack: [[1, 0], [1, 0], [0, 0], [0, 0]], currByte: 44 }; - generatePassCase(input3, output, 3, "stack height less than specified"); + let input3 = { stack: [[1, 0], [1, 0], [0, 0], [0, 0]], currByte: 44, depth: 3 }; + generatePassCase(input3, output, "stack height less than specified"); - let input4 = { stack: [[1, 0], [2, 0], [1, 0], [0, 0]], currByte: 34 }; - generatePassCase(input4, { out: 0 }, 2, "incorrect currByte"); + let input4 = { stack: [[1, 0], [2, 0], [1, 0], [0, 0]], currByte: 34, depth: 2 }; + generatePassCase(input4, { out: 0 }, "incorrect currByte"); }); describe("KeyMatch", async () => { diff --git a/src/codegen/json.rs b/src/codegen/json.rs index 4fc726f..3e9df17 100644 --- a/src/codegen/json.rs +++ b/src/codegen/json.rs @@ -422,7 +422,7 @@ fn build_json_circuit( Key::String(_) => { num_objects += 1; circuit_buffer += &format!(" is_key{}_match[0] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen{}, depth{})(data, key{}, 0, parsing_key[0], State[0].next_stack);\n", i+1, i+1, i+1, i+1); - circuit_buffer += &format!(" is_next_pair_at_depth{}[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth{})(State[0].next_stack, data[0]);\n", i+1, i+1); + circuit_buffer += &format!(" is_next_pair_at_depth{}[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT)(State[0].next_stack, data[0], depth{});\n", i+1, i+1); circuit_buffer += &format!(" is_key{}_match_for_value[1] <== Mux1()([is_key{}_match_for_value[0] * (1-is_next_pair_at_depth{}[0]), is_key{}_match[0] * (1-is_next_pair_at_depth{}[0])], is_key{}_match[0]);\n", i+1, i+1, i+1, i+1, i+1, i+1); if debug { circuit_buffer += &format!(" // log(\"is_key{}_match_for_value\", is_key{}_match_for_value[1]);\n\n", i + 1, i + 1); @@ -572,7 +572,7 @@ fn build_json_circuit( Key::String(_) => { num_objects += 1; circuit_buffer += &format!(" is_key{}_match[data_idx] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen{}, depth{})(data, key{}, data_idx, parsing_key[data_idx], State[data_idx].next_stack);\n", i+1, i+1, i+1, i+1); - circuit_buffer += &format!(" is_next_pair_at_depth{}[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth{})(State[data_idx].next_stack, data[data_idx]);\n", i+1, i+1); + circuit_buffer += &format!(" is_next_pair_at_depth{}[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT)(State[data_idx].next_stack, data[data_idx], depth{});\n", i+1, i+1); circuit_buffer += &format!(" is_key{}_match_for_value[data_idx+1] <== Mux1()([is_key{}_match_for_value[data_idx] * (1-is_next_pair_at_depth{}[data_idx]), is_key{}_match[data_idx] * (1-is_next_pair_at_depth{}[data_idx])], is_key{}_match[data_idx]);\n", i+1, i+1, i+1, i+1, i+1, i+1); if debug { circuit_buffer += &format!(" // log(\"is_key{}_match_for_value\", is_key{}_match_for_value[data_idx+1]);\n\n", i + 1, i + 1);