diff --git a/circuits/json/extractor.circom b/circuits/json/extractor.circom new file mode 100644 index 0000000..7f5fafd --- /dev/null +++ b/circuits/json/extractor.circom @@ -0,0 +1,161 @@ +pragma circom 2.1.9; + +include "interpreter.circom"; + +template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, maxValueLen) { + // Declaration of signals. + signal input data[DATA_BYTES]; + signal input key[keyLen]; + + signal output value[maxValueLen]; + + // Constraints. + signal value_starting_index[DATA_BYTES]; + // flag determining whether this byte is matched value + signal is_value_match[DATA_BYTES]; + // final mask + signal mask[DATA_BYTES]; + + component State[DATA_BYTES]; + State[0] = StateUpdate(MAX_STACK_HEIGHT); + State[0].byte <== data[0]; + for(var i = 0; i < MAX_STACK_HEIGHT; i++) { + State[0].stack[i] <== [0,0]; + } + State[0].parsing_string <== 0; + State[0].parsing_number <== 0; + + signal parsing_key[DATA_BYTES]; + signal parsing_value[DATA_BYTES]; + signal parsing_object_value[DATA_BYTES]; + signal is_key_match[DATA_BYTES]; + signal is_key_match_for_value[DATA_BYTES+1]; + is_key_match_for_value[0] <== 0; + signal is_next_pair_at_depth[DATA_BYTES]; + signal or[DATA_BYTES]; + + // initialise first iteration + + // check inside key or value + parsing_key[0] <== InsideKey(MAX_STACK_HEIGHT)(State[0].next_stack, State[0].next_parsing_string, State[0].next_parsing_number); + parsing_value[0] <== InsideValueObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number); + + is_key_match[0] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, 0)(data, key, 0, parsing_key[0], State[0].next_stack); + is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[0].next_stack, data[0]); + is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); + is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; + + mask[0] <== data[0] * is_value_match[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + State[data_idx] = StateUpdate(MAX_STACK_HEIGHT); + State[data_idx].byte <== data[data_idx]; + State[data_idx].stack <== State[data_idx - 1].next_stack; + State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string; + State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number; + + // - parsing key + // - parsing value (different for string/numbers and array) + // - key match (key 1, key 2) + // - is next pair + // - is key match for value + // - value_mask + // - mask + + // check if inside key or not + parsing_key[data_idx] <== InsideKey(MAX_STACK_HEIGHT)(State[data_idx].next_stack, State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); + // check if inside value + parsing_value[data_idx] <== InsideValueObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); + + // to get correct value, check: + // - key matches at current index and depth of key is as specified + // - whether next KV pair starts + // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) + is_key_match[data_idx] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, 0)(data, key, data_idx, parsing_key[data_idx], State[data_idx].next_stack); + is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[data_idx].next_stack, data[data_idx]); + is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); + is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; + + or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx - 1]); + + // mask = currently parsing value and all subsequent keys matched + mask[data_idx] <== data[data_idx] * or[data_idx]; + } + + // find starting index of value in data by matching mask + signal is_zero_mask[DATA_BYTES]; + signal is_prev_starting_index[DATA_BYTES]; + value_starting_index[0] <== 0; + is_prev_starting_index[0] <== 0; + is_zero_mask[0] <== IsZero()(mask[0]); + for (var i=1 ; i= 2); + + signal input data[DATA_BYTES]; + signal input index; + + signal output value[maxValueLen]; + + // value starting index in `data` + signal value_starting_index[DATA_BYTES]; + // final mask + signal mask[DATA_BYTES]; + + component State[DATA_BYTES]; + State[0] = StateUpdate(MAX_STACK_HEIGHT); + State[0].byte <== data[0]; + for(var i = 0; i < MAX_STACK_HEIGHT; i++) { + State[0].stack[i] <== [0,0]; + } + State[0].parsing_string <== 0; + State[0].parsing_number <== 0; + + signal parsing_array[DATA_BYTES]; + signal or[DATA_BYTES]; + + parsing_array[0] <== InsideArrayIndexObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number, index); + mask[0] <== data[0] * parsing_array[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + State[data_idx] = StateUpdate(MAX_STACK_HEIGHT); + State[data_idx].byte <== data[data_idx]; + State[data_idx].stack <== State[data_idx - 1].next_stack; + State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string; + State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number; + + parsing_array[data_idx] <== InsideArrayIndexObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number, index); + + or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]); + mask[data_idx] <== data[data_idx] * or[data_idx]; + } + + signal is_zero_mask[DATA_BYTES]; + signal is_prev_starting_index[DATA_BYTES]; + value_starting_index[0] <== 0; + is_prev_starting_index[0] <== 0; + is_zero_mask[0] <== IsZero()(mask[0]); + for (var i=1 ; i { await json_circuit.expectPass({ data: inputJson, key1: key[0], key2: key[1], key4: key[3], key5: key[4] }, { value: output }); }); +}); + +describe("array-only", async () => { + let circuit: WitnessTester<["data", "index"], ["value"]>; + let jsonFilename = "array_only"; + let inputJson: number[] = []; + let maxValueLen = 30; + + before(async () => { + let [jsonFile, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + 0 + ] + ); + inputJson = jsonFile; + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ArrayIndexExtractor", + params: [inputJson.length, 2, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + }); + + it("response-matcher index: 0", async () => { + let outputs = [52, 50, 44]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 0 }, { value: outputs }); + }); + + it("response-matcher index: 1", async () => { + let outputs = [123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 97, 34, 58, 32, 34, 98, 34, 10, 32, 32, 32, 32, 125]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 1 }, { value: outputs }); + }); + + it("response-matcher index: 2", async () => { + /* + [ + 0, + 1 + ] + */ + let outputs = [91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 48, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 49, 10, 32, 32, 32, 32, 93]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 2 }, { value: outputs }); + }); + + it("response-matcher index: 3", async () => { + // "foobar" + let outputs = [34, 102, 111, 111, 98, 97, 114, 34]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, index: 3 }, { value: outputs }); + }); +}); + +describe("object-extractor", async () => { + let circuit: WitnessTester<["data", "key"], ["value"]>; + let jsonFilename = "value_object"; + let maxValueLen = 30; + + it("key: \"a\", value: \"{ \"d\" : \"e\", \"e\": \"c\" }\"", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "a" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 1, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // { "d" : "e", "e": "c" } + let outputs = [123, 32, 34, 100, 34, 32, 58, 32, 34, 101, 34, 44, 32, 34, 101, 34, 58, 32, 34, 99, 34, 32, 125]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); + + it("key: \"g\", value: \"{ \"h\": { \"a\": \"c\" }}\"", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "g" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 1, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // { "h": { "a": "c" }} + let outputs = [123, 32, 34, 104, 34, 58, 32, 123, 32, 34, 97, 34, 58, 32, 34, 99, 34, 32, 125, 125]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); + + it("key: \"ab\", value: \"foobar\"", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "ab" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 2, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // "foobar" + let outputs = [34, 102, 111, 111, 98, 97, 114, 34]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); + + it("key: \"ab\", value: 42", async () => { + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "bc" + ] + ); + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `json/extractor`, + template: "ObjectExtractor", + params: [inputJson.length, 3, 2, maxValueLen], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // "foobar" + let outputs = [52, 50]; + outputs.fill(0, outputs.length, maxValueLen); + + await circuit.expectPass({ data: inputJson, key: key }, { value: outputs }); + }); }); \ No newline at end of file diff --git a/circuits/test/json/parser/parser.test.ts b/circuits/test/json/parser/parser.test.ts index f266144..eccbc99 100644 --- a/circuits/test/json/parser/parser.test.ts +++ b/circuits/test/json/parser/parser.test.ts @@ -7,9 +7,9 @@ describe("json-parser", () => { let filename = "array_only"; let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, [0]); - circuit = await circomkit.WitnessTester(`StateUpdate`, { - file: "json/parser/machine", - template: "StateUpdate", + circuit = await circomkit.WitnessTester(`Parser`, { + file: "json/parser/parser", + template: "Parser", params: [input.length, 2], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -18,4 +18,20 @@ describe("json-parser", () => { data: input }); }); + + it(`object input`, async () => { + let filename = "value_object"; + let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a"]); + + circuit = await circomkit.WitnessTester(`Parser`, { + file: "json/parser/parser", + template: "Parser", + params: [input.length, 3], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + await circuit.expectPass({ + data: input + }); + }); }) \ No newline at end of file