From 7fa287defa581af4e86fe1c074673a70460a163a Mon Sep 17 00:00:00 2001 From: Sambhav Dusad Date: Thu, 26 Sep 2024 23:43:44 +0530 Subject: [PATCH] fix: circuit warnings (#93) * remove slice * remove syntax * feat: json improvements * feat(http): remove syntax use * feat(http): remove hasher substring matching * fix: delete extractor * fix: codegen * feat: remove hashing based substring matching * fix: spotify test * fix: tests --- circuits/http/extractor.circom | 165 --------------- circuits/http/interpreter.circom | 19 +- circuits/http/locker.circom | 8 +- circuits/http/parser/machine.circom | 23 +- circuits/json/interpreter.circom | 46 ++-- circuits/json/parser/machine.circom | 19 +- circuits/test/http/codegen.test.ts | 47 +++++ circuits/test/http/extractor.test.ts | 81 ------- circuits/test/http/interpreter.test.ts | 6 +- .../test/json/extractor/extractor.test.ts | 32 +++ .../test/json/extractor/interpreter.test.ts | 66 +++--- circuits/test/spotify_top_artists.test.ts | 74 ------- circuits/test/utils/array.test.ts | 30 --- circuits/test/utils/search.test.ts | 39 +++- circuits/utils/array.circom | 29 --- circuits/utils/hash.circom | 10 +- circuits/utils/search.circom | 37 +++- docs/json.md | 4 +- src/codegen/http.rs | 136 ++++++++---- src/codegen/json.rs | 198 ++++++++++-------- 20 files changed, 457 insertions(+), 612 deletions(-) delete mode 100644 circuits/http/extractor.circom delete mode 100644 circuits/test/http/extractor.test.ts diff --git a/circuits/http/extractor.circom b/circuits/http/extractor.circom deleted file mode 100644 index 160e8ee..0000000 --- a/circuits/http/extractor.circom +++ /dev/null @@ -1,165 +0,0 @@ -pragma circom 2.1.9; - -include "interpreter.circom"; -include "parser/machine.circom"; -include "../utils/bytes.circom"; -include "../utils/search.circom"; -include "circomlib/circuits/gates.circom"; -include "@zk-email/circuits/utils/array.circom"; - -// TODO: -// - handle CRLF in response data - -template ExtractResponse(DATA_BYTES, maxContentLength) { - signal input data[DATA_BYTES]; - signal output response[maxContentLength]; - - //--------------------------------------------------------------------------------------------// - //-CONSTRAINTS--------------------------------------------------------------------------------// - //--------------------------------------------------------------------------------------------// - component dataASCII = ASCII(DATA_BYTES); - dataASCII.in <== data; - //--------------------------------------------------------------------------------------------// - - // Initialze the parser - component State[DATA_BYTES]; - State[0] = HttpStateUpdate(); - State[0].byte <== data[0]; - State[0].parsing_start <== 1; - State[0].parsing_header <== 0; - State[0].parsing_field_name <== 0; - State[0].parsing_field_value <== 0; - State[0].parsing_body <== 0; - State[0].line_status <== 0; - - signal dataMask[DATA_BYTES]; - dataMask[0] <== 0; - - for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { - State[data_idx] = HttpStateUpdate(); - State[data_idx].byte <== data[data_idx]; - State[data_idx].parsing_start <== State[data_idx - 1].next_parsing_start; - State[data_idx].parsing_header <== State[data_idx - 1].next_parsing_header; - State[data_idx].parsing_field_name <== State[data_idx-1].next_parsing_field_name; - State[data_idx].parsing_field_value <== State[data_idx-1].next_parsing_field_value; - State[data_idx].parsing_body <== State[data_idx - 1].next_parsing_body; - State[data_idx].line_status <== State[data_idx - 1].next_line_status; - - // apply body mask to data - dataMask[data_idx] <== data[data_idx] * State[data_idx].next_parsing_body; - - // Debugging - log("State[", data_idx, "].parsing_start ", "= ", State[data_idx].parsing_start); - log("State[", data_idx, "].parsing_header ", "= ", State[data_idx].parsing_header); - log("State[", data_idx, "].parsing_field_name ", "= ", State[data_idx].parsing_field_name); - log("State[", data_idx, "].parsing_field_value", "= ", State[data_idx].parsing_field_value); - log("State[", data_idx, "].parsing_body ", "= ", State[data_idx].parsing_body); - log("State[", data_idx, "].line_status ", "= ", State[data_idx].line_status); - log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - } - - // Debugging - log("State[", DATA_BYTES, "].parsing_start ", "= ", State[DATA_BYTES-1].next_parsing_start); - log("State[", DATA_BYTES, "].parsing_header ", "= ", State[DATA_BYTES-1].next_parsing_header); - log("State[", DATA_BYTES, "].parsing_field_name ", "= ", State[DATA_BYTES-1].parsing_field_name); - log("State[", DATA_BYTES, "].parsing_field_value", "= ", State[DATA_BYTES-1].parsing_field_value); - log("State[", DATA_BYTES, "].parsing_body ", "= ", State[DATA_BYTES-1].next_parsing_body); - log("State[", DATA_BYTES, "].line_status ", "= ", State[DATA_BYTES-1].next_line_status); - log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - - signal valueStartingIndex[DATA_BYTES]; - signal isZeroMask[DATA_BYTES]; - signal isPrevStartingIndex[DATA_BYTES]; - valueStartingIndex[0] <== 0; - isZeroMask[0] <== IsZero()(dataMask[0]); - for (var i=1 ; i < DATA_BYTES; i++) { - isZeroMask[i] <== IsZero()(dataMask[i]); - isPrevStartingIndex[i] <== IsZero()(valueStartingIndex[i-1]); - valueStartingIndex[i] <== valueStartingIndex[i-1] + i * (1-isZeroMask[i]) * isPrevStartingIndex[i]; - } - - response <== SelectSubArray(DATA_BYTES, maxContentLength)(dataMask, valueStartingIndex[DATA_BYTES-1]+1, maxContentLength); -} - -template ExtractHeaderValue(DATA_BYTES, headerNameLength, maxValueLength) { - signal input data[DATA_BYTES]; - signal input header[headerNameLength]; - - signal output value[maxValueLength]; - - //--------------------------------------------------------------------------------------------// - //-CONSTRAINTS--------------------------------------------------------------------------------// - //--------------------------------------------------------------------------------------------// - component dataASCII = ASCII(DATA_BYTES); - dataASCII.in <== data; - //--------------------------------------------------------------------------------------------// - - // Initialze the parser - component State[DATA_BYTES]; - State[0] = HttpStateUpdate(); - State[0].byte <== data[0]; - State[0].parsing_start <== 1; - State[0].parsing_header <== 0; - State[0].parsing_field_name <== 0; - State[0].parsing_field_value <== 0; - State[0].parsing_body <== 0; - State[0].line_status <== 0; - - signal headerMatch[DATA_BYTES]; - headerMatch[0] <== 0; - signal isHeaderNameMatch[DATA_BYTES]; - isHeaderNameMatch[0] <== 0; - signal readCRLF[DATA_BYTES]; - readCRLF[0] <== 0; - signal valueMask[DATA_BYTES]; - valueMask[0] <== 0; - - for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { - State[data_idx] = HttpStateUpdate(); - State[data_idx].byte <== data[data_idx]; - State[data_idx].parsing_start <== State[data_idx - 1].next_parsing_start; - State[data_idx].parsing_header <== State[data_idx - 1].next_parsing_header; - State[data_idx].parsing_field_name <== State[data_idx-1].next_parsing_field_name; - State[data_idx].parsing_field_value <== State[data_idx-1].next_parsing_field_value; - State[data_idx].parsing_body <== State[data_idx - 1].next_parsing_body; - State[data_idx].line_status <== State[data_idx - 1].next_line_status; - - // apply value mask to data - // TODO: change r - headerMatch[data_idx] <== HeaderFieldNameMatch(DATA_BYTES, headerNameLength)(data, header, 100, data_idx); - readCRLF[data_idx] <== IsEqual()([State[data_idx].line_status, 2]); - isHeaderNameMatch[data_idx] <== Mux1()([isHeaderNameMatch[data_idx-1] * (1-readCRLF[data_idx]), 1], headerMatch[data_idx]); - valueMask[data_idx] <== MultiAND(3)([data[data_idx], isHeaderNameMatch[data_idx], State[data_idx].parsing_field_value]); - - // Debugging - log("State[", data_idx, "].parsing_start ", "= ", State[data_idx].parsing_start); - log("State[", data_idx, "].parsing_header ", "= ", State[data_idx].parsing_header); - log("State[", data_idx, "].parsing_field_name ", "= ", State[data_idx].parsing_field_name); - log("State[", data_idx, "].parsing_field_value", "= ", State[data_idx].parsing_field_value); - log("State[", data_idx, "].parsing_body ", "= ", State[data_idx].parsing_body); - log("State[", data_idx, "].line_status ", "= ", State[data_idx].line_status); - log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - } - - // Debugging - log("State[", DATA_BYTES, "].parsing_start ", "= ", State[DATA_BYTES-1].next_parsing_start); - log("State[", DATA_BYTES, "].parsing_header ", "= ", State[DATA_BYTES-1].next_parsing_header); - log("State[", DATA_BYTES, "].parsing_field_name ", "= ", State[DATA_BYTES-1].parsing_field_name); - log("State[", DATA_BYTES, "].parsing_field_value", "= ", State[DATA_BYTES-1].parsing_field_value); - log("State[", DATA_BYTES, "].parsing_body ", "= ", State[DATA_BYTES-1].next_parsing_body); - log("State[", DATA_BYTES, "].line_status ", "= ", State[DATA_BYTES-1].next_line_status); - log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - - signal valueStartingIndex[DATA_BYTES]; - signal isZeroMask[DATA_BYTES]; - signal isPrevStartingIndex[DATA_BYTES]; - valueStartingIndex[0] <== 0; - isZeroMask[0] <== IsZero()(valueMask[0]); - for (var i=1 ; i 44` - component syntax = Syntax(); - signal isComma <== IsEqual()([currByte, syntax.COMMA]); + // `,` -> 44 + signal isComma <== IsEqual()([currByte, 44]); // pointer <= depth + // TODO: `LessThan` circuit warning signal atLessDepth <== LessEqThan(logMaxDepth)([pointer-1, depth]); // current depth is less than key depth signal isCommaAtDepthLessThanCurrent <== isComma * atLessDepth; @@ -244,19 +245,17 @@ template NextKVPairAtDepth(n, depth) { template KeyMatch(dataLen, keyLen) { signal input data[dataLen]; signal input key[keyLen]; - signal input r; signal input index; signal input parsing_key; - component syntax = Syntax(); - + // `"` -> 34 signal end_of_key <== IndexSelector(dataLen)(data, index + keyLen); - signal is_end_of_key_equal_to_quote <== IsEqual()([end_of_key, syntax.QUOTE]); + signal is_end_of_key_equal_to_quote <== IsEqual()([end_of_key, 34]); signal start_of_key <== IndexSelector(dataLen)(data, index - 1); - signal is_start_of_key_equal_to_quote <== IsEqual()([start_of_key, syntax.QUOTE]); + signal is_start_of_key_equal_to_quote <== IsEqual()([start_of_key, 34]); - signal substring_match <== SubstringMatchWithIndex(dataLen, keyLen)(data, key, r, index); + signal substring_match <== SubstringMatchWithIndex(dataLen, keyLen)(data, key, index); signal is_key_between_quotes <== is_start_of_key_equal_to_quote * is_end_of_key_equal_to_quote; signal is_parsing_correct_key <== is_key_between_quotes * parsing_key; @@ -285,7 +284,6 @@ template KeyMatch(dataLen, keyLen) { template KeyMatchAtDepth(dataLen, n, keyLen, depth) { signal input data[dataLen]; signal input key[keyLen]; - signal input r; signal input index; signal input parsing_key; signal input stack[n][2]; @@ -293,19 +291,20 @@ template KeyMatchAtDepth(dataLen, n, keyLen, depth) { component topOfStack = GetTopOfStack(n); topOfStack.stack <== stack; signal pointer <== topOfStack.pointer; + _ <== topOfStack.value; - component syntax = Syntax(); + // `"` -> 34 // end of key equals `"` signal end_of_key <== IndexSelector(dataLen)(data, index + keyLen); - signal is_end_of_key_equal_to_quote <== IsEqual()([end_of_key, syntax.QUOTE]); + signal is_end_of_key_equal_to_quote <== IsEqual()([end_of_key, 34]); // start of key equals `"` signal start_of_key <== IndexSelector(dataLen)(data, index - 1); - signal is_start_of_key_equal_to_quote <== IsEqual()([start_of_key, syntax.QUOTE]); + signal is_start_of_key_equal_to_quote <== IsEqual()([start_of_key, 34]); // key matches - signal substring_match <== SubstringMatchWithIndex(dataLen, keyLen)(data, key, r, index); + signal substring_match <== SubstringMatchWithIndex(dataLen, keyLen)(data, key, index); // key should be a string signal is_key_between_quotes <== is_start_of_key_equal_to_quote * is_end_of_key_equal_to_quote; @@ -316,7 +315,6 @@ template KeyMatchAtDepth(dataLen, n, keyLen, depth) { signal is_key_at_depth <== IsEqual()([pointer-1, depth]); signal is_parsing_correct_key_at_depth <== is_parsing_correct_key * is_key_at_depth; - // log("key match", index, end_of_key, is_end_of_key_equal_to_quote, substring_match); signal output out <== substring_match * is_parsing_correct_key_at_depth; } \ No newline at end of file diff --git a/circuits/json/parser/machine.circom b/circuits/json/parser/machine.circom index 78a38c6..21c8a87 100644 --- a/circuits/json/parser/machine.circom +++ b/circuits/json/parser/machine.circom @@ -56,29 +56,28 @@ template StateUpdate(MAX_STACK_HEIGHT) { signal output next_parsing_string; signal output next_parsing_number; - component Syntax = Syntax(); component Command = Command(); //--------------------------------------------------------------------------------------------// // Break down what was read // * read in a start brace `{` * component readStartBrace = IsEqual(); - readStartBrace.in <== [byte, Syntax.START_BRACE]; + readStartBrace.in <== [byte, 123]; // * read in an end brace `}` * component readEndBrace = IsEqual(); - readEndBrace.in <== [byte, Syntax.END_BRACE]; + readEndBrace.in <== [byte, 125]; // * read in a start bracket `[` * component readStartBracket = IsEqual(); - readStartBracket.in <== [byte, Syntax.START_BRACKET]; + readStartBracket.in <== [byte, 91]; // * read in an end bracket `]` * component readEndBracket = IsEqual(); - readEndBracket.in <== [byte, Syntax.END_BRACKET]; + readEndBracket.in <== [byte, 93]; // * read in a colon `:` * component readColon = IsEqual(); - readColon.in <== [byte, Syntax.COLON]; + readColon.in <== [byte, 58]; // * read in a comma `,` * component readComma = IsEqual(); - readComma.in <== [byte, Syntax.COMMA]; + readComma.in <== [byte, 44]; // * read in some delimeter * signal readDelimeter <== readStartBrace.out + readEndBrace.out + readStartBracket.out + readEndBracket.out + readColon.out + readComma.out; @@ -88,7 +87,7 @@ template StateUpdate(MAX_STACK_HEIGHT) { readNumber.range <== [48, 57]; // This is the range where ASCII digits are // * read in a quote `"` * component readQuote = IsEqual(); - readQuote.in <== [byte, Syntax.QUOTE]; + readQuote.in <== [byte, 34]; component readOther = IsZero(); readOther.in <== readDelimeter + readNumber.out + readQuote.out; //--------------------------------------------------------------------------------------------// @@ -240,6 +239,7 @@ template GetTopOfStack(n) { atTop.vals[i] <== stack[i]; } atTop.case <== selector; + _ <== atTop.match; value <== atTop.out; pointer <== selector; } @@ -283,9 +283,6 @@ template RewriteStack(n) { signal pointer <== topOfStack.pointer; signal current_value[2] <== topOfStack.value; // * check if we are currently in a value of an object * - component inObjectValue = IsEqualArray(2); - inObjectValue.in[0] <== current_value; - inObjectValue.in[1] <== [1,1]; // * check if value indicates currently in an array * component inArray = IsEqual(); inArray.in[0] <== current_value[0]; diff --git a/circuits/test/http/codegen.test.ts b/circuits/test/http/codegen.test.ts index 2db0a23..16f750d 100644 --- a/circuits/test/http/codegen.test.ts +++ b/circuits/test/http/codegen.test.ts @@ -213,4 +213,51 @@ describe("HTTP :: Codegen :: Response", async () => { circuitInput.value1 = toByte("/aip"); await circuit.expectFail(circuitInput); }); +}); + +describe("spotify_top_artists_http", async () => { + let http_circuit: WitnessTester<["data", "version", "status", "message", "header1", "value1"], ["body"]>; + + it("POST response body", async () => { + let httpLockfile = "spotify.lock" + let httpInputFile = "spotify_top_artists_response.http"; + let httpCircuitName = "spotify_top_artists"; + + await executeCodegen(`${httpCircuitName}_test`, httpInputFile, `${httpLockfile}.json`); + + const lockData = readLockFile(`${httpLockfile}.json`); + + const http = readHTTPInputFile(`${httpInputFile}`); + const inputHttp = http.input; + + const headers = getHeaders(lockData); + + const params = [inputHttp.length, http.bodyBytes.length, lockData.version.length, lockData.status.length, lockData.message.length]; + headers.forEach(header => { + params.push(header[0].length); + params.push(header[1].length); + }); + + http_circuit = await circomkit.WitnessTester(`Extract`, { + file: `main/http_${httpCircuitName}_test`, + template: "LockHTTPResponse", + params: params, + }); + console.log("#constraints:", await http_circuit.getConstraintCount()); + + // match circuit output to original JSON value + const circuitInput: any = { + data: inputHttp, + version: toByte(lockData.version), + status: toByte(lockData.status), + message: toByte(lockData.message), + }; + + headers.forEach((header, index) => { + circuitInput[`header${index + 1}`] = toByte(header[0]); + circuitInput[`value${index + 1}`] = toByte(header[1]); + }); + + await http_circuit.expectPass(circuitInput, { body: http.bodyBytes }); + }); }); \ No newline at end of file diff --git a/circuits/test/http/extractor.test.ts b/circuits/test/http/extractor.test.ts deleted file mode 100644 index 20750f1..0000000 --- a/circuits/test/http/extractor.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { circomkit, WitnessTester, generateDescription, toByte } from "../common"; -import { readHTTPInputFile } from "../common/http"; - -describe("HTTP :: body Extractor", async () => { - let circuit: WitnessTester<["data"], ["response"]>; - - - function generatePassCase(input: number[], expected: any, desc: string) { - const description = generateDescription(input); - - it(`(valid) witness: ${description} ${desc}`, async () => { - circuit = await circomkit.WitnessTester(`ExtractResponseData`, { - file: "http/extractor", - template: "ExtractResponse", - params: [input.length, expected.length], - }); - console.log("#constraints:", await circuit.getConstraintCount()); - - await circuit.expectPass({ data: input }, { response: expected }); - }); - } - - describe("response", async () => { - - let parsedHttp = readHTTPInputFile("get_response.http"); - - generatePassCase(parsedHttp.input, parsedHttp.bodyBytes, ""); - - let output2 = parsedHttp.bodyBytes.slice(0); - output2.push(0, 0, 0, 0); - generatePassCase(parsedHttp.input, output2, "output length more than actual length"); - - let output3 = parsedHttp.bodyBytes.slice(0); - output3.pop(); - output3.pop(); - generatePassCase(parsedHttp.input, output3, "output length less than actual length"); - }); - - describe("request", async () => { - let parsedHttp = readHTTPInputFile("post_request.http"); - - generatePassCase(parsedHttp.input, parsedHttp.bodyBytes, ""); - - let output2 = parsedHttp.bodyBytes.slice(0); - output2.push(0, 0, 0, 0, 0, 0); - generatePassCase(parsedHttp.input, output2, "output length more than actual length"); - - console.log(parsedHttp.bodyBytes.length); - let output3 = parsedHttp.bodyBytes.slice(0); - output3.pop(); - output3.pop(); - generatePassCase(parsedHttp.input, output3, "output length less than actual length"); - }); -}); - -describe("HTTP :: header Extractor", async () => { - let circuit: WitnessTester<["data", "header"], ["value"]>; - - function generatePassCase(input: number[], headerName: number[], headerValue: number[], desc: string) { - const description = generateDescription(input); - - it(`(valid) witness: ${description} ${desc}`, async () => { - circuit = await circomkit.WitnessTester(`ExtractHeaderValue`, { - file: "http/extractor", - template: "ExtractHeaderValue", - params: [input.length, headerName.length, headerValue.length], - }); - console.log("#constraints:", await circuit.getConstraintCount()); - - await circuit.expectPass({ data: input, header: headerName }, { value: headerValue }); - }); - } - - describe("response", async () => { - - let parsedHttp = readHTTPInputFile("get_response.http"); - - generatePassCase(parsedHttp.input, toByte("Content-Length"), toByte(parsedHttp.headers["content-length"]), ""); - }); -}); - diff --git a/circuits/test/http/interpreter.test.ts b/circuits/test/http/interpreter.test.ts index 5986eac..24113c7 100644 --- a/circuits/test/http/interpreter.test.ts +++ b/circuits/test/http/interpreter.test.ts @@ -3,7 +3,7 @@ import { readHTTPInputFile } from "../common/http"; describe("HTTP :: Interpreter", async () => { describe("MethodMatch", async () => { - let circuit: WitnessTester<["data", "method", "r", "index"], []>; + let circuit: WitnessTester<["data", "method", "index"], []>; function generatePassCase(input: number[], method: number[], index: number, desc: string) { const description = generateDescription(input); @@ -16,7 +16,7 @@ describe("HTTP :: Interpreter", async () => { }); console.log("#constraints:", await circuit.getConstraintCount()); - await circuit.expectPass({ data: input, method: method, r: 100, index: index }, {}); + await circuit.expectPass({ data: input, method: method, index: index }, {}); }); } @@ -31,7 +31,7 @@ describe("HTTP :: Interpreter", async () => { }); console.log("#constraints:", await circuit.getConstraintCount()); - await circuit.expectFail({ data: input, method: method, r: 100, index: index }); + await circuit.expectFail({ data: input, method: method, index: index }); }); } diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index ec9860c..f5db3b9 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -197,4 +197,36 @@ describe("ExtractValueArrayObject", () => { await circuit.expectPass({ data: input, key1: keyUnicode[0], key3: keyUnicode[2] }, { value: num }); }); +}); + +describe("spotify_top_artists_json", async () => { + let json_circuit: WitnessTester<["data", "key1", "key2", "key4", "key5"], ["value"]>; + + it("response matcher", async () => { + let jsonFilename = "spotify"; + + await executeCodegen(`${jsonFilename}_test`, `${jsonFilename}.json`, `${jsonFilename}.json`); + + let index_0 = 0; + + let [inputJson, key, output] = readJSONInputFile( + `${jsonFilename}.json`, + [ + "data", + "items", + index_0, + "profile", + "name" + ] + ); + + json_circuit = await circomkit.WitnessTester(`Extract`, { + file: `main/json_${jsonFilename}_test`, + template: "ExtractStringValue", + params: [inputJson.length, 5, 4, 0, 5, 1, index_0, 2, 7, 3, 4, 4, 12], + }); + console.log("#constraints:", await json_circuit.getConstraintCount()); + + await json_circuit.expectPass({ data: inputJson, key1: key[0], key2: key[1], key4: key[3], key5: key[4] }, { value: output }); + }); }); \ No newline at end of file diff --git a/circuits/test/json/extractor/interpreter.test.ts b/circuits/test/json/extractor/interpreter.test.ts index a4e1b1f..bf6a220 100644 --- a/circuits/test/json/extractor/interpreter.test.ts +++ b/circuits/test/json/extractor/interpreter.test.ts @@ -41,13 +41,13 @@ describe("Interpreter", async () => { generatePassCase(input5, { out: 0 }, "parsing number as a key"); }); - describe("InsideValue", async () => { + describe("InsideValueAtTop", async () => { let circuit: WitnessTester<["stack", "parsing_string", "parsing_number"], ["out"]>; before(async () => { - circuit = await circomkit.WitnessTester(`InsideValue`, { + circuit = await circomkit.WitnessTester(`InsideValueAtTop`, { file: "json/interpreter", - template: "InsideValue", + template: "InsideValueAtTop", params: [4], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -80,20 +80,21 @@ describe("Interpreter", async () => { generatePassCase(input5, { out: 0 }, "parsing number and key both"); }); - describe("InsideValueAtDepth", async () => { + describe("InsideValue", async () => { let circuit: WitnessTester<["stack", "parsing_string", "parsing_number"], ["out"]>; function generatePassCase(input: any, expected: any, depth: number, desc: string) { const description = generateDescription(input); it(`(valid) witness: ${description} ${desc}`, async () => { - circuit = await circomkit.WitnessTester(`InsideValueAtDepth`, { + circuit = await circomkit.WitnessTester(`InsideValue`, { file: "json/interpreter", - template: "InsideValueAtDepth", - params: [4, depth], + template: "InsideValue", }); console.log("#constraints:", await circuit.getConstraintCount()); + input.stack = input.stack[depth]; + await circuit.expectPass(input, expected); }); } @@ -117,16 +118,16 @@ describe("Interpreter", async () => { generatePassCase(input5, { out: 0 }, 3, "parsing number and key both"); }); - describe("InsideArrayIndex", async () => { + describe("InsideArrayIndexAtTop", async () => { let circuit: WitnessTester<["stack", "parsing_string", "parsing_number"], ["out"]>; function generatePassCase(input: any, expected: any, index: number, desc: string) { const description = generateDescription(input); it(`(valid) witness: ${description} ${desc}`, async () => { - circuit = await circomkit.WitnessTester(`InsideArrayIndex`, { + circuit = await circomkit.WitnessTester(`InsideArrayIndexAtTop`, { file: "json/interpreter", - template: "InsideArrayIndex", + template: "InsideArrayIndexAtTop", params: [4, index], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -157,20 +158,22 @@ describe("Interpreter", async () => { generatePassCase(input6, { out: 0 }, 3, "incorrect index"); }); - describe("InsideArrayIndexAtDepth", async () => { + describe("InsideArrayIndex", async () => { let circuit: WitnessTester<["stack", "parsing_string", "parsing_number"], ["out"]>; function generatePassCase(input: any, expected: any, index: number, depth: number, desc: string) { const description = generateDescription(input); it(`(valid) witness: ${description} ${desc}`, async () => { - circuit = await circomkit.WitnessTester(`InsideArrayIndexAtDepth`, { + circuit = await circomkit.WitnessTester(`InsideArrayIndex`, { file: "json/interpreter", - template: "InsideArrayIndexAtDepth", - params: [4, index, depth], + template: "InsideArrayIndex", + params: [index], }); console.log("#constraints:", await circuit.getConstraintCount()); + input.stack = input.stack[depth] + await circuit.expectPass(input, expected); }); } @@ -266,7 +269,7 @@ describe("Interpreter", async () => { }); describe("KeyMatch", async () => { - let circuit: WitnessTester<["data", "key", "r", "index", "parsing_key"], ["out"]>; + let circuit: WitnessTester<["data", "key", "index", "parsing_key"], ["out"]>; function generatePassCase(input: any, expected: any, desc: string) { const description = generateDescription(input); @@ -284,30 +287,28 @@ describe("Interpreter", async () => { } let input = readJSONInputFile("value_array_object.json", ["a"]); - const concatenatedInput = input[1][0].concat(input[0]); - const hashResult = PoseidonModular(concatenatedInput); let output = { out: 1 }; - let input1 = { data: input[0], key: input[1][0], r: hashResult, index: 2, parsing_key: 1 }; + let input1 = { data: input[0], key: input[1][0], index: 2, parsing_key: 1 }; generatePassCase(input1, output, ""); - let input2 = { data: input[0], key: [99], r: hashResult, index: 20, parsing_key: 1 }; + let input2 = { data: input[0], key: [99], index: 20, parsing_key: 1 }; generatePassCase(input2, output, ""); // fail cases - let input3 = { data: input[0], key: input[1][0], r: hashResult, index: 3, parsing_key: 1 }; + let input3 = { data: input[0], key: input[1][0], index: 3, parsing_key: 1 }; generatePassCase(input3, { out: 0 }, "wrong index"); - let input4 = { data: input[0], key: [98], r: hashResult, index: 2, parsing_key: 1 }; + let input4 = { data: input[0], key: [98], index: 2, parsing_key: 1 }; generatePassCase(input4, { out: 0 }, "wrong key"); - let input5 = { data: input[0], key: [97], r: hashResult, index: 2, parsing_key: 0 }; + let input5 = { data: input[0], key: [97], index: 2, parsing_key: 0 }; generatePassCase(input5, { out: 0 }, "not parsing key"); }); describe("KeyMatchAtDepth", async () => { - let circuit: WitnessTester<["data", "key", "r", "index", "parsing_key", "stack"], ["out"]>; + let circuit: WitnessTester<["data", "key", "index", "parsing_key", "stack"], ["out"]>; function generatePassCase(input: any, expected: any, depth: number, desc: string) { const description = generateDescription(input); @@ -325,33 +326,32 @@ describe("Interpreter", async () => { } let input = readJSONInputFile("value_array_object.json", ["a", 0, "b", 0]); - const concatenatedInput = input[1][0].concat(input[0]); - const hashResult = PoseidonModular(concatenatedInput); let output = { out: 1 }; - let input1 = { data: input[0], key: input[1][0], r: hashResult, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; + let input1 = { data: input[0], key: input[1][0], index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; generatePassCase(input1, output, 0, ""); - let input2 = { data: input[0], key: input[1][2], r: hashResult, index: 8, parsing_key: 1, stack: [[1, 1], [2, 0], [1, 0], [0, 0]] }; + let input2 = { data: input[0], key: input[1][2], index: 8, parsing_key: 1, stack: [[1, 1], [2, 0], [1, 0], [0, 0]] }; generatePassCase(input2, output, 2, ""); - let input3 = { data: input[0], key: [99], r: hashResult, index: 20, parsing_key: 1, stack: [[1, 1], [2, 1], [1, 1], [0, 0]] }; - generatePassCase(input3, { out: 1 }, 2, "wrong stack"); + let input3 = { data: input[0], key: [99], index: 20, parsing_key: 1, stack: [[1, 1], [2, 1], [1, 1], [0, 0]] }; + generatePassCase(input3, output, 2, "wrong stack"); // fail cases - let input4 = { data: input[0], key: input[1][1], r: hashResult, index: 3, parsing_key: 1, stack: [[1, 0], [2, 0], [1, 0], [0, 0]] }; + let input4 = { data: input[0], key: input[1][1], index: 3, parsing_key: 1, stack: [[1, 0], [2, 0], [1, 0], [0, 0]] }; generatePassCase(input4, { out: 0 }, 2, "wrong key"); - let input5 = { data: input[0], key: [97], r: hashResult, index: 12, parsing_key: 0, stack: [[1, 1], [2, 0], [1, 1], [0, 0]] }; + let input5 = { data: input[0], key: [97], index: 12, parsing_key: 0, stack: [[1, 1], [2, 0], [1, 1], [0, 0]] }; generatePassCase(input5, { out: 0 }, 3, "not parsing key"); let input6Data = input[0].slice(0); - let input6 = { data: input6Data.splice(1, 1, 35), key: input[1][0], r: hashResult, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; + input6Data.splice(1, 1, 35); + let input6 = { data: input6Data, key: input[1][0], index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; generatePassCase(input6, { out: 0 }, 0, "invalid key (not surrounded by quotes)"); - let input7 = { data: input[0], key: input[1][0], r: hashResult, index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; + let input7 = { data: input[0], key: input[1][0], index: 2, parsing_key: 1, stack: [[1, 0], [0, 0], [0, 0], [0, 0]] }; generatePassCase(input6, { out: 0 }, 1, "wrong depth"); }); }); \ No newline at end of file diff --git a/circuits/test/spotify_top_artists.test.ts b/circuits/test/spotify_top_artists.test.ts index 208cfd4..ef4f491 100644 --- a/circuits/test/spotify_top_artists.test.ts +++ b/circuits/test/spotify_top_artists.test.ts @@ -32,80 +32,6 @@ async function extendedLockfileCodegen(circuitName: string, inputFileName: strin }) } -// describe("spotify top artists separate", async () => { -// let http_circuit: WitnessTester<["data", "version", "status", "message", "header1", "value1"], ["body"]>; -// let json_circuit: WitnessTester<["data", "key1", "key2", "key4", "key5"], ["value"]>; - -// it("POST response body extraction", async () => { -// let httpLockfile = "spotify.lock" -// let httpInputFile = "spotify_top_artists_response.http"; -// let httpCircuitName = "spotify_top_artists"; - -// await httpLockfileCodegen(httpCircuitName, httpInputFile, `${httpLockfile}.json`); - -// let jsonFilename = "spotify"; - -// await jsonLockfileCodegen(`${jsonFilename}_test`, `${jsonFilename}.json`, `${jsonFilename}.json`); - -// const lockData = readLockFile(`${httpLockfile}.json`); - -// const http = readHTTPInputFile(`${httpInputFile}`); -// const inputHttp = http.input; - -// const headers = getHttpHeaders(lockData); - -// const params = [inputHttp.length, http.bodyBytes.length, lockData.version.length, lockData.status.length, lockData.message.length]; -// headers.forEach(header => { -// params.push(header[0].length); -// params.push(header[1].length); -// }); - -// http_circuit = await circomkit.WitnessTester(`Extract`, { -// file: `main/http_${httpCircuitName}`, -// template: "LockHTTPResponse", -// params: params, -// }); -// console.log("#constraints:", await http_circuit.getConstraintCount()); - -// // match circuit output to original JSON value -// const circuitInput: any = { -// data: inputHttp, -// version: toByte(lockData.version), -// status: toByte(lockData.status), -// message: toByte(lockData.message), -// }; - -// headers.forEach((header, index) => { -// circuitInput[`header${index + 1}`] = toByte(header[0]); -// circuitInput[`value${index + 1}`] = toByte(header[1]); -// }); - -// await http_circuit.expectPass(circuitInput, { body: http.bodyBytes }); - -// let index_0 = 0; - -// let [inputJson, key, output] = readJSONInputFile( -// `${jsonFilename}.json`, -// [ -// "data", -// "items", -// index_0, -// "profile", -// "name" -// ] -// ); - -// json_circuit = await circomkit.WitnessTester(`Extract`, { -// file: `main/json_${jsonFilename}_test`, -// template: "ExtractStringValue", -// params: [inputJson.length, 5, 4, 0, 5, 1, index_0, 2, 7, 3, 4, 4, 12], -// }); -// console.log("#constraints:", await json_circuit.getConstraintCount()); - -// await json_circuit.expectPass({ data: inputJson, key1: key[0], key2: key[1], key4: key[3], key5: key[4] }, { value: output }); -// }); -// }); - interface JsonLockfile { keys: any[], valueType: string, diff --git a/circuits/test/utils/array.test.ts b/circuits/test/utils/array.test.ts index c951452..02b2fce 100644 --- a/circuits/test/utils/array.test.ts +++ b/circuits/test/utils/array.test.ts @@ -1,34 +1,4 @@ import { circomkit, WitnessTester } from "../common"; - -describe("array", () => { - describe("Slice", () => { - let circuit: WitnessTester<["in"], ["out"]>; - before(async () => { - circuit = await circomkit.WitnessTester(`Slice`, { - file: "utils/array", - template: "Slice", - params: [10, 2, 4], - }); - console.log("#constraints:", await circuit.getConstraintCount()); - }); - - it("witness: [random*10], start: 2, end: 4", async () => { - const input = Array.from({ length: 10 }, () => Math.floor(Math.random() * 256)); - await circuit.expectPass( - { in: input }, - { out: input.slice(2, 4) } - ); - }); - - it("witness: [random*9], start: 2, end: 4", async () => { - const input = Array.from({ length: 9 }, () => Math.floor(Math.random() * 256)); - await circuit.expectFail( - { in: input }, - ); - }); - }); -}); - describe("IsEqualArray", () => { let circuit: WitnessTester<["in"], ["out"]>; before(async () => { diff --git a/circuits/test/utils/search.test.ts b/circuits/test/utils/search.test.ts index e6cbbe1..91f2ded 100644 --- a/circuits/test/utils/search.test.ts +++ b/circuits/test/utils/search.test.ts @@ -80,13 +80,13 @@ describe("search", () => { }); }); - describe("SubstringMatchWithIndex", () => { + describe("SubstringMatchWithHasher", () => { let circuit: WitnessTester<["data", "key", "r", "start"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`SubstringSearch`, { file: "utils/search", - template: "SubstringMatchWithIndex", + template: "SubstringMatchWithHasher", params: [787, 10], }); console.log("#constraints:", await circuit.getConstraintCount()); @@ -117,6 +117,41 @@ describe("search", () => { }); }); + describe("SubstringMatchWithIndex", () => { + let circuit: WitnessTester<["data", "key", "start"], ["out"]>; + + before(async () => { + circuit = await circomkit.WitnessTester(`SubstringSearch`, { + file: "utils/search", + template: "SubstringMatchWithIndex", + params: [787, 10], + }); + console.log("#constraints:", await circuit.getConstraintCount()); + }); + + it("data = witness.json:data, key = witness.json:key, r = hash(key+data)", async () => { + await circuit.expectPass( + { + data: witness["data"], + key: witness["key"], + start: 6 + }, + { out: 1 }, + ); + }); + + it("data = witness.json:data, key = witness.json:key, r = hash(key+data), output false", async () => { + await circuit.expectPass( + { + data: witness["data"], + key: witness["key"], + start: 98 + }, + { out: 0 } + ); + }); + }); + describe("SubstringMatch", () => { let circuit: WitnessTester<["data", "key"], ["position"]>; diff --git a/circuits/utils/array.circom b/circuits/utils/array.circom index 6a9ae48..ff45809 100644 --- a/circuits/utils/array.circom +++ b/circuits/utils/array.circom @@ -2,35 +2,6 @@ pragma circom 2.1.9; include "circomlib/circuits/comparators.circom"; -/// Extract a fixed portion of an array -/// -/// # Note -/// Unlike SelectSubArray, Slice uses compile-time known indices and doesn't pad the output. -/// Slice is more efficient for fixed ranges, while SelectSubArray offers runtime flexibility -/// -/// # Parameters -/// - `n`: The length of the input array -/// - `start`: The starting index of the slice (inclusive) -/// - `end`: The ending index of the slice (exclusive) -/// -/// # Inputs -/// - `in`: The input array of length n -/// -/// # Output -/// - `out`: The sliced array of length (end - start) -template Slice(n, start, end) { - assert(n >= end); - assert(start >= 0); - assert(end >= start); - - signal input in[n]; - signal output out[end - start]; - - for (var i = start; i < end; i++) { - out[i - start] <== in[i]; - } -} - /* This template is an indicator for two equal array inputs. diff --git a/circuits/utils/hash.circom b/circuits/utils/hash.circom index 3d0ef40..e2e1823 100644 --- a/circuits/utils/hash.circom +++ b/circuits/utils/hash.circom @@ -36,10 +36,16 @@ template PoseidonModular(numElements) { if (end > numElements) { // last chunk end = numElements; - var last_chunk[last_chunk_size] = Slice(numElements, start, end)(in); + var last_chunk[last_chunk_size]; + for (var i=start ; i::new(); let re = Regex::new(r":\s+").unwrap(); for &header in headers { - println!("header: {:?}", header); let key_value: Vec<&str> = re.split(header).collect(); assert_eq!(key_value.len(), 2); - println!("key: {:?}", key_value); headers_map.insert(key_value[0].to_string(), key_value[1].to_string()); } @@ -181,7 +179,6 @@ impl HttpData { input_file: &Path, codegen_filename: &str, ) -> Result> { - println!("input_ifle: {:?}", input_file); let input = FileType::Http.read_input(input_file)?; let circuit_template_name = match self { @@ -346,7 +343,6 @@ fn build_http_circuit( signal output body[maxContentLength]; signal bodyMask[DATA_BYTES]; - bodyMask[0] <== 0; "#; } } @@ -366,9 +362,9 @@ fn build_http_circuit( signal targetMask[DATA_BYTES]; signal versionMask[DATA_BYTES]; - var target_start_counter = 1; - var target_end_counter = 1; - var version_end_counter = 1; + var target_start_counter = 0; + var target_end_counter = 0; + var version_end_counter = 0; "#; } HttpData::Response(_) => { @@ -383,16 +379,24 @@ fn build_http_circuit( signal statusMask[DATA_BYTES]; signal messageMask[DATA_BYTES]; - var status_start_counter = 1; - var status_end_counter = 1; - var message_end_counter = 1; + var status_start_counter = 0; + var status_end_counter = 0; + var message_end_counter = 0; "#; } } + + // Create header match signals + { + for (i, _header) in data.headers().iter().enumerate() { + circuit_buffer += + &format!(" signal headerNameValueMatch{}[DATA_BYTES];\n", i + 1); + circuit_buffer += &format!(" var hasMatchedHeaderValue{} = 0;\n\n", i + 1); + } + } } - circuit_buffer += r#" - component State[DATA_BYTES]; + circuit_buffer += r#" component State[DATA_BYTES]; State[0] = HttpStateUpdate(); State[0].byte <== data[0]; State[0].parsing_start <== 1; @@ -401,15 +405,71 @@ fn build_http_circuit( State[0].parsing_field_value <== 0; State[0].parsing_body <== 0; State[0].line_status <== 0; +"#; + // If parsing a `Response`, create a mask of the body bytes + { + if let HttpData::Response(_) = data { + circuit_buffer += r#" + // Mask if parser is in the body of response + bodyMask[0] <== data[0] * State[0].next_parsing_body; "#; + } + } - // Create header match signals + // Start line matches { - for (i, _header) in data.headers().iter().enumerate() { - circuit_buffer += &format!(" signal headerNameValueMatch{}[DATA_BYTES];\n", i + 1); - circuit_buffer += &format!(" headerNameValueMatch{}[0] <== 0;\n", i + 1); - circuit_buffer += &format!(" var hasMatchedHeaderValue{} = 0;\n\n", i + 1); + match data { + HttpData::Request(_) => { + circuit_buffer += r#" + // Check remaining method bytes + // if(data_idx < methodLen) { + // methodIsEqual[data_idx] <== IsEqual()([data[data_idx], method[data_idx]]); + // methodIsEqual[data_idx] === 1; + // } + + // Get the target bytes + startLineMask[0] <== inStartLine()(State[0].next_parsing_start); + targetMask[0] <== inStartMiddle()(State[0].next_parsing_start); + versionMask[0] <== inStartEnd()(State[0].next_parsing_start); + target_start_counter += startLineMask[0] - targetMask[0] - versionMask[0]; + + // Get the version bytes + target_end_counter += startLineMask[0] - versionMask[0]; + version_end_counter += startLineMask[0]; +"#; + } + HttpData::Response(_) => { + circuit_buffer += r#" + // Check remaining version bytes + // if(data_idx < versionLen) { + // versionIsEqual[data_idx] <== IsEqual()([data[data_idx], version[data_idx]]); + // versionIsEqual[data_idx] === 1; + // } + + // Get the status bytes + startLineMask[0] <== inStartLine()(State[0].next_parsing_start); + statusMask[0] <== inStartMiddle()(State[0].next_parsing_start); + messageMask[0] <== inStartEnd()(State[0].next_parsing_start); + status_start_counter += startLineMask[0] - statusMask[0] - messageMask[0]; + + // Get the message bytes + status_end_counter += startLineMask[0] - messageMask[0]; + message_end_counter += startLineMask[0]; +"#; + } + } + + // Header matches + { + for (i, _header) in data.headers().iter().enumerate() { + circuit_buffer += &format!(" headerNameValueMatch{}[0] <== HeaderFieldNameValueMatch(DATA_BYTES, headerNameLen{}, headerValueLen{})(data, header{}, value{}, 0);\n", i + 1, i + 1, i + 1, i + 1, i + 1); + circuit_buffer += &format!( + " hasMatchedHeaderValue{} += headerNameValueMatch{}[0];\n", + i + 1, + i + 1 + ); + } } } @@ -451,9 +511,9 @@ fn build_http_circuit( } // Get the target bytes - startLineMask[data_idx] <== inStartLine()(State[data_idx].parsing_start); - targetMask[data_idx] <== inStartMiddle()(State[data_idx].parsing_start); - versionMask[data_idx] <== inStartEnd()(State[data_idx].parsing_start); + startLineMask[data_idx] <== inStartLine()(State[data_idx].next_parsing_start); + targetMask[data_idx] <== inStartMiddle()(State[data_idx].next_parsing_start); + versionMask[data_idx] <== inStartEnd()(State[data_idx].next_parsing_start); target_start_counter += startLineMask[data_idx] - targetMask[data_idx] - versionMask[data_idx]; // Get the version bytes @@ -470,9 +530,9 @@ fn build_http_circuit( } // Get the status bytes - startLineMask[data_idx] <== inStartLine()(State[data_idx].parsing_start); - statusMask[data_idx] <== inStartMiddle()(State[data_idx].parsing_start); - messageMask[data_idx] <== inStartEnd()(State[data_idx].parsing_start); + startLineMask[data_idx] <== inStartLine()(State[data_idx].next_parsing_start); + statusMask[data_idx] <== inStartMiddle()(State[data_idx].next_parsing_start); + messageMask[data_idx] <== inStartEnd()(State[data_idx].next_parsing_start); status_start_counter += startLineMask[data_idx] - statusMask[data_idx] - messageMask[data_idx]; // Get the message bytes @@ -486,7 +546,7 @@ fn build_http_circuit( // Header matches { for (i, _header) in data.headers().iter().enumerate() { - circuit_buffer += &format!(" headerNameValueMatch{}[data_idx] <== HeaderFieldNameValueMatch(DATA_BYTES, headerNameLen{}, headerValueLen{})(data, header{}, value{}, 100, data_idx);\n", i + 1,i + 1,i + 1,i + 1,i + 1); + circuit_buffer += &format!(" headerNameValueMatch{}[data_idx] <== HeaderFieldNameValueMatch(DATA_BYTES, headerNameLen{}, headerValueLen{})(data, header{}, value{}, data_idx);\n", i + 1, i + 1, i + 1, i + 1, i + 1); circuit_buffer += &format!( " hasMatchedHeaderValue{} += headerNameValueMatch{}[data_idx];\n", i + 1, @@ -509,7 +569,13 @@ fn build_http_circuit( "#; } - circuit_buffer += " }"; + circuit_buffer += " } + + _ <== State[DATA_BYTES-1].next_line_status; + _ <== State[DATA_BYTES-1].next_parsing_start; + _ <== State[DATA_BYTES-1].next_parsing_header; + _ <== State[DATA_BYTES-1].next_parsing_field_name; + _ <== State[DATA_BYTES-1].next_parsing_field_value;\n"; // debugging if debug { @@ -530,10 +596,12 @@ fn build_http_circuit( { if let HttpData::Response(_) = data { circuit_buffer += r#" + signal bodyStartingIndex[DATA_BYTES]; signal isZeroMask[DATA_BYTES]; signal isPrevStartingIndex[DATA_BYTES]; bodyStartingIndex[0] <== 0; + isPrevStartingIndex[0] <== 0; isZeroMask[0] <== IsZero()(bodyMask[0]); for (var i=1 ; i < DATA_BYTES; i++) { isZeroMask[i] <== IsZero()(bodyMask[i]); @@ -560,17 +628,15 @@ fn build_http_circuit( HttpData::Request(_) => { circuit_buffer += r#" // Verify method had correct length - methodLen === target_start_counter - 1; + methodLen === target_start_counter; // Check target is correct by substring match and length check - // TODO: change r - signal targetMatch <== SubstringMatchWithIndex(DATA_BYTES, targetLen)(data, target, 100, target_start_counter); + signal targetMatch <== SubstringMatchWithIndex(DATA_BYTES, targetLen)(data, target, target_start_counter + 1); targetMatch === 1; targetLen === target_end_counter - target_start_counter - 1; // Check version is correct by substring match and length check - // TODO: change r - signal versionMatch <== SubstringMatchWithIndex(DATA_BYTES, versionLen)(data, version, 100, target_end_counter); + signal versionMatch <== SubstringMatchWithIndex(DATA_BYTES, versionLen)(data, version, target_end_counter + 1); versionMatch === 1; // -2 here for the CRLF versionLen === version_end_counter - target_end_counter - 2; @@ -579,17 +645,15 @@ fn build_http_circuit( HttpData::Response(_) => { circuit_buffer += r#" // Verify version had correct length - versionLen === status_start_counter - 1; + versionLen === status_start_counter; // Check status is correct by substring match and length check - // TODO: change r - signal statusMatch <== SubstringMatchWithIndex(DATA_BYTES, statusLen)(data, status, 100, status_start_counter); + signal statusMatch <== SubstringMatchWithIndex(DATA_BYTES, statusLen)(data, status, status_start_counter + 1); statusMatch === 1; statusLen === status_end_counter - status_start_counter - 1; // Check message is correct by substring match and length check - // TODO: change r - signal messageMatch <== SubstringMatchWithIndex(DATA_BYTES, messageLen)(data, message, 100, status_end_counter); + signal messageMatch <== SubstringMatchWithIndex(DATA_BYTES, messageLen)(data, message, status_end_counter + 1); messageMatch === 1; // -2 here for the CRLF messageLen === message_end_counter - status_end_counter - 2; diff --git a/src/codegen/json.rs b/src/codegen/json.rs index 5f99cad..a90d545 100644 --- a/src/codegen/json.rs +++ b/src/codegen/json.rs @@ -5,7 +5,6 @@ use std::{ collections::HashMap, error::Error, fs::{self, create_dir_all}, - str::FromStr, }; use crate::{circuit_config::CircomkitCircuitConfig, ExtractorArgs}; @@ -208,11 +207,11 @@ fn extract_string( } *circuit_buffer += r#" - value <== SelectSubArray(DATA_BYTES, maxValueLen)(data, value_starting_index[DATA_BYTES-2]+1, maxValueLen);"#; + value <== SelectSubArray(DATA_BYTES, maxValueLen)(data, value_starting_index[DATA_BYTES-1]+1, maxValueLen);"#; if debug { *circuit_buffer += r#" - log("value_starting_index", value_starting_index[DATA_BYTES-2]); + log("value_starting_index", value_starting_index[DATA_BYTES-1]+1); for (var i=0 ; i circuit_buffer += &format!(" keyLen{} +", i + 1), - Key::Num(_) => (), - } - } - circuit_buffer.pop(); - circuit_buffer.pop(); - circuit_buffer += ");\n"; - - let mut key_len_counter_str = String::from_str("i")?; - for (i, key) in data.keys.iter().enumerate() { - match key { - Key::String(_) => { - circuit_buffer += &format!(" for (var i = 0 ; i < keyLen{} ; i++) {{\n rHasher.in[{}] <== key{}[i];\n }}\n", i+1, key_len_counter_str, i+1); - key_len_counter_str += &format!(" + keyLen{}", i + 1); - } - Key::Num(_) => (), - } - } - - circuit_buffer += &format!(" for (var i = 0 ; i < DATA_BYTES ; i++) {{\n rHasher.in[{}] <== data[i];\n }}\n", key_len_counter_str); - } - - circuit_buffer += r#" signal r <== rHasher.out; - + circuit_buffer += r#" // value starting index in `data` signal output value_starting_index[DATA_BYTES]; - + // flag determining whether this byte is matched value + signal is_value_match[DATA_BYTES]; + // final mask signal mask[DATA_BYTES]; - // mask[0] <== 0; - - var logDataLen = log2Ceil(DATA_BYTES); component State[DATA_BYTES]; State[0] = StateUpdate(MAX_STACK_HEIGHT); @@ -413,18 +370,93 @@ fn build_json_circuit( for (i, key) in data.keys.iter().enumerate() { match key { - Key::String(_) => circuit_buffer += &format!(" signal is_key{}_match[DATA_BYTES];\n signal is_key{}_match_for_value[DATA_BYTES];\n is_key{}_match_for_value[0] <== 0;\n signal is_next_pair_at_depth{}[DATA_BYTES];\n", i+1, i+1, i+1, i+1), + Key::String(_) => circuit_buffer += &format!(" signal is_key{}_match[DATA_BYTES];\n signal is_key{}_match_for_value[DATA_BYTES+1];\n is_key{}_match_for_value[0] <== 0;\n signal is_next_pair_at_depth{}[DATA_BYTES];\n", i+1, i+1, i+1, i+1), Key::Num(_) => (), } } } + let mut num_objects = 0; + + // initialise first iteration + { + // parsing_key and parsing_object{i}_value + circuit_buffer += r#" + // initialise first iteration + parsing_key[0] <== InsideKey(MAX_STACK_HEIGHT)(State[0].next_stack, State[0].next_parsing_string, State[0].next_parsing_number); + +"#; + + for (i, key) in data.keys.iter().enumerate() { + match key { + Key::String(_) => { + circuit_buffer += &format!(" parsing_object{}_value[0] <== InsideValue()(State[0].next_stack[0], State[0].next_parsing_string, State[0].next_parsing_number);\n", i+1); + } + Key::Num(_) => { + circuit_buffer += &format!(" parsing_array{}[0] <== InsideArrayIndex(index{})(State[0].next_stack[0], State[0].next_parsing_string, State[0].next_parsing_number);\n", i+1, i+1); + } + } + } + + // parsing_value[0] <== MultiAND(5)([parsing_object1_value[0], parsing_object2_value[0], parsing_array3[0], parsing_object4_value[0], parsing_object5_value[0]]); + circuit_buffer += &format!( + " // parsing correct value = AND(all individual stack values)\n parsing_value[0] <== MultiAND({})([", + data.keys.len() + ); + for (i, key) in data.keys.iter().take(data.keys.len() - 1).enumerate() { + match key { + Key::String(_) => circuit_buffer += &format!("parsing_object{}_value[0], ", i + 1), + Key::Num(_) => circuit_buffer += &format!("parsing_array{}[0], ", i + 1), + } + } + match data.keys[data.keys.len() - 1] { + Key::String(_) => { + circuit_buffer += &format!("parsing_object{}_value[0]]);\n\n", data.keys.len()) + } + Key::Num(_) => circuit_buffer += &format!("parsing_array{}[0]]);\n\n", data.keys.len()), + } + + // is_key{i}_match_for_value + for (i, key) in data.keys.iter().enumerate() { + match key { + Key::String(_) => { + num_objects += 1; + circuit_buffer += &format!(" is_key{}_match[0] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen{}, depth{})(data, key{}, 0, parsing_key[0], State[0].next_stack);\n", i+1, i+1, i+1, i+1); + circuit_buffer += &format!(" is_next_pair_at_depth{}[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth{})(State[0].next_stack, data[0]);\n", i+1, i+1); + circuit_buffer += &format!(" is_key{}_match_for_value[1] <== Mux1()([is_key{}_match_for_value[0] * (1-is_next_pair_at_depth{}[0]), is_key{}_match[0] * (1-is_next_pair_at_depth{}[0])], is_key{}_match[0]);\n", i+1, i+1, i+1, i+1, i+1, i+1); + if debug { + circuit_buffer += &format!(" // log(\"is_key{}_match_for_value\", is_key{}_match_for_value[1]);\n\n", i + 1, i + 1); + } + } + Key::Num(_) => (), + } + } + + // is_value_match[data_idx] <== MultiAND(2)([is_key1_match_for_value[data_idx], is_key3_match_for_value[data_idx]]); + { + circuit_buffer += &format!(" is_value_match[0] <== MultiAND({})([", num_objects); + for (i, key) in data.keys.iter().enumerate() { + match key { + Key::String(_) => { + circuit_buffer += &format!("is_key{}_match_for_value[1], ", i + 1) + } + Key::Num(_) => (), + } + } + + // remove last 2 chars `, ` from string buffer + circuit_buffer.pop(); + circuit_buffer.pop(); + circuit_buffer += "]);\n"; + } + + circuit_buffer += r#" + mask[0] <== parsing_value[0] * is_value_match[0]; +"#; + } + // debugging circuit_buffer += r#" - signal is_value_match[DATA_BYTES]; - is_value_match[0] <== 0; - signal value_mask[DATA_BYTES]; - for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) {"#; if debug { @@ -455,22 +487,22 @@ fn build_json_circuit( // - mask // check if inside key or not - parsing_key[data_idx-1] <== InsideKey(MAX_STACK_HEIGHT)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number); + parsing_key[data_idx] <== InsideKey(MAX_STACK_HEIGHT)(State[data_idx].next_stack, State[data_idx].next_parsing_string, State[data_idx].next_parsing_number); "#; /* Determining wheter parsing correct value and array index - parsing_object1_value[data_idx-1] <== InsideValueAtDepth(MAX_STACK_HEIGHT, depth1)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number); - parsing_array2[data_idx-1] <== InsideArrayIndexAtDepth(MAX_STACK_HEIGHT, index2, depth2)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number); + parsing_object1_value[data_idx-1] <== InsideValue(MAX_STACK_HEIGHT, depth1)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number); + parsing_array2[data_idx-1] <== InsideArrayIndex(MAX_STACK_HEIGHT, index2, depth2)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number); */ { for (i, key) in data.keys.iter().enumerate() { match key { Key::String(_) => { - circuit_buffer += &format!(" parsing_object{}_value[data_idx-1] <== InsideValueAtDepth(MAX_STACK_HEIGHT, depth{})(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number);\n", i+1, i+1); + circuit_buffer += &format!(" parsing_object{}_value[data_idx] <== InsideValue()(State[data_idx].next_stack[depth{}], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number);\n", i+1, i+1); } Key::Num(_) => { - circuit_buffer += &format!(" parsing_array{}[data_idx-1] <== InsideArrayIndexAtDepth(MAX_STACK_HEIGHT, index{}, depth{})(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number);\n", i+1, i+1, i+1); + circuit_buffer += &format!(" parsing_array{}[data_idx] <== InsideArrayIndex(index{})(State[data_idx].next_stack[depth{}], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number);\n", i+1, i+1, i+1); } } } @@ -480,25 +512,24 @@ fn build_json_circuit( // parsing_value[data_idx-1] <== MultiAND(4)([parsing_object1_value[data_idx-1], parsing_array2[data_idx-1], parsing_object3_value[data_idx-1], parsing_array4[data_idx-1]]); { circuit_buffer += &format!( - " // parsing correct value = AND(all individual stack values)\n parsing_value[data_idx-1] <== MultiAND({})([", - data.keys.len() - ); + " // parsing correct value = AND(all individual stack values)\n parsing_value[data_idx] <== MultiAND({})([", + data.keys.len() + ); for (i, key) in data.keys.iter().take(data.keys.len() - 1).enumerate() { match key { Key::String(_) => { - circuit_buffer += &format!("parsing_object{}_value[data_idx-1], ", i + 1) + circuit_buffer += &format!("parsing_object{}_value[data_idx], ", i + 1) } - Key::Num(_) => circuit_buffer += &format!("parsing_array{}[data_idx-1], ", i + 1), + Key::Num(_) => circuit_buffer += &format!("parsing_array{}[data_idx], ", i + 1), } } match data.keys[data.keys.len() - 1] { Key::String(_) => { - circuit_buffer += - &format!("parsing_object{}_value[data_idx-1]]);\n", data.keys.len()) + circuit_buffer += &format!("parsing_object{}_value[data_idx]]);\n", data.keys.len()) } Key::Num(_) => { - circuit_buffer += &format!("parsing_array{}[data_idx-1]]);\n", data.keys.len()) + circuit_buffer += &format!("parsing_array{}[data_idx]]);\n", data.keys.len()) } } @@ -508,14 +539,12 @@ fn build_json_circuit( for (i, key) in data.keys.iter().enumerate() { match key { Key::String(_) => { - circuit_buffer += &format!("parsing_object{}_value[data_idx-1], ", i + 1) - } - Key::Num(_) => { - circuit_buffer += &format!("parsing_array{}[data_idx-1], ", i + 1) + circuit_buffer += &format!("parsing_object{}_value[data_idx], ", i + 1) } + Key::Num(_) => circuit_buffer += &format!("parsing_array{}[data_idx], ", i + 1), } } - circuit_buffer += "parsing_value[data_idx-1]);\n\n"; + circuit_buffer += "parsing_value[data_idx]);\n\n"; } } @@ -526,7 +555,7 @@ fn build_json_circuit( - key matches at current index and depth of key is as specified - whether next KV pair starts - whether key matched for a value (propogate key match until new KV pair of lower depth starts) - is_key1_match[data_idx-1] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen1, depth1)(data, key1, r, data_idx-1, parsing_key[data_idx-1], State[data_idx].stack); + is_key1_match[data_idx-1] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen1, depth1)(data, key1, data_idx-1, parsing_key[data_idx-1], State[data_idx].stack); is_next_pair_at_depth1[data_idx-1] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth1)(State[data_idx].stack, data[data_idx-1]); is_key1_match_for_value[data_idx] <== Mux1()([is_key1_match_for_value[data_idx-1] * (1-is_next_pair_at_depth1[data_idx-1]), is_key1_match[data_idx-1] * (1-is_next_pair_at_depth1[data_idx-1])], is_key1_match[data_idx-1]); */ @@ -542,11 +571,11 @@ fn build_json_circuit( match key { Key::String(_) => { num_objects += 1; - circuit_buffer += &format!(" is_key{}_match[data_idx-1] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen{}, depth{})(data, key{}, r, data_idx-1, parsing_key[data_idx-1], State[data_idx].stack);\n", i+1, i+1, i+1, i+1); - circuit_buffer += &format!(" is_next_pair_at_depth{}[data_idx-1] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth{})(State[data_idx].stack, data[data_idx-1]);\n", i+1, i+1); - circuit_buffer += &format!(" is_key{}_match_for_value[data_idx] <== Mux1()([is_key{}_match_for_value[data_idx-1] * (1-is_next_pair_at_depth{}[data_idx-1]), is_key{}_match[data_idx-1] * (1-is_next_pair_at_depth{}[data_idx-1])], is_key{}_match[data_idx-1]);\n", i+1, i+1, i+1, i+1, i+1, i+1); + circuit_buffer += &format!(" is_key{}_match[data_idx] <== KeyMatchAtDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen{}, depth{})(data, key{}, data_idx, parsing_key[data_idx], State[data_idx].next_stack);\n", i+1, i+1, i+1, i+1); + circuit_buffer += &format!(" is_next_pair_at_depth{}[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth{})(State[data_idx].next_stack, data[data_idx]);\n", i+1, i+1); + circuit_buffer += &format!(" is_key{}_match_for_value[data_idx+1] <== Mux1()([is_key{}_match_for_value[data_idx] * (1-is_next_pair_at_depth{}[data_idx]), is_key{}_match[data_idx] * (1-is_next_pair_at_depth{}[data_idx])], is_key{}_match[data_idx]);\n", i+1, i+1, i+1, i+1, i+1, i+1); if debug { - circuit_buffer += &format!(" // log(\"is_key{}_match_for_value\", is_key{}_match_for_value[data_idx]);\n\n", i + 1, i + 1); + circuit_buffer += &format!(" // log(\"is_key{}_match_for_value\", is_key{}_match_for_value[data_idx+1]);\n\n", i + 1, i + 1); } } Key::Num(_) => (), @@ -563,7 +592,7 @@ fn build_json_circuit( for (i, key) in data.keys.iter().enumerate() { match key { Key::String(_) => { - circuit_buffer += &format!("is_key{}_match_for_value[data_idx], ", i + 1) + circuit_buffer += &format!("is_key{}_match_for_value[data_idx+1], ", i + 1) } Key::Num(_) => (), } @@ -578,9 +607,8 @@ fn build_json_circuit( // debugging and output bytes { circuit_buffer += r#" - // mask[i] = data[i] * parsing_value[i] * is_key_match_for_value[i] - value_mask[data_idx-1] <== data[data_idx-1] * parsing_value[data_idx-1]; - mask[data_idx-1] <== value_mask[data_idx-1] * is_value_match[data_idx]; + // mask = currently parsing value and all subsequent keys matched + mask[data_idx] <== parsing_value[data_idx] * is_value_match[data_idx]; }"#; // Debugging @@ -597,11 +625,13 @@ fn build_json_circuit( circuit_buffer += r#" + // find starting index of value in data by matching mask signal is_zero_mask[DATA_BYTES]; signal is_prev_starting_index[DATA_BYTES]; value_starting_index[0] <== 0; + is_prev_starting_index[0] <== 0; is_zero_mask[0] <== IsZero()(mask[0]); - for (var i=1 ; i