From 5fd9651e6a6890701347300479fe3de1bd4710e2 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Tue, 22 Oct 2024 14:31:12 -0600 Subject: [PATCH] feat: NIVC circuits + tests (#96) * simplify `JsonMaskObjectNIVC` * passing `JsonMaskArrayIndexNIVC` * bug: incorrect mask for "profile" key * bug: not extracting `"profile"` * add extra key index to match end quote * fix: masking tests pass * fix: final extraction * cleanup: unused signals, consistency Found unused signals with the circom-witnesscalc `build-circuit` binary. These are now removed. * feat: passing HTTP NIVC tests * fix: unallocated signals `HTTPParseAndLockStartLine` had unused signals. I set these properly now. * rename SubstringMatchWithIndex * add `HeaderFieldNameValueMatchPadded` * add a todo in json interpreter * change `LockHeader` to handle max key and value length * use circuit.compute to find witness and simplify test --------- Co-authored-by: lonerapier --- .gitignore | 7 +- circuits/http/interpreter.circom | 27 ++- circuits/http/nivc/body_mask.circom | 36 +++ circuits/http/nivc/lock_header.circom | 113 +++++++++ .../nivc/parse_and_lock_start_line.circom | 139 +++++++++++ circuits/json/interpreter.circom | 19 +- circuits/json/nivc/extractor.circom | 47 ++++ circuits/json/nivc/masker.circom | 217 ++++++++++++++++++ circuits/json/nivc/parse.circom | 61 +++++ circuits/test/common/index.ts | 19 ++ circuits/test/http/interpreter.test.ts | 69 ++++++ circuits/test/http/nivc/nivc.test.ts | 105 +++++++++ circuits/test/json/nivc/masker_nivc.test.ts | 105 +++++++++ circuits/test/utils/search.test.ts | 4 +- circuits/utils/array.circom | 31 +++ circuits/utils/search.circom | 2 +- 16 files changed, 992 insertions(+), 9 deletions(-) create mode 100644 circuits/http/nivc/body_mask.circom create mode 100644 circuits/http/nivc/lock_header.circom create mode 100644 circuits/http/nivc/parse_and_lock_start_line.circom create mode 100644 circuits/json/nivc/extractor.circom create mode 100644 circuits/json/nivc/masker.circom create mode 100644 circuits/json/nivc/parse.circom create mode 100644 circuits/test/http/nivc/nivc.test.ts create mode 100644 circuits/test/json/nivc/masker_nivc.test.ts diff --git a/.gitignore b/.gitignore index faa5f81..3fe3e13 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,9 @@ circuits/main/* # Rust generated inputs/**/*.json -!inputs/search/witness.json \ No newline at end of file +!inputs/search/witness.json + +# Circom-witnesscalc generated +ir_log/* +log_input_signals.txt +*.bin \ No newline at end of file diff --git a/circuits/http/interpreter.circom b/circuits/http/interpreter.circom index ddcd52b..79e6278 100644 --- a/circuits/http/interpreter.circom +++ b/circuits/http/interpreter.circom @@ -47,8 +47,6 @@ template HeaderFieldNameValueMatch(dataLen, nameLen, valueLen) { signal input headerValue[valueLen]; signal input index; - // signal output value[valueLen]; - // is name matches signal headerNameMatch <== SubstringMatchWithIndex(dataLen, nameLen)(data, headerName, index); @@ -65,6 +63,31 @@ template HeaderFieldNameValueMatch(dataLen, nameLen, valueLen) { signal output out <== headerNameMatchAndNextByteColon * headerValueMatch; } +// https://www.rfc-editor.org/rfc/rfc9112.html#name-field-syntax +template HeaderFieldNameValueMatchPadded(dataLen, maxNameLen, maxValueLen) { + signal input data[dataLen]; + signal input headerName[maxNameLen]; + signal input nameLen; + signal input headerValue[maxValueLen]; + signal input valueLen; + signal input index; + + // is name matchesnameLen + signal headerNameMatch <== SubstringMatchWithIndexPadded(dataLen, maxNameLen)(data, headerName, nameLen, index); + + // next byte to name should be COLON + signal endOfHeaderName <== IndexSelector(dataLen)(data, index + nameLen); + signal isNextByteColon <== IsEqual()([endOfHeaderName, 58]); + + signal headerNameMatchAndNextByteColon <== headerNameMatch * isNextByteColon; + + // field-name: SP field-value + signal headerValueMatch <== SubstringMatchWithIndexPadded(dataLen, maxValueLen)(data, headerValue, valueLen, index + nameLen + 2); + + // header name matches + header value matches + signal output out <== headerNameMatchAndNextByteColon * headerValueMatch; +} + // https://www.rfc-editor.org/rfc/rfc9112.html#name-field-syntax template HeaderFieldNameMatch(dataLen, nameLen) { signal input data[dataLen]; diff --git a/circuits/http/nivc/body_mask.circom b/circuits/http/nivc/body_mask.circom new file mode 100644 index 0000000..5ac67f9 --- /dev/null +++ b/circuits/http/nivc/body_mask.circom @@ -0,0 +1,36 @@ +pragma circom 2.1.9; + +include "../interpreter.circom"; + +template HTTPMaskBodyNIVC(DATA_BYTES, MAX_STACK_HEIGHT) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (HttpParseAndLockStartLine or HTTPLockHeader) + signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; + signal output step_out[TOTAL_BYTES_ACROSS_NIVC]; + + signal data[DATA_BYTES]; + signal parsing_body[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + parsing_body[i] <== step_in[DATA_BYTES + i * 5 + 4]; // `parsing_body` stored in every 5th slot of step_in/out + } + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Write out to next NIVC step + for (var i = 0 ; i < DATA_BYTES ; i++) { + step_out[i] <== data[i] * parsing_body[i]; + } + // Write out padded with zeros + for (var i = DATA_BYTES ; i < TOTAL_BYTES_ACROSS_NIVC ; i++) { + step_out[i] <== 0; + } +} + diff --git a/circuits/http/nivc/lock_header.circom b/circuits/http/nivc/lock_header.circom new file mode 100644 index 0000000..fed4294 --- /dev/null +++ b/circuits/http/nivc/lock_header.circom @@ -0,0 +1,113 @@ +pragma circom 2.1.9; + +include "../interpreter.circom"; +include "../../utils/array.circom"; + +// TODO: should use a MAX_HEADER_NAME_LENGTH and a MAX_HEADER_VALUE_LENGTH +template LockHeader(DATA_BYTES, MAX_STACK_HEIGHT, MAX_HEADER_NAME_LENGTH, MAX_HEADER_VALUE_LENGTH) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + /* 5 is for the variables: + next_parsing_start + next_parsing_header + next_parsing_field_name + next_parsing_field_value + State[i].next_parsing_body + */ + var TOTAL_BYTES_HTTP_STATE = DATA_BYTES * (5 + 1); // data + parser vars + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (HttpParseAndLockStartLine or HTTPLockHeader) + signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; + signal output step_out[TOTAL_BYTES_ACROSS_NIVC]; + + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + signal httpParserState[DATA_BYTES * 5]; + for (var i = 0 ; i < DATA_BYTES * 5 ; i++) { + httpParserState[i] <== step_in[DATA_BYTES + i]; + } + + // TODO: Better naming for these variables + signal input header[MAX_HEADER_NAME_LENGTH]; + signal input headerNameLength; + signal input value[MAX_HEADER_VALUE_LENGTH]; + signal input headerValueLength; + + // find header location + signal headerNameLocation <== FirstStringMatch(DATA_BYTES, MAX_HEADER_NAME_LENGTH)(data, header); + + // This is the assertion that we have locked down the correct header + signal headerFieldNameValueMatch <== HeaderFieldNameValueMatchPadded(DATA_BYTES, MAX_HEADER_NAME_LENGTH, MAX_HEADER_VALUE_LENGTH)(data, header, headerNameLength, value, headerValueLength, headerNameLocation); + headerFieldNameValueMatch === 1; + + // parser state should be parsing header + signal isParsingHeader <== IndexSelector(DATA_BYTES * 5)(httpParserState, headerNameLocation * 5 + 1); + isParsingHeader === 1; + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Write out to next NIVC step + for (var i = 0 ; i < DATA_BYTES ; i++) { + // add plaintext http input to step_out + step_out[i] <== step_in[i]; + + // add parser state + step_out[DATA_BYTES + i * 5] <== step_in[DATA_BYTES + i * 5]; + step_out[DATA_BYTES + i * 5 + 1] <== step_in[DATA_BYTES + i * 5 + 1]; + step_out[DATA_BYTES + i * 5 + 2] <== step_in[DATA_BYTES + i * 5 + 2]; + step_out[DATA_BYTES + i * 5 + 3] <== step_in[DATA_BYTES + i * 5 + 3]; + step_out[DATA_BYTES + i * 5 + 4] <== step_in[DATA_BYTES + i * 5 + 4]; + } + // Pad remaining with zeros + for (var i = TOTAL_BYTES_HTTP_STATE ; i < TOTAL_BYTES_ACROSS_NIVC ; i++ ) { + step_out[i] <== 0; + } +} + +// TODO: Handrolled template that I haven't tested YOLO. +template FirstStringMatch(dataLen, maxKeyLen) { + signal input data[dataLen]; + signal input key[maxKeyLen]; + signal output position; + + signal paddedData[dataLen + maxKeyLen]; + for (var i = 0 ; i < dataLen ; i++) { + paddedData[i] <== data[i]; + } + for (var i = 0 ; i < maxKeyLen ; i++) { + paddedData[dataLen + i] <== 0; + } + + var matched = 0; + var counter = 0; + component stringMatch[dataLen]; + component hasMatched[dataLen]; + signal isKeyOutOfBounds[maxKeyLen]; + signal isFirstMatchAndInsideBound[dataLen * maxKeyLen]; + for (var i = 0 ; i < maxKeyLen ; i++) { + isKeyOutOfBounds[i] <== IsZero()(key[i]); + } + + for (var idx = 0 ; idx < dataLen ; idx++) { + stringMatch[idx] = IsEqualArray(maxKeyLen); + stringMatch[idx].in[0] <== key; + for (var key_idx = 0 ; key_idx < maxKeyLen ; key_idx++) { + isFirstMatchAndInsideBound[idx * maxKeyLen + key_idx] <== (1 - matched) * (1 - isKeyOutOfBounds[key_idx]); + stringMatch[idx].in[1][key_idx] <== paddedData[idx + key_idx] * isFirstMatchAndInsideBound[idx * maxKeyLen + key_idx]; + } + hasMatched[idx] = IsEqual(); + hasMatched[idx].in <== [stringMatch[idx].out, 1]; + matched += hasMatched[idx].out; + counter += (1 - matched); // TODO: Off by one? Move before? + } + position <== counter; +} + + diff --git a/circuits/http/nivc/parse_and_lock_start_line.circom b/circuits/http/nivc/parse_and_lock_start_line.circom new file mode 100644 index 0000000..d16ae6d --- /dev/null +++ b/circuits/http/nivc/parse_and_lock_start_line.circom @@ -0,0 +1,139 @@ +pragma circom 2.1.9; + +include "../parser/machine.circom"; +include "../interpreter.circom"; +include "../../utils/bytes.circom"; + +// TODO: Note that TOTAL_BYTES will match what we have for AESGCMFOLD step_out +// I have not gone through to double check the sizes of everything yet. +template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, MIDDLE_LENGTH, FINAL_LENGTH) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + // var AES_BYTES = DATA_BYTES + 50; // TODO: Might be wrong, but good enough for now + /* 5 is for the variables: + next_parsing_start + next_parsing_header + next_parsing_field_name + next_parsing_field_value + State[i].next_parsing_body + */ + var TOTAL_BYTES_HTTP_STATE = DATA_BYTES * (5 + 1); // data + parser vars + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (JsonParseNIVC) + signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; + signal output step_out[TOTAL_BYTES_ACROSS_NIVC]; + + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + // data[i] <== step_in[50 + i]; // THIS WAS OFFSET FOR AES, WHICH WE NEED TO TAKE INTO ACCOUNT + data[i] <== step_in[i]; + } + + // // TODO: check if these needs to here or not + // DON'T THINK WE NEED THIS SINCE AES SHOULD OUTPUT ASCII OR FAIL + // component dataASCII = ASCII(DATA_BYTES); + // dataASCII.in <== data; + + signal input beginning[BEGINNING_LENGTH]; + signal input middle[MIDDLE_LENGTH]; + signal input final[FINAL_LENGTH]; + + // Initialze the parser + component State[DATA_BYTES]; + State[0] = HttpStateUpdate(); + State[0].byte <== data[0]; + State[0].parsing_start <== 1; + State[0].parsing_header <== 0; + State[0].parsing_field_name <== 0; + State[0].parsing_field_value <== 0; + State[0].parsing_body <== 0; + State[0].line_status <== 0; + + /* + Note, because we know a beginning is the very first thing in a request + we can make this more efficient by just comparing the first `BEGINNING_LENGTH` bytes + of the data ASCII against the beginning ASCII itself. + */ + // Check first beginning byte + signal beginningIsEqual[BEGINNING_LENGTH]; + beginningIsEqual[0] <== IsEqual()([data[0],beginning[0]]); + beginningIsEqual[0] === 1; + + // Setup to check middle bytes + signal startLineMask[DATA_BYTES]; + signal middleMask[DATA_BYTES]; + signal finalMask[DATA_BYTES]; + startLineMask[0] <== inStartLine()(State[0].parsing_start); + middleMask[0] <== inStartMiddle()(State[0].parsing_start); + finalMask[0] <== inStartEnd()(State[0].parsing_start); + + + var middle_start_counter = 1; + var middle_end_counter = 1; + var final_end_counter = 1; + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + State[data_idx] = HttpStateUpdate(); + State[data_idx].byte <== data[data_idx]; + State[data_idx].parsing_start <== State[data_idx - 1].next_parsing_start; + State[data_idx].parsing_header <== State[data_idx - 1].next_parsing_header; + State[data_idx].parsing_field_name <== State[data_idx - 1].next_parsing_field_name; + State[data_idx].parsing_field_value <== State[data_idx - 1].next_parsing_field_value; + State[data_idx].parsing_body <== State[data_idx - 1].next_parsing_body; + State[data_idx].line_status <== State[data_idx - 1].next_line_status; + + // Check remaining beginning bytes + if(data_idx < BEGINNING_LENGTH) { + beginningIsEqual[data_idx] <== IsEqual()([data[data_idx], beginning[data_idx]]); + beginningIsEqual[data_idx] === 1; + } + + // Set the masks based on parser state + startLineMask[data_idx] <== inStartLine()(State[data_idx].parsing_start); + middleMask[data_idx] <== inStartMiddle()(State[data_idx].parsing_start); + finalMask[data_idx] <== inStartEnd()(State[data_idx].parsing_start); + + // Increment counters based on mask information + middle_start_counter += startLineMask[data_idx] - middleMask[data_idx] - finalMask[data_idx]; + middle_end_counter += startLineMask[data_idx] - finalMask[data_idx]; + final_end_counter += startLineMask[data_idx]; + } + + // Additionally verify beginning had correct length + BEGINNING_LENGTH === middle_start_counter - 1; + + // Check middle is correct by substring match and length check + signal middleMatch <== SubstringMatchWithIndex(DATA_BYTES, MIDDLE_LENGTH)(data, middle, middle_start_counter); + middleMatch === 1; + MIDDLE_LENGTH === middle_end_counter - middle_start_counter - 1; + + // Check final is correct by substring match and length check + signal finalMatch <== SubstringMatchWithIndex(DATA_BYTES, FINAL_LENGTH)(data, final, middle_end_counter); + finalMatch === 1; + // -2 here for the CRLF + FINAL_LENGTH === final_end_counter - middle_end_counter - 2; + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Write out to next NIVC step (Lock Header) + for (var i = 0 ; i < DATA_BYTES ; i++) { + // add plaintext http input to step_out + // step_out[i] <== step_in[50 + i]; // AGAIN, NEED TO ACCOUNT FOR AES VARIABLES POSSIBLY + step_out[i] <== step_in[i]; + + // add parser state + step_out[DATA_BYTES + i * 5] <== State[i].next_parsing_start; + step_out[DATA_BYTES + i * 5 + 1] <== State[i].next_parsing_header; + step_out[DATA_BYTES + i * 5 + 2] <== State[i].next_parsing_field_name; + step_out[DATA_BYTES + i * 5 + 3] <== State[i].next_parsing_field_value; + step_out[DATA_BYTES + i * 5 + 4] <== State[i].next_parsing_body; + } + // Pad remaining with zeros + for (var i = TOTAL_BYTES_HTTP_STATE ; i < TOTAL_BYTES_ACROSS_NIVC ; i++ ) { + step_out[i] <== 0; + } +} diff --git a/circuits/json/interpreter.circom b/circuits/json/interpreter.circom index ce0e34c..8d8d18b 100644 --- a/circuits/json/interpreter.circom +++ b/circuits/json/interpreter.circom @@ -393,7 +393,10 @@ template KeyMatchAtDepth(dataLen, n, keyLen, depth) { signal output out <== substring_match * is_parsing_correct_key_at_depth; } +// TODO: Not checking start of key is quote since that is handled by `parsing_key`? template MatchPaddedKey(n) { + // TODO: If key is not padded at all, then `in[1]` will not contain an end quote. + // Perhaps we modify this to handle that, or just always pad the key at least once. signal input in[2][n]; signal input keyLen; signal output out; @@ -412,8 +415,10 @@ template MatchPaddedKey(n) { isQuote[i] <== IsEqual()([in[1][i], 34]); endOfKeyAccum[i+1] <== endOfKeyAccum[i] + isEndOfKey[i] * isQuote[i]; + // TODO: might not be right to check for zero, instead check for -1? isPaddedElement[i] = IsZero(); isPaddedElement[i].in <== in[0][i]; + equalComponent[i] = IsEqual(); equalComponent[i].in[0] <== in[0][i]; equalComponent[i].in[1] <== in[1][i] * (1-isPaddedElement[i].out); @@ -421,6 +426,7 @@ template MatchPaddedKey(n) { } signal isEndOfKeyEqualToQuote <== IsEqual()([endOfKeyAccum[n], 1]); + // log("isEndOfKeyEqualToQuote", isEndOfKeyEqualToQuote); component totalEqual = IsEqual(); totalEqual.in[0] <== n; @@ -448,6 +454,11 @@ template KeyMatchAtIndex(dataLen, maxKeyLen, index) { signal input keyLen; signal input parsing_key; + signal paddedKey[maxKeyLen + 1]; + for (var i = 0 ; i < maxKeyLen ; i++) { + paddedKey[i] <== key[i]; + } + paddedKey[maxKeyLen] <== 0; // `"` -> 34 // start of key equal to quote @@ -455,12 +466,14 @@ template KeyMatchAtIndex(dataLen, maxKeyLen, index) { signal isParsingCorrectKey <== parsing_key * startOfKeyEqualToQuote; // key matches - component isSubstringMatch = MatchPaddedKey(maxKeyLen); - isSubstringMatch.in[0] <== key; + component isSubstringMatch = MatchPaddedKey(maxKeyLen+1); + isSubstringMatch.in[0] <== paddedKey; isSubstringMatch.keyLen <== keyLen; - for(var matcher_idx = 0; matcher_idx < maxKeyLen; matcher_idx++) { + for(var matcher_idx = 0; matcher_idx <= maxKeyLen; matcher_idx++) { + // log("matcher_idx", index, matcher_idx, data[index + matcher_idx]); isSubstringMatch.in[1][matcher_idx] <== data[index + matcher_idx]; } + // log("keyMatchAtIndex", isParsingCorrectKey, isSubstringMatch.out); signal output out <== isSubstringMatch.out * isParsingCorrectKey; } \ No newline at end of file diff --git a/circuits/json/nivc/extractor.circom b/circuits/json/nivc/extractor.circom new file mode 100644 index 0000000..aadb436 --- /dev/null +++ b/circuits/json/nivc/extractor.circom @@ -0,0 +1,47 @@ +pragma circom 2.1.9; + +include "circomlib/circuits/gates.circom"; +include "@zk-email/circuits/utils/array.circom"; + +template MaskExtractFinal(DATA_BYTES, MAX_STACK_HEIGHT, MAX_VALUE_LENGTH) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + assert(MAX_STACK_HEIGHT >= 2); + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + // ------------------------------------------------------------------------------------------------------------------ // + signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; + signal output step_out[TOTAL_BYTES_ACROSS_NIVC]; + + signal is_zero_mask[DATA_BYTES]; + signal is_prev_starting_index[DATA_BYTES]; + signal value_starting_index[DATA_BYTES]; + + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + + value_starting_index[0] <== 0; + is_prev_starting_index[0] <== 0; + is_zero_mask[0] <== IsZero()(step_in[0]); + for (var i=1 ; i < DATA_BYTES ; i++) { + is_zero_mask[i] <== IsZero()(step_in[i]); + is_prev_starting_index[i] <== IsZero()(value_starting_index[i-1]); + value_starting_index[i] <== value_starting_index[i-1] + i * (1-is_zero_mask[i]) * is_prev_starting_index[i]; + } + // TODO: Clear step out? + signal output value[MAX_VALUE_LENGTH] <== SelectSubArray(DATA_BYTES, MAX_VALUE_LENGTH)(data, value_starting_index[DATA_BYTES-1], MAX_VALUE_LENGTH); + for (var i = 0 ; i < MAX_VALUE_LENGTH ; i++) { + // log(i, value[i]); + step_out[i] <== value[i]; + } + for (var i = MAX_VALUE_LENGTH ; i < TOTAL_BYTES_ACROSS_NIVC ; i++) { + step_out[i] <== 0; + } + // TODO: Do anything with last depth? + // step_out[TOTAL_BYTES_ACROSS_NIVC - 1] <== 0; +} + +// component main { public [step_in] } = MaskExtractFinal(4160, 320, 200); \ No newline at end of file diff --git a/circuits/json/nivc/masker.circom b/circuits/json/nivc/masker.circom new file mode 100644 index 0000000..66dbf81 --- /dev/null +++ b/circuits/json/nivc/masker.circom @@ -0,0 +1,217 @@ +pragma circom 2.1.9; + +include "../interpreter.circom"; +include "../../utils/array.circom"; + +template JsonMaskObjectNIVC(DATA_BYTES, MAX_STACK_HEIGHT, MAX_KEY_LENGTH) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + assert(MAX_STACK_HEIGHT >= 2); + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (JsonParseNIVC) + signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; + signal output step_out[TOTAL_BYTES_ACROSS_NIVC]; + + // Grab the raw data bytes from the `step_in` variable + var paddedDataLen = DATA_BYTES + MAX_KEY_LENGTH + 1; + signal data[paddedDataLen]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + for (var i = 0 ; i < MAX_KEY_LENGTH + 1 ; i++) { + data[DATA_BYTES + i] <== 0; + } + + // Decode the encoded data in `step_in` back into parser variables + signal stack[DATA_BYTES][MAX_STACK_HEIGHT + 1][2]; + signal parsingData[DATA_BYTES][2]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + for (var j = 0 ; j < MAX_STACK_HEIGHT + 1 ; j++) { + if (j < MAX_STACK_HEIGHT) { + stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; + stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; + } else { + // Add one extra stack element without doing this while parsing. + // Stack under/overflow caught in parsing. + stack[i][j][0] <== 0; + stack[i][j][1] <== 0; + } + + } + parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; + parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; + } + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Object masking ~ + // Key data to use to point to which object to extract + signal input key[MAX_KEY_LENGTH]; + signal input keyLen; + + // Signals to detect if we are parsing a key or value with initial setup + signal parsing_key[DATA_BYTES]; + signal parsing_value[DATA_BYTES]; + + // Flags at each byte to indicate if we are matching correct key and in subsequent value + signal is_key_match[DATA_BYTES]; + signal is_value_match[DATA_BYTES]; + + signal is_next_pair_at_depth[DATA_BYTES]; + signal is_key_match_for_value[DATA_BYTES + 1]; + is_key_match_for_value[0] <== 0; + + // Initialize values knowing 0th bit of data will never be a key/value + parsing_key[0] <== 0; + parsing_value[0] <== 0; + is_key_match[0] <== 0; + + component stackSelector[DATA_BYTES]; + stackSelector[0] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + stackSelector[0].in <== stack[0]; + stackSelector[0].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1]; + + component nextStackSelector[DATA_BYTES]; + nextStackSelector[0] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + nextStackSelector[0].in <== stack[0]; + nextStackSelector[0].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1] + 1; + + is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT + 1)(stack[0], data[0],step_in[TOTAL_BYTES_ACROSS_NIVC - 1]); + is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); + is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; + + signal or[DATA_BYTES]; + or[0] <== is_value_match[0]; + step_out[0] <== data[0] * or[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + // Grab the stack at the indicated height (from `step_in`) + stackSelector[data_idx] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + stackSelector[data_idx].in <== stack[data_idx]; + stackSelector[data_idx].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1]; + + nextStackSelector[data_idx] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + nextStackSelector[data_idx].in <== stack[data_idx]; + nextStackSelector[data_idx].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1] + 1; + + // Detect if we are parsing + parsing_key[data_idx] <== InsideKey()(stackSelector[data_idx].out, parsingData[data_idx][0], parsingData[data_idx][1]); + parsing_value[data_idx] <== InsideValueObject()(stackSelector[data_idx].out, nextStackSelector[data_idx].out, parsingData[data_idx][0], parsingData[data_idx][1]); + + // to get correct value, check: + // - key matches at current index and depth of key is as specified + // - whether next KV pair starts + // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) + is_key_match[data_idx] <== KeyMatchAtIndex(paddedDataLen, MAX_KEY_LENGTH, data_idx)(data, key, keyLen, parsing_key[data_idx]); + is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT + 1)(stack[data_idx], data[data_idx], step_in[TOTAL_BYTES_ACROSS_NIVC - 1]); + + is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); + is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; + + // Set the next NIVC step to only have the masked data + or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx -1]); + step_out[data_idx] <== data[data_idx] * or[data_idx]; + } + // Append the parser state back on `step_out` + for (var i = DATA_BYTES ; i < TOTAL_BYTES_ACROSS_NIVC - 1 ; i++) { + step_out[i] <== step_in[i]; + } + // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_ACROSS_NIVC + + // Finally, update the current depth we are extracting from + step_out[TOTAL_BYTES_ACROSS_NIVC - 1] <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1] + 1; +} + +template JsonMaskArrayIndexNIVC(DATA_BYTES, MAX_STACK_HEIGHT) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + assert(MAX_STACK_HEIGHT >= 2); + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (JsonParseNIVC) + signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; + signal output step_out[TOTAL_BYTES_ACROSS_NIVC]; + + // Grab the raw data bytes from the `step_in` variable + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + + // Decode the encoded data in `step_in` back into parser variables + signal stack[DATA_BYTES][MAX_STACK_HEIGHT + 1][2]; + signal parsingData[DATA_BYTES][2]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + for (var j = 0 ; j < MAX_STACK_HEIGHT + 1 ; j++) { + if (j < MAX_STACK_HEIGHT) { + stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; + stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; + } else { + // Add one extra stack element without doing this while parsing. + // Stack under/overflow caught in parsing. + stack[i][j][0] <== 0; + stack[i][j][1] <== 0; + } + + } + parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; + parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; + } + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Array index masking ~ + signal input index; + + signal parsing_array[DATA_BYTES]; + + component stackSelector[DATA_BYTES]; + stackSelector[0] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + stackSelector[0].in <== stack[0]; + stackSelector[0].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1]; + + component nextStackSelector[DATA_BYTES]; + nextStackSelector[0] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + nextStackSelector[0].in <== stack[0]; + nextStackSelector[0].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1] + 1; + + parsing_array[0] <== InsideArrayIndexObject()(stackSelector[0].out, nextStackSelector[0].out, parsingData[0][0], parsingData[0][1], index); + + signal or[DATA_BYTES]; + or[0] <== parsing_array[0]; + step_out[0] <== data[0] * or[0]; + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + stackSelector[data_idx] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + stackSelector[data_idx].in <== stack[data_idx]; + stackSelector[data_idx].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1]; + + nextStackSelector[data_idx] = ArraySelector(MAX_STACK_HEIGHT + 1, 2); + nextStackSelector[data_idx].in <== stack[data_idx]; + nextStackSelector[data_idx].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1] + 1; + + parsing_array[data_idx] <== InsideArrayIndexObject()(stackSelector[data_idx].out, nextStackSelector[data_idx].out, parsingData[data_idx][0], parsingData[data_idx][1], index); + + or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]); + step_out[data_idx] <== data[data_idx] * or[data_idx]; + } + + // Write the `step_out` with masked data + + // Append the parser state back on `step_out` + for (var i = DATA_BYTES ; i < TOTAL_BYTES_ACROSS_NIVC - 1 ; i++) { + step_out[i] <== step_in[i]; + } + // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED + step_out[TOTAL_BYTES_ACROSS_NIVC - 1] <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1] + 1; +} diff --git a/circuits/json/nivc/parse.circom b/circuits/json/nivc/parse.circom new file mode 100644 index 0000000..4650dfa --- /dev/null +++ b/circuits/json/nivc/parse.circom @@ -0,0 +1,61 @@ +pragma circom 2.1.9; + +include "../parser/parser.circom"; + +template JsonParseNIVC(DATA_BYTES, MAX_STACK_HEIGHT) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + // ------------------------------------------------------------------------------------------------------------------ // + + // Read in from previous NIVC step (AESNIVC) + signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Parse JSON ~ + // Initialize the parser + component State[DATA_BYTES]; + State[0] = StateUpdate(MAX_STACK_HEIGHT); + for(var i = 0; i < MAX_STACK_HEIGHT; i++) { + State[0].stack[i] <== [0,0]; + } + State[0].parsing_string <== 0; + State[0].parsing_number <== 0; + State[0].byte <== step_in[0]; + + // Parse all the data to generate the complete parser state + for(var i = 1; i < DATA_BYTES; i++) { + State[i] = StateUpdate(MAX_STACK_HEIGHT); + State[i].byte <== step_in[i]; + State[i].stack <== State[i - 1].next_stack; + State[i].parsing_string <== State[i - 1].next_parsing_string; + State[i].parsing_number <== State[i - 1].next_parsing_number; + } + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Write to `step_out` for next NIVC step + // Pass the data bytes back out in the first `step_out` signals + signal output step_out[TOTAL_BYTES_ACROSS_NIVC]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + step_out[i] <== step_in[i]; + } + + // Decode the parser state into the `step_out` remaining signals + for (var i = 0 ; i < DATA_BYTES ; i++) { + for (var j = 0 ; j < MAX_STACK_HEIGHT ; j++) { + step_out[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2] <== State[i].next_stack[j][0]; + step_out[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1] <== State[i].next_stack[j][1]; + } + step_out[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2] <== State[i].next_parsing_string; + step_out[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1] <== State[i].next_parsing_number; + } + // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED + step_out[TOTAL_BYTES_ACROSS_NIVC - 1] <== 0; // Initial depth set to 0 for extraction + // ------------------------------------------------------------------------------------------------------------------ // +} + +// component main { public [step_in] } = JsonParseNIVC(320, 5); + diff --git a/circuits/test/common/index.ts b/circuits/test/common/index.ts index ecc66f5..65f1bdb 100644 --- a/circuits/test/common/index.ts +++ b/circuits/test/common/index.ts @@ -61,6 +61,25 @@ export function readJSONInputFile(filename: string, key: any[]): [number[], numb return [input, keyUnicode, output]; } +import fs from 'fs'; + +export function readJsonFile(filePath: string): T { + // Read the file synchronously + const fileContents = fs.readFileSync(filePath, 'utf-8'); + + // Parse the JSON content + const jsonData = JSON.parse(fileContents, (key, value) => { + // Check if the value is a string that ends with 'n' (for BigInt) + if (typeof value === 'string' && value.endsWith('n')) { + // Convert it back to a BigInt + return BigInt(value.slice(0, -1)); + } + return value; + }); + + return jsonData as T; +} + export function toByte(data: string): number[] { const byteArray = []; for (let i = 0; i < data.length; i++) { diff --git a/circuits/test/http/interpreter.test.ts b/circuits/test/http/interpreter.test.ts index 24113c7..5bc6968 100644 --- a/circuits/test/http/interpreter.test.ts +++ b/circuits/test/http/interpreter.test.ts @@ -39,4 +39,73 @@ describe("HTTP :: Interpreter", async () => { generatePassCase(parsedHttp.input, toByte("GET"), 0, ""); generateFailCase(parsedHttp.input, toByte("POST"), 0, ""); }); +}); + +// 320 bytes in the HTTP response +let http_response_plaintext = [ + 72, 84, 84, 80, 47, 49, 46, 49, 32, 50, 48, 48, 32, 79, 75, 13, 10, 99, 111, 110, 116, 101, 110, + 116, 45, 116, 121, 112, 101, 58, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 106, + 115, 111, 110, 59, 32, 99, 104, 97, 114, 115, 101, 116, 61, 117, 116, 102, 45, 56, 13, 10, 99, + 111, 110, 116, 101, 110, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 103, 122, 105, + 112, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, + 32, 99, 104, 117, 110, 107, 101, 100, 13, 10, 13, 10, 123, 13, 10, 32, 32, 32, 34, 100, 97, 116, + 97, 34, 58, 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, 32, 34, 105, 116, 101, 109, 115, 34, 58, 32, + 91, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 97, 116, 97, 34, 58, 32, 34, 65, 114, 116, 105, 115, + 116, 34, 44, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 112, 114, + 111, 102, 105, 108, 101, 34, 58, 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 84, 97, 121, 108, 111, 114, 32, 83, 119, + 105, 102, 116, 34, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 13, + 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 13, 10, 32, 32, 32, 32, 32, 32, 32, 93, 13, + 10, 32, 32, 32, 125, 13, 10, 125]; + +describe("HeaderFieldNameValueMatchPadded", async () => { + let circuit: WitnessTester<["data", "headerName", "nameLen", "headerValue", "valueLen", "index"], ["out"]>; + + let DATA_BYTES = 320; + let MAX_NAME_LENGTH = 20; + let MAX_VALUE_LENGTH = 35; + + before(async () => { + circuit = await circomkit.WitnessTester(`HeaderFieldNameValueMatchPadded`, { + file: "http/interpreter", + template: "HeaderFieldNameValueMatchPadded", + params: [DATA_BYTES, MAX_NAME_LENGTH, MAX_VALUE_LENGTH], + }); + }); + + function generatePassCase(input: any, expected: any, desc: string) { + const description = generateDescription(input); + input["headerName"] = input["headerName"].concat(Array(MAX_NAME_LENGTH - input["headerName"].length).fill(0)); + input["headerValue"] = input["headerValue"].concat(Array(MAX_VALUE_LENGTH - input["headerValue"].length).fill(0)); + + it(`(valid) witness: ${desc}`, async () => { + // console.log(JSON.stringify(await circuit.compute(input, ["step_out"]))) + await circuit.expectPass(input, expected); + }); + + } + + let header_name = toByte("content-type"); + let header_value = toByte("application/json; charset=utf-8"); + + let input = { + data: http_response_plaintext, + headerName: header_name, + nameLen: header_name.length, + headerValue: header_value, + valueLen: header_value.length, + index: 17, + } + generatePassCase(input, { out: 1 }, "header name and value matches"); + + let input2 = { + data: http_response_plaintext, + headerName: header_name, + nameLen: header_name.length, + headerValue: header_value, + valueLen: header_value.length, + index: 16, + } + generatePassCase(input2, { out: 0 }, "incorrect index"); }); \ No newline at end of file diff --git a/circuits/test/http/nivc/nivc.test.ts b/circuits/test/http/nivc/nivc.test.ts new file mode 100644 index 0000000..f9c12b3 --- /dev/null +++ b/circuits/test/http/nivc/nivc.test.ts @@ -0,0 +1,105 @@ +import { circomkit, WitnessTester, generateDescription, readJsonFile, toByte } from "../../common"; +import { join } from "path"; + +// HTTP/1.1 200 OK +// content-type: application/json; charset=utf-8 +// content-encoding: gzip +// Transfer-Encoding: chunked +// +// { +// "data": { +// "items": [ +// { +// "data": "Artist", +// "profile": { +// "name": "Taylor Swift" +// } +// } +// ] +// } +// } + +// 320 bytes in the HTTP response +let http_response_plaintext = [ + 72, 84, 84, 80, 47, 49, 46, 49, 32, 50, 48, 48, 32, 79, 75, 13, 10, 99, 111, 110, 116, 101, 110, + 116, 45, 116, 121, 112, 101, 58, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 106, + 115, 111, 110, 59, 32, 99, 104, 97, 114, 115, 101, 116, 61, 117, 116, 102, 45, 56, 13, 10, 99, + 111, 110, 116, 101, 110, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 103, 122, 105, + 112, 13, 10, 84, 114, 97, 110, 115, 102, 101, 114, 45, 69, 110, 99, 111, 100, 105, 110, 103, 58, + 32, 99, 104, 117, 110, 107, 101, 100, 13, 10, 13, 10, 123, 13, 10, 32, 32, 32, 34, 100, 97, 116, + 97, 34, 58, 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, 32, 34, 105, 116, 101, 109, 115, 34, 58, 32, + 91, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 97, 116, 97, 34, 58, 32, 34, 65, 114, 116, 105, 115, + 116, 34, 44, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 112, 114, + 111, 102, 105, 108, 101, 34, 58, 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 84, 97, 121, 108, 111, 114, 32, 83, 119, + 105, 102, 116, 34, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 13, + 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 13, 10, 32, 32, 32, 32, 32, 32, 32, 93, 13, + 10, 32, 32, 32, 125, 13, 10, 125]; + +describe("HTTPParseAndLockStartLineNIVC", async () => { + let httpParseAndLockStartLineCircuit: WitnessTester<["step_in", "beginning", "middle", "final"], ["step_out"]>; + let lockHeaderCircuit: WitnessTester<["step_in", "header", "headerNameLength", "value", "headerValueLength"], ["step_out"]>; + let bodyMaskCircuit: WitnessTester<["step_in"], ["step_out"]>; + + const DATA_BYTES = 320; + const MAX_STACK_HEIGHT = 5; + const PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + const TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + + const MAX_HEADER_NAME_LENGTH = 20; + const MAX_HEADER_VALUE_LENGTH = 35; + + const beginning = [72, 84, 84, 80, 47, 49, 46, 49]; // HTTP/1.1 + const BEGINNING_LENGTH = 8; + const middle = [50, 48, 48]; // 200 + const MIDDLE_LENGTH = 3; + const final = [79, 75]; // OK + const FINAL_LENGTH = 2; + + before(async () => { + httpParseAndLockStartLineCircuit = await circomkit.WitnessTester(`ParseAndLockStartLine`, { + file: "http/nivc/parse_and_lock_start_line", + template: "ParseAndLockStartLine", + params: [DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, MIDDLE_LENGTH, FINAL_LENGTH], + }); + console.log("#constraints:", await httpParseAndLockStartLineCircuit.getConstraintCount()); + + lockHeaderCircuit = await circomkit.WitnessTester(`LockHeader`, { + file: "http/nivc/lock_header", + template: "LockHeader", + params: [DATA_BYTES, MAX_STACK_HEIGHT, MAX_HEADER_NAME_LENGTH, MAX_HEADER_VALUE_LENGTH], + }); + console.log("#constraints:", await lockHeaderCircuit.getConstraintCount()); + + bodyMaskCircuit = await circomkit.WitnessTester(`BodyMask`, { + file: "http/nivc/body_mask", + template: "HTTPMaskBodyNIVC", + params: [DATA_BYTES, MAX_STACK_HEIGHT], + }); + console.log("#constraints:", await bodyMaskCircuit.getConstraintCount()); + }); + + let extendedJsonInput = http_response_plaintext.concat(Array(Math.max(0, TOTAL_BYTES_ACROSS_NIVC - http_response_plaintext.length)).fill(0)); + + let headerName = toByte("content-type"); + let headerValue = toByte("application/json; charset=utf-8"); + + let headerNamePadded = headerName.concat(Array(MAX_HEADER_NAME_LENGTH - headerName.length).fill(0)); + let headerValuePadded = headerValue.concat(Array(MAX_HEADER_VALUE_LENGTH - headerValue.length).fill(0)); + it("HTTPParseAndExtract", async () => { + let parseAndLockStartLine = await httpParseAndLockStartLineCircuit.compute({ step_in: extendedJsonInput, beginning: beginning, middle: middle, final: final }, ["step_out"]); + + let lockHeader = await lockHeaderCircuit.compute({ step_in: parseAndLockStartLine.step_out, header: headerNamePadded, headerNameLength: headerName.length, value: headerValuePadded, headerValueLength: headerValue.length }, ["step_out"]); + + let bodyMask = await bodyMaskCircuit.compute({ step_in: lockHeader.step_out }, ["step_out"]); + + let bodyMaskOut = bodyMask.step_out as number[]; + let idx = bodyMaskOut.indexOf('{'.charCodeAt(0)); + + let extended_json_input_2 = extendedJsonInput.fill(0, 0, idx); + extended_json_input_2 = extended_json_input_2.fill(0, 320); + + bodyMaskOut === extended_json_input_2; + }); +}); \ No newline at end of file diff --git a/circuits/test/json/nivc/masker_nivc.test.ts b/circuits/test/json/nivc/masker_nivc.test.ts new file mode 100644 index 0000000..42e6dfe --- /dev/null +++ b/circuits/test/json/nivc/masker_nivc.test.ts @@ -0,0 +1,105 @@ +import { circomkit, WitnessTester, generateDescription, readJsonFile, toByte } from "../../common"; +import { join } from "path"; + +// HTTP/1.1 200 OK +// content-type: application/json; charset=utf-8 +// content-encoding: gzip +// Transfer-Encoding: chunked +// +// { +// "data": { +// "items": [ +// { +// "data": "Artist", +// "profile": { +// "name": "Taylor Swift" +// } +// } +// ] +// } +// } + +// 202 bytes in the JSON +let json_input = [123, 13, 10, 32, 32, 32, 34, 100, 97, 116, 97, 34, 58, 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, + 32, 34, 105, 116, 101, 109, 115, 34, 58, 32, 91, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 123, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 97, 116, + 97, 34, 58, 32, 34, 65, 114, 116, 105, 115, 116, 34, 44, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 34, 112, 114, 111, 102, 105, 108, 101, 34, 58, 32, 123, 13, 10, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, + 34, 84, 97, 121, 108, 111, 114, 32, 83, 119, 105, 102, 116, 34, 13, 10, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, + 13, 10, 32, 32, 32, 32, 32, 32, 32, 93, 13, 10, 32, 32, 32, 125, 13, 10, 125]; + +describe("NIVC Extract", async () => { + let parse_circuit: WitnessTester<["step_in"], ["step_out"]>; + let json_mask_object_circuit: WitnessTester<["step_in", "key", "keyLen"], ["step_out"]>; + let json_mask_arr_circuit: WitnessTester<["step_in", "index"], ["step_out"]>; + let extract_value_circuit: WitnessTester<["step_in"], ["step_out"]>; + + const DATA_BYTES = 202; + const MAX_STACK_HEIGHT = 5; + const MAX_KEY_LENGTH = 8; + const MAX_VALUE_LENGTH = 35; + const PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + const TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1; + + before(async () => { + parse_circuit = await circomkit.WitnessTester(`JsonParseNIVC`, { + file: "json/nivc/parse", + template: "JsonParseNIVC", + params: [DATA_BYTES, MAX_STACK_HEIGHT], + }); + console.log("#constraints:", await parse_circuit.getConstraintCount()); + + json_mask_arr_circuit = await circomkit.WitnessTester(`JsonMaskArrayIndexNIVC`, { + file: "json/nivc/masker", + template: "JsonMaskArrayIndexNIVC", + params: [DATA_BYTES, MAX_STACK_HEIGHT], + }); + console.log("#constraints:", await json_mask_arr_circuit.getConstraintCount()); + + json_mask_object_circuit = await circomkit.WitnessTester(`JsonMaskObjectNIVC`, { + file: "json/nivc/masker", + template: "JsonMaskObjectNIVC", + params: [DATA_BYTES, MAX_STACK_HEIGHT, MAX_KEY_LENGTH], + }); + console.log("#constraints:", await json_mask_object_circuit.getConstraintCount()); + + extract_value_circuit = await circomkit.WitnessTester(`JsonMaskExtractFinal`, { + file: "json/nivc/extractor", + template: "MaskExtractFinal", + params: [DATA_BYTES, MAX_STACK_HEIGHT, MAX_VALUE_LENGTH], + }); + console.log("#constraints:", await extract_value_circuit.getConstraintCount()); + }); + + let extended_json_input = json_input.concat(Array(Math.max(0, TOTAL_BYTES_ACROSS_NIVC - json_input.length)).fill(0)); + + let key0 = [100, 97, 116, 97, 0, 0, 0, 0]; // "data" + let key0Len = 4; + let key1 = [105, 116, 101, 109, 115, 0, 0, 0]; // "items" + let key1Len = 5; + let key2 = [112, 114, 111, 102, 105, 108, 101, 0]; // "profile" + let key2Len = 7; + let key3 = [110, 97, 109, 101, 0, 0, 0, 0]; "name" + let key3Len = 4; + + let value = toByte("\"Taylor Swift\""); + + it("parse and mask", async () => { + let json_parse = await parse_circuit.compute({ step_in: extended_json_input }, ["step_out"]); + + let json_extract_key0 = await json_mask_object_circuit.compute({ step_in: json_parse.step_out, key: key0, keyLen: key0Len }, ["step_out"]); + + let json_extract_key1 = await json_mask_object_circuit.compute({ step_in: json_extract_key0.step_out, key: key1, keyLen: key1Len }, ["step_out"]); + + let json_extract_arr = await json_mask_arr_circuit.compute({ step_in: json_extract_key1.step_out, index: 0 }, ["step_out"]); + + let json_extract_key2 = await json_mask_object_circuit.compute({ step_in: json_extract_arr.step_out, key: key2, keyLen: key2Len }, ["step_out"]); + + let json_extract_key3 = await json_mask_object_circuit.compute({ step_in: json_extract_key2.step_out, key: key3, keyLen: key3Len }, ["step_out"]); + + value = value.concat(Array(MAX_VALUE_LENGTH - value.length).fill(0)); + await extract_value_circuit.expectPass({ step_in: json_extract_key3.step_out }, { step_out: value }); + }); +}); \ No newline at end of file diff --git a/circuits/test/utils/search.test.ts b/circuits/test/utils/search.test.ts index 98b8bd8..f172fa1 100644 --- a/circuits/test/utils/search.test.ts +++ b/circuits/test/utils/search.test.ts @@ -152,14 +152,14 @@ describe("search", () => { }); }); - describe("SubstringMatchWithIndexx", () => { + describe("SubstringMatchWithIndexPadded", () => { let circuit: WitnessTester<["data", "key", "keyLen", "start"], ["out"]>; let maxKeyLen = 30; before(async () => { circuit = await circomkit.WitnessTester(`SubstringSearch`, { file: "utils/search", - template: "SubstringMatchWithIndexx", + template: "SubstringMatchWithIndexPadded", params: [787, maxKeyLen], }); console.log("#constraints:", await circuit.getConstraintCount()); diff --git a/circuits/utils/array.circom b/circuits/utils/array.circom index d19b4d6..fb28002 100644 --- a/circuits/utils/array.circom +++ b/circuits/utils/array.circom @@ -196,6 +196,8 @@ template ScalarArrayMul(n) { } } +// TODO: Below really needs documentation. + // from: https://github.com/pluto/aes-proof/blob/main/circuits/aes-gcm/helper_functions.circom template SumMultiple(n) { signal input nums[n]; @@ -229,4 +231,33 @@ template IndexSelector(total) { } out <== calcTotal.sum; +} + +template ArraySelector(m, n) { + signal input in[m][n]; + signal input index; + signal output out[n]; + assert(index >= 0 && index < m); + + signal selector[m]; + component Equal[m]; + for (var i = 0; i < m; i++) { + selector[i] <== IsEqual()([index, i]); + } + + var sum = 0; + for (var i = 0; i < m; i++) { + sum += selector[i]; + } + sum === 1; + + signal sums[n][m+1]; + // note: loop order is column-wise, not row-wise + for (var j = 0; j < n; j++) { + sums[j][0] <== 0; + for (var i = 0; i < m; i++) { + sums[j][i+1] <== sums[j][i] + in[i][j] * selector[i]; + } + out[j] <== sums[j][m]; + } } \ No newline at end of file diff --git a/circuits/utils/search.circom b/circuits/utils/search.circom index 821051d..6b9e1aa 100644 --- a/circuits/utils/search.circom +++ b/circuits/utils/search.circom @@ -232,7 +232,7 @@ template SubstringMatchWithIndex(dataLen, keyLen) { signal output out <== isStartLessThanMaxLength * isSubarrayMatch; } -template SubstringMatchWithIndexx(dataLen, maxKeyLen) { +template SubstringMatchWithIndexPadded(dataLen, maxKeyLen) { signal input data[dataLen]; signal input key[maxKeyLen]; signal input keyLen;