feat: http parse max lengths

circuits/http/nivc/lock_header.circom

@@ -2,6 +2,7 @@ pragma circom 2.1.9;
 
 include "../interpreter.circom";
 include "../../utils/array.circom";
+include "circomlib/circuits/comparators.circom";
 
 // TODO: should use a MAX_HEADER_NAME_LENGTH and a MAX_HEADER_VALUE_LENGTH
 template LockHeader(DATA_BYTES, MAX_STACK_HEIGHT, MAX_HEADER_NAME_LENGTH, MAX_HEADER_VALUE_LENGTH) {
@@ -48,9 +49,10 @@ template LockHeader(DATA_BYTES, MAX_STACK_HEIGHT, MAX_HEADER_NAME_LENGTH, MAX_HE
     signal headerFieldNameValueMatch <==  HeaderFieldNameValueMatchPadded(DATA_BYTES, MAX_HEADER_NAME_LENGTH, MAX_HEADER_VALUE_LENGTH)(data, header, headerNameLength, value, headerValueLength, headerNameLocation);
     headerFieldNameValueMatch === 1;
 
-    // parser state should be parsing header
+    // parser state should be parsing header upto 2^10 max headers
     signal isParsingHeader <== IndexSelector(DATA_BYTES * 5)(httpParserState, headerNameLocation * 5 + 1);
-    isParsingHeader === 1;
+    signal parsingHeader <== GreaterThan(10)([isParsingHeader, 0]);
+    parsingHeader === 1;
 
     // ------------------------------------------------------------------------------------------------------------------ //
     // ~ Write out to next NIVC step

circuits/http/nivc/parse_and_lock_start_line.circom

@@ -6,7 +6,7 @@ include "../../utils/bytes.circom";
 
 // TODO: Note that TOTAL_BYTES will match what we have for AESGCMFOLD step_out
 // I have not gone through to double check the sizes of everything yet.
-template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, MIDDLE_LENGTH, FINAL_LENGTH) {
+template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, MAX_BEGINNING_LENGTH, MAX_MIDDLE_LENGTH, MAX_FINAL_LENGTH) {
     // ------------------------------------------------------------------------------------------------------------------ //
     // ~~ Set sizes at compile time ~~
     // Total number of variables in the parser for each byte of data
@@ -26,7 +26,7 @@ template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, M
     // ------------------------------------------------------------------------------------------------------------------ //
     // ~ Unravel from previous NIVC step ~
     // Read in from previous NIVC step (JsonParseNIVC)
-    signal input step_in[TOTAL_BYTES_ACROSS_NIVC]; 
+    signal input step_in[TOTAL_BYTES_ACROSS_NIVC];
     signal output step_out[TOTAL_BYTES_ACROSS_NIVC];
 
     signal data[DATA_BYTES];
@@ -40,9 +40,12 @@ template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, M
     // component dataASCII = ASCII(DATA_BYTES);
     // dataASCII.in <== data;
 
-    signal input beginning[BEGINNING_LENGTH];
-    signal input middle[MIDDLE_LENGTH];
-    signal input final[FINAL_LENGTH];
+    signal input beginning[MAX_BEGINNING_LENGTH];
+    signal input beginning_length;
+    signal input middle[MAX_MIDDLE_LENGTH];
+    signal input middle_length;
+    signal input final[MAX_FINAL_LENGTH];
+    signal input final_length;
 
     // Initialze the parser
     component State[DATA_BYTES];
@@ -60,10 +63,6 @@ template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, M
     we can make this more efficient by just comparing the first `BEGINNING_LENGTH` bytes
     of the data ASCII against the beginning ASCII itself.
     */
-    // Check first beginning byte
-    signal beginningIsEqual[BEGINNING_LENGTH];
-    beginningIsEqual[0] <== IsEqual()([data[0],beginning[0]]);
-    beginningIsEqual[0] === 1;
 
     // Setup to check middle bytes
     signal startLineMask[DATA_BYTES];
@@ -87,12 +86,6 @@ template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, M
         State[data_idx].parsing_body        <== State[data_idx - 1].next_parsing_body;
         State[data_idx].line_status         <== State[data_idx - 1].next_line_status;
 
-        // Check remaining beginning bytes
-        if(data_idx < BEGINNING_LENGTH) {
-            beginningIsEqual[data_idx] <== IsEqual()([data[data_idx], beginning[data_idx]]);
-            beginningIsEqual[data_idx] === 1;
-        }
-
         // Set the masks based on parser state
         startLineMask[data_idx] <== inStartLine()(State[data_idx].parsing_start);
         middleMask[data_idx]    <== inStartMiddle()(State[data_idx].parsing_start);
@@ -105,18 +98,20 @@ template ParseAndLockStartLine(DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, M
     }
 
     // Additionally verify beginning had correct length
-    BEGINNING_LENGTH === middle_start_counter - 1;
+    beginning_length === middle_start_counter - 1;
+
+    signal beginningMatch <== SubstringMatchWithIndexPadded(DATA_BYTES, MAX_BEGINNING_LENGTH)(data, beginning, beginning_length, 0);
 
     // Check middle is correct by substring match and length check
-    signal middleMatch <== SubstringMatchWithIndex(DATA_BYTES, MIDDLE_LENGTH)(data, middle, middle_start_counter);
+    signal middleMatch <== SubstringMatchWithIndexPadded(DATA_BYTES, MAX_MIDDLE_LENGTH)(data, middle, middle_length, middle_start_counter);
     middleMatch === 1;
-    MIDDLE_LENGTH === middle_end_counter - middle_start_counter - 1;
+    middle_length === middle_end_counter - middle_start_counter - 1;
 
     // Check final is correct by substring match and length check
-    signal finalMatch <== SubstringMatchWithIndex(DATA_BYTES, FINAL_LENGTH)(data, final, middle_end_counter);
+    signal finalMatch <== SubstringMatchWithIndexPadded(DATA_BYTES, MAX_FINAL_LENGTH)(data, final, final_length, middle_end_counter);
     finalMatch === 1;
     // -2 here for the CRLF
-    FINAL_LENGTH === final_end_counter - middle_end_counter - 2;
+    final_length === final_end_counter - middle_end_counter - 2;
 
     // ------------------------------------------------------------------------------------------------------------------ //
     // ~ Write out to next NIVC step (Lock Header)

circuits/test/common/http.ts

@@ -56,7 +56,7 @@ export function readHTTPInputFile(filename: string) {
 
         headerLines.forEach(line => {
             const [key, value] = line.split(/:\s(.+)/);
-            if (key) headers[key.toLowerCase()] = value ? value : '';
+            if (key) headers[key] = value ? value : '';
         });
 
         return headers;

circuits/test/http/codegen.test.ts

@@ -138,7 +138,7 @@ describe("HTTP :: Codegen :: Response", async () => {
 
         const headers = getHeaders(lockData);
 
-        const params = [input.length, parseInt(http.headers["Content-Length".toLowerCase()]), lockData.version.length, lockData.status.length, lockData.message.length];
+        const params = [input.length, parseInt(http.headers["Content-Length"]), lockData.version.length, lockData.status.length, lockData.message.length];
         headers.forEach(header => {
             params.push(header[0].length);
             params.push(header[1].length);
@@ -184,7 +184,7 @@ describe("HTTP :: Codegen :: Response", async () => {
 
         const headers = getHeaders(lockData);
 
-        const params = [input.length, parseInt(http.headers["Content-Length".toLowerCase()]), lockData.version.length, lockData.status.length, lockData.message.length];
+        const params = [input.length, parseInt(http.headers["Content-Length"]), lockData.version.length, lockData.status.length, lockData.message.length];
         headers.forEach(header => {
             params.push(header[0].length);
             params.push(header[1].length);

circuits/test/http/nivc/nivc.test.ts → circuits/test/http/nivc/body_mask.test.ts

@@ -1,5 +1,4 @@
-import { circomkit, WitnessTester, generateDescription, readJsonFile, toByte } from "../../common";
-import { join } from "path";
+import { circomkit, WitnessTester, toByte } from "../../common";
 
 // HTTP/1.1 200 OK
 // content-type: application/json; charset=utf-8
@@ -37,8 +36,8 @@ let http_response_plaintext = [
     10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 13, 10, 32, 32, 32, 32, 32, 32, 32, 93, 13,
     10, 32, 32, 32, 125, 13, 10, 125];
 
-describe("HTTPParseAndLockStartLineNIVC", async () => {
-    let httpParseAndLockStartLineCircuit: WitnessTester<["step_in", "beginning", "middle", "final"], ["step_out"]>;
+describe("NIVC_HTTP", async () => {
+    let httpParseAndLockStartLineCircuit: WitnessTester<["step_in", "beginning", "beginning_length", "middle", "middle_length", "final", "final_length"], ["step_out"]>;
     let lockHeaderCircuit: WitnessTester<["step_in", "header", "headerNameLength", "value", "headerValueLength"], ["step_out"]>;
     let bodyMaskCircuit: WitnessTester<["step_in"], ["step_out"]>;
 
@@ -49,19 +48,19 @@ describe("HTTPParseAndLockStartLineNIVC", async () => {
 
     const MAX_HEADER_NAME_LENGTH = 20;
     const MAX_HEADER_VALUE_LENGTH = 35;
+    const MAX_BEGINNING_LENGTH = 10;
+    const MAX_MIDDLE_LENGTH = 30;
+    const MAX_FINAL_LENGTH = 10;
 
     const beginning = [72, 84, 84, 80, 47, 49, 46, 49]; // HTTP/1.1
-    const BEGINNING_LENGTH = 8;
     const middle = [50, 48, 48]; // 200
-    const MIDDLE_LENGTH = 3;
     const final = [79, 75]; // OK
-    const FINAL_LENGTH = 2;
 
     before(async () => {
         httpParseAndLockStartLineCircuit = await circomkit.WitnessTester(`ParseAndLockStartLine`, {
             file: "http/nivc/parse_and_lock_start_line",
             template: "ParseAndLockStartLine",
-            params: [DATA_BYTES, MAX_STACK_HEIGHT, BEGINNING_LENGTH, MIDDLE_LENGTH, FINAL_LENGTH],
+            params: [DATA_BYTES, MAX_STACK_HEIGHT, MAX_BEGINNING_LENGTH, MAX_MIDDLE_LENGTH, MAX_FINAL_LENGTH],
         });
         console.log("#constraints:", await httpParseAndLockStartLineCircuit.getConstraintCount());
 
@@ -87,8 +86,11 @@ describe("HTTPParseAndLockStartLineNIVC", async () => {
 
     let headerNamePadded = headerName.concat(Array(MAX_HEADER_NAME_LENGTH - headerName.length).fill(0));
     let headerValuePadded = headerValue.concat(Array(MAX_HEADER_VALUE_LENGTH - headerValue.length).fill(0));
+    let beginningPadded = beginning.concat(Array(MAX_BEGINNING_LENGTH - beginning.length).fill(0));
+    let middlePadded = middle.concat(Array(MAX_MIDDLE_LENGTH - middle.length).fill(0));
+    let finalPadded = final.concat(Array(MAX_FINAL_LENGTH - final.length).fill(0));
     it("HTTPParseAndExtract", async () => {
-        let parseAndLockStartLine = await httpParseAndLockStartLineCircuit.compute({ step_in: extendedJsonInput, beginning: beginning, middle: middle, final: final }, ["step_out"]);
+        let parseAndLockStartLine = await httpParseAndLockStartLineCircuit.compute({ step_in: extendedJsonInput, beginning: beginningPadded, beginning_length: beginning.length, middle: middlePadded, middle_length: middle.length, final: finalPadded, final_length: final.length }, ["step_out"]);
 
         let lockHeader = await lockHeaderCircuit.compute({ step_in: parseAndLockStartLine.step_out, header: headerNamePadded, headerNameLength: headerName.length, value: headerValuePadded, headerValueLength: headerValue.length }, ["step_out"]);
 
@@ -97,9 +99,9 @@ describe("HTTPParseAndLockStartLineNIVC", async () => {
         let bodyMaskOut = bodyMask.step_out as number[];
         let idx = bodyMaskOut.indexOf('{'.charCodeAt(0));
 
-        let extended_json_input_2 = extendedJsonInput.fill(0, 0, idx);
-        extended_json_input_2 = extended_json_input_2.fill(0, 320);
+        let maskedInput = extendedJsonInput.fill(0, 0, idx);
+        maskedInput = maskedInput.fill(0, 320);
 
-        bodyMaskOut === extended_json_input_2;
+        bodyMaskOut === maskedInput;
     });
 });

circuits/test/http/nivc/lock_header.test.ts

@@ -0,0 +1,100 @@
+import { circomkit, WitnessTester, toByte } from "../../common";
+import { readHTTPInputFile } from "../../common/http";
+
+describe("HTTPLockHeader", async () => {
+    let httpParseAndLockStartLineCircuit: WitnessTester<["step_in", "beginning", "beginning_length", "middle", "middle_length", "final", "final_length"], ["step_out"]>;
+    let lockHeaderCircuit: WitnessTester<["step_in", "header", "headerNameLength", "value", "headerValueLength"], ["step_out"]>;
+
+    const DATA_BYTES = 320;
+    const MAX_STACK_HEIGHT = 5;
+    const PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2;
+    const TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1;
+
+    const MAX_BEGINNING_LENGTH = 10;
+    const MAX_MIDDLE_LENGTH = 50;
+    const MAX_FINAL_LENGTH = 10;
+    const MAX_HEADER_NAME_LENGTH = 20;
+    const MAX_HEADER_VALUE_LENGTH = 35;
+
+    before(async () => {
+        httpParseAndLockStartLineCircuit = await circomkit.WitnessTester(`ParseAndLockStartLine`, {
+            file: "http/nivc/parse_and_lock_start_line",
+            template: "ParseAndLockStartLine",
+            params: [DATA_BYTES, MAX_STACK_HEIGHT, MAX_BEGINNING_LENGTH, MAX_MIDDLE_LENGTH, MAX_FINAL_LENGTH],
+        });
+        console.log("#constraints:", await httpParseAndLockStartLineCircuit.getConstraintCount());
+
+        lockHeaderCircuit = await circomkit.WitnessTester(`LockHeader`, {
+            file: "http/nivc/lock_header",
+            template: "LockHeader",
+            params: [DATA_BYTES, MAX_STACK_HEIGHT, MAX_HEADER_NAME_LENGTH, MAX_HEADER_VALUE_LENGTH],
+        });
+        console.log("#constraints:", await lockHeaderCircuit.getConstraintCount());
+    });
+
+    function generatePassCase(input: number[], beginning: number[], middle: number[], final: number[], headerName: number[], headerValue: number[], desc: string) {
+        it(`should pass: \"${headerName}: ${headerValue}\", ${desc}`, async () => {
+            let extendedInput = input.concat(Array(Math.max(0, TOTAL_BYTES_ACROSS_NIVC - input.length)).fill(0));
+
+            let beginningPadded = beginning.concat(Array(MAX_BEGINNING_LENGTH - beginning.length).fill(0));
+            let middlePadded = middle.concat(Array(MAX_MIDDLE_LENGTH - middle.length).fill(0));
+            let finalPadded = final.concat(Array(MAX_FINAL_LENGTH - final.length).fill(0));
+
+            let headerNamePadded = headerName.concat(Array(MAX_HEADER_NAME_LENGTH - headerName.length).fill(0));
+            let headerValuePadded = headerValue.concat(Array(MAX_HEADER_VALUE_LENGTH - headerValue.length).fill(0));
+
+            let parseAndLockStartLine = await httpParseAndLockStartLineCircuit.compute({ step_in: extendedInput, beginning: beginningPadded, beginning_length: beginning.length, middle: middlePadded, middle_length: middle.length, final: finalPadded, final_length: final.length }, ["step_out"]);
+
+            await lockHeaderCircuit.expectPass({ step_in: parseAndLockStartLine.step_out, header: headerNamePadded, headerNameLength: headerName.length, value: headerValuePadded, headerValueLength: headerValue.length });
+        });
+    }
+
+    function generateFailCase(input: number[], beginning: number[], middle: number[], final: number[], headerName: number[], headerValue: number[], desc: string) {
+        it(`should fail: ${desc}`, async () => {
+            let extendedInput = input.concat(Array(Math.max(0, TOTAL_BYTES_ACROSS_NIVC - input.length)).fill(0));
+
+            let beginningPadded = beginning.concat(Array(MAX_BEGINNING_LENGTH - beginning.length).fill(0));
+            let middlePadded = middle.concat(Array(MAX_MIDDLE_LENGTH - middle.length).fill(0));
+            let finalPadded = final.concat(Array(MAX_FINAL_LENGTH - final.length).fill(0));
+
+            let headerNamePadded = headerName.concat(Array(MAX_HEADER_NAME_LENGTH - headerName.length).fill(0));
+            let headerValuePadded = headerValue.concat(Array(MAX_HEADER_VALUE_LENGTH - headerValue.length).fill(0));
+
+            let parseAndLockStartLine = await httpParseAndLockStartLineCircuit.compute({ step_in: extendedInput, beginning: beginningPadded, beginning_length: beginning.length, middle: middlePadded, middle_length: middle.length, final: finalPadded, final_length: final.length }, ["step_out"]);
+
+            await lockHeaderCircuit.expectFail({ step_in: parseAndLockStartLine.step_out, header: headerNamePadded, headerNameLength: headerName.length, value: headerValuePadded, headerValueLength: headerValue.length });
+        });
+    }
+
+    describe("request", async () => {
+        let { input, headers } = readHTTPInputFile("post_request.http");
+
+        let beginning = toByte("POST");
+        let middle = toByte("/contact_form.php");
+        let final = toByte("HTTP/1.1");
+
+        let headerName = toByte("Host");
+        let headerValue = toByte("developer.mozilla.org");
+
+        for (const [key, value] of Object.entries(headers)) {
+            generatePassCase(input, beginning, middle, final, toByte(key), toByte(value), "request");
+        }
+        let incorrectHeaderValue = toByte("application/json");
+        generateFailCase(input, beginning, middle, final, headerName, incorrectHeaderValue, "incorrect header value");
+    });
+
+    describe("response", async () => {
+        let { input, headers } = readHTTPInputFile("spotify_top_artists_response.http");
+        let beginning = toByte("HTTP/1.1");
+        let middle = toByte("200");
+        let final = toByte("OK");
+
+        for (const [key, value] of Object.entries(headers)) {
+            generatePassCase(input, beginning, middle, final, toByte(key), toByte(value), "response");
+        }
+
+        let headerName = toByte("content-encoding");
+        let invalidHeaderValue = toByte("chunked");
+        generateFailCase(input, beginning, middle, final, headerName, invalidHeaderValue, "should fail: invalid header value");
+    });
+});