From d839e846dd7b04f460a34d9b9f648345bde40d1f Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Oct 2024 17:03:35 -0600 Subject: [PATCH 1/6] mv: `*.circom` into `circuits/` --- aes_gcm.circom => circuits/aes_gcm.circom | 0 extract_value.circom => circuits/extract_value.circom | 0 http_body_mask.circom => circuits/http_body_mask.circom | 0 http_lock_header.circom => circuits/http_lock_header.circom | 0 .../http_parse_and_lock_start_line.circom | 0 .../json_mask_array_index.circom | 0 json_mask_object.circom => circuits/json_mask_object.circom | 0 json_parse.circom => circuits/json_parse.circom | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename aes_gcm.circom => circuits/aes_gcm.circom (100%) rename extract_value.circom => circuits/extract_value.circom (100%) rename http_body_mask.circom => circuits/http_body_mask.circom (100%) rename http_lock_header.circom => circuits/http_lock_header.circom (100%) rename http_parse_and_lock_start_line.circom => circuits/http_parse_and_lock_start_line.circom (100%) rename json_mask_array_index.circom => circuits/json_mask_array_index.circom (100%) rename json_mask_object.circom => circuits/json_mask_object.circom (100%) rename json_parse.circom => circuits/json_parse.circom (100%) diff --git a/aes_gcm.circom b/circuits/aes_gcm.circom similarity index 100% rename from aes_gcm.circom rename to circuits/aes_gcm.circom diff --git a/extract_value.circom b/circuits/extract_value.circom similarity index 100% rename from extract_value.circom rename to circuits/extract_value.circom diff --git a/http_body_mask.circom b/circuits/http_body_mask.circom similarity index 100% rename from http_body_mask.circom rename to circuits/http_body_mask.circom diff --git a/http_lock_header.circom b/circuits/http_lock_header.circom similarity index 100% rename from http_lock_header.circom rename to circuits/http_lock_header.circom diff --git a/http_parse_and_lock_start_line.circom b/circuits/http_parse_and_lock_start_line.circom similarity index 100% rename from http_parse_and_lock_start_line.circom rename to circuits/http_parse_and_lock_start_line.circom diff --git a/json_mask_array_index.circom b/circuits/json_mask_array_index.circom similarity index 100% rename from json_mask_array_index.circom rename to circuits/json_mask_array_index.circom diff --git a/json_mask_object.circom b/circuits/json_mask_object.circom similarity index 100% rename from json_mask_object.circom rename to circuits/json_mask_object.circom diff --git a/json_parse.circom b/circuits/json_parse.circom similarity index 100% rename from json_parse.circom rename to circuits/json_parse.circom From 15c754ec68716c996f4e141ea68ce8990a96501a Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Oct 2024 17:04:46 -0600 Subject: [PATCH 2/6] Create artifacts.yaml --- .github/workflows/artifacts.yaml | 86 ++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/artifacts.yaml diff --git a/.github/workflows/artifacts.yaml b/.github/workflows/artifacts.yaml new file mode 100644 index 0000000..e8f6676 --- /dev/null +++ b/.github/workflows/artifacts.yaml @@ -0,0 +1,86 @@ +name: build-circuits + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Protocol Buffers + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libprotobuf-dev + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2024-06-10 + + - name: Install Circom + run: | + CIRCOM_VERSION=2.1.9 + curl -L https://github.com/iden3/circom/releases/download/v$CIRCOM_VERSION/circom-linux-amd64 -o circom + chmod +x circom + sudo mv circom /usr/local/bin/ + circom --version + + - name: Install circom-witnesscalc + run: | + cd .. && git clone https://github.com/pluto/circom-witnesscalc.git + cd circom-witnesscalc + cargo install --path . + echo $(which build-circuit) + + - name: Install Node.js dependencies + run: | + npm install + + - name: Compile Circom circuits + run: | + mkdir -p artifacts + for circuit in circuits/*.circom; do + if [ -f "$circuit" ]; then + filename=$(basename "$circuit" .circom) + output_dir="artifacts/$filename" + mkdir -p "$output_dir" + + echo "Processing $filename..." + + # Run circom compilation + circom "$circuit" --r1cs --wasm -o "$output_dir" -l node_modules + + # Run witness calculator build + build-circuit "$circuit" "$output_dir/$filename.bin" -l node_modules + fi + done + + - name: Create release artifacts + run: | + cd artifacts + zip -r ../circom-artifacts.zip ./* + cd .. + + - name: Create Release + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: softprops/action-gh-release@v1 + with: + files: circom-artifacts.zip + name: Circuit Artifacts ${{ github.sha }} + tag_name: v${{ github.run_number }} + body: | + Automated release of compiled Circom circuits + Commit: ${{ github.sha }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From c1fb2273e0bcec573fa3bd6b06292803eae26562 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Oct 2024 17:19:12 -0600 Subject: [PATCH 3/6] fix: missing circuit import --- circuits/json_mask_array_index.circom | 2 +- circuits/json_mask_object.circom | 2 +- circuits/json_universal.circom | 174 ++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 circuits/json_universal.circom diff --git a/circuits/json_mask_array_index.circom b/circuits/json_mask_array_index.circom index 5633232..73ffab7 100644 --- a/circuits/json_mask_array_index.circom +++ b/circuits/json_mask_array_index.circom @@ -1,5 +1,5 @@ pragma circom 2.1.9; -include "../json_universal.circom"; +include "json_universal.circom"; component main { public [step_in] } = JsonMaskArrayIndexNIVC(4160, 320, 5); \ No newline at end of file diff --git a/circuits/json_mask_object.circom b/circuits/json_mask_object.circom index 52622eb..7054f35 100644 --- a/circuits/json_mask_object.circom +++ b/circuits/json_mask_object.circom @@ -1,5 +1,5 @@ pragma circom 2.1.9; -include "../json_universal.circom"; +include "json_universal.circom"; component main { public [step_in] } = JsonMaskObjectNIVC(4160, 320, 5, 10); \ No newline at end of file diff --git a/circuits/json_universal.circom b/circuits/json_universal.circom new file mode 100644 index 0000000..de21a44 --- /dev/null +++ b/circuits/json_universal.circom @@ -0,0 +1,174 @@ +pragma circom 2.1.9; + +include "parser-attestor/circuits/json/interpreter.circom"; + +template JsonMaskObjectNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + assert(MAX_STACK_HEIGHT >= 2); + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_USED = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1); // data + parser vars + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (JsonParseNIVC) + signal input step_in[TOTAL_BYTES]; + + // Grab the raw data bytes from the `step_in` variable + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + + // Decode the encoded data in `step_in` back into parser variables + signal stack[DATA_BYTES][MAX_STACK_HEIGHT][2]; + signal parsingData[DATA_BYTES][2]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + for (var j = 0 ; j < MAX_STACK_HEIGHT ; j++) { + stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; + stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; + } + parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; + parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; + } + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Object masking ~ + // Key data to use to point to which object to extract + signal input key[maxKeyLen]; + signal input keyLen; + + // flag determining whether this byte is matched value + signal is_value_match[DATA_BYTES - maxKeyLen]; + // final mask + signal mask[DATA_BYTES - maxKeyLen]; + + + // signal parsing_object_value[DATA_BYTES - maxKeyLen]; + signal is_key_match[DATA_BYTES - maxKeyLen]; + signal is_key_match_for_value[DATA_BYTES + 1 - maxKeyLen]; + is_key_match_for_value[0] <== 0; + signal is_next_pair_at_depth[DATA_BYTES - maxKeyLen]; + signal or[DATA_BYTES - maxKeyLen]; + + // Signals to detect if we are parsing a key or value with initial setup + signal parsing_key[DATA_BYTES - maxKeyLen]; + signal parsing_value[DATA_BYTES - maxKeyLen]; + // TODO: Can't these just be 0 since the start of object can't be either of these? + // parsing_key[0] <== InsideKey()(stack[0][0], parsingData[0][0], parsingData[0][1]); + // parsing_value[0] <== InsideValueObject()(stack[0][0], stack[0][1], parsingData[0][0], parsingData[0][1]); + + // Initialize values knowing 0th bit of data will never be a key/value + parsing_key[0] <== 0; + parsing_value[0] <== 0; + is_key_match[0] <== 0; + + is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(stack[0], data[0]); + is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); + is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; + + mask[0] <== data[0] * is_value_match[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES - maxKeyLen; data_idx++) { + parsing_key[data_idx] <== InsideKey()(stack[data_idx][0], parsingData[data_idx][0], parsingData[data_idx][1]); + parsing_value[data_idx] <== InsideValueObject()(stack[data_idx][0], stack[data_idx][1], parsingData[data_idx][0], parsingData[data_idx][1]); + + // to get correct value, check: + // - key matches at current index and depth of key is as specified + // - whether next KV pair starts + // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) + is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, maxKeyLen, data_idx)(data, key, keyLen, parsing_key[data_idx]); + is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(stack[data_idx], data[data_idx]); + is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); + is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; + + or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx - 1]); + + // mask = currently parsing value and all subsequent keys matched + mask[data_idx] <== data[data_idx] * or[data_idx]; + + } + + // Write the `step_out` with masked data + signal output step_out[TOTAL_BYTES]; + for (var i = 0 ; i < DATA_BYTES - maxKeyLen ; i++) { + step_out[i] <== mask[i]; + } + for (var i = 0 ; i < maxKeyLen ; i++) { + step_out[DATA_BYTES - maxKeyLen + i] <== 0; + } + // Append the parser state back on `step_out` + for (var i = DATA_BYTES ; i < TOTAL_BYTES ; i++) { + step_out[i] <== step_in[i]; + } + // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED +} + +template JsonMaskArrayIndexNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + assert(MAX_STACK_HEIGHT >= 2); + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_USED = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1); + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (JsonParseNIVC) + signal input step_in[TOTAL_BYTES]; + + // Grab the raw data bytes from the `step_in` variable + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + + // Decode the encoded data in `step_in` back into parser variables + signal stack[DATA_BYTES][MAX_STACK_HEIGHT][2]; + signal parsingData[DATA_BYTES][2]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + for (var j = 0 ; j < MAX_STACK_HEIGHT ; j++) { + stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; + stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; + } + parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; + parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; + } + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Array index masking ~ + signal input index; + + // value starting index in `data` + signal value_starting_index[DATA_BYTES]; + signal mask[DATA_BYTES]; + + signal parsing_array[DATA_BYTES]; + signal or[DATA_BYTES]; + + parsing_array[0] <== InsideArrayIndexObject()(stack[0][0], stack[0][1], parsingData[0][0], parsingData[0][1], index); + mask[0] <== data[0] * parsing_array[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + parsing_array[data_idx] <== InsideArrayIndexObject()(stack[data_idx][0], stack[data_idx][1], parsingData[data_idx][0], parsingData[data_idx][1], index); + + or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]); + mask[data_idx] <== data[data_idx] * or[data_idx]; + } + + // Write the `step_out` with masked data + signal output step_out[TOTAL_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + step_out[i] <== mask[i]; + } + // Append the parser state back on `step_out` + for (var i = DATA_BYTES ; i < TOTAL_BYTES ; i++) { + step_out[i] <== step_in[i]; + } + // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED +} \ No newline at end of file From 997270f99fe96e20bfbd644d4704541050f57c31 Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Oct 2024 17:29:47 -0600 Subject: [PATCH 4/6] redo circuits --- circuits/json_mask_array_index.circom | 66 +++++++++- circuits/json_mask_object.circom | 109 +++++++++++++++- circuits/json_universal.circom | 174 -------------------------- 3 files changed, 172 insertions(+), 177 deletions(-) delete mode 100644 circuits/json_universal.circom diff --git a/circuits/json_mask_array_index.circom b/circuits/json_mask_array_index.circom index 73ffab7..ba53e08 100644 --- a/circuits/json_mask_array_index.circom +++ b/circuits/json_mask_array_index.circom @@ -1,5 +1,69 @@ pragma circom 2.1.9; -include "json_universal.circom"; +template JsonMaskArrayIndexNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + assert(MAX_STACK_HEIGHT >= 2); + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_USED = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1); + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (JsonParseNIVC) + signal input step_in[TOTAL_BYTES]; + + // Grab the raw data bytes from the `step_in` variable + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + + // Decode the encoded data in `step_in` back into parser variables + signal stack[DATA_BYTES][MAX_STACK_HEIGHT][2]; + signal parsingData[DATA_BYTES][2]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + for (var j = 0 ; j < MAX_STACK_HEIGHT ; j++) { + stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; + stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; + } + parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; + parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; + } + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Array index masking ~ + signal input index; + + // value starting index in `data` + signal value_starting_index[DATA_BYTES]; + signal mask[DATA_BYTES]; + + signal parsing_array[DATA_BYTES]; + signal or[DATA_BYTES]; + + parsing_array[0] <== InsideArrayIndexObject()(stack[0][0], stack[0][1], parsingData[0][0], parsingData[0][1], index); + mask[0] <== data[0] * parsing_array[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { + parsing_array[data_idx] <== InsideArrayIndexObject()(stack[data_idx][0], stack[data_idx][1], parsingData[data_idx][0], parsingData[data_idx][1], index); + + or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]); + mask[data_idx] <== data[data_idx] * or[data_idx]; + } + + // Write the `step_out` with masked data + signal output step_out[TOTAL_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + step_out[i] <== mask[i]; + } + // Append the parser state back on `step_out` + for (var i = DATA_BYTES ; i < TOTAL_BYTES ; i++) { + step_out[i] <== step_in[i]; + } + // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED +} component main { public [step_in] } = JsonMaskArrayIndexNIVC(4160, 320, 5); \ No newline at end of file diff --git a/circuits/json_mask_object.circom b/circuits/json_mask_object.circom index 7054f35..3cd1e63 100644 --- a/circuits/json_mask_object.circom +++ b/circuits/json_mask_object.circom @@ -1,5 +1,110 @@ pragma circom 2.1.9; -include "json_universal.circom"; +include "parser-attestor/circuits/json/interpreter.circom"; -component main { public [step_in] } = JsonMaskObjectNIVC(4160, 320, 5, 10); \ No newline at end of file +template JsonMaskObjectNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen) { + // ------------------------------------------------------------------------------------------------------------------ // + // ~~ Set sizes at compile time ~~ + // Total number of variables in the parser for each byte of data + assert(MAX_STACK_HEIGHT >= 2); + var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; + var TOTAL_BYTES_USED = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1); // data + parser vars + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Unravel from previous NIVC step ~ + // Read in from previous NIVC step (JsonParseNIVC) + signal input step_in[TOTAL_BYTES]; + + // Grab the raw data bytes from the `step_in` variable + signal data[DATA_BYTES]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + data[i] <== step_in[i]; + } + + // Decode the encoded data in `step_in` back into parser variables + signal stack[DATA_BYTES][MAX_STACK_HEIGHT][2]; + signal parsingData[DATA_BYTES][2]; + for (var i = 0 ; i < DATA_BYTES ; i++) { + for (var j = 0 ; j < MAX_STACK_HEIGHT ; j++) { + stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; + stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; + } + parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; + parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; + } + // ------------------------------------------------------------------------------------------------------------------ // + + // ------------------------------------------------------------------------------------------------------------------ // + // ~ Object masking ~ + // Key data to use to point to which object to extract + signal input key[maxKeyLen]; + signal input keyLen; + + // flag determining whether this byte is matched value + signal is_value_match[DATA_BYTES - maxKeyLen]; + // final mask + signal mask[DATA_BYTES - maxKeyLen]; + + + // signal parsing_object_value[DATA_BYTES - maxKeyLen]; + signal is_key_match[DATA_BYTES - maxKeyLen]; + signal is_key_match_for_value[DATA_BYTES + 1 - maxKeyLen]; + is_key_match_for_value[0] <== 0; + signal is_next_pair_at_depth[DATA_BYTES - maxKeyLen]; + signal or[DATA_BYTES - maxKeyLen]; + + // Signals to detect if we are parsing a key or value with initial setup + signal parsing_key[DATA_BYTES - maxKeyLen]; + signal parsing_value[DATA_BYTES - maxKeyLen]; + // TODO: Can't these just be 0 since the start of object can't be either of these? + // parsing_key[0] <== InsideKey()(stack[0][0], parsingData[0][0], parsingData[0][1]); + // parsing_value[0] <== InsideValueObject()(stack[0][0], stack[0][1], parsingData[0][0], parsingData[0][1]); + + // Initialize values knowing 0th bit of data will never be a key/value + parsing_key[0] <== 0; + parsing_value[0] <== 0; + is_key_match[0] <== 0; + + is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(stack[0], data[0]); + is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); + is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; + + mask[0] <== data[0] * is_value_match[0]; + + for(var data_idx = 1; data_idx < DATA_BYTES - maxKeyLen; data_idx++) { + parsing_key[data_idx] <== InsideKey()(stack[data_idx][0], parsingData[data_idx][0], parsingData[data_idx][1]); + parsing_value[data_idx] <== InsideValueObject()(stack[data_idx][0], stack[data_idx][1], parsingData[data_idx][0], parsingData[data_idx][1]); + + // to get correct value, check: + // - key matches at current index and depth of key is as specified + // - whether next KV pair starts + // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) + is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, maxKeyLen, data_idx)(data, key, keyLen, parsing_key[data_idx]); + is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(stack[data_idx], data[data_idx]); + is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); + is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; + + or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx - 1]); + + // mask = currently parsing value and all subsequent keys matched + mask[data_idx] <== data[data_idx] * or[data_idx]; + + } + + // Write the `step_out` with masked data + signal output step_out[TOTAL_BYTES]; + for (var i = 0 ; i < DATA_BYTES - maxKeyLen ; i++) { + step_out[i] <== mask[i]; + } + for (var i = 0 ; i < maxKeyLen ; i++) { + step_out[DATA_BYTES - maxKeyLen + i] <== 0; + } + // Append the parser state back on `step_out` + for (var i = DATA_BYTES ; i < TOTAL_BYTES ; i++) { + step_out[i] <== step_in[i]; + } + // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED +} + +component main { public [step_in] } = JsonMaskArrayIndexNIVC(4160, 320, 5); diff --git a/circuits/json_universal.circom b/circuits/json_universal.circom deleted file mode 100644 index de21a44..0000000 --- a/circuits/json_universal.circom +++ /dev/null @@ -1,174 +0,0 @@ -pragma circom 2.1.9; - -include "parser-attestor/circuits/json/interpreter.circom"; - -template JsonMaskObjectNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen) { - // ------------------------------------------------------------------------------------------------------------------ // - // ~~ Set sizes at compile time ~~ - // Total number of variables in the parser for each byte of data - assert(MAX_STACK_HEIGHT >= 2); - var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; - var TOTAL_BYTES_USED = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1); // data + parser vars - // ------------------------------------------------------------------------------------------------------------------ // - - // ------------------------------------------------------------------------------------------------------------------ // - // ~ Unravel from previous NIVC step ~ - // Read in from previous NIVC step (JsonParseNIVC) - signal input step_in[TOTAL_BYTES]; - - // Grab the raw data bytes from the `step_in` variable - signal data[DATA_BYTES]; - for (var i = 0 ; i < DATA_BYTES ; i++) { - data[i] <== step_in[i]; - } - - // Decode the encoded data in `step_in` back into parser variables - signal stack[DATA_BYTES][MAX_STACK_HEIGHT][2]; - signal parsingData[DATA_BYTES][2]; - for (var i = 0 ; i < DATA_BYTES ; i++) { - for (var j = 0 ; j < MAX_STACK_HEIGHT ; j++) { - stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; - stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; - } - parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; - parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; - } - // ------------------------------------------------------------------------------------------------------------------ // - - // ------------------------------------------------------------------------------------------------------------------ // - // ~ Object masking ~ - // Key data to use to point to which object to extract - signal input key[maxKeyLen]; - signal input keyLen; - - // flag determining whether this byte is matched value - signal is_value_match[DATA_BYTES - maxKeyLen]; - // final mask - signal mask[DATA_BYTES - maxKeyLen]; - - - // signal parsing_object_value[DATA_BYTES - maxKeyLen]; - signal is_key_match[DATA_BYTES - maxKeyLen]; - signal is_key_match_for_value[DATA_BYTES + 1 - maxKeyLen]; - is_key_match_for_value[0] <== 0; - signal is_next_pair_at_depth[DATA_BYTES - maxKeyLen]; - signal or[DATA_BYTES - maxKeyLen]; - - // Signals to detect if we are parsing a key or value with initial setup - signal parsing_key[DATA_BYTES - maxKeyLen]; - signal parsing_value[DATA_BYTES - maxKeyLen]; - // TODO: Can't these just be 0 since the start of object can't be either of these? - // parsing_key[0] <== InsideKey()(stack[0][0], parsingData[0][0], parsingData[0][1]); - // parsing_value[0] <== InsideValueObject()(stack[0][0], stack[0][1], parsingData[0][0], parsingData[0][1]); - - // Initialize values knowing 0th bit of data will never be a key/value - parsing_key[0] <== 0; - parsing_value[0] <== 0; - is_key_match[0] <== 0; - - is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(stack[0], data[0]); - is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]); - is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1]; - - mask[0] <== data[0] * is_value_match[0]; - - for(var data_idx = 1; data_idx < DATA_BYTES - maxKeyLen; data_idx++) { - parsing_key[data_idx] <== InsideKey()(stack[data_idx][0], parsingData[data_idx][0], parsingData[data_idx][1]); - parsing_value[data_idx] <== InsideValueObject()(stack[data_idx][0], stack[data_idx][1], parsingData[data_idx][0], parsingData[data_idx][1]); - - // to get correct value, check: - // - key matches at current index and depth of key is as specified - // - whether next KV pair starts - // - whether key matched for a value (propogate key match until new KV pair of lower depth starts) - is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, maxKeyLen, data_idx)(data, key, keyLen, parsing_key[data_idx]); - is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(stack[data_idx], data[data_idx]); - is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]); - is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx]; - - or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx - 1]); - - // mask = currently parsing value and all subsequent keys matched - mask[data_idx] <== data[data_idx] * or[data_idx]; - - } - - // Write the `step_out` with masked data - signal output step_out[TOTAL_BYTES]; - for (var i = 0 ; i < DATA_BYTES - maxKeyLen ; i++) { - step_out[i] <== mask[i]; - } - for (var i = 0 ; i < maxKeyLen ; i++) { - step_out[DATA_BYTES - maxKeyLen + i] <== 0; - } - // Append the parser state back on `step_out` - for (var i = DATA_BYTES ; i < TOTAL_BYTES ; i++) { - step_out[i] <== step_in[i]; - } - // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED -} - -template JsonMaskArrayIndexNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT) { - // ------------------------------------------------------------------------------------------------------------------ // - // ~~ Set sizes at compile time ~~ - // Total number of variables in the parser for each byte of data - assert(MAX_STACK_HEIGHT >= 2); - var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2; - var TOTAL_BYTES_USED = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1); - // ------------------------------------------------------------------------------------------------------------------ // - - // ------------------------------------------------------------------------------------------------------------------ // - // ~ Unravel from previous NIVC step ~ - // Read in from previous NIVC step (JsonParseNIVC) - signal input step_in[TOTAL_BYTES]; - - // Grab the raw data bytes from the `step_in` variable - signal data[DATA_BYTES]; - for (var i = 0 ; i < DATA_BYTES ; i++) { - data[i] <== step_in[i]; - } - - // Decode the encoded data in `step_in` back into parser variables - signal stack[DATA_BYTES][MAX_STACK_HEIGHT][2]; - signal parsingData[DATA_BYTES][2]; - for (var i = 0 ; i < DATA_BYTES ; i++) { - for (var j = 0 ; j < MAX_STACK_HEIGHT ; j++) { - stack[i][j][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2]; - stack[i][j][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + j * 2 + 1]; - } - parsingData[i][0] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2]; - parsingData[i][1] <== step_in[DATA_BYTES + i * PER_ITERATION_DATA_LENGTH + MAX_STACK_HEIGHT * 2 + 1]; - } - // ------------------------------------------------------------------------------------------------------------------ // - - // ------------------------------------------------------------------------------------------------------------------ // - // ~ Array index masking ~ - signal input index; - - // value starting index in `data` - signal value_starting_index[DATA_BYTES]; - signal mask[DATA_BYTES]; - - signal parsing_array[DATA_BYTES]; - signal or[DATA_BYTES]; - - parsing_array[0] <== InsideArrayIndexObject()(stack[0][0], stack[0][1], parsingData[0][0], parsingData[0][1], index); - mask[0] <== data[0] * parsing_array[0]; - - for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) { - parsing_array[data_idx] <== InsideArrayIndexObject()(stack[data_idx][0], stack[data_idx][1], parsingData[data_idx][0], parsingData[data_idx][1], index); - - or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]); - mask[data_idx] <== data[data_idx] * or[data_idx]; - } - - // Write the `step_out` with masked data - signal output step_out[TOTAL_BYTES]; - for (var i = 0 ; i < DATA_BYTES ; i++) { - step_out[i] <== mask[i]; - } - // Append the parser state back on `step_out` - for (var i = DATA_BYTES ; i < TOTAL_BYTES ; i++) { - step_out[i] <== step_in[i]; - } - // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED -} \ No newline at end of file From f3f69c1ee8b830ab5c7cd8d634569c52c455a99f Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Oct 2024 17:50:12 -0600 Subject: [PATCH 5/6] fix circuits again --- circuits/json_mask_array_index.circom | 2 ++ circuits/json_mask_object.circom | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/circuits/json_mask_array_index.circom b/circuits/json_mask_array_index.circom index ba53e08..5cf36c7 100644 --- a/circuits/json_mask_array_index.circom +++ b/circuits/json_mask_array_index.circom @@ -1,5 +1,7 @@ pragma circom 2.1.9; +include "parser-attestor/circuits/json/interpreter.circom"; + template JsonMaskArrayIndexNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT) { // ------------------------------------------------------------------------------------------------------------------ // // ~~ Set sizes at compile time ~~ diff --git a/circuits/json_mask_object.circom b/circuits/json_mask_object.circom index 3cd1e63..89b7135 100644 --- a/circuits/json_mask_object.circom +++ b/circuits/json_mask_object.circom @@ -107,4 +107,4 @@ template JsonMaskObjectNIVC(TOTAL_BYTES, DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen // No need to pad as this is currently when TOTAL_BYTES == TOTAL_BYTES_USED } -component main { public [step_in] } = JsonMaskArrayIndexNIVC(4160, 320, 5); +component main { public [step_in] } = JsonMaskObjectNIVC(4160, 320, 5, 10); From 6be1efe0461c874eedfa775efe2bac9c0a958d7b Mon Sep 17 00:00:00 2001 From: Colin Roberts Date: Wed, 23 Oct 2024 18:01:14 -0600 Subject: [PATCH 6/6] Update artifacts.yaml --- .github/workflows/artifacts.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/artifacts.yaml b/.github/workflows/artifacts.yaml index e8f6676..9423987 100644 --- a/.github/workflows/artifacts.yaml +++ b/.github/workflows/artifacts.yaml @@ -72,6 +72,16 @@ jobs: zip -r ../circom-artifacts.zip ./* cd .. + # Upload artifacts for PR + - name: Upload artifacts for PR + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: circom-artifacts + path: artifacts/ + retention-days: 5 + + # Create Release only on push to main - name: Create Release if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: softprops/action-gh-release@v1