Skip to content

Commit

Permalink
add extra key index to match end quote
Browse files Browse the repository at this point in the history
  • Loading branch information
@lonerapier @Autoparallel
lonerapier authored and Autoparallel committed Oct 20, 2024
1 parent b40464d commit 93a6e1d
Show file tree
Hide file tree
Showing 4 changed files with 2,675 additions and 20 deletions.
14 changes: 11 additions & 3 deletions circuits/json/interpreter.circom
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,7 @@ template MatchPaddedKey(n) {
}

signal isEndOfKeyEqualToQuote <== IsEqual()([endOfKeyAccum[n], 1]);
// log("isEndOfKeyEqualToQuote", isEndOfKeyEqualToQuote);

component totalEqual = IsEqual();
totalEqual.in[0] <== n;
Expand Down Expand Up @@ -448,19 +449,26 @@ template KeyMatchAtIndex(dataLen, maxKeyLen, index) {
signal input keyLen;
signal input parsing_key;

signal paddedKey[maxKeyLen + 1];
for (var i = 0 ; i < maxKeyLen ; i++) {
paddedKey[i] <== key[i];
}
paddedKey[maxKeyLen] <== 0;
// `"` -> 34

// start of key equal to quote
signal startOfKeyEqualToQuote <== IsEqual()([data[index - 1], 34]);
signal isParsingCorrectKey <== parsing_key * startOfKeyEqualToQuote;

// key matches
component isSubstringMatch = MatchPaddedKey(maxKeyLen);
isSubstringMatch.in[0] <== key;
component isSubstringMatch = MatchPaddedKey(maxKeyLen+1);
isSubstringMatch.in[0] <== paddedKey;
isSubstringMatch.keyLen <== keyLen;
for(var matcher_idx = 0; matcher_idx < maxKeyLen; matcher_idx++) {
for(var matcher_idx = 0; matcher_idx <= maxKeyLen; matcher_idx++) {
// log("matcher_idx", index, matcher_idx, data[index + matcher_idx]);
isSubstringMatch.in[1][matcher_idx] <== data[index + matcher_idx];
}
// log("keyMatchAtIndex", isParsingCorrectKey, isSubstringMatch.out);

signal output out <== isSubstringMatch.out * isParsingCorrectKey;
}
21 changes: 14 additions & 7 deletions circuits/json/nivc/masker.circom
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,23 @@ template JsonMaskObjectNIVC(DATA_BYTES, MAX_STACK_HEIGHT, MAX_KEY_LENGTH) {
// Total number of variables in the parser for each byte of data
assert(MAX_STACK_HEIGHT >= 2);
var PER_ITERATION_DATA_LENGTH = MAX_STACK_HEIGHT * 2 + 2;
var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1;
var TOTAL_BYTES_ACROSS_NIVC = DATA_BYTES * (PER_ITERATION_DATA_LENGTH + 1) + 1;
// ------------------------------------------------------------------------------------------------------------------ //

// ------------------------------------------------------------------------------------------------------------------ //
// ~ Unravel from previous NIVC step ~
// Read in from previous NIVC step (JsonParseNIVC)
signal input step_in[TOTAL_BYTES_ACROSS_NIVC];
signal input step_in[TOTAL_BYTES_ACROSS_NIVC];

// Grab the raw data bytes from the `step_in` variable
signal data[DATA_BYTES];
var paddedDataLen = DATA_BYTES + MAX_KEY_LENGTH + 1;
signal data[paddedDataLen];
for (var i = 0 ; i < DATA_BYTES ; i++) {
data[i] <== step_in[i];
}
for (var i = 0 ; i <= MAX_KEY_LENGTH ; i++) {
data[DATA_BYTES + i] <== 0;
}

// Decode the encoded data in `step_in` back into parser variables
signal stack[DATA_BYTES][MAX_STACK_HEIGHT][2];
Expand All @@ -39,7 +43,7 @@ template JsonMaskObjectNIVC(DATA_BYTES, MAX_STACK_HEIGHT, MAX_KEY_LENGTH) {
// ~ Object masking ~
// Key data to use to point to which object to extract
signal input key[MAX_KEY_LENGTH];
signal input keyLen;
signal input keyLen;

// Signals to detect if we are parsing a key or value with initial setup
signal parsing_key[DATA_BYTES - MAX_KEY_LENGTH];
Expand All @@ -56,7 +60,7 @@ template JsonMaskObjectNIVC(DATA_BYTES, MAX_STACK_HEIGHT, MAX_KEY_LENGTH) {
// Initialize values knowing 0th bit of data will never be a key/value
parsing_key[0] <== 0;
parsing_value[0] <== 0;
is_key_match[0] <== 0;
is_key_match[0] <== 0;

component stackSelector[DATA_BYTES];
stackSelector[0] = ArraySelector(MAX_STACK_HEIGHT, 2);
Expand All @@ -76,6 +80,7 @@ template JsonMaskObjectNIVC(DATA_BYTES, MAX_STACK_HEIGHT, MAX_KEY_LENGTH) {
stackSelector[data_idx].in <== stack[data_idx];
stackSelector[data_idx].index <== step_in[TOTAL_BYTES_ACROSS_NIVC - 1];

log("step_in[", data_idx, "] =", step_in[data_idx]);
log("stackSelector[", data_idx, "].out[0] = ", stackSelector[data_idx].out[0]);
log("stackSelector[", data_idx, "].out[1] = ", stackSelector[data_idx].out[1]);

Expand All @@ -90,17 +95,19 @@ template JsonMaskObjectNIVC(DATA_BYTES, MAX_STACK_HEIGHT, MAX_KEY_LENGTH) {
// - key matches at current index and depth of key is as specified
// - whether next KV pair starts
// - whether key matched for a value (propogate key match until new KV pair of lower depth starts)
is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, MAX_KEY_LENGTH, data_idx)(data, key, keyLen, parsing_key[data_idx]);
is_key_match[data_idx] <== KeyMatchAtIndex(paddedDataLen, MAX_KEY_LENGTH, data_idx)(data, key, keyLen, parsing_key[data_idx]);
is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT)(stack[data_idx], data[data_idx], step_in[TOTAL_BYTES_ACROSS_NIVC - 1]);

log("is_key_match[", data_idx, "] = ", is_key_match[data_idx]);
log("is_next_pair_at_depth[", data_idx, "] = ", is_next_pair_at_depth[data_idx]);

is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]);
is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx];

// Set the next NIVC step to only have the masked data
log("is_value_match", is_value_match[data_idx]);
step_out[data_idx] <== data[data_idx] * is_value_match[data_idx];
log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
}
for (var i = 0 ; i < MAX_KEY_LENGTH ; i++) {
step_out[DATA_BYTES - MAX_KEY_LENGTH + i] <== 0;
Expand Down
29 changes: 19 additions & 10 deletions circuits/test/json/nivc/masker_nivc.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import { join } from "path";
// Notes:
// - "data"'s object appears at byte 14
// - colon after "items" appears at byte 31
// - 0th index of arr appears at byte 47
// - 0th index of arr appears at byte 47
// - byte 64 is `"` for the data inside the array obj
// - byte 81 is where `Artist",` ends
// - byte 100 is where `"profile"` starts
Expand All @@ -46,6 +46,7 @@ let nivc_parse = readJsonFile<NIVCData>(join(__dirname, "..", "nivc/nivc_parse.j
let nivc_extract_key0 = readJsonFile<NIVCData>(join(__dirname, "..", "nivc/nivc_extract_key0.json"));
let nivc_extract_key1 = readJsonFile<NIVCData>(join(__dirname, "..", "nivc/nivc_extract_key1.json"));
let nivc_extract_arr = readJsonFile<NIVCData>(join(__dirname, "..", "nivc/nivc_extract_arr.json"));
let nivc_extract_key3 = readJsonFile<NIVCData>(join(__dirname, "..", "nivc/nivc_extract_key3.json"));

describe("JsonParseNIVC", async () => {
let circuit: WitnessTester<["step_in"], ["step_out"]>;
Expand Down Expand Up @@ -77,7 +78,7 @@ describe("JsonParseNIVC", async () => {

let extended_json_input = json_input.concat(Array(Math.max(0, TOTAL_BYTES_ACROSS_NIVC - json_input.length)).fill(0));

await generatePassCase({ step_in: extended_json_input }, { step_out: nivc_parse.step_out }, "parsing JSON");
generatePassCase({ step_in: extended_json_input }, { step_out: nivc_parse.step_out }, "parsing JSON");

});

Expand All @@ -87,6 +88,7 @@ describe("JsonMaskObjectNIVC", async () => {
let DATA_BYTES = 202;
let MAX_STACK_HEIGHT = 5;
let MAX_KEY_LENGTH = 7;
let step_out: bigint[] = [];

before(async () => {
circuit = await circomkit.WitnessTester(`JsonMaskObjectNIVC`, {
Expand All @@ -100,24 +102,31 @@ describe("JsonMaskObjectNIVC", async () => {
function generatePassCase(input: any, expected: any, desc: string) {
const description = generateDescription(input);

it(`(valid) witness: ${description} ${desc}`, async () => {
it(`(valid) witness: ${desc}`, async () => {
// console.log(JSON.stringify(await circuit.compute(input, ["step_out"])))
await circuit.expectPass(input, expected);
let wit = await circuit.calculateWitness(input);
console.log("wit", wit.slice(0, 100));
// step_out = wit;
// await circuit.expectPass(input, expected);
});
}

// let key0 = [100, 97, 116, 97, 0, 0, 0]; // "data"
// let key0Len = 4;
// generatePassCase({ step_in: nivc_parse.step_out, key: key0, keyLen: key0Len }, { step_out: nivc_extract_key0.step_out }, "masking json object at depth 0");

// let key1 = [105, 116, 101, 109, 115, 0, 0]; // "items"
// let key1Len = 5;
// generatePassCase({ step_in: nivc_extract_key0.step_out, key: key1, keyLen: key1Len }, { step_out: nivc_extract_key1.step_out }, "masking json object at depth 0");
let key1 = [105, 116, 101, 109, 115, 0, 0]; // "items"
let key1Len = 5;
generatePassCase({ step_in: nivc_extract_key0.step_out, key: key1, keyLen: key1Len }, { step_out: nivc_extract_key1.step_out }, "masking json object at depth 0");

// Ran after doing arr masking
let key2 = [112, 114, 111, 102, 105, 108, 101]; // "profile"
let key2Len = 7;
generatePassCase({ step_in: nivc_extract_arr.step_out, key: key2, keyLen: key2Len }, { step_out: nivc_extract_key1.step_out }, "masking json object at depth 0");
// let key2 = [112, 114, 111, 102, 105, 108, 101]; // "profile"
// let key2Len = 7;
// generatePassCase({ step_in: nivc_extract_arr.step_out, key: key2, keyLen: key2Len }, { step_out: nivc_extract_key1.step_out }, "masking json object at depth 0");

// let key3 = [110, 97, 109, 101, 0, 0, 0]; // "name"
// let key3Len = 4;
// generatePassCase({ step_in: nivc_extract_key3.step_out, key: key3, keyLen: key3Len }, {}, "masking json at depth 4");
});

describe("JsonMaskArrayIndexNIVC", async () => {
Expand Down
Loading

0 comments on commit 93a6e1d

Please sign in to comment.