Skip to content

Commit

Permalink
extract nested array working
Browse files Browse the repository at this point in the history
  • Loading branch information
@lonerapier
lonerapier committed Aug 21, 2024
1 parent 8875441 commit 62a4dfd
Show file tree
Hide file tree
Showing 2 changed files with 175 additions and 21 deletions.
145 changes: 141 additions & 4 deletions circuits/fetcher.circom
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,26 @@ template InsideArrayIndex(n, index) {
out <== inside_array_index * (parsing_string + parsing_number);
}

template InsideArrayIndexAtDepth(n, index, depth) {
signal input stack[n][2];
signal input parsing_string;
signal input parsing_number;

signal output out;

// component topOfStack = GetTopOfStack(n);
// topOfStack.stack <== stack;
// signal current_val[2] <== topOfStack.value;
// signal pointer <== topOfStack.pointer;

signal inside_array <== IsEqual()([stack[depth][0], 2]);
signal inside_index <== IsEqual()([stack[depth][1], index]);
signal inside_array_index <== inside_array * inside_index;
// signal at_depth <== IsEqual()([pointer, depth]);
// signal inside_array_index_at_depth <== inside_array_index * at_depth;
out <== inside_array_index * (parsing_string + parsing_number);
}

template NextKVPair(n) {
signal input stack[n][2];
signal input curr_byte;
Expand Down Expand Up @@ -299,7 +319,7 @@ template ExtractNumber(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, maxValueLen) {
}

template ExtractArray(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, index, maxValueLen) {
signal input data[DATA_BYTES];
signal input data[DATA_BYTES];
signal input key[keyLen];

signal value_starting_index[DATA_BYTES];
Expand Down Expand Up @@ -482,7 +502,7 @@ template ExtractMultiDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen1, depth1, keyLen
// eg: `{ "a": { "d" : "e", "e": "c" }, "e": { "f": "a", "e": "2" } }`
is_next_pair_at_depth1[data_idx-1] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth1)(State[data_idx].stack, data[data_idx-1]);
is_next_pair_at_depth2[data_idx-1] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, depth2)(State[data_idx].stack, data[data_idx-1]);
log("is_new_kv_pair:", is_next_pair_at_depth1[data_idx-1], is_next_pair_at_depth2[data_idx-1]);
// log("is_new_kv_pair:", is_next_pair_at_depth1[data_idx-1], is_next_pair_at_depth2[data_idx-1]);

// is the value getting parsed has a matched key?
// use mux1 to carry parse_key forward to value
Expand All @@ -493,10 +513,10 @@ template ExtractMultiDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen1, depth1, keyLen
// all the keys should match for the correct value
is_key1_match_for_value[data_idx] <== Mux1()([is_key1_match_for_value[data_idx-1] * (1-is_next_pair_at_depth1[data_idx-1]), is_key1_match[data_idx-1] * (1-is_next_pair_at_depth1[data_idx-1])], is_key1_match[data_idx-1]);
is_key2_match_for_value[data_idx] <== Mux1()([is_key2_match_for_value[data_idx-1] * (1-is_next_pair_at_depth2[data_idx-1]), is_key2_match[data_idx-1] * (1-is_next_pair_at_depth2[data_idx-1])], is_key2_match[data_idx-1]);
log("is_key_match_for_value:", is_key1_match_for_value[data_idx], is_key2_match_for_value[data_idx]);
// log("is_key_match_for_value:", is_key1_match_for_value[data_idx], is_key2_match_for_value[data_idx]);

is_value_match[data_idx] <== is_key1_match_for_value[data_idx] * is_key2_match_for_value[data_idx];
log("is_value_match", is_value_match[data_idx]);
// log("is_value_match", is_value_match[data_idx]);

// mask[i] = data[i] * parsing_value[i] * is_key_match_for_value[i]
value_mask[data_idx-1] <== data[data_idx-1] * parsing_value[data_idx-1];
Expand Down Expand Up @@ -541,4 +561,121 @@ template ExtractStringMultiDepth(DATA_BYTES, MAX_STACK_HEIGHT, keyLen1, depth1,
for (var i=0 ; i<maxValueLen; i++) {
log("value[",i,"]=", value[i]);
}
}

template ExtractNestedArray(DATA_BYTES, MAX_STACK_HEIGHT, keyLen, index1, depth1, index2, depth2, maxValueLen) {
signal input data[DATA_BYTES];
signal input key[keyLen];

signal value_starting_index[DATA_BYTES];
signal output value[maxValueLen];


signal mask[DATA_BYTES];
// mask[0] <== 0;

var logDataLen = log2Ceil(DATA_BYTES);

component State[DATA_BYTES];
State[0] = StateUpdate(MAX_STACK_HEIGHT);
State[0].byte <== data[0];
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
State[0].stack[i] <== [0,0];
}
State[0].parsing_string <== 0;
State[0].parsing_number <== 0;

signal parsing_key[DATA_BYTES];
signal parsing_array1[DATA_BYTES];
signal parsing_array2[DATA_BYTES];
signal parsing_value[DATA_BYTES];
signal is_key_match[DATA_BYTES];
signal is_key_match_and_inside_key[DATA_BYTES];
signal is_key_match_for_value[DATA_BYTES];
is_key_match_for_value[0] <== 0; // TODO: this might not be correct way to initialise
signal value_mask[DATA_BYTES];
signal is_next_pair[DATA_BYTES];
for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) {
// Debugging
for(var i = 0; i<MAX_STACK_HEIGHT; i++) {
log("State[", data_idx-1, "].stack[", i,"] ", "= [",State[data_idx-1].next_stack[i][0], "][", State[data_idx-1].next_stack[i][1],"]" );
}
log("State[", data_idx-1, "].byte", "= ", data[data_idx-1]);
log("State[", data_idx-1, "].parsing_string", "= ", State[data_idx-1].next_parsing_string);
log("State[", data_idx-1, "].parsing_number", "= ", State[data_idx-1].next_parsing_number);

State[data_idx] = StateUpdate(MAX_STACK_HEIGHT);
State[data_idx].byte <== data[data_idx];
State[data_idx].stack <== State[data_idx - 1].next_stack;
State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string;
State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number;

parsing_key[data_idx-1] <== InsideKey(MAX_STACK_HEIGHT)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number);
// log("parsing key:", parsing_key[data_idx]);

parsing_array1[data_idx-1] <== InsideArrayIndexAtDepth(MAX_STACK_HEIGHT, index1, depth1)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number);
parsing_array2[data_idx-1] <== InsideArrayIndexAtDepth(MAX_STACK_HEIGHT, index2, depth2)(State[data_idx].stack, State[data_idx].parsing_string, State[data_idx].parsing_number);
parsing_value[data_idx-1] <== parsing_array1[data_idx-1] * parsing_array2[data_idx-1];
log("parsing value:", parsing_value[data_idx-1]);

is_key_match[data_idx-1] <== KeyMatch(DATA_BYTES, keyLen)(data, key, 100, data_idx-1, parsing_key[data_idx-1]);
// log("is_key_match", is_key_match[data_idx]);

// is the value getting parsed has a matched key?
// use mux1 to carry parse_key forward to value
// is_key_match_for_value should reset when moving to next kv pair
// `is_key_match = 0` -> 0
// `is_key_match = 1` -> 1 until new kv pair
// `new kv pair = 1` -> 0
is_next_pair[data_idx-1] <== NextKVPair(MAX_STACK_HEIGHT)(State[data_idx].stack, data[data_idx-1]);
// log("is_new_kv_pair:", is_next_pair[data_idx]);

is_key_match_for_value[data_idx] <== Mux1()([is_key_match_for_value[data_idx-1] * (1-is_next_pair[data_idx-1]), is_key_match[data_idx-1] * (1-is_next_pair[data_idx-1])], is_key_match[data_idx-1]);
// log("is_key_match_for_value:", is_key_match_for_value[data_idx]);

// mask[i] = data[i] * parsing_value[i] * is_key_match_for_value[i]
value_mask[data_idx-1] <== data[data_idx-1] * parsing_value[data_idx-1];
mask[data_idx-1] <== value_mask[data_idx-1] * is_key_match_for_value[data_idx];
log("mask", mask[data_idx-1]);
log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
}

// Debugging
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
log("State[", DATA_BYTES-1, "].stack[", i,"] ", "= [",State[DATA_BYTES -1].next_stack[i][0], "][", State[DATA_BYTES - 1].next_stack[i][1],"]" );
}
log("State[", DATA_BYTES-1, "].parsing_string", "= ", State[DATA_BYTES-1].next_parsing_string);
log("State[", DATA_BYTES-1, "].parsing_number", "= ", State[DATA_BYTES-1].next_parsing_number);
log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");

// signal value_starting_index[DATA_BYTES];
signal is_zero_mask[DATA_BYTES];
signal is_prev_starting_index[DATA_BYTES];
value_starting_index[0] <== 0;
is_zero_mask[0] <== IsZero()(mask[0]);
for (var i=1 ; i<DATA_BYTES-1 ; i++) {
is_zero_mask[i] <== IsZero()(mask[i]);
is_prev_starting_index[i] <== IsZero()(value_starting_index[i-1]);
value_starting_index[i] <== value_starting_index[i-1] + i * (1-is_zero_mask[i]) * is_prev_starting_index[i];
}

signal value_string[maxValueLen];

log("value_starting_index:", value_starting_index[DATA_BYTES-2]);

value_string <== SelectSubArray(DATA_BYTES, maxValueLen)(data, value_starting_index[DATA_BYTES-2], maxValueLen);

for (var i=0 ; i<maxValueLen; i++) {
log("value[",i,"]=", value_string[i]);
}

value <== value_string;

// signal number_value[maxValueLen];
// number_value[0] <== (value_string[0]-48);
// for (var i=1 ; i<maxValueLen ; i++) {
// number_value[i] <== number_value[i-1] * 10 + (value_string[i]-48);
// }

// value <== number_value[maxValueLen-1];
}
51 changes: 34 additions & 17 deletions circuits/test/fetcher.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { circomkit, WitnessTester, generateDescription } from "./common";
import { readFileSync } from "fs";
import { join } from "path";

function readInputFile(filename: string, key: string[], index: number | undefined): [number[], number[][], number[]] {
function readInputFile(filename: string, key: any[]): [number[], number[][], number[]] {
const value_string_path = join(__dirname, "..", "..", "json_examples", "test", filename);

let input: number[] = [];
Expand All @@ -13,9 +13,10 @@ function readInputFile(filename: string, key: string[], index: number | undefine
let keyUnicode: number[][] = [];
for (let i = 0; i < key.length; i++) {
keyUnicode[i] = [];
for (let j = 0; j < key[i].length; j++) {
let key_string = key.toString();
for (let j = 0; j < key_string[i].length; j++) {

keyUnicode[i].push(key[i].charCodeAt(j));
keyUnicode[i].push(key_string[i].charCodeAt(j));
}
}

Expand All @@ -26,13 +27,7 @@ function readInputFile(filename: string, key: string[], index: number | undefine
input = byteArray;

let jsonFile = JSON.parse(data);
let jsonValue: string = key.reduce((acc, key) => acc && acc[key], jsonFile);
let value: string = "";
if (typeof jsonValue === "number" || typeof jsonValue === "string") {
value = jsonValue.toString();
} else if (typeof jsonValue === "object") {
value = jsonValue[index as number];
}
let value: string = key.reduce((acc, key) => acc && acc[key], jsonFile).toString();
for (let i = 0; i < value.length; i++) {
output.push(value.charCodeAt(i));
}
Expand All @@ -44,7 +39,7 @@ describe("ExtractValue", () => {
let circuit: WitnessTester<["data", "key"], ["value"]>;

it("value_string: {\"a\": \"b\"}", async () => {
let [input, keyUnicode, output] = readInputFile("value_string.json", ["k"], undefined);
let [input, keyUnicode, output] = readInputFile("value_string.json", ["k"]);
circuit = await circomkit.WitnessTester(`Extract`, {
file: "circuits/fetcher",
template: "ExtractString",
Expand All @@ -60,7 +55,7 @@ describe("ExtractValue", () => {
});

it("two_keys: {\"key1\": \"abc\", \"key2\": \"def\" }", async () => {
let [input1, keyUnicode1, output1] = readInputFile("two_keys.json", ["key1"], undefined);
let [input1, keyUnicode1, output1] = readInputFile("two_keys.json", ["key1"]);

circuit = await circomkit.WitnessTester(`Extract`, {
file: "circuits/fetcher",
Expand All @@ -71,12 +66,12 @@ describe("ExtractValue", () => {

await circuit.expectPass({ data: input1, key: keyUnicode1 }, { value: output1 });

let [input2, keyUnicode2, output2] = readInputFile("two_keys.json", ["key2"], undefined);
let [input2, keyUnicode2, output2] = readInputFile("two_keys.json", ["key2"]);
await circuit.expectPass({ data: input2, key: keyUnicode2 }, { value: output2 });
});

it("value_number: {\"k\": 69 }", async () => {
let [input1, keyUnicode1, output1] = readInputFile("value_number.json", ["k"], undefined);
let [input1, keyUnicode1, output1] = readInputFile("value_number.json", ["k"]);
circuit = await circomkit.WitnessTester(`Extract`, {
file: "circuits/fetcher",
template: "ExtractNumber",
Expand All @@ -91,7 +86,7 @@ describe("ExtractValue", () => {

it("value_array: { \"k\" : [ 420 , 69 , 4200 , 600 ], \"b\": [ \"ab\" , \"ba\", \"ccc\", \"d\" ] }", async () => {
for (let i = 0; i < 4; i++) {
let [input, keyUnicode, output] = readInputFile("value_array.json", ["b"], i);
let [input, keyUnicode, output] = readInputFile("value_array.json", ["b", i]);

circuit = await circomkit.WitnessTester(`Extract`, {
file: "circuits/fetcher",
Expand All @@ -109,7 +104,7 @@ describe("ExtractValueMultiDepth", () => {
let circuit: WitnessTester<["data", "key1", "key2"], ["value"]>;

it("value_object: { \"a\": { \"d\" : \"e\", \"e\": \"c\" }, \"e\": { \"f\": \"a\", \"e\": \"2\" } }", async () => {
let [input, keyUnicode, output] = readInputFile("value_object.json", ["e", "e"], undefined);
let [input, keyUnicode, output] = readInputFile("value_object.json", ["e", "e"]);

circuit = await circomkit.WitnessTester(`Extract`, {
file: "circuits/fetcher",
Expand All @@ -120,7 +115,29 @@ describe("ExtractValueMultiDepth", () => {

await circuit.expectPass({ data: input, key1: keyUnicode[0], key2: keyUnicode[1] }, { value: output });

let [input1, keyUnicode1, output1] = readInputFile("value_object.json", ["e", "f"], undefined);
let [input1, keyUnicode1, output1] = readInputFile("value_object.json", ["e", "f"]);
await circuit.expectPass({ data: input1, key1: keyUnicode1[0], key2: keyUnicode1[1] }, { value: output1 });
});
});

describe("ExtractValueMultiDepth", () => {
let circuit: WitnessTester<["data", "key"], ["value"]>;

it("value_array_nested: { \"a\": [[1,0],[0,1,3]] }", async () => {
let index_0 = 1;
let index_1 = 0;
let [input, keyUnicode, output] = readInputFile("value_array_nested.json", ["a", index_0, index_1]);
console.log(input, keyUnicode, output);
circuit = await circomkit.WitnessTester(`Extract`, {
file: "circuits/fetcher",
template: "ExtractNestedArray",
params: [input.length, 4, 1, index_0, 1, index_1, 2, 1],
});
console.log("#constraints:", await circuit.getConstraintCount());


// "b".0.0."d"
console.log("input", input, "key:", keyUnicode, "output:", output);
await circuit.expectPass({ data: input, key: keyUnicode[0] }, { value: output });
});
});

0 comments on commit 62a4dfd

Please sign in to comment.