Skip to content

Commit

Permalink
feat: general value extractor (#95)
Browse files Browse the repository at this point in the history 
* init example

* feat: add single value extractor

* test: add example

* feat: remove keyLen and add maxKeyLen

* add optimisations from web-prover

* `depth` as signal in `NextKVPairAtDepth`

* fix broken tests
  • Loading branch information
@lonerapier
lonerapier authored Oct 18, 2024
1 parent 7fa287d commit af08a7a
Show file tree
Hide file tree
Showing 14 changed files with 905 additions and 62 deletions.
229 changes: 214 additions & 15 deletions circuits.json
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,216 @@
{
"json-parser": {
"file": "json/parser/parser",
"template": "Parser",
"params": [
157,
13
]
},
"http-parser": {
"file": "http/parser/parser",
"template": "Parser",
"params": [
60
]
}
"value_number_test": {
"file": "main/json_value_number_test",
"template": "ExtractNumValue",
"params": [
12,
1,
1,
0,
2
]
},
"json-parser": {
"file": "json/parser/parser",
"template": "Parser",
"params": [
157,
13
]
},
"value_string_test": {
"file": "main/json_value_string_test",
"template": "ExtractStringValue",
"params": [
12,
1,
1,
0,
1
]
},
"spotify_test": {
"file": "main/json_spotify_test",
"template": "ExtractStringValue",
"params": [
85,
5,
4,
0,
5,
1,
0,
2,
7,
3,
4,
4,
12
]
},
"http-parser": {
"file": "http/parser/parser",
"template": "Parser",
"params": [
60
]
},
"spotify_top_artists_test": {
"file": "main/http_spotify_top_artists_test",
"template": "LockHTTPResponse",
"params": [
203,
85,
8,
3,
2,
12,
31
]
},
"value_array_number_test": {
"file": "main/json_value_array_number_test",
"template": "ExtractNumValue",
"params": [
73,
2,
1,
0,
2,
1,
4
]
},
"value_array_object_test": {
"file": "main/json_value_array_object_test",
"template": "ExtractNumValue",
"params": [
29,
4,
1,
0,
0,
1,
1,
2,
0,
3,
1
]
},
"value_string": {
"file": "main/json_value_string",
"template": "ExtractStringValue",
"params": [
12,
1,
1,
0,
1
]
},
"two_keys_test": {
"file": "main/json_two_keys_test",
"template": "ExtractStringValue",
"params": [
40,
1,
4,
0,
3
]
},
"get_request_test": {
"file": "main/http_get_request_test",
"template": "LockHTTPRequest",
"params": [
60,
3,
4,
8,
6,
16,
4,
9
]
},
"value_array_nested_test": {
"file": "main/json_value_array_nested_test",
"template": "ExtractNumValue",
"params": [
24,
3,
1,
0,
0,
1,
0,
2,
1
]
},
"spotify_top_artists": {
"file": "main/extended_spotify_top_artists",
"template": "HttpJson",
"params": [
203,
85,
8,
3,
2,
12,
31,
5,
4,
0,
5,
1,
0,
2,
7,
3,
4,
4,
12
]
},
"value_object_test": {
"file": "main/json_value_object_test",
"template": "ExtractStringValue",
"params": [
134,
3,
1,
0,
1,
1,
1
]
},
"value_array_string_test": {
"file": "main/json_value_array_string_test",
"template": "ExtractStringValue",
"params": [
73,
2,
1,
0,
1,
1,
2
]
},
"get_response_test": {
"file": "main/http_get_response_test",
"template": "LockHTTPResponse",
"params": [
89,
18,
8,
3,
2,
12,
16
]
}
}
164 changes: 164 additions & 0 deletions circuits/json/extractor.circom
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
pragma circom 2.1.9;

include "interpreter.circom";

template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) {
assert(MAX_STACK_HEIGHT >= 2);

// Declaration of signals.
signal input data[DATA_BYTES];
signal input key[maxKeyLen];
signal input keyLen;

signal output value[maxValueLen];

// Constraints.
signal value_starting_index[DATA_BYTES - maxKeyLen];
// flag determining whether this byte is matched value
signal is_value_match[DATA_BYTES - maxKeyLen];
// final mask
signal mask[DATA_BYTES - maxKeyLen];

component State[DATA_BYTES - maxKeyLen];
State[0] = StateUpdate(MAX_STACK_HEIGHT);
State[0].byte <== data[0];
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
State[0].stack[i] <== [0,0];
}
State[0].parsing_string <== 0;
State[0].parsing_number <== 0;

signal parsing_key[DATA_BYTES - maxKeyLen];
signal parsing_value[DATA_BYTES - maxKeyLen];
signal parsing_object_value[DATA_BYTES - maxKeyLen];
signal is_key_match[DATA_BYTES - maxKeyLen];
signal is_key_match_for_value[DATA_BYTES+1 - maxKeyLen];
is_key_match_for_value[0] <== 0;
signal is_next_pair_at_depth[DATA_BYTES - maxKeyLen];
signal or[DATA_BYTES - maxKeyLen];

// initialise first iteration

// check inside key or value
parsing_key[0] <== InsideKey()(State[0].next_stack[0], State[0].next_parsing_string, State[0].next_parsing_number);
parsing_value[0] <== InsideValueObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number);

is_key_match[0] <== 0;
is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT)(State[0].next_stack, data[0], 0);
is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]);
is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1];

mask[0] <== data[0] * is_value_match[0];

for(var data_idx = 1; data_idx < DATA_BYTES - maxKeyLen; data_idx++) {
State[data_idx] = StateUpdate(MAX_STACK_HEIGHT);
State[data_idx].byte <== data[data_idx];
State[data_idx].stack <== State[data_idx - 1].next_stack;
State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string;
State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number;

// - parsing key
// - parsing value (different for string/numbers and array)
// - key match (key 1, key 2)
// - is next pair
// - is key match for value
// - value_mask
// - mask

// check if inside key or not
parsing_key[data_idx] <== InsideKey()(State[data_idx].next_stack[0], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number);
// check if inside value
parsing_value[data_idx] <== InsideValueObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number);

// to get correct value, check:
// - key matches at current index and depth of key is as specified
// - whether next KV pair starts
// - whether key matched for a value (propogate key match until new KV pair of lower depth starts)
is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, maxKeyLen, data_idx)(data, key, keyLen, parsing_key[data_idx]);
is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT)(State[data_idx].next_stack, data[data_idx], 0);
is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]);
is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx];

or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx - 1]);

// mask = currently parsing value and all subsequent keys matched
mask[data_idx] <== data[data_idx] * or[data_idx];
}

// find starting index of value in data by matching mask
signal is_zero_mask[DATA_BYTES];
signal is_prev_starting_index[DATA_BYTES];
value_starting_index[0] <== 0;
is_prev_starting_index[0] <== 0;
is_zero_mask[0] <== IsZero()(mask[0]);
for (var i=1 ; i<DATA_BYTES - maxKeyLen ; i++) {
is_zero_mask[i] <== IsZero()(mask[i]);
is_prev_starting_index[i] <== IsZero()(value_starting_index[i-1]);
value_starting_index[i] <== value_starting_index[i-1] + i * (1-is_zero_mask[i]) * is_prev_starting_index[i];
}

log("value starting index", value_starting_index[DATA_BYTES-1 - maxKeyLen]);
value <== SelectSubArray(DATA_BYTES - maxKeyLen, maxValueLen)(mask, value_starting_index[DATA_BYTES-1 - maxKeyLen], maxValueLen);
for (var i = 0 ; i < maxValueLen ; i++) {
log(i, value[i]);
}
}

template ArrayIndexExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxValueLen) {
assert(MAX_STACK_HEIGHT >= 2);

signal input data[DATA_BYTES];
signal input index;

signal output value[maxValueLen];

// value starting index in `data`
signal value_starting_index[DATA_BYTES];
// final mask
signal mask[DATA_BYTES];

component State[DATA_BYTES];
State[0] = StateUpdate(MAX_STACK_HEIGHT);
State[0].byte <== data[0];
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
State[0].stack[i] <== [0,0];
}
State[0].parsing_string <== 0;
State[0].parsing_number <== 0;

signal parsing_array[DATA_BYTES];
signal or[DATA_BYTES];

parsing_array[0] <== InsideArrayIndexObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number, index);
mask[0] <== data[0] * parsing_array[0];

for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) {
State[data_idx] = StateUpdate(MAX_STACK_HEIGHT);
State[data_idx].byte <== data[data_idx];
State[data_idx].stack <== State[data_idx - 1].next_stack;
State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string;
State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number;

parsing_array[data_idx] <== InsideArrayIndexObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number, index);

or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]);
mask[data_idx] <== data[data_idx] * or[data_idx];
}

signal is_zero_mask[DATA_BYTES];
signal is_prev_starting_index[DATA_BYTES];
value_starting_index[0] <== 0;
is_prev_starting_index[0] <== 0;
is_zero_mask[0] <== IsZero()(mask[0]);
for (var i=1 ; i<DATA_BYTES ; i++) {
is_zero_mask[i] <== IsZero()(mask[i]);
is_prev_starting_index[i] <== IsZero()(value_starting_index[i-1]);
value_starting_index[i] <== value_starting_index[i-1] + i * (1-is_zero_mask[i]) * is_prev_starting_index[i];
}

log("value starting index", value_starting_index[DATA_BYTES-1]);
value <== SelectSubArray(DATA_BYTES, maxValueLen)(mask, value_starting_index[DATA_BYTES-1], maxValueLen);
for (var i = 0 ; i < maxValueLen ; i++) {
log(i, value[i]);
}
}
Loading

0 comments on commit af08a7a

Please sign in to comment.