Skip to content

Commit

Permalink
Added jsmn_parse_next feature
Browse files Browse the repository at this point in the history 
With jsmn_parse_next it is possible to parse large streams of JSON
objects allocating and reusing only a small array of tokens.

Developed by Roman Orlov (https://github.com/compmaniak),
see zserge#148
  • Loading branch information
@pegro
pegro committed Jan 23, 2019
1 parent 2fa360d commit d541063
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 6 deletions.
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ simple_example: example/simple.c jsmn.h
jsondump: example/jsondump.c jsmn.h
$(CC) $(LDFLAGS) $< -o $@

json_sequence: example/json_sequence.o jsmn.h
$(CC) $(LDFLAGS) $< -o $@

fmt:
clang-format -i jsmn.h test/*.[ch] example/*.[ch]

Expand All @@ -31,6 +34,7 @@ clean:
rm -f *.o example/*.o
rm -f simple_example
rm -f jsondump
rm -f json_sequence

.PHONY: clean test

34 changes: 34 additions & 0 deletions example/json_sequence.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../jsmn.h"

static const char *JSON_STRING =
"{\"@timestamp\":\"2018-11-25T18:45:00\", \"programname\":\"my_prog\", \"procid\":\"123\", \"severity\":\"info\", \"message\":\"Started\"}\n"
"{\"@timestamp\":\"2018-11-25T18:45:01\", \"programname\":\"my_prog\", \"procid\":\"123\", \"severity\":\"warn\", \"message\":\"File is too large\"}\n"
"{\"@timestamp\":\"2018-11-25T18:45:03\", \"programname\":\"oom_killer\", \"procid\":\"42\", \"severity\":\"info\", \"message\":\"Process 123 (my_prog) was killed\"}";

int main() {
int r;
jsmn_parser p;
jsmntok_t t[11]; /* We expect no more than 11 tokens in one JSON object */

jsmn_init(&p);
size_t len = strlen(JSON_STRING);

r = jsmn_parse_next(&p, JSON_STRING, len, t, 11);
while (r > 0) {
printf("%.*s %.*s[%.*s]: <%.*s> %.*s\n",
t[2].end - t[2].start, JSON_STRING + t[2].start,
t[4].end - t[4].start, JSON_STRING + t[4].start,
t[6].end - t[6].start, JSON_STRING + t[6].start,
t[8].end - t[8].start, JSON_STRING + t[8].start,
t[10].end - t[10].start, JSON_STRING + t[10].start);
r = jsmn_parse_next(&p, JSON_STRING, len, t, 11);
}
if (r < 0) {
printf("Failed to parse JSON: %d\n", r);
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
46 changes: 40 additions & 6 deletions jsmn.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,13 @@ JSMN_API void jsmn_init(jsmn_parser *parser);
JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
jsmntok_t *tokens, unsigned int num_tokens);

/**
* Run JSON parser. Unlike jsmn_parse it stops after a next complete JSON
* object is parsed.
*/
JSMN_API int jsmn_parse_next(jsmn_parser *parser, const char *js, size_t len,
jsmntok_t *tokens, unsigned int num_tokens);

#ifndef JSMN_HEADER
/**
* Allocates a fresh unused token from the token pool.
Expand Down Expand Up @@ -260,8 +267,9 @@ static int jsmn_parse_string(jsmn_parser *parser, const char *js, size_t len,
/**
* Parse JSON string and fill tokens.
*/
JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
jsmntok_t *tokens, unsigned int num_tokens) {
static int jsmn_parse_full(jsmn_parser *parser, const char *js, size_t len,
jsmntok_t *tokens, unsigned int num_tokens,
int parse_next) {
int r;
int i;
jsmntok_t *token;
Expand Down Expand Up @@ -342,19 +350,27 @@ JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
}
}
#endif
if (parse_next && parser->toksuper == -1)
len = parser->pos;
break;
case '\"':
r = jsmn_parse_string(parser, js, len, tokens, num_tokens);
if (r < 0)
return r;
count++;
if (parser->toksuper != -1 && tokens != NULL)
tokens[parser->toksuper].size++;
if (parser->toksuper != -1) {
if (tokens != NULL)
tokens[parser->toksuper].size++;
} else if (parse_next) {
len = parser->pos;
}
break;
case '\t':
case '\r':
case '\n':
case ' ':
if (parse_next && parser->toksuper == -1 && count > 0)
len = parser->pos;
break;
case ':':
parser->toksuper = parser->toknext - 1;
Expand Down Expand Up @@ -409,8 +425,12 @@ JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
if (r < 0)
return r;
count++;
if (parser->toksuper != -1 && tokens != NULL)
tokens[parser->toksuper].size++;
if (parser->toksuper != -1) {
if (tokens != NULL)
tokens[parser->toksuper].size++;
} else if (parse_next) {
len = parser->pos;
}
break;

#ifdef JSMN_STRICT
Expand All @@ -433,6 +453,20 @@ JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
return count;
}

JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
jsmntok_t *tokens, unsigned int num_tokens) {
return jsmn_parse_full(parser, js, len, tokens, num_tokens, 0);
}

JSMN_API int jsmn_parse_next(jsmn_parser *parser, const char *js, size_t len,
jsmntok_t *tokens, unsigned int num_tokens) {
int r;
r = jsmn_parse_full(parser, js, len, tokens, num_tokens, 1);
if (r >= 0)
parser->toknext = 0;
return r;
}

/**
* Creates a new parser based over a given buffer with an array of tokens
* available.
Expand Down
64 changes: 64 additions & 0 deletions test/tests.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,68 @@ int test_unmatched_brackets(void) {
return 0;
}

int test_json_sequence(void) {
const char *js;
js = " null\t\"2\"[\r][ 3 ]\r\n[ {\n}] [ 4, { } ]";
check(parse(js, 10, 10,
JSMN_PRIMITIVE, "null",
JSMN_STRING, "2", 0,
JSMN_ARRAY, 9, 12, 0,
JSMN_ARRAY, 12, 17, 1,
JSMN_PRIMITIVE, "3",
JSMN_ARRAY, 19, 25, 1,
JSMN_OBJECT, 21, 24, 0,
JSMN_ARRAY, 26, 36, 2,
JSMN_PRIMITIVE, "4",
JSMN_OBJECT, 31, 34, 0));
return 0;
}

int test_json_sequence_by_one(void) {
const char *js;
size_t js_len;
int r;
jsmn_parser p;
jsmntok_t tokens[3];

js = " null\t\"2\"[\r][ 3 ]\r\n[ {\n}] [ 4, { } ]";
js_len = strlen(js);

jsmn_init(&p);

r = jsmn_parse_next(&p, js, js_len, tokens, 3);
check(r == 1);
check(tokeq(js, tokens, r,
JSMN_PRIMITIVE, "null"));
r = jsmn_parse_next(&p, js, js_len, tokens, 3);
check(r == 1);
check(tokeq(js, tokens, r,
JSMN_STRING, "2", 0));
r = jsmn_parse_next(&p, js, js_len, tokens, 3);
check(r == 1);
check(tokeq(js, tokens, r,
JSMN_ARRAY, 9, 12, 0));
r = jsmn_parse_next(&p, js, js_len, tokens, 3);
check(r == 2);
check(tokeq(js, tokens, r,
JSMN_ARRAY, 12, 17, 1,
JSMN_PRIMITIVE, "3"));
r = jsmn_parse_next(&p, js, js_len, tokens, 3);
check(r == 2);
check(tokeq(js, tokens, r,
JSMN_ARRAY, 19, 25, 1,
JSMN_OBJECT, 21, 24, 0));
r = jsmn_parse_next(&p, js, js_len, tokens, 3);
check(r == 3);
check(tokeq(js, tokens, r,
JSMN_ARRAY, 26, 36, 2,
JSMN_PRIMITIVE, "4",
JSMN_OBJECT, 31, 34, 0));
r = jsmn_parse_next(&p, js, js_len, tokens, 3);
check(r == 0);
return 0;
}

int main(void) {
test(test_empty, "test for a empty JSON objects/arrays");
test(test_object, "test for a JSON objects");
Expand All @@ -334,6 +396,8 @@ int main(void) {
test(test_count, "test tokens count estimation");
test(test_nonstrict, "test for non-strict mode");
test(test_unmatched_brackets, "test for unmatched brackets");
test(test_json_sequence, "test for many JSONs in one string");
test(test_json_sequence_by_one, "test for many JSONs in one string by one");
printf("\nPASSED: %d\nFAILED: %d\n", test_passed, test_failed);
return (test_failed > 0);
}

0 comments on commit d541063

Please sign in to comment.