2 * Copyright (c) 2009-2011 Petri Lehtinen <petri@digip.org>
4 * Jansson is free software; you can redistribute it and/or modify
5 * it under the terms of the MIT license. See LICENSE for details.
18 #include "jansson_private.h"
19 #include "strbuffer.h"
22 #define STREAM_STATE_OK 0
23 #define STREAM_STATE_EOF -1
24 #define STREAM_STATE_ERROR -2
26 #define TOKEN_INVALID -1
28 #define TOKEN_STRING 256
29 #define TOKEN_INTEGER 257
30 #define TOKEN_REAL 258
31 #define TOKEN_TRUE 259
32 #define TOKEN_FALSE 260
33 #define TOKEN_NULL 261
35 /* Read one byte from stream, convert to unsigned char, then int, and
36 return. return EOF on end of file. This corresponds to the
37 behaviour of fgetc(). */
38 typedef int (*get_func)(void *data);
47 int column, last_column;
53 strbuffer_t saved_text;
62 #define stream_to_lex(stream) container_of(stream, lex_t, stream)
65 /*** error reporting ***/
67 static void error_set(json_error_t *error, const lex_t *lex,
71 char msg_text[JSON_ERROR_TEXT_LENGTH];
72 char msg_with_context[JSON_ERROR_TEXT_LENGTH];
74 int line = -1, col = -1;
76 const char *result = msg_text;
82 vsnprintf(msg_text, JSON_ERROR_TEXT_LENGTH, msg, ap);
87 const char *saved_text = strbuffer_value(&lex->saved_text);
89 line = lex->stream.line;
90 col = lex->stream.column;
91 pos = lex->stream.position;
93 if(saved_text && saved_text[0])
95 if(lex->saved_text.length <= 20) {
96 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
97 "%s near '%s'", msg_text, saved_text);
98 result = msg_with_context;
103 if(lex->stream.state == STREAM_STATE_ERROR) {
104 /* No context for UTF-8 decoding errors */
108 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
109 "%s near end of file", msg_text);
110 result = msg_with_context;
115 jsonp_error_set(error, line, col, pos, "%s", result);
119 /*** lexical analyzer ***/
122 stream_init(stream_t *stream, get_func get, void *data)
126 stream->buffer[0] = '\0';
127 stream->buffer_pos = 0;
129 stream->state = STREAM_STATE_OK;
132 stream->position = 0;
135 static int stream_get(stream_t *stream, json_error_t *error)
139 if(stream->state != STREAM_STATE_OK)
140 return stream->state;
142 if(!stream->buffer[stream->buffer_pos])
144 c = stream->get(stream->data);
146 stream->state = STREAM_STATE_EOF;
147 return STREAM_STATE_EOF;
150 stream->buffer[0] = c;
151 stream->buffer_pos = 0;
153 if(0x80 <= c && c <= 0xFF)
155 /* multi-byte UTF-8 sequence */
158 count = utf8_check_first(c);
164 for(i = 1; i < count; i++)
165 stream->buffer[i] = stream->get(stream->data);
167 if(!utf8_check_full(stream->buffer, count, NULL))
170 stream->buffer[count] = '\0';
173 stream->buffer[1] = '\0';
176 c = stream->buffer[stream->buffer_pos++];
181 stream->last_column = stream->column;
184 else if(utf8_check_first(c)) {
185 /* track the Unicode character column, so increment only if
186 this is the first character of a UTF-8 sequence */
193 stream->state = STREAM_STATE_ERROR;
194 error_set(error, stream_to_lex(stream), "unable to decode byte 0x%x", c);
195 return STREAM_STATE_ERROR;
198 static void stream_unget(stream_t *stream, int c)
200 if(c == STREAM_STATE_EOF || c == STREAM_STATE_ERROR)
206 stream->column = stream->last_column;
208 else if(utf8_check_first(c))
211 assert(stream->buffer_pos > 0);
212 stream->buffer_pos--;
213 assert(stream->buffer[stream->buffer_pos] == c);
217 static int lex_get(lex_t *lex, json_error_t *error)
219 return stream_get(&lex->stream, error);
222 static void lex_save(lex_t *lex, int c)
224 strbuffer_append_byte(&lex->saved_text, c);
227 static int lex_get_save(lex_t *lex, json_error_t *error)
229 int c = stream_get(&lex->stream, error);
230 if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR)
235 static void lex_unget(lex_t *lex, int c)
237 stream_unget(&lex->stream, c);
240 static void lex_unget_unsave(lex_t *lex, int c)
242 if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR) {
244 stream_unget(&lex->stream, c);
245 d = strbuffer_pop(&lex->saved_text);
250 static void lex_save_cached(lex_t *lex)
252 while(lex->stream.buffer[lex->stream.buffer_pos] != '\0')
254 lex_save(lex, lex->stream.buffer[lex->stream.buffer_pos]);
255 lex->stream.buffer_pos++;
256 lex->stream.position++;
260 /* assumes that str points to 'u' plus at least 4 valid hex digits */
261 static int32_t decode_unicode_escape(const char *str)
266 assert(str[0] == 'u');
268 for(i = 1; i <= 4; i++) {
274 value += c - 'a' + 10;
276 value += c - 'A' + 10;
284 static void lex_scan_string(lex_t *lex, json_error_t *error)
291 lex->value.string = NULL;
292 lex->token = TOKEN_INVALID;
294 c = lex_get_save(lex, error);
297 if(c == STREAM_STATE_ERROR)
300 else if(c == STREAM_STATE_EOF) {
301 error_set(error, lex, "premature end of input");
305 else if(0 <= c && c <= 0x1F) {
306 /* control character */
307 lex_unget_unsave(lex, c);
309 error_set(error, lex, "unexpected newline", c);
311 error_set(error, lex, "control character 0x%x", c);
316 c = lex_get_save(lex, error);
318 c = lex_get_save(lex, error);
319 for(i = 0; i < 4; i++) {
321 error_set(error, lex, "invalid escape");
324 c = lex_get_save(lex, error);
327 else if(c == '"' || c == '\\' || c == '/' || c == 'b' ||
328 c == 'f' || c == 'n' || c == 'r' || c == 't')
329 c = lex_get_save(lex, error);
331 error_set(error, lex, "invalid escape");
336 c = lex_get_save(lex, error);
339 /* the actual value is at most of the same length as the source
341 - shortcut escapes (e.g. "\t") (length 2) are converted to 1 byte
342 - a single \uXXXX escape (length 6) is converted to at most 3 bytes
343 - two \uXXXX escapes (length 12) forming an UTF-16 surrogate pair
344 are converted to 4 bytes
346 lex->value.string = jsonp_malloc(lex->saved_text.length + 1);
347 if(!lex->value.string) {
348 /* this is not very nice, since TOKEN_INVALID is returned */
353 t = lex->value.string;
355 /* + 1 to skip the " */
356 p = strbuffer_value(&lex->saved_text) + 1;
366 value = decode_unicode_escape(p);
369 if(0xD800 <= value && value <= 0xDBFF) {
371 if(*p == '\\' && *(p + 1) == 'u') {
372 int32_t value2 = decode_unicode_escape(++p);
375 if(0xDC00 <= value2 && value2 <= 0xDFFF) {
376 /* valid second surrogate */
378 ((value - 0xD800) << 10) +
383 /* invalid second surrogate */
384 error_set(error, lex,
385 "invalid Unicode '\\u%04X\\u%04X'",
391 /* no second surrogate */
392 error_set(error, lex, "invalid Unicode '\\u%04X'",
397 else if(0xDC00 <= value && value <= 0xDFFF) {
398 error_set(error, lex, "invalid Unicode '\\u%04X'", value);
403 error_set(error, lex, "\\u0000 is not allowed");
407 if(utf8_encode(value, buffer, &length))
410 memcpy(t, buffer, length);
415 case '"': case '\\': case '/':
417 case 'b': *t = '\b'; break;
418 case 'f': *t = '\f'; break;
419 case 'n': *t = '\n'; break;
420 case 'r': *t = '\r'; break;
421 case 't': *t = '\t'; break;
432 lex->token = TOKEN_STRING;
436 jsonp_free(lex->value.string);
439 #if JSON_INTEGER_IS_LONG_LONG
440 #define json_strtoint strtoll
442 #define json_strtoint strtol
445 static int lex_scan_number(lex_t *lex, int c, json_error_t *error)
447 const char *saved_text;
451 lex->token = TOKEN_INVALID;
454 c = lex_get_save(lex, error);
457 c = lex_get_save(lex, error);
459 lex_unget_unsave(lex, c);
463 else if(isdigit(c)) {
464 c = lex_get_save(lex, error);
466 c = lex_get_save(lex, error);
469 lex_unget_unsave(lex, c);
473 if(c != '.' && c != 'E' && c != 'e') {
476 lex_unget_unsave(lex, c);
478 saved_text = strbuffer_value(&lex->saved_text);
481 value = json_strtoint(saved_text, &end, 10);
482 if(errno == ERANGE) {
484 error_set(error, lex, "too big negative integer");
486 error_set(error, lex, "too big integer");
490 assert(end == saved_text + lex->saved_text.length);
492 lex->token = TOKEN_INTEGER;
493 lex->value.integer = value;
498 c = lex_get(lex, error);
505 c = lex_get_save(lex, error);
507 c = lex_get_save(lex, error);
510 if(c == 'E' || c == 'e') {
511 c = lex_get_save(lex, error);
512 if(c == '+' || c == '-')
513 c = lex_get_save(lex, error);
516 lex_unget_unsave(lex, c);
520 c = lex_get_save(lex, error);
522 c = lex_get_save(lex, error);
525 lex_unget_unsave(lex, c);
527 saved_text = strbuffer_value(&lex->saved_text);
529 value = strtod(saved_text, &end);
530 assert(end == saved_text + lex->saved_text.length);
532 if(errno == ERANGE && value != 0) {
533 error_set(error, lex, "real number overflow");
537 lex->token = TOKEN_REAL;
538 lex->value.real = value;
545 static int lex_scan(lex_t *lex, json_error_t *error)
549 strbuffer_clear(&lex->saved_text);
551 if(lex->token == TOKEN_STRING) {
552 jsonp_free(lex->value.string);
553 lex->value.string = NULL;
556 c = lex_get(lex, error);
557 while(c == ' ' || c == '\t' || c == '\n' || c == '\r')
558 c = lex_get(lex, error);
560 if(c == STREAM_STATE_EOF) {
561 lex->token = TOKEN_EOF;
565 if(c == STREAM_STATE_ERROR) {
566 lex->token = TOKEN_INVALID;
572 if(c == '{' || c == '}' || c == '[' || c == ']' || c == ':' || c == ',')
576 lex_scan_string(lex, error);
578 else if(isdigit(c) || c == '-') {
579 if(lex_scan_number(lex, c, error))
583 else if(isupper(c) || islower(c)) {
584 /* eat up the whole identifier for clearer error messages */
585 const char *saved_text;
587 c = lex_get_save(lex, error);
588 while(isupper(c) || islower(c))
589 c = lex_get_save(lex, error);
590 lex_unget_unsave(lex, c);
592 saved_text = strbuffer_value(&lex->saved_text);
594 if(strcmp(saved_text, "true") == 0)
595 lex->token = TOKEN_TRUE;
596 else if(strcmp(saved_text, "false") == 0)
597 lex->token = TOKEN_FALSE;
598 else if(strcmp(saved_text, "null") == 0)
599 lex->token = TOKEN_NULL;
601 lex->token = TOKEN_INVALID;
605 /* save the rest of the input UTF-8 sequence to get an error
606 message of valid UTF-8 */
607 lex_save_cached(lex);
608 lex->token = TOKEN_INVALID;
615 static char *lex_steal_string(lex_t *lex)
618 if(lex->token == TOKEN_STRING)
620 result = lex->value.string;
621 lex->value.string = NULL;
626 static int lex_init(lex_t *lex, get_func get, void *data)
628 stream_init(&lex->stream, get, data);
629 if(strbuffer_init(&lex->saved_text))
632 lex->token = TOKEN_INVALID;
636 static void lex_close(lex_t *lex)
638 if(lex->token == TOKEN_STRING)
639 jsonp_free(lex->value.string);
640 strbuffer_close(&lex->saved_text);
646 static json_t *parse_value(lex_t *lex, size_t flags, json_error_t *error);
648 static json_t *parse_object(lex_t *lex, size_t flags, json_error_t *error)
650 json_t *object = json_object();
654 lex_scan(lex, error);
655 if(lex->token == '}')
662 if(lex->token != TOKEN_STRING) {
663 error_set(error, lex, "string or '}' expected");
667 key = lex_steal_string(lex);
671 if(flags & JSON_REJECT_DUPLICATES) {
672 if(json_object_get(object, key)) {
674 error_set(error, lex, "duplicate object key");
679 lex_scan(lex, error);
680 if(lex->token != ':') {
682 error_set(error, lex, "':' expected");
686 lex_scan(lex, error);
687 value = parse_value(lex, flags, error);
693 if(json_object_set_nocheck(object, key, value)) {
702 lex_scan(lex, error);
703 if(lex->token != ',')
706 lex_scan(lex, error);
709 if(lex->token != '}') {
710 error_set(error, lex, "'}' expected");
721 static json_t *parse_array(lex_t *lex, size_t flags, json_error_t *error)
723 json_t *array = json_array();
727 lex_scan(lex, error);
728 if(lex->token == ']')
732 json_t *elem = parse_value(lex, flags, error);
736 if(json_array_append(array, elem)) {
742 lex_scan(lex, error);
743 if(lex->token != ',')
746 lex_scan(lex, error);
749 if(lex->token != ']') {
750 error_set(error, lex, "']' expected");
761 static json_t *parse_value(lex_t *lex, size_t flags, json_error_t *error)
767 json = json_string_nocheck(lex->value.string);
771 case TOKEN_INTEGER: {
772 json = json_integer(lex->value.integer);
777 json = json_real(lex->value.real);
794 json = parse_object(lex, flags, error);
798 json = parse_array(lex, flags, error);
802 error_set(error, lex, "invalid token");
806 error_set(error, lex, "unexpected token");
816 static json_t *parse_json(lex_t *lex, size_t flags, json_error_t *error)
820 lex_scan(lex, error);
821 if(lex->token != '[' && lex->token != '{') {
822 error_set(error, lex, "'[' or '{' expected");
826 result = parse_value(lex, flags, error);
830 if(!(flags & JSON_DISABLE_EOF_CHECK)) {
831 lex_scan(lex, error);
832 if(lex->token != TOKEN_EOF) {
833 error_set(error, lex, "end of file expected");
848 static int string_get(void *data)
851 string_data_t *stream = (string_data_t *)data;
852 c = stream->data[stream->pos];
858 return (unsigned char)c;
862 json_t *json_loads(const char *string, size_t flags, json_error_t *error)
866 string_data_t stream_data;
868 stream_data.data = string;
871 if(lex_init(&lex, string_get, (void *)&stream_data))
874 jsonp_error_init(error, "<string>");
875 result = parse_json(&lex, flags, error);
888 static int buffer_get(void *data)
891 buffer_data_t *stream = data;
892 if(stream->pos >= stream->len)
895 c = stream->data[stream->pos];
897 return (unsigned char)c;
900 json_t *json_loadb(const char *buffer, size_t buflen, size_t flags, json_error_t *error)
904 buffer_data_t stream_data;
906 stream_data.data = buffer;
908 stream_data.len = buflen;
910 if(lex_init(&lex, buffer_get, (void *)&stream_data))
913 jsonp_error_init(error, "<buffer>");
914 result = parse_json(&lex, flags, error);
920 json_t *json_loadf(FILE *input, size_t flags, json_error_t *error)
926 if(lex_init(&lex, (get_func)fgetc, input))
934 jsonp_error_init(error, source);
935 result = parse_json(&lex, flags, error);
941 json_t *json_load_file(const char *path, size_t flags, json_error_t *error)
946 jsonp_error_init(error, path);
948 fp = fopen(path, "r");
951 error_set(error, NULL, "unable to open %s: %s",
952 path, strerror(errno));
956 result = json_loadf(fp, flags, error);