2 * Copyright (c) 2009-2012 Petri Lehtinen <petri@digip.org>
4 * Jansson is free software; you can redistribute it and/or modify
5 * it under the terms of the MIT license. See LICENSE for details.
17 #include "jansson_private.h"
18 #include "strbuffer.h"
21 #define STREAM_STATE_OK 0
22 #define STREAM_STATE_EOF -1
23 #define STREAM_STATE_ERROR -2
25 #define TOKEN_INVALID -1
27 #define TOKEN_STRING 256
28 #define TOKEN_INTEGER 257
29 #define TOKEN_REAL 258
30 #define TOKEN_TRUE 259
31 #define TOKEN_FALSE 260
32 #define TOKEN_NULL 261
34 /* Locale independent versions of isxxx() functions */
35 #define l_isupper(c) ('A' <= (c) && (c) <= 'Z')
36 #define l_islower(c) ('a' <= (c) && (c) <= 'z')
37 #define l_isalpha(c) (l_isupper(c) || l_islower(c))
38 #define l_isdigit(c) ('0' <= (c) && (c) <= '9')
39 #define l_isxdigit(c) \
40 (l_isdigit(c) || 'A' <= (c) || (c) <= 'F' || 'a' <= (c) || (c) <= 'f')
42 /* Read one byte from stream, convert to unsigned char, then int, and
43 return. return EOF on end of file. This corresponds to the
44 behaviour of fgetc(). */
45 typedef int (*get_func)(void *data);
54 int column, last_column;
60 strbuffer_t saved_text;
69 #define stream_to_lex(stream) container_of(stream, lex_t, stream)
72 /*** error reporting ***/
74 static void error_set(json_error_t *error, const lex_t *lex,
78 char msg_text[JSON_ERROR_TEXT_LENGTH];
79 char msg_with_context[JSON_ERROR_TEXT_LENGTH];
81 int line = -1, col = -1;
83 const char *result = msg_text;
89 vsnprintf(msg_text, JSON_ERROR_TEXT_LENGTH, msg, ap);
90 msg_text[JSON_ERROR_TEXT_LENGTH - 1] = '\0';
95 const char *saved_text = strbuffer_value(&lex->saved_text);
97 line = lex->stream.line;
98 col = lex->stream.column;
99 pos = lex->stream.position;
101 if(saved_text && saved_text[0])
103 if(lex->saved_text.length <= 20) {
104 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
105 "%s near '%s'", msg_text, saved_text);
106 msg_with_context[JSON_ERROR_TEXT_LENGTH - 1] = '\0';
107 result = msg_with_context;
112 if(lex->stream.state == STREAM_STATE_ERROR) {
113 /* No context for UTF-8 decoding errors */
117 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
118 "%s near end of file", msg_text);
119 msg_with_context[JSON_ERROR_TEXT_LENGTH - 1] = '\0';
120 result = msg_with_context;
125 jsonp_error_set(error, line, col, pos, "%s", result);
129 /*** lexical analyzer ***/
132 stream_init(stream_t *stream, get_func get, void *data)
136 stream->buffer[0] = '\0';
137 stream->buffer_pos = 0;
139 stream->state = STREAM_STATE_OK;
142 stream->position = 0;
145 static int stream_get(stream_t *stream, json_error_t *error)
149 if(stream->state != STREAM_STATE_OK)
150 return stream->state;
152 if(!stream->buffer[stream->buffer_pos])
154 c = stream->get(stream->data);
156 stream->state = STREAM_STATE_EOF;
157 return STREAM_STATE_EOF;
160 stream->buffer[0] = c;
161 stream->buffer_pos = 0;
163 if(0x80 <= c && c <= 0xFF)
165 /* multi-byte UTF-8 sequence */
168 count = utf8_check_first(c);
174 for(i = 1; i < count; i++)
175 stream->buffer[i] = stream->get(stream->data);
177 if(!utf8_check_full(stream->buffer, count, NULL))
180 stream->buffer[count] = '\0';
183 stream->buffer[1] = '\0';
186 c = stream->buffer[stream->buffer_pos++];
191 stream->last_column = stream->column;
194 else if(utf8_check_first(c)) {
195 /* track the Unicode character column, so increment only if
196 this is the first character of a UTF-8 sequence */
203 stream->state = STREAM_STATE_ERROR;
204 error_set(error, stream_to_lex(stream), "unable to decode byte 0x%x", c);
205 return STREAM_STATE_ERROR;
208 static void stream_unget(stream_t *stream, int c)
210 if(c == STREAM_STATE_EOF || c == STREAM_STATE_ERROR)
216 stream->column = stream->last_column;
218 else if(utf8_check_first(c))
221 assert(stream->buffer_pos > 0);
222 stream->buffer_pos--;
223 assert(stream->buffer[stream->buffer_pos] == c);
227 static int lex_get(lex_t *lex, json_error_t *error)
229 return stream_get(&lex->stream, error);
232 static void lex_save(lex_t *lex, int c)
234 strbuffer_append_byte(&lex->saved_text, c);
237 static int lex_get_save(lex_t *lex, json_error_t *error)
239 int c = stream_get(&lex->stream, error);
240 if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR)
245 static void lex_unget(lex_t *lex, int c)
247 stream_unget(&lex->stream, c);
250 static void lex_unget_unsave(lex_t *lex, int c)
252 if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR) {
254 stream_unget(&lex->stream, c);
255 d = strbuffer_pop(&lex->saved_text);
260 static void lex_save_cached(lex_t *lex)
262 while(lex->stream.buffer[lex->stream.buffer_pos] != '\0')
264 lex_save(lex, lex->stream.buffer[lex->stream.buffer_pos]);
265 lex->stream.buffer_pos++;
266 lex->stream.position++;
270 /* assumes that str points to 'u' plus at least 4 valid hex digits */
271 static int32_t decode_unicode_escape(const char *str)
276 assert(str[0] == 'u');
278 for(i = 1; i <= 4; i++) {
283 else if(l_islower(c))
284 value += c - 'a' + 10;
285 else if(l_isupper(c))
286 value += c - 'A' + 10;
294 static void lex_scan_string(lex_t *lex, json_error_t *error)
301 lex->value.string = NULL;
302 lex->token = TOKEN_INVALID;
304 c = lex_get_save(lex, error);
307 if(c == STREAM_STATE_ERROR)
310 else if(c == STREAM_STATE_EOF) {
311 error_set(error, lex, "premature end of input");
315 else if(0 <= c && c <= 0x1F) {
316 /* control character */
317 lex_unget_unsave(lex, c);
319 error_set(error, lex, "unexpected newline", c);
321 error_set(error, lex, "control character 0x%x", c);
326 c = lex_get_save(lex, error);
328 c = lex_get_save(lex, error);
329 for(i = 0; i < 4; i++) {
331 error_set(error, lex, "invalid escape");
334 c = lex_get_save(lex, error);
337 else if(c == '"' || c == '\\' || c == '/' || c == 'b' ||
338 c == 'f' || c == 'n' || c == 'r' || c == 't')
339 c = lex_get_save(lex, error);
341 error_set(error, lex, "invalid escape");
346 c = lex_get_save(lex, error);
349 /* the actual value is at most of the same length as the source
351 - shortcut escapes (e.g. "\t") (length 2) are converted to 1 byte
352 - a single \uXXXX escape (length 6) is converted to at most 3 bytes
353 - two \uXXXX escapes (length 12) forming an UTF-16 surrogate pair
354 are converted to 4 bytes
356 lex->value.string = jsonp_malloc(lex->saved_text.length + 1);
357 if(!lex->value.string) {
358 /* this is not very nice, since TOKEN_INVALID is returned */
363 t = lex->value.string;
365 /* + 1 to skip the " */
366 p = strbuffer_value(&lex->saved_text) + 1;
376 value = decode_unicode_escape(p);
379 if(0xD800 <= value && value <= 0xDBFF) {
381 if(*p == '\\' && *(p + 1) == 'u') {
382 int32_t value2 = decode_unicode_escape(++p);
385 if(0xDC00 <= value2 && value2 <= 0xDFFF) {
386 /* valid second surrogate */
388 ((value - 0xD800) << 10) +
393 /* invalid second surrogate */
394 error_set(error, lex,
395 "invalid Unicode '\\u%04X\\u%04X'",
401 /* no second surrogate */
402 error_set(error, lex, "invalid Unicode '\\u%04X'",
407 else if(0xDC00 <= value && value <= 0xDFFF) {
408 error_set(error, lex, "invalid Unicode '\\u%04X'", value);
413 error_set(error, lex, "\\u0000 is not allowed");
417 if(utf8_encode(value, buffer, &length))
420 memcpy(t, buffer, length);
425 case '"': case '\\': case '/':
427 case 'b': *t = '\b'; break;
428 case 'f': *t = '\f'; break;
429 case 'n': *t = '\n'; break;
430 case 'r': *t = '\r'; break;
431 case 't': *t = '\t'; break;
442 lex->token = TOKEN_STRING;
446 jsonp_free(lex->value.string);
449 #if JSON_INTEGER_IS_LONG_LONG
450 #ifdef _MSC_VER // Microsoft Visual Studio
451 #define json_strtoint _strtoi64
453 #define json_strtoint strtoll
456 #define json_strtoint strtol
459 static int lex_scan_number(lex_t *lex, int c, json_error_t *error)
461 const char *saved_text;
465 lex->token = TOKEN_INVALID;
468 c = lex_get_save(lex, error);
471 c = lex_get_save(lex, error);
473 lex_unget_unsave(lex, c);
477 else if(l_isdigit(c)) {
478 c = lex_get_save(lex, error);
480 c = lex_get_save(lex, error);
483 lex_unget_unsave(lex, c);
487 if(c != '.' && c != 'E' && c != 'e') {
490 lex_unget_unsave(lex, c);
492 saved_text = strbuffer_value(&lex->saved_text);
495 value = json_strtoint(saved_text, &end, 10);
496 if(errno == ERANGE) {
498 error_set(error, lex, "too big negative integer");
500 error_set(error, lex, "too big integer");
504 assert(end == saved_text + lex->saved_text.length);
506 lex->token = TOKEN_INTEGER;
507 lex->value.integer = value;
512 c = lex_get(lex, error);
519 c = lex_get_save(lex, error);
521 c = lex_get_save(lex, error);
524 if(c == 'E' || c == 'e') {
525 c = lex_get_save(lex, error);
526 if(c == '+' || c == '-')
527 c = lex_get_save(lex, error);
530 lex_unget_unsave(lex, c);
534 c = lex_get_save(lex, error);
536 c = lex_get_save(lex, error);
539 lex_unget_unsave(lex, c);
541 if(jsonp_strtod(&lex->saved_text, &value)) {
542 error_set(error, lex, "real number overflow");
546 lex->token = TOKEN_REAL;
547 lex->value.real = value;
554 static int lex_scan(lex_t *lex, json_error_t *error)
558 strbuffer_clear(&lex->saved_text);
560 if(lex->token == TOKEN_STRING) {
561 jsonp_free(lex->value.string);
562 lex->value.string = NULL;
565 c = lex_get(lex, error);
566 while(c == ' ' || c == '\t' || c == '\n' || c == '\r')
567 c = lex_get(lex, error);
569 if(c == STREAM_STATE_EOF) {
570 lex->token = TOKEN_EOF;
574 if(c == STREAM_STATE_ERROR) {
575 lex->token = TOKEN_INVALID;
581 if(c == '{' || c == '}' || c == '[' || c == ']' || c == ':' || c == ',')
585 lex_scan_string(lex, error);
587 else if(l_isdigit(c) || c == '-') {
588 if(lex_scan_number(lex, c, error))
592 else if(l_isalpha(c)) {
593 /* eat up the whole identifier for clearer error messages */
594 const char *saved_text;
596 c = lex_get_save(lex, error);
598 c = lex_get_save(lex, error);
599 lex_unget_unsave(lex, c);
601 saved_text = strbuffer_value(&lex->saved_text);
603 if(strcmp(saved_text, "true") == 0)
604 lex->token = TOKEN_TRUE;
605 else if(strcmp(saved_text, "false") == 0)
606 lex->token = TOKEN_FALSE;
607 else if(strcmp(saved_text, "null") == 0)
608 lex->token = TOKEN_NULL;
610 lex->token = TOKEN_INVALID;
614 /* save the rest of the input UTF-8 sequence to get an error
615 message of valid UTF-8 */
616 lex_save_cached(lex);
617 lex->token = TOKEN_INVALID;
624 static char *lex_steal_string(lex_t *lex)
627 if(lex->token == TOKEN_STRING)
629 result = lex->value.string;
630 lex->value.string = NULL;
635 static int lex_init(lex_t *lex, get_func get, void *data)
637 stream_init(&lex->stream, get, data);
638 if(strbuffer_init(&lex->saved_text))
641 lex->token = TOKEN_INVALID;
645 static void lex_close(lex_t *lex)
647 if(lex->token == TOKEN_STRING)
648 jsonp_free(lex->value.string);
649 strbuffer_close(&lex->saved_text);
655 static json_t *parse_value(lex_t *lex, size_t flags, json_error_t *error);
657 static json_t *parse_object(lex_t *lex, size_t flags, json_error_t *error)
659 json_t *object = json_object();
663 lex_scan(lex, error);
664 if(lex->token == '}')
671 if(lex->token != TOKEN_STRING) {
672 error_set(error, lex, "string or '}' expected");
676 key = lex_steal_string(lex);
680 if(flags & JSON_REJECT_DUPLICATES) {
681 if(json_object_get(object, key)) {
683 error_set(error, lex, "duplicate object key");
688 lex_scan(lex, error);
689 if(lex->token != ':') {
691 error_set(error, lex, "':' expected");
695 lex_scan(lex, error);
696 value = parse_value(lex, flags, error);
702 if(json_object_set_nocheck(object, key, value)) {
711 lex_scan(lex, error);
712 if(lex->token != ',')
715 lex_scan(lex, error);
718 if(lex->token != '}') {
719 error_set(error, lex, "'}' expected");
730 static json_t *parse_array(lex_t *lex, size_t flags, json_error_t *error)
732 json_t *array = json_array();
736 lex_scan(lex, error);
737 if(lex->token == ']')
741 json_t *elem = parse_value(lex, flags, error);
745 if(json_array_append(array, elem)) {
751 lex_scan(lex, error);
752 if(lex->token != ',')
755 lex_scan(lex, error);
758 if(lex->token != ']') {
759 error_set(error, lex, "']' expected");
770 static json_t *parse_value(lex_t *lex, size_t flags, json_error_t *error)
776 json = json_string_nocheck(lex->value.string);
780 case TOKEN_INTEGER: {
781 json = json_integer(lex->value.integer);
786 json = json_real(lex->value.real);
803 json = parse_object(lex, flags, error);
807 json = parse_array(lex, flags, error);
811 error_set(error, lex, "invalid token");
815 error_set(error, lex, "unexpected token");
825 static json_t *parse_json(lex_t *lex, size_t flags, json_error_t *error)
829 lex_scan(lex, error);
830 if(!(flags & JSON_DECODE_ANY)) {
831 if(lex->token != '[' && lex->token != '{') {
832 error_set(error, lex, "'[' or '{' expected");
837 result = parse_value(lex, flags, error);
841 if(!(flags & JSON_DISABLE_EOF_CHECK)) {
842 lex_scan(lex, error);
843 if(lex->token != TOKEN_EOF) {
844 error_set(error, lex, "end of file expected");
851 /* Save the position even though there was no error */
852 error->position = lex->stream.position;
864 static int string_get(void *data)
867 string_data_t *stream = (string_data_t *)data;
868 c = stream->data[stream->pos];
874 return (unsigned char)c;
878 json_t *json_loads(const char *string, size_t flags, json_error_t *error)
882 string_data_t stream_data;
884 jsonp_error_init(error, "<string>");
886 if (string == NULL) {
887 error_set(error, NULL, "wrong arguments");
891 stream_data.data = string;
894 if(lex_init(&lex, string_get, (void *)&stream_data))
897 result = parse_json(&lex, flags, error);
910 static int buffer_get(void *data)
913 buffer_data_t *stream = data;
914 if(stream->pos >= stream->len)
917 c = stream->data[stream->pos];
919 return (unsigned char)c;
922 json_t *json_loadb(const char *buffer, size_t buflen, size_t flags, json_error_t *error)
926 buffer_data_t stream_data;
928 jsonp_error_init(error, "<buffer>");
930 if (buffer == NULL) {
931 error_set(error, NULL, "wrong arguments");
935 stream_data.data = buffer;
937 stream_data.len = buflen;
939 if(lex_init(&lex, buffer_get, (void *)&stream_data))
942 result = parse_json(&lex, flags, error);
948 json_t *json_loadf(FILE *input, size_t flags, json_error_t *error)
959 jsonp_error_init(error, source);
962 error_set(error, NULL, "wrong arguments");
966 if(lex_init(&lex, (get_func)fgetc, input))
969 result = parse_json(&lex, flags, error);
975 json_t *json_load_file(const char *path, size_t flags, json_error_t *error)
980 jsonp_error_init(error, path);
983 error_set(error, NULL, "wrong arguments");
987 fp = fopen(path, "rb");
990 error_set(error, NULL, "unable to open %s: %s",
991 path, strerror(errno));
995 result = json_loadf(fp, flags, error);
1001 #define MAX_BUF_LEN 1024
1005 char data[MAX_BUF_LEN];
1008 json_load_callback_t callback;
1012 static int callback_get(void *data)
1015 callback_data_t *stream = data;
1017 if(stream->pos >= stream->len) {
1019 stream->len = stream->callback(stream->data, MAX_BUF_LEN, stream->arg);
1020 if(stream->len == 0 || stream->len == (size_t)-1)
1024 c = stream->data[stream->pos];
1026 return (unsigned char)c;
1029 json_t *json_load_callback(json_load_callback_t callback, void *arg, size_t flags, json_error_t *error)
1034 callback_data_t stream_data;
1036 memset(&stream_data, 0, sizeof(stream_data));
1037 stream_data.callback = callback;
1038 stream_data.arg = arg;
1040 jsonp_error_init(error, "<callback>");
1042 if (callback == NULL) {
1043 error_set(error, NULL, "wrong arguments");
1047 if(lex_init(&lex, (get_func)callback_get, &stream_data))
1050 result = parse_json(&lex, flags, error);