json_tokener.c

   1 /*
   2  * $Id: json_tokener.c,v 1.20 2006/07/25 03:24:50 mclark Exp $
   3  *
   4  * Copyright (c) 2004, 2005 Metaparadigm Pte. Ltd.
   5  * Michael Clark <michael@metaparadigm.com>
   6  *
   7  * This library is free software; you can redistribute it and/or modify
   8  * it under the terms of the MIT license. See COPYING for details.
   9  *
  10  *
  11  * Copyright (c) 2008-2009 Yahoo! Inc.  All rights reserved.
  12  * The copyrights to the contents of this file are licensed under the MIT License
  13  * (http://www.opensource.org/licenses/mit-license.php)
  14  */
  15
  16 #include "config.h"
  17
  18 #include <math.h>
  19 #include <stdio.h>
  20 #include <stdlib.h>
  21 #include <stddef.h>
  22 #include <ctype.h>
  23 #include <string.h>
  24 #include <limits.h>
  25
  26 #include "bits.h"
  27 #include "debug.h"
  28 #include "printbuf.h"
  29 #include "arraylist.h"
  30 #include "json_inttypes.h"
  31 #include "json_object.h"
  32 #include "json_tokener.h"
  33 #include "json_util.h"
  34
  35 #ifdef HAVE_LOCALE_H
  36 #include <locale.h>
  37 #endif /* HAVE_LOCALE_H */
  38
  39 #if !HAVE_STRDUP && defined(_MSC_VER)
  40   /* MSC has the version as _strdup */
  41 # define strdup _strdup
  42 #elif !HAVE_STRDUP
  43 # error You do not have strdup on your system.
  44 #endif /* HAVE_STRDUP */
  45
  46 #if !HAVE_STRNCASECMP && defined(_MSC_VER)
  47   /* MSC has the version as _strnicmp */
  48 # define strncasecmp _strnicmp
  49 #elif !HAVE_STRNCASECMP
  50 # error You do not have strncasecmp on your system.
  51 #endif /* HAVE_STRNCASECMP */
  52
  53 /* Use C99 NAN by default; if not available, nan("") should work too. */
  54 #ifndef NAN
  55 #define NAN nan("")
  56 #endif /* !NAN */
  57
  58 static const char json_null_str[] = "null";
  59 static const int json_null_str_len = sizeof(json_null_str) - 1;
  60 static const char json_inf_str[] = "Infinity";
  61 static const int json_inf_str_len = sizeof(json_inf_str) - 1;
  62 static const char json_nan_str[] = "NaN";
  63 static const int json_nan_str_len = sizeof(json_nan_str) - 1;
  64 static const char json_true_str[] = "true";
  65 static const int json_true_str_len = sizeof(json_true_str) - 1;
  66 static const char json_false_str[] = "false";
  67 static const int json_false_str_len = sizeof(json_false_str) - 1;
  68
  69 static const char* json_tokener_errors[] = {
  70   "success",
  71   "continue",
  72   "nesting too deep",
  73   "unexpected end of data",
  74   "unexpected character",
  75   "null expected",
  76   "boolean expected",
  77   "number expected",
  78   "array value separator ',' expected",
  79   "quoted object property name expected",
  80   "object property name separator ':' expected",
  81   "object value separator ',' expected",
  82   "invalid string sequence",
  83   "expected comment",
  84   "buffer size overflow"
  85 };
  86
  87 const char *json_tokener_error_desc(enum json_tokener_error jerr)
  88 {
  89         int jerr_int = (int)jerr;
  90         if (jerr_int < 0 || jerr_int >= (int)(sizeof(json_tokener_errors) / sizeof(json_tokener_errors[0])))
  91                 return "Unknown error, invalid json_tokener_error value passed to json_tokener_error_desc()";
  92         return json_tokener_errors[jerr];
  93 }
  94
  95 enum json_tokener_error json_tokener_get_error(json_tokener *tok)
  96 {
  97         return tok->err;
  98 }
  99
 100 /* Stuff for decoding unicode sequences */
 101 #define IS_HIGH_SURROGATE(uc) (((uc) & 0xFC00) == 0xD800)
 102 #define IS_LOW_SURROGATE(uc)  (((uc) & 0xFC00) == 0xDC00)
 103 #define DECODE_SURROGATE_PAIR(hi,lo) ((((hi) & 0x3FF) << 10) + ((lo) & 0x3FF) + 0x10000)
 104 static unsigned char utf8_replacement_char[3] = { 0xEF, 0xBF, 0xBD };
 105
 106 struct json_tokener* json_tokener_new_ex(int depth)
 107 {
 108   struct json_tokener *tok;
 109
 110   tok = (struct json_tokener*)calloc(1, sizeof(struct json_tokener));
 111   if (!tok) return NULL;
 112   tok->stack = (struct json_tokener_srec *)calloc(depth, sizeof(struct json_tokener_srec));
 113   if (!tok->stack) {
 114     free(tok);
 115     return NULL;
 116   }
 117   tok->pb = printbuf_new();
 118   tok->max_depth = depth;
 119   json_tokener_reset(tok);
 120   return tok;
 121 }
 122
 123 struct json_tokener* json_tokener_new(void)
 124 {
 125   return json_tokener_new_ex(JSON_TOKENER_DEFAULT_DEPTH);
 126 }
 127
 128 void json_tokener_free(struct json_tokener *tok)
 129 {
 130   json_tokener_reset(tok);
 131   if (tok->pb) printbuf_free(tok->pb);
 132   if (tok->stack) free(tok->stack);
 133   free(tok);
 134 }
 135
 136 static void json_tokener_reset_level(struct json_tokener *tok, int depth)
 137 {
 138   tok->stack[depth].state = json_tokener_state_eatws;
 139   tok->stack[depth].saved_state = json_tokener_state_start;
 140   json_object_put(tok->stack[depth].current);
 141   tok->stack[depth].current = NULL;
 142   free(tok->stack[depth].obj_field_name);
 143   tok->stack[depth].obj_field_name = NULL;
 144 }
 145
 146 void json_tokener_reset(struct json_tokener *tok)
 147 {
 148   int i;
 149   if (!tok)
 150     return;
 151
 152   for(i = tok->depth; i >= 0; i--)
 153     json_tokener_reset_level(tok, i);
 154   tok->depth = 0;
 155   tok->err = json_tokener_success;
 156 }
 157
 158 struct json_object* json_tokener_parse(const char *str)
 159 {
 160     enum json_tokener_error jerr_ignored;
 161     struct json_object* obj;
 162     obj = json_tokener_parse_verbose(str, &jerr_ignored);
 163     return obj;
 164 }
 165
 166 struct json_object* json_tokener_parse_verbose(const char *str, enum json_tokener_error *error)
 167 {
 168     struct json_tokener* tok;
 169     struct json_object* obj;
 170
 171     tok = json_tokener_new();
 172     if (!tok)
 173       return NULL;
 174     obj = json_tokener_parse_ex(tok, str, -1);
 175     *error = tok->err;
 176     if(tok->err != json_tokener_success) {
 177                 if (obj != NULL)
 178                         json_object_put(obj);
 179         obj = NULL;
 180     }
 181
 182     json_tokener_free(tok);
 183     return obj;
 184 }
 185
 186 #define state  tok->stack[tok->depth].state
 187 #define saved_state  tok->stack[tok->depth].saved_state
 188 #define current tok->stack[tok->depth].current
 189 #define obj_field_name tok->stack[tok->depth].obj_field_name
 190
 191 /* Optimization:
 192  * json_tokener_parse_ex() consumed a lot of CPU in its main loop,
 193  * iterating character-by character.  A large performance boost is
 194  * achieved by using tighter loops to locally handle units such as
 195  * comments and strings.  Loops that handle an entire token within
 196  * their scope also gather entire strings and pass them to
 197  * printbuf_memappend() in a single call, rather than calling
 198  * printbuf_memappend() one char at a time.
 199  *
 200  * PEEK_CHAR() and ADVANCE_CHAR() macros are used for code that is
 201  * common to both the main loop and the tighter loops.
 202  */
 203
 204 /* PEEK_CHAR(dest, tok) macro:
 205  *   Peeks at the current char and stores it in dest.
 206  *   Returns 1 on success, sets tok->err and returns 0 if no more chars.
 207  *   Implicit inputs:  str, len vars
 208  */
 209 #define PEEK_CHAR(dest, tok)                                                  \
 210   (((tok)->char_offset == len) ?                                          \
 211    (((tok)->depth == 0 && state == json_tokener_state_eatws && saved_state == json_tokener_state_finish) ? \
 212     (((tok)->err = json_tokener_success), 0)                              \
 213     :                                                                   \
 214     (((tok)->err = json_tokener_continue), 0)                             \
 215     ) :                                                                 \
 216    (((dest) = *str), 1)                                                 \
 217    )
 218
 219 /* ADVANCE_CHAR() macro:
 220  *   Incrementes str & tok->char_offset.
 221  *   For convenience of existing conditionals, returns the old value of c (0 on eof)
 222  *   Implicit inputs:  c var
 223  */
 224 #define ADVANCE_CHAR(str, tok) \
 225   ( ++(str), ((tok)->char_offset)++, c)
 226
 227
 228 /* End optimization macro defs */
 229
 230
 231 struct json_object* json_tokener_parse_ex(struct json_tokener *tok,
 232                                           const char *str, int len)
 233 {
 234   struct json_object *obj = NULL;
 235   char c = '\1';
 236 #ifdef HAVE_SETLOCALE
 237   char *oldlocale=NULL, *tmplocale;
 238
 239   tmplocale = setlocale(LC_NUMERIC, NULL);
 240   if (tmplocale) oldlocale = strdup(tmplocale);
 241   setlocale(LC_NUMERIC, "C");
 242 #endif
 243
 244   tok->char_offset = 0;
 245   tok->err = json_tokener_success;
 246
 247   /* this interface is presently not 64-bit clean due to the int len argument
 248      and the internal printbuf interface that takes 32-bit int len arguments
 249      so the function limits the maximum string size to INT32_MAX (2GB).
 250      If the function is called with len == -1 then strlen is called to check
 251      the string length is less than INT32_MAX (2GB) */
 252   if ((len < -1) || (len == -1 && strlen(str) > INT32_MAX)) {
 253     tok->err = json_tokener_error_size;
 254     return NULL;
 255   }
 256
 257   while (PEEK_CHAR(c, tok)) {
 258
 259   redo_char:
 260     switch(state) {
 261
 262     case json_tokener_state_eatws:
 263       /* Advance until we change state */
 264       while (isspace((int)c)) {
 265         if ((!ADVANCE_CHAR(str, tok)) || (!PEEK_CHAR(c, tok)))
 266           goto out;
 267       }
 268       if(c == '/' && !(tok->flags & JSON_TOKENER_STRICT)) {
 269         printbuf_reset(tok->pb);
 270         printbuf_memappend_fast(tok->pb, &c, 1);
 271         state = json_tokener_state_comment_start;
 272       } else {
 273         state = saved_state;
 274         goto redo_char;
 275       }
 276       break;
 277
 278     case json_tokener_state_start:
 279       switch(c) {
 280       case '{':
 281         state = json_tokener_state_eatws;
 282         saved_state = json_tokener_state_object_field_start;
 283         current = json_object_new_object();
 284         break;
 285       case '[':
 286         state = json_tokener_state_eatws;
 287         saved_state = json_tokener_state_array;
 288         current = json_object_new_array();
 289         break;
 290       case 'I':
 291       case 'i':
 292         state = json_tokener_state_inf;
 293         printbuf_reset(tok->pb);
 294         tok->st_pos = 0;
 295         goto redo_char;
 296       case 'N':
 297       case 'n':
 298         state = json_tokener_state_null; // or NaN
 299         printbuf_reset(tok->pb);
 300         tok->st_pos = 0;
 301         goto redo_char;
 302       case '\'':
 303         if (tok->flags & JSON_TOKENER_STRICT) {
 304             /* in STRICT mode only double-quote are allowed */
 305             tok->err = json_tokener_error_parse_unexpected;
 306             goto out;
 307         }
 308       case '"':
 309         state = json_tokener_state_string;
 310         printbuf_reset(tok->pb);
 311         tok->quote_char = c;
 312         break;
 313       case 'T':
 314       case 't':
 315       case 'F':
 316       case 'f':
 317         state = json_tokener_state_boolean;
 318         printbuf_reset(tok->pb);
 319         tok->st_pos = 0;
 320         goto redo_char;
 321 #if defined(__GNUC__)
 322           case '0' ... '9':
 323 #else
 324           case '0':
 325       case '1':
 326       case '2':
 327       case '3':
 328       case '4':
 329       case '5':
 330       case '6':
 331       case '7':
 332       case '8':
 333       case '9':
 334 #endif
 335       case '-':
 336         state = json_tokener_state_number;
 337         printbuf_reset(tok->pb);
 338         tok->is_double = 0;
 339         goto redo_char;
 340       default:
 341         tok->err = json_tokener_error_parse_unexpected;
 342         goto out;
 343       }
 344       break;
 345
 346     case json_tokener_state_finish:
 347       if(tok->depth == 0) goto out;
 348       obj = json_object_get(current);
 349       json_tokener_reset_level(tok, tok->depth);
 350       tok->depth--;
 351       goto redo_char;
 352
 353     case json_tokener_state_inf: /* aka starts with 'i' */
 354       {
 355         int size;
 356         int size_inf;
 357         int is_negative = 0;
 358
 359         printbuf_memappend_fast(tok->pb, &c, 1);
 360         size = json_min(tok->st_pos+1, json_null_str_len);
 361         size_inf = json_min(tok->st_pos+1, json_inf_str_len);
 362         char *infbuf = tok->pb->buf;
 363         if (*infbuf == '-')
 364         {
 365                 infbuf++;
 366                 is_negative = 1;
 367         }
 368         if ((!(tok->flags & JSON_TOKENER_STRICT) &&
 369                   strncasecmp(json_inf_str, infbuf, size_inf) == 0) ||
 370                  (strncmp(json_inf_str, infbuf, size_inf) == 0)
 371                 )
 372         {
 373                 if (tok->st_pos == json_inf_str_len)
 374                 {
 375                         current = json_object_new_double(is_negative ? -INFINITY : INFINITY);
 376                         saved_state = json_tokener_state_finish;
 377                         state = json_tokener_state_eatws;
 378                         goto redo_char;
 379                 }
 380         } else {
 381                 tok->err = json_tokener_error_parse_unexpected;
 382                 goto out;
 383         }
 384         tok->st_pos++;
 385       }
 386       break;
 387     case json_tokener_state_null: /* aka starts with 'n' */
 388       {
 389         int size;
 390         int size_nan;
 391         printbuf_memappend_fast(tok->pb, &c, 1);
 392         size = json_min(tok->st_pos+1, json_null_str_len);
 393         size_nan = json_min(tok->st_pos+1, json_nan_str_len);
 394         if((!(tok->flags & JSON_TOKENER_STRICT) &&
 395           strncasecmp(json_null_str, tok->pb->buf, size) == 0)
 396           || (strncmp(json_null_str, tok->pb->buf, size) == 0)
 397           ) {
 398           if (tok->st_pos == json_null_str_len) {
 399             current = NULL;
 400             saved_state = json_tokener_state_finish;
 401             state = json_tokener_state_eatws;
 402             goto redo_char;
 403           }
 404         }
 405         else if ((!(tok->flags & JSON_TOKENER_STRICT) &&
 406                   strncasecmp(json_nan_str, tok->pb->buf, size_nan) == 0) ||
 407                  (strncmp(json_nan_str, tok->pb->buf, size_nan) == 0)
 408                 )
 409         {
 410                 if (tok->st_pos == json_nan_str_len)
 411                 {
 412                         current = json_object_new_double(NAN);
 413                         saved_state = json_tokener_state_finish;
 414                         state = json_tokener_state_eatws;
 415                         goto redo_char;
 416                 }
 417         } else {
 418           tok->err = json_tokener_error_parse_null;
 419           goto out;
 420         }
 421         tok->st_pos++;
 422       }
 423       break;
 424
 425     case json_tokener_state_comment_start:
 426       if(c == '*') {
 427         state = json_tokener_state_comment;
 428       } else if(c == '/') {
 429         state = json_tokener_state_comment_eol;
 430       } else {
 431         tok->err = json_tokener_error_parse_comment;
 432         goto out;
 433       }
 434       printbuf_memappend_fast(tok->pb, &c, 1);
 435       break;
 436
 437     case json_tokener_state_comment:
 438               {
 439           /* Advance until we change state */
 440           const char *case_start = str;
 441           while(c != '*') {
 442             if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 443               printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 444               goto out;
 445             }
 446           }
 447           printbuf_memappend_fast(tok->pb, case_start, 1+str-case_start);
 448           state = json_tokener_state_comment_end;
 449         }
 450             break;
 451
 452     case json_tokener_state_comment_eol:
 453       {
 454         /* Advance until we change state */
 455         const char *case_start = str;
 456         while(c != '\n') {
 457           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 458             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 459             goto out;
 460           }
 461         }
 462         printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 463         MC_DEBUG("json_tokener_comment: %s\n", tok->pb->buf);
 464         state = json_tokener_state_eatws;
 465       }
 466       break;
 467
 468     case json_tokener_state_comment_end:
 469       printbuf_memappend_fast(tok->pb, &c, 1);
 470       if(c == '/') {
 471         MC_DEBUG("json_tokener_comment: %s\n", tok->pb->buf);
 472         state = json_tokener_state_eatws;
 473       } else {
 474         state = json_tokener_state_comment;
 475       }
 476       break;
 477
 478     case json_tokener_state_string:
 479       {
 480         /* Advance until we change state */
 481         const char *case_start = str;
 482         while(1) {
 483           if(c == tok->quote_char) {
 484             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 485             current = json_object_new_string_len(tok->pb->buf, tok->pb->bpos);
 486             saved_state = json_tokener_state_finish;
 487             state = json_tokener_state_eatws;
 488             break;
 489           } else if(c == '\\') {
 490             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 491             saved_state = json_tokener_state_string;
 492             state = json_tokener_state_string_escape;
 493             break;
 494           }
 495           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 496             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 497             goto out;
 498           }
 499         }
 500       }
 501       break;
 502
 503     case json_tokener_state_string_escape:
 504       switch(c) {
 505       case '"':
 506       case '\\':
 507       case '/':
 508         printbuf_memappend_fast(tok->pb, &c, 1);
 509         state = saved_state;
 510         break;
 511       case 'b':
 512       case 'n':
 513       case 'r':
 514       case 't':
 515       case 'f':
 516         if(c == 'b') printbuf_memappend_fast(tok->pb, "\b", 1);
 517         else if(c == 'n') printbuf_memappend_fast(tok->pb, "\n", 1);
 518         else if(c == 'r') printbuf_memappend_fast(tok->pb, "\r", 1);
 519         else if(c == 't') printbuf_memappend_fast(tok->pb, "\t", 1);
 520         else if(c == 'f') printbuf_memappend_fast(tok->pb, "\f", 1);
 521         state = saved_state;
 522         break;
 523       case 'u':
 524         tok->ucs_char = 0;
 525         tok->st_pos = 0;
 526         state = json_tokener_state_escape_unicode;
 527         break;
 528       default:
 529         tok->err = json_tokener_error_parse_string;
 530         goto out;
 531       }
 532       break;
 533
 534     case json_tokener_state_escape_unicode:
 535         {
 536           unsigned int got_hi_surrogate = 0;
 537
 538           /* Handle a 4-byte sequence, or two sequences if a surrogate pair */
 539           while(1) {
 540             if(strchr(json_hex_chars, c)) {
 541               tok->ucs_char += ((unsigned int)hexdigit(c) << ((3-tok->st_pos++)*4));
 542               if(tok->st_pos == 4) {
 543                 unsigned char unescaped_utf[4];
 544
 545                 if (got_hi_surrogate) {
 546                   if (IS_LOW_SURROGATE(tok->ucs_char)) {
 547                     /* Recalculate the ucs_char, then fall thru to process normally */
 548                     tok->ucs_char = DECODE_SURROGATE_PAIR(got_hi_surrogate, tok->ucs_char);
 549                   } else {
 550                     /* Hi surrogate was not followed by a low surrogate */
 551                     /* Replace the hi and process the rest normally */
 552                     printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 553                   }
 554                   got_hi_surrogate = 0;
 555                 }
 556
 557                 if (tok->ucs_char < 0x80) {
 558                   unescaped_utf[0] = tok->ucs_char;
 559                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 1);
 560                 } else if (tok->ucs_char < 0x800) {
 561                   unescaped_utf[0] = 0xc0 | (tok->ucs_char >> 6);
 562                   unescaped_utf[1] = 0x80 | (tok->ucs_char & 0x3f);
 563                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 2);
 564                 } else if (IS_HIGH_SURROGATE(tok->ucs_char)) {
 565                   /* Got a high surrogate.  Remember it and look for the
 566                    * the beginning of another sequence, which should be the
 567                    * low surrogate.
 568                    */
 569                   got_hi_surrogate = tok->ucs_char;
 570                   /* Not at end, and the next two chars should be "\u" */
 571                   if ((tok->char_offset+1 != len) &&
 572                       (tok->char_offset+2 != len) &&
 573                       (str[1] == '\\') &&
 574                       (str[2] == 'u'))
 575                   {
 576                 /* Advance through the 16 bit surrogate, and move on to the
 577                  * next sequence. The next step is to process the following
 578                  * characters.
 579                  */
 580                     if( !ADVANCE_CHAR(str, tok) || !ADVANCE_CHAR(str, tok) ) {
 581                     printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 582                 }
 583                     /* Advance to the first char of the next sequence and
 584                      * continue processing with the next sequence.
 585                      */
 586                     if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 587                       printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 588                       goto out;
 589                     }
 590                     tok->ucs_char = 0;
 591                     tok->st_pos = 0;
 592                     continue; /* other json_tokener_state_escape_unicode */
 593                   } else {
 594                     /* Got a high surrogate without another sequence following
 595                      * it.  Put a replacement char in for the hi surrogate
 596                      * and pretend we finished.
 597                      */
 598                     printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 599                   }
 600                 } else if (IS_LOW_SURROGATE(tok->ucs_char)) {
 601                   /* Got a low surrogate not preceded by a high */
 602                   printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 603                 } else if (tok->ucs_char < 0x10000) {
 604                   unescaped_utf[0] = 0xe0 | (tok->ucs_char >> 12);
 605                   unescaped_utf[1] = 0x80 | ((tok->ucs_char >> 6) & 0x3f);
 606                   unescaped_utf[2] = 0x80 | (tok->ucs_char & 0x3f);
 607                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 3);
 608                 } else if (tok->ucs_char < 0x110000) {
 609                   unescaped_utf[0] = 0xf0 | ((tok->ucs_char >> 18) & 0x07);
 610                   unescaped_utf[1] = 0x80 | ((tok->ucs_char >> 12) & 0x3f);
 611                   unescaped_utf[2] = 0x80 | ((tok->ucs_char >> 6) & 0x3f);
 612                   unescaped_utf[3] = 0x80 | (tok->ucs_char & 0x3f);
 613                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 4);
 614                 } else {
 615                   /* Don't know what we got--insert the replacement char */
 616                   printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 617                 }
 618                 state = saved_state;
 619                 break;
 620               }
 621             } else {
 622               tok->err = json_tokener_error_parse_string;
 623               goto out;
 624             }
 625           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 626             if (got_hi_surrogate) /* Clean up any pending chars */
 627               printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 628             goto out;
 629           }
 630         }
 631       }
 632       break;
 633
 634     case json_tokener_state_boolean:
 635       {
 636         int size1, size2;
 637         printbuf_memappend_fast(tok->pb, &c, 1);
 638         size1 = json_min(tok->st_pos+1, json_true_str_len);
 639         size2 = json_min(tok->st_pos+1, json_false_str_len);
 640         if((!(tok->flags & JSON_TOKENER_STRICT) &&
 641           strncasecmp(json_true_str, tok->pb->buf, size1) == 0)
 642           || (strncmp(json_true_str, tok->pb->buf, size1) == 0)
 643           ) {
 644           if(tok->st_pos == json_true_str_len) {
 645             current = json_object_new_boolean(1);
 646             saved_state = json_tokener_state_finish;
 647             state = json_tokener_state_eatws;
 648             goto redo_char;
 649           }
 650         } else if((!(tok->flags & JSON_TOKENER_STRICT) &&
 651           strncasecmp(json_false_str, tok->pb->buf, size2) == 0)
 652           || (strncmp(json_false_str, tok->pb->buf, size2) == 0)) {
 653           if(tok->st_pos == json_false_str_len) {
 654             current = json_object_new_boolean(0);
 655             saved_state = json_tokener_state_finish;
 656             state = json_tokener_state_eatws;
 657             goto redo_char;
 658           }
 659         } else {
 660           tok->err = json_tokener_error_parse_boolean;
 661           goto out;
 662         }
 663         tok->st_pos++;
 664       }
 665       break;
 666
 667     case json_tokener_state_number:
 668       {
 669         /* Advance until we change state */
 670         const char *case_start = str;
 671         int case_len=0;
 672         while(c && strchr(json_number_chars, c)) {
 673           ++case_len;
 674           if(c == '.' || c == 'e' || c == 'E')
 675             tok->is_double = 1;
 676           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 677             printbuf_memappend_fast(tok->pb, case_start, case_len);
 678             goto out;
 679           }
 680         }
 681         if (case_len>0)
 682           printbuf_memappend_fast(tok->pb, case_start, case_len);
 683
 684         // Check for -Infinity
 685         if (tok->pb->buf[0] == '-' && case_len == 1 &&
 686             (c == 'i' || c == 'I'))
 687         {
 688                 state = json_tokener_state_inf;
 689                 goto redo_char;
 690         }
 691       }
 692       {
 693         int64_t num64;
 694         double  numd;
 695         if (!tok->is_double && json_parse_int64(tok->pb->buf, &num64) == 0) {
 696                 if (num64 && tok->pb->buf[0]=='0' && (tok->flags & JSON_TOKENER_STRICT)) {
 697                         /* in strict mode, number must not start with 0 */
 698                         tok->err = json_tokener_error_parse_number;
 699                         goto out;
 700                 }
 701                 current = json_object_new_int64(num64);
 702         }
 703         else if(tok->is_double && json_parse_double(tok->pb->buf, &numd) == 0)
 704         {
 705           current = json_object_new_double_s(numd, tok->pb->buf);
 706         } else {
 707           tok->err = json_tokener_error_parse_number;
 708           goto out;
 709         }
 710         saved_state = json_tokener_state_finish;
 711         state = json_tokener_state_eatws;
 712         goto redo_char;
 713       }
 714       break;
 715
 716     case json_tokener_state_array_after_sep:
 717     case json_tokener_state_array:
 718       if(c == ']') {
 719                 if (state == json_tokener_state_array_after_sep &&
 720                         (tok->flags & JSON_TOKENER_STRICT))
 721                 {
 722                         tok->err = json_tokener_error_parse_unexpected;
 723                         goto out;
 724                 }
 725         saved_state = json_tokener_state_finish;
 726         state = json_tokener_state_eatws;
 727       } else {
 728         if(tok->depth >= tok->max_depth-1) {
 729           tok->err = json_tokener_error_depth;
 730           goto out;
 731         }
 732         state = json_tokener_state_array_add;
 733         tok->depth++;
 734         json_tokener_reset_level(tok, tok->depth);
 735         goto redo_char;
 736       }
 737       break;
 738
 739     case json_tokener_state_array_add:
 740       json_object_array_add(current, obj);
 741       saved_state = json_tokener_state_array_sep;
 742       state = json_tokener_state_eatws;
 743       goto redo_char;
 744
 745     case json_tokener_state_array_sep:
 746       if(c == ']') {
 747         saved_state = json_tokener_state_finish;
 748         state = json_tokener_state_eatws;
 749       } else if(c == ',') {
 750         saved_state = json_tokener_state_array_after_sep;
 751         state = json_tokener_state_eatws;
 752       } else {
 753         tok->err = json_tokener_error_parse_array;
 754         goto out;
 755       }
 756       break;
 757
 758     case json_tokener_state_object_field_start:
 759     case json_tokener_state_object_field_start_after_sep:
 760       if(c == '}') {
 761                 if (state == json_tokener_state_object_field_start_after_sep &&
 762                     (tok->flags & JSON_TOKENER_STRICT))
 763                 {
 764                         tok->err = json_tokener_error_parse_unexpected;
 765                         goto out;
 766                 }
 767         saved_state = json_tokener_state_finish;
 768         state = json_tokener_state_eatws;
 769       } else if (c == '"' || c == '\'') {
 770         tok->quote_char = c;
 771         printbuf_reset(tok->pb);
 772         state = json_tokener_state_object_field;
 773       } else {
 774         tok->err = json_tokener_error_parse_object_key_name;
 775         goto out;
 776       }
 777       break;
 778
 779     case json_tokener_state_object_field:
 780       {
 781         /* Advance until we change state */
 782         const char *case_start = str;
 783         while(1) {
 784           if(c == tok->quote_char) {
 785             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 786             obj_field_name = strdup(tok->pb->buf);
 787             saved_state = json_tokener_state_object_field_end;
 788             state = json_tokener_state_eatws;
 789             break;
 790           } else if(c == '\\') {
 791             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 792             saved_state = json_tokener_state_object_field;
 793             state = json_tokener_state_string_escape;
 794             break;
 795           }
 796           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 797             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 798             goto out;
 799           }
 800         }
 801       }
 802       break;
 803
 804     case json_tokener_state_object_field_end:
 805       if(c == ':') {
 806         saved_state = json_tokener_state_object_value;
 807         state = json_tokener_state_eatws;
 808       } else {
 809         tok->err = json_tokener_error_parse_object_key_sep;
 810         goto out;
 811       }
 812       break;
 813
 814     case json_tokener_state_object_value:
 815       if(tok->depth >= tok->max_depth-1) {
 816         tok->err = json_tokener_error_depth;
 817         goto out;
 818       }
 819       state = json_tokener_state_object_value_add;
 820       tok->depth++;
 821       json_tokener_reset_level(tok, tok->depth);
 822       goto redo_char;
 823
 824     case json_tokener_state_object_value_add:
 825       json_object_object_add(current, obj_field_name, obj);
 826       free(obj_field_name);
 827       obj_field_name = NULL;
 828       saved_state = json_tokener_state_object_sep;
 829       state = json_tokener_state_eatws;
 830       goto redo_char;
 831
 832     case json_tokener_state_object_sep:
 833       if(c == '}') {
 834         saved_state = json_tokener_state_finish;
 835         state = json_tokener_state_eatws;
 836       } else if(c == ',') {
 837         saved_state = json_tokener_state_object_field_start_after_sep;
 838         state = json_tokener_state_eatws;
 839       } else {
 840         tok->err = json_tokener_error_parse_object_value_sep;
 841         goto out;
 842       }
 843       break;
 844
 845     }
 846     if (!ADVANCE_CHAR(str, tok))
 847       goto out;
 848   } /* while(POP_CHAR) */
 849
 850  out:
 851   if (c &&
 852      (state == json_tokener_state_finish) &&
 853      (tok->depth == 0) &&
 854      (tok->flags & JSON_TOKENER_STRICT)) {
 855       /* unexpected char after JSON data */
 856       tok->err = json_tokener_error_parse_unexpected;
 857   }
 858   if (!c) { /* We hit an eof char (0) */
 859     if(state != json_tokener_state_finish &&
 860        saved_state != json_tokener_state_finish)
 861       tok->err = json_tokener_error_parse_eof;
 862   }
 863
 864 #ifdef HAVE_SETLOCALE
 865   setlocale(LC_NUMERIC, oldlocale);
 866   if (oldlocale) free(oldlocale);
 867 #endif
 868
 869   if (tok->err == json_tokener_success)
 870   {
 871     json_object *ret = json_object_get(current);
 872         int ii;
 873
 874         /* Partially reset, so we parse additional objects on subsequent calls. */
 875     for(ii = tok->depth; ii >= 0; ii--)
 876       json_tokener_reset_level(tok, ii);
 877     return ret;
 878   }
 879
 880   MC_DEBUG("json_tokener_parse_ex: error %s at offset %d\n",
 881            json_tokener_errors[tok->err], tok->char_offset);
 882   return NULL;
 883 }
 884
 885 void json_tokener_set_flags(struct json_tokener *tok, int flags)
 886 {
 887         tok->flags = flags;
 888 }