json_tokener.c

   1 /*
   2  * $Id: json_tokener.c,v 1.20 2006/07/25 03:24:50 mclark Exp $
   3  *
   4  * Copyright (c) 2004, 2005 Metaparadigm Pte. Ltd.
   5  * Michael Clark <michael@metaparadigm.com>
   6  *
   7  * This library is free software; you can redistribute it and/or modify
   8  * it under the terms of the MIT license. See COPYING for details.
   9  *
  10  *
  11  * Copyright (c) 2008-2009 Yahoo! Inc.  All rights reserved.
  12  * The copyrights to the contents of this file are licensed under the MIT License
  13  * (http://www.opensource.org/licenses/mit-license.php)
  14  */
  15
  16 #include "config.h"
  17
  18 #include <math.h>
  19 #include <stdio.h>
  20 #include <stdlib.h>
  21 #include <stddef.h>
  22 #include <ctype.h>
  23 #include <string.h>
  24 #include <limits.h>
  25
  26 #include "bits.h"
  27 #include "debug.h"
  28 #include "printbuf.h"
  29 #include "arraylist.h"
  30 #include "json_inttypes.h"
  31 #include "json_object.h"
  32 #include "json_tokener.h"
  33 #include "json_util.h"
  34
  35 #ifdef HAVE_LOCALE_H
  36 #include <locale.h>
  37 #endif /* HAVE_LOCALE_H */
  38
  39 #if !HAVE_STRDUP && defined(_MSC_VER)
  40   /* MSC has the version as _strdup */
  41 # define strdup _strdup
  42 #elif !HAVE_STRDUP
  43 # error You do not have strdup on your system.
  44 #endif /* HAVE_STRDUP */
  45
  46 #if !HAVE_STRNCASECMP && defined(_MSC_VER)
  47   /* MSC has the version as _strnicmp */
  48 # define strncasecmp _strnicmp
  49 #elif !HAVE_STRNCASECMP
  50 # error You do not have strncasecmp on your system.
  51 #endif /* HAVE_STRNCASECMP */
  52
  53 /* Use C99 NAN by default; if not available, nan("") should work too. */
  54 #ifndef NAN
  55 #define NAN nan("")
  56 #endif /* !NAN */
  57
  58 static const char json_null_str[] = "null";
  59 static const int json_null_str_len = sizeof(json_null_str) - 1;
  60 static const char json_inf_str[] = "Infinity";
  61 static const int json_inf_str_len = sizeof(json_inf_str) - 1;
  62 static const char json_nan_str[] = "NaN";
  63 static const int json_nan_str_len = sizeof(json_nan_str) - 1;
  64 static const char json_true_str[] = "true";
  65 static const int json_true_str_len = sizeof(json_true_str) - 1;
  66 static const char json_false_str[] = "false";
  67 static const int json_false_str_len = sizeof(json_false_str) - 1;
  68
  69 static const char* json_tokener_errors[] = {
  70   "success",
  71   "continue",
  72   "nesting too deep",
  73   "unexpected end of data",
  74   "unexpected character",
  75   "null expected",
  76   "boolean expected",
  77   "number expected",
  78   "array value separator ',' expected",
  79   "quoted object property name expected",
  80   "object property name separator ':' expected",
  81   "object value separator ',' expected",
  82   "invalid string sequence",
  83   "expected comment",
  84   "buffer size overflow"
  85 };
  86
  87 const char *json_tokener_error_desc(enum json_tokener_error jerr)
  88 {
  89         int jerr_int = (int)jerr;
  90         if (jerr_int < 0 || jerr_int >= (int)(sizeof(json_tokener_errors) / sizeof(json_tokener_errors[0])))
  91                 return "Unknown error, invalid json_tokener_error value passed to json_tokener_error_desc()";
  92         return json_tokener_errors[jerr];
  93 }
  94
  95 enum json_tokener_error json_tokener_get_error(json_tokener *tok)
  96 {
  97         return tok->err;
  98 }
  99
 100 /* Stuff for decoding unicode sequences */
 101 #define IS_HIGH_SURROGATE(uc) (((uc) & 0xFC00) == 0xD800)
 102 #define IS_LOW_SURROGATE(uc)  (((uc) & 0xFC00) == 0xDC00)
 103 #define DECODE_SURROGATE_PAIR(hi,lo) ((((hi) & 0x3FF) << 10) + ((lo) & 0x3FF) + 0x10000)
 104 static unsigned char utf8_replacement_char[3] = { 0xEF, 0xBF, 0xBD };
 105
 106 struct json_tokener* json_tokener_new_ex(int depth)
 107 {
 108   struct json_tokener *tok;
 109
 110   tok = (struct json_tokener*)calloc(1, sizeof(struct json_tokener));
 111   if (!tok) return NULL;
 112   tok->stack = (struct json_tokener_srec *)calloc(depth, sizeof(struct json_tokener_srec));
 113   if (!tok->stack) {
 114     free(tok);
 115     return NULL;
 116   }
 117   tok->pb = printbuf_new();
 118   tok->max_depth = depth;
 119   json_tokener_reset(tok);
 120   return tok;
 121 }
 122
 123 struct json_tokener* json_tokener_new(void)
 124 {
 125   return json_tokener_new_ex(JSON_TOKENER_DEFAULT_DEPTH);
 126 }
 127
 128 void json_tokener_free(struct json_tokener *tok)
 129 {
 130   json_tokener_reset(tok);
 131   if (tok->pb) printbuf_free(tok->pb);
 132   if (tok->stack) free(tok->stack);
 133   free(tok);
 134 }
 135
 136 static void json_tokener_reset_level(struct json_tokener *tok, int depth)
 137 {
 138   tok->stack[depth].state = json_tokener_state_eatws;
 139   tok->stack[depth].saved_state = json_tokener_state_start;
 140   json_object_put(tok->stack[depth].current);
 141   tok->stack[depth].current = NULL;
 142   free(tok->stack[depth].obj_field_name);
 143   tok->stack[depth].obj_field_name = NULL;
 144 }
 145
 146 void json_tokener_reset(struct json_tokener *tok)
 147 {
 148   int i;
 149   if (!tok)
 150     return;
 151
 152   for(i = tok->depth; i >= 0; i--)
 153     json_tokener_reset_level(tok, i);
 154   tok->depth = 0;
 155   tok->err = json_tokener_success;
 156 }
 157
 158 struct json_object* json_tokener_parse(const char *str)
 159 {
 160     enum json_tokener_error jerr_ignored;
 161     struct json_object* obj;
 162     obj = json_tokener_parse_verbose(str, &jerr_ignored);
 163     return obj;
 164 }
 165
 166 struct json_object* json_tokener_parse_verbose(const char *str, enum json_tokener_error *error)
 167 {
 168     struct json_tokener* tok;
 169     struct json_object* obj;
 170
 171     tok = json_tokener_new();
 172     if (!tok)
 173       return NULL;
 174     obj = json_tokener_parse_ex(tok, str, -1);
 175     *error = tok->err;
 176     if(tok->err != json_tokener_success) {
 177                 if (obj != NULL)
 178                         json_object_put(obj);
 179         obj = NULL;
 180     }
 181
 182     json_tokener_free(tok);
 183     return obj;
 184 }
 185
 186 #define state  tok->stack[tok->depth].state
 187 #define saved_state  tok->stack[tok->depth].saved_state
 188 #define current tok->stack[tok->depth].current
 189 #define obj_field_name tok->stack[tok->depth].obj_field_name
 190
 191 /* Optimization:
 192  * json_tokener_parse_ex() consumed a lot of CPU in its main loop,
 193  * iterating character-by character.  A large performance boost is
 194  * achieved by using tighter loops to locally handle units such as
 195  * comments and strings.  Loops that handle an entire token within
 196  * their scope also gather entire strings and pass them to
 197  * printbuf_memappend() in a single call, rather than calling
 198  * printbuf_memappend() one char at a time.
 199  *
 200  * PEEK_CHAR() and ADVANCE_CHAR() macros are used for code that is
 201  * common to both the main loop and the tighter loops.
 202  */
 203
 204 /* PEEK_CHAR(dest, tok) macro:
 205  *   Peeks at the current char and stores it in dest.
 206  *   Returns 1 on success, sets tok->err and returns 0 if no more chars.
 207  *   Implicit inputs:  str, len vars
 208  */
 209 #define PEEK_CHAR(dest, tok)                                                  \
 210   (((tok)->char_offset == len) ?                                          \
 211    (((tok)->depth == 0 && state == json_tokener_state_eatws && saved_state == json_tokener_state_finish) ? \
 212     (((tok)->err = json_tokener_success), 0)                              \
 213     :                                                                   \
 214     (((tok)->err = json_tokener_continue), 0)                             \
 215     ) :                                                                 \
 216    (((dest) = *str), 1)                                                 \
 217    )
 218
 219 /* ADVANCE_CHAR() macro:
 220  *   Incrementes str & tok->char_offset.
 221  *   For convenience of existing conditionals, returns the old value of c (0 on eof)
 222  *   Implicit inputs:  c var
 223  */
 224 #define ADVANCE_CHAR(str, tok) \
 225   ( ++(str), ((tok)->char_offset)++, c)
 226
 227
 228 /* End optimization macro defs */
 229
 230
 231 struct json_object* json_tokener_parse_ex(struct json_tokener *tok,
 232                                           const char *str, int len)
 233 {
 234   struct json_object *obj = NULL;
 235   char c = '\1';
 236 #ifdef HAVE_SETLOCALE
 237   char *oldlocale=NULL, *tmplocale;
 238
 239   tmplocale = setlocale(LC_NUMERIC, NULL);
 240   if (tmplocale) oldlocale = strdup(tmplocale);
 241   setlocale(LC_NUMERIC, "C");
 242 #endif
 243
 244   tok->char_offset = 0;
 245   tok->err = json_tokener_success;
 246
 247   /* this interface is presently not 64-bit clean due to the int len argument
 248      and the internal printbuf interface that takes 32-bit int len arguments
 249      so the function limits the maximum string size to INT32_MAX (2GB).
 250      If the function is called with len == -1 then strlen is called to check
 251      the string length is less than INT32_MAX (2GB) */
 252   if ((len < -1) || (len == -1 && strlen(str) > INT32_MAX)) {
 253     tok->err = json_tokener_error_size;
 254     return NULL;
 255   }
 256
 257   while (PEEK_CHAR(c, tok)) {
 258
 259   redo_char:
 260     switch(state) {
 261
 262     case json_tokener_state_eatws:
 263       /* Advance until we change state */
 264       while (isspace((int)c)) {
 265         if ((!ADVANCE_CHAR(str, tok)) || (!PEEK_CHAR(c, tok)))
 266           goto out;
 267       }
 268       if(c == '/' && !(tok->flags & JSON_TOKENER_STRICT)) {
 269         printbuf_reset(tok->pb);
 270         printbuf_memappend_fast(tok->pb, &c, 1);
 271         state = json_tokener_state_comment_start;
 272       } else {
 273         state = saved_state;
 274         goto redo_char;
 275       }
 276       break;
 277
 278     case json_tokener_state_start:
 279       switch(c) {
 280       case '{':
 281         state = json_tokener_state_eatws;
 282         saved_state = json_tokener_state_object_field_start;
 283         current = json_object_new_object();
 284         break;
 285       case '[':
 286         state = json_tokener_state_eatws;
 287         saved_state = json_tokener_state_array;
 288         current = json_object_new_array();
 289         break;
 290       case 'I':
 291       case 'i':
 292         state = json_tokener_state_inf;
 293         printbuf_reset(tok->pb);
 294         tok->st_pos = 0;
 295         goto redo_char;
 296       case 'N':
 297       case 'n':
 298         state = json_tokener_state_null; // or NaN
 299         printbuf_reset(tok->pb);
 300         tok->st_pos = 0;
 301         goto redo_char;
 302       case '\'':
 303         if (tok->flags & JSON_TOKENER_STRICT) {
 304             /* in STRICT mode only double-quote are allowed */
 305             tok->err = json_tokener_error_parse_unexpected;
 306             goto out;
 307         }
 308       case '"':
 309         state = json_tokener_state_string;
 310         printbuf_reset(tok->pb);
 311         tok->quote_char = c;
 312         break;
 313       case 'T':
 314       case 't':
 315       case 'F':
 316       case 'f':
 317         state = json_tokener_state_boolean;
 318         printbuf_reset(tok->pb);
 319         tok->st_pos = 0;
 320         goto redo_char;
 321 #if defined(__GNUC__)
 322           case '0' ... '9':
 323 #else
 324           case '0':
 325       case '1':
 326       case '2':
 327       case '3':
 328       case '4':
 329       case '5':
 330       case '6':
 331       case '7':
 332       case '8':
 333       case '9':
 334 #endif
 335       case '-':
 336         state = json_tokener_state_number;
 337         printbuf_reset(tok->pb);
 338         tok->is_double = 0;
 339         goto redo_char;
 340       default:
 341         tok->err = json_tokener_error_parse_unexpected;
 342         goto out;
 343       }
 344       break;
 345
 346     case json_tokener_state_finish:
 347       if(tok->depth == 0) goto out;
 348       obj = json_object_get(current);
 349       json_tokener_reset_level(tok, tok->depth);
 350       tok->depth--;
 351       goto redo_char;
 352
 353     case json_tokener_state_inf: /* aka starts with 'i' */
 354       {
 355         int size_inf;
 356         int is_negative = 0;
 357
 358         printbuf_memappend_fast(tok->pb, &c, 1);
 359         size_inf = json_min(tok->st_pos+1, json_inf_str_len);
 360         char *infbuf = tok->pb->buf;
 361         if (*infbuf == '-')
 362         {
 363                 infbuf++;
 364                 is_negative = 1;
 365         }
 366         if ((!(tok->flags & JSON_TOKENER_STRICT) &&
 367                   strncasecmp(json_inf_str, infbuf, size_inf) == 0) ||
 368                  (strncmp(json_inf_str, infbuf, size_inf) == 0)
 369                 )
 370         {
 371                 if (tok->st_pos == json_inf_str_len)
 372                 {
 373                         current = json_object_new_double(is_negative ? -INFINITY : INFINITY);
 374                         saved_state = json_tokener_state_finish;
 375                         state = json_tokener_state_eatws;
 376                         goto redo_char;
 377                 }
 378         } else {
 379                 tok->err = json_tokener_error_parse_unexpected;
 380                 goto out;
 381         }
 382         tok->st_pos++;
 383       }
 384       break;
 385     case json_tokener_state_null: /* aka starts with 'n' */
 386       {
 387         int size;
 388         int size_nan;
 389         printbuf_memappend_fast(tok->pb, &c, 1);
 390         size = json_min(tok->st_pos+1, json_null_str_len);
 391         size_nan = json_min(tok->st_pos+1, json_nan_str_len);
 392         if((!(tok->flags & JSON_TOKENER_STRICT) &&
 393           strncasecmp(json_null_str, tok->pb->buf, size) == 0)
 394           || (strncmp(json_null_str, tok->pb->buf, size) == 0)
 395           ) {
 396           if (tok->st_pos == json_null_str_len) {
 397             current = NULL;
 398             saved_state = json_tokener_state_finish;
 399             state = json_tokener_state_eatws;
 400             goto redo_char;
 401           }
 402         }
 403         else if ((!(tok->flags & JSON_TOKENER_STRICT) &&
 404                   strncasecmp(json_nan_str, tok->pb->buf, size_nan) == 0) ||
 405                  (strncmp(json_nan_str, tok->pb->buf, size_nan) == 0)
 406                 )
 407         {
 408                 if (tok->st_pos == json_nan_str_len)
 409                 {
 410                         current = json_object_new_double(NAN);
 411                         saved_state = json_tokener_state_finish;
 412                         state = json_tokener_state_eatws;
 413                         goto redo_char;
 414                 }
 415         } else {
 416           tok->err = json_tokener_error_parse_null;
 417           goto out;
 418         }
 419         tok->st_pos++;
 420       }
 421       break;
 422
 423     case json_tokener_state_comment_start:
 424       if(c == '*') {
 425         state = json_tokener_state_comment;
 426       } else if(c == '/') {
 427         state = json_tokener_state_comment_eol;
 428       } else {
 429         tok->err = json_tokener_error_parse_comment;
 430         goto out;
 431       }
 432       printbuf_memappend_fast(tok->pb, &c, 1);
 433       break;
 434
 435     case json_tokener_state_comment:
 436               {
 437           /* Advance until we change state */
 438           const char *case_start = str;
 439           while(c != '*') {
 440             if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 441               printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 442               goto out;
 443             }
 444           }
 445           printbuf_memappend_fast(tok->pb, case_start, 1+str-case_start);
 446           state = json_tokener_state_comment_end;
 447         }
 448             break;
 449
 450     case json_tokener_state_comment_eol:
 451       {
 452         /* Advance until we change state */
 453         const char *case_start = str;
 454         while(c != '\n') {
 455           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 456             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 457             goto out;
 458           }
 459         }
 460         printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 461         MC_DEBUG("json_tokener_comment: %s\n", tok->pb->buf);
 462         state = json_tokener_state_eatws;
 463       }
 464       break;
 465
 466     case json_tokener_state_comment_end:
 467       printbuf_memappend_fast(tok->pb, &c, 1);
 468       if(c == '/') {
 469         MC_DEBUG("json_tokener_comment: %s\n", tok->pb->buf);
 470         state = json_tokener_state_eatws;
 471       } else {
 472         state = json_tokener_state_comment;
 473       }
 474       break;
 475
 476     case json_tokener_state_string:
 477       {
 478         /* Advance until we change state */
 479         const char *case_start = str;
 480         while(1) {
 481           if(c == tok->quote_char) {
 482             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 483             current = json_object_new_string_len(tok->pb->buf, tok->pb->bpos);
 484             saved_state = json_tokener_state_finish;
 485             state = json_tokener_state_eatws;
 486             break;
 487           } else if(c == '\\') {
 488             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 489             saved_state = json_tokener_state_string;
 490             state = json_tokener_state_string_escape;
 491             break;
 492           }
 493           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 494             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 495             goto out;
 496           }
 497         }
 498       }
 499       break;
 500
 501     case json_tokener_state_string_escape:
 502       switch(c) {
 503       case '"':
 504       case '\\':
 505       case '/':
 506         printbuf_memappend_fast(tok->pb, &c, 1);
 507         state = saved_state;
 508         break;
 509       case 'b':
 510       case 'n':
 511       case 'r':
 512       case 't':
 513       case 'f':
 514         if(c == 'b') printbuf_memappend_fast(tok->pb, "\b", 1);
 515         else if(c == 'n') printbuf_memappend_fast(tok->pb, "\n", 1);
 516         else if(c == 'r') printbuf_memappend_fast(tok->pb, "\r", 1);
 517         else if(c == 't') printbuf_memappend_fast(tok->pb, "\t", 1);
 518         else if(c == 'f') printbuf_memappend_fast(tok->pb, "\f", 1);
 519         state = saved_state;
 520         break;
 521       case 'u':
 522         tok->ucs_char = 0;
 523         tok->st_pos = 0;
 524         state = json_tokener_state_escape_unicode;
 525         break;
 526       default:
 527         tok->err = json_tokener_error_parse_string;
 528         goto out;
 529       }
 530       break;
 531
 532     case json_tokener_state_escape_unicode:
 533         {
 534           unsigned int got_hi_surrogate = 0;
 535
 536           /* Handle a 4-byte sequence, or two sequences if a surrogate pair */
 537           while(1) {
 538             if(strchr(json_hex_chars, c)) {
 539               tok->ucs_char += ((unsigned int)hexdigit(c) << ((3-tok->st_pos++)*4));
 540               if(tok->st_pos == 4) {
 541                 unsigned char unescaped_utf[4];
 542
 543                 if (got_hi_surrogate) {
 544                   if (IS_LOW_SURROGATE(tok->ucs_char)) {
 545                     /* Recalculate the ucs_char, then fall thru to process normally */
 546                     tok->ucs_char = DECODE_SURROGATE_PAIR(got_hi_surrogate, tok->ucs_char);
 547                   } else {
 548                     /* Hi surrogate was not followed by a low surrogate */
 549                     /* Replace the hi and process the rest normally */
 550                     printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 551                   }
 552                   got_hi_surrogate = 0;
 553                 }
 554
 555                 if (tok->ucs_char < 0x80) {
 556                   unescaped_utf[0] = tok->ucs_char;
 557                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 1);
 558                 } else if (tok->ucs_char < 0x800) {
 559                   unescaped_utf[0] = 0xc0 | (tok->ucs_char >> 6);
 560                   unescaped_utf[1] = 0x80 | (tok->ucs_char & 0x3f);
 561                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 2);
 562                 } else if (IS_HIGH_SURROGATE(tok->ucs_char)) {
 563                   /* Got a high surrogate.  Remember it and look for the
 564                    * the beginning of another sequence, which should be the
 565                    * low surrogate.
 566                    */
 567                   got_hi_surrogate = tok->ucs_char;
 568                   /* Not at end, and the next two chars should be "\u" */
 569                   if ((tok->char_offset+1 != len) &&
 570                       (tok->char_offset+2 != len) &&
 571                       (str[1] == '\\') &&
 572                       (str[2] == 'u'))
 573                   {
 574                 /* Advance through the 16 bit surrogate, and move on to the
 575                  * next sequence. The next step is to process the following
 576                  * characters.
 577                  */
 578                     if( !ADVANCE_CHAR(str, tok) || !ADVANCE_CHAR(str, tok) ) {
 579                     printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 580                 }
 581                     /* Advance to the first char of the next sequence and
 582                      * continue processing with the next sequence.
 583                      */
 584                     if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 585                       printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 586                       goto out;
 587                     }
 588                     tok->ucs_char = 0;
 589                     tok->st_pos = 0;
 590                     continue; /* other json_tokener_state_escape_unicode */
 591                   } else {
 592                     /* Got a high surrogate without another sequence following
 593                      * it.  Put a replacement char in for the hi surrogate
 594                      * and pretend we finished.
 595                      */
 596                     printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 597                   }
 598                 } else if (IS_LOW_SURROGATE(tok->ucs_char)) {
 599                   /* Got a low surrogate not preceded by a high */
 600                   printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 601                 } else if (tok->ucs_char < 0x10000) {
 602                   unescaped_utf[0] = 0xe0 | (tok->ucs_char >> 12);
 603                   unescaped_utf[1] = 0x80 | ((tok->ucs_char >> 6) & 0x3f);
 604                   unescaped_utf[2] = 0x80 | (tok->ucs_char & 0x3f);
 605                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 3);
 606                 } else if (tok->ucs_char < 0x110000) {
 607                   unescaped_utf[0] = 0xf0 | ((tok->ucs_char >> 18) & 0x07);
 608                   unescaped_utf[1] = 0x80 | ((tok->ucs_char >> 12) & 0x3f);
 609                   unescaped_utf[2] = 0x80 | ((tok->ucs_char >> 6) & 0x3f);
 610                   unescaped_utf[3] = 0x80 | (tok->ucs_char & 0x3f);
 611                   printbuf_memappend_fast(tok->pb, (char*)unescaped_utf, 4);
 612                 } else {
 613                   /* Don't know what we got--insert the replacement char */
 614                   printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 615                 }
 616                 state = saved_state;
 617                 break;
 618               }
 619             } else {
 620               tok->err = json_tokener_error_parse_string;
 621               goto out;
 622             }
 623           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 624             if (got_hi_surrogate) /* Clean up any pending chars */
 625               printbuf_memappend_fast(tok->pb, (char*)utf8_replacement_char, 3);
 626             goto out;
 627           }
 628         }
 629       }
 630       break;
 631
 632     case json_tokener_state_boolean:
 633       {
 634         int size1, size2;
 635         printbuf_memappend_fast(tok->pb, &c, 1);
 636         size1 = json_min(tok->st_pos+1, json_true_str_len);
 637         size2 = json_min(tok->st_pos+1, json_false_str_len);
 638         if((!(tok->flags & JSON_TOKENER_STRICT) &&
 639           strncasecmp(json_true_str, tok->pb->buf, size1) == 0)
 640           || (strncmp(json_true_str, tok->pb->buf, size1) == 0)
 641           ) {
 642           if(tok->st_pos == json_true_str_len) {
 643             current = json_object_new_boolean(1);
 644             saved_state = json_tokener_state_finish;
 645             state = json_tokener_state_eatws;
 646             goto redo_char;
 647           }
 648         } else if((!(tok->flags & JSON_TOKENER_STRICT) &&
 649           strncasecmp(json_false_str, tok->pb->buf, size2) == 0)
 650           || (strncmp(json_false_str, tok->pb->buf, size2) == 0)) {
 651           if(tok->st_pos == json_false_str_len) {
 652             current = json_object_new_boolean(0);
 653             saved_state = json_tokener_state_finish;
 654             state = json_tokener_state_eatws;
 655             goto redo_char;
 656           }
 657         } else {
 658           tok->err = json_tokener_error_parse_boolean;
 659           goto out;
 660         }
 661         tok->st_pos++;
 662       }
 663       break;
 664
 665     case json_tokener_state_number:
 666       {
 667         /* Advance until we change state */
 668         const char *case_start = str;
 669         int case_len=0;
 670         while(c && strchr(json_number_chars, c)) {
 671           ++case_len;
 672           if(c == '.' || c == 'e' || c == 'E')
 673             tok->is_double = 1;
 674           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 675             printbuf_memappend_fast(tok->pb, case_start, case_len);
 676             goto out;
 677           }
 678         }
 679         if (case_len>0)
 680           printbuf_memappend_fast(tok->pb, case_start, case_len);
 681
 682         // Check for -Infinity
 683         if (tok->pb->buf[0] == '-' && case_len == 1 &&
 684             (c == 'i' || c == 'I'))
 685         {
 686                 state = json_tokener_state_inf;
 687                 goto redo_char;
 688         }
 689       }
 690       {
 691         int64_t num64;
 692         double  numd;
 693         if (!tok->is_double && json_parse_int64(tok->pb->buf, &num64) == 0) {
 694                 if (num64 && tok->pb->buf[0]=='0' && (tok->flags & JSON_TOKENER_STRICT)) {
 695                         /* in strict mode, number must not start with 0 */
 696                         tok->err = json_tokener_error_parse_number;
 697                         goto out;
 698                 }
 699                 current = json_object_new_int64(num64);
 700         }
 701         else if(tok->is_double && json_parse_double(tok->pb->buf, &numd) == 0)
 702         {
 703           current = json_object_new_double_s(numd, tok->pb->buf);
 704         } else {
 705           tok->err = json_tokener_error_parse_number;
 706           goto out;
 707         }
 708         saved_state = json_tokener_state_finish;
 709         state = json_tokener_state_eatws;
 710         goto redo_char;
 711       }
 712       break;
 713
 714     case json_tokener_state_array_after_sep:
 715     case json_tokener_state_array:
 716       if(c == ']') {
 717                 if (state == json_tokener_state_array_after_sep &&
 718                         (tok->flags & JSON_TOKENER_STRICT))
 719                 {
 720                         tok->err = json_tokener_error_parse_unexpected;
 721                         goto out;
 722                 }
 723         saved_state = json_tokener_state_finish;
 724         state = json_tokener_state_eatws;
 725       } else {
 726         if(tok->depth >= tok->max_depth-1) {
 727           tok->err = json_tokener_error_depth;
 728           goto out;
 729         }
 730         state = json_tokener_state_array_add;
 731         tok->depth++;
 732         json_tokener_reset_level(tok, tok->depth);
 733         goto redo_char;
 734       }
 735       break;
 736
 737     case json_tokener_state_array_add:
 738       json_object_array_add(current, obj);
 739       saved_state = json_tokener_state_array_sep;
 740       state = json_tokener_state_eatws;
 741       goto redo_char;
 742
 743     case json_tokener_state_array_sep:
 744       if(c == ']') {
 745         saved_state = json_tokener_state_finish;
 746         state = json_tokener_state_eatws;
 747       } else if(c == ',') {
 748         saved_state = json_tokener_state_array_after_sep;
 749         state = json_tokener_state_eatws;
 750       } else {
 751         tok->err = json_tokener_error_parse_array;
 752         goto out;
 753       }
 754       break;
 755
 756     case json_tokener_state_object_field_start:
 757     case json_tokener_state_object_field_start_after_sep:
 758       if(c == '}') {
 759                 if (state == json_tokener_state_object_field_start_after_sep &&
 760                     (tok->flags & JSON_TOKENER_STRICT))
 761                 {
 762                         tok->err = json_tokener_error_parse_unexpected;
 763                         goto out;
 764                 }
 765         saved_state = json_tokener_state_finish;
 766         state = json_tokener_state_eatws;
 767       } else if (c == '"' || c == '\'') {
 768         tok->quote_char = c;
 769         printbuf_reset(tok->pb);
 770         state = json_tokener_state_object_field;
 771       } else {
 772         tok->err = json_tokener_error_parse_object_key_name;
 773         goto out;
 774       }
 775       break;
 776
 777     case json_tokener_state_object_field:
 778       {
 779         /* Advance until we change state */
 780         const char *case_start = str;
 781         while(1) {
 782           if(c == tok->quote_char) {
 783             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 784             obj_field_name = strdup(tok->pb->buf);
 785             saved_state = json_tokener_state_object_field_end;
 786             state = json_tokener_state_eatws;
 787             break;
 788           } else if(c == '\\') {
 789             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 790             saved_state = json_tokener_state_object_field;
 791             state = json_tokener_state_string_escape;
 792             break;
 793           }
 794           if (!ADVANCE_CHAR(str, tok) || !PEEK_CHAR(c, tok)) {
 795             printbuf_memappend_fast(tok->pb, case_start, str-case_start);
 796             goto out;
 797           }
 798         }
 799       }
 800       break;
 801
 802     case json_tokener_state_object_field_end:
 803       if(c == ':') {
 804         saved_state = json_tokener_state_object_value;
 805         state = json_tokener_state_eatws;
 806       } else {
 807         tok->err = json_tokener_error_parse_object_key_sep;
 808         goto out;
 809       }
 810       break;
 811
 812     case json_tokener_state_object_value:
 813       if(tok->depth >= tok->max_depth-1) {
 814         tok->err = json_tokener_error_depth;
 815         goto out;
 816       }
 817       state = json_tokener_state_object_value_add;
 818       tok->depth++;
 819       json_tokener_reset_level(tok, tok->depth);
 820       goto redo_char;
 821
 822     case json_tokener_state_object_value_add:
 823       json_object_object_add(current, obj_field_name, obj);
 824       free(obj_field_name);
 825       obj_field_name = NULL;
 826       saved_state = json_tokener_state_object_sep;
 827       state = json_tokener_state_eatws;
 828       goto redo_char;
 829
 830     case json_tokener_state_object_sep:
 831       if(c == '}') {
 832         saved_state = json_tokener_state_finish;
 833         state = json_tokener_state_eatws;
 834       } else if(c == ',') {
 835         saved_state = json_tokener_state_object_field_start_after_sep;
 836         state = json_tokener_state_eatws;
 837       } else {
 838         tok->err = json_tokener_error_parse_object_value_sep;
 839         goto out;
 840       }
 841       break;
 842
 843     }
 844     if (!ADVANCE_CHAR(str, tok))
 845       goto out;
 846   } /* while(POP_CHAR) */
 847
 848  out:
 849   if (c &&
 850      (state == json_tokener_state_finish) &&
 851      (tok->depth == 0) &&
 852      (tok->flags & JSON_TOKENER_STRICT)) {
 853       /* unexpected char after JSON data */
 854       tok->err = json_tokener_error_parse_unexpected;
 855   }
 856   if (!c) { /* We hit an eof char (0) */
 857     if(state != json_tokener_state_finish &&
 858        saved_state != json_tokener_state_finish)
 859       tok->err = json_tokener_error_parse_eof;
 860   }
 861
 862 #ifdef HAVE_SETLOCALE
 863   setlocale(LC_NUMERIC, oldlocale);
 864   if (oldlocale) free(oldlocale);
 865 #endif
 866
 867   if (tok->err == json_tokener_success)
 868   {
 869     json_object *ret = json_object_get(current);
 870         int ii;
 871
 872         /* Partially reset, so we parse additional objects on subsequent calls. */
 873     for(ii = tok->depth; ii >= 0; ii--)
 874       json_tokener_reset_level(tok, ii);
 875     return ret;
 876   }
 877
 878   MC_DEBUG("json_tokener_parse_ex: error %s at offset %d\n",
 879            json_tokener_errors[tok->err], tok->char_offset);
 880   return NULL;
 881 }
 882
 883 void json_tokener_set_flags(struct json_tokener *tok, int flags)
 884 {
 885         tok->flags = flags;
 886 }