glib/gscanner.c

   1 /* GLIB - Library of useful routines for C programming
   2  * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
   3  *
   4  * GScanner: Flexible lexical scanner for general purpose.
   5  * Copyright (C) 1997, 1998 Tim Janik
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Library General Public
  18  * License along with this library; if not, write to the
  19  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  20  * Boston, MA 02111-1307, USA.
  21  */
  22 #define         __gscanner_c__
  23
  24 #include        <stdlib.h>
  25 #include        <stdarg.h>
  26 #include        <string.h>
  27 #include        <stdio.h>
  28 #include        <unistd.h>
  29 #include        <errno.h>
  30 #include        <sys/types.h>   /* needed for sys/stat.h */
  31 #include        <sys/stat.h>
  32 #include        "glib.h"
  33
  34
  35
  36 /* --- defines --- */
  37 #define to_lower(c)                             ( \
  38         (guchar) (                                                      \
  39           ( (((guchar)(c))>='A' && ((guchar)(c))<='Z') * ('a'-'A') ) |  \
  40           ( (((guchar)(c))>=192 && ((guchar)(c))<=214) * (224-192) ) |  \
  41           ( (((guchar)(c))>=216 && ((guchar)(c))<=222) * (248-216) ) |  \
  42           ((guchar)(c))                                                 \
  43         )                                                               \
  44 )
  45
  46
  47 /* --- typedefs --- */
  48 typedef struct  _GScannerHashVal        GScannerHashVal;
  49
  50 struct  _GScannerHashVal
  51 {
  52   gchar         *key;
  53   gpointer      value;
  54 };
  55
  56
  57
  58 /* --- variables --- */
  59 static  GScannerConfig  g_scanner_config_template =
  60 {
  61   (
  62    " \t\n"
  63    )                    /* cset_skip_characters */,
  64   (
  65    G_CSET_a_2_z
  66    "_"
  67    G_CSET_A_2_Z
  68    )                    /* cset_identifier_first */,
  69   (
  70    G_CSET_a_2_z
  71    "_0123456789"
  72    G_CSET_A_2_Z
  73    G_CSET_LATINS
  74    G_CSET_LATINC
  75    )                    /* cset_identifier_nth */,
  76   ( "#\n" )             /* cpair_comment_single */,
  77
  78   FALSE                 /* case_sensitive */,
  79
  80   TRUE                  /* skip_comment_multi */,
  81   TRUE                  /* skip_comment_single */,
  82   TRUE                  /* scan_comment_multi */,
  83   TRUE                  /* scan_identifier */,
  84   FALSE                 /* scan_identifier_1char */,
  85   FALSE                 /* scan_identifier_NULL */,
  86   TRUE                  /* scan_symbols */,
  87   FALSE                 /* scan_binary */,
  88   TRUE                  /* scan_octal */,
  89   TRUE                  /* scan_float */,
  90   TRUE                  /* scan_hex */,
  91   FALSE                 /* scan_hex_dollar */,
  92   TRUE                  /* scan_string_sq */,
  93   TRUE                  /* scan_string_dq */,
  94   TRUE                  /* numbers_2_int */,
  95   FALSE                 /* int_2_float */,
  96   FALSE                 /* identifier_2_string */,
  97   TRUE                  /* char_2_token */,
  98   FALSE                 /* symbol_2_token */,
  99 };
 100
 101
 102 /* --- prototypes --- */
 103 extern char* g_vsprintf (gchar *fmt, va_list *args, va_list *args2);
 104 static  GScannerHashVal* g_scanner_lookup_internal (GScanner    *scanner,
 105                                                     const gchar *symbol);
 106 static  void    g_scanner_get_token_ll  (GScanner       *scanner,
 107                                          GTokenType     *token_p,
 108                                          GValue         *value_p,
 109                                          guint          *line_p,
 110                                          guint          *position_p);
 111 static  void    g_scanner_get_token_i   (GScanner       *scanner,
 112                                          GTokenType     *token_p,
 113                                          GValue         *value_p,
 114                                          guint          *line_p,
 115                                          guint          *position_p);
 116 static  void    g_scanner_free_value    (GTokenType     *token_p,
 117                                          GValue         *value_p);
 118
 119 static  inline
 120 gint            g_scanner_char_2_num    (guchar         c,
 121                                          guchar         base);
 122 static  guchar  g_scanner_peek_next_char(GScanner       *scanner);
 123 static  guchar  g_scanner_get_char      (GScanner       *scanner,
 124                                          guint          *line_p,
 125                                          guint          *position_p);
 126 static  void    g_scanner_msg_handler   (GScanner       *scanner,
 127                                          gchar          *message,
 128                                          gint            is_error);
 129
 130
 131 /* --- functions --- */
 132 static gint
 133 g_scanner_char_2_num (guchar    c,
 134                       guchar    base)
 135 {
 136   if (c >= '0' && c <= '9')
 137     c -= '0';
 138   else if (c >= 'A' && c <= 'Z')
 139     c -= 'A' - 10;
 140   else if (c >= 'a' && c <= 'z')
 141     c -= 'a' - 10;
 142   else
 143     return -1;
 144
 145   if (c < base)
 146     return c;
 147
 148   return -1;
 149 }
 150
 151 GScanner*
 152 g_scanner_new (GScannerConfig   *config_templ)
 153 {
 154   register GScanner     *scanner;
 155
 156   if (!config_templ)
 157     config_templ = &g_scanner_config_template;
 158
 159   scanner = g_new0 (GScanner, 1);
 160
 161   scanner->user_data = NULL;
 162   scanner->max_parse_errors = 0;
 163   scanner->parse_errors = 0;
 164   scanner->input_name = NULL;
 165   scanner->derived_data = NULL;
 166
 167   scanner->config = g_new0 (GScannerConfig, 1);
 168
 169   scanner->config->case_sensitive       = config_templ->case_sensitive;
 170   scanner->config->cset_skip_characters = config_templ->cset_skip_characters;
 171   scanner->config->cset_identifier_first= config_templ->cset_identifier_first;
 172   scanner->config->cset_identifier_nth  = config_templ->cset_identifier_nth;
 173   scanner->config->cpair_comment_single = config_templ->cpair_comment_single;
 174   scanner->config->skip_comment_multi   = config_templ->skip_comment_multi;
 175   scanner->config->skip_comment_single  = config_templ->skip_comment_single;
 176   scanner->config->scan_comment_multi   = config_templ->scan_comment_multi;
 177   scanner->config->scan_identifier      = config_templ->scan_identifier;
 178   scanner->config->scan_identifier_1char= config_templ->scan_identifier_1char;
 179   scanner->config->scan_identifier_NULL = config_templ->scan_identifier_NULL;
 180   scanner->config->scan_symbols         = config_templ->scan_symbols;
 181   scanner->config->scan_binary          = config_templ->scan_binary;
 182   scanner->config->scan_octal           = config_templ->scan_octal;
 183   scanner->config->scan_float           = config_templ->scan_float;
 184   scanner->config->scan_hex             = config_templ->scan_hex;
 185   scanner->config->scan_hex_dollar      = config_templ->scan_hex_dollar;
 186   scanner->config->scan_string_sq       = config_templ->scan_string_sq;
 187   scanner->config->scan_string_dq       = config_templ->scan_string_dq;
 188   scanner->config->numbers_2_int        = config_templ->numbers_2_int;
 189   scanner->config->int_2_float          = config_templ->int_2_float;
 190   scanner->config->identifier_2_string  = config_templ->identifier_2_string;
 191   scanner->config->char_2_token         = config_templ->char_2_token;
 192   scanner->config->symbol_2_token       = config_templ->symbol_2_token;
 193
 194   scanner->token = G_TOKEN_NONE;
 195   scanner->value.v_int = 0;
 196   scanner->line = 1;
 197   scanner->position = 0;
 198
 199   scanner->next_token = G_TOKEN_NONE;
 200   scanner->next_value.v_int = 0;
 201   scanner->next_line = 1;
 202   scanner->next_position = 0;
 203
 204   scanner->symbol_table = g_hash_table_new (g_str_hash, g_str_equal);
 205   scanner->text = NULL;
 206   scanner->text_len = 0;
 207   scanner->input_fd = -1;
 208   scanner->peeked_char = -1;
 209
 210   scanner->msg_handler = g_scanner_msg_handler;
 211
 212   return scanner;
 213 }
 214
 215 static void
 216 g_scanner_destroy_symbol_table_entry (gpointer key,
 217                                       gpointer value,
 218                                       gpointer user_data)
 219 {
 220   g_free (key);
 221   g_free (value);
 222 }
 223
 224 void
 225 g_scanner_destroy (GScanner     *scanner)
 226 {
 227   g_return_if_fail (scanner != NULL);
 228
 229   g_hash_table_foreach (scanner->symbol_table,
 230                         g_scanner_destroy_symbol_table_entry, NULL);
 231   g_hash_table_destroy (scanner->symbol_table);
 232   g_scanner_free_value (&scanner->token, &scanner->value);
 233   g_scanner_free_value (&scanner->next_token, &scanner->next_value);
 234   g_free (scanner->config);
 235   g_free (scanner);
 236 }
 237
 238 static void
 239 g_scanner_msg_handler (GScanner         *scanner,
 240                        gchar            *message,
 241                        gint              is_error)
 242 {
 243   g_return_if_fail (scanner != NULL);
 244
 245   fprintf (stdout, "%s:%d: ", scanner->input_name, scanner->line);
 246   if (is_error)
 247     fprintf (stdout, "error: ");
 248   fprintf (stdout, "%s\n", message);
 249 }
 250
 251 void
 252 g_scanner_error (GScanner       *scanner,
 253                  const gchar    *format,
 254                  ...)
 255 {
 256   g_return_if_fail (scanner != NULL);
 257   g_return_if_fail (format != NULL);
 258
 259   scanner->parse_errors++;
 260
 261   if (scanner->msg_handler)
 262     {
 263       va_list args, args2;
 264       gchar *string;
 265
 266       va_start (args, format);
 267       va_start (args2, format);
 268       string = g_vsprintf ((gchar*) format, &args, &args2);
 269       va_end (args);
 270       va_end (args2);
 271
 272       string = g_strdup (string);
 273
 274       scanner->msg_handler (scanner, string, TRUE);
 275
 276       g_free (string);
 277     }
 278 }
 279
 280 void
 281 g_scanner_warn (GScanner       *scanner,
 282                 const gchar    *format,
 283                 ...)
 284 {
 285   g_return_if_fail (scanner != NULL);
 286   g_return_if_fail (format != NULL);
 287
 288   if (scanner->msg_handler)
 289     {
 290       va_list args, args2;
 291       gchar *string;
 292
 293       va_start (args, format);
 294       va_start (args2, format);
 295       string = g_vsprintf ((gchar*) format, &args, &args2);
 296       va_end (args);
 297       va_end (args2);
 298
 299       string = g_strdup (string);
 300
 301       scanner->msg_handler (scanner, string, FALSE);
 302
 303       g_free (string);
 304     }
 305 }
 306
 307 void
 308 g_scanner_input_file (GScanner  *scanner,
 309                       gint      input_fd)
 310 {
 311   g_return_if_fail (input_fd >= 0);
 312
 313   scanner->token = G_TOKEN_NONE;
 314   scanner->value.v_int = 0;
 315   scanner->line = 1;
 316   scanner->position = 0;
 317   scanner->next_token = G_TOKEN_NONE;
 318
 319   scanner->text = NULL;
 320   scanner->text_len = 0;
 321   scanner->input_fd = input_fd;
 322   scanner->peeked_char = -1;
 323 }
 324
 325 void
 326 g_scanner_input_text (GScanner       *scanner,
 327                       const  gchar   *text,
 328                       guint           text_len)
 329 {
 330   g_return_if_fail (text != NULL);
 331
 332   scanner->token = G_TOKEN_NONE;
 333   scanner->value.v_int = 0;
 334   scanner->line = 1;
 335   scanner->position = 0;
 336   scanner->next_token = G_TOKEN_NONE;
 337
 338   scanner->text = text;
 339   scanner->text_len = text_len;
 340   scanner->input_fd = -1;
 341   scanner->peeked_char = -1;
 342 }
 343
 344 void
 345 g_scanner_add_symbol (GScanner          *scanner,
 346                       const gchar       *symbol,
 347                       gpointer          value)
 348 {
 349   register GScannerHashVal      *hash_val;
 350
 351   g_return_if_fail (scanner != NULL);
 352   g_return_if_fail (symbol != NULL);
 353
 354   hash_val = g_scanner_lookup_internal (scanner, symbol);
 355
 356   if (!hash_val)
 357     {
 358       hash_val = g_new (GScannerHashVal, 1);
 359       hash_val->key = g_strdup (symbol);
 360       hash_val->value = value;
 361       if (!scanner->config->case_sensitive)
 362         {
 363           register guint        i, l;
 364
 365           l = strlen (hash_val->key);
 366           for (i = 0; i < l; i++)
 367             hash_val->key[i] = to_lower (hash_val->key[i]);
 368         }
 369       g_hash_table_insert (scanner->symbol_table, hash_val->key, hash_val);
 370     }
 371   else
 372     hash_val->value = value;
 373 }
 374
 375 gpointer
 376 g_scanner_lookup_symbol (GScanner       *scanner,
 377                          const gchar    *symbol)
 378 {
 379   register GScannerHashVal      *hash_val;
 380
 381   g_return_val_if_fail (scanner != NULL, NULL);
 382
 383   if (!symbol)
 384     return NULL;
 385
 386   hash_val = g_scanner_lookup_internal (scanner, symbol);
 387
 388   if (hash_val)
 389     return hash_val->value;
 390   else
 391     return NULL;
 392 }
 393
 394 static void
 395 g_scanner_foreach_internal (gpointer  key,
 396                             gpointer  value,
 397                             gpointer  user_data)
 398 {
 399   register GScannerHashVal *hash_val;
 400   register GHFunc func;
 401   register gpointer func_data;
 402   register gpointer *d;
 403
 404   d = user_data;
 405   func = (GHFunc)d[0];
 406   func_data = d[1];
 407   hash_val = value;
 408
 409   func (key, hash_val->value, func_data);
 410 }
 411
 412 void
 413 g_scanner_foreach_symbol (GScanner       *scanner,
 414                           GHFunc          func,
 415                           gpointer        func_data)
 416 {
 417   gpointer d[2];
 418
 419   g_return_if_fail (scanner != NULL);
 420
 421   d[0] = (gpointer)func;
 422   d[1] = func_data;
 423
 424   g_hash_table_foreach (scanner->symbol_table, g_scanner_foreach_internal, d);
 425 }
 426
 427 void
 428 g_scanner_remove_symbol (GScanner       *scanner,
 429                          const gchar    *symbol)
 430 {
 431   register GScannerHashVal      *hash_val;
 432
 433   g_return_if_fail (scanner != NULL);
 434
 435   hash_val = g_scanner_lookup_internal (scanner, symbol);
 436
 437   if (hash_val)
 438     {
 439       g_hash_table_remove (scanner->symbol_table, hash_val->key);
 440       g_free (hash_val->key);
 441       g_free (hash_val);
 442     }
 443 }
 444
 445 void
 446 g_scanner_freeze_symbol_table (GScanner *scanner)
 447 {
 448   g_return_if_fail (scanner != NULL);
 449
 450   g_hash_table_freeze (scanner->symbol_table);
 451 }
 452
 453 void
 454 g_scanner_thaw_symbol_table (GScanner *scanner)
 455 {
 456   g_return_if_fail (scanner != NULL);
 457
 458   g_hash_table_thaw (scanner->symbol_table);
 459 }
 460
 461 GTokenType
 462 g_scanner_peek_next_token (GScanner     *scanner)
 463 {
 464   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 465
 466   if (scanner->next_token == G_TOKEN_NONE)
 467     {
 468       scanner->next_line = scanner->line;
 469       scanner->next_position = scanner->position;
 470       g_scanner_get_token_i (scanner,
 471                              &scanner->next_token,
 472                              &scanner->next_value,
 473                              &scanner->next_line,
 474                              &scanner->next_position);
 475     }
 476
 477   return scanner->next_token;
 478 }
 479
 480 GTokenType
 481 g_scanner_get_next_token (GScanner      *scanner)
 482 {
 483   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 484
 485   if (scanner->next_token != G_TOKEN_NONE)
 486     {
 487       g_scanner_free_value (&scanner->token, &scanner->value);
 488
 489       scanner->token = scanner->next_token;
 490       scanner->value = scanner->next_value;
 491       scanner->line = scanner->next_line;
 492       scanner->position = scanner->next_position;
 493       scanner->next_token = G_TOKEN_NONE;
 494     }
 495   else
 496     g_scanner_get_token_i (scanner,
 497                            &scanner->token,
 498                            &scanner->value,
 499                            &scanner->line,
 500                            &scanner->position);
 501
 502   return scanner->token;
 503 }
 504
 505 GTokenType
 506 g_scanner_cur_token (GScanner *scanner)
 507 {
 508   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 509
 510   return scanner->token;
 511 }
 512
 513 GValue
 514 g_scanner_cur_value (GScanner *scanner)
 515 {
 516   register GValue v;
 517
 518   v.v_int = 0;
 519   g_return_val_if_fail (scanner != NULL, v);
 520
 521   return scanner->value;
 522 }
 523
 524 guint
 525 g_scanner_cur_line (GScanner *scanner)
 526 {
 527   g_return_val_if_fail (scanner != NULL, 0);
 528
 529   return scanner->line;
 530 }
 531
 532 guint
 533 g_scanner_cur_position (GScanner *scanner)
 534 {
 535   g_return_val_if_fail (scanner != NULL, 0);
 536
 537   return scanner->position;
 538 }
 539
 540 gboolean
 541 g_scanner_eof (GScanner *scanner)
 542 {
 543   g_return_val_if_fail (scanner != NULL, TRUE);
 544
 545   return scanner->token == G_TOKEN_EOF;
 546 }
 547
 548 static GScannerHashVal*
 549 g_scanner_lookup_internal (GScanner     *scanner,
 550                            const gchar  *symbol)
 551 {
 552   register GScannerHashVal      *hash_val;
 553
 554   if (!scanner->config->case_sensitive)
 555     {
 556       register gchar *buffer;
 557       register guint i, l;
 558
 559       l = strlen (symbol);
 560       buffer = g_new (gchar, l + 1);
 561       for (i = 0; i < l; i++)
 562         buffer[i] = to_lower (symbol[i]);
 563       buffer[i] = 0;
 564       hash_val = g_hash_table_lookup (scanner->symbol_table, buffer);
 565       g_free (buffer);
 566     }
 567   else
 568     hash_val = g_hash_table_lookup (scanner->symbol_table, (gchar*) symbol);
 569
 570   return hash_val;
 571 }
 572
 573 static guchar
 574 g_scanner_peek_next_char (GScanner *scanner)
 575 {
 576   guchar fchar;
 577
 578   if (scanner->text_len)
 579     {
 580       fchar = scanner->text[0];
 581     }
 582   else if (scanner->input_fd >= 0)
 583     {
 584       if (scanner->peeked_char < 0)
 585         {
 586           register gint count;
 587
 588           do
 589             {
 590               count = read (scanner->input_fd, &fchar, 1);
 591             }
 592           while (count == -1 &&
 593                  (errno == EINTR ||
 594                   errno == EAGAIN));
 595
 596           if (count != 1)
 597             fchar = 0;
 598
 599           scanner->peeked_char = fchar;
 600         }
 601       else
 602         fchar = scanner->peeked_char;
 603     }
 604   else
 605     fchar = 0;
 606
 607   return fchar;
 608 }
 609
 610 static guchar
 611 g_scanner_get_char (GScanner    *scanner,
 612                     guint       *line_p,
 613                     guint       *position_p)
 614 {
 615   guchar fchar;
 616
 617   if (scanner->text_len)
 618     {
 619       fchar = *(scanner->text++);
 620       scanner->text_len--;
 621     }
 622   else if (scanner->input_fd >= 0)
 623     {
 624       if (scanner->peeked_char < 0)
 625         {
 626           register gint count;
 627
 628           do
 629             {
 630               count = read (scanner->input_fd, &fchar, 1);
 631             }
 632           while (count == -1 &&
 633                  (errno == EINTR ||
 634                   errno == EAGAIN));
 635           if (count != 1 || fchar == 0)
 636             {
 637               fchar = 0;
 638               scanner->peeked_char = 0;
 639             }
 640         }
 641       else
 642         {
 643           fchar = scanner->peeked_char;
 644           if (fchar)
 645             scanner->peeked_char = -1;
 646         }
 647     }
 648   else
 649     fchar = 0;
 650
 651   if (fchar == '\n')
 652     {
 653       (*position_p) = 0;
 654       (*line_p)++;
 655     }
 656   else if (fchar)
 657     {
 658       (*position_p)++;
 659     }
 660
 661   return fchar;
 662 }
 663
 664 void
 665 g_scanner_unexp_token (GScanner         *scanner,
 666                        GTokenType        expected_token,
 667                        const gchar      *identifier_spec,
 668                        const gchar      *symbol_spec,
 669                        const gchar      *symbol_name,
 670                        const gchar      *message,
 671                        gint              is_error)
 672 {
 673   register gchar        *token_string;
 674   register guint        token_string_len;
 675   register gchar        *expected_string;
 676   register guint        expected_string_len;
 677   register gchar        *message_prefix;
 678   register gboolean     print_unexp;
 679   void (*msg_handler)   (GScanner*, const gchar*, ...);
 680
 681   g_return_if_fail (scanner != NULL);
 682
 683   if (is_error)
 684     msg_handler = g_scanner_error;
 685   else
 686     msg_handler = g_scanner_warn;
 687
 688   if (!identifier_spec)
 689     identifier_spec = "identifier";
 690   if (!symbol_spec)
 691     symbol_spec = "symbol";
 692
 693   token_string_len = 56;
 694   token_string = g_new (gchar, token_string_len + 1);
 695   expected_string_len = 64;
 696   expected_string = g_new (gchar, expected_string_len + 1);
 697   print_unexp = TRUE;
 698
 699   switch (scanner->token)
 700     {
 701     case  G_TOKEN_EOF:
 702       g_snprintf (token_string, token_string_len, "end of file");
 703       break;
 704
 705     default:
 706       if (scanner->token >= 1 && scanner->token <= 255)
 707         {
 708           if ((scanner->token >= ' ' && scanner->token <= '~') ||
 709               strchr (scanner->config->cset_identifier_first, scanner->token) ||
 710               strchr (scanner->config->cset_identifier_nth, scanner->token))
 711             g_snprintf (token_string, expected_string_len, "character `%c'", scanner->token);
 712           else
 713             g_snprintf (token_string, expected_string_len, "character `\\%o'", scanner->token);
 714           break;
 715         }
 716       else if (!scanner->config->symbol_2_token)
 717         {
 718           g_snprintf (token_string, token_string_len, "(unknown) token <%d>", scanner->token);
 719           break;
 720         }
 721       /* fall through */
 722     case  G_TOKEN_SYMBOL:
 723       if (expected_token == G_TOKEN_SYMBOL ||
 724           (scanner->config->symbol_2_token &&
 725            expected_token > G_TOKEN_LAST))
 726         print_unexp = FALSE;
 727       if (symbol_name)
 728         g_snprintf (token_string,
 729                     token_string_len,
 730                     "%s%s `%s'",
 731                     print_unexp ? "" : "invalid ",
 732                     symbol_spec,
 733                     symbol_name);
 734       else
 735         g_snprintf (token_string,
 736                     token_string_len,
 737                     "%s%s",
 738                     print_unexp ? "" : "invalid ",
 739                     symbol_spec);
 740       break;
 741
 742     case  G_TOKEN_ERROR:
 743       print_unexp = FALSE;
 744       expected_token = G_TOKEN_NONE;
 745       switch (scanner->value.v_error)
 746         {
 747         case  G_ERR_UNEXP_EOF:
 748           g_snprintf (token_string, token_string_len, "scanner: unexpected end of file");
 749           break;
 750
 751         case  G_ERR_UNEXP_EOF_IN_STRING:
 752           g_snprintf (token_string, token_string_len, "scanner: unterminated string constant");
 753           break;
 754
 755         case  G_ERR_UNEXP_EOF_IN_COMMENT:
 756           g_snprintf (token_string, token_string_len, "scanner: unterminated comment");
 757           break;
 758
 759         case  G_ERR_NON_DIGIT_IN_CONST:
 760           g_snprintf (token_string, token_string_len, "scanner: non digit in constant");
 761           break;
 762
 763         case  G_ERR_FLOAT_RADIX:
 764           g_snprintf (token_string, token_string_len, "scanner: invalid radix for floating constant");
 765           break;
 766
 767         case  G_ERR_FLOAT_MALFORMED:
 768           g_snprintf (token_string, token_string_len, "scanner: malformed floating constant");
 769           break;
 770
 771         case  G_ERR_DIGIT_RADIX:
 772           g_snprintf (token_string, token_string_len, "scanner: digit is beyond radix");
 773           break;
 774
 775         case  G_ERR_UNKNOWN:
 776         default:
 777           g_snprintf (token_string, token_string_len, "scanner: unknown error");
 778           break;
 779         }
 780       break;
 781
 782     case  G_TOKEN_CHAR:
 783       g_snprintf (token_string, token_string_len, "character `%c'", scanner->value.v_char);
 784       break;
 785
 786     case  G_TOKEN_IDENTIFIER:
 787       if (expected_token == G_TOKEN_IDENTIFIER)
 788         print_unexp = FALSE;
 789       g_snprintf (token_string,
 790                   token_string_len,
 791                   "%s%s `%s'",
 792                   print_unexp ? "" : "invalid ",
 793                   identifier_spec,
 794                   scanner->value.v_string);
 795       break;
 796
 797     case  G_TOKEN_BINARY:
 798     case  G_TOKEN_OCTAL:
 799     case  G_TOKEN_INT:
 800     case  G_TOKEN_HEX:
 801       g_snprintf (token_string, token_string_len, "number `%ld'", scanner->value.v_int);
 802       break;
 803
 804     case  G_TOKEN_FLOAT:
 805       g_snprintf (token_string, token_string_len, "number `%.3f'", scanner->value.v_float);
 806       break;
 807
 808     case  G_TOKEN_STRING:
 809       if (expected_token == G_TOKEN_STRING)
 810         print_unexp = FALSE;
 811       g_snprintf (token_string,
 812                   token_string_len,
 813                   "%s%sstring constant \"%s\"",
 814                   print_unexp ? "" : "invalid ",
 815                   scanner->value.v_string[0] == 0 ? "empty " : "",
 816                   scanner->value.v_string);
 817       token_string[token_string_len - 2] = '"';
 818       token_string[token_string_len - 1] = 0;
 819       break;
 820
 821     case  G_TOKEN_COMMENT_SINGLE:
 822     case  G_TOKEN_COMMENT_MULTI:
 823       g_snprintf (token_string, token_string_len, "comment");
 824       break;
 825
 826     case  G_TOKEN_NONE:
 827       /* somehow the user's parsing code is screwed, there isn't much
 828        * we can do about it.
 829        * Note, a common case to trigger this is
 830        * g_scanner_peek_next_token(); g_scanner_unexp_token();
 831        * without an intermediate g_scanner_get_next_token().
 832        */
 833       g_assert_not_reached ();
 834       break;
 835     }
 836
 837
 838   switch (expected_token)
 839     {
 840       gboolean need_valid;
 841
 842     default:
 843       if (expected_token >= 1 && expected_token <= 255)
 844         {
 845           if ((expected_token >= ' ' && expected_token <= '~') ||
 846               strchr (scanner->config->cset_identifier_first, expected_token) ||
 847               strchr (scanner->config->cset_identifier_nth, expected_token))
 848             g_snprintf (expected_string, expected_string_len, "character `%c'", expected_token);
 849           else
 850             g_snprintf (expected_string, expected_string_len, "character `\\%o'", expected_token);
 851           break;
 852         }
 853       else if (!scanner->config->symbol_2_token)
 854         {
 855           g_snprintf (expected_string, expected_string_len, "(unknown) token <%d>", expected_token);
 856           break;
 857         }
 858       /* fall through */
 859     case  G_TOKEN_SYMBOL:
 860       need_valid = (scanner->token == G_TOKEN_SYMBOL ||
 861                     (scanner->config->symbol_2_token &&
 862                      scanner->token > G_TOKEN_LAST));
 863       g_snprintf (expected_string,
 864                   expected_string_len,
 865                   "%s%s",
 866                   need_valid ? "valid " : "",
 867                   symbol_spec);
 868       /* FIXME: should we attempt to lookup the symbol_name for symbol_2_token? */
 869       break;
 870
 871     case  G_TOKEN_INT:
 872       g_snprintf (expected_string, expected_string_len, "number (integer)");
 873       break;
 874
 875     case  G_TOKEN_FLOAT:
 876       g_snprintf (expected_string, expected_string_len, "number (float)");
 877       break;
 878
 879     case  G_TOKEN_STRING:
 880       g_snprintf (expected_string,
 881                   expected_string_len,
 882                   "%sstring constant",
 883                   scanner->token == G_TOKEN_STRING ? "valid " : "");
 884       break;
 885
 886     case  G_TOKEN_IDENTIFIER:
 887       g_snprintf (expected_string,
 888                   expected_string_len,
 889                   "%s%s",
 890                   scanner->token == G_TOKEN_IDENTIFIER ? "valid " : "",
 891                   identifier_spec);
 892       break;
 893
 894     case  G_TOKEN_NONE:
 895       break;
 896     }
 897
 898   if (message && message[0] != 0)
 899     message_prefix = " - ";
 900   else
 901     {
 902       message_prefix = "";
 903       message = "";
 904     }
 905
 906   if (expected_token != G_TOKEN_NONE)
 907     {
 908       if (print_unexp)
 909         msg_handler (scanner,
 910                      "unexpected %s, expected %s%s%s",
 911                      token_string,
 912                      expected_string,
 913                      message_prefix,
 914                      message);
 915       else
 916         msg_handler (scanner,
 917                      "%s, expected %s%s%s",
 918                      token_string,
 919                      expected_string,
 920                      message_prefix,
 921                      message);
 922     }
 923   else
 924     {
 925       if (print_unexp)
 926         msg_handler (scanner,
 927                      "unexpected %s%s%s",
 928                      token_string,
 929                      message_prefix,
 930                      message);
 931       else
 932         msg_handler (scanner,
 933                      "%s%s%s",
 934                      token_string,
 935                      message_prefix,
 936                      message);
 937     }
 938
 939   g_free (token_string);
 940   g_free (expected_string);
 941 }
 942
 943 gint
 944 g_scanner_stat_mode (const gchar *filename)
 945 {
 946   struct stat  *stat_buf;
 947   gint          st_mode;
 948
 949   stat_buf = g_new0 (struct stat, 1);
 950
 951   lstat (filename, stat_buf);
 952
 953   st_mode = stat_buf->st_mode;
 954
 955   g_free (stat_buf);
 956
 957   return st_mode;
 958 }
 959
 960 static void
 961 g_scanner_free_value (GTokenType     *token_p,
 962                       GValue         *value_p)
 963 {
 964   switch (*token_p)
 965     {
 966     case  G_TOKEN_STRING:
 967     case  G_TOKEN_IDENTIFIER:
 968     case  G_TOKEN_IDENTIFIER_NULL:
 969     case  G_TOKEN_COMMENT_SINGLE:
 970     case  G_TOKEN_COMMENT_MULTI:
 971       g_free (value_p->v_string);
 972       break;
 973
 974     default:
 975       break;
 976     }
 977
 978   *token_p = G_TOKEN_NONE;
 979 }
 980
 981 static void
 982 g_scanner_get_token_i (GScanner *scanner,
 983                        GTokenType       *token_p,
 984                        GValue           *value_p,
 985                        guint            *line_p,
 986                        guint            *position_p)
 987 {
 988   do
 989     {
 990       g_scanner_free_value (token_p, value_p);
 991       g_scanner_get_token_ll (scanner, token_p, value_p, line_p, position_p);
 992     }
 993   while (((*token_p > 0 && *token_p < 256) &&
 994           strchr (scanner->config->cset_skip_characters, *token_p)) ||
 995          (*token_p == G_TOKEN_CHAR &&
 996           strchr (scanner->config->cset_skip_characters, value_p->v_char)) ||
 997          (*token_p == G_TOKEN_COMMENT_MULTI &&
 998           scanner->config->skip_comment_multi) ||
 999          (*token_p == G_TOKEN_COMMENT_SINGLE &&
1000           scanner->config->skip_comment_single));
1001
1002   switch (*token_p)
1003     {
1004     case        G_TOKEN_IDENTIFIER:
1005       if (scanner->config->identifier_2_string)
1006         *token_p = G_TOKEN_STRING;
1007       break;
1008
1009     case        G_TOKEN_SYMBOL:
1010       if (scanner->config->symbol_2_token)
1011         *token_p = (GTokenType) value_p->v_symbol;
1012       break;
1013
1014     case        G_TOKEN_BINARY:
1015     case        G_TOKEN_OCTAL:
1016     case        G_TOKEN_HEX:
1017       if (scanner->config->numbers_2_int)
1018         *token_p = G_TOKEN_INT;
1019       break;
1020
1021     default:
1022       break;
1023     }
1024
1025   if (*token_p == G_TOKEN_INT &&
1026       scanner->config->int_2_float)
1027     {
1028       *token_p = G_TOKEN_FLOAT;
1029       value_p->v_float = value_p->v_int;
1030     }
1031
1032   errno = 0;
1033 }
1034
1035 static void
1036 g_scanner_get_token_ll  (GScanner       *scanner,
1037                          GTokenType     *token_p,
1038                          GValue         *value_p,
1039                          guint          *line_p,
1040                          guint          *position_p)
1041 {
1042   register GScannerConfig       *config;
1043   register gboolean             in_comment_multi;
1044   register gboolean             in_comment_single;
1045   register gboolean             in_string_sq;
1046   register gboolean             in_string_dq;
1047   static   guchar               ch;
1048   register GTokenType           token;
1049   register GValue               value;
1050   register GString              *gstring;
1051
1052   config = scanner->config;
1053   (*value_p).v_int = 0;
1054
1055   if (scanner->token == G_TOKEN_EOF ||
1056       (!scanner->text_len &&
1057        (scanner->input_fd < 0 ||
1058         scanner->peeked_char == 0)))
1059     {
1060       *token_p = G_TOKEN_EOF;
1061       return;
1062     }
1063
1064   in_comment_multi = FALSE;
1065   in_comment_single = FALSE;
1066   in_string_sq = FALSE;
1067   in_string_dq = FALSE;
1068   gstring = NULL;
1069
1070   do
1071     {
1072       register gboolean         dotted_float = FALSE;
1073
1074       ch = g_scanner_get_char (scanner, line_p, position_p);
1075
1076       value.v_int = 0;
1077       token = G_TOKEN_NONE;
1078
1079       /* this is *evil*, but needed ;(
1080        * we first check for identifier first character, because  it
1081        * might interfere with other key chars like slashes or numbers
1082        */
1083       if (config->scan_identifier &&
1084           ch && strchr (config->cset_identifier_first, ch))
1085         goto identifier_precedence;
1086
1087       switch (ch)
1088         {
1089           register gboolean     in_number;
1090           static         gchar          *endptr;
1091
1092         case  0:
1093           token = G_TOKEN_EOF;
1094           (*position_p)++;
1095           ch = 0;
1096           break;
1097
1098         case  '/':
1099           if (!config->scan_comment_multi ||
1100               g_scanner_peek_next_char (scanner) != '*')
1101             goto default_case;
1102           g_scanner_get_char (scanner, line_p, position_p);
1103           token = G_TOKEN_COMMENT_MULTI;
1104           in_comment_multi = TRUE;
1105           gstring = g_string_new ("");
1106           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1107             {
1108               if (ch == '*' && g_scanner_peek_next_char (scanner) == '/')
1109                 {
1110                   g_scanner_get_char (scanner, line_p, position_p);
1111                   in_comment_multi = FALSE;
1112                   break;
1113                 }
1114               else
1115                 gstring = g_string_append_c (gstring, ch);
1116             }
1117           ch = 0;
1118           break;
1119
1120         case  '\'':
1121           if (!config->scan_string_sq)
1122             goto default_case;
1123           token = G_TOKEN_STRING;
1124           in_string_sq = TRUE;
1125           gstring = g_string_new ("");
1126           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1127             {
1128               if (ch == '\'')
1129                 {
1130                   in_string_sq = FALSE;
1131                   break;
1132                 }
1133               else
1134                 gstring = g_string_append_c (gstring, ch);
1135             }
1136           ch = 0;
1137           break;
1138
1139         case  '"':
1140           if (!config->scan_string_dq)
1141             goto default_case;
1142           token = G_TOKEN_STRING;
1143           in_string_dq = TRUE;
1144           gstring = g_string_new ("");
1145           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1146             {
1147               if (ch == '"')
1148                 {
1149                   in_string_dq = FALSE;
1150                   break;
1151                 }
1152               else
1153                 {
1154                   if (ch == '\\')
1155                     {
1156                       ch = g_scanner_get_char (scanner, line_p, position_p);
1157                       switch (ch)
1158                         {
1159                           register guint        i;
1160                           register guint        fchar;
1161
1162                         case  0:
1163                           break;
1164
1165                         case  '\\':
1166                           gstring = g_string_append_c (gstring, '\\');
1167                           break;
1168
1169                         case  'n':
1170                           gstring = g_string_append_c (gstring, '\n');
1171                           break;
1172
1173                         case  't':
1174                           gstring = g_string_append_c (gstring, '\t');
1175                           break;
1176
1177                         case  'r':
1178                           gstring = g_string_append_c (gstring, '\r');
1179                           break;
1180
1181                         case  'b':
1182                           gstring = g_string_append_c (gstring, '\b');
1183                           break;
1184
1185                         case  'f':
1186                           gstring = g_string_append_c (gstring, '\f');
1187                           break;
1188
1189                         case  '0':
1190                         case  '1':
1191                         case  '2':
1192                         case  '3':
1193                         case  '4':
1194                         case  '5':
1195                         case  '6':
1196                         case  '7':
1197                           i = ch - '0';
1198                           fchar = g_scanner_peek_next_char (scanner);
1199                           if (fchar >= '0' && fchar <= '7')
1200                             {
1201                               ch = g_scanner_get_char (scanner, line_p, position_p);
1202                               i= i * 8 + ch - '0';
1203                               fchar = g_scanner_peek_next_char (scanner);
1204                               if (fchar >= '0' && fchar <= '7')
1205                                 {
1206                                   ch = g_scanner_get_char (scanner, line_p, position_p);
1207                                   i = i * 8 + ch - '0';
1208                                 }
1209                             }
1210                           gstring = g_string_append_c (gstring, i);
1211                           break;
1212
1213                         default:
1214                           gstring = g_string_append_c (gstring, ch);
1215                           break;
1216                         }
1217                     }
1218                   else
1219                     gstring = g_string_append_c (gstring, ch);
1220                 }
1221             }
1222           ch = 0;
1223           break;
1224
1225         case  '.':
1226           if (!config->scan_float)
1227             goto default_case;
1228           token = G_TOKEN_FLOAT;
1229           dotted_float = TRUE;
1230           ch = g_scanner_get_char (scanner, line_p, position_p);
1231           goto number_parsing;
1232
1233         case  '$':
1234           if (!config->scan_hex_dollar)
1235             goto default_case;
1236           token = G_TOKEN_HEX;
1237           ch = g_scanner_get_char (scanner, line_p, position_p);
1238           goto number_parsing;
1239
1240         case  '0':
1241           if (config->scan_octal)
1242             token = G_TOKEN_OCTAL;
1243           else
1244             token = G_TOKEN_INT;
1245           ch = g_scanner_peek_next_char (scanner);
1246           if (config->scan_hex && (ch == 'x' || ch == 'X'))
1247             {
1248               token = G_TOKEN_HEX;
1249               g_scanner_get_char (scanner, line_p, position_p);
1250               ch = g_scanner_get_char (scanner, line_p, position_p);
1251               if (ch == 0)
1252                 {
1253                   token = G_TOKEN_ERROR;
1254                   value.v_error = G_ERR_UNEXP_EOF;
1255                   (*position_p)++;
1256                   break;
1257                 }
1258               if (g_scanner_char_2_num (ch, 16) < 0)
1259                 {
1260                   token = G_TOKEN_ERROR;
1261                   value.v_error = G_ERR_DIGIT_RADIX;
1262                   ch = 0;
1263                   break;
1264                 }
1265             }
1266           else if (config->scan_binary && (ch == 'b' || ch == 'B'))
1267             {
1268               token = G_TOKEN_BINARY;
1269               g_scanner_get_char (scanner, line_p, position_p);
1270               ch = g_scanner_get_char (scanner, line_p, position_p);
1271               if (ch == 0)
1272                 {
1273                   token = G_TOKEN_ERROR;
1274                   value.v_error = G_ERR_UNEXP_EOF;
1275                   (*position_p)++;
1276                   break;
1277                 }
1278               if (g_scanner_char_2_num (ch, 10) < 0)
1279                 {
1280                   token = G_TOKEN_ERROR;
1281                   value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1282                   ch = 0;
1283                   break;
1284                 }
1285             }
1286           else
1287             ch = '0';
1288           /* fall through */
1289         case  '1':
1290         case  '2':
1291         case  '3':
1292         case  '4':
1293         case  '5':
1294         case  '6':
1295         case  '7':
1296         case  '8':
1297         case  '9':
1298         number_parsing:
1299         if (token == G_TOKEN_NONE)
1300           token = G_TOKEN_INT;
1301
1302         gstring = g_string_new (dotted_float ? "0." : "");
1303         gstring = g_string_append_c (gstring, ch);
1304         in_number = TRUE;
1305         while (in_number)
1306           {
1307             register gboolean is_E;
1308
1309             is_E = (ch == 'e' || ch == 'E') && token == G_TOKEN_FLOAT;
1310             ch = g_scanner_peek_next_char (scanner);
1311
1312             if (g_scanner_char_2_num (ch, 36) >= 0 ||
1313                 (config->scan_float && ch == '.') ||
1314                 (is_E && ch == '+') ||
1315                 (is_E && ch == '-') )
1316               ch = g_scanner_get_char (scanner, line_p, position_p);
1317             else
1318               in_number = FALSE;
1319
1320             if (in_number)
1321               switch (ch)
1322                 {
1323                 case  '.':
1324                   if (token != G_TOKEN_INT &&
1325                       token != G_TOKEN_OCTAL)
1326                     {
1327                       token = G_TOKEN_ERROR;
1328                       if (token == G_TOKEN_FLOAT)
1329                         value.v_error = G_ERR_FLOAT_MALFORMED;
1330                       else
1331                         value.v_error = G_ERR_FLOAT_RADIX;
1332                       in_number = FALSE;
1333                     }
1334                   else
1335                     {
1336                       token = G_TOKEN_FLOAT;
1337                       gstring = g_string_append_c (gstring, ch);
1338                     }
1339                   break;
1340
1341                 case    '0':
1342                 case  '1':
1343                 case  '2':
1344                 case  '3':
1345                 case  '4':
1346                 case  '5':
1347                 case  '6':
1348                 case  '7':
1349                 case  '8':
1350                 case  '9':
1351                   gstring = g_string_append_c (gstring, ch);
1352                   break;
1353
1354                 case    '-':
1355                 case    '+':
1356                   if (token != G_TOKEN_FLOAT)
1357                     {
1358                       token = G_TOKEN_ERROR;
1359                       value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1360                       in_number = FALSE;
1361                     }
1362                   else
1363                     gstring = g_string_append_c (gstring, ch);
1364                   break;
1365
1366                 case    'e':
1367                 case    'E':
1368                   if ((token != G_TOKEN_HEX && !config->scan_float) ||
1369                       (token != G_TOKEN_HEX &&
1370                        token != G_TOKEN_OCTAL &&
1371                        token != G_TOKEN_FLOAT &&
1372                        token != G_TOKEN_INT))
1373                     {
1374                       token = G_TOKEN_ERROR;
1375                       value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1376                       in_number = FALSE;
1377                     }
1378                   else
1379                     {
1380                       if (token != G_TOKEN_HEX)
1381                         token = G_TOKEN_FLOAT;
1382                       gstring = g_string_append_c (gstring, ch);
1383                     }
1384                   break;
1385
1386                 default:
1387                   if (token != G_TOKEN_HEX)
1388                     {
1389                       token = G_TOKEN_ERROR;
1390                       value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1391                       in_number = FALSE;
1392                     }
1393                   else
1394                     gstring = g_string_append_c (gstring, ch);
1395                   break;
1396                 }
1397           }
1398         endptr = NULL;
1399         switch (token)
1400           {
1401           case  G_TOKEN_BINARY:
1402             value.v_binary = strtol (gstring->str, &endptr, 2);
1403             break;
1404
1405           case  G_TOKEN_OCTAL:
1406             value.v_octal = strtol (gstring->str, &endptr, 8);
1407             break;
1408
1409           case  G_TOKEN_INT:
1410             value.v_int = strtol (gstring->str, &endptr, 10);
1411             break;
1412
1413           case  G_TOKEN_FLOAT:
1414             value.v_float = g_strtod (gstring->str, &endptr);
1415             break;
1416
1417           case  G_TOKEN_HEX:
1418             value.v_hex = strtol (gstring->str, &endptr, 16);
1419             break;
1420
1421           default:
1422             break;
1423           }
1424         if (endptr && *endptr)
1425           {
1426             token = G_TOKEN_ERROR;
1427             if (*endptr == 'e' || *endptr == 'E')
1428               value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1429             else
1430               value.v_error = G_ERR_DIGIT_RADIX;
1431           }
1432         g_string_free (gstring, TRUE);
1433         gstring = NULL;
1434         ch = 0;
1435         break;
1436
1437         default:
1438         default_case:
1439         if (config->cpair_comment_single &&
1440             ch == config->cpair_comment_single[0])
1441           {
1442             token = G_TOKEN_COMMENT_SINGLE;
1443             in_comment_single = TRUE;
1444             gstring = g_string_new ("");
1445             while ((ch = g_scanner_get_char (scanner,
1446                                              line_p,
1447                                              position_p)) != 0)
1448               {
1449                 if (ch == config->cpair_comment_single[1])
1450                   {
1451                     in_comment_single = FALSE;
1452                     ch = 0;
1453                     break;
1454                   }
1455
1456                 gstring = g_string_append_c (gstring, ch);
1457                 ch = 0;
1458               }
1459           }
1460         else if (config->scan_identifier && ch &&
1461                  strchr (config->cset_identifier_first, ch))
1462           {
1463           identifier_precedence:
1464
1465             if (config->cset_identifier_nth && ch &&
1466                 strchr (config->cset_identifier_nth,
1467                         g_scanner_peek_next_char (scanner)))
1468               {
1469                 token = G_TOKEN_IDENTIFIER;
1470                 gstring = g_string_new ("");
1471                 gstring = g_string_append_c (gstring, ch);
1472                 do
1473                   {
1474                     ch = g_scanner_get_char (scanner, line_p, position_p);
1475                     gstring = g_string_append_c (gstring, ch);
1476                     ch = g_scanner_peek_next_char (scanner);
1477                   }
1478                 while (ch && strchr (config->cset_identifier_nth, ch));
1479                 ch = 0;
1480               }
1481             else if (config->scan_identifier_1char)
1482               {
1483                 token = G_TOKEN_IDENTIFIER;
1484                 value.v_identifier = g_new0 (gchar, 2);
1485                 value.v_identifier[0] = ch;
1486                 ch = 0;
1487               }
1488           }
1489         if (ch)
1490           {
1491             if (config->char_2_token)
1492               token = ch;
1493             else
1494               {
1495                 token = G_TOKEN_CHAR;
1496                 value.v_char = ch;
1497               }
1498             ch = 0;
1499           }
1500         break;
1501         }
1502       g_assert (ch == 0 && token != G_TOKEN_NONE);
1503     }
1504   while (ch != 0);
1505
1506   if (in_comment_multi ||
1507       in_comment_single ||
1508       in_string_sq ||
1509       in_string_dq)
1510     {
1511       token = G_TOKEN_ERROR;
1512       if (gstring)
1513         {
1514           g_string_free (gstring, TRUE);
1515           gstring = NULL;
1516         }
1517       (*position_p)++;
1518       if (in_comment_multi || in_comment_single)
1519         value.v_error = G_ERR_UNEXP_EOF_IN_COMMENT;
1520       else if (in_string_sq || in_string_dq)
1521         value.v_error = G_ERR_UNEXP_EOF_IN_STRING;
1522     }
1523
1524   if (gstring)
1525     {
1526       value.v_string = gstring->str;
1527       g_string_free (gstring, FALSE);
1528       gstring = NULL;
1529     }
1530
1531   if (token == G_TOKEN_IDENTIFIER &&
1532       config->scan_symbols)
1533     {
1534       register GScannerHashVal  *hash_val;
1535
1536       hash_val = g_scanner_lookup_internal (scanner, value.v_identifier);
1537
1538       if (hash_val)
1539         {
1540           g_free (value.v_identifier);
1541           token = G_TOKEN_SYMBOL;
1542           value.v_symbol = hash_val->value;
1543         }
1544     }
1545
1546   if (token == G_TOKEN_IDENTIFIER &&
1547       config->scan_identifier_NULL &&
1548       strlen (value.v_identifier) == 4)
1549     {
1550       gchar *null_upper = "NULL";
1551       gchar *null_lower = "null";
1552
1553       if (scanner->config->case_sensitive)
1554         {
1555           if (value.v_identifier[0] == null_upper[0] &&
1556               value.v_identifier[1] == null_upper[1] &&
1557               value.v_identifier[2] == null_upper[2] &&
1558               value.v_identifier[3] == null_upper[3])
1559             token = G_TOKEN_IDENTIFIER_NULL;
1560         }
1561       else
1562         {
1563           if ((value.v_identifier[0] == null_upper[0] ||
1564                value.v_identifier[0] == null_lower[0]) &&
1565               (value.v_identifier[1] == null_upper[1] ||
1566                value.v_identifier[1] == null_lower[1]) &&
1567               (value.v_identifier[2] == null_upper[2] ||
1568                value.v_identifier[2] == null_lower[2]) &&
1569               (value.v_identifier[3] == null_upper[3] ||
1570                value.v_identifier[3] == null_lower[3]))
1571             token = G_TOKEN_IDENTIFIER_NULL;
1572         }
1573     }
1574
1575   *token_p = token;
1576   *value_p = value;
1577 }