glib/gscanner.c

   1 /* GLIB - Library of useful routines for C programming
   2  * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
   3  *
   4  * GScanner: Flexible lexical scanner for general purpose.
   5  * Copyright (C) 1997, 1998 Tim Janik
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the
  19  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  20  * Boston, MA 02111-1307, USA.
  21  */
  22
  23 /*
  24  * Modified by the GLib Team and others 1997-2000.  See the AUTHORS
  25  * file for a list of people on the GLib Team.  See the ChangeLog
  26  * files for a list of changes.  These files are distributed with
  27  * GLib at ftp://ftp.gtk.org/pub/gtk/.
  28  */
  29
  30 /*
  31  * MT safe
  32  */
  33
  34 #ifdef HAVE_CONFIG_H
  35 #include <config.h>
  36 #endif
  37
  38 #include        <stdlib.h>
  39 #include        <stdarg.h>
  40 #include        <string.h>
  41 #include        <stdio.h>
  42 #include        "glib.h"
  43 #include        "gprintfint.h"
  44 #ifdef HAVE_UNISTD_H
  45 #include        <unistd.h>
  46 #endif
  47 #include        <errno.h>
  48 #ifdef G_OS_WIN32
  49 #include        <io.h>          /* For _read() */
  50 #endif
  51
  52 /* --- defines --- */
  53 #define to_lower(c)                             ( \
  54         (guchar) (                                                      \
  55           ( (((guchar)(c))>='A' && ((guchar)(c))<='Z') * ('a'-'A') ) |  \
  56           ( (((guchar)(c))>=192 && ((guchar)(c))<=214) * (224-192) ) |  \
  57           ( (((guchar)(c))>=216 && ((guchar)(c))<=222) * (248-216) ) |  \
  58           ((guchar)(c))                                                 \
  59         )                                                               \
  60 )
  61 #define READ_BUFFER_SIZE        (4000)
  62
  63
  64 /* --- typedefs --- */
  65 typedef struct  _GScannerKey    GScannerKey;
  66
  67 struct  _GScannerKey
  68 {
  69   guint          scope_id;
  70   gchar         *symbol;
  71   gpointer       value;
  72 };
  73
  74
  75 /* --- variables --- */
  76 static GScannerConfig g_scanner_config_template =
  77 {
  78   (
  79    " \t\r\n"
  80    )                    /* cset_skip_characters */,
  81   (
  82    G_CSET_a_2_z
  83    "_"
  84    G_CSET_A_2_Z
  85    )                    /* cset_identifier_first */,
  86   (
  87    G_CSET_a_2_z
  88    "_"
  89    G_CSET_A_2_Z
  90    G_CSET_DIGITS
  91    G_CSET_LATINS
  92    G_CSET_LATINC
  93    )                    /* cset_identifier_nth */,
  94   ( "#\n" )             /* cpair_comment_single */,
  95
  96   FALSE                 /* case_sensitive */,
  97
  98   TRUE                  /* skip_comment_multi */,
  99   TRUE                  /* skip_comment_single */,
 100   TRUE                  /* scan_comment_multi */,
 101   TRUE                  /* scan_identifier */,
 102   FALSE                 /* scan_identifier_1char */,
 103   FALSE                 /* scan_identifier_NULL */,
 104   TRUE                  /* scan_symbols */,
 105   FALSE                 /* scan_binary */,
 106   TRUE                  /* scan_octal */,
 107   TRUE                  /* scan_float */,
 108   TRUE                  /* scan_hex */,
 109   FALSE                 /* scan_hex_dollar */,
 110   TRUE                  /* scan_string_sq */,
 111   TRUE                  /* scan_string_dq */,
 112   TRUE                  /* numbers_2_int */,
 113   FALSE                 /* int_2_float */,
 114   FALSE                 /* identifier_2_string */,
 115   TRUE                  /* char_2_token */,
 116   FALSE                 /* symbol_2_token */,
 117   FALSE                 /* scope_0_fallback */,
 118   FALSE                 /* store_int64 */,
 119 };
 120
 121
 122 /* --- prototypes --- */
 123 static inline
 124 GScannerKey*    g_scanner_lookup_internal (GScanner     *scanner,
 125                                            guint         scope_id,
 126                                            const gchar  *symbol);
 127 static gboolean g_scanner_key_equal       (gconstpointer v1,
 128                                            gconstpointer v2);
 129 static guint    g_scanner_key_hash        (gconstpointer v);
 130 static void     g_scanner_get_token_ll    (GScanner     *scanner,
 131                                            GTokenType   *token_p,
 132                                            GTokenValue  *value_p,
 133                                            guint        *line_p,
 134                                            guint        *position_p);
 135 static void     g_scanner_get_token_i     (GScanner     *scanner,
 136                                            GTokenType   *token_p,
 137                                            GTokenValue  *value_p,
 138                                            guint        *line_p,
 139                                            guint        *position_p);
 140
 141 static guchar   g_scanner_peek_next_char  (GScanner     *scanner);
 142 static guchar   g_scanner_get_char        (GScanner     *scanner,
 143                                            guint        *line_p,
 144                                            guint        *position_p);
 145 static void     g_scanner_msg_handler     (GScanner     *scanner,
 146                                            gchar        *message,
 147                                            gboolean      is_error);
 148
 149
 150 /* --- functions --- */
 151 static inline gint
 152 g_scanner_char_2_num (guchar    c,
 153                       guchar    base)
 154 {
 155   if (c >= '0' && c <= '9')
 156     c -= '0';
 157   else if (c >= 'A' && c <= 'Z')
 158     c -= 'A' - 10;
 159   else if (c >= 'a' && c <= 'z')
 160     c -= 'a' - 10;
 161   else
 162     return -1;
 163
 164   if (c < base)
 165     return c;
 166
 167   return -1;
 168 }
 169
 170 GScanner*
 171 g_scanner_new (const GScannerConfig *config_templ)
 172 {
 173   GScanner *scanner;
 174
 175   if (!config_templ)
 176     config_templ = &g_scanner_config_template;
 177
 178   scanner = g_new0 (GScanner, 1);
 179
 180   scanner->user_data = NULL;
 181   scanner->max_parse_errors = 1;
 182   scanner->parse_errors = 0;
 183   scanner->input_name = NULL;
 184   g_datalist_init (&scanner->qdata);
 185
 186   scanner->config = g_new0 (GScannerConfig, 1);
 187
 188   scanner->config->case_sensitive        = config_templ->case_sensitive;
 189   scanner->config->cset_skip_characters  = config_templ->cset_skip_characters;
 190   if (!scanner->config->cset_skip_characters)
 191     scanner->config->cset_skip_characters = "";
 192   scanner->config->cset_identifier_first = config_templ->cset_identifier_first;
 193   scanner->config->cset_identifier_nth   = config_templ->cset_identifier_nth;
 194   scanner->config->cpair_comment_single  = config_templ->cpair_comment_single;
 195   scanner->config->skip_comment_multi    = config_templ->skip_comment_multi;
 196   scanner->config->skip_comment_single   = config_templ->skip_comment_single;
 197   scanner->config->scan_comment_multi    = config_templ->scan_comment_multi;
 198   scanner->config->scan_identifier       = config_templ->scan_identifier;
 199   scanner->config->scan_identifier_1char = config_templ->scan_identifier_1char;
 200   scanner->config->scan_identifier_NULL  = config_templ->scan_identifier_NULL;
 201   scanner->config->scan_symbols          = config_templ->scan_symbols;
 202   scanner->config->scan_binary           = config_templ->scan_binary;
 203   scanner->config->scan_octal            = config_templ->scan_octal;
 204   scanner->config->scan_float            = config_templ->scan_float;
 205   scanner->config->scan_hex              = config_templ->scan_hex;
 206   scanner->config->scan_hex_dollar       = config_templ->scan_hex_dollar;
 207   scanner->config->scan_string_sq        = config_templ->scan_string_sq;
 208   scanner->config->scan_string_dq        = config_templ->scan_string_dq;
 209   scanner->config->numbers_2_int         = config_templ->numbers_2_int;
 210   scanner->config->int_2_float           = config_templ->int_2_float;
 211   scanner->config->identifier_2_string   = config_templ->identifier_2_string;
 212   scanner->config->char_2_token          = config_templ->char_2_token;
 213   scanner->config->symbol_2_token        = config_templ->symbol_2_token;
 214   scanner->config->scope_0_fallback      = config_templ->scope_0_fallback;
 215   scanner->config->store_int64           = config_templ->store_int64;
 216
 217   scanner->token = G_TOKEN_NONE;
 218   scanner->value.v_int64 = 0;
 219   scanner->line = 1;
 220   scanner->position = 0;
 221
 222   scanner->next_token = G_TOKEN_NONE;
 223   scanner->next_value.v_int64 = 0;
 224   scanner->next_line = 1;
 225   scanner->next_position = 0;
 226
 227   scanner->symbol_table = g_hash_table_new (g_scanner_key_hash, g_scanner_key_equal);
 228   scanner->input_fd = -1;
 229   scanner->text = NULL;
 230   scanner->text_end = NULL;
 231   scanner->buffer = NULL;
 232   scanner->scope_id = 0;
 233
 234   scanner->msg_handler = g_scanner_msg_handler;
 235
 236   return scanner;
 237 }
 238
 239 static inline void
 240 g_scanner_free_value (GTokenType     *token_p,
 241                       GTokenValue     *value_p)
 242 {
 243   switch (*token_p)
 244     {
 245     case G_TOKEN_STRING:
 246     case G_TOKEN_IDENTIFIER:
 247     case G_TOKEN_IDENTIFIER_NULL:
 248     case G_TOKEN_COMMENT_SINGLE:
 249     case G_TOKEN_COMMENT_MULTI:
 250       g_free (value_p->v_string);
 251       break;
 252
 253     default:
 254       break;
 255     }
 256
 257   *token_p = G_TOKEN_NONE;
 258 }
 259
 260 static void
 261 g_scanner_destroy_symbol_table_entry (gpointer _key,
 262                                       gpointer _value,
 263                                       gpointer _data)
 264 {
 265   GScannerKey *key = _key;
 266
 267   g_free (key->symbol);
 268   g_free (key);
 269 }
 270
 271 void
 272 g_scanner_destroy (GScanner     *scanner)
 273 {
 274   g_return_if_fail (scanner != NULL);
 275
 276   g_datalist_clear (&scanner->qdata);
 277   g_hash_table_foreach (scanner->symbol_table,
 278                         g_scanner_destroy_symbol_table_entry, NULL);
 279   g_hash_table_destroy (scanner->symbol_table);
 280   g_scanner_free_value (&scanner->token, &scanner->value);
 281   g_scanner_free_value (&scanner->next_token, &scanner->next_value);
 282   g_free (scanner->config);
 283   g_free (scanner->buffer);
 284   g_free (scanner);
 285 }
 286
 287 static void
 288 g_scanner_msg_handler (GScanner         *scanner,
 289                        gchar            *message,
 290                        gboolean         is_error)
 291 {
 292   g_return_if_fail (scanner != NULL);
 293
 294   _g_fprintf (stderr, "%s:%d: ",
 295               scanner->input_name ? scanner->input_name : "<memory>",
 296               scanner->line);
 297   if (is_error)
 298     _g_fprintf (stderr, "error: ");
 299   _g_fprintf (stderr, "%s\n", message);
 300 }
 301
 302 void
 303 g_scanner_error (GScanner       *scanner,
 304                  const gchar    *format,
 305                  ...)
 306 {
 307   g_return_if_fail (scanner != NULL);
 308   g_return_if_fail (format != NULL);
 309
 310   scanner->parse_errors++;
 311
 312   if (scanner->msg_handler)
 313     {
 314       va_list args;
 315       gchar *string;
 316
 317       va_start (args, format);
 318       string = g_strdup_vprintf (format, args);
 319       va_end (args);
 320
 321       scanner->msg_handler (scanner, string, TRUE);
 322
 323       g_free (string);
 324     }
 325 }
 326
 327 void
 328 g_scanner_warn (GScanner       *scanner,
 329                 const gchar    *format,
 330                 ...)
 331 {
 332   g_return_if_fail (scanner != NULL);
 333   g_return_if_fail (format != NULL);
 334
 335   if (scanner->msg_handler)
 336     {
 337       va_list args;
 338       gchar *string;
 339
 340       va_start (args, format);
 341       string = g_strdup_vprintf (format, args);
 342       va_end (args);
 343
 344       scanner->msg_handler (scanner, string, FALSE);
 345
 346       g_free (string);
 347     }
 348 }
 349
 350 static gboolean
 351 g_scanner_key_equal (gconstpointer v1,
 352                      gconstpointer v2)
 353 {
 354   const GScannerKey *key1 = v1;
 355   const GScannerKey *key2 = v2;
 356
 357   return (key1->scope_id == key2->scope_id) && (strcmp (key1->symbol, key2->symbol) == 0);
 358 }
 359
 360 static guint
 361 g_scanner_key_hash (gconstpointer v)
 362 {
 363   const GScannerKey *key = v;
 364   gchar *c;
 365   guint h;
 366
 367   h = key->scope_id;
 368   for (c = key->symbol; *c; c++)
 369     h = (h << 5) - h + *c;
 370
 371   return h;
 372 }
 373
 374 static inline GScannerKey*
 375 g_scanner_lookup_internal (GScanner     *scanner,
 376                            guint         scope_id,
 377                            const gchar  *symbol)
 378 {
 379   GScannerKey   *key_p;
 380   GScannerKey key;
 381
 382   key.scope_id = scope_id;
 383
 384   if (!scanner->config->case_sensitive)
 385     {
 386       gchar *d;
 387       const gchar *c;
 388
 389       key.symbol = g_new (gchar, strlen (symbol) + 1);
 390       for (d = key.symbol, c = symbol; *c; c++, d++)
 391         *d = to_lower (*c);
 392       *d = 0;
 393       key_p = g_hash_table_lookup (scanner->symbol_table, &key);
 394       g_free (key.symbol);
 395     }
 396   else
 397     {
 398       key.symbol = (gchar*) symbol;
 399       key_p = g_hash_table_lookup (scanner->symbol_table, &key);
 400     }
 401
 402   return key_p;
 403 }
 404
 405 void
 406 g_scanner_scope_add_symbol (GScanner    *scanner,
 407                             guint        scope_id,
 408                             const gchar *symbol,
 409                             gpointer     value)
 410 {
 411   GScannerKey   *key;
 412
 413   g_return_if_fail (scanner != NULL);
 414   g_return_if_fail (symbol != NULL);
 415
 416   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 417
 418   if (!key)
 419     {
 420       key = g_new (GScannerKey, 1);
 421       key->scope_id = scope_id;
 422       key->symbol = g_strdup (symbol);
 423       key->value = value;
 424       if (!scanner->config->case_sensitive)
 425         {
 426           gchar *c;
 427
 428           c = key->symbol;
 429           while (*c != 0)
 430             {
 431               *c = to_lower (*c);
 432               c++;
 433             }
 434         }
 435       g_hash_table_insert (scanner->symbol_table, key, key);
 436     }
 437   else
 438     key->value = value;
 439 }
 440
 441 void
 442 g_scanner_scope_remove_symbol (GScanner    *scanner,
 443                                guint        scope_id,
 444                                const gchar *symbol)
 445 {
 446   GScannerKey   *key;
 447
 448   g_return_if_fail (scanner != NULL);
 449   g_return_if_fail (symbol != NULL);
 450
 451   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 452
 453   if (key)
 454     {
 455       g_hash_table_remove (scanner->symbol_table, key);
 456       g_free (key->symbol);
 457       g_free (key);
 458     }
 459 }
 460
 461 gpointer
 462 g_scanner_lookup_symbol (GScanner       *scanner,
 463                          const gchar    *symbol)
 464 {
 465   GScannerKey   *key;
 466   guint scope_id;
 467
 468   g_return_val_if_fail (scanner != NULL, NULL);
 469
 470   if (!symbol)
 471     return NULL;
 472
 473   scope_id = scanner->scope_id;
 474   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 475   if (!key && scope_id && scanner->config->scope_0_fallback)
 476     key = g_scanner_lookup_internal (scanner, 0, symbol);
 477
 478   if (key)
 479     return key->value;
 480   else
 481     return NULL;
 482 }
 483
 484 gpointer
 485 g_scanner_scope_lookup_symbol (GScanner       *scanner,
 486                                guint           scope_id,
 487                                const gchar    *symbol)
 488 {
 489   GScannerKey   *key;
 490
 491   g_return_val_if_fail (scanner != NULL, NULL);
 492
 493   if (!symbol)
 494     return NULL;
 495
 496   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 497
 498   if (key)
 499     return key->value;
 500   else
 501     return NULL;
 502 }
 503
 504 guint
 505 g_scanner_set_scope (GScanner       *scanner,
 506                      guint           scope_id)
 507 {
 508   guint old_scope_id;
 509
 510   g_return_val_if_fail (scanner != NULL, 0);
 511
 512   old_scope_id = scanner->scope_id;
 513   scanner->scope_id = scope_id;
 514
 515   return old_scope_id;
 516 }
 517
 518 static void
 519 g_scanner_foreach_internal (gpointer  _key,
 520                             gpointer  _value,
 521                             gpointer  _user_data)
 522 {
 523   GScannerKey *key;
 524   gpointer *d;
 525   GHFunc func;
 526   gpointer user_data;
 527   guint *scope_id;
 528
 529   d = _user_data;
 530   func = (GHFunc) d[0];
 531   user_data = d[1];
 532   scope_id = d[2];
 533   key = _value;
 534
 535   if (key->scope_id == *scope_id)
 536     func (key->symbol, key->value, user_data);
 537 }
 538
 539 void
 540 g_scanner_scope_foreach_symbol (GScanner       *scanner,
 541                                 guint           scope_id,
 542                                 GHFunc          func,
 543                                 gpointer        user_data)
 544 {
 545   gpointer d[3];
 546
 547   g_return_if_fail (scanner != NULL);
 548
 549   d[0] = (gpointer) func;
 550   d[1] = user_data;
 551   d[2] = &scope_id;
 552
 553   g_hash_table_foreach (scanner->symbol_table, g_scanner_foreach_internal, d);
 554 }
 555
 556 GTokenType
 557 g_scanner_peek_next_token (GScanner     *scanner)
 558 {
 559   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 560
 561   if (scanner->next_token == G_TOKEN_NONE)
 562     {
 563       scanner->next_line = scanner->line;
 564       scanner->next_position = scanner->position;
 565       g_scanner_get_token_i (scanner,
 566                              &scanner->next_token,
 567                              &scanner->next_value,
 568                              &scanner->next_line,
 569                              &scanner->next_position);
 570     }
 571
 572   return scanner->next_token;
 573 }
 574
 575 GTokenType
 576 g_scanner_get_next_token (GScanner      *scanner)
 577 {
 578   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 579
 580   if (scanner->next_token != G_TOKEN_NONE)
 581     {
 582       g_scanner_free_value (&scanner->token, &scanner->value);
 583
 584       scanner->token = scanner->next_token;
 585       scanner->value = scanner->next_value;
 586       scanner->line = scanner->next_line;
 587       scanner->position = scanner->next_position;
 588       scanner->next_token = G_TOKEN_NONE;
 589     }
 590   else
 591     g_scanner_get_token_i (scanner,
 592                            &scanner->token,
 593                            &scanner->value,
 594                            &scanner->line,
 595                            &scanner->position);
 596
 597   return scanner->token;
 598 }
 599
 600 GTokenType
 601 g_scanner_cur_token (GScanner *scanner)
 602 {
 603   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 604
 605   return scanner->token;
 606 }
 607
 608 GTokenValue
 609 g_scanner_cur_value (GScanner *scanner)
 610 {
 611   GTokenValue v;
 612
 613   v.v_int64 = 0;
 614
 615   g_return_val_if_fail (scanner != NULL, v);
 616
 617   /* MSC isn't capable of handling return scanner->value; ? */
 618
 619   v = scanner->value;
 620
 621   return v;
 622 }
 623
 624 guint
 625 g_scanner_cur_line (GScanner *scanner)
 626 {
 627   g_return_val_if_fail (scanner != NULL, 0);
 628
 629   return scanner->line;
 630 }
 631
 632 guint
 633 g_scanner_cur_position (GScanner *scanner)
 634 {
 635   g_return_val_if_fail (scanner != NULL, 0);
 636
 637   return scanner->position;
 638 }
 639
 640 gboolean
 641 g_scanner_eof (GScanner *scanner)
 642 {
 643   g_return_val_if_fail (scanner != NULL, TRUE);
 644
 645   return scanner->token == G_TOKEN_EOF || scanner->token == G_TOKEN_ERROR;
 646 }
 647
 648 void
 649 g_scanner_input_file (GScanner *scanner,
 650                       gint      input_fd)
 651 {
 652   g_return_if_fail (scanner != NULL);
 653   g_return_if_fail (input_fd >= 0);
 654
 655   if (scanner->input_fd >= 0)
 656     g_scanner_sync_file_offset (scanner);
 657
 658   scanner->token = G_TOKEN_NONE;
 659   scanner->value.v_int64 = 0;
 660   scanner->line = 1;
 661   scanner->position = 0;
 662   scanner->next_token = G_TOKEN_NONE;
 663
 664   scanner->input_fd = input_fd;
 665   scanner->text = NULL;
 666   scanner->text_end = NULL;
 667
 668   if (!scanner->buffer)
 669     scanner->buffer = g_new (gchar, READ_BUFFER_SIZE + 1);
 670 }
 671
 672 void
 673 g_scanner_input_text (GScanner    *scanner,
 674                       const gchar *text,
 675                       guint        text_len)
 676 {
 677   g_return_if_fail (scanner != NULL);
 678   if (text_len)
 679     g_return_if_fail (text != NULL);
 680   else
 681     text = NULL;
 682
 683   if (scanner->input_fd >= 0)
 684     g_scanner_sync_file_offset (scanner);
 685
 686   scanner->token = G_TOKEN_NONE;
 687   scanner->value.v_int64 = 0;
 688   scanner->line = 1;
 689   scanner->position = 0;
 690   scanner->next_token = G_TOKEN_NONE;
 691
 692   scanner->input_fd = -1;
 693   scanner->text = text;
 694   scanner->text_end = text + text_len;
 695
 696   if (scanner->buffer)
 697     {
 698       g_free (scanner->buffer);
 699       scanner->buffer = NULL;
 700     }
 701 }
 702
 703 static guchar
 704 g_scanner_peek_next_char (GScanner *scanner)
 705 {
 706   if (scanner->text < scanner->text_end)
 707     {
 708       return *scanner->text;
 709     }
 710   else if (scanner->input_fd >= 0)
 711     {
 712       gint count;
 713       gchar *buffer;
 714
 715       buffer = scanner->buffer;
 716       do
 717         {
 718           count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
 719         }
 720       while (count == -1 && (errno == EINTR || errno == EAGAIN));
 721
 722       if (count < 1)
 723         {
 724           scanner->input_fd = -1;
 725
 726           return 0;
 727         }
 728       else
 729         {
 730           scanner->text = buffer;
 731           scanner->text_end = buffer + count;
 732
 733           return *buffer;
 734         }
 735     }
 736   else
 737     return 0;
 738 }
 739
 740 void
 741 g_scanner_sync_file_offset (GScanner *scanner)
 742 {
 743   g_return_if_fail (scanner != NULL);
 744
 745   /* for file input, rewind the filedescriptor to the current
 746    * buffer position and blow the file read ahead buffer. usefull for
 747    * third party uses of our filedescriptor, which hooks onto the current
 748    * scanning position.
 749    */
 750
 751   if (scanner->input_fd >= 0 && scanner->text_end > scanner->text)
 752     {
 753       gint buffered;
 754
 755       buffered = scanner->text_end - scanner->text;
 756       if (lseek (scanner->input_fd, - buffered, SEEK_CUR) >= 0)
 757         {
 758           /* we succeeded, blow our buffer's contents now */
 759           scanner->text = NULL;
 760           scanner->text_end = NULL;
 761         }
 762       else
 763         errno = 0;
 764     }
 765 }
 766
 767 static guchar
 768 g_scanner_get_char (GScanner    *scanner,
 769                     guint       *line_p,
 770                     guint       *position_p)
 771 {
 772   guchar fchar;
 773
 774   if (scanner->text < scanner->text_end)
 775     fchar = *(scanner->text++);
 776   else if (scanner->input_fd >= 0)
 777     {
 778       gint count;
 779       gchar *buffer;
 780
 781       buffer = scanner->buffer;
 782       do
 783         {
 784           count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
 785         }
 786       while (count == -1 && (errno == EINTR || errno == EAGAIN));
 787
 788       if (count < 1)
 789         {
 790           scanner->input_fd = -1;
 791           fchar = 0;
 792         }
 793       else
 794         {
 795           scanner->text = buffer + 1;
 796           scanner->text_end = buffer + count;
 797           fchar = *buffer;
 798           if (!fchar)
 799             {
 800               g_scanner_sync_file_offset (scanner);
 801               scanner->text_end = scanner->text;
 802               scanner->input_fd = -1;
 803             }
 804         }
 805     }
 806   else
 807     fchar = 0;
 808
 809   if (fchar == '\n')
 810     {
 811       (*position_p) = 0;
 812       (*line_p)++;
 813     }
 814   else if (fchar)
 815     {
 816       (*position_p)++;
 817     }
 818
 819   return fchar;
 820 }
 821
 822 void
 823 g_scanner_unexp_token (GScanner         *scanner,
 824                        GTokenType        expected_token,
 825                        const gchar      *identifier_spec,
 826                        const gchar      *symbol_spec,
 827                        const gchar      *symbol_name,
 828                        const gchar      *message,
 829                        gint              is_error)
 830 {
 831   gchar *token_string;
 832   guint token_string_len;
 833   gchar *expected_string;
 834   guint expected_string_len;
 835   gchar *message_prefix;
 836   gboolean print_unexp;
 837   void (*msg_handler)   (GScanner*, const gchar*, ...);
 838
 839   g_return_if_fail (scanner != NULL);
 840
 841   if (is_error)
 842     msg_handler = g_scanner_error;
 843   else
 844     msg_handler = g_scanner_warn;
 845
 846   if (!identifier_spec)
 847     identifier_spec = "identifier";
 848   if (!symbol_spec)
 849     symbol_spec = "symbol";
 850
 851   token_string_len = 56;
 852   token_string = g_new (gchar, token_string_len + 1);
 853   expected_string_len = 64;
 854   expected_string = g_new (gchar, expected_string_len + 1);
 855   print_unexp = TRUE;
 856
 857   switch (scanner->token)
 858     {
 859     case G_TOKEN_EOF:
 860       _g_snprintf (token_string, token_string_len, "end of file");
 861       break;
 862
 863     default:
 864       if (scanner->token >= 1 && scanner->token <= 255)
 865         {
 866           if ((scanner->token >= ' ' && scanner->token <= '~') ||
 867               strchr (scanner->config->cset_identifier_first, scanner->token) ||
 868               strchr (scanner->config->cset_identifier_nth, scanner->token))
 869             _g_snprintf (token_string, token_string_len, "character `%c'", scanner->token);
 870           else
 871             _g_snprintf (token_string, token_string_len, "character `\\%o'", scanner->token);
 872           break;
 873         }
 874       else if (!scanner->config->symbol_2_token)
 875         {
 876           _g_snprintf (token_string, token_string_len, "(unknown) token <%d>", scanner->token);
 877           break;
 878         }
 879       /* fall through */
 880     case G_TOKEN_SYMBOL:
 881       if (expected_token == G_TOKEN_SYMBOL ||
 882           (scanner->config->symbol_2_token &&
 883            expected_token > G_TOKEN_LAST))
 884         print_unexp = FALSE;
 885       if (symbol_name)
 886         _g_snprintf (token_string,
 887                      token_string_len,
 888                      "%s%s `%s'",
 889                      print_unexp ? "" : "invalid ",
 890                      symbol_spec,
 891                      symbol_name);
 892       else
 893         _g_snprintf (token_string,
 894                      token_string_len,
 895                      "%s%s",
 896                      print_unexp ? "" : "invalid ",
 897                      symbol_spec);
 898       break;
 899
 900     case G_TOKEN_ERROR:
 901       print_unexp = FALSE;
 902       expected_token = G_TOKEN_NONE;
 903       switch (scanner->value.v_error)
 904         {
 905         case G_ERR_UNEXP_EOF:
 906           _g_snprintf (token_string, token_string_len, "scanner: unexpected end of file");
 907           break;
 908
 909         case G_ERR_UNEXP_EOF_IN_STRING:
 910           _g_snprintf (token_string, token_string_len, "scanner: unterminated string constant");
 911           break;
 912
 913         case G_ERR_UNEXP_EOF_IN_COMMENT:
 914           _g_snprintf (token_string, token_string_len, "scanner: unterminated comment");
 915           break;
 916
 917         case G_ERR_NON_DIGIT_IN_CONST:
 918           _g_snprintf (token_string, token_string_len, "scanner: non digit in constant");
 919           break;
 920
 921         case G_ERR_FLOAT_RADIX:
 922           _g_snprintf (token_string, token_string_len, "scanner: invalid radix for floating constant");
 923           break;
 924
 925         case G_ERR_FLOAT_MALFORMED:
 926           _g_snprintf (token_string, token_string_len, "scanner: malformed floating constant");
 927           break;
 928
 929         case G_ERR_DIGIT_RADIX:
 930           _g_snprintf (token_string, token_string_len, "scanner: digit is beyond radix");
 931           break;
 932
 933         case G_ERR_UNKNOWN:
 934         default:
 935           _g_snprintf (token_string, token_string_len, "scanner: unknown error");
 936           break;
 937         }
 938       break;
 939
 940     case G_TOKEN_CHAR:
 941       _g_snprintf (token_string, token_string_len, "character `%c'", scanner->value.v_char);
 942       break;
 943
 944     case G_TOKEN_IDENTIFIER:
 945     case G_TOKEN_IDENTIFIER_NULL:
 946       if (expected_token == G_TOKEN_IDENTIFIER ||
 947           expected_token == G_TOKEN_IDENTIFIER_NULL)
 948         print_unexp = FALSE;
 949       _g_snprintf (token_string,
 950                   token_string_len,
 951                   "%s%s `%s'",
 952                   print_unexp ? "" : "invalid ",
 953                   identifier_spec,
 954                   scanner->token == G_TOKEN_IDENTIFIER ? scanner->value.v_string : "null");
 955       break;
 956
 957     case G_TOKEN_BINARY:
 958     case G_TOKEN_OCTAL:
 959     case G_TOKEN_INT:
 960     case G_TOKEN_HEX:
 961       if (scanner->config->store_int64)
 962         _g_snprintf (token_string, token_string_len, "number `%llu'", scanner->value.v_int64);
 963       else
 964         _g_snprintf (token_string, token_string_len, "number `%lu'", scanner->value.v_int);
 965       break;
 966
 967     case G_TOKEN_FLOAT:
 968       _g_snprintf (token_string, token_string_len, "number `%.3f'", scanner->value.v_float);
 969       break;
 970
 971     case G_TOKEN_STRING:
 972       if (expected_token == G_TOKEN_STRING)
 973         print_unexp = FALSE;
 974       _g_snprintf (token_string,
 975                    token_string_len,
 976                    "%s%sstring constant \"%s\"",
 977                    print_unexp ? "" : "invalid ",
 978                    scanner->value.v_string[0] == 0 ? "empty " : "",
 979                    scanner->value.v_string);
 980       token_string[token_string_len - 2] = '"';
 981       token_string[token_string_len - 1] = 0;
 982       break;
 983
 984     case G_TOKEN_COMMENT_SINGLE:
 985     case G_TOKEN_COMMENT_MULTI:
 986       _g_snprintf (token_string, token_string_len, "comment");
 987       break;
 988
 989     case G_TOKEN_NONE:
 990       /* somehow the user's parsing code is screwed, there isn't much
 991        * we can do about it.
 992        * Note, a common case to trigger this is
 993        * g_scanner_peek_next_token(); g_scanner_unexp_token();
 994        * without an intermediate g_scanner_get_next_token().
 995        */
 996       g_assert_not_reached ();
 997       break;
 998     }
 999
1000
1001   switch (expected_token)
1002     {
1003       gboolean need_valid;
1004       gchar *tstring;
1005     case G_TOKEN_EOF:
1006       _g_snprintf (expected_string, expected_string_len, "end of file");
1007       break;
1008     default:
1009       if (expected_token >= 1 && expected_token <= 255)
1010         {
1011           if ((expected_token >= ' ' && expected_token <= '~') ||
1012               strchr (scanner->config->cset_identifier_first, expected_token) ||
1013               strchr (scanner->config->cset_identifier_nth, expected_token))
1014             _g_snprintf (expected_string, expected_string_len, "character `%c'", expected_token);
1015           else
1016             _g_snprintf (expected_string, expected_string_len, "character `\\%o'", expected_token);
1017           break;
1018         }
1019       else if (!scanner->config->symbol_2_token)
1020         {
1021           _g_snprintf (expected_string, expected_string_len, "(unknown) token <%d>", expected_token);
1022           break;
1023         }
1024       /* fall through */
1025     case G_TOKEN_SYMBOL:
1026       need_valid = (scanner->token == G_TOKEN_SYMBOL ||
1027                     (scanner->config->symbol_2_token &&
1028                      scanner->token > G_TOKEN_LAST));
1029       _g_snprintf (expected_string,
1030                    expected_string_len,
1031                    "%s%s",
1032                    need_valid ? "valid " : "",
1033                    symbol_spec);
1034       /* FIXME: should we attempt to lookup the symbol_name for symbol_2_token? */
1035       break;
1036     case G_TOKEN_CHAR:
1037       _g_snprintf (expected_string, expected_string_len, "%scharacter",
1038                    scanner->token == G_TOKEN_CHAR ? "valid " : "");
1039       break;
1040     case G_TOKEN_BINARY:
1041       tstring = "binary";
1042       _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1043                    scanner->token == expected_token ? "valid " : "", tstring);
1044       break;
1045     case G_TOKEN_OCTAL:
1046       tstring = "octal";
1047       _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1048                    scanner->token == expected_token ? "valid " : "", tstring);
1049       break;
1050     case G_TOKEN_INT:
1051       tstring = "integer";
1052       _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1053                    scanner->token == expected_token ? "valid " : "", tstring);
1054       break;
1055     case G_TOKEN_HEX:
1056       tstring = "hexadecimal";
1057       _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1058                    scanner->token == expected_token ? "valid " : "", tstring);
1059       break;
1060     case G_TOKEN_FLOAT:
1061       tstring = "float";
1062       _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1063                    scanner->token == expected_token ? "valid " : "", tstring);
1064       break;
1065     case G_TOKEN_STRING:
1066       _g_snprintf (expected_string,
1067                    expected_string_len,
1068                    "%sstring constant",
1069                    scanner->token == G_TOKEN_STRING ? "valid " : "");
1070       break;
1071     case G_TOKEN_IDENTIFIER:
1072     case G_TOKEN_IDENTIFIER_NULL:
1073       need_valid = (scanner->token == G_TOKEN_IDENTIFIER_NULL ||
1074                     scanner->token == G_TOKEN_IDENTIFIER);
1075       _g_snprintf (expected_string,
1076                    expected_string_len,
1077                    "%s%s",
1078                    need_valid ? "valid " : "",
1079                    identifier_spec);
1080       break;
1081     case G_TOKEN_COMMENT_SINGLE:
1082       tstring = "single-line";
1083       _g_snprintf (expected_string, expected_string_len, "%scomment (%s)",
1084                    scanner->token == expected_token ? "valid " : "", tstring);
1085       break;
1086     case G_TOKEN_COMMENT_MULTI:
1087       tstring = "multi-line";
1088       _g_snprintf (expected_string, expected_string_len, "%scomment (%s)",
1089                    scanner->token == expected_token ? "valid " : "", tstring);
1090       break;
1091     case G_TOKEN_NONE:
1092     case G_TOKEN_ERROR:
1093       /* this is handled upon printout */
1094       break;
1095     }
1096
1097   if (message && message[0] != 0)
1098     message_prefix = " - ";
1099   else
1100     {
1101       message_prefix = "";
1102       message = "";
1103     }
1104   if (expected_token == G_TOKEN_ERROR)
1105     {
1106       msg_handler (scanner,
1107                    "failure around %s%s%s",
1108                    token_string,
1109                    message_prefix,
1110                    message);
1111     }
1112   else if (expected_token == G_TOKEN_NONE)
1113     {
1114       if (print_unexp)
1115         msg_handler (scanner,
1116                      "unexpected %s%s%s",
1117                      token_string,
1118                      message_prefix,
1119                      message);
1120       else
1121         msg_handler (scanner,
1122                      "%s%s%s",
1123                      token_string,
1124                      message_prefix,
1125                      message);
1126     }
1127   else
1128     {
1129       if (print_unexp)
1130         msg_handler (scanner,
1131                      "unexpected %s, expected %s%s%s",
1132                      token_string,
1133                      expected_string,
1134                      message_prefix,
1135                      message);
1136       else
1137         msg_handler (scanner,
1138                      "%s, expected %s%s%s",
1139                      token_string,
1140                      expected_string,
1141                      message_prefix,
1142                      message);
1143     }
1144
1145   g_free (token_string);
1146   g_free (expected_string);
1147 }
1148
1149 static void
1150 g_scanner_get_token_i (GScanner *scanner,
1151                        GTokenType       *token_p,
1152                        GTokenValue      *value_p,
1153                        guint            *line_p,
1154                        guint            *position_p)
1155 {
1156   do
1157     {
1158       g_scanner_free_value (token_p, value_p);
1159       g_scanner_get_token_ll (scanner, token_p, value_p, line_p, position_p);
1160     }
1161   while (((*token_p > 0 && *token_p < 256) &&
1162           strchr (scanner->config->cset_skip_characters, *token_p)) ||
1163          (*token_p == G_TOKEN_CHAR &&
1164           strchr (scanner->config->cset_skip_characters, value_p->v_char)) ||
1165          (*token_p == G_TOKEN_COMMENT_MULTI &&
1166           scanner->config->skip_comment_multi) ||
1167          (*token_p == G_TOKEN_COMMENT_SINGLE &&
1168           scanner->config->skip_comment_single));
1169
1170   switch (*token_p)
1171     {
1172     case G_TOKEN_IDENTIFIER:
1173       if (scanner->config->identifier_2_string)
1174         *token_p = G_TOKEN_STRING;
1175       break;
1176
1177     case G_TOKEN_SYMBOL:
1178       if (scanner->config->symbol_2_token)
1179         *token_p = (GTokenType) value_p->v_symbol;
1180       break;
1181
1182     case G_TOKEN_BINARY:
1183     case G_TOKEN_OCTAL:
1184     case G_TOKEN_HEX:
1185       if (scanner->config->numbers_2_int)
1186         *token_p = G_TOKEN_INT;
1187       break;
1188
1189     default:
1190       break;
1191     }
1192
1193   if (*token_p == G_TOKEN_INT &&
1194       scanner->config->int_2_float)
1195     {
1196       *token_p = G_TOKEN_FLOAT;
1197       if (scanner->config->store_int64)
1198         value_p->v_float = value_p->v_int64;
1199       else
1200         value_p->v_float = value_p->v_int;
1201     }
1202
1203   errno = 0;
1204 }
1205
1206 static void
1207 g_scanner_get_token_ll  (GScanner       *scanner,
1208                          GTokenType     *token_p,
1209                          GTokenValue    *value_p,
1210                          guint          *line_p,
1211                          guint          *position_p)
1212 {
1213   GScannerConfig *config;
1214   GTokenType       token;
1215   gboolean         in_comment_multi;
1216   gboolean         in_comment_single;
1217   gboolean         in_string_sq;
1218   gboolean         in_string_dq;
1219   GString         *gstring;
1220   GTokenValue      value;
1221   guchar           ch;
1222
1223   config = scanner->config;
1224   (*value_p).v_int64 = 0;
1225
1226   if ((scanner->text >= scanner->text_end && scanner->input_fd < 0) ||
1227       scanner->token == G_TOKEN_EOF)
1228     {
1229       *token_p = G_TOKEN_EOF;
1230       return;
1231     }
1232
1233   in_comment_multi = FALSE;
1234   in_comment_single = FALSE;
1235   in_string_sq = FALSE;
1236   in_string_dq = FALSE;
1237   gstring = NULL;
1238
1239   do /* while (ch != 0) */
1240     {
1241       gboolean dotted_float = FALSE;
1242
1243       ch = g_scanner_get_char (scanner, line_p, position_p);
1244
1245       value.v_int64 = 0;
1246       token = G_TOKEN_NONE;
1247
1248       /* this is *evil*, but needed ;(
1249        * we first check for identifier first character, because  it
1250        * might interfere with other key chars like slashes or numbers
1251        */
1252       if (config->scan_identifier &&
1253           ch && strchr (config->cset_identifier_first, ch))
1254         goto identifier_precedence;
1255
1256       switch (ch)
1257         {
1258         case 0:
1259           token = G_TOKEN_EOF;
1260           (*position_p)++;
1261           /* ch = 0; */
1262           break;
1263
1264         case '/':
1265           if (!config->scan_comment_multi ||
1266               g_scanner_peek_next_char (scanner) != '*')
1267             goto default_case;
1268           g_scanner_get_char (scanner, line_p, position_p);
1269           token = G_TOKEN_COMMENT_MULTI;
1270           in_comment_multi = TRUE;
1271           gstring = g_string_new ("");
1272           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1273             {
1274               if (ch == '*' && g_scanner_peek_next_char (scanner) == '/')
1275                 {
1276                   g_scanner_get_char (scanner, line_p, position_p);
1277                   in_comment_multi = FALSE;
1278                   break;
1279                 }
1280               else
1281                 gstring = g_string_append_c (gstring, ch);
1282             }
1283           ch = 0;
1284           break;
1285
1286         case '\'':
1287           if (!config->scan_string_sq)
1288             goto default_case;
1289           token = G_TOKEN_STRING;
1290           in_string_sq = TRUE;
1291           gstring = g_string_new ("");
1292           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1293             {
1294               if (ch == '\'')
1295                 {
1296                   in_string_sq = FALSE;
1297                   break;
1298                 }
1299               else
1300                 gstring = g_string_append_c (gstring, ch);
1301             }
1302           ch = 0;
1303           break;
1304
1305         case '"':
1306           if (!config->scan_string_dq)
1307             goto default_case;
1308           token = G_TOKEN_STRING;
1309           in_string_dq = TRUE;
1310           gstring = g_string_new ("");
1311           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1312             {
1313               if (ch == '"')
1314                 {
1315                   in_string_dq = FALSE;
1316                   break;
1317                 }
1318               else
1319                 {
1320                   if (ch == '\\')
1321                     {
1322                       ch = g_scanner_get_char (scanner, line_p, position_p);
1323                       switch (ch)
1324                         {
1325                           guint i;
1326                           guint fchar;
1327
1328                         case 0:
1329                           break;
1330
1331                         case '\\':
1332                           gstring = g_string_append_c (gstring, '\\');
1333                           break;
1334
1335                         case 'n':
1336                           gstring = g_string_append_c (gstring, '\n');
1337                           break;
1338
1339                         case 't':
1340                           gstring = g_string_append_c (gstring, '\t');
1341                           break;
1342
1343                         case 'r':
1344                           gstring = g_string_append_c (gstring, '\r');
1345                           break;
1346
1347                         case 'b':
1348                           gstring = g_string_append_c (gstring, '\b');
1349                           break;
1350
1351                         case 'f':
1352                           gstring = g_string_append_c (gstring, '\f');
1353                           break;
1354
1355                         case '0':
1356                         case '1':
1357                         case '2':
1358                         case '3':
1359                         case '4':
1360                         case '5':
1361                         case '6':
1362                         case '7':
1363                           i = ch - '0';
1364                           fchar = g_scanner_peek_next_char (scanner);
1365                           if (fchar >= '0' && fchar <= '7')
1366                             {
1367                               ch = g_scanner_get_char (scanner, line_p, position_p);
1368                               i = i * 8 + ch - '0';
1369                               fchar = g_scanner_peek_next_char (scanner);
1370                               if (fchar >= '0' && fchar <= '7')
1371                                 {
1372                                   ch = g_scanner_get_char (scanner, line_p, position_p);
1373                                   i = i * 8 + ch - '0';
1374                                 }
1375                             }
1376                           gstring = g_string_append_c (gstring, i);
1377                           break;
1378
1379                         default:
1380                           gstring = g_string_append_c (gstring, ch);
1381                           break;
1382                         }
1383                     }
1384                   else
1385                     gstring = g_string_append_c (gstring, ch);
1386                 }
1387             }
1388           ch = 0;
1389           break;
1390
1391         case '.':
1392           if (!config->scan_float)
1393             goto default_case;
1394           token = G_TOKEN_FLOAT;
1395           dotted_float = TRUE;
1396           ch = g_scanner_get_char (scanner, line_p, position_p);
1397           goto number_parsing;
1398
1399         case '$':
1400           if (!config->scan_hex_dollar)
1401             goto default_case;
1402           token = G_TOKEN_HEX;
1403           ch = g_scanner_get_char (scanner, line_p, position_p);
1404           goto number_parsing;
1405
1406         case '0':
1407           if (config->scan_octal)
1408             token = G_TOKEN_OCTAL;
1409           else
1410             token = G_TOKEN_INT;
1411           ch = g_scanner_peek_next_char (scanner);
1412           if (config->scan_hex && (ch == 'x' || ch == 'X'))
1413             {
1414               token = G_TOKEN_HEX;
1415               g_scanner_get_char (scanner, line_p, position_p);
1416               ch = g_scanner_get_char (scanner, line_p, position_p);
1417               if (ch == 0)
1418                 {
1419                   token = G_TOKEN_ERROR;
1420                   value.v_error = G_ERR_UNEXP_EOF;
1421                   (*position_p)++;
1422                   break;
1423                 }
1424               if (g_scanner_char_2_num (ch, 16) < 0)
1425                 {
1426                   token = G_TOKEN_ERROR;
1427                   value.v_error = G_ERR_DIGIT_RADIX;
1428                   ch = 0;
1429                   break;
1430                 }
1431             }
1432           else if (config->scan_binary && (ch == 'b' || ch == 'B'))
1433             {
1434               token = G_TOKEN_BINARY;
1435               g_scanner_get_char (scanner, line_p, position_p);
1436               ch = g_scanner_get_char (scanner, line_p, position_p);
1437               if (ch == 0)
1438                 {
1439                   token = G_TOKEN_ERROR;
1440                   value.v_error = G_ERR_UNEXP_EOF;
1441                   (*position_p)++;
1442                   break;
1443                 }
1444               if (g_scanner_char_2_num (ch, 10) < 0)
1445                 {
1446                   token = G_TOKEN_ERROR;
1447                   value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1448                   ch = 0;
1449                   break;
1450                 }
1451             }
1452           else
1453             ch = '0';
1454           /* fall through */
1455         case '1':
1456         case '2':
1457         case '3':
1458         case '4':
1459         case '5':
1460         case '6':
1461         case '7':
1462         case '8':
1463         case '9':
1464         number_parsing:
1465         {
1466           gboolean in_number = TRUE;
1467           gchar *endptr;
1468
1469           if (token == G_TOKEN_NONE)
1470             token = G_TOKEN_INT;
1471
1472           gstring = g_string_new (dotted_float ? "0." : "");
1473           gstring = g_string_append_c (gstring, ch);
1474
1475           do /* while (in_number) */
1476             {
1477               gboolean is_E;
1478
1479               is_E = token == G_TOKEN_FLOAT && (ch == 'e' || ch == 'E');
1480
1481               ch = g_scanner_peek_next_char (scanner);
1482
1483               if (g_scanner_char_2_num (ch, 36) >= 0 ||
1484                   (config->scan_float && ch == '.') ||
1485                   (is_E && (ch == '+' || ch == '-')))
1486                 {
1487                   ch = g_scanner_get_char (scanner, line_p, position_p);
1488
1489                   switch (ch)
1490                     {
1491                     case '.':
1492                       if (token != G_TOKEN_INT && token != G_TOKEN_OCTAL)
1493                         {
1494                           value.v_error = token == G_TOKEN_FLOAT ? G_ERR_FLOAT_MALFORMED : G_ERR_FLOAT_RADIX;
1495                           token = G_TOKEN_ERROR;
1496                           in_number = FALSE;
1497                         }
1498                       else
1499                         {
1500                           token = G_TOKEN_FLOAT;
1501                           gstring = g_string_append_c (gstring, ch);
1502                         }
1503                       break;
1504
1505                     case '0':
1506                     case '1':
1507                     case '2':
1508                     case '3':
1509                     case '4':
1510                     case '5':
1511                     case '6':
1512                     case '7':
1513                     case '8':
1514                     case '9':
1515                       gstring = g_string_append_c (gstring, ch);
1516                       break;
1517
1518                     case '-':
1519                     case '+':
1520                       if (token != G_TOKEN_FLOAT)
1521                         {
1522                           token = G_TOKEN_ERROR;
1523                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1524                           in_number = FALSE;
1525                         }
1526                       else
1527                         gstring = g_string_append_c (gstring, ch);
1528                       break;
1529
1530                     case 'e':
1531                     case 'E':
1532                       if ((token != G_TOKEN_HEX && !config->scan_float) ||
1533                           (token != G_TOKEN_HEX &&
1534                            token != G_TOKEN_OCTAL &&
1535                            token != G_TOKEN_FLOAT &&
1536                            token != G_TOKEN_INT))
1537                         {
1538                           token = G_TOKEN_ERROR;
1539                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1540                           in_number = FALSE;
1541                         }
1542                       else
1543                         {
1544                           if (token != G_TOKEN_HEX)
1545                             token = G_TOKEN_FLOAT;
1546                           gstring = g_string_append_c (gstring, ch);
1547                         }
1548                       break;
1549
1550                     default:
1551                       if (token != G_TOKEN_HEX)
1552                         {
1553                           token = G_TOKEN_ERROR;
1554                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1555                           in_number = FALSE;
1556                         }
1557                       else
1558                         gstring = g_string_append_c (gstring, ch);
1559                       break;
1560                     }
1561                 }
1562               else
1563                 in_number = FALSE;
1564             }
1565           while (in_number);
1566
1567           endptr = NULL;
1568           if (token == G_TOKEN_FLOAT)
1569             value.v_float = g_strtod (gstring->str, &endptr);
1570           else
1571             {
1572               guint64 ui64 = 0;
1573               switch (token)
1574                 {
1575                 case G_TOKEN_BINARY:
1576                   ui64 = g_ascii_strtoull (gstring->str, &endptr, 2);
1577                   break;
1578                 case G_TOKEN_OCTAL:
1579                   ui64 = g_ascii_strtoull (gstring->str, &endptr, 8);
1580                   break;
1581                 case G_TOKEN_INT:
1582                   ui64 = g_ascii_strtoull (gstring->str, &endptr, 10);
1583                   break;
1584                 case G_TOKEN_HEX:
1585                   ui64 = g_ascii_strtoull (gstring->str, &endptr, 16);
1586                   break;
1587                 default: ;
1588                 }
1589               if (scanner->config->store_int64)
1590                 value.v_int64 = ui64;
1591               else
1592                 value.v_int = ui64;
1593             }
1594           if (endptr && *endptr)
1595             {
1596               token = G_TOKEN_ERROR;
1597               if (*endptr == 'e' || *endptr == 'E')
1598                 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1599               else
1600                 value.v_error = G_ERR_DIGIT_RADIX;
1601             }
1602           g_string_free (gstring, TRUE);
1603           gstring = NULL;
1604           ch = 0;
1605         } /* number_parsing:... */
1606         break;
1607
1608         default:
1609         default_case:
1610         {
1611           if (config->cpair_comment_single &&
1612               ch == config->cpair_comment_single[0])
1613             {
1614               token = G_TOKEN_COMMENT_SINGLE;
1615               in_comment_single = TRUE;
1616               gstring = g_string_new ("");
1617               ch = g_scanner_get_char (scanner, line_p, position_p);
1618               while (ch != 0)
1619                 {
1620                   if (ch == config->cpair_comment_single[1])
1621                     {
1622                       in_comment_single = FALSE;
1623                       ch = 0;
1624                       break;
1625                     }
1626
1627                   gstring = g_string_append_c (gstring, ch);
1628                   ch = g_scanner_get_char (scanner, line_p, position_p);
1629                 }
1630             }
1631           else if (config->scan_identifier && ch &&
1632                    strchr (config->cset_identifier_first, ch))
1633             {
1634             identifier_precedence:
1635
1636               if (config->cset_identifier_nth && ch &&
1637                   strchr (config->cset_identifier_nth,
1638                           g_scanner_peek_next_char (scanner)))
1639                 {
1640                   token = G_TOKEN_IDENTIFIER;
1641                   gstring = g_string_new ("");
1642                   gstring = g_string_append_c (gstring, ch);
1643                   do
1644                     {
1645                       ch = g_scanner_get_char (scanner, line_p, position_p);
1646                       gstring = g_string_append_c (gstring, ch);
1647                       ch = g_scanner_peek_next_char (scanner);
1648                     }
1649                   while (ch && strchr (config->cset_identifier_nth, ch));
1650                   ch = 0;
1651                 }
1652               else if (config->scan_identifier_1char)
1653                 {
1654                   token = G_TOKEN_IDENTIFIER;
1655                   value.v_identifier = g_new0 (gchar, 2);
1656                   value.v_identifier[0] = ch;
1657                   ch = 0;
1658                 }
1659             }
1660           if (ch)
1661             {
1662               if (config->char_2_token)
1663                 token = ch;
1664               else
1665                 {
1666                   token = G_TOKEN_CHAR;
1667                   value.v_char = ch;
1668                 }
1669               ch = 0;
1670             }
1671         } /* default_case:... */
1672         break;
1673         }
1674       g_assert (ch == 0 && token != G_TOKEN_NONE); /* paranoid */
1675     }
1676   while (ch != 0);
1677
1678   if (in_comment_multi || in_comment_single ||
1679       in_string_sq || in_string_dq)
1680     {
1681       token = G_TOKEN_ERROR;
1682       if (gstring)
1683         {
1684           g_string_free (gstring, TRUE);
1685           gstring = NULL;
1686         }
1687       (*position_p)++;
1688       if (in_comment_multi || in_comment_single)
1689         value.v_error = G_ERR_UNEXP_EOF_IN_COMMENT;
1690       else /* (in_string_sq || in_string_dq) */
1691         value.v_error = G_ERR_UNEXP_EOF_IN_STRING;
1692     }
1693
1694   if (gstring)
1695     {
1696       value.v_string = gstring->str;
1697       g_string_free (gstring, FALSE);
1698       gstring = NULL;
1699     }
1700
1701   if (token == G_TOKEN_IDENTIFIER)
1702     {
1703       if (config->scan_symbols)
1704         {
1705           GScannerKey *key;
1706           guint scope_id;
1707
1708           scope_id = scanner->scope_id;
1709           key = g_scanner_lookup_internal (scanner, scope_id, value.v_identifier);
1710           if (!key && scope_id && scanner->config->scope_0_fallback)
1711             key = g_scanner_lookup_internal (scanner, 0, value.v_identifier);
1712
1713           if (key)
1714             {
1715               g_free (value.v_identifier);
1716               token = G_TOKEN_SYMBOL;
1717               value.v_symbol = key->value;
1718             }
1719         }
1720
1721       if (token == G_TOKEN_IDENTIFIER &&
1722           config->scan_identifier_NULL &&
1723           strlen (value.v_identifier) == 4)
1724         {
1725           gchar *null_upper = "NULL";
1726           gchar *null_lower = "null";
1727
1728           if (scanner->config->case_sensitive)
1729             {
1730               if (value.v_identifier[0] == null_upper[0] &&
1731                   value.v_identifier[1] == null_upper[1] &&
1732                   value.v_identifier[2] == null_upper[2] &&
1733                   value.v_identifier[3] == null_upper[3])
1734                 token = G_TOKEN_IDENTIFIER_NULL;
1735             }
1736           else
1737             {
1738               if ((value.v_identifier[0] == null_upper[0] ||
1739                    value.v_identifier[0] == null_lower[0]) &&
1740                   (value.v_identifier[1] == null_upper[1] ||
1741                    value.v_identifier[1] == null_lower[1]) &&
1742                   (value.v_identifier[2] == null_upper[2] ||
1743                    value.v_identifier[2] == null_lower[2]) &&
1744                   (value.v_identifier[3] == null_upper[3] ||
1745                    value.v_identifier[3] == null_lower[3]))
1746                 token = G_TOKEN_IDENTIFIER_NULL;
1747             }
1748         }
1749     }
1750
1751   *token_p = token;
1752   *value_p = value;
1753 }