gscanner.c

   1 /* GLIB - Library of useful routines for C programming
   2  * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
   3  *
   4  * GScanner: Flexible lexical scanner for general purpose.
   5  * Copyright (C) 1997, 1998 Tim Janik
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Library General Public
  18  * License along with this library; if not, write to the
  19  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  20  * Boston, MA 02111-1307, USA.
  21  */
  22
  23 /*
  24  * MT safe
  25  */
  26
  27 #define         __gscanner_c__
  28
  29 #ifdef HAVE_CONFIG_H
  30 #include <config.h>
  31 #endif
  32
  33 #include        <stdlib.h>
  34 #include        <stdarg.h>
  35 #include        <string.h>
  36 #include        <stdio.h>
  37 #include        "glib.h"
  38 #ifdef HAVE_UNISTD_H
  39 #include        <unistd.h>
  40 #endif
  41 #include        <errno.h>
  42 #include        <sys/types.h>   /* needed for sys/stat.h */
  43 #include        <sys/stat.h>
  44 #ifdef _MSC_VER
  45 #include        <io.h>          /* For _read() */
  46 #endif
  47
  48 /* --- defines --- */
  49 #define to_lower(c)                             ( \
  50         (guchar) (                                                      \
  51           ( (((guchar)(c))>='A' && ((guchar)(c))<='Z') * ('a'-'A') ) |  \
  52           ( (((guchar)(c))>=192 && ((guchar)(c))<=214) * (224-192) ) |  \
  53           ( (((guchar)(c))>=216 && ((guchar)(c))<=222) * (248-216) ) |  \
  54           ((guchar)(c))                                                 \
  55         )                                                               \
  56 )
  57 #define READ_BUFFER_SIZE        (4000)
  58
  59
  60 /* --- typedefs --- */
  61 typedef struct  _GScannerKey    GScannerKey;
  62
  63 struct  _GScannerKey
  64 {
  65   guint          scope_id;
  66   gchar         *symbol;
  67   gpointer       value;
  68 };
  69
  70
  71
  72 /* --- variables --- */
  73 static GScannerConfig g_scanner_config_template =
  74 {
  75   (
  76    " \t\r\n"
  77    )                    /* cset_skip_characters */,
  78   (
  79    G_CSET_a_2_z
  80    "_"
  81    G_CSET_A_2_Z
  82    )                    /* cset_identifier_first */,
  83   (
  84    G_CSET_a_2_z
  85    "_0123456789"
  86    G_CSET_A_2_Z
  87    G_CSET_LATINS
  88    G_CSET_LATINC
  89    )                    /* cset_identifier_nth */,
  90   ( "#\n" )             /* cpair_comment_single */,
  91
  92   FALSE                 /* case_sensitive */,
  93
  94   TRUE                  /* skip_comment_multi */,
  95   TRUE                  /* skip_comment_single */,
  96   TRUE                  /* scan_comment_multi */,
  97   TRUE                  /* scan_identifier */,
  98   FALSE                 /* scan_identifier_1char */,
  99   FALSE                 /* scan_identifier_NULL */,
 100   TRUE                  /* scan_symbols */,
 101   FALSE                 /* scan_binary */,
 102   TRUE                  /* scan_octal */,
 103   TRUE                  /* scan_float */,
 104   TRUE                  /* scan_hex */,
 105   FALSE                 /* scan_hex_dollar */,
 106   TRUE                  /* scan_string_sq */,
 107   TRUE                  /* scan_string_dq */,
 108   TRUE                  /* numbers_2_int */,
 109   FALSE                 /* int_2_float */,
 110   FALSE                 /* identifier_2_string */,
 111   TRUE                  /* char_2_token */,
 112   FALSE                 /* symbol_2_token */,
 113   FALSE                 /* scope_0_fallback */,
 114 };
 115
 116
 117 /* --- prototypes --- */
 118 static inline
 119 GScannerKey*    g_scanner_lookup_internal (GScanner     *scanner,
 120                                            guint         scope_id,
 121                                            const gchar  *symbol);
 122 static gint     g_scanner_key_equal       (gconstpointer v1,
 123                                            gconstpointer v2);
 124 static guint    g_scanner_key_hash        (gconstpointer v);
 125 static void     g_scanner_get_token_ll    (GScanner     *scanner,
 126                                            GTokenType   *token_p,
 127                                            GTokenValue  *value_p,
 128                                            guint        *line_p,
 129                                            guint        *position_p);
 130 static void     g_scanner_get_token_i     (GScanner     *scanner,
 131                                            GTokenType   *token_p,
 132                                            GTokenValue  *value_p,
 133                                            guint        *line_p,
 134                                            guint        *position_p);
 135
 136 static guchar   g_scanner_peek_next_char  (GScanner     *scanner);
 137 static guchar   g_scanner_get_char        (GScanner     *scanner,
 138                                            guint        *line_p,
 139                                            guint        *position_p);
 140 static void     g_scanner_msg_handler     (GScanner     *scanner,
 141                                            gchar        *message,
 142                                            gint          is_error);
 143
 144
 145 /* --- functions --- */
 146 static inline gint
 147 g_scanner_char_2_num (guchar    c,
 148                       guchar    base)
 149 {
 150   if (c >= '0' && c <= '9')
 151     c -= '0';
 152   else if (c >= 'A' && c <= 'Z')
 153     c -= 'A' - 10;
 154   else if (c >= 'a' && c <= 'z')
 155     c -= 'a' - 10;
 156   else
 157     return -1;
 158
 159   if (c < base)
 160     return c;
 161
 162   return -1;
 163 }
 164
 165 GScanner*
 166 g_scanner_new (GScannerConfig   *config_templ)
 167 {
 168   GScanner *scanner;
 169
 170   if (!config_templ)
 171     config_templ = &g_scanner_config_template;
 172
 173   scanner = g_new0 (GScanner, 1);
 174
 175   scanner->user_data = NULL;
 176   scanner->max_parse_errors = 0;
 177   scanner->parse_errors = 0;
 178   scanner->input_name = NULL;
 179   scanner->derived_data = NULL;
 180
 181   scanner->config = g_new0 (GScannerConfig, 1);
 182
 183   scanner->config->case_sensitive       = config_templ->case_sensitive;
 184   scanner->config->cset_skip_characters = config_templ->cset_skip_characters;
 185   scanner->config->cset_identifier_first= config_templ->cset_identifier_first;
 186   scanner->config->cset_identifier_nth  = config_templ->cset_identifier_nth;
 187   scanner->config->cpair_comment_single = config_templ->cpair_comment_single;
 188   scanner->config->skip_comment_multi   = config_templ->skip_comment_multi;
 189   scanner->config->skip_comment_single  = config_templ->skip_comment_single;
 190   scanner->config->scan_comment_multi   = config_templ->scan_comment_multi;
 191   scanner->config->scan_identifier      = config_templ->scan_identifier;
 192   scanner->config->scan_identifier_1char= config_templ->scan_identifier_1char;
 193   scanner->config->scan_identifier_NULL = config_templ->scan_identifier_NULL;
 194   scanner->config->scan_symbols         = config_templ->scan_symbols;
 195   scanner->config->scan_binary          = config_templ->scan_binary;
 196   scanner->config->scan_octal           = config_templ->scan_octal;
 197   scanner->config->scan_float           = config_templ->scan_float;
 198   scanner->config->scan_hex             = config_templ->scan_hex;
 199   scanner->config->scan_hex_dollar      = config_templ->scan_hex_dollar;
 200   scanner->config->scan_string_sq       = config_templ->scan_string_sq;
 201   scanner->config->scan_string_dq       = config_templ->scan_string_dq;
 202   scanner->config->numbers_2_int        = config_templ->numbers_2_int;
 203   scanner->config->int_2_float          = config_templ->int_2_float;
 204   scanner->config->identifier_2_string  = config_templ->identifier_2_string;
 205   scanner->config->char_2_token         = config_templ->char_2_token;
 206   scanner->config->symbol_2_token       = config_templ->symbol_2_token;
 207   scanner->config->scope_0_fallback     = config_templ->scope_0_fallback;
 208
 209   scanner->token = G_TOKEN_NONE;
 210   scanner->value.v_int = 0;
 211   scanner->line = 1;
 212   scanner->position = 0;
 213
 214   scanner->next_token = G_TOKEN_NONE;
 215   scanner->next_value.v_int = 0;
 216   scanner->next_line = 1;
 217   scanner->next_position = 0;
 218
 219   scanner->symbol_table = g_hash_table_new (g_scanner_key_hash, g_scanner_key_equal);
 220   scanner->input_fd = -1;
 221   scanner->text = NULL;
 222   scanner->text_end = NULL;
 223   scanner->buffer = NULL;
 224   scanner->scope_id = 0;
 225
 226   scanner->msg_handler = g_scanner_msg_handler;
 227
 228   return scanner;
 229 }
 230
 231 static inline void
 232 g_scanner_free_value (GTokenType     *token_p,
 233                       GTokenValue     *value_p)
 234 {
 235   switch (*token_p)
 236     {
 237     case G_TOKEN_STRING:
 238     case G_TOKEN_IDENTIFIER:
 239     case G_TOKEN_IDENTIFIER_NULL:
 240     case G_TOKEN_COMMENT_SINGLE:
 241     case G_TOKEN_COMMENT_MULTI:
 242       g_free (value_p->v_string);
 243       break;
 244
 245     default:
 246       break;
 247     }
 248
 249   *token_p = G_TOKEN_NONE;
 250 }
 251
 252 static void
 253 g_scanner_destroy_symbol_table_entry (gpointer _key,
 254                                       gpointer _value,
 255                                       gpointer _data)
 256 {
 257   GScannerKey *key = _key;
 258
 259   g_free (key->symbol);
 260   g_free (key);
 261 }
 262
 263 void
 264 g_scanner_destroy (GScanner     *scanner)
 265 {
 266   g_return_if_fail (scanner != NULL);
 267
 268   g_hash_table_foreach (scanner->symbol_table,
 269                         g_scanner_destroy_symbol_table_entry, NULL);
 270   g_hash_table_destroy (scanner->symbol_table);
 271   g_scanner_free_value (&scanner->token, &scanner->value);
 272   g_scanner_free_value (&scanner->next_token, &scanner->next_value);
 273   g_free (scanner->config);
 274   g_free (scanner->buffer);
 275   g_free (scanner);
 276 }
 277
 278 static void
 279 g_scanner_msg_handler (GScanner         *scanner,
 280                        gchar            *message,
 281                        gint              is_error)
 282 {
 283   g_return_if_fail (scanner != NULL);
 284
 285   fprintf (stdout, "%s:%d: ", scanner->input_name, scanner->line);
 286   if (is_error)
 287     fprintf (stdout, "error: ");
 288   fprintf (stdout, "%s\n", message);
 289 }
 290
 291 void
 292 g_scanner_error (GScanner       *scanner,
 293                  const gchar    *format,
 294                  ...)
 295 {
 296   g_return_if_fail (scanner != NULL);
 297   g_return_if_fail (format != NULL);
 298
 299   scanner->parse_errors++;
 300
 301   if (scanner->msg_handler)
 302     {
 303       va_list args;
 304       gchar *string;
 305
 306       va_start (args, format);
 307       string = g_strdup_vprintf (format, args);
 308       va_end (args);
 309
 310       scanner->msg_handler (scanner, string, TRUE);
 311
 312       g_free (string);
 313     }
 314 }
 315
 316 void
 317 g_scanner_warn (GScanner       *scanner,
 318                 const gchar    *format,
 319                 ...)
 320 {
 321   g_return_if_fail (scanner != NULL);
 322   g_return_if_fail (format != NULL);
 323
 324   if (scanner->msg_handler)
 325     {
 326       va_list args;
 327       gchar *string;
 328
 329       va_start (args, format);
 330       string = g_strdup_vprintf (format, args);
 331       va_end (args);
 332
 333       scanner->msg_handler (scanner, string, FALSE);
 334
 335       g_free (string);
 336     }
 337 }
 338
 339 static gint
 340 g_scanner_key_equal (gconstpointer v1,
 341                      gconstpointer v2)
 342 {
 343   const GScannerKey *key1 = v1;
 344   const GScannerKey *key2 = v2;
 345
 346   return (key1->scope_id == key2->scope_id) && (strcmp (key1->symbol, key2->symbol) == 0);
 347 }
 348
 349 static guint
 350 g_scanner_key_hash (gconstpointer v)
 351 {
 352   const GScannerKey *key = v;
 353   gchar *c;
 354   guint h;
 355
 356   h = key->scope_id;
 357   for (c = key->symbol; *c; c++)
 358     {
 359       guint g;
 360
 361       h = (h << 4) + *c;
 362       g = h & 0xf0000000;
 363       if (g)
 364         {
 365           h = h ^ (g >> 24);
 366           h = h ^ g;
 367         }
 368     }
 369
 370   return h;
 371 }
 372
 373 static inline GScannerKey*
 374 g_scanner_lookup_internal (GScanner     *scanner,
 375                            guint         scope_id,
 376                            const gchar  *symbol)
 377 {
 378   GScannerKey   *key_p;
 379   GScannerKey key;
 380
 381   key.scope_id = scope_id;
 382
 383   if (!scanner->config->case_sensitive)
 384     {
 385       gchar *d;
 386       const gchar *c;
 387
 388       key.symbol = g_new (gchar, strlen (symbol) + 1);
 389       for (d = key.symbol, c = symbol; *c; c++, d++)
 390         *d = to_lower (*c);
 391       *d = 0;
 392       key_p = g_hash_table_lookup (scanner->symbol_table, &key);
 393       g_free (key.symbol);
 394     }
 395   else
 396     {
 397       key.symbol = (gchar*) symbol;
 398       key_p = g_hash_table_lookup (scanner->symbol_table, &key);
 399     }
 400
 401   return key_p;
 402 }
 403
 404 void
 405 g_scanner_scope_add_symbol (GScanner    *scanner,
 406                             guint        scope_id,
 407                             const gchar *symbol,
 408                             gpointer     value)
 409 {
 410   GScannerKey   *key;
 411
 412   g_return_if_fail (scanner != NULL);
 413   g_return_if_fail (symbol != NULL);
 414
 415   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 416
 417   if (!key)
 418     {
 419       key = g_new (GScannerKey, 1);
 420       key->scope_id = scope_id;
 421       key->symbol = g_strdup (symbol);
 422       key->value = value;
 423       if (!scanner->config->case_sensitive)
 424         {
 425           gchar *c;
 426
 427           c = key->symbol;
 428           while (*c != 0)
 429             {
 430               *c = to_lower (*c);
 431               c++;
 432             }
 433         }
 434       g_hash_table_insert (scanner->symbol_table, key, key);
 435     }
 436   else
 437     key->value = value;
 438 }
 439
 440 void
 441 g_scanner_scope_remove_symbol (GScanner    *scanner,
 442                                guint        scope_id,
 443                                const gchar *symbol)
 444 {
 445   GScannerKey   *key;
 446
 447   g_return_if_fail (scanner != NULL);
 448   g_return_if_fail (symbol != NULL);
 449
 450   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 451
 452   if (key)
 453     {
 454       g_hash_table_remove (scanner->symbol_table, key);
 455       g_free (key->symbol);
 456       g_free (key);
 457     }
 458 }
 459
 460 gpointer
 461 g_scanner_lookup_symbol (GScanner       *scanner,
 462                          const gchar    *symbol)
 463 {
 464   GScannerKey   *key;
 465   guint scope_id;
 466
 467   g_return_val_if_fail (scanner != NULL, NULL);
 468
 469   if (!symbol)
 470     return NULL;
 471
 472   scope_id = scanner->scope_id;
 473   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 474   if (!key && scope_id && scanner->config->scope_0_fallback)
 475     key = g_scanner_lookup_internal (scanner, 0, symbol);
 476
 477   if (key)
 478     return key->value;
 479   else
 480     return NULL;
 481 }
 482
 483 gpointer
 484 g_scanner_scope_lookup_symbol (GScanner       *scanner,
 485                                guint           scope_id,
 486                                const gchar    *symbol)
 487 {
 488   GScannerKey   *key;
 489
 490   g_return_val_if_fail (scanner != NULL, NULL);
 491
 492   if (!symbol)
 493     return NULL;
 494
 495   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 496
 497   if (key)
 498     return key->value;
 499   else
 500     return NULL;
 501 }
 502
 503 guint
 504 g_scanner_set_scope (GScanner       *scanner,
 505                      guint           scope_id)
 506 {
 507   guint old_scope_id;
 508
 509   g_return_val_if_fail (scanner != NULL, 0);
 510
 511   old_scope_id = scanner->scope_id;
 512   scanner->scope_id = scope_id;
 513
 514   return old_scope_id;
 515 }
 516
 517 static void
 518 g_scanner_foreach_internal (gpointer  _key,
 519                             gpointer  _value,
 520                             gpointer  _user_data)
 521 {
 522   GScannerKey *key;
 523   gpointer *d;
 524   GHFunc func;
 525   gpointer user_data;
 526   guint *scope_id;
 527
 528   d = _user_data;
 529   func = (GHFunc) d[0];
 530   user_data = d[1];
 531   scope_id = d[2];
 532   key = _value;
 533
 534   if (key->scope_id == *scope_id)
 535     func (key->symbol, key->value, user_data);
 536 }
 537
 538 void
 539 g_scanner_scope_foreach_symbol (GScanner       *scanner,
 540                                 guint           scope_id,
 541                                 GHFunc          func,
 542                                 gpointer        user_data)
 543 {
 544   gpointer d[3];
 545
 546   g_return_if_fail (scanner != NULL);
 547
 548   d[0] = (gpointer) func;
 549   d[1] = user_data;
 550   d[2] = &scope_id;
 551
 552   g_hash_table_foreach (scanner->symbol_table, g_scanner_foreach_internal, d);
 553 }
 554
 555 void
 556 g_scanner_freeze_symbol_table (GScanner *scanner)
 557 {
 558   g_return_if_fail (scanner != NULL);
 559
 560   g_hash_table_freeze (scanner->symbol_table);
 561 }
 562
 563 void
 564 g_scanner_thaw_symbol_table (GScanner *scanner)
 565 {
 566   g_return_if_fail (scanner != NULL);
 567
 568   g_hash_table_thaw (scanner->symbol_table);
 569 }
 570
 571 GTokenType
 572 g_scanner_peek_next_token (GScanner     *scanner)
 573 {
 574   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 575
 576   if (scanner->next_token == G_TOKEN_NONE)
 577     {
 578       scanner->next_line = scanner->line;
 579       scanner->next_position = scanner->position;
 580       g_scanner_get_token_i (scanner,
 581                              &scanner->next_token,
 582                              &scanner->next_value,
 583                              &scanner->next_line,
 584                              &scanner->next_position);
 585     }
 586
 587   return scanner->next_token;
 588 }
 589
 590 GTokenType
 591 g_scanner_get_next_token (GScanner      *scanner)
 592 {
 593   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 594
 595   if (scanner->next_token != G_TOKEN_NONE)
 596     {
 597       g_scanner_free_value (&scanner->token, &scanner->value);
 598
 599       scanner->token = scanner->next_token;
 600       scanner->value = scanner->next_value;
 601       scanner->line = scanner->next_line;
 602       scanner->position = scanner->next_position;
 603       scanner->next_token = G_TOKEN_NONE;
 604     }
 605   else
 606     g_scanner_get_token_i (scanner,
 607                            &scanner->token,
 608                            &scanner->value,
 609                            &scanner->line,
 610                            &scanner->position);
 611
 612   return scanner->token;
 613 }
 614
 615 GTokenType
 616 g_scanner_cur_token (GScanner *scanner)
 617 {
 618   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 619
 620   return scanner->token;
 621 }
 622
 623 GTokenValue
 624 g_scanner_cur_value (GScanner *scanner)
 625 {
 626   GTokenValue v;
 627
 628   v.v_int = 0;
 629
 630   g_return_val_if_fail (scanner != NULL, v);
 631
 632   /* MSC isn't capable of handling return scanner->value; ? */
 633
 634   v = scanner->value;
 635
 636   return v;
 637 }
 638
 639 guint
 640 g_scanner_cur_line (GScanner *scanner)
 641 {
 642   g_return_val_if_fail (scanner != NULL, 0);
 643
 644   return scanner->line;
 645 }
 646
 647 guint
 648 g_scanner_cur_position (GScanner *scanner)
 649 {
 650   g_return_val_if_fail (scanner != NULL, 0);
 651
 652   return scanner->position;
 653 }
 654
 655 gboolean
 656 g_scanner_eof (GScanner *scanner)
 657 {
 658   g_return_val_if_fail (scanner != NULL, TRUE);
 659
 660   return scanner->token == G_TOKEN_EOF;
 661 }
 662
 663 void
 664 g_scanner_input_file (GScanner *scanner,
 665                       gint      input_fd)
 666 {
 667   g_return_if_fail (scanner != NULL);
 668   g_return_if_fail (input_fd >= 0);
 669
 670   if (scanner->input_fd >= 0)
 671     g_scanner_sync_file_offset (scanner);
 672
 673   scanner->token = G_TOKEN_NONE;
 674   scanner->value.v_int = 0;
 675   scanner->line = 1;
 676   scanner->position = 0;
 677   scanner->next_token = G_TOKEN_NONE;
 678
 679   scanner->input_fd = input_fd;
 680   scanner->text = NULL;
 681   scanner->text_end = NULL;
 682
 683   if (!scanner->buffer)
 684     scanner->buffer = g_new (gchar, READ_BUFFER_SIZE + 1);
 685 }
 686
 687 void
 688 g_scanner_input_text (GScanner    *scanner,
 689                       const gchar *text,
 690                       guint        text_len)
 691 {
 692   g_return_if_fail (scanner != NULL);
 693   if (text_len)
 694     g_return_if_fail (text != NULL);
 695   else
 696     text = NULL;
 697
 698   if (scanner->input_fd >= 0)
 699     g_scanner_sync_file_offset (scanner);
 700
 701   scanner->token = G_TOKEN_NONE;
 702   scanner->value.v_int = 0;
 703   scanner->line = 1;
 704   scanner->position = 0;
 705   scanner->next_token = G_TOKEN_NONE;
 706
 707   scanner->input_fd = -1;
 708   scanner->text = text;
 709   scanner->text_end = text + text_len;
 710
 711   if (scanner->buffer)
 712     {
 713       g_free (scanner->buffer);
 714       scanner->buffer = NULL;
 715     }
 716 }
 717
 718 static guchar
 719 g_scanner_peek_next_char (GScanner *scanner)
 720 {
 721   if (scanner->text < scanner->text_end)
 722     {
 723       return *scanner->text;
 724     }
 725   else if (scanner->input_fd >= 0)
 726     {
 727       gint count;
 728       gchar *buffer;
 729
 730       buffer = scanner->buffer;
 731       do
 732         {
 733           count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
 734         }
 735       while (count == -1 && (errno == EINTR || errno == EAGAIN));
 736
 737       if (count < 1)
 738         {
 739           scanner->input_fd = -1;
 740
 741           return 0;
 742         }
 743       else
 744         {
 745           scanner->text = buffer;
 746           scanner->text_end = buffer + count;
 747
 748           return *buffer;
 749         }
 750     }
 751   else
 752     return 0;
 753 }
 754
 755 void
 756 g_scanner_sync_file_offset (GScanner *scanner)
 757 {
 758   g_return_if_fail (scanner != NULL);
 759
 760   /* for file input, rewind the filedescriptor to the current
 761    * buffer position and blow the file read ahead buffer. usefull for
 762    * third party uses of our filedescriptor, which hooks onto the current
 763    * scanning position.
 764    */
 765
 766   if (scanner->input_fd >= 0 && scanner->text_end > scanner->text)
 767     {
 768       gint buffered;
 769
 770       buffered = scanner->text_end - scanner->text;
 771       if (lseek (scanner->input_fd, - buffered, SEEK_CUR) >= 0)
 772         {
 773           /* we succeeded, blow our buffer's contents now */
 774           scanner->text = NULL;
 775           scanner->text_end = NULL;
 776         }
 777       else
 778         errno = 0;
 779     }
 780 }
 781
 782 static guchar
 783 g_scanner_get_char (GScanner    *scanner,
 784                     guint       *line_p,
 785                     guint       *position_p)
 786 {
 787   guchar fchar;
 788
 789   if (scanner->text < scanner->text_end)
 790     fchar = *(scanner->text++);
 791   else if (scanner->input_fd >= 0)
 792     {
 793       gint count;
 794       gchar *buffer;
 795
 796       buffer = scanner->buffer;
 797       do
 798         {
 799           count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
 800         }
 801       while (count == -1 && (errno == EINTR || errno == EAGAIN));
 802
 803       if (count < 1)
 804         {
 805           scanner->input_fd = -1;
 806           fchar = 0;
 807         }
 808       else
 809         {
 810           scanner->text = buffer + 1;
 811           scanner->text_end = buffer + count;
 812           fchar = *buffer;
 813           if (!fchar)
 814             {
 815               g_scanner_sync_file_offset (scanner);
 816               scanner->text_end = scanner->text;
 817               scanner->input_fd = -1;
 818             }
 819         }
 820     }
 821   else
 822     fchar = 0;
 823
 824   if (fchar == '\n')
 825     {
 826       (*position_p) = 0;
 827       (*line_p)++;
 828     }
 829   else if (fchar)
 830     {
 831       (*position_p)++;
 832     }
 833
 834   return fchar;
 835 }
 836
 837 void
 838 g_scanner_unexp_token (GScanner         *scanner,
 839                        GTokenType        expected_token,
 840                        const gchar      *identifier_spec,
 841                        const gchar      *symbol_spec,
 842                        const gchar      *symbol_name,
 843                        const gchar      *message,
 844                        gint              is_error)
 845 {
 846   gchar *token_string;
 847   guint token_string_len;
 848   gchar *expected_string;
 849   guint expected_string_len;
 850   gchar *message_prefix;
 851   gboolean print_unexp;
 852   void (*msg_handler)   (GScanner*, const gchar*, ...);
 853
 854   g_return_if_fail (scanner != NULL);
 855
 856   if (is_error)
 857     msg_handler = g_scanner_error;
 858   else
 859     msg_handler = g_scanner_warn;
 860
 861   if (!identifier_spec)
 862     identifier_spec = "identifier";
 863   if (!symbol_spec)
 864     symbol_spec = "symbol";
 865
 866   token_string_len = 56;
 867   token_string = g_new (gchar, token_string_len + 1);
 868   expected_string_len = 64;
 869   expected_string = g_new (gchar, expected_string_len + 1);
 870   print_unexp = TRUE;
 871
 872   switch (scanner->token)
 873     {
 874     case G_TOKEN_EOF:
 875       g_snprintf (token_string, token_string_len, "end of file");
 876       break;
 877
 878     default:
 879       if (scanner->token >= 1 && scanner->token <= 255)
 880         {
 881           if ((scanner->token >= ' ' && scanner->token <= '~') ||
 882               strchr (scanner->config->cset_identifier_first, scanner->token) ||
 883               strchr (scanner->config->cset_identifier_nth, scanner->token))
 884             g_snprintf (token_string, expected_string_len, "character `%c'", scanner->token);
 885           else
 886             g_snprintf (token_string, expected_string_len, "character `\\%o'", scanner->token);
 887           break;
 888         }
 889       else if (!scanner->config->symbol_2_token)
 890         {
 891           g_snprintf (token_string, token_string_len, "(unknown) token <%d>", scanner->token);
 892           break;
 893         }
 894       /* fall through */
 895     case G_TOKEN_SYMBOL:
 896       if (expected_token == G_TOKEN_SYMBOL ||
 897           (scanner->config->symbol_2_token &&
 898            expected_token > G_TOKEN_LAST))
 899         print_unexp = FALSE;
 900       if (symbol_name)
 901         g_snprintf (token_string,
 902                     token_string_len,
 903                     "%s%s `%s'",
 904                     print_unexp ? "" : "invalid ",
 905                     symbol_spec,
 906                     symbol_name);
 907       else
 908         g_snprintf (token_string,
 909                     token_string_len,
 910                     "%s%s",
 911                     print_unexp ? "" : "invalid ",
 912                     symbol_spec);
 913       break;
 914
 915     case G_TOKEN_ERROR:
 916       print_unexp = FALSE;
 917       expected_token = G_TOKEN_NONE;
 918       switch (scanner->value.v_error)
 919         {
 920         case G_ERR_UNEXP_EOF:
 921           g_snprintf (token_string, token_string_len, "scanner: unexpected end of file");
 922           break;
 923
 924         case G_ERR_UNEXP_EOF_IN_STRING:
 925           g_snprintf (token_string, token_string_len, "scanner: unterminated string constant");
 926           break;
 927
 928         case G_ERR_UNEXP_EOF_IN_COMMENT:
 929           g_snprintf (token_string, token_string_len, "scanner: unterminated comment");
 930           break;
 931
 932         case G_ERR_NON_DIGIT_IN_CONST:
 933           g_snprintf (token_string, token_string_len, "scanner: non digit in constant");
 934           break;
 935
 936         case G_ERR_FLOAT_RADIX:
 937           g_snprintf (token_string, token_string_len, "scanner: invalid radix for floating constant");
 938           break;
 939
 940         case G_ERR_FLOAT_MALFORMED:
 941           g_snprintf (token_string, token_string_len, "scanner: malformed floating constant");
 942           break;
 943
 944         case G_ERR_DIGIT_RADIX:
 945           g_snprintf (token_string, token_string_len, "scanner: digit is beyond radix");
 946           break;
 947
 948         case G_ERR_UNKNOWN:
 949         default:
 950           g_snprintf (token_string, token_string_len, "scanner: unknown error");
 951           break;
 952         }
 953       break;
 954
 955     case G_TOKEN_CHAR:
 956       g_snprintf (token_string, token_string_len, "character `%c'", scanner->value.v_char);
 957       break;
 958
 959     case G_TOKEN_IDENTIFIER:
 960     case G_TOKEN_IDENTIFIER_NULL:
 961       if (expected_token == G_TOKEN_IDENTIFIER ||
 962           expected_token == G_TOKEN_IDENTIFIER_NULL)
 963         print_unexp = FALSE;
 964       g_snprintf (token_string,
 965                   token_string_len,
 966                   "%s%s `%s'",
 967                   print_unexp ? "" : "invalid ",
 968                   identifier_spec,
 969                   scanner->value.v_string);
 970       break;
 971
 972     case G_TOKEN_BINARY:
 973     case G_TOKEN_OCTAL:
 974     case G_TOKEN_INT:
 975     case G_TOKEN_HEX:
 976       g_snprintf (token_string, token_string_len, "number `%ld'", scanner->value.v_int);
 977       break;
 978
 979     case G_TOKEN_FLOAT:
 980       g_snprintf (token_string, token_string_len, "number `%.3f'", scanner->value.v_float);
 981       break;
 982
 983     case G_TOKEN_STRING:
 984       if (expected_token == G_TOKEN_STRING)
 985         print_unexp = FALSE;
 986       g_snprintf (token_string,
 987                   token_string_len,
 988                   "%s%sstring constant \"%s\"",
 989                   print_unexp ? "" : "invalid ",
 990                   scanner->value.v_string[0] == 0 ? "empty " : "",
 991                   scanner->value.v_string);
 992       token_string[token_string_len - 2] = '"';
 993       token_string[token_string_len - 1] = 0;
 994       break;
 995
 996     case G_TOKEN_COMMENT_SINGLE:
 997     case G_TOKEN_COMMENT_MULTI:
 998       g_snprintf (token_string, token_string_len, "comment");
 999       break;
1000
1001     case G_TOKEN_NONE:
1002       /* somehow the user's parsing code is screwed, there isn't much
1003        * we can do about it.
1004        * Note, a common case to trigger this is
1005        * g_scanner_peek_next_token(); g_scanner_unexp_token();
1006        * without an intermediate g_scanner_get_next_token().
1007        */
1008       g_assert_not_reached ();
1009       break;
1010     }
1011
1012
1013   switch (expected_token)
1014     {
1015       gboolean need_valid;
1016
1017     default:
1018       if (expected_token >= 1 && expected_token <= 255)
1019         {
1020           if ((expected_token >= ' ' && expected_token <= '~') ||
1021               strchr (scanner->config->cset_identifier_first, expected_token) ||
1022               strchr (scanner->config->cset_identifier_nth, expected_token))
1023             g_snprintf (expected_string, expected_string_len, "character `%c'", expected_token);
1024           else
1025             g_snprintf (expected_string, expected_string_len, "character `\\%o'", expected_token);
1026           break;
1027         }
1028       else if (!scanner->config->symbol_2_token)
1029         {
1030           g_snprintf (expected_string, expected_string_len, "(unknown) token <%d>", expected_token);
1031           break;
1032         }
1033       /* fall through */
1034     case G_TOKEN_SYMBOL:
1035       need_valid = (scanner->token == G_TOKEN_SYMBOL ||
1036                     (scanner->config->symbol_2_token &&
1037                      scanner->token > G_TOKEN_LAST));
1038       g_snprintf (expected_string,
1039                   expected_string_len,
1040                   "%s%s",
1041                   need_valid ? "valid " : "",
1042                   symbol_spec);
1043       /* FIXME: should we attempt to lookup the symbol_name for symbol_2_token? */
1044       break;
1045
1046     case G_TOKEN_INT:
1047       g_snprintf (expected_string, expected_string_len, "number (integer)");
1048       break;
1049
1050     case G_TOKEN_FLOAT:
1051       g_snprintf (expected_string, expected_string_len, "number (float)");
1052       break;
1053
1054     case G_TOKEN_STRING:
1055       g_snprintf (expected_string,
1056                   expected_string_len,
1057                   "%sstring constant",
1058                   scanner->token == G_TOKEN_STRING ? "valid " : "");
1059       break;
1060
1061     case G_TOKEN_IDENTIFIER:
1062     case G_TOKEN_IDENTIFIER_NULL:
1063       g_snprintf (expected_string,
1064                   expected_string_len,
1065                   "%s%s",
1066                   (scanner->token == G_TOKEN_IDENTIFIER_NULL ||
1067                    scanner->token == G_TOKEN_IDENTIFIER ? "valid " : ""),
1068                   identifier_spec);
1069       break;
1070
1071     case G_TOKEN_NONE:
1072       break;
1073     }
1074
1075   if (message && message[0] != 0)
1076     message_prefix = " - ";
1077   else
1078     {
1079       message_prefix = "";
1080       message = "";
1081     }
1082
1083   if (expected_token != G_TOKEN_NONE)
1084     {
1085       if (print_unexp)
1086         msg_handler (scanner,
1087                      "unexpected %s, expected %s%s%s",
1088                      token_string,
1089                      expected_string,
1090                      message_prefix,
1091                      message);
1092       else
1093         msg_handler (scanner,
1094                      "%s, expected %s%s%s",
1095                      token_string,
1096                      expected_string,
1097                      message_prefix,
1098                      message);
1099     }
1100   else
1101     {
1102       if (print_unexp)
1103         msg_handler (scanner,
1104                      "unexpected %s%s%s",
1105                      token_string,
1106                      message_prefix,
1107                      message);
1108       else
1109         msg_handler (scanner,
1110                      "%s%s%s",
1111                      token_string,
1112                      message_prefix,
1113                      message);
1114     }
1115
1116   g_free (token_string);
1117   g_free (expected_string);
1118 }
1119
1120 gint
1121 g_scanner_stat_mode (const gchar *filename)
1122 {
1123   struct stat  *stat_buf;
1124   gint          st_mode;
1125
1126   stat_buf = g_new0 (struct stat, 1);
1127 #ifdef HAVE_LSTAT
1128   lstat (filename, stat_buf);
1129 #else
1130   stat (filename, stat_buf);
1131 #endif
1132   st_mode = stat_buf->st_mode;
1133
1134   g_free (stat_buf);
1135
1136   return st_mode;
1137 }
1138
1139 static void
1140 g_scanner_get_token_i (GScanner *scanner,
1141                        GTokenType       *token_p,
1142                        GTokenValue      *value_p,
1143                        guint            *line_p,
1144                        guint            *position_p)
1145 {
1146   do
1147     {
1148       g_scanner_free_value (token_p, value_p);
1149       g_scanner_get_token_ll (scanner, token_p, value_p, line_p, position_p);
1150     }
1151   while (((*token_p > 0 && *token_p < 256) &&
1152           strchr (scanner->config->cset_skip_characters, *token_p)) ||
1153          (*token_p == G_TOKEN_CHAR &&
1154           strchr (scanner->config->cset_skip_characters, value_p->v_char)) ||
1155          (*token_p == G_TOKEN_COMMENT_MULTI &&
1156           scanner->config->skip_comment_multi) ||
1157          (*token_p == G_TOKEN_COMMENT_SINGLE &&
1158           scanner->config->skip_comment_single));
1159
1160   switch (*token_p)
1161     {
1162     case G_TOKEN_IDENTIFIER:
1163       if (scanner->config->identifier_2_string)
1164         *token_p = G_TOKEN_STRING;
1165       break;
1166
1167     case G_TOKEN_SYMBOL:
1168       if (scanner->config->symbol_2_token)
1169         *token_p = (GTokenType) value_p->v_symbol;
1170       break;
1171
1172     case G_TOKEN_BINARY:
1173     case G_TOKEN_OCTAL:
1174     case G_TOKEN_HEX:
1175       if (scanner->config->numbers_2_int)
1176         *token_p = G_TOKEN_INT;
1177       break;
1178
1179     default:
1180       break;
1181     }
1182
1183   if (*token_p == G_TOKEN_INT &&
1184       scanner->config->int_2_float)
1185     {
1186       *token_p = G_TOKEN_FLOAT;
1187       value_p->v_float = value_p->v_int;
1188     }
1189
1190   errno = 0;
1191 }
1192
1193 static void
1194 g_scanner_get_token_ll  (GScanner       *scanner,
1195                          GTokenType     *token_p,
1196                          GTokenValue    *value_p,
1197                          guint          *line_p,
1198                          guint          *position_p)
1199 {
1200   GScannerConfig *config;
1201   GTokenType       token;
1202   gboolean         in_comment_multi;
1203   gboolean         in_comment_single;
1204   gboolean         in_string_sq;
1205   gboolean         in_string_dq;
1206   GString         *gstring;
1207   GTokenValue      value;
1208   guchar           ch;
1209
1210   config = scanner->config;
1211   (*value_p).v_int = 0;
1212
1213   if ((scanner->text >= scanner->text_end && scanner->input_fd < 0) ||
1214       scanner->token == G_TOKEN_EOF)
1215     {
1216       *token_p = G_TOKEN_EOF;
1217       return;
1218     }
1219
1220   in_comment_multi = FALSE;
1221   in_comment_single = FALSE;
1222   in_string_sq = FALSE;
1223   in_string_dq = FALSE;
1224   gstring = NULL;
1225
1226   do /* while (ch != 0) */
1227     {
1228       gboolean dotted_float = FALSE;
1229
1230       ch = g_scanner_get_char (scanner, line_p, position_p);
1231
1232       value.v_int = 0;
1233       token = G_TOKEN_NONE;
1234
1235       /* this is *evil*, but needed ;(
1236        * we first check for identifier first character, because  it
1237        * might interfere with other key chars like slashes or numbers
1238        */
1239       if (config->scan_identifier &&
1240           ch && strchr (config->cset_identifier_first, ch))
1241         goto identifier_precedence;
1242
1243       switch (ch)
1244         {
1245         case 0:
1246           token = G_TOKEN_EOF;
1247           (*position_p)++;
1248           /* ch = 0; */
1249           break;
1250
1251         case '/':
1252           if (!config->scan_comment_multi ||
1253               g_scanner_peek_next_char (scanner) != '*')
1254             goto default_case;
1255           g_scanner_get_char (scanner, line_p, position_p);
1256           token = G_TOKEN_COMMENT_MULTI;
1257           in_comment_multi = TRUE;
1258           gstring = g_string_new ("");
1259           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1260             {
1261               if (ch == '*' && g_scanner_peek_next_char (scanner) == '/')
1262                 {
1263                   g_scanner_get_char (scanner, line_p, position_p);
1264                   in_comment_multi = FALSE;
1265                   break;
1266                 }
1267               else
1268                 gstring = g_string_append_c (gstring, ch);
1269             }
1270           ch = 0;
1271           break;
1272
1273         case '\'':
1274           if (!config->scan_string_sq)
1275             goto default_case;
1276           token = G_TOKEN_STRING;
1277           in_string_sq = TRUE;
1278           gstring = g_string_new ("");
1279           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1280             {
1281               if (ch == '\'')
1282                 {
1283                   in_string_sq = FALSE;
1284                   break;
1285                 }
1286               else
1287                 gstring = g_string_append_c (gstring, ch);
1288             }
1289           ch = 0;
1290           break;
1291
1292         case '"':
1293           if (!config->scan_string_dq)
1294             goto default_case;
1295           token = G_TOKEN_STRING;
1296           in_string_dq = TRUE;
1297           gstring = g_string_new ("");
1298           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1299             {
1300               if (ch == '"')
1301                 {
1302                   in_string_dq = FALSE;
1303                   break;
1304                 }
1305               else
1306                 {
1307                   if (ch == '\\')
1308                     {
1309                       ch = g_scanner_get_char (scanner, line_p, position_p);
1310                       switch (ch)
1311                         {
1312                           guint i;
1313                           guint fchar;
1314
1315                         case 0:
1316                           break;
1317
1318                         case '\\':
1319                           gstring = g_string_append_c (gstring, '\\');
1320                           break;
1321
1322                         case 'n':
1323                           gstring = g_string_append_c (gstring, '\n');
1324                           break;
1325
1326                         case 't':
1327                           gstring = g_string_append_c (gstring, '\t');
1328                           break;
1329
1330                         case 'r':
1331                           gstring = g_string_append_c (gstring, '\r');
1332                           break;
1333
1334                         case 'b':
1335                           gstring = g_string_append_c (gstring, '\b');
1336                           break;
1337
1338                         case 'f':
1339                           gstring = g_string_append_c (gstring, '\f');
1340                           break;
1341
1342                         case '0':
1343                         case '1':
1344                         case '2':
1345                         case '3':
1346                         case '4':
1347                         case '5':
1348                         case '6':
1349                         case '7':
1350                           i = ch - '0';
1351                           fchar = g_scanner_peek_next_char (scanner);
1352                           if (fchar >= '0' && fchar <= '7')
1353                             {
1354                               ch = g_scanner_get_char (scanner, line_p, position_p);
1355                               i = i * 8 + ch - '0';
1356                               fchar = g_scanner_peek_next_char (scanner);
1357                               if (fchar >= '0' && fchar <= '7')
1358                                 {
1359                                   ch = g_scanner_get_char (scanner, line_p, position_p);
1360                                   i = i * 8 + ch - '0';
1361                                 }
1362                             }
1363                           gstring = g_string_append_c (gstring, i);
1364                           break;
1365
1366                         default:
1367                           gstring = g_string_append_c (gstring, ch);
1368                           break;
1369                         }
1370                     }
1371                   else
1372                     gstring = g_string_append_c (gstring, ch);
1373                 }
1374             }
1375           ch = 0;
1376           break;
1377
1378         case '.':
1379           if (!config->scan_float)
1380             goto default_case;
1381           token = G_TOKEN_FLOAT;
1382           dotted_float = TRUE;
1383           ch = g_scanner_get_char (scanner, line_p, position_p);
1384           goto number_parsing;
1385
1386         case '$':
1387           if (!config->scan_hex_dollar)
1388             goto default_case;
1389           token = G_TOKEN_HEX;
1390           ch = g_scanner_get_char (scanner, line_p, position_p);
1391           goto number_parsing;
1392
1393         case '0':
1394           if (config->scan_octal)
1395             token = G_TOKEN_OCTAL;
1396           else
1397             token = G_TOKEN_INT;
1398           ch = g_scanner_peek_next_char (scanner);
1399           if (config->scan_hex && (ch == 'x' || ch == 'X'))
1400             {
1401               token = G_TOKEN_HEX;
1402               g_scanner_get_char (scanner, line_p, position_p);
1403               ch = g_scanner_get_char (scanner, line_p, position_p);
1404               if (ch == 0)
1405                 {
1406                   token = G_TOKEN_ERROR;
1407                   value.v_error = G_ERR_UNEXP_EOF;
1408                   (*position_p)++;
1409                   break;
1410                 }
1411               if (g_scanner_char_2_num (ch, 16) < 0)
1412                 {
1413                   token = G_TOKEN_ERROR;
1414                   value.v_error = G_ERR_DIGIT_RADIX;
1415                   ch = 0;
1416                   break;
1417                 }
1418             }
1419           else if (config->scan_binary && (ch == 'b' || ch == 'B'))
1420             {
1421               token = G_TOKEN_BINARY;
1422               g_scanner_get_char (scanner, line_p, position_p);
1423               ch = g_scanner_get_char (scanner, line_p, position_p);
1424               if (ch == 0)
1425                 {
1426                   token = G_TOKEN_ERROR;
1427                   value.v_error = G_ERR_UNEXP_EOF;
1428                   (*position_p)++;
1429                   break;
1430                 }
1431               if (g_scanner_char_2_num (ch, 10) < 0)
1432                 {
1433                   token = G_TOKEN_ERROR;
1434                   value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1435                   ch = 0;
1436                   break;
1437                 }
1438             }
1439           else
1440             ch = '0';
1441           /* fall through */
1442         case '1':
1443         case '2':
1444         case '3':
1445         case '4':
1446         case '5':
1447         case '6':
1448         case '7':
1449         case '8':
1450         case '9':
1451         number_parsing:
1452         {
1453           gboolean in_number = TRUE;
1454           gchar *endptr;
1455
1456           if (token == G_TOKEN_NONE)
1457             token = G_TOKEN_INT;
1458
1459           gstring = g_string_new (dotted_float ? "0." : "");
1460           gstring = g_string_append_c (gstring, ch);
1461
1462           do /* while (in_number) */
1463             {
1464               gboolean is_E;
1465
1466               is_E = token == G_TOKEN_FLOAT && (ch == 'e' || ch == 'E');
1467
1468               ch = g_scanner_peek_next_char (scanner);
1469
1470               if (g_scanner_char_2_num (ch, 36) >= 0 ||
1471                   (config->scan_float && ch == '.') ||
1472                   (is_E && (ch == '+' || ch == '-')))
1473                 {
1474                   ch = g_scanner_get_char (scanner, line_p, position_p);
1475
1476                   switch (ch)
1477                     {
1478                     case '.':
1479                       if (token != G_TOKEN_INT && token != G_TOKEN_OCTAL)
1480                         {
1481                           value.v_error = token == G_TOKEN_FLOAT ? G_ERR_FLOAT_MALFORMED : G_ERR_FLOAT_RADIX;
1482                           token = G_TOKEN_ERROR;
1483                           in_number = FALSE;
1484                         }
1485                       else
1486                         {
1487                           token = G_TOKEN_FLOAT;
1488                           gstring = g_string_append_c (gstring, ch);
1489                         }
1490                       break;
1491
1492                     case '0':
1493                     case '1':
1494                     case '2':
1495                     case '3':
1496                     case '4':
1497                     case '5':
1498                     case '6':
1499                     case '7':
1500                     case '8':
1501                     case '9':
1502                       gstring = g_string_append_c (gstring, ch);
1503                       break;
1504
1505                     case '-':
1506                     case '+':
1507                       if (token != G_TOKEN_FLOAT)
1508                         {
1509                           token = G_TOKEN_ERROR;
1510                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1511                           in_number = FALSE;
1512                         }
1513                       else
1514                         gstring = g_string_append_c (gstring, ch);
1515                       break;
1516
1517                     case 'e':
1518                     case 'E':
1519                       if ((token != G_TOKEN_HEX && !config->scan_float) ||
1520                           (token != G_TOKEN_HEX &&
1521                            token != G_TOKEN_OCTAL &&
1522                            token != G_TOKEN_FLOAT &&
1523                            token != G_TOKEN_INT))
1524                         {
1525                           token = G_TOKEN_ERROR;
1526                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1527                           in_number = FALSE;
1528                         }
1529                       else
1530                         {
1531                           if (token != G_TOKEN_HEX)
1532                             token = G_TOKEN_FLOAT;
1533                           gstring = g_string_append_c (gstring, ch);
1534                         }
1535                       break;
1536
1537                     default:
1538                       if (token != G_TOKEN_HEX)
1539                         {
1540                           token = G_TOKEN_ERROR;
1541                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1542                           in_number = FALSE;
1543                         }
1544                       else
1545                         gstring = g_string_append_c (gstring, ch);
1546                       break;
1547                     }
1548                 }
1549               else
1550                 in_number = FALSE;
1551             }
1552           while (in_number);
1553
1554           endptr = NULL;
1555           switch (token)
1556             {
1557             case G_TOKEN_BINARY:
1558               value.v_binary = strtol (gstring->str, &endptr, 2);
1559               break;
1560
1561             case G_TOKEN_OCTAL:
1562               value.v_octal = strtol (gstring->str, &endptr, 8);
1563               break;
1564
1565             case G_TOKEN_INT:
1566               value.v_int = strtol (gstring->str, &endptr, 10);
1567               break;
1568
1569             case G_TOKEN_FLOAT:
1570               value.v_float = g_strtod (gstring->str, &endptr);
1571               break;
1572
1573             case G_TOKEN_HEX:
1574               value.v_hex = strtol (gstring->str, &endptr, 16);
1575               break;
1576
1577             default:
1578               break;
1579             }
1580           if (endptr && *endptr)
1581             {
1582               token = G_TOKEN_ERROR;
1583               if (*endptr == 'e' || *endptr == 'E')
1584                 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1585               else
1586                 value.v_error = G_ERR_DIGIT_RADIX;
1587             }
1588           g_string_free (gstring, TRUE);
1589           gstring = NULL;
1590           ch = 0;
1591         } /* number_parsing:... */
1592         break;
1593
1594         default:
1595         default_case:
1596         {
1597           if (config->cpair_comment_single &&
1598               ch == config->cpair_comment_single[0])
1599             {
1600               token = G_TOKEN_COMMENT_SINGLE;
1601               in_comment_single = TRUE;
1602               gstring = g_string_new ("");
1603               while ((ch = g_scanner_get_char (scanner,
1604                                                line_p,
1605                                                position_p)) != 0)
1606                 {
1607                   if (ch == config->cpair_comment_single[1])
1608                     {
1609                       in_comment_single = FALSE;
1610                       ch = 0;
1611                       break;
1612                     }
1613
1614                   gstring = g_string_append_c (gstring, ch);
1615                   ch = 0;
1616                 }
1617             }
1618           else if (config->scan_identifier && ch &&
1619                    strchr (config->cset_identifier_first, ch))
1620             {
1621             identifier_precedence:
1622
1623               if (config->cset_identifier_nth && ch &&
1624                   strchr (config->cset_identifier_nth,
1625                           g_scanner_peek_next_char (scanner)))
1626                 {
1627                   token = G_TOKEN_IDENTIFIER;
1628                   gstring = g_string_new ("");
1629                   gstring = g_string_append_c (gstring, ch);
1630                   do
1631                     {
1632                       ch = g_scanner_get_char (scanner, line_p, position_p);
1633                       gstring = g_string_append_c (gstring, ch);
1634                       ch = g_scanner_peek_next_char (scanner);
1635                     }
1636                   while (ch && strchr (config->cset_identifier_nth, ch));
1637                   ch = 0;
1638                 }
1639               else if (config->scan_identifier_1char)
1640                 {
1641                   token = G_TOKEN_IDENTIFIER;
1642                   value.v_identifier = g_new0 (gchar, 2);
1643                   value.v_identifier[0] = ch;
1644                   ch = 0;
1645                 }
1646             }
1647           if (ch)
1648             {
1649               if (config->char_2_token)
1650                 token = ch;
1651               else
1652                 {
1653                   token = G_TOKEN_CHAR;
1654                   value.v_char = ch;
1655                 }
1656               ch = 0;
1657             }
1658         } /* default_case:... */
1659         break;
1660         }
1661       g_assert (ch == 0 && token != G_TOKEN_NONE); /* paranoid */
1662     }
1663   while (ch != 0);
1664
1665   if (in_comment_multi || in_comment_single ||
1666       in_string_sq || in_string_dq)
1667     {
1668       token = G_TOKEN_ERROR;
1669       if (gstring)
1670         {
1671           g_string_free (gstring, TRUE);
1672           gstring = NULL;
1673         }
1674       (*position_p)++;
1675       if (in_comment_multi || in_comment_single)
1676         value.v_error = G_ERR_UNEXP_EOF_IN_COMMENT;
1677       else /* (in_string_sq || in_string_dq) */
1678         value.v_error = G_ERR_UNEXP_EOF_IN_STRING;
1679     }
1680
1681   if (gstring)
1682     {
1683       value.v_string = gstring->str;
1684       g_string_free (gstring, FALSE);
1685       gstring = NULL;
1686     }
1687
1688   if (token == G_TOKEN_IDENTIFIER)
1689     {
1690       if (config->scan_symbols)
1691         {
1692           GScannerKey *key;
1693           guint scope_id;
1694
1695           scope_id = scanner->scope_id;
1696           key = g_scanner_lookup_internal (scanner, scope_id, value.v_identifier);
1697           if (!key && scope_id && scanner->config->scope_0_fallback)
1698             key = g_scanner_lookup_internal (scanner, 0, value.v_identifier);
1699
1700           if (key)
1701             {
1702               g_free (value.v_identifier);
1703               token = G_TOKEN_SYMBOL;
1704               value.v_symbol = key->value;
1705             }
1706         }
1707
1708       if (token == G_TOKEN_IDENTIFIER &&
1709           config->scan_identifier_NULL &&
1710           strlen (value.v_identifier) == 4)
1711         {
1712           gchar *null_upper = "NULL";
1713           gchar *null_lower = "null";
1714
1715           if (scanner->config->case_sensitive)
1716             {
1717               if (value.v_identifier[0] == null_upper[0] &&
1718                   value.v_identifier[1] == null_upper[1] &&
1719                   value.v_identifier[2] == null_upper[2] &&
1720                   value.v_identifier[3] == null_upper[3])
1721                 token = G_TOKEN_IDENTIFIER_NULL;
1722             }
1723           else
1724             {
1725               if ((value.v_identifier[0] == null_upper[0] ||
1726                    value.v_identifier[0] == null_lower[0]) &&
1727                   (value.v_identifier[1] == null_upper[1] ||
1728                    value.v_identifier[1] == null_lower[1]) &&
1729                   (value.v_identifier[2] == null_upper[2] ||
1730                    value.v_identifier[2] == null_lower[2]) &&
1731                   (value.v_identifier[3] == null_upper[3] ||
1732                    value.v_identifier[3] == null_lower[3]))
1733                 token = G_TOKEN_IDENTIFIER_NULL;
1734             }
1735         }
1736     }
1737
1738   *token_p = token;
1739   *value_p = value;
1740 }