gscanner.c

   1 /* GLIB - Library of useful routines for C programming
   2  * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
   3  *
   4  * GScanner: Flexible lexical scanner for general purpose.
   5  * Copyright (C) 1997, 1998 Tim Janik
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Library General Public
  18  * License along with this library; if not, write to the
  19  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  20  * Boston, MA 02111-1307, USA.
  21  */
  22
  23 /*
  24  * MT safe
  25  */
  26
  27 #define         __gscanner_c__
  28
  29 #ifdef HAVE_CONFIG_H
  30 #include <config.h>
  31 #endif
  32
  33 #include        <stdlib.h>
  34 #include        <stdarg.h>
  35 #include        <string.h>
  36 #include        <stdio.h>
  37 #include        "glib.h"
  38 #ifdef HAVE_UNISTD_H
  39 #include        <unistd.h>
  40 #endif
  41 #include        <errno.h>
  42 #include        <sys/types.h>   /* needed for sys/stat.h */
  43 #include        <sys/stat.h>
  44 #ifdef _MSC_VER
  45 #include        <io.h>          /* For _read() */
  46 #endif
  47
  48 /* --- defines --- */
  49 #define to_lower(c)                             ( \
  50         (guchar) (                                                      \
  51           ( (((guchar)(c))>='A' && ((guchar)(c))<='Z') * ('a'-'A') ) |  \
  52           ( (((guchar)(c))>=192 && ((guchar)(c))<=214) * (224-192) ) |  \
  53           ( (((guchar)(c))>=216 && ((guchar)(c))<=222) * (248-216) ) |  \
  54           ((guchar)(c))                                                 \
  55         )                                                               \
  56 )
  57 #define READ_BUFFER_SIZE        (4000)
  58
  59
  60 /* --- typedefs --- */
  61 typedef struct  _GScannerKey    GScannerKey;
  62
  63 struct  _GScannerKey
  64 {
  65   guint          scope_id;
  66   gchar         *symbol;
  67   gpointer       value;
  68 };
  69
  70
  71
  72 /* --- variables --- */
  73 static GScannerConfig g_scanner_config_template =
  74 {
  75   (
  76    " \t\r\n"
  77    )                    /* cset_skip_characters */,
  78   (
  79    G_CSET_a_2_z
  80    "_"
  81    G_CSET_A_2_Z
  82    )                    /* cset_identifier_first */,
  83   (
  84    G_CSET_a_2_z
  85    "_0123456789"
  86    G_CSET_A_2_Z
  87    G_CSET_LATINS
  88    G_CSET_LATINC
  89    )                    /* cset_identifier_nth */,
  90   ( "#\n" )             /* cpair_comment_single */,
  91
  92   FALSE                 /* case_sensitive */,
  93
  94   TRUE                  /* skip_comment_multi */,
  95   TRUE                  /* skip_comment_single */,
  96   TRUE                  /* scan_comment_multi */,
  97   TRUE                  /* scan_identifier */,
  98   FALSE                 /* scan_identifier_1char */,
  99   FALSE                 /* scan_identifier_NULL */,
 100   TRUE                  /* scan_symbols */,
 101   FALSE                 /* scan_binary */,
 102   TRUE                  /* scan_octal */,
 103   TRUE                  /* scan_float */,
 104   TRUE                  /* scan_hex */,
 105   FALSE                 /* scan_hex_dollar */,
 106   TRUE                  /* scan_string_sq */,
 107   TRUE                  /* scan_string_dq */,
 108   TRUE                  /* numbers_2_int */,
 109   FALSE                 /* int_2_float */,
 110   FALSE                 /* identifier_2_string */,
 111   TRUE                  /* char_2_token */,
 112   FALSE                 /* symbol_2_token */,
 113   FALSE                 /* scope_0_fallback */,
 114 };
 115
 116
 117 /* --- prototypes --- */
 118 static inline
 119 GScannerKey*    g_scanner_lookup_internal (GScanner     *scanner,
 120                                            guint         scope_id,
 121                                            const gchar  *symbol);
 122 static gint     g_scanner_key_equal       (gconstpointer v1,
 123                                            gconstpointer v2);
 124 static guint    g_scanner_key_hash        (gconstpointer v);
 125 static void     g_scanner_get_token_ll    (GScanner     *scanner,
 126                                            GTokenType   *token_p,
 127                                            GTokenValue  *value_p,
 128                                            guint        *line_p,
 129                                            guint        *position_p);
 130 static void     g_scanner_get_token_i     (GScanner     *scanner,
 131                                            GTokenType   *token_p,
 132                                            GTokenValue  *value_p,
 133                                            guint        *line_p,
 134                                            guint        *position_p);
 135
 136 static guchar   g_scanner_peek_next_char  (GScanner     *scanner);
 137 static guchar   g_scanner_get_char        (GScanner     *scanner,
 138                                            guint        *line_p,
 139                                            guint        *position_p);
 140 static void     g_scanner_msg_handler     (GScanner     *scanner,
 141                                            gchar        *message,
 142                                            gint          is_error);
 143
 144
 145 /* --- functions --- */
 146 static inline gint
 147 g_scanner_char_2_num (guchar    c,
 148                       guchar    base)
 149 {
 150   if (c >= '0' && c <= '9')
 151     c -= '0';
 152   else if (c >= 'A' && c <= 'Z')
 153     c -= 'A' - 10;
 154   else if (c >= 'a' && c <= 'z')
 155     c -= 'a' - 10;
 156   else
 157     return -1;
 158
 159   if (c < base)
 160     return c;
 161
 162   return -1;
 163 }
 164
 165 GScanner*
 166 g_scanner_new (GScannerConfig   *config_templ)
 167 {
 168   register GScanner     *scanner;
 169
 170   if (!config_templ)
 171     config_templ = &g_scanner_config_template;
 172
 173   scanner = g_new0 (GScanner, 1);
 174
 175   scanner->user_data = NULL;
 176   scanner->max_parse_errors = 0;
 177   scanner->parse_errors = 0;
 178   scanner->input_name = NULL;
 179   scanner->derived_data = NULL;
 180
 181   scanner->config = g_new0 (GScannerConfig, 1);
 182
 183   scanner->config->case_sensitive       = config_templ->case_sensitive;
 184   scanner->config->cset_skip_characters = config_templ->cset_skip_characters;
 185   scanner->config->cset_identifier_first= config_templ->cset_identifier_first;
 186   scanner->config->cset_identifier_nth  = config_templ->cset_identifier_nth;
 187   scanner->config->cpair_comment_single = config_templ->cpair_comment_single;
 188   scanner->config->skip_comment_multi   = config_templ->skip_comment_multi;
 189   scanner->config->skip_comment_single  = config_templ->skip_comment_single;
 190   scanner->config->scan_comment_multi   = config_templ->scan_comment_multi;
 191   scanner->config->scan_identifier      = config_templ->scan_identifier;
 192   scanner->config->scan_identifier_1char= config_templ->scan_identifier_1char;
 193   scanner->config->scan_identifier_NULL = config_templ->scan_identifier_NULL;
 194   scanner->config->scan_symbols         = config_templ->scan_symbols;
 195   scanner->config->scan_binary          = config_templ->scan_binary;
 196   scanner->config->scan_octal           = config_templ->scan_octal;
 197   scanner->config->scan_float           = config_templ->scan_float;
 198   scanner->config->scan_hex             = config_templ->scan_hex;
 199   scanner->config->scan_hex_dollar      = config_templ->scan_hex_dollar;
 200   scanner->config->scan_string_sq       = config_templ->scan_string_sq;
 201   scanner->config->scan_string_dq       = config_templ->scan_string_dq;
 202   scanner->config->numbers_2_int        = config_templ->numbers_2_int;
 203   scanner->config->int_2_float          = config_templ->int_2_float;
 204   scanner->config->identifier_2_string  = config_templ->identifier_2_string;
 205   scanner->config->char_2_token         = config_templ->char_2_token;
 206   scanner->config->symbol_2_token       = config_templ->symbol_2_token;
 207   scanner->config->scope_0_fallback     = config_templ->scope_0_fallback;
 208
 209   scanner->token = G_TOKEN_NONE;
 210   scanner->value.v_int = 0;
 211   scanner->line = 1;
 212   scanner->position = 0;
 213
 214   scanner->next_token = G_TOKEN_NONE;
 215   scanner->next_value.v_int = 0;
 216   scanner->next_line = 1;
 217   scanner->next_position = 0;
 218
 219   scanner->symbol_table = g_hash_table_new (g_scanner_key_hash, g_scanner_key_equal);
 220   scanner->input_fd = -1;
 221   scanner->text = NULL;
 222   scanner->text_end = NULL;
 223   scanner->buffer = NULL;
 224   scanner->scope_id = 0;
 225
 226   scanner->msg_handler = g_scanner_msg_handler;
 227
 228   return scanner;
 229 }
 230
 231 static inline void
 232 g_scanner_free_value (GTokenType     *token_p,
 233                       GTokenValue     *value_p)
 234 {
 235   switch (*token_p)
 236     {
 237     case  G_TOKEN_STRING:
 238     case  G_TOKEN_IDENTIFIER:
 239     case  G_TOKEN_IDENTIFIER_NULL:
 240     case  G_TOKEN_COMMENT_SINGLE:
 241     case  G_TOKEN_COMMENT_MULTI:
 242       g_free (value_p->v_string);
 243       break;
 244
 245     default:
 246       break;
 247     }
 248
 249   *token_p = G_TOKEN_NONE;
 250 }
 251
 252 static void
 253 g_scanner_destroy_symbol_table_entry (gpointer _key,
 254                                       gpointer _value,
 255                                       gpointer _data)
 256 {
 257   GScannerKey *key = _key;
 258
 259   g_free (key->symbol);
 260   g_free (key);
 261 }
 262
 263 void
 264 g_scanner_destroy (GScanner     *scanner)
 265 {
 266   g_return_if_fail (scanner != NULL);
 267
 268   g_hash_table_foreach (scanner->symbol_table,
 269                         g_scanner_destroy_symbol_table_entry, NULL);
 270   g_hash_table_destroy (scanner->symbol_table);
 271   g_scanner_free_value (&scanner->token, &scanner->value);
 272   g_scanner_free_value (&scanner->next_token, &scanner->next_value);
 273   g_free (scanner->config);
 274   g_free (scanner->buffer);
 275   g_free (scanner);
 276 }
 277
 278 static void
 279 g_scanner_msg_handler (GScanner         *scanner,
 280                        gchar            *message,
 281                        gint              is_error)
 282 {
 283   g_return_if_fail (scanner != NULL);
 284
 285   fprintf (stdout, "%s:%d: ", scanner->input_name, scanner->line);
 286   if (is_error)
 287     fprintf (stdout, "error: ");
 288   fprintf (stdout, "%s\n", message);
 289 }
 290
 291 void
 292 g_scanner_error (GScanner       *scanner,
 293                  const gchar    *format,
 294                  ...)
 295 {
 296   g_return_if_fail (scanner != NULL);
 297   g_return_if_fail (format != NULL);
 298
 299   scanner->parse_errors++;
 300
 301   if (scanner->msg_handler)
 302     {
 303       va_list args;
 304       gchar *string;
 305
 306       va_start (args, format);
 307       string = g_strdup_vprintf (format, args);
 308       va_end (args);
 309
 310       scanner->msg_handler (scanner, string, TRUE);
 311
 312       g_free (string);
 313     }
 314 }
 315
 316 void
 317 g_scanner_warn (GScanner       *scanner,
 318                 const gchar    *format,
 319                 ...)
 320 {
 321   g_return_if_fail (scanner != NULL);
 322   g_return_if_fail (format != NULL);
 323
 324   if (scanner->msg_handler)
 325     {
 326       va_list args;
 327       gchar *string;
 328
 329       va_start (args, format);
 330       string = g_strdup_vprintf (format, args);
 331       va_end (args);
 332
 333       scanner->msg_handler (scanner, string, FALSE);
 334
 335       g_free (string);
 336     }
 337 }
 338
 339 static gint
 340 g_scanner_key_equal (gconstpointer v1,
 341                      gconstpointer v2)
 342 {
 343   register const GScannerKey *key1 = v1;
 344   register const GScannerKey *key2 = v2;
 345
 346   return (key1->scope_id == key2->scope_id) && (strcmp (key1->symbol, key2->symbol) == 0);
 347 }
 348
 349 static guint
 350 g_scanner_key_hash (gconstpointer v)
 351 {
 352   register const GScannerKey *key = v;
 353   register gchar *c;
 354   register guint h;
 355
 356   h = key->scope_id;
 357   for (c = key->symbol; *c; c++)
 358     {
 359       register guint g;
 360
 361       h = (h << 4) + *c;
 362       g = h & 0xf0000000;
 363       if (g)
 364         {
 365           h = h ^ (g >> 24);
 366           h = h ^ g;
 367         }
 368     }
 369
 370   return h;
 371 }
 372
 373 static inline GScannerKey*
 374 g_scanner_lookup_internal (GScanner     *scanner,
 375                            guint         scope_id,
 376                            const gchar  *symbol)
 377 {
 378   register GScannerKey  *key_p;
 379   GScannerKey key;
 380
 381   key.scope_id = scope_id;
 382
 383   if (!scanner->config->case_sensitive)
 384     {
 385       register gchar *d;
 386       register const gchar *c;
 387
 388       key.symbol = g_new (gchar, strlen (symbol) + 1);
 389       for (d = key.symbol, c = symbol; *c; c++, d++)
 390         *d = to_lower (*c);
 391       *d = 0;
 392       key_p = g_hash_table_lookup (scanner->symbol_table, &key);
 393       g_free (key.symbol);
 394     }
 395   else
 396     {
 397       key.symbol = (gchar*) symbol;
 398       key_p = g_hash_table_lookup (scanner->symbol_table, &key);
 399     }
 400
 401   return key_p;
 402 }
 403
 404 void
 405 g_scanner_scope_add_symbol (GScanner    *scanner,
 406                             guint        scope_id,
 407                             const gchar *symbol,
 408                             gpointer     value)
 409 {
 410   register GScannerKey  *key;
 411
 412   g_return_if_fail (scanner != NULL);
 413   g_return_if_fail (symbol != NULL);
 414
 415   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 416
 417   if (!key)
 418     {
 419       key = g_new (GScannerKey, 1);
 420       key->scope_id = scope_id;
 421       key->symbol = g_strdup (symbol);
 422       key->value = value;
 423       if (!scanner->config->case_sensitive)
 424         {
 425           register gchar *c;
 426
 427           c = key->symbol;
 428           while (*c != 0)
 429             {
 430               *c = to_lower (*c);
 431               c++;
 432             }
 433         }
 434       g_hash_table_insert (scanner->symbol_table, key, key);
 435     }
 436   else
 437     key->value = value;
 438 }
 439
 440 void
 441 g_scanner_scope_remove_symbol (GScanner    *scanner,
 442                                guint        scope_id,
 443                                const gchar *symbol)
 444 {
 445   register GScannerKey  *key;
 446
 447   g_return_if_fail (scanner != NULL);
 448   g_return_if_fail (symbol != NULL);
 449
 450   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 451
 452   if (key)
 453     {
 454       g_hash_table_remove (scanner->symbol_table, key);
 455       g_free (key->symbol);
 456       g_free (key);
 457     }
 458 }
 459
 460 gpointer
 461 g_scanner_lookup_symbol (GScanner       *scanner,
 462                          const gchar    *symbol)
 463 {
 464   register GScannerKey  *key;
 465   register guint scope_id;
 466
 467   g_return_val_if_fail (scanner != NULL, NULL);
 468
 469   if (!symbol)
 470     return NULL;
 471
 472   scope_id = scanner->scope_id;
 473   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 474   if (!key && scope_id && scanner->config->scope_0_fallback)
 475     key = g_scanner_lookup_internal (scanner, 0, symbol);
 476
 477   if (key)
 478     return key->value;
 479   else
 480     return NULL;
 481 }
 482
 483 gpointer
 484 g_scanner_scope_lookup_symbol (GScanner       *scanner,
 485                                guint           scope_id,
 486                                const gchar    *symbol)
 487 {
 488   register GScannerKey  *key;
 489
 490   g_return_val_if_fail (scanner != NULL, NULL);
 491
 492   if (!symbol)
 493     return NULL;
 494
 495   key = g_scanner_lookup_internal (scanner, scope_id, symbol);
 496
 497   if (key)
 498     return key->value;
 499   else
 500     return NULL;
 501 }
 502
 503 guint
 504 g_scanner_set_scope (GScanner       *scanner,
 505                      guint           scope_id)
 506 {
 507   register guint old_scope_id;
 508
 509   g_return_val_if_fail (scanner != NULL, 0);
 510
 511   old_scope_id = scanner->scope_id;
 512   scanner->scope_id = scope_id;
 513
 514   return old_scope_id;
 515 }
 516
 517 static void
 518 g_scanner_foreach_internal (gpointer  _key,
 519                             gpointer  _value,
 520                             gpointer  _user_data)
 521 {
 522   register GScannerKey *key;
 523   register gpointer *d;
 524   register GHFunc func;
 525   register gpointer func_data;
 526   register guint *scope_id;
 527
 528   d = _user_data;
 529   func = (GHFunc) d[0];
 530   func_data = d[1];
 531   scope_id = d[2];
 532   key = _value;
 533
 534   if (key->scope_id == *scope_id)
 535     func (key->symbol, key->value, func_data);
 536 }
 537
 538 void
 539 g_scanner_scope_foreach_symbol (GScanner       *scanner,
 540                                 guint           scope_id,
 541                                 GHFunc          func,
 542                                 gpointer        func_data)
 543 {
 544   gpointer d[3];
 545
 546   g_return_if_fail (scanner != NULL);
 547
 548   d[0] = (gpointer) func;
 549   d[1] = func_data;
 550   d[2] = &scope_id;
 551
 552   g_hash_table_foreach (scanner->symbol_table, g_scanner_foreach_internal, d);
 553 }
 554
 555 void
 556 g_scanner_freeze_symbol_table (GScanner *scanner)
 557 {
 558   g_return_if_fail (scanner != NULL);
 559
 560   g_hash_table_freeze (scanner->symbol_table);
 561 }
 562
 563 void
 564 g_scanner_thaw_symbol_table (GScanner *scanner)
 565 {
 566   g_return_if_fail (scanner != NULL);
 567
 568   g_hash_table_thaw (scanner->symbol_table);
 569 }
 570
 571 GTokenType
 572 g_scanner_peek_next_token (GScanner     *scanner)
 573 {
 574   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 575
 576   if (scanner->next_token == G_TOKEN_NONE)
 577     {
 578       scanner->next_line = scanner->line;
 579       scanner->next_position = scanner->position;
 580       g_scanner_get_token_i (scanner,
 581                              &scanner->next_token,
 582                              &scanner->next_value,
 583                              &scanner->next_line,
 584                              &scanner->next_position);
 585     }
 586
 587   return scanner->next_token;
 588 }
 589
 590 GTokenType
 591 g_scanner_get_next_token (GScanner      *scanner)
 592 {
 593   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 594
 595   if (scanner->next_token != G_TOKEN_NONE)
 596     {
 597       g_scanner_free_value (&scanner->token, &scanner->value);
 598
 599       scanner->token = scanner->next_token;
 600       scanner->value = scanner->next_value;
 601       scanner->line = scanner->next_line;
 602       scanner->position = scanner->next_position;
 603       scanner->next_token = G_TOKEN_NONE;
 604     }
 605   else
 606     g_scanner_get_token_i (scanner,
 607                            &scanner->token,
 608                            &scanner->value,
 609                            &scanner->line,
 610                            &scanner->position);
 611
 612   return scanner->token;
 613 }
 614
 615 GTokenType
 616 g_scanner_cur_token (GScanner *scanner)
 617 {
 618   g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
 619
 620   return scanner->token;
 621 }
 622
 623 GTokenValue
 624 g_scanner_cur_value (GScanner *scanner)
 625 {
 626   GTokenValue v;
 627
 628   v.v_int = 0;
 629
 630   g_return_val_if_fail (scanner != NULL, v);
 631
 632   /* MSC isn't capable of handling return scanner->value; ? */
 633
 634   v = scanner->value;
 635
 636   return v;
 637 }
 638
 639 guint
 640 g_scanner_cur_line (GScanner *scanner)
 641 {
 642   g_return_val_if_fail (scanner != NULL, 0);
 643
 644   return scanner->line;
 645 }
 646
 647 guint
 648 g_scanner_cur_position (GScanner *scanner)
 649 {
 650   g_return_val_if_fail (scanner != NULL, 0);
 651
 652   return scanner->position;
 653 }
 654
 655 gboolean
 656 g_scanner_eof (GScanner *scanner)
 657 {
 658   g_return_val_if_fail (scanner != NULL, TRUE);
 659
 660   return scanner->token == G_TOKEN_EOF;
 661 }
 662
 663 void
 664 g_scanner_input_file (GScanner *scanner,
 665                       gint      input_fd)
 666 {
 667   g_return_if_fail (scanner != NULL);
 668   g_return_if_fail (input_fd >= 0);
 669
 670   if (scanner->input_fd >= 0)
 671     g_scanner_sync_file_offset (scanner);
 672
 673   scanner->token = G_TOKEN_NONE;
 674   scanner->value.v_int = 0;
 675   scanner->line = 1;
 676   scanner->position = 0;
 677   scanner->next_token = G_TOKEN_NONE;
 678
 679   scanner->input_fd = input_fd;
 680   scanner->text = NULL;
 681   scanner->text_end = NULL;
 682
 683   if (!scanner->buffer)
 684     scanner->buffer = g_new (gchar, READ_BUFFER_SIZE + 1);
 685 }
 686
 687 void
 688 g_scanner_input_text (GScanner    *scanner,
 689                       const gchar *text,
 690                       guint        text_len)
 691 {
 692   g_return_if_fail (scanner != NULL);
 693   if (text_len)
 694     g_return_if_fail (text != NULL);
 695   else
 696     text = NULL;
 697
 698   if (scanner->input_fd >= 0)
 699     g_scanner_sync_file_offset (scanner);
 700
 701   scanner->token = G_TOKEN_NONE;
 702   scanner->value.v_int = 0;
 703   scanner->line = 1;
 704   scanner->position = 0;
 705   scanner->next_token = G_TOKEN_NONE;
 706
 707   scanner->input_fd = -1;
 708   scanner->text = text;
 709   scanner->text_end = text + text_len;
 710
 711   if (scanner->buffer)
 712     {
 713       g_free (scanner->buffer);
 714       scanner->buffer = NULL;
 715     }
 716 }
 717
 718 static guchar
 719 g_scanner_peek_next_char (GScanner *scanner)
 720 {
 721   if (scanner->text < scanner->text_end)
 722     {
 723       return *scanner->text;
 724     }
 725   else if (scanner->input_fd >= 0)
 726     {
 727       register gint count;
 728       register gchar *buffer;
 729
 730       buffer = scanner->buffer;
 731       do
 732         {
 733           count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
 734         }
 735       while (count == -1 && (errno == EINTR || errno == EAGAIN));
 736
 737       if (count < 1)
 738         {
 739           scanner->input_fd = -1;
 740
 741           return 0;
 742         }
 743       else
 744         {
 745           scanner->text = buffer;
 746           scanner->text_end = buffer + count;
 747
 748           return *buffer;
 749         }
 750     }
 751   else
 752     return 0;
 753 }
 754
 755 void
 756 g_scanner_sync_file_offset (GScanner *scanner)
 757 {
 758   g_return_if_fail (scanner != NULL);
 759
 760   /* for file input, rewind the filedescriptor to the current
 761    * buffer position and blow the file read ahead buffer. usefull for
 762    * third party uses of our filedescriptor, which hooks onto the current
 763    * scanning position.
 764    */
 765
 766   if (scanner->input_fd >= 0 && scanner->text_end > scanner->text)
 767     {
 768       gint buffered;
 769
 770       buffered = scanner->text_end - scanner->text;
 771       if (lseek (scanner->input_fd, - buffered, SEEK_CUR) >= 0)
 772         {
 773           /* we succeeded, blow our buffer's contents now */
 774           scanner->text = NULL;
 775           scanner->text_end = NULL;
 776         }
 777       else
 778         errno = 0;
 779     }
 780 }
 781
 782 static guchar
 783 g_scanner_get_char (GScanner    *scanner,
 784                     guint       *line_p,
 785                     guint       *position_p)
 786 {
 787   register guchar fchar;
 788
 789   if (scanner->text < scanner->text_end)
 790     fchar = *(scanner->text++);
 791   else if (scanner->input_fd >= 0)
 792     {
 793       register gint count;
 794       register gchar *buffer;
 795
 796       buffer = scanner->buffer;
 797       do
 798         {
 799           count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
 800         }
 801       while (count == -1 && (errno == EINTR || errno == EAGAIN));
 802
 803       if (count < 1)
 804         {
 805           scanner->input_fd = -1;
 806           fchar = 0;
 807         }
 808       else
 809         {
 810           scanner->text = buffer + 1;
 811           scanner->text_end = buffer + count;
 812           fchar = *buffer;
 813           if (!fchar)
 814             {
 815               g_scanner_sync_file_offset (scanner);
 816               scanner->text_end = scanner->text;
 817               scanner->input_fd = -1;
 818             }
 819         }
 820     }
 821   else
 822     fchar = 0;
 823
 824   if (fchar == '\n')
 825     {
 826       (*position_p) = 0;
 827       (*line_p)++;
 828     }
 829   else if (fchar)
 830     {
 831       (*position_p)++;
 832     }
 833
 834   return fchar;
 835 }
 836
 837 void
 838 g_scanner_unexp_token (GScanner         *scanner,
 839                        GTokenType        expected_token,
 840                        const gchar      *identifier_spec,
 841                        const gchar      *symbol_spec,
 842                        const gchar      *symbol_name,
 843                        const gchar      *message,
 844                        gint              is_error)
 845 {
 846   register gchar        *token_string;
 847   register guint        token_string_len;
 848   register gchar        *expected_string;
 849   register guint        expected_string_len;
 850   register gchar        *message_prefix;
 851   register gboolean     print_unexp;
 852   void (*msg_handler)   (GScanner*, const gchar*, ...);
 853
 854   g_return_if_fail (scanner != NULL);
 855
 856   if (is_error)
 857     msg_handler = g_scanner_error;
 858   else
 859     msg_handler = g_scanner_warn;
 860
 861   if (!identifier_spec)
 862     identifier_spec = "identifier";
 863   if (!symbol_spec)
 864     symbol_spec = "symbol";
 865
 866   token_string_len = 56;
 867   token_string = g_new (gchar, token_string_len + 1);
 868   expected_string_len = 64;
 869   expected_string = g_new (gchar, expected_string_len + 1);
 870   print_unexp = TRUE;
 871
 872   switch (scanner->token)
 873     {
 874     case  G_TOKEN_EOF:
 875       g_snprintf (token_string, token_string_len, "end of file");
 876       break;
 877
 878     default:
 879       if (scanner->token >= 1 && scanner->token <= 255)
 880         {
 881           if ((scanner->token >= ' ' && scanner->token <= '~') ||
 882               strchr (scanner->config->cset_identifier_first, scanner->token) ||
 883               strchr (scanner->config->cset_identifier_nth, scanner->token))
 884             g_snprintf (token_string, expected_string_len, "character `%c'", scanner->token);
 885           else
 886             g_snprintf (token_string, expected_string_len, "character `\\%o'", scanner->token);
 887           break;
 888         }
 889       else if (!scanner->config->symbol_2_token)
 890         {
 891           g_snprintf (token_string, token_string_len, "(unknown) token <%d>", scanner->token);
 892           break;
 893         }
 894       /* fall through */
 895     case  G_TOKEN_SYMBOL:
 896       if (expected_token == G_TOKEN_SYMBOL ||
 897           (scanner->config->symbol_2_token &&
 898            expected_token > G_TOKEN_LAST))
 899         print_unexp = FALSE;
 900       if (symbol_name)
 901         g_snprintf (token_string,
 902                     token_string_len,
 903                     "%s%s `%s'",
 904                     print_unexp ? "" : "invalid ",
 905                     symbol_spec,
 906                     symbol_name);
 907       else
 908         g_snprintf (token_string,
 909                     token_string_len,
 910                     "%s%s",
 911                     print_unexp ? "" : "invalid ",
 912                     symbol_spec);
 913       break;
 914
 915     case  G_TOKEN_ERROR:
 916       print_unexp = FALSE;
 917       expected_token = G_TOKEN_NONE;
 918       switch (scanner->value.v_error)
 919         {
 920         case  G_ERR_UNEXP_EOF:
 921           g_snprintf (token_string, token_string_len, "scanner: unexpected end of file");
 922           break;
 923
 924         case  G_ERR_UNEXP_EOF_IN_STRING:
 925           g_snprintf (token_string, token_string_len, "scanner: unterminated string constant");
 926           break;
 927
 928         case  G_ERR_UNEXP_EOF_IN_COMMENT:
 929           g_snprintf (token_string, token_string_len, "scanner: unterminated comment");
 930           break;
 931
 932         case  G_ERR_NON_DIGIT_IN_CONST:
 933           g_snprintf (token_string, token_string_len, "scanner: non digit in constant");
 934           break;
 935
 936         case  G_ERR_FLOAT_RADIX:
 937           g_snprintf (token_string, token_string_len, "scanner: invalid radix for floating constant");
 938           break;
 939
 940         case  G_ERR_FLOAT_MALFORMED:
 941           g_snprintf (token_string, token_string_len, "scanner: malformed floating constant");
 942           break;
 943
 944         case  G_ERR_DIGIT_RADIX:
 945           g_snprintf (token_string, token_string_len, "scanner: digit is beyond radix");
 946           break;
 947
 948         case  G_ERR_UNKNOWN:
 949         default:
 950           g_snprintf (token_string, token_string_len, "scanner: unknown error");
 951           break;
 952         }
 953       break;
 954
 955     case  G_TOKEN_CHAR:
 956       g_snprintf (token_string, token_string_len, "character `%c'", scanner->value.v_char);
 957       break;
 958
 959     case  G_TOKEN_IDENTIFIER:
 960       if (expected_token == G_TOKEN_IDENTIFIER)
 961         print_unexp = FALSE;
 962       g_snprintf (token_string,
 963                   token_string_len,
 964                   "%s%s `%s'",
 965                   print_unexp ? "" : "invalid ",
 966                   identifier_spec,
 967                   scanner->value.v_string);
 968       break;
 969
 970     case  G_TOKEN_BINARY:
 971     case  G_TOKEN_OCTAL:
 972     case  G_TOKEN_INT:
 973     case  G_TOKEN_HEX:
 974       g_snprintf (token_string, token_string_len, "number `%ld'", scanner->value.v_int);
 975       break;
 976
 977     case  G_TOKEN_FLOAT:
 978       g_snprintf (token_string, token_string_len, "number `%.3f'", scanner->value.v_float);
 979       break;
 980
 981     case  G_TOKEN_STRING:
 982       if (expected_token == G_TOKEN_STRING)
 983         print_unexp = FALSE;
 984       g_snprintf (token_string,
 985                   token_string_len,
 986                   "%s%sstring constant \"%s\"",
 987                   print_unexp ? "" : "invalid ",
 988                   scanner->value.v_string[0] == 0 ? "empty " : "",
 989                   scanner->value.v_string);
 990       token_string[token_string_len - 2] = '"';
 991       token_string[token_string_len - 1] = 0;
 992       break;
 993
 994     case  G_TOKEN_COMMENT_SINGLE:
 995     case  G_TOKEN_COMMENT_MULTI:
 996       g_snprintf (token_string, token_string_len, "comment");
 997       break;
 998
 999     case  G_TOKEN_NONE:
1000       /* somehow the user's parsing code is screwed, there isn't much
1001        * we can do about it.
1002        * Note, a common case to trigger this is
1003        * g_scanner_peek_next_token(); g_scanner_unexp_token();
1004        * without an intermediate g_scanner_get_next_token().
1005        */
1006       g_assert_not_reached ();
1007       break;
1008     }
1009
1010
1011   switch (expected_token)
1012     {
1013       gboolean need_valid;
1014
1015     default:
1016       if (expected_token >= 1 && expected_token <= 255)
1017         {
1018           if ((expected_token >= ' ' && expected_token <= '~') ||
1019               strchr (scanner->config->cset_identifier_first, expected_token) ||
1020               strchr (scanner->config->cset_identifier_nth, expected_token))
1021             g_snprintf (expected_string, expected_string_len, "character `%c'", expected_token);
1022           else
1023             g_snprintf (expected_string, expected_string_len, "character `\\%o'", expected_token);
1024           break;
1025         }
1026       else if (!scanner->config->symbol_2_token)
1027         {
1028           g_snprintf (expected_string, expected_string_len, "(unknown) token <%d>", expected_token);
1029           break;
1030         }
1031       /* fall through */
1032     case  G_TOKEN_SYMBOL:
1033       need_valid = (scanner->token == G_TOKEN_SYMBOL ||
1034                     (scanner->config->symbol_2_token &&
1035                      scanner->token > G_TOKEN_LAST));
1036       g_snprintf (expected_string,
1037                   expected_string_len,
1038                   "%s%s",
1039                   need_valid ? "valid " : "",
1040                   symbol_spec);
1041       /* FIXME: should we attempt to lookup the symbol_name for symbol_2_token? */
1042       break;
1043
1044     case  G_TOKEN_INT:
1045       g_snprintf (expected_string, expected_string_len, "number (integer)");
1046       break;
1047
1048     case  G_TOKEN_FLOAT:
1049       g_snprintf (expected_string, expected_string_len, "number (float)");
1050       break;
1051
1052     case  G_TOKEN_STRING:
1053       g_snprintf (expected_string,
1054                   expected_string_len,
1055                   "%sstring constant",
1056                   scanner->token == G_TOKEN_STRING ? "valid " : "");
1057       break;
1058
1059     case  G_TOKEN_IDENTIFIER:
1060       g_snprintf (expected_string,
1061                   expected_string_len,
1062                   "%s%s",
1063                   scanner->token == G_TOKEN_IDENTIFIER ? "valid " : "",
1064                   identifier_spec);
1065       break;
1066
1067     case  G_TOKEN_NONE:
1068       break;
1069     }
1070
1071   if (message && message[0] != 0)
1072     message_prefix = " - ";
1073   else
1074     {
1075       message_prefix = "";
1076       message = "";
1077     }
1078
1079   if (expected_token != G_TOKEN_NONE)
1080     {
1081       if (print_unexp)
1082         msg_handler (scanner,
1083                      "unexpected %s, expected %s%s%s",
1084                      token_string,
1085                      expected_string,
1086                      message_prefix,
1087                      message);
1088       else
1089         msg_handler (scanner,
1090                      "%s, expected %s%s%s",
1091                      token_string,
1092                      expected_string,
1093                      message_prefix,
1094                      message);
1095     }
1096   else
1097     {
1098       if (print_unexp)
1099         msg_handler (scanner,
1100                      "unexpected %s%s%s",
1101                      token_string,
1102                      message_prefix,
1103                      message);
1104       else
1105         msg_handler (scanner,
1106                      "%s%s%s",
1107                      token_string,
1108                      message_prefix,
1109                      message);
1110     }
1111
1112   g_free (token_string);
1113   g_free (expected_string);
1114 }
1115
1116 gint
1117 g_scanner_stat_mode (const gchar *filename)
1118 {
1119   struct stat  *stat_buf;
1120   gint          st_mode;
1121
1122   stat_buf = g_new0 (struct stat, 1);
1123 #ifdef HAVE_LSTAT
1124   lstat (filename, stat_buf);
1125 #else
1126   stat (filename, stat_buf);
1127 #endif
1128   st_mode = stat_buf->st_mode;
1129
1130   g_free (stat_buf);
1131
1132   return st_mode;
1133 }
1134
1135 static void
1136 g_scanner_get_token_i (GScanner *scanner,
1137                        GTokenType       *token_p,
1138                        GTokenValue      *value_p,
1139                        guint            *line_p,
1140                        guint            *position_p)
1141 {
1142   do
1143     {
1144       g_scanner_free_value (token_p, value_p);
1145       g_scanner_get_token_ll (scanner, token_p, value_p, line_p, position_p);
1146     }
1147   while (((*token_p > 0 && *token_p < 256) &&
1148           strchr (scanner->config->cset_skip_characters, *token_p)) ||
1149          (*token_p == G_TOKEN_CHAR &&
1150           strchr (scanner->config->cset_skip_characters, value_p->v_char)) ||
1151          (*token_p == G_TOKEN_COMMENT_MULTI &&
1152           scanner->config->skip_comment_multi) ||
1153          (*token_p == G_TOKEN_COMMENT_SINGLE &&
1154           scanner->config->skip_comment_single));
1155
1156   switch (*token_p)
1157     {
1158     case G_TOKEN_IDENTIFIER:
1159       if (scanner->config->identifier_2_string)
1160         *token_p = G_TOKEN_STRING;
1161       break;
1162
1163     case G_TOKEN_SYMBOL:
1164       if (scanner->config->symbol_2_token)
1165         *token_p = (GTokenType) value_p->v_symbol;
1166       break;
1167
1168     case G_TOKEN_BINARY:
1169     case G_TOKEN_OCTAL:
1170     case G_TOKEN_HEX:
1171       if (scanner->config->numbers_2_int)
1172         *token_p = G_TOKEN_INT;
1173       break;
1174
1175     default:
1176       break;
1177     }
1178
1179   if (*token_p == G_TOKEN_INT &&
1180       scanner->config->int_2_float)
1181     {
1182       *token_p = G_TOKEN_FLOAT;
1183       value_p->v_float = value_p->v_int;
1184     }
1185
1186   errno = 0;
1187 }
1188
1189 static void
1190 g_scanner_get_token_ll  (GScanner       *scanner,
1191                          GTokenType     *token_p,
1192                          GTokenValue    *value_p,
1193                          guint          *line_p,
1194                          guint          *position_p)
1195 {
1196   register GScannerConfig *config;
1197   register GTokenType      token;
1198   register gboolean        in_comment_multi;
1199   register gboolean        in_comment_single;
1200   register gboolean        in_string_sq;
1201   register gboolean        in_string_dq;
1202   register GString        *gstring;
1203   GTokenValue              value;
1204   guchar                   ch;
1205
1206   config = scanner->config;
1207   (*value_p).v_int = 0;
1208
1209   if ((scanner->text >= scanner->text_end && scanner->input_fd < 0) ||
1210       scanner->token == G_TOKEN_EOF)
1211     {
1212       *token_p = G_TOKEN_EOF;
1213       return;
1214     }
1215
1216   in_comment_multi = FALSE;
1217   in_comment_single = FALSE;
1218   in_string_sq = FALSE;
1219   in_string_dq = FALSE;
1220   gstring = NULL;
1221
1222   do /* while (ch != 0) */
1223     {
1224       register gboolean         dotted_float = FALSE;
1225
1226       ch = g_scanner_get_char (scanner, line_p, position_p);
1227
1228       value.v_int = 0;
1229       token = G_TOKEN_NONE;
1230
1231       /* this is *evil*, but needed ;(
1232        * we first check for identifier first character, because  it
1233        * might interfere with other key chars like slashes or numbers
1234        */
1235       if (config->scan_identifier &&
1236           ch && strchr (config->cset_identifier_first, ch))
1237         goto identifier_precedence;
1238
1239       switch (ch)
1240         {
1241         case  0:
1242           token = G_TOKEN_EOF;
1243           (*position_p)++;
1244           /* ch = 0; */
1245           break;
1246
1247         case  '/':
1248           if (!config->scan_comment_multi ||
1249               g_scanner_peek_next_char (scanner) != '*')
1250             goto default_case;
1251           g_scanner_get_char (scanner, line_p, position_p);
1252           token = G_TOKEN_COMMENT_MULTI;
1253           in_comment_multi = TRUE;
1254           gstring = g_string_new ("");
1255           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1256             {
1257               if (ch == '*' && g_scanner_peek_next_char (scanner) == '/')
1258                 {
1259                   g_scanner_get_char (scanner, line_p, position_p);
1260                   in_comment_multi = FALSE;
1261                   break;
1262                 }
1263               else
1264                 gstring = g_string_append_c (gstring, ch);
1265             }
1266           ch = 0;
1267           break;
1268
1269         case  '\'':
1270           if (!config->scan_string_sq)
1271             goto default_case;
1272           token = G_TOKEN_STRING;
1273           in_string_sq = TRUE;
1274           gstring = g_string_new ("");
1275           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1276             {
1277               if (ch == '\'')
1278                 {
1279                   in_string_sq = FALSE;
1280                   break;
1281                 }
1282               else
1283                 gstring = g_string_append_c (gstring, ch);
1284             }
1285           ch = 0;
1286           break;
1287
1288         case  '"':
1289           if (!config->scan_string_dq)
1290             goto default_case;
1291           token = G_TOKEN_STRING;
1292           in_string_dq = TRUE;
1293           gstring = g_string_new ("");
1294           while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1295             {
1296               if (ch == '"')
1297                 {
1298                   in_string_dq = FALSE;
1299                   break;
1300                 }
1301               else
1302                 {
1303                   if (ch == '\\')
1304                     {
1305                       ch = g_scanner_get_char (scanner, line_p, position_p);
1306                       switch (ch)
1307                         {
1308                           register guint        i;
1309                           register guint        fchar;
1310
1311                         case  0:
1312                           break;
1313
1314                         case  '\\':
1315                           gstring = g_string_append_c (gstring, '\\');
1316                           break;
1317
1318                         case  'n':
1319                           gstring = g_string_append_c (gstring, '\n');
1320                           break;
1321
1322                         case  't':
1323                           gstring = g_string_append_c (gstring, '\t');
1324                           break;
1325
1326                         case  'r':
1327                           gstring = g_string_append_c (gstring, '\r');
1328                           break;
1329
1330                         case  'b':
1331                           gstring = g_string_append_c (gstring, '\b');
1332                           break;
1333
1334                         case  'f':
1335                           gstring = g_string_append_c (gstring, '\f');
1336                           break;
1337
1338                         case  '0':
1339                         case  '1':
1340                         case  '2':
1341                         case  '3':
1342                         case  '4':
1343                         case  '5':
1344                         case  '6':
1345                         case  '7':
1346                           i = ch - '0';
1347                           fchar = g_scanner_peek_next_char (scanner);
1348                           if (fchar >= '0' && fchar <= '7')
1349                             {
1350                               ch = g_scanner_get_char (scanner, line_p, position_p);
1351                               i = i * 8 + ch - '0';
1352                               fchar = g_scanner_peek_next_char (scanner);
1353                               if (fchar >= '0' && fchar <= '7')
1354                                 {
1355                                   ch = g_scanner_get_char (scanner, line_p, position_p);
1356                                   i = i * 8 + ch - '0';
1357                                 }
1358                             }
1359                           gstring = g_string_append_c (gstring, i);
1360                           break;
1361
1362                         default:
1363                           gstring = g_string_append_c (gstring, ch);
1364                           break;
1365                         }
1366                     }
1367                   else
1368                     gstring = g_string_append_c (gstring, ch);
1369                 }
1370             }
1371           ch = 0;
1372           break;
1373
1374         case  '.':
1375           if (!config->scan_float)
1376             goto default_case;
1377           token = G_TOKEN_FLOAT;
1378           dotted_float = TRUE;
1379           ch = g_scanner_get_char (scanner, line_p, position_p);
1380           goto number_parsing;
1381
1382         case  '$':
1383           if (!config->scan_hex_dollar)
1384             goto default_case;
1385           token = G_TOKEN_HEX;
1386           ch = g_scanner_get_char (scanner, line_p, position_p);
1387           goto number_parsing;
1388
1389         case  '0':
1390           if (config->scan_octal)
1391             token = G_TOKEN_OCTAL;
1392           else
1393             token = G_TOKEN_INT;
1394           ch = g_scanner_peek_next_char (scanner);
1395           if (config->scan_hex && (ch == 'x' || ch == 'X'))
1396             {
1397               token = G_TOKEN_HEX;
1398               g_scanner_get_char (scanner, line_p, position_p);
1399               ch = g_scanner_get_char (scanner, line_p, position_p);
1400               if (ch == 0)
1401                 {
1402                   token = G_TOKEN_ERROR;
1403                   value.v_error = G_ERR_UNEXP_EOF;
1404                   (*position_p)++;
1405                   break;
1406                 }
1407               if (g_scanner_char_2_num (ch, 16) < 0)
1408                 {
1409                   token = G_TOKEN_ERROR;
1410                   value.v_error = G_ERR_DIGIT_RADIX;
1411                   ch = 0;
1412                   break;
1413                 }
1414             }
1415           else if (config->scan_binary && (ch == 'b' || ch == 'B'))
1416             {
1417               token = G_TOKEN_BINARY;
1418               g_scanner_get_char (scanner, line_p, position_p);
1419               ch = g_scanner_get_char (scanner, line_p, position_p);
1420               if (ch == 0)
1421                 {
1422                   token = G_TOKEN_ERROR;
1423                   value.v_error = G_ERR_UNEXP_EOF;
1424                   (*position_p)++;
1425                   break;
1426                 }
1427               if (g_scanner_char_2_num (ch, 10) < 0)
1428                 {
1429                   token = G_TOKEN_ERROR;
1430                   value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1431                   ch = 0;
1432                   break;
1433                 }
1434             }
1435           else
1436             ch = '0';
1437           /* fall through */
1438         case  '1':
1439         case  '2':
1440         case  '3':
1441         case  '4':
1442         case  '5':
1443         case  '6':
1444         case  '7':
1445         case  '8':
1446         case  '9':
1447         number_parsing:
1448         {
1449           register gboolean in_number = TRUE;
1450           gchar *endptr;
1451
1452           if (token == G_TOKEN_NONE)
1453             token = G_TOKEN_INT;
1454
1455           gstring = g_string_new (dotted_float ? "0." : "");
1456           gstring = g_string_append_c (gstring, ch);
1457
1458           do /* while (in_number) */
1459             {
1460               register gboolean is_E;
1461
1462               is_E = token == G_TOKEN_FLOAT && (ch == 'e' || ch == 'E');
1463
1464               ch = g_scanner_peek_next_char (scanner);
1465
1466               if (g_scanner_char_2_num (ch, 36) >= 0 ||
1467                   (config->scan_float && ch == '.') ||
1468                   (is_E && (ch == '+' || ch == '-')))
1469                 {
1470                   ch = g_scanner_get_char (scanner, line_p, position_p);
1471
1472                   switch (ch)
1473                     {
1474                     case '.':
1475                       if (token != G_TOKEN_INT && token != G_TOKEN_OCTAL)
1476                         {
1477                           value.v_error = token == G_TOKEN_FLOAT ? G_ERR_FLOAT_MALFORMED : G_ERR_FLOAT_RADIX;
1478                           token = G_TOKEN_ERROR;
1479                           in_number = FALSE;
1480                         }
1481                       else
1482                         {
1483                           token = G_TOKEN_FLOAT;
1484                           gstring = g_string_append_c (gstring, ch);
1485                         }
1486                       break;
1487
1488                     case '0':
1489                     case '1':
1490                     case '2':
1491                     case '3':
1492                     case '4':
1493                     case '5':
1494                     case '6':
1495                     case '7':
1496                     case '8':
1497                     case '9':
1498                       gstring = g_string_append_c (gstring, ch);
1499                       break;
1500
1501                     case '-':
1502                     case '+':
1503                       if (token != G_TOKEN_FLOAT)
1504                         {
1505                           token = G_TOKEN_ERROR;
1506                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1507                           in_number = FALSE;
1508                         }
1509                       else
1510                         gstring = g_string_append_c (gstring, ch);
1511                       break;
1512
1513                     case 'e':
1514                     case 'E':
1515                       if ((token != G_TOKEN_HEX && !config->scan_float) ||
1516                           (token != G_TOKEN_HEX &&
1517                            token != G_TOKEN_OCTAL &&
1518                            token != G_TOKEN_FLOAT &&
1519                            token != G_TOKEN_INT))
1520                         {
1521                           token = G_TOKEN_ERROR;
1522                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1523                           in_number = FALSE;
1524                         }
1525                       else
1526                         {
1527                           if (token != G_TOKEN_HEX)
1528                             token = G_TOKEN_FLOAT;
1529                           gstring = g_string_append_c (gstring, ch);
1530                         }
1531                       break;
1532
1533                     default:
1534                       if (token != G_TOKEN_HEX)
1535                         {
1536                           token = G_TOKEN_ERROR;
1537                           value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1538                           in_number = FALSE;
1539                         }
1540                       else
1541                         gstring = g_string_append_c (gstring, ch);
1542                       break;
1543                     }
1544                 }
1545               else
1546                 in_number = FALSE;
1547             }
1548           while (in_number);
1549
1550           endptr = NULL;
1551           switch (token)
1552             {
1553             case G_TOKEN_BINARY:
1554               value.v_binary = strtol (gstring->str, &endptr, 2);
1555               break;
1556
1557             case G_TOKEN_OCTAL:
1558               value.v_octal = strtol (gstring->str, &endptr, 8);
1559               break;
1560
1561             case G_TOKEN_INT:
1562               value.v_int = strtol (gstring->str, &endptr, 10);
1563               break;
1564
1565             case G_TOKEN_FLOAT:
1566               value.v_float = g_strtod (gstring->str, &endptr);
1567               break;
1568
1569             case G_TOKEN_HEX:
1570               value.v_hex = strtol (gstring->str, &endptr, 16);
1571               break;
1572
1573             default:
1574               break;
1575             }
1576           if (endptr && *endptr)
1577             {
1578               token = G_TOKEN_ERROR;
1579               if (*endptr == 'e' || *endptr == 'E')
1580                 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1581               else
1582                 value.v_error = G_ERR_DIGIT_RADIX;
1583             }
1584           g_string_free (gstring, TRUE);
1585           gstring = NULL;
1586           ch = 0;
1587         } /* number_parsing:... */
1588         break;
1589
1590         default:
1591         default_case:
1592         {
1593           if (config->cpair_comment_single &&
1594               ch == config->cpair_comment_single[0])
1595             {
1596               token = G_TOKEN_COMMENT_SINGLE;
1597               in_comment_single = TRUE;
1598               gstring = g_string_new ("");
1599               while ((ch = g_scanner_get_char (scanner,
1600                                                line_p,
1601                                                position_p)) != 0)
1602                 {
1603                   if (ch == config->cpair_comment_single[1])
1604                     {
1605                       in_comment_single = FALSE;
1606                       ch = 0;
1607                       break;
1608                     }
1609
1610                   gstring = g_string_append_c (gstring, ch);
1611                   ch = 0;
1612                 }
1613             }
1614           else if (config->scan_identifier && ch &&
1615                    strchr (config->cset_identifier_first, ch))
1616             {
1617             identifier_precedence:
1618
1619               if (config->cset_identifier_nth && ch &&
1620                   strchr (config->cset_identifier_nth,
1621                           g_scanner_peek_next_char (scanner)))
1622                 {
1623                   token = G_TOKEN_IDENTIFIER;
1624                   gstring = g_string_new ("");
1625                   gstring = g_string_append_c (gstring, ch);
1626                   do
1627                     {
1628                       ch = g_scanner_get_char (scanner, line_p, position_p);
1629                       gstring = g_string_append_c (gstring, ch);
1630                       ch = g_scanner_peek_next_char (scanner);
1631                     }
1632                   while (ch && strchr (config->cset_identifier_nth, ch));
1633                   ch = 0;
1634                 }
1635               else if (config->scan_identifier_1char)
1636                 {
1637                   token = G_TOKEN_IDENTIFIER;
1638                   value.v_identifier = g_new0 (gchar, 2);
1639                   value.v_identifier[0] = ch;
1640                   ch = 0;
1641                 }
1642             }
1643           if (ch)
1644             {
1645               if (config->char_2_token)
1646                 token = ch;
1647               else
1648                 {
1649                   token = G_TOKEN_CHAR;
1650                   value.v_char = ch;
1651                 }
1652               ch = 0;
1653             }
1654         } /* default_case:... */
1655         break;
1656         }
1657       g_assert (ch == 0 && token != G_TOKEN_NONE); /* paranoid */
1658     }
1659   while (ch != 0);
1660
1661   if (in_comment_multi || in_comment_single ||
1662       in_string_sq || in_string_dq)
1663     {
1664       token = G_TOKEN_ERROR;
1665       if (gstring)
1666         {
1667           g_string_free (gstring, TRUE);
1668           gstring = NULL;
1669         }
1670       (*position_p)++;
1671       if (in_comment_multi || in_comment_single)
1672         value.v_error = G_ERR_UNEXP_EOF_IN_COMMENT;
1673       else /* (in_string_sq || in_string_dq) */
1674         value.v_error = G_ERR_UNEXP_EOF_IN_STRING;
1675     }
1676
1677   if (gstring)
1678     {
1679       value.v_string = gstring->str;
1680       g_string_free (gstring, FALSE);
1681       gstring = NULL;
1682     }
1683
1684   if (token == G_TOKEN_IDENTIFIER)
1685     {
1686       if (config->scan_symbols)
1687         {
1688           register GScannerKey  *key;
1689           register guint scope_id;
1690
1691           scope_id = scanner->scope_id;
1692           key = g_scanner_lookup_internal (scanner, scope_id, value.v_identifier);
1693           if (!key && scope_id && scanner->config->scope_0_fallback)
1694             key = g_scanner_lookup_internal (scanner, 0, value.v_identifier);
1695
1696           if (key)
1697             {
1698               g_free (value.v_identifier);
1699               token = G_TOKEN_SYMBOL;
1700               value.v_symbol = key->value;
1701             }
1702         }
1703
1704       if (token == G_TOKEN_IDENTIFIER &&
1705           config->scan_identifier_NULL &&
1706           strlen (value.v_identifier) == 4)
1707         {
1708           gchar *null_upper = "NULL";
1709           gchar *null_lower = "null";
1710
1711           if (scanner->config->case_sensitive)
1712             {
1713               if (value.v_identifier[0] == null_upper[0] &&
1714                   value.v_identifier[1] == null_upper[1] &&
1715                   value.v_identifier[2] == null_upper[2] &&
1716                   value.v_identifier[3] == null_upper[3])
1717                 token = G_TOKEN_IDENTIFIER_NULL;
1718             }
1719           else
1720             {
1721               if ((value.v_identifier[0] == null_upper[0] ||
1722                    value.v_identifier[0] == null_lower[0]) &&
1723                   (value.v_identifier[1] == null_upper[1] ||
1724                    value.v_identifier[1] == null_lower[1]) &&
1725                   (value.v_identifier[2] == null_upper[2] ||
1726                    value.v_identifier[2] == null_lower[2]) &&
1727                   (value.v_identifier[3] == null_upper[3] ||
1728                    value.v_identifier[3] == null_lower[3]))
1729                 token = G_TOKEN_IDENTIFIER_NULL;
1730             }
1731         }
1732     }
1733
1734   *token_p = token;
1735   *value_p = value;
1736 }