1 /* GLIB - Library of useful routines for C programming
2 * Copyright (C) 1995-1997 Peter Mattis, Spencer Kimball and Josh MacDonald
4 * GScanner: Flexible lexical scanner for general purpose.
5 * Copyright (C) 1997, 1998 Tim Janik
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
20 * Boston, MA 02111-1307, USA.
24 * Modified by the GLib Team and others 1997-2000. See the AUTHORS
25 * file for a list of people on the GLib Team. See the ChangeLog
26 * files for a list of changes. These files are distributed with
27 * GLib at ftp://ftp.gtk.org/pub/gtk/.
48 #include <io.h> /* For _read() */
52 #define to_lower(c) ( \
54 ( (((guchar)(c))>='A' && ((guchar)(c))<='Z') * ('a'-'A') ) | \
55 ( (((guchar)(c))>=192 && ((guchar)(c))<=214) * (224-192) ) | \
56 ( (((guchar)(c))>=216 && ((guchar)(c))<=222) * (248-216) ) | \
60 #define READ_BUFFER_SIZE (4000)
63 /* --- typedefs --- */
64 typedef struct _GScannerKey GScannerKey;
75 /* --- variables --- */
76 static GScannerConfig g_scanner_config_template =
80 ) /* cset_skip_characters */,
85 ) /* cset_identifier_first */,
93 ) /* cset_identifier_nth */,
94 ( "#\n" ) /* cpair_comment_single */,
96 FALSE /* case_sensitive */,
98 TRUE /* skip_comment_multi */,
99 TRUE /* skip_comment_single */,
100 TRUE /* scan_comment_multi */,
101 TRUE /* scan_identifier */,
102 FALSE /* scan_identifier_1char */,
103 FALSE /* scan_identifier_NULL */,
104 TRUE /* scan_symbols */,
105 FALSE /* scan_binary */,
106 TRUE /* scan_octal */,
107 TRUE /* scan_float */,
109 FALSE /* scan_hex_dollar */,
110 TRUE /* scan_string_sq */,
111 TRUE /* scan_string_dq */,
112 TRUE /* numbers_2_int */,
113 FALSE /* int_2_float */,
114 FALSE /* identifier_2_string */,
115 TRUE /* char_2_token */,
116 FALSE /* symbol_2_token */,
117 FALSE /* scope_0_fallback */,
121 /* --- prototypes --- */
123 GScannerKey* g_scanner_lookup_internal (GScanner *scanner,
125 const gchar *symbol);
126 static gboolean g_scanner_key_equal (gconstpointer v1,
128 static guint g_scanner_key_hash (gconstpointer v);
129 static void g_scanner_get_token_ll (GScanner *scanner,
131 GTokenValue *value_p,
134 static void g_scanner_get_token_i (GScanner *scanner,
136 GTokenValue *value_p,
140 static guchar g_scanner_peek_next_char (GScanner *scanner);
141 static guchar g_scanner_get_char (GScanner *scanner,
144 static void g_scanner_msg_handler (GScanner *scanner,
149 /* --- functions --- */
151 g_scanner_char_2_num (guchar c,
154 if (c >= '0' && c <= '9')
156 else if (c >= 'A' && c <= 'Z')
158 else if (c >= 'a' && c <= 'z')
170 g_scanner_new (const GScannerConfig *config_templ)
175 config_templ = &g_scanner_config_template;
177 scanner = g_new0 (GScanner, 1);
179 scanner->user_data = NULL;
180 scanner->max_parse_errors = 1;
181 scanner->parse_errors = 0;
182 scanner->input_name = NULL;
183 g_datalist_init (&scanner->qdata);
185 scanner->config = g_new0 (GScannerConfig, 1);
187 scanner->config->case_sensitive = config_templ->case_sensitive;
188 scanner->config->cset_skip_characters = config_templ->cset_skip_characters;
189 if (!scanner->config->cset_skip_characters)
190 scanner->config->cset_skip_characters = "";
191 scanner->config->cset_identifier_first = config_templ->cset_identifier_first;
192 scanner->config->cset_identifier_nth = config_templ->cset_identifier_nth;
193 scanner->config->cpair_comment_single = config_templ->cpair_comment_single;
194 scanner->config->skip_comment_multi = config_templ->skip_comment_multi;
195 scanner->config->skip_comment_single = config_templ->skip_comment_single;
196 scanner->config->scan_comment_multi = config_templ->scan_comment_multi;
197 scanner->config->scan_identifier = config_templ->scan_identifier;
198 scanner->config->scan_identifier_1char = config_templ->scan_identifier_1char;
199 scanner->config->scan_identifier_NULL = config_templ->scan_identifier_NULL;
200 scanner->config->scan_symbols = config_templ->scan_symbols;
201 scanner->config->scan_binary = config_templ->scan_binary;
202 scanner->config->scan_octal = config_templ->scan_octal;
203 scanner->config->scan_float = config_templ->scan_float;
204 scanner->config->scan_hex = config_templ->scan_hex;
205 scanner->config->scan_hex_dollar = config_templ->scan_hex_dollar;
206 scanner->config->scan_string_sq = config_templ->scan_string_sq;
207 scanner->config->scan_string_dq = config_templ->scan_string_dq;
208 scanner->config->numbers_2_int = config_templ->numbers_2_int;
209 scanner->config->int_2_float = config_templ->int_2_float;
210 scanner->config->identifier_2_string = config_templ->identifier_2_string;
211 scanner->config->char_2_token = config_templ->char_2_token;
212 scanner->config->symbol_2_token = config_templ->symbol_2_token;
213 scanner->config->scope_0_fallback = config_templ->scope_0_fallback;
215 scanner->token = G_TOKEN_NONE;
216 scanner->value.v_int = 0;
218 scanner->position = 0;
220 scanner->next_token = G_TOKEN_NONE;
221 scanner->next_value.v_int = 0;
222 scanner->next_line = 1;
223 scanner->next_position = 0;
225 scanner->symbol_table = g_hash_table_new (g_scanner_key_hash, g_scanner_key_equal);
226 scanner->input_fd = -1;
227 scanner->text = NULL;
228 scanner->text_end = NULL;
229 scanner->buffer = NULL;
230 scanner->scope_id = 0;
232 scanner->msg_handler = g_scanner_msg_handler;
238 g_scanner_free_value (GTokenType *token_p,
239 GTokenValue *value_p)
244 case G_TOKEN_IDENTIFIER:
245 case G_TOKEN_IDENTIFIER_NULL:
246 case G_TOKEN_COMMENT_SINGLE:
247 case G_TOKEN_COMMENT_MULTI:
248 g_free (value_p->v_string);
255 *token_p = G_TOKEN_NONE;
259 g_scanner_destroy_symbol_table_entry (gpointer _key,
263 GScannerKey *key = _key;
265 g_free (key->symbol);
270 g_scanner_destroy (GScanner *scanner)
272 g_return_if_fail (scanner != NULL);
274 g_datalist_clear (&scanner->qdata);
275 g_hash_table_foreach (scanner->symbol_table,
276 g_scanner_destroy_symbol_table_entry, NULL);
277 g_hash_table_destroy (scanner->symbol_table);
278 g_scanner_free_value (&scanner->token, &scanner->value);
279 g_scanner_free_value (&scanner->next_token, &scanner->next_value);
280 g_free (scanner->config);
281 g_free (scanner->buffer);
286 g_scanner_msg_handler (GScanner *scanner,
290 g_return_if_fail (scanner != NULL);
292 if (scanner->input_name)
293 fprintf (stderr, "%s:%d: ", scanner->input_name, scanner->line);
295 fprintf (stderr, "error: ");
296 fprintf (stderr, "%s\n", message);
300 g_scanner_error (GScanner *scanner,
304 g_return_if_fail (scanner != NULL);
305 g_return_if_fail (format != NULL);
307 scanner->parse_errors++;
309 if (scanner->msg_handler)
314 va_start (args, format);
315 string = g_strdup_vprintf (format, args);
318 scanner->msg_handler (scanner, string, TRUE);
325 g_scanner_warn (GScanner *scanner,
329 g_return_if_fail (scanner != NULL);
330 g_return_if_fail (format != NULL);
332 if (scanner->msg_handler)
337 va_start (args, format);
338 string = g_strdup_vprintf (format, args);
341 scanner->msg_handler (scanner, string, FALSE);
348 g_scanner_key_equal (gconstpointer v1,
351 const GScannerKey *key1 = v1;
352 const GScannerKey *key2 = v2;
354 return (key1->scope_id == key2->scope_id) && (strcmp (key1->symbol, key2->symbol) == 0);
358 g_scanner_key_hash (gconstpointer v)
360 const GScannerKey *key = v;
365 for (c = key->symbol; *c; c++)
366 h = (h << 5) - h + *c;
371 static inline GScannerKey*
372 g_scanner_lookup_internal (GScanner *scanner,
379 key.scope_id = scope_id;
381 if (!scanner->config->case_sensitive)
386 key.symbol = g_new (gchar, strlen (symbol) + 1);
387 for (d = key.symbol, c = symbol; *c; c++, d++)
390 key_p = g_hash_table_lookup (scanner->symbol_table, &key);
395 key.symbol = (gchar*) symbol;
396 key_p = g_hash_table_lookup (scanner->symbol_table, &key);
403 g_scanner_scope_add_symbol (GScanner *scanner,
410 g_return_if_fail (scanner != NULL);
411 g_return_if_fail (symbol != NULL);
413 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
417 key = g_new (GScannerKey, 1);
418 key->scope_id = scope_id;
419 key->symbol = g_strdup (symbol);
421 if (!scanner->config->case_sensitive)
432 g_hash_table_insert (scanner->symbol_table, key, key);
439 g_scanner_scope_remove_symbol (GScanner *scanner,
445 g_return_if_fail (scanner != NULL);
446 g_return_if_fail (symbol != NULL);
448 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
452 g_hash_table_remove (scanner->symbol_table, key);
453 g_free (key->symbol);
459 g_scanner_lookup_symbol (GScanner *scanner,
465 g_return_val_if_fail (scanner != NULL, NULL);
470 scope_id = scanner->scope_id;
471 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
472 if (!key && scope_id && scanner->config->scope_0_fallback)
473 key = g_scanner_lookup_internal (scanner, 0, symbol);
482 g_scanner_scope_lookup_symbol (GScanner *scanner,
488 g_return_val_if_fail (scanner != NULL, NULL);
493 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
502 g_scanner_set_scope (GScanner *scanner,
507 g_return_val_if_fail (scanner != NULL, 0);
509 old_scope_id = scanner->scope_id;
510 scanner->scope_id = scope_id;
516 g_scanner_foreach_internal (gpointer _key,
527 func = (GHFunc) d[0];
532 if (key->scope_id == *scope_id)
533 func (key->symbol, key->value, user_data);
537 g_scanner_scope_foreach_symbol (GScanner *scanner,
544 g_return_if_fail (scanner != NULL);
546 d[0] = (gpointer) func;
550 g_hash_table_foreach (scanner->symbol_table, g_scanner_foreach_internal, d);
554 g_scanner_peek_next_token (GScanner *scanner)
556 g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
558 if (scanner->next_token == G_TOKEN_NONE)
560 scanner->next_line = scanner->line;
561 scanner->next_position = scanner->position;
562 g_scanner_get_token_i (scanner,
563 &scanner->next_token,
564 &scanner->next_value,
566 &scanner->next_position);
569 return scanner->next_token;
573 g_scanner_get_next_token (GScanner *scanner)
575 g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
577 if (scanner->next_token != G_TOKEN_NONE)
579 g_scanner_free_value (&scanner->token, &scanner->value);
581 scanner->token = scanner->next_token;
582 scanner->value = scanner->next_value;
583 scanner->line = scanner->next_line;
584 scanner->position = scanner->next_position;
585 scanner->next_token = G_TOKEN_NONE;
588 g_scanner_get_token_i (scanner,
594 return scanner->token;
598 g_scanner_cur_token (GScanner *scanner)
600 g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
602 return scanner->token;
606 g_scanner_cur_value (GScanner *scanner)
612 g_return_val_if_fail (scanner != NULL, v);
614 /* MSC isn't capable of handling return scanner->value; ? */
622 g_scanner_cur_line (GScanner *scanner)
624 g_return_val_if_fail (scanner != NULL, 0);
626 return scanner->line;
630 g_scanner_cur_position (GScanner *scanner)
632 g_return_val_if_fail (scanner != NULL, 0);
634 return scanner->position;
638 g_scanner_eof (GScanner *scanner)
640 g_return_val_if_fail (scanner != NULL, TRUE);
642 return scanner->token == G_TOKEN_EOF || scanner->token == G_TOKEN_ERROR;
646 g_scanner_input_file (GScanner *scanner,
649 g_return_if_fail (scanner != NULL);
650 g_return_if_fail (input_fd >= 0);
652 if (scanner->input_fd >= 0)
653 g_scanner_sync_file_offset (scanner);
655 scanner->token = G_TOKEN_NONE;
656 scanner->value.v_int = 0;
658 scanner->position = 0;
659 scanner->next_token = G_TOKEN_NONE;
661 scanner->input_fd = input_fd;
662 scanner->text = NULL;
663 scanner->text_end = NULL;
665 if (!scanner->buffer)
666 scanner->buffer = g_new (gchar, READ_BUFFER_SIZE + 1);
670 g_scanner_input_text (GScanner *scanner,
674 g_return_if_fail (scanner != NULL);
676 g_return_if_fail (text != NULL);
680 if (scanner->input_fd >= 0)
681 g_scanner_sync_file_offset (scanner);
683 scanner->token = G_TOKEN_NONE;
684 scanner->value.v_int = 0;
686 scanner->position = 0;
687 scanner->next_token = G_TOKEN_NONE;
689 scanner->input_fd = -1;
690 scanner->text = text;
691 scanner->text_end = text + text_len;
695 g_free (scanner->buffer);
696 scanner->buffer = NULL;
701 g_scanner_peek_next_char (GScanner *scanner)
703 if (scanner->text < scanner->text_end)
705 return *scanner->text;
707 else if (scanner->input_fd >= 0)
712 buffer = scanner->buffer;
715 count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
717 while (count == -1 && (errno == EINTR || errno == EAGAIN));
721 scanner->input_fd = -1;
727 scanner->text = buffer;
728 scanner->text_end = buffer + count;
738 g_scanner_sync_file_offset (GScanner *scanner)
740 g_return_if_fail (scanner != NULL);
742 /* for file input, rewind the filedescriptor to the current
743 * buffer position and blow the file read ahead buffer. usefull for
744 * third party uses of our filedescriptor, which hooks onto the current
748 if (scanner->input_fd >= 0 && scanner->text_end > scanner->text)
752 buffered = scanner->text_end - scanner->text;
753 if (lseek (scanner->input_fd, - buffered, SEEK_CUR) >= 0)
755 /* we succeeded, blow our buffer's contents now */
756 scanner->text = NULL;
757 scanner->text_end = NULL;
765 g_scanner_get_char (GScanner *scanner,
771 if (scanner->text < scanner->text_end)
772 fchar = *(scanner->text++);
773 else if (scanner->input_fd >= 0)
778 buffer = scanner->buffer;
781 count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
783 while (count == -1 && (errno == EINTR || errno == EAGAIN));
787 scanner->input_fd = -1;
792 scanner->text = buffer + 1;
793 scanner->text_end = buffer + count;
797 g_scanner_sync_file_offset (scanner);
798 scanner->text_end = scanner->text;
799 scanner->input_fd = -1;
820 g_scanner_unexp_token (GScanner *scanner,
821 GTokenType expected_token,
822 const gchar *identifier_spec,
823 const gchar *symbol_spec,
824 const gchar *symbol_name,
825 const gchar *message,
829 guint token_string_len;
830 gchar *expected_string;
831 guint expected_string_len;
832 gchar *message_prefix;
833 gboolean print_unexp;
834 void (*msg_handler) (GScanner*, const gchar*, ...);
836 g_return_if_fail (scanner != NULL);
839 msg_handler = g_scanner_error;
841 msg_handler = g_scanner_warn;
843 if (!identifier_spec)
844 identifier_spec = "identifier";
846 symbol_spec = "symbol";
848 token_string_len = 56;
849 token_string = g_new (gchar, token_string_len + 1);
850 expected_string_len = 64;
851 expected_string = g_new (gchar, expected_string_len + 1);
854 switch (scanner->token)
857 g_snprintf (token_string, token_string_len, "end of file");
861 if (scanner->token >= 1 && scanner->token <= 255)
863 if ((scanner->token >= ' ' && scanner->token <= '~') ||
864 strchr (scanner->config->cset_identifier_first, scanner->token) ||
865 strchr (scanner->config->cset_identifier_nth, scanner->token))
866 g_snprintf (token_string, token_string_len, "character `%c'", scanner->token);
868 g_snprintf (token_string, token_string_len, "character `\\%o'", scanner->token);
871 else if (!scanner->config->symbol_2_token)
873 g_snprintf (token_string, token_string_len, "(unknown) token <%d>", scanner->token);
878 if (expected_token == G_TOKEN_SYMBOL ||
879 (scanner->config->symbol_2_token &&
880 expected_token > G_TOKEN_LAST))
883 g_snprintf (token_string,
886 print_unexp ? "" : "invalid ",
890 g_snprintf (token_string,
893 print_unexp ? "" : "invalid ",
899 expected_token = G_TOKEN_NONE;
900 switch (scanner->value.v_error)
902 case G_ERR_UNEXP_EOF:
903 g_snprintf (token_string, token_string_len, "scanner: unexpected end of file");
906 case G_ERR_UNEXP_EOF_IN_STRING:
907 g_snprintf (token_string, token_string_len, "scanner: unterminated string constant");
910 case G_ERR_UNEXP_EOF_IN_COMMENT:
911 g_snprintf (token_string, token_string_len, "scanner: unterminated comment");
914 case G_ERR_NON_DIGIT_IN_CONST:
915 g_snprintf (token_string, token_string_len, "scanner: non digit in constant");
918 case G_ERR_FLOAT_RADIX:
919 g_snprintf (token_string, token_string_len, "scanner: invalid radix for floating constant");
922 case G_ERR_FLOAT_MALFORMED:
923 g_snprintf (token_string, token_string_len, "scanner: malformed floating constant");
926 case G_ERR_DIGIT_RADIX:
927 g_snprintf (token_string, token_string_len, "scanner: digit is beyond radix");
932 g_snprintf (token_string, token_string_len, "scanner: unknown error");
938 g_snprintf (token_string, token_string_len, "character `%c'", scanner->value.v_char);
941 case G_TOKEN_IDENTIFIER:
942 case G_TOKEN_IDENTIFIER_NULL:
943 if (expected_token == G_TOKEN_IDENTIFIER ||
944 expected_token == G_TOKEN_IDENTIFIER_NULL)
946 g_snprintf (token_string,
949 print_unexp ? "" : "invalid ",
951 scanner->token == G_TOKEN_IDENTIFIER ? scanner->value.v_string : "null");
958 g_snprintf (token_string, token_string_len, "number `%ld'", scanner->value.v_int);
962 g_snprintf (token_string, token_string_len, "number `%.3f'", scanner->value.v_float);
966 if (expected_token == G_TOKEN_STRING)
968 g_snprintf (token_string,
970 "%s%sstring constant \"%s\"",
971 print_unexp ? "" : "invalid ",
972 scanner->value.v_string[0] == 0 ? "empty " : "",
973 scanner->value.v_string);
974 token_string[token_string_len - 2] = '"';
975 token_string[token_string_len - 1] = 0;
978 case G_TOKEN_COMMENT_SINGLE:
979 case G_TOKEN_COMMENT_MULTI:
980 g_snprintf (token_string, token_string_len, "comment");
984 /* somehow the user's parsing code is screwed, there isn't much
985 * we can do about it.
986 * Note, a common case to trigger this is
987 * g_scanner_peek_next_token(); g_scanner_unexp_token();
988 * without an intermediate g_scanner_get_next_token().
990 g_assert_not_reached ();
995 switch (expected_token)
1000 g_snprintf (expected_string, expected_string_len, "end of file");
1003 if (expected_token >= 1 && expected_token <= 255)
1005 if ((expected_token >= ' ' && expected_token <= '~') ||
1006 strchr (scanner->config->cset_identifier_first, expected_token) ||
1007 strchr (scanner->config->cset_identifier_nth, expected_token))
1008 g_snprintf (expected_string, expected_string_len, "character `%c'", expected_token);
1010 g_snprintf (expected_string, expected_string_len, "character `\\%o'", expected_token);
1013 else if (!scanner->config->symbol_2_token)
1015 g_snprintf (expected_string, expected_string_len, "(unknown) token <%d>", expected_token);
1019 case G_TOKEN_SYMBOL:
1020 need_valid = (scanner->token == G_TOKEN_SYMBOL ||
1021 (scanner->config->symbol_2_token &&
1022 scanner->token > G_TOKEN_LAST));
1023 g_snprintf (expected_string,
1024 expected_string_len,
1026 need_valid ? "valid " : "",
1028 /* FIXME: should we attempt to lookup the symbol_name for symbol_2_token? */
1031 g_snprintf (expected_string, expected_string_len, "%scharacter",
1032 scanner->token == G_TOKEN_CHAR ? "valid " : "");
1034 case G_TOKEN_BINARY:
1036 g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1037 scanner->token == expected_token ? "valid " : "", tstring);
1041 g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1042 scanner->token == expected_token ? "valid " : "", tstring);
1045 tstring = "integer";
1046 g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1047 scanner->token == expected_token ? "valid " : "", tstring);
1050 tstring = "hexadecimal";
1051 g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1052 scanner->token == expected_token ? "valid " : "", tstring);
1056 g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1057 scanner->token == expected_token ? "valid " : "", tstring);
1059 case G_TOKEN_STRING:
1060 g_snprintf (expected_string,
1061 expected_string_len,
1062 "%sstring constant",
1063 scanner->token == G_TOKEN_STRING ? "valid " : "");
1065 case G_TOKEN_IDENTIFIER:
1066 case G_TOKEN_IDENTIFIER_NULL:
1067 need_valid = (scanner->token == G_TOKEN_IDENTIFIER_NULL ||
1068 scanner->token == G_TOKEN_IDENTIFIER);
1069 g_snprintf (expected_string,
1070 expected_string_len,
1072 need_valid ? "valid " : "",
1075 case G_TOKEN_COMMENT_SINGLE:
1076 tstring = "single-line";
1077 g_snprintf (expected_string, expected_string_len, "%scomment (%s)",
1078 scanner->token == expected_token ? "valid " : "", tstring);
1080 case G_TOKEN_COMMENT_MULTI:
1081 tstring = "multi-line";
1082 g_snprintf (expected_string, expected_string_len, "%scomment (%s)",
1083 scanner->token == expected_token ? "valid " : "", tstring);
1087 /* this is handled upon printout */
1091 if (message && message[0] != 0)
1092 message_prefix = " - ";
1095 message_prefix = "";
1098 if (expected_token == G_TOKEN_ERROR)
1100 msg_handler (scanner,
1101 "failure around %s%s%s",
1106 else if (expected_token == G_TOKEN_NONE)
1109 msg_handler (scanner,
1110 "unexpected %s%s%s",
1115 msg_handler (scanner,
1124 msg_handler (scanner,
1125 "unexpected %s, expected %s%s%s",
1131 msg_handler (scanner,
1132 "%s, expected %s%s%s",
1139 g_free (token_string);
1140 g_free (expected_string);
1144 g_scanner_get_token_i (GScanner *scanner,
1145 GTokenType *token_p,
1146 GTokenValue *value_p,
1152 g_scanner_free_value (token_p, value_p);
1153 g_scanner_get_token_ll (scanner, token_p, value_p, line_p, position_p);
1155 while (((*token_p > 0 && *token_p < 256) &&
1156 strchr (scanner->config->cset_skip_characters, *token_p)) ||
1157 (*token_p == G_TOKEN_CHAR &&
1158 strchr (scanner->config->cset_skip_characters, value_p->v_char)) ||
1159 (*token_p == G_TOKEN_COMMENT_MULTI &&
1160 scanner->config->skip_comment_multi) ||
1161 (*token_p == G_TOKEN_COMMENT_SINGLE &&
1162 scanner->config->skip_comment_single));
1166 case G_TOKEN_IDENTIFIER:
1167 if (scanner->config->identifier_2_string)
1168 *token_p = G_TOKEN_STRING;
1171 case G_TOKEN_SYMBOL:
1172 if (scanner->config->symbol_2_token)
1173 *token_p = (GTokenType) value_p->v_symbol;
1176 case G_TOKEN_BINARY:
1179 if (scanner->config->numbers_2_int)
1180 *token_p = G_TOKEN_INT;
1187 if (*token_p == G_TOKEN_INT &&
1188 scanner->config->int_2_float)
1190 *token_p = G_TOKEN_FLOAT;
1191 value_p->v_float = value_p->v_int;
1198 g_scanner_get_token_ll (GScanner *scanner,
1199 GTokenType *token_p,
1200 GTokenValue *value_p,
1204 GScannerConfig *config;
1206 gboolean in_comment_multi;
1207 gboolean in_comment_single;
1208 gboolean in_string_sq;
1209 gboolean in_string_dq;
1214 config = scanner->config;
1215 (*value_p).v_int = 0;
1217 if ((scanner->text >= scanner->text_end && scanner->input_fd < 0) ||
1218 scanner->token == G_TOKEN_EOF)
1220 *token_p = G_TOKEN_EOF;
1224 in_comment_multi = FALSE;
1225 in_comment_single = FALSE;
1226 in_string_sq = FALSE;
1227 in_string_dq = FALSE;
1230 do /* while (ch != 0) */
1232 gboolean dotted_float = FALSE;
1234 ch = g_scanner_get_char (scanner, line_p, position_p);
1237 token = G_TOKEN_NONE;
1239 /* this is *evil*, but needed ;(
1240 * we first check for identifier first character, because it
1241 * might interfere with other key chars like slashes or numbers
1243 if (config->scan_identifier &&
1244 ch && strchr (config->cset_identifier_first, ch))
1245 goto identifier_precedence;
1250 token = G_TOKEN_EOF;
1256 if (!config->scan_comment_multi ||
1257 g_scanner_peek_next_char (scanner) != '*')
1259 g_scanner_get_char (scanner, line_p, position_p);
1260 token = G_TOKEN_COMMENT_MULTI;
1261 in_comment_multi = TRUE;
1262 gstring = g_string_new ("");
1263 while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1265 if (ch == '*' && g_scanner_peek_next_char (scanner) == '/')
1267 g_scanner_get_char (scanner, line_p, position_p);
1268 in_comment_multi = FALSE;
1272 gstring = g_string_append_c (gstring, ch);
1278 if (!config->scan_string_sq)
1280 token = G_TOKEN_STRING;
1281 in_string_sq = TRUE;
1282 gstring = g_string_new ("");
1283 while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1287 in_string_sq = FALSE;
1291 gstring = g_string_append_c (gstring, ch);
1297 if (!config->scan_string_dq)
1299 token = G_TOKEN_STRING;
1300 in_string_dq = TRUE;
1301 gstring = g_string_new ("");
1302 while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1306 in_string_dq = FALSE;
1313 ch = g_scanner_get_char (scanner, line_p, position_p);
1323 gstring = g_string_append_c (gstring, '\\');
1327 gstring = g_string_append_c (gstring, '\n');
1331 gstring = g_string_append_c (gstring, '\t');
1335 gstring = g_string_append_c (gstring, '\r');
1339 gstring = g_string_append_c (gstring, '\b');
1343 gstring = g_string_append_c (gstring, '\f');
1355 fchar = g_scanner_peek_next_char (scanner);
1356 if (fchar >= '0' && fchar <= '7')
1358 ch = g_scanner_get_char (scanner, line_p, position_p);
1359 i = i * 8 + ch - '0';
1360 fchar = g_scanner_peek_next_char (scanner);
1361 if (fchar >= '0' && fchar <= '7')
1363 ch = g_scanner_get_char (scanner, line_p, position_p);
1364 i = i * 8 + ch - '0';
1367 gstring = g_string_append_c (gstring, i);
1371 gstring = g_string_append_c (gstring, ch);
1376 gstring = g_string_append_c (gstring, ch);
1383 if (!config->scan_float)
1385 token = G_TOKEN_FLOAT;
1386 dotted_float = TRUE;
1387 ch = g_scanner_get_char (scanner, line_p, position_p);
1388 goto number_parsing;
1391 if (!config->scan_hex_dollar)
1393 token = G_TOKEN_HEX;
1394 ch = g_scanner_get_char (scanner, line_p, position_p);
1395 goto number_parsing;
1398 if (config->scan_octal)
1399 token = G_TOKEN_OCTAL;
1401 token = G_TOKEN_INT;
1402 ch = g_scanner_peek_next_char (scanner);
1403 if (config->scan_hex && (ch == 'x' || ch == 'X'))
1405 token = G_TOKEN_HEX;
1406 g_scanner_get_char (scanner, line_p, position_p);
1407 ch = g_scanner_get_char (scanner, line_p, position_p);
1410 token = G_TOKEN_ERROR;
1411 value.v_error = G_ERR_UNEXP_EOF;
1415 if (g_scanner_char_2_num (ch, 16) < 0)
1417 token = G_TOKEN_ERROR;
1418 value.v_error = G_ERR_DIGIT_RADIX;
1423 else if (config->scan_binary && (ch == 'b' || ch == 'B'))
1425 token = G_TOKEN_BINARY;
1426 g_scanner_get_char (scanner, line_p, position_p);
1427 ch = g_scanner_get_char (scanner, line_p, position_p);
1430 token = G_TOKEN_ERROR;
1431 value.v_error = G_ERR_UNEXP_EOF;
1435 if (g_scanner_char_2_num (ch, 10) < 0)
1437 token = G_TOKEN_ERROR;
1438 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1457 gboolean in_number = TRUE;
1460 if (token == G_TOKEN_NONE)
1461 token = G_TOKEN_INT;
1463 gstring = g_string_new (dotted_float ? "0." : "");
1464 gstring = g_string_append_c (gstring, ch);
1466 do /* while (in_number) */
1470 is_E = token == G_TOKEN_FLOAT && (ch == 'e' || ch == 'E');
1472 ch = g_scanner_peek_next_char (scanner);
1474 if (g_scanner_char_2_num (ch, 36) >= 0 ||
1475 (config->scan_float && ch == '.') ||
1476 (is_E && (ch == '+' || ch == '-')))
1478 ch = g_scanner_get_char (scanner, line_p, position_p);
1483 if (token != G_TOKEN_INT && token != G_TOKEN_OCTAL)
1485 value.v_error = token == G_TOKEN_FLOAT ? G_ERR_FLOAT_MALFORMED : G_ERR_FLOAT_RADIX;
1486 token = G_TOKEN_ERROR;
1491 token = G_TOKEN_FLOAT;
1492 gstring = g_string_append_c (gstring, ch);
1506 gstring = g_string_append_c (gstring, ch);
1511 if (token != G_TOKEN_FLOAT)
1513 token = G_TOKEN_ERROR;
1514 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1518 gstring = g_string_append_c (gstring, ch);
1523 if ((token != G_TOKEN_HEX && !config->scan_float) ||
1524 (token != G_TOKEN_HEX &&
1525 token != G_TOKEN_OCTAL &&
1526 token != G_TOKEN_FLOAT &&
1527 token != G_TOKEN_INT))
1529 token = G_TOKEN_ERROR;
1530 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1535 if (token != G_TOKEN_HEX)
1536 token = G_TOKEN_FLOAT;
1537 gstring = g_string_append_c (gstring, ch);
1542 if (token != G_TOKEN_HEX)
1544 token = G_TOKEN_ERROR;
1545 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1549 gstring = g_string_append_c (gstring, ch);
1561 case G_TOKEN_BINARY:
1562 value.v_binary = strtol (gstring->str, &endptr, 2);
1566 value.v_octal = strtol (gstring->str, &endptr, 8);
1570 value.v_int = strtol (gstring->str, &endptr, 10);
1574 value.v_float = g_strtod (gstring->str, &endptr);
1578 value.v_hex = strtol (gstring->str, &endptr, 16);
1584 if (endptr && *endptr)
1586 token = G_TOKEN_ERROR;
1587 if (*endptr == 'e' || *endptr == 'E')
1588 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1590 value.v_error = G_ERR_DIGIT_RADIX;
1592 g_string_free (gstring, TRUE);
1595 } /* number_parsing:... */
1601 if (config->cpair_comment_single &&
1602 ch == config->cpair_comment_single[0])
1604 token = G_TOKEN_COMMENT_SINGLE;
1605 in_comment_single = TRUE;
1606 gstring = g_string_new ("");
1607 ch = g_scanner_get_char (scanner, line_p, position_p);
1610 if (ch == config->cpair_comment_single[1])
1612 in_comment_single = FALSE;
1617 gstring = g_string_append_c (gstring, ch);
1618 ch = g_scanner_get_char (scanner, line_p, position_p);
1621 else if (config->scan_identifier && ch &&
1622 strchr (config->cset_identifier_first, ch))
1624 identifier_precedence:
1626 if (config->cset_identifier_nth && ch &&
1627 strchr (config->cset_identifier_nth,
1628 g_scanner_peek_next_char (scanner)))
1630 token = G_TOKEN_IDENTIFIER;
1631 gstring = g_string_new ("");
1632 gstring = g_string_append_c (gstring, ch);
1635 ch = g_scanner_get_char (scanner, line_p, position_p);
1636 gstring = g_string_append_c (gstring, ch);
1637 ch = g_scanner_peek_next_char (scanner);
1639 while (ch && strchr (config->cset_identifier_nth, ch));
1642 else if (config->scan_identifier_1char)
1644 token = G_TOKEN_IDENTIFIER;
1645 value.v_identifier = g_new0 (gchar, 2);
1646 value.v_identifier[0] = ch;
1652 if (config->char_2_token)
1656 token = G_TOKEN_CHAR;
1661 } /* default_case:... */
1664 g_assert (ch == 0 && token != G_TOKEN_NONE); /* paranoid */
1668 if (in_comment_multi || in_comment_single ||
1669 in_string_sq || in_string_dq)
1671 token = G_TOKEN_ERROR;
1674 g_string_free (gstring, TRUE);
1678 if (in_comment_multi || in_comment_single)
1679 value.v_error = G_ERR_UNEXP_EOF_IN_COMMENT;
1680 else /* (in_string_sq || in_string_dq) */
1681 value.v_error = G_ERR_UNEXP_EOF_IN_STRING;
1686 value.v_string = gstring->str;
1687 g_string_free (gstring, FALSE);
1691 if (token == G_TOKEN_IDENTIFIER)
1693 if (config->scan_symbols)
1698 scope_id = scanner->scope_id;
1699 key = g_scanner_lookup_internal (scanner, scope_id, value.v_identifier);
1700 if (!key && scope_id && scanner->config->scope_0_fallback)
1701 key = g_scanner_lookup_internal (scanner, 0, value.v_identifier);
1705 g_free (value.v_identifier);
1706 token = G_TOKEN_SYMBOL;
1707 value.v_symbol = key->value;
1711 if (token == G_TOKEN_IDENTIFIER &&
1712 config->scan_identifier_NULL &&
1713 strlen (value.v_identifier) == 4)
1715 gchar *null_upper = "NULL";
1716 gchar *null_lower = "null";
1718 if (scanner->config->case_sensitive)
1720 if (value.v_identifier[0] == null_upper[0] &&
1721 value.v_identifier[1] == null_upper[1] &&
1722 value.v_identifier[2] == null_upper[2] &&
1723 value.v_identifier[3] == null_upper[3])
1724 token = G_TOKEN_IDENTIFIER_NULL;
1728 if ((value.v_identifier[0] == null_upper[0] ||
1729 value.v_identifier[0] == null_lower[0]) &&
1730 (value.v_identifier[1] == null_upper[1] ||
1731 value.v_identifier[1] == null_lower[1]) &&
1732 (value.v_identifier[2] == null_upper[2] ||
1733 value.v_identifier[2] == null_lower[2]) &&
1734 (value.v_identifier[3] == null_upper[3] ||
1735 value.v_identifier[3] == null_lower[3]))
1736 token = G_TOKEN_IDENTIFIER_NULL;