1 /* GLIB - Library of useful routines for C programming
2 * Copyright (C) 1995-1997 Peter Mattis, Spencer Kimball and Josh MacDonald
4 * GScanner: Flexible lexical scanner for general purpose.
5 * Copyright (C) 1997, 1998 Tim Janik
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
20 * Boston, MA 02111-1307, USA.
24 * Modified by the GLib Team and others 1997-2000. See the AUTHORS
25 * file for a list of people on the GLib Team. See the ChangeLog
26 * files for a list of changes. These files are distributed with
27 * GLib at ftp://ftp.gtk.org/pub/gtk/.
43 #include "gprintfint.h"
49 #include <io.h> /* For _read() */
53 #define to_lower(c) ( \
55 ( (((guchar)(c))>='A' && ((guchar)(c))<='Z') * ('a'-'A') ) | \
56 ( (((guchar)(c))>=192 && ((guchar)(c))<=214) * (224-192) ) | \
57 ( (((guchar)(c))>=216 && ((guchar)(c))<=222) * (248-216) ) | \
61 #define READ_BUFFER_SIZE (4000)
64 /* --- typedefs --- */
65 typedef struct _GScannerKey GScannerKey;
75 /* --- variables --- */
76 static GScannerConfig g_scanner_config_template =
80 ) /* cset_skip_characters */,
85 ) /* cset_identifier_first */,
93 ) /* cset_identifier_nth */,
94 ( "#\n" ) /* cpair_comment_single */,
96 FALSE /* case_sensitive */,
98 TRUE /* skip_comment_multi */,
99 TRUE /* skip_comment_single */,
100 TRUE /* scan_comment_multi */,
101 TRUE /* scan_identifier */,
102 FALSE /* scan_identifier_1char */,
103 FALSE /* scan_identifier_NULL */,
104 TRUE /* scan_symbols */,
105 FALSE /* scan_binary */,
106 TRUE /* scan_octal */,
107 TRUE /* scan_float */,
109 FALSE /* scan_hex_dollar */,
110 TRUE /* scan_string_sq */,
111 TRUE /* scan_string_dq */,
112 TRUE /* numbers_2_int */,
113 FALSE /* int_2_float */,
114 FALSE /* identifier_2_string */,
115 TRUE /* char_2_token */,
116 FALSE /* symbol_2_token */,
117 FALSE /* scope_0_fallback */,
118 FALSE /* store_int64 */,
122 /* --- prototypes --- */
124 GScannerKey* g_scanner_lookup_internal (GScanner *scanner,
126 const gchar *symbol);
127 static gboolean g_scanner_key_equal (gconstpointer v1,
129 static guint g_scanner_key_hash (gconstpointer v);
130 static void g_scanner_get_token_ll (GScanner *scanner,
132 GTokenValue *value_p,
135 static void g_scanner_get_token_i (GScanner *scanner,
137 GTokenValue *value_p,
141 static guchar g_scanner_peek_next_char (GScanner *scanner);
142 static guchar g_scanner_get_char (GScanner *scanner,
145 static void g_scanner_msg_handler (GScanner *scanner,
150 /* --- functions --- */
152 g_scanner_char_2_num (guchar c,
155 if (c >= '0' && c <= '9')
157 else if (c >= 'A' && c <= 'Z')
159 else if (c >= 'a' && c <= 'z')
171 g_scanner_new (const GScannerConfig *config_templ)
176 config_templ = &g_scanner_config_template;
178 scanner = g_new0 (GScanner, 1);
180 scanner->user_data = NULL;
181 scanner->max_parse_errors = 1;
182 scanner->parse_errors = 0;
183 scanner->input_name = NULL;
184 g_datalist_init (&scanner->qdata);
186 scanner->config = g_new0 (GScannerConfig, 1);
188 scanner->config->case_sensitive = config_templ->case_sensitive;
189 scanner->config->cset_skip_characters = config_templ->cset_skip_characters;
190 if (!scanner->config->cset_skip_characters)
191 scanner->config->cset_skip_characters = "";
192 scanner->config->cset_identifier_first = config_templ->cset_identifier_first;
193 scanner->config->cset_identifier_nth = config_templ->cset_identifier_nth;
194 scanner->config->cpair_comment_single = config_templ->cpair_comment_single;
195 scanner->config->skip_comment_multi = config_templ->skip_comment_multi;
196 scanner->config->skip_comment_single = config_templ->skip_comment_single;
197 scanner->config->scan_comment_multi = config_templ->scan_comment_multi;
198 scanner->config->scan_identifier = config_templ->scan_identifier;
199 scanner->config->scan_identifier_1char = config_templ->scan_identifier_1char;
200 scanner->config->scan_identifier_NULL = config_templ->scan_identifier_NULL;
201 scanner->config->scan_symbols = config_templ->scan_symbols;
202 scanner->config->scan_binary = config_templ->scan_binary;
203 scanner->config->scan_octal = config_templ->scan_octal;
204 scanner->config->scan_float = config_templ->scan_float;
205 scanner->config->scan_hex = config_templ->scan_hex;
206 scanner->config->scan_hex_dollar = config_templ->scan_hex_dollar;
207 scanner->config->scan_string_sq = config_templ->scan_string_sq;
208 scanner->config->scan_string_dq = config_templ->scan_string_dq;
209 scanner->config->numbers_2_int = config_templ->numbers_2_int;
210 scanner->config->int_2_float = config_templ->int_2_float;
211 scanner->config->identifier_2_string = config_templ->identifier_2_string;
212 scanner->config->char_2_token = config_templ->char_2_token;
213 scanner->config->symbol_2_token = config_templ->symbol_2_token;
214 scanner->config->scope_0_fallback = config_templ->scope_0_fallback;
215 scanner->config->store_int64 = config_templ->store_int64;
217 scanner->token = G_TOKEN_NONE;
218 scanner->value.v_int64 = 0;
220 scanner->position = 0;
222 scanner->next_token = G_TOKEN_NONE;
223 scanner->next_value.v_int64 = 0;
224 scanner->next_line = 1;
225 scanner->next_position = 0;
227 scanner->symbol_table = g_hash_table_new (g_scanner_key_hash, g_scanner_key_equal);
228 scanner->input_fd = -1;
229 scanner->text = NULL;
230 scanner->text_end = NULL;
231 scanner->buffer = NULL;
232 scanner->scope_id = 0;
234 scanner->msg_handler = g_scanner_msg_handler;
240 g_scanner_free_value (GTokenType *token_p,
241 GTokenValue *value_p)
246 case G_TOKEN_IDENTIFIER:
247 case G_TOKEN_IDENTIFIER_NULL:
248 case G_TOKEN_COMMENT_SINGLE:
249 case G_TOKEN_COMMENT_MULTI:
250 g_free (value_p->v_string);
257 *token_p = G_TOKEN_NONE;
261 g_scanner_destroy_symbol_table_entry (gpointer _key,
265 GScannerKey *key = _key;
267 g_free (key->symbol);
272 g_scanner_destroy (GScanner *scanner)
274 g_return_if_fail (scanner != NULL);
276 g_datalist_clear (&scanner->qdata);
277 g_hash_table_foreach (scanner->symbol_table,
278 g_scanner_destroy_symbol_table_entry, NULL);
279 g_hash_table_destroy (scanner->symbol_table);
280 g_scanner_free_value (&scanner->token, &scanner->value);
281 g_scanner_free_value (&scanner->next_token, &scanner->next_value);
282 g_free (scanner->config);
283 g_free (scanner->buffer);
288 g_scanner_msg_handler (GScanner *scanner,
292 g_return_if_fail (scanner != NULL);
294 _g_fprintf (stderr, "%s:%d: ",
295 scanner->input_name ? scanner->input_name : "<memory>",
298 _g_fprintf (stderr, "error: ");
299 _g_fprintf (stderr, "%s\n", message);
303 g_scanner_error (GScanner *scanner,
307 g_return_if_fail (scanner != NULL);
308 g_return_if_fail (format != NULL);
310 scanner->parse_errors++;
312 if (scanner->msg_handler)
317 va_start (args, format);
318 string = g_strdup_vprintf (format, args);
321 scanner->msg_handler (scanner, string, TRUE);
328 g_scanner_warn (GScanner *scanner,
332 g_return_if_fail (scanner != NULL);
333 g_return_if_fail (format != NULL);
335 if (scanner->msg_handler)
340 va_start (args, format);
341 string = g_strdup_vprintf (format, args);
344 scanner->msg_handler (scanner, string, FALSE);
351 g_scanner_key_equal (gconstpointer v1,
354 const GScannerKey *key1 = v1;
355 const GScannerKey *key2 = v2;
357 return (key1->scope_id == key2->scope_id) && (strcmp (key1->symbol, key2->symbol) == 0);
361 g_scanner_key_hash (gconstpointer v)
363 const GScannerKey *key = v;
368 for (c = key->symbol; *c; c++)
369 h = (h << 5) - h + *c;
374 static inline GScannerKey*
375 g_scanner_lookup_internal (GScanner *scanner,
382 key.scope_id = scope_id;
384 if (!scanner->config->case_sensitive)
389 key.symbol = g_new (gchar, strlen (symbol) + 1);
390 for (d = key.symbol, c = symbol; *c; c++, d++)
393 key_p = g_hash_table_lookup (scanner->symbol_table, &key);
398 key.symbol = (gchar*) symbol;
399 key_p = g_hash_table_lookup (scanner->symbol_table, &key);
406 g_scanner_scope_add_symbol (GScanner *scanner,
413 g_return_if_fail (scanner != NULL);
414 g_return_if_fail (symbol != NULL);
416 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
420 key = g_new (GScannerKey, 1);
421 key->scope_id = scope_id;
422 key->symbol = g_strdup (symbol);
424 if (!scanner->config->case_sensitive)
435 g_hash_table_insert (scanner->symbol_table, key, key);
442 g_scanner_scope_remove_symbol (GScanner *scanner,
448 g_return_if_fail (scanner != NULL);
449 g_return_if_fail (symbol != NULL);
451 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
455 g_hash_table_remove (scanner->symbol_table, key);
456 g_free (key->symbol);
462 g_scanner_lookup_symbol (GScanner *scanner,
468 g_return_val_if_fail (scanner != NULL, NULL);
473 scope_id = scanner->scope_id;
474 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
475 if (!key && scope_id && scanner->config->scope_0_fallback)
476 key = g_scanner_lookup_internal (scanner, 0, symbol);
485 g_scanner_scope_lookup_symbol (GScanner *scanner,
491 g_return_val_if_fail (scanner != NULL, NULL);
496 key = g_scanner_lookup_internal (scanner, scope_id, symbol);
505 g_scanner_set_scope (GScanner *scanner,
510 g_return_val_if_fail (scanner != NULL, 0);
512 old_scope_id = scanner->scope_id;
513 scanner->scope_id = scope_id;
519 g_scanner_foreach_internal (gpointer _key,
530 func = (GHFunc) d[0];
535 if (key->scope_id == *scope_id)
536 func (key->symbol, key->value, user_data);
540 g_scanner_scope_foreach_symbol (GScanner *scanner,
547 g_return_if_fail (scanner != NULL);
549 d[0] = (gpointer) func;
553 g_hash_table_foreach (scanner->symbol_table, g_scanner_foreach_internal, d);
557 g_scanner_peek_next_token (GScanner *scanner)
559 g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
561 if (scanner->next_token == G_TOKEN_NONE)
563 scanner->next_line = scanner->line;
564 scanner->next_position = scanner->position;
565 g_scanner_get_token_i (scanner,
566 &scanner->next_token,
567 &scanner->next_value,
569 &scanner->next_position);
572 return scanner->next_token;
576 g_scanner_get_next_token (GScanner *scanner)
578 g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
580 if (scanner->next_token != G_TOKEN_NONE)
582 g_scanner_free_value (&scanner->token, &scanner->value);
584 scanner->token = scanner->next_token;
585 scanner->value = scanner->next_value;
586 scanner->line = scanner->next_line;
587 scanner->position = scanner->next_position;
588 scanner->next_token = G_TOKEN_NONE;
591 g_scanner_get_token_i (scanner,
597 return scanner->token;
601 g_scanner_cur_token (GScanner *scanner)
603 g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);
605 return scanner->token;
609 g_scanner_cur_value (GScanner *scanner)
615 g_return_val_if_fail (scanner != NULL, v);
617 /* MSC isn't capable of handling return scanner->value; ? */
625 g_scanner_cur_line (GScanner *scanner)
627 g_return_val_if_fail (scanner != NULL, 0);
629 return scanner->line;
633 g_scanner_cur_position (GScanner *scanner)
635 g_return_val_if_fail (scanner != NULL, 0);
637 return scanner->position;
641 g_scanner_eof (GScanner *scanner)
643 g_return_val_if_fail (scanner != NULL, TRUE);
645 return scanner->token == G_TOKEN_EOF || scanner->token == G_TOKEN_ERROR;
649 g_scanner_input_file (GScanner *scanner,
652 g_return_if_fail (scanner != NULL);
653 g_return_if_fail (input_fd >= 0);
655 if (scanner->input_fd >= 0)
656 g_scanner_sync_file_offset (scanner);
658 scanner->token = G_TOKEN_NONE;
659 scanner->value.v_int64 = 0;
661 scanner->position = 0;
662 scanner->next_token = G_TOKEN_NONE;
664 scanner->input_fd = input_fd;
665 scanner->text = NULL;
666 scanner->text_end = NULL;
668 if (!scanner->buffer)
669 scanner->buffer = g_new (gchar, READ_BUFFER_SIZE + 1);
673 g_scanner_input_text (GScanner *scanner,
677 g_return_if_fail (scanner != NULL);
679 g_return_if_fail (text != NULL);
683 if (scanner->input_fd >= 0)
684 g_scanner_sync_file_offset (scanner);
686 scanner->token = G_TOKEN_NONE;
687 scanner->value.v_int64 = 0;
689 scanner->position = 0;
690 scanner->next_token = G_TOKEN_NONE;
692 scanner->input_fd = -1;
693 scanner->text = text;
694 scanner->text_end = text + text_len;
698 g_free (scanner->buffer);
699 scanner->buffer = NULL;
704 g_scanner_peek_next_char (GScanner *scanner)
706 if (scanner->text < scanner->text_end)
708 return *scanner->text;
710 else if (scanner->input_fd >= 0)
715 buffer = scanner->buffer;
718 count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
720 while (count == -1 && (errno == EINTR || errno == EAGAIN));
724 scanner->input_fd = -1;
730 scanner->text = buffer;
731 scanner->text_end = buffer + count;
741 g_scanner_sync_file_offset (GScanner *scanner)
743 g_return_if_fail (scanner != NULL);
745 /* for file input, rewind the filedescriptor to the current
746 * buffer position and blow the file read ahead buffer. usefull for
747 * third party uses of our filedescriptor, which hooks onto the current
751 if (scanner->input_fd >= 0 && scanner->text_end > scanner->text)
755 buffered = scanner->text_end - scanner->text;
756 if (lseek (scanner->input_fd, - buffered, SEEK_CUR) >= 0)
758 /* we succeeded, blow our buffer's contents now */
759 scanner->text = NULL;
760 scanner->text_end = NULL;
768 g_scanner_get_char (GScanner *scanner,
774 if (scanner->text < scanner->text_end)
775 fchar = *(scanner->text++);
776 else if (scanner->input_fd >= 0)
781 buffer = scanner->buffer;
784 count = read (scanner->input_fd, buffer, READ_BUFFER_SIZE);
786 while (count == -1 && (errno == EINTR || errno == EAGAIN));
790 scanner->input_fd = -1;
795 scanner->text = buffer + 1;
796 scanner->text_end = buffer + count;
800 g_scanner_sync_file_offset (scanner);
801 scanner->text_end = scanner->text;
802 scanner->input_fd = -1;
823 g_scanner_unexp_token (GScanner *scanner,
824 GTokenType expected_token,
825 const gchar *identifier_spec,
826 const gchar *symbol_spec,
827 const gchar *symbol_name,
828 const gchar *message,
832 guint token_string_len;
833 gchar *expected_string;
834 guint expected_string_len;
835 gchar *message_prefix;
836 gboolean print_unexp;
837 void (*msg_handler) (GScanner*, const gchar*, ...);
839 g_return_if_fail (scanner != NULL);
842 msg_handler = g_scanner_error;
844 msg_handler = g_scanner_warn;
846 if (!identifier_spec)
847 identifier_spec = "identifier";
849 symbol_spec = "symbol";
851 token_string_len = 56;
852 token_string = g_new (gchar, token_string_len + 1);
853 expected_string_len = 64;
854 expected_string = g_new (gchar, expected_string_len + 1);
857 switch (scanner->token)
860 _g_snprintf (token_string, token_string_len, "end of file");
864 if (scanner->token >= 1 && scanner->token <= 255)
866 if ((scanner->token >= ' ' && scanner->token <= '~') ||
867 strchr (scanner->config->cset_identifier_first, scanner->token) ||
868 strchr (scanner->config->cset_identifier_nth, scanner->token))
869 _g_snprintf (token_string, token_string_len, "character `%c'", scanner->token);
871 _g_snprintf (token_string, token_string_len, "character `\\%o'", scanner->token);
874 else if (!scanner->config->symbol_2_token)
876 _g_snprintf (token_string, token_string_len, "(unknown) token <%d>", scanner->token);
881 if (expected_token == G_TOKEN_SYMBOL ||
882 (scanner->config->symbol_2_token &&
883 expected_token > G_TOKEN_LAST))
886 _g_snprintf (token_string,
889 print_unexp ? "" : "invalid ",
893 _g_snprintf (token_string,
896 print_unexp ? "" : "invalid ",
902 expected_token = G_TOKEN_NONE;
903 switch (scanner->value.v_error)
905 case G_ERR_UNEXP_EOF:
906 _g_snprintf (token_string, token_string_len, "scanner: unexpected end of file");
909 case G_ERR_UNEXP_EOF_IN_STRING:
910 _g_snprintf (token_string, token_string_len, "scanner: unterminated string constant");
913 case G_ERR_UNEXP_EOF_IN_COMMENT:
914 _g_snprintf (token_string, token_string_len, "scanner: unterminated comment");
917 case G_ERR_NON_DIGIT_IN_CONST:
918 _g_snprintf (token_string, token_string_len, "scanner: non digit in constant");
921 case G_ERR_FLOAT_RADIX:
922 _g_snprintf (token_string, token_string_len, "scanner: invalid radix for floating constant");
925 case G_ERR_FLOAT_MALFORMED:
926 _g_snprintf (token_string, token_string_len, "scanner: malformed floating constant");
929 case G_ERR_DIGIT_RADIX:
930 _g_snprintf (token_string, token_string_len, "scanner: digit is beyond radix");
935 _g_snprintf (token_string, token_string_len, "scanner: unknown error");
941 _g_snprintf (token_string, token_string_len, "character `%c'", scanner->value.v_char);
944 case G_TOKEN_IDENTIFIER:
945 case G_TOKEN_IDENTIFIER_NULL:
946 if (expected_token == G_TOKEN_IDENTIFIER ||
947 expected_token == G_TOKEN_IDENTIFIER_NULL)
949 _g_snprintf (token_string,
952 print_unexp ? "" : "invalid ",
954 scanner->token == G_TOKEN_IDENTIFIER ? scanner->value.v_string : "null");
961 if (scanner->config->store_int64)
962 _g_snprintf (token_string, token_string_len, "number `%llu'", scanner->value.v_int64);
964 _g_snprintf (token_string, token_string_len, "number `%lu'", scanner->value.v_int);
968 _g_snprintf (token_string, token_string_len, "number `%.3f'", scanner->value.v_float);
972 if (expected_token == G_TOKEN_STRING)
974 _g_snprintf (token_string,
976 "%s%sstring constant \"%s\"",
977 print_unexp ? "" : "invalid ",
978 scanner->value.v_string[0] == 0 ? "empty " : "",
979 scanner->value.v_string);
980 token_string[token_string_len - 2] = '"';
981 token_string[token_string_len - 1] = 0;
984 case G_TOKEN_COMMENT_SINGLE:
985 case G_TOKEN_COMMENT_MULTI:
986 _g_snprintf (token_string, token_string_len, "comment");
990 /* somehow the user's parsing code is screwed, there isn't much
991 * we can do about it.
992 * Note, a common case to trigger this is
993 * g_scanner_peek_next_token(); g_scanner_unexp_token();
994 * without an intermediate g_scanner_get_next_token().
996 g_assert_not_reached ();
1001 switch (expected_token)
1003 gboolean need_valid;
1006 _g_snprintf (expected_string, expected_string_len, "end of file");
1009 if (expected_token >= 1 && expected_token <= 255)
1011 if ((expected_token >= ' ' && expected_token <= '~') ||
1012 strchr (scanner->config->cset_identifier_first, expected_token) ||
1013 strchr (scanner->config->cset_identifier_nth, expected_token))
1014 _g_snprintf (expected_string, expected_string_len, "character `%c'", expected_token);
1016 _g_snprintf (expected_string, expected_string_len, "character `\\%o'", expected_token);
1019 else if (!scanner->config->symbol_2_token)
1021 _g_snprintf (expected_string, expected_string_len, "(unknown) token <%d>", expected_token);
1025 case G_TOKEN_SYMBOL:
1026 need_valid = (scanner->token == G_TOKEN_SYMBOL ||
1027 (scanner->config->symbol_2_token &&
1028 scanner->token > G_TOKEN_LAST));
1029 _g_snprintf (expected_string,
1030 expected_string_len,
1032 need_valid ? "valid " : "",
1034 /* FIXME: should we attempt to lookup the symbol_name for symbol_2_token? */
1037 _g_snprintf (expected_string, expected_string_len, "%scharacter",
1038 scanner->token == G_TOKEN_CHAR ? "valid " : "");
1040 case G_TOKEN_BINARY:
1042 _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1043 scanner->token == expected_token ? "valid " : "", tstring);
1047 _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1048 scanner->token == expected_token ? "valid " : "", tstring);
1051 tstring = "integer";
1052 _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1053 scanner->token == expected_token ? "valid " : "", tstring);
1056 tstring = "hexadecimal";
1057 _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1058 scanner->token == expected_token ? "valid " : "", tstring);
1062 _g_snprintf (expected_string, expected_string_len, "%snumber (%s)",
1063 scanner->token == expected_token ? "valid " : "", tstring);
1065 case G_TOKEN_STRING:
1066 _g_snprintf (expected_string,
1067 expected_string_len,
1068 "%sstring constant",
1069 scanner->token == G_TOKEN_STRING ? "valid " : "");
1071 case G_TOKEN_IDENTIFIER:
1072 case G_TOKEN_IDENTIFIER_NULL:
1073 need_valid = (scanner->token == G_TOKEN_IDENTIFIER_NULL ||
1074 scanner->token == G_TOKEN_IDENTIFIER);
1075 _g_snprintf (expected_string,
1076 expected_string_len,
1078 need_valid ? "valid " : "",
1081 case G_TOKEN_COMMENT_SINGLE:
1082 tstring = "single-line";
1083 _g_snprintf (expected_string, expected_string_len, "%scomment (%s)",
1084 scanner->token == expected_token ? "valid " : "", tstring);
1086 case G_TOKEN_COMMENT_MULTI:
1087 tstring = "multi-line";
1088 _g_snprintf (expected_string, expected_string_len, "%scomment (%s)",
1089 scanner->token == expected_token ? "valid " : "", tstring);
1093 /* this is handled upon printout */
1097 if (message && message[0] != 0)
1098 message_prefix = " - ";
1101 message_prefix = "";
1104 if (expected_token == G_TOKEN_ERROR)
1106 msg_handler (scanner,
1107 "failure around %s%s%s",
1112 else if (expected_token == G_TOKEN_NONE)
1115 msg_handler (scanner,
1116 "unexpected %s%s%s",
1121 msg_handler (scanner,
1130 msg_handler (scanner,
1131 "unexpected %s, expected %s%s%s",
1137 msg_handler (scanner,
1138 "%s, expected %s%s%s",
1145 g_free (token_string);
1146 g_free (expected_string);
1150 g_scanner_get_token_i (GScanner *scanner,
1151 GTokenType *token_p,
1152 GTokenValue *value_p,
1158 g_scanner_free_value (token_p, value_p);
1159 g_scanner_get_token_ll (scanner, token_p, value_p, line_p, position_p);
1161 while (((*token_p > 0 && *token_p < 256) &&
1162 strchr (scanner->config->cset_skip_characters, *token_p)) ||
1163 (*token_p == G_TOKEN_CHAR &&
1164 strchr (scanner->config->cset_skip_characters, value_p->v_char)) ||
1165 (*token_p == G_TOKEN_COMMENT_MULTI &&
1166 scanner->config->skip_comment_multi) ||
1167 (*token_p == G_TOKEN_COMMENT_SINGLE &&
1168 scanner->config->skip_comment_single));
1172 case G_TOKEN_IDENTIFIER:
1173 if (scanner->config->identifier_2_string)
1174 *token_p = G_TOKEN_STRING;
1177 case G_TOKEN_SYMBOL:
1178 if (scanner->config->symbol_2_token)
1179 *token_p = (GTokenType) value_p->v_symbol;
1182 case G_TOKEN_BINARY:
1185 if (scanner->config->numbers_2_int)
1186 *token_p = G_TOKEN_INT;
1193 if (*token_p == G_TOKEN_INT &&
1194 scanner->config->int_2_float)
1196 *token_p = G_TOKEN_FLOAT;
1197 if (scanner->config->store_int64)
1198 value_p->v_float = value_p->v_int64;
1200 value_p->v_float = value_p->v_int;
1207 g_scanner_get_token_ll (GScanner *scanner,
1208 GTokenType *token_p,
1209 GTokenValue *value_p,
1213 GScannerConfig *config;
1215 gboolean in_comment_multi;
1216 gboolean in_comment_single;
1217 gboolean in_string_sq;
1218 gboolean in_string_dq;
1223 config = scanner->config;
1224 (*value_p).v_int64 = 0;
1226 if ((scanner->text >= scanner->text_end && scanner->input_fd < 0) ||
1227 scanner->token == G_TOKEN_EOF)
1229 *token_p = G_TOKEN_EOF;
1233 in_comment_multi = FALSE;
1234 in_comment_single = FALSE;
1235 in_string_sq = FALSE;
1236 in_string_dq = FALSE;
1239 do /* while (ch != 0) */
1241 gboolean dotted_float = FALSE;
1243 ch = g_scanner_get_char (scanner, line_p, position_p);
1246 token = G_TOKEN_NONE;
1248 /* this is *evil*, but needed ;(
1249 * we first check for identifier first character, because it
1250 * might interfere with other key chars like slashes or numbers
1252 if (config->scan_identifier &&
1253 ch && strchr (config->cset_identifier_first, ch))
1254 goto identifier_precedence;
1259 token = G_TOKEN_EOF;
1265 if (!config->scan_comment_multi ||
1266 g_scanner_peek_next_char (scanner) != '*')
1268 g_scanner_get_char (scanner, line_p, position_p);
1269 token = G_TOKEN_COMMENT_MULTI;
1270 in_comment_multi = TRUE;
1271 gstring = g_string_new ("");
1272 while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1274 if (ch == '*' && g_scanner_peek_next_char (scanner) == '/')
1276 g_scanner_get_char (scanner, line_p, position_p);
1277 in_comment_multi = FALSE;
1281 gstring = g_string_append_c (gstring, ch);
1287 if (!config->scan_string_sq)
1289 token = G_TOKEN_STRING;
1290 in_string_sq = TRUE;
1291 gstring = g_string_new ("");
1292 while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1296 in_string_sq = FALSE;
1300 gstring = g_string_append_c (gstring, ch);
1306 if (!config->scan_string_dq)
1308 token = G_TOKEN_STRING;
1309 in_string_dq = TRUE;
1310 gstring = g_string_new ("");
1311 while ((ch = g_scanner_get_char (scanner, line_p, position_p)) != 0)
1315 in_string_dq = FALSE;
1322 ch = g_scanner_get_char (scanner, line_p, position_p);
1332 gstring = g_string_append_c (gstring, '\\');
1336 gstring = g_string_append_c (gstring, '\n');
1340 gstring = g_string_append_c (gstring, '\t');
1344 gstring = g_string_append_c (gstring, '\r');
1348 gstring = g_string_append_c (gstring, '\b');
1352 gstring = g_string_append_c (gstring, '\f');
1364 fchar = g_scanner_peek_next_char (scanner);
1365 if (fchar >= '0' && fchar <= '7')
1367 ch = g_scanner_get_char (scanner, line_p, position_p);
1368 i = i * 8 + ch - '0';
1369 fchar = g_scanner_peek_next_char (scanner);
1370 if (fchar >= '0' && fchar <= '7')
1372 ch = g_scanner_get_char (scanner, line_p, position_p);
1373 i = i * 8 + ch - '0';
1376 gstring = g_string_append_c (gstring, i);
1380 gstring = g_string_append_c (gstring, ch);
1385 gstring = g_string_append_c (gstring, ch);
1392 if (!config->scan_float)
1394 token = G_TOKEN_FLOAT;
1395 dotted_float = TRUE;
1396 ch = g_scanner_get_char (scanner, line_p, position_p);
1397 goto number_parsing;
1400 if (!config->scan_hex_dollar)
1402 token = G_TOKEN_HEX;
1403 ch = g_scanner_get_char (scanner, line_p, position_p);
1404 goto number_parsing;
1407 if (config->scan_octal)
1408 token = G_TOKEN_OCTAL;
1410 token = G_TOKEN_INT;
1411 ch = g_scanner_peek_next_char (scanner);
1412 if (config->scan_hex && (ch == 'x' || ch == 'X'))
1414 token = G_TOKEN_HEX;
1415 g_scanner_get_char (scanner, line_p, position_p);
1416 ch = g_scanner_get_char (scanner, line_p, position_p);
1419 token = G_TOKEN_ERROR;
1420 value.v_error = G_ERR_UNEXP_EOF;
1424 if (g_scanner_char_2_num (ch, 16) < 0)
1426 token = G_TOKEN_ERROR;
1427 value.v_error = G_ERR_DIGIT_RADIX;
1432 else if (config->scan_binary && (ch == 'b' || ch == 'B'))
1434 token = G_TOKEN_BINARY;
1435 g_scanner_get_char (scanner, line_p, position_p);
1436 ch = g_scanner_get_char (scanner, line_p, position_p);
1439 token = G_TOKEN_ERROR;
1440 value.v_error = G_ERR_UNEXP_EOF;
1444 if (g_scanner_char_2_num (ch, 10) < 0)
1446 token = G_TOKEN_ERROR;
1447 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1466 gboolean in_number = TRUE;
1469 if (token == G_TOKEN_NONE)
1470 token = G_TOKEN_INT;
1472 gstring = g_string_new (dotted_float ? "0." : "");
1473 gstring = g_string_append_c (gstring, ch);
1475 do /* while (in_number) */
1479 is_E = token == G_TOKEN_FLOAT && (ch == 'e' || ch == 'E');
1481 ch = g_scanner_peek_next_char (scanner);
1483 if (g_scanner_char_2_num (ch, 36) >= 0 ||
1484 (config->scan_float && ch == '.') ||
1485 (is_E && (ch == '+' || ch == '-')))
1487 ch = g_scanner_get_char (scanner, line_p, position_p);
1492 if (token != G_TOKEN_INT && token != G_TOKEN_OCTAL)
1494 value.v_error = token == G_TOKEN_FLOAT ? G_ERR_FLOAT_MALFORMED : G_ERR_FLOAT_RADIX;
1495 token = G_TOKEN_ERROR;
1500 token = G_TOKEN_FLOAT;
1501 gstring = g_string_append_c (gstring, ch);
1515 gstring = g_string_append_c (gstring, ch);
1520 if (token != G_TOKEN_FLOAT)
1522 token = G_TOKEN_ERROR;
1523 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1527 gstring = g_string_append_c (gstring, ch);
1532 if ((token != G_TOKEN_HEX && !config->scan_float) ||
1533 (token != G_TOKEN_HEX &&
1534 token != G_TOKEN_OCTAL &&
1535 token != G_TOKEN_FLOAT &&
1536 token != G_TOKEN_INT))
1538 token = G_TOKEN_ERROR;
1539 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1544 if (token != G_TOKEN_HEX)
1545 token = G_TOKEN_FLOAT;
1546 gstring = g_string_append_c (gstring, ch);
1551 if (token != G_TOKEN_HEX)
1553 token = G_TOKEN_ERROR;
1554 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1558 gstring = g_string_append_c (gstring, ch);
1568 if (token == G_TOKEN_FLOAT)
1569 value.v_float = g_strtod (gstring->str, &endptr);
1575 case G_TOKEN_BINARY:
1576 ui64 = g_ascii_strtoull (gstring->str, &endptr, 2);
1579 ui64 = g_ascii_strtoull (gstring->str, &endptr, 8);
1582 ui64 = g_ascii_strtoull (gstring->str, &endptr, 10);
1585 ui64 = g_ascii_strtoull (gstring->str, &endptr, 16);
1589 if (scanner->config->store_int64)
1590 value.v_int64 = ui64;
1594 if (endptr && *endptr)
1596 token = G_TOKEN_ERROR;
1597 if (*endptr == 'e' || *endptr == 'E')
1598 value.v_error = G_ERR_NON_DIGIT_IN_CONST;
1600 value.v_error = G_ERR_DIGIT_RADIX;
1602 g_string_free (gstring, TRUE);
1605 } /* number_parsing:... */
1611 if (config->cpair_comment_single &&
1612 ch == config->cpair_comment_single[0])
1614 token = G_TOKEN_COMMENT_SINGLE;
1615 in_comment_single = TRUE;
1616 gstring = g_string_new ("");
1617 ch = g_scanner_get_char (scanner, line_p, position_p);
1620 if (ch == config->cpair_comment_single[1])
1622 in_comment_single = FALSE;
1627 gstring = g_string_append_c (gstring, ch);
1628 ch = g_scanner_get_char (scanner, line_p, position_p);
1631 else if (config->scan_identifier && ch &&
1632 strchr (config->cset_identifier_first, ch))
1634 identifier_precedence:
1636 if (config->cset_identifier_nth && ch &&
1637 strchr (config->cset_identifier_nth,
1638 g_scanner_peek_next_char (scanner)))
1640 token = G_TOKEN_IDENTIFIER;
1641 gstring = g_string_new ("");
1642 gstring = g_string_append_c (gstring, ch);
1645 ch = g_scanner_get_char (scanner, line_p, position_p);
1646 gstring = g_string_append_c (gstring, ch);
1647 ch = g_scanner_peek_next_char (scanner);
1649 while (ch && strchr (config->cset_identifier_nth, ch));
1652 else if (config->scan_identifier_1char)
1654 token = G_TOKEN_IDENTIFIER;
1655 value.v_identifier = g_new0 (gchar, 2);
1656 value.v_identifier[0] = ch;
1662 if (config->char_2_token)
1666 token = G_TOKEN_CHAR;
1671 } /* default_case:... */
1674 g_assert (ch == 0 && token != G_TOKEN_NONE); /* paranoid */
1678 if (in_comment_multi || in_comment_single ||
1679 in_string_sq || in_string_dq)
1681 token = G_TOKEN_ERROR;
1684 g_string_free (gstring, TRUE);
1688 if (in_comment_multi || in_comment_single)
1689 value.v_error = G_ERR_UNEXP_EOF_IN_COMMENT;
1690 else /* (in_string_sq || in_string_dq) */
1691 value.v_error = G_ERR_UNEXP_EOF_IN_STRING;
1696 value.v_string = gstring->str;
1697 g_string_free (gstring, FALSE);
1701 if (token == G_TOKEN_IDENTIFIER)
1703 if (config->scan_symbols)
1708 scope_id = scanner->scope_id;
1709 key = g_scanner_lookup_internal (scanner, scope_id, value.v_identifier);
1710 if (!key && scope_id && scanner->config->scope_0_fallback)
1711 key = g_scanner_lookup_internal (scanner, 0, value.v_identifier);
1715 g_free (value.v_identifier);
1716 token = G_TOKEN_SYMBOL;
1717 value.v_symbol = key->value;
1721 if (token == G_TOKEN_IDENTIFIER &&
1722 config->scan_identifier_NULL &&
1723 strlen (value.v_identifier) == 4)
1725 gchar *null_upper = "NULL";
1726 gchar *null_lower = "null";
1728 if (scanner->config->case_sensitive)
1730 if (value.v_identifier[0] == null_upper[0] &&
1731 value.v_identifier[1] == null_upper[1] &&
1732 value.v_identifier[2] == null_upper[2] &&
1733 value.v_identifier[3] == null_upper[3])
1734 token = G_TOKEN_IDENTIFIER_NULL;
1738 if ((value.v_identifier[0] == null_upper[0] ||
1739 value.v_identifier[0] == null_lower[0]) &&
1740 (value.v_identifier[1] == null_upper[1] ||
1741 value.v_identifier[1] == null_lower[1]) &&
1742 (value.v_identifier[2] == null_upper[2] ||
1743 value.v_identifier[2] == null_lower[2]) &&
1744 (value.v_identifier[3] == null_upper[3] ||
1745 value.v_identifier[3] == null_lower[3]))
1746 token = G_TOKEN_IDENTIFIER_NULL;