1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
7 Single-pass line tokenization by Neil Booth, April 2000
9 This program is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 2, or (at your option) any
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
23 /* This lexer works with a single pass of the file. Recently I
24 re-wrote it to minimize the places where we step backwards in the
25 input stream, to make future changes to support multi-byte
26 character sets fairly straight-forward.
28 There is now only one routine where we do step backwards:
29 skip_escaped_newlines. This routine could probably also be changed
30 so that it doesn't need to step back. One possibility is to use a
31 trick similar to that used in lex_period and lex_percent. Two
32 extra characters might be needed, but skip_escaped_newlines itself
33 would probably be the only place that needs to be aware of that,
34 and changes to the remaining routines would probably only be needed
35 if they process a backslash. */
43 /* Tokens with SPELL_STRING store their spelling in the token list,
44 and it's length in the token->val.name.len. */
56 enum spell_type category;
57 const unsigned char *name;
60 const unsigned char *digraph_spellings [] = {U"%:", U"%:%:", U"<:",
63 #define OP(e, s) { SPELL_OPERATOR, U s },
64 #define TK(e, s) { s, U STRINGX (e) },
65 const struct token_spelling token_spellings [N_TTYPES] = {TTYPE_TABLE };
69 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
70 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
72 static cppchar_t handle_newline PARAMS ((cpp_buffer *, cppchar_t));
73 static cppchar_t skip_escaped_newlines PARAMS ((cpp_buffer *, cppchar_t));
74 static cppchar_t get_effective_char PARAMS ((cpp_buffer *));
76 static int skip_block_comment PARAMS ((cpp_reader *));
77 static int skip_line_comment PARAMS ((cpp_reader *));
78 static void adjust_column PARAMS ((cpp_reader *));
79 static void skip_whitespace PARAMS ((cpp_reader *, cppchar_t));
80 static cpp_hashnode *parse_identifier PARAMS ((cpp_reader *, cppchar_t));
81 static void parse_number PARAMS ((cpp_reader *, cpp_string *, cppchar_t, int));
82 static int unescaped_terminator_p PARAMS ((cpp_reader *, const U_CHAR *));
83 static void parse_string PARAMS ((cpp_reader *, cpp_token *, cppchar_t));
84 static void unterminated PARAMS ((cpp_reader *, int));
85 static int trigraph_ok PARAMS ((cpp_reader *, cppchar_t));
86 static void save_comment PARAMS ((cpp_reader *, cpp_token *, const U_CHAR *));
87 static void lex_percent PARAMS ((cpp_buffer *, cpp_token *));
88 static void lex_dot PARAMS ((cpp_reader *, cpp_token *));
89 static int name_p PARAMS ((cpp_reader *, const cpp_string *));
91 static cpp_chunk *new_chunk PARAMS ((unsigned int));
92 static int chunk_suitable PARAMS ((cpp_pool *, cpp_chunk *, unsigned int));
96 Compares, the token TOKEN to the NUL-terminated string STRING.
97 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
100 cpp_ideq (token, string)
101 const cpp_token *token;
104 if (token->type != CPP_NAME)
107 return !ustrcmp (token->val.node->name, (const U_CHAR *) string);
110 /* Call when meeting a newline. Returns the character after the newline
111 (or carriage-return newline combination), or EOF. */
113 handle_newline (buffer, newline_char)
115 cppchar_t newline_char;
117 cppchar_t next = EOF;
119 buffer->col_adjust = 0;
121 buffer->line_base = buffer->cur;
123 /* Handle CR-LF and LF-CR combinations, get the next character. */
124 if (buffer->cur < buffer->rlimit)
126 next = *buffer->cur++;
127 if (next + newline_char == '\r' + '\n')
129 buffer->line_base = buffer->cur;
130 if (buffer->cur < buffer->rlimit)
131 next = *buffer->cur++;
137 buffer->read_ahead = next;
141 /* Subroutine of skip_escaped_newlines; called when a trigraph is
142 encountered. It warns if necessary, and returns true if the
143 trigraph should be honoured. FROM_CHAR is the third character of a
144 trigraph, and presumed to be the previous character for position
147 trigraph_ok (pfile, from_char)
151 int accept = CPP_OPTION (pfile, trigraphs);
153 /* Don't warn about trigraphs in comments. */
154 if (CPP_OPTION (pfile, warn_trigraphs) && !pfile->state.lexing_comment)
156 cpp_buffer *buffer = pfile->buffer;
158 cpp_warning_with_line (pfile, buffer->lineno, CPP_BUF_COL (buffer) - 2,
159 "trigraph ??%c converted to %c",
161 (int) _cpp_trigraph_map[from_char]);
163 cpp_warning_with_line (pfile, buffer->lineno, CPP_BUF_COL (buffer) - 2,
164 "trigraph ??%c ignored", (int) from_char);
170 /* Assumes local variables buffer and result. */
171 #define ACCEPT_CHAR(t) \
172 do { result->type = t; buffer->read_ahead = EOF; } while (0)
174 /* When we move to multibyte character sets, add to these something
175 that saves and restores the state of the multibyte conversion
176 library. This probably involves saving and restoring a "cookie".
177 In the case of glibc it is an 8-byte structure, so is not a high
178 overhead operation. In any case, it's out of the fast path. */
179 #define SAVE_STATE() do { saved_cur = buffer->cur; } while (0)
180 #define RESTORE_STATE() do { buffer->cur = saved_cur; } while (0)
182 /* Skips any escaped newlines introduced by NEXT, which is either a
183 '?' or a '\\'. Returns the next character, which will also have
184 been placed in buffer->read_ahead. This routine performs
185 preprocessing stages 1 and 2 of the ISO C standard. */
187 skip_escaped_newlines (buffer, next)
191 /* Only do this if we apply stages 1 and 2. */
192 if (!buffer->from_stage3)
195 const unsigned char *saved_cur;
200 if (buffer->cur == buffer->rlimit)
206 next1 = *buffer->cur++;
207 if (next1 != '?' || buffer->cur == buffer->rlimit)
213 next1 = *buffer->cur++;
214 if (!_cpp_trigraph_map[next1]
215 || !trigraph_ok (buffer->pfile, next1))
221 /* We have a full trigraph here. */
222 next = _cpp_trigraph_map[next1];
223 if (next != '\\' || buffer->cur == buffer->rlimit)
228 /* We have a backslash, and room for at least one more character. */
232 next1 = *buffer->cur++;
233 if (!is_nvspace (next1))
237 while (buffer->cur < buffer->rlimit);
239 if (!is_vspace (next1))
246 cpp_warning (buffer->pfile,
247 "backslash and newline separated by space");
249 next = handle_newline (buffer, next1);
251 cpp_pedwarn (buffer->pfile, "backslash-newline at end of file");
253 while (next == '\\' || next == '?');
256 buffer->read_ahead = next;
260 /* Obtain the next character, after trigraph conversion and skipping
261 an arbitrary string of escaped newlines. The common case of no
262 trigraphs or escaped newlines falls through quickly. */
264 get_effective_char (buffer)
267 cppchar_t next = EOF;
269 if (buffer->cur < buffer->rlimit)
271 next = *buffer->cur++;
273 /* '?' can introduce trigraphs (and therefore backslash); '\\'
274 can introduce escaped newlines, which we want to skip, or
275 UCNs, which, depending upon lexer state, we will handle in
277 if (next == '?' || next == '\\')
278 next = skip_escaped_newlines (buffer, next);
281 buffer->read_ahead = next;
285 /* Skip a C-style block comment. We find the end of the comment by
286 seeing if an asterisk is before every '/' we encounter. Returns
287 non-zero if comment terminated by EOF, zero otherwise. */
289 skip_block_comment (pfile)
292 cpp_buffer *buffer = pfile->buffer;
293 cppchar_t c = EOF, prevc = EOF;
295 pfile->state.lexing_comment = 1;
296 while (buffer->cur != buffer->rlimit)
298 prevc = c, c = *buffer->cur++;
301 /* FIXME: For speed, create a new character class of characters
302 of interest inside block comments. */
303 if (c == '?' || c == '\\')
304 c = skip_escaped_newlines (buffer, c);
306 /* People like decorating comments with '*', so check for '/'
307 instead for efficiency. */
313 /* Warn about potential nested comments, but not if the '/'
314 comes immediately before the true comment delimeter.
315 Don't bother to get it right across escaped newlines. */
316 if (CPP_OPTION (pfile, warn_comments)
317 && buffer->cur != buffer->rlimit)
319 prevc = c, c = *buffer->cur++;
320 if (c == '*' && buffer->cur != buffer->rlimit)
322 prevc = c, c = *buffer->cur++;
324 cpp_warning_with_line (pfile, CPP_BUF_LINE (buffer),
325 CPP_BUF_COL (buffer),
326 "\"/*\" within comment");
331 else if (is_vspace (c))
333 prevc = c, c = handle_newline (buffer, c);
337 adjust_column (pfile);
340 pfile->state.lexing_comment = 0;
341 buffer->read_ahead = EOF;
342 return c != '/' || prevc != '*';
345 /* Skip a C++ line comment. Handles escaped newlines. Returns
346 non-zero if a multiline comment. The following new line, if any,
347 is left in buffer->read_ahead. */
349 skip_line_comment (pfile)
352 cpp_buffer *buffer = pfile->buffer;
353 unsigned int orig_lineno = buffer->lineno;
356 pfile->state.lexing_comment = 1;
360 if (buffer->cur == buffer->rlimit)
364 if (c == '?' || c == '\\')
365 c = skip_escaped_newlines (buffer, c);
367 while (!is_vspace (c));
369 pfile->state.lexing_comment = 0;
370 buffer->read_ahead = c; /* Leave any newline for caller. */
371 return orig_lineno != buffer->lineno;
374 /* pfile->buffer->cur is one beyond the \t character. Update
375 col_adjust so we track the column correctly. */
377 adjust_column (pfile)
380 cpp_buffer *buffer = pfile->buffer;
381 unsigned int col = CPP_BUF_COL (buffer) - 1; /* Zero-based column. */
383 /* Round it up to multiple of the tabstop, but subtract 1 since the
384 tab itself occupies a character position. */
385 buffer->col_adjust += (CPP_OPTION (pfile, tabstop)
386 - col % CPP_OPTION (pfile, tabstop)) - 1;
389 /* Skips whitespace, saving the next non-whitespace character.
390 Adjusts pfile->col_adjust to account for tabs. Without this,
391 tokens might be assigned an incorrect column. */
393 skip_whitespace (pfile, c)
397 cpp_buffer *buffer = pfile->buffer;
398 unsigned int warned = 0;
402 /* Horizontal space always OK. */
406 adjust_column (pfile);
407 /* Just \f \v or \0 left. */
412 cpp_warning (pfile, "null character(s) ignored");
416 else if (pfile->state.in_directive && CPP_PEDANTIC (pfile))
417 cpp_pedwarn_with_line (pfile, CPP_BUF_LINE (buffer),
418 CPP_BUF_COL (buffer),
419 "%s in preprocessing directive",
420 c == '\f' ? "form feed" : "vertical tab");
423 if (buffer->cur == buffer->rlimit)
427 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
428 while (is_nvspace (c));
430 /* Remember the next character. */
431 buffer->read_ahead = c;
434 /* See if the characters of a number token are valid in a name (no
437 name_p (pfile, string)
439 const cpp_string *string;
443 for (i = 0; i < string->len; i++)
444 if (!is_idchar (string->text[i]))
450 /* Parse an identifier, skipping embedded backslash-newlines.
451 Calculate the hash value of the token while parsing, for improved
452 performance. The hashing algorithm *must* match cpp_lookup(). */
454 static cpp_hashnode *
455 parse_identifier (pfile, c)
459 cpp_hashnode *result;
460 cpp_buffer *buffer = pfile->buffer;
461 unsigned char *dest, *limit;
462 unsigned int r = 0, saw_dollar = 0;
464 dest = POOL_FRONT (&pfile->ident_pool);
465 limit = POOL_LIMIT (&pfile->ident_pool);
471 /* Need room for terminating null. */
472 if (dest + 1 >= limit)
473 limit = _cpp_next_chunk (&pfile->ident_pool, 0, &dest);
482 if (buffer->cur == buffer->rlimit)
487 while (is_idchar (c));
489 /* Potential escaped newline? */
490 if (c != '?' && c != '\\')
492 c = skip_escaped_newlines (buffer, c);
494 while (is_idchar (c));
496 /* Remember the next character. */
497 buffer->read_ahead = c;
499 /* $ is not a identifier character in the standard, but is commonly
500 accepted as an extension. Don't warn about it in skipped
501 conditional blocks. */
502 if (saw_dollar && CPP_PEDANTIC (pfile) && ! pfile->skipping)
503 cpp_pedwarn (pfile, "'$' character(s) in identifier");
505 /* Identifiers are null-terminated. */
508 /* This routine commits the memory if necessary. */
509 result = _cpp_lookup_with_hash (pfile,
510 dest - POOL_FRONT (&pfile->ident_pool), r);
512 /* Some identifiers require diagnostics when lexed. */
513 if (result->flags & NODE_DIAGNOSTIC && !pfile->skipping)
515 /* It is allowed to poison the same identifier twice. */
516 if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
517 cpp_error (pfile, "attempt to use poisoned \"%s\"", result->name);
519 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
520 replacement list of a variable-arguments macro. */
521 if (result == pfile->spec_nodes.n__VA_ARGS__
522 && !pfile->state.va_args_ok)
523 cpp_pedwarn (pfile, "__VA_ARGS__ can only appear in the expansion of a C99 variable-argument macro");
529 /* Parse a number, skipping embedded backslash-newlines. */
531 parse_number (pfile, number, c, leading_period)
537 cpp_buffer *buffer = pfile->buffer;
538 cpp_pool *pool = pfile->string_pool;
539 unsigned char *dest, *limit;
541 dest = POOL_FRONT (pool);
542 limit = POOL_LIMIT (pool);
544 /* Place a leading period. */
548 limit = _cpp_next_chunk (pool, 0, &dest);
556 /* Need room for terminating null. */
557 if (dest + 1 >= limit)
558 limit = _cpp_next_chunk (pool, 0, &dest);
562 if (buffer->cur == buffer->rlimit)
567 while (is_numchar (c) || c == '.' || VALID_SIGN (c, dest[-1]));
569 /* Potential escaped newline? */
570 if (c != '?' && c != '\\')
572 c = skip_escaped_newlines (buffer, c);
574 while (is_numchar (c) || c == '.' || VALID_SIGN (c, dest[-1]));
576 /* Remember the next character. */
577 buffer->read_ahead = c;
579 /* Null-terminate the number. */
582 number->text = POOL_FRONT (pool);
583 number->len = dest - number->text;
584 POOL_COMMIT (pool, number->len + 1);
587 /* Subroutine of parse_string. Emits error for unterminated strings. */
589 unterminated (pfile, term)
593 cpp_error (pfile, "missing terminating %c character", term);
595 if (term == '\"' && pfile->mlstring_pos.line
596 && pfile->mlstring_pos.line != pfile->lexer_pos.line)
598 cpp_error_with_line (pfile, pfile->mlstring_pos.line,
599 pfile->mlstring_pos.col,
600 "possible start of unterminated string literal");
601 pfile->mlstring_pos.line = 0;
605 /* Subroutine of parse_string. */
607 unescaped_terminator_p (pfile, dest)
609 const unsigned char *dest;
611 const unsigned char *start, *temp;
613 /* In #include-style directives, terminators are not escapeable. */
614 if (pfile->state.angled_headers)
617 start = POOL_FRONT (pfile->string_pool);
619 /* An odd number of consecutive backslashes represents an escaped
621 for (temp = dest; temp > start && temp[-1] == '\\'; temp--)
624 return ((dest - temp) & 1) == 0;
627 /* Parses a string, character constant, or angle-bracketed header file
628 name. Handles embedded trigraphs and escaped newlines.
630 Multi-line strings are allowed, but they are deprecated within
633 parse_string (pfile, token, terminator)
636 cppchar_t terminator;
638 cpp_buffer *buffer = pfile->buffer;
639 cpp_pool *pool = pfile->string_pool;
640 unsigned char *dest, *limit;
642 unsigned int nulls = 0;
644 dest = POOL_FRONT (pool);
645 limit = POOL_LIMIT (pool);
649 if (buffer->cur == buffer->rlimit)
652 unterminated (pfile, terminator);
658 /* Handle trigraphs, escaped newlines etc. */
659 if (c == '?' || c == '\\')
660 c = skip_escaped_newlines (buffer, c);
662 if (c == terminator && unescaped_terminator_p (pfile, dest))
667 else if (is_vspace (c))
669 /* In assembly language, silently terminate string and
670 character literals at end of line. This is a kludge
671 around not knowing where comments are. */
672 if (CPP_OPTION (pfile, lang_asm) && terminator != '>')
675 /* Character constants and header names may not extend over
676 multiple lines. In Standard C, neither may strings.
677 Unfortunately, we accept multiline strings as an
679 if (terminator != '"')
681 unterminated (pfile, terminator);
685 if (pfile->mlstring_pos.line == 0)
687 pfile->mlstring_pos = pfile->lexer_pos;
688 if (CPP_PEDANTIC (pfile))
689 cpp_pedwarn (pfile, "multi-line string constant");
692 handle_newline (buffer, c); /* Stores to read_ahead. */
698 cpp_warning (pfile, "null character(s) preserved in literal");
701 /* No terminating null for strings - they could contain nulls. */
703 limit = _cpp_next_chunk (pool, 0, &dest);
706 /* If we had a new line, the next character is in read_ahead. */
709 c = buffer->read_ahead;
714 /* Remember the next character. */
715 buffer->read_ahead = c;
717 token->val.str.text = POOL_FRONT (pool);
718 token->val.str.len = dest - token->val.str.text;
719 POOL_COMMIT (pool, token->val.str.len);
722 /* The stored comment includes the comment start and any terminator. */
724 save_comment (pfile, token, from)
727 const unsigned char *from;
729 unsigned char *buffer;
732 len = pfile->buffer->cur - from + 1; /* + 1 for the initial '/'. */
733 /* C++ comments probably (not definitely) have moved past a new
734 line, which we don't want to save in the comment. */
735 if (pfile->buffer->read_ahead != EOF)
737 buffer = _cpp_pool_alloc (pfile->string_pool, len);
739 token->type = CPP_COMMENT;
740 token->val.str.len = len;
741 token->val.str.text = buffer;
744 memcpy (buffer + 1, from, len - 1);
747 /* Subroutine of lex_token to handle '%'. A little tricky, since we
748 want to avoid stepping back when lexing %:%X. */
750 lex_percent (buffer, result)
756 result->type = CPP_MOD;
757 /* Parsing %:%X could leave an extra character. */
758 if (buffer->extra_char == EOF)
759 c = get_effective_char (buffer);
762 c = buffer->read_ahead = buffer->extra_char;
763 buffer->extra_char = EOF;
767 ACCEPT_CHAR (CPP_MOD_EQ);
768 else if (CPP_OPTION (buffer->pfile, digraphs))
772 result->flags |= DIGRAPH;
773 ACCEPT_CHAR (CPP_HASH);
774 if (get_effective_char (buffer) == '%')
776 buffer->extra_char = get_effective_char (buffer);
777 if (buffer->extra_char == ':')
779 buffer->extra_char = EOF;
780 ACCEPT_CHAR (CPP_PASTE);
783 /* We'll catch the extra_char when we're called back. */
784 buffer->read_ahead = '%';
789 result->flags |= DIGRAPH;
790 ACCEPT_CHAR (CPP_CLOSE_BRACE);
795 /* Subroutine of lex_token to handle '.'. This is tricky, since we
796 want to avoid stepping back when lexing '...' or '.123'. In the
797 latter case we should also set a flag for parse_number. */
799 lex_dot (pfile, result)
803 cpp_buffer *buffer = pfile->buffer;
806 /* Parsing ..X could leave an extra character. */
807 if (buffer->extra_char == EOF)
808 c = get_effective_char (buffer);
811 c = buffer->read_ahead = buffer->extra_char;
812 buffer->extra_char = EOF;
815 /* All known character sets have 0...9 contiguous. */
816 if (c >= '0' && c <= '9')
818 result->type = CPP_NUMBER;
819 parse_number (pfile, &result->val.str, c, 1);
823 result->type = CPP_DOT;
826 buffer->extra_char = get_effective_char (buffer);
827 if (buffer->extra_char == '.')
829 buffer->extra_char = EOF;
830 ACCEPT_CHAR (CPP_ELLIPSIS);
833 /* We'll catch the extra_char when we're called back. */
834 buffer->read_ahead = '.';
836 else if (c == '*' && CPP_OPTION (pfile, cplusplus))
837 ACCEPT_CHAR (CPP_DOT_STAR);
842 _cpp_lex_token (pfile, result)
847 cpp_buffer *buffer = pfile->buffer;
848 const unsigned char *comment_start;
849 unsigned char was_skip_newlines = pfile->state.skip_newlines;
850 unsigned char newline_in_args = 0;
852 pfile->state.skip_newlines = 0;
855 pfile->lexer_pos.line = buffer->lineno;
857 pfile->lexer_pos.col = CPP_BUF_COLUMN (buffer, buffer->cur);
859 c = buffer->read_ahead;
860 if (c == EOF && buffer->cur < buffer->rlimit)
863 pfile->lexer_pos.col++;
867 buffer->read_ahead = EOF;
871 /* Non-empty files should end in a newline. Ignore for command
872 line and _Pragma buffers. */
873 if (pfile->lexer_pos.col != 0 && !buffer->from_stage3)
874 cpp_pedwarn (pfile, "no newline at end of file");
875 pfile->state.skip_newlines = 1;
876 result->type = CPP_EOF;
879 case ' ': case '\t': case '\f': case '\v': case '\0':
880 skip_whitespace (pfile, c);
881 result->flags |= PREV_WHITE;
884 case '\n': case '\r':
885 /* Don't let directives spill over to the next line. */
886 if (pfile->state.in_directive)
887 buffer->read_ahead = c;
890 handle_newline (buffer, c);
892 pfile->lexer_pos.output_line = buffer->lineno;
894 /* Skip newlines in macro arguments (except in directives). */
895 if (pfile->state.parsing_args)
897 /* Set the whitespace flag. */
899 result->flags |= PREV_WHITE;
903 if (was_skip_newlines)
905 /* Clear any whitespace flag. */
906 result->flags &= ~PREV_WHITE;
911 /* Next we're at BOL, so skip new lines. */
912 pfile->state.skip_newlines = 1;
913 result->type = CPP_EOF;
918 /* These could start an escaped newline, or '?' a trigraph. Let
919 skip_escaped_newlines do all the work. */
921 unsigned int lineno = buffer->lineno;
923 c = skip_escaped_newlines (buffer, c);
924 if (lineno != buffer->lineno)
925 /* We had at least one escaped newline of some sort, and the
926 next character is in buffer->read_ahead. Update the
927 token's line and column. */
930 /* We are either the original '?' or '\\', or a trigraph. */
931 result->type = CPP_QUERY;
932 buffer->read_ahead = EOF;
940 case '0': case '1': case '2': case '3': case '4':
941 case '5': case '6': case '7': case '8': case '9':
942 result->type = CPP_NUMBER;
943 parse_number (pfile, &result->val.str, c, 0);
947 if (!CPP_OPTION (pfile, dollars_in_ident))
949 /* Fall through... */
952 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
953 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
954 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
955 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
957 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
958 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
959 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
960 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
962 result->type = CPP_NAME;
963 result->val.node = parse_identifier (pfile, c);
965 /* 'L' may introduce wide characters or strings. */
966 if (result->val.node == pfile->spec_nodes.n_L)
968 c = buffer->read_ahead; /* For make_string. */
969 if (c == '\'' || c == '"')
971 ACCEPT_CHAR (c == '"' ? CPP_WSTRING: CPP_WCHAR);
975 /* Convert named operators to their proper types. */
976 else if (result->val.node->flags & NODE_OPERATOR)
978 result->flags |= NAMED_OP;
979 result->type = result->val.node->value.operator;
985 result->type = c == '"' ? CPP_STRING: CPP_CHAR;
987 parse_string (pfile, result, c);
991 /* A potential block or line comment. */
992 comment_start = buffer->cur;
993 result->type = CPP_DIV;
994 c = get_effective_char (buffer);
996 ACCEPT_CHAR (CPP_DIV_EQ);
997 if (c != '/' && c != '*')
1002 if (skip_block_comment (pfile))
1003 cpp_error_with_line (pfile, pfile->lexer_pos.line,
1004 pfile->lexer_pos.col,
1005 "unterminated comment");
1009 if (!CPP_OPTION (pfile, cplusplus_comments)
1010 && !CPP_IN_SYSTEM_HEADER (pfile))
1013 /* We silently allow C++ comments in system headers,
1014 irrespective of conformance mode, because lots of
1015 broken systems do that and trying to clean it up in
1016 fixincludes is a nightmare. */
1017 if (CPP_OPTION (pfile, c89) && CPP_PEDANTIC (pfile)
1018 && ! buffer->warned_cplusplus_comments)
1021 "C++ style comments are not allowed in ISO C89");
1023 "(this will be reported only once per input file)");
1024 buffer->warned_cplusplus_comments = 1;
1027 /* Skip_line_comment updates buffer->read_ahead. */
1028 if (skip_line_comment (pfile))
1029 cpp_warning_with_line (pfile, pfile->lexer_pos.line,
1030 pfile->lexer_pos.col,
1031 "multi-line comment");
1034 /* Skipping the comment has updated buffer->read_ahead. */
1035 if (!pfile->state.save_comments)
1037 result->flags |= PREV_WHITE;
1041 /* Save the comment as a token in its own right. */
1042 save_comment (pfile, result, comment_start);
1046 if (pfile->state.angled_headers)
1048 result->type = CPP_HEADER_NAME;
1049 c = '>'; /* terminator. */
1053 result->type = CPP_LESS;
1054 c = get_effective_char (buffer);
1056 ACCEPT_CHAR (CPP_LESS_EQ);
1059 ACCEPT_CHAR (CPP_LSHIFT);
1060 if (get_effective_char (buffer) == '=')
1061 ACCEPT_CHAR (CPP_LSHIFT_EQ);
1063 else if (c == '?' && CPP_OPTION (pfile, cplusplus))
1065 ACCEPT_CHAR (CPP_MIN);
1066 if (get_effective_char (buffer) == '=')
1067 ACCEPT_CHAR (CPP_MIN_EQ);
1069 else if (c == ':' && CPP_OPTION (pfile, digraphs))
1071 ACCEPT_CHAR (CPP_OPEN_SQUARE);
1072 result->flags |= DIGRAPH;
1074 else if (c == '%' && CPP_OPTION (pfile, digraphs))
1076 ACCEPT_CHAR (CPP_OPEN_BRACE);
1077 result->flags |= DIGRAPH;
1082 result->type = CPP_GREATER;
1083 c = get_effective_char (buffer);
1085 ACCEPT_CHAR (CPP_GREATER_EQ);
1088 ACCEPT_CHAR (CPP_RSHIFT);
1089 if (get_effective_char (buffer) == '=')
1090 ACCEPT_CHAR (CPP_RSHIFT_EQ);
1092 else if (c == '?' && CPP_OPTION (pfile, cplusplus))
1094 ACCEPT_CHAR (CPP_MAX);
1095 if (get_effective_char (buffer) == '=')
1096 ACCEPT_CHAR (CPP_MAX_EQ);
1101 lex_percent (buffer, result);
1102 if (result->type == CPP_HASH)
1107 lex_dot (pfile, result);
1111 result->type = CPP_PLUS;
1112 c = get_effective_char (buffer);
1114 ACCEPT_CHAR (CPP_PLUS_EQ);
1116 ACCEPT_CHAR (CPP_PLUS_PLUS);
1120 result->type = CPP_MINUS;
1121 c = get_effective_char (buffer);
1124 ACCEPT_CHAR (CPP_DEREF);
1125 if (CPP_OPTION (pfile, cplusplus)
1126 && get_effective_char (buffer) == '*')
1127 ACCEPT_CHAR (CPP_DEREF_STAR);
1130 ACCEPT_CHAR (CPP_MINUS_EQ);
1132 ACCEPT_CHAR (CPP_MINUS_MINUS);
1136 result->type = CPP_MULT;
1137 if (get_effective_char (buffer) == '=')
1138 ACCEPT_CHAR (CPP_MULT_EQ);
1142 result->type = CPP_EQ;
1143 if (get_effective_char (buffer) == '=')
1144 ACCEPT_CHAR (CPP_EQ_EQ);
1148 result->type = CPP_NOT;
1149 if (get_effective_char (buffer) == '=')
1150 ACCEPT_CHAR (CPP_NOT_EQ);
1154 result->type = CPP_AND;
1155 c = get_effective_char (buffer);
1157 ACCEPT_CHAR (CPP_AND_EQ);
1159 ACCEPT_CHAR (CPP_AND_AND);
1163 if (get_effective_char (buffer) == '#')
1164 ACCEPT_CHAR (CPP_PASTE);
1167 result->type = CPP_HASH;
1169 /* CPP_DHASH is the hash introducing a directive. */
1170 if (was_skip_newlines || newline_in_args)
1172 result->type = CPP_DHASH;
1173 /* Get whitespace right - newline_in_args sets it. */
1174 if (pfile->lexer_pos.col == 1)
1175 result->flags &= ~PREV_WHITE;
1181 result->type = CPP_OR;
1182 c = get_effective_char (buffer);
1184 ACCEPT_CHAR (CPP_OR_EQ);
1186 ACCEPT_CHAR (CPP_OR_OR);
1190 result->type = CPP_XOR;
1191 if (get_effective_char (buffer) == '=')
1192 ACCEPT_CHAR (CPP_XOR_EQ);
1196 result->type = CPP_COLON;
1197 c = get_effective_char (buffer);
1198 if (c == ':' && CPP_OPTION (pfile, cplusplus))
1199 ACCEPT_CHAR (CPP_SCOPE);
1200 else if (c == '>' && CPP_OPTION (pfile, digraphs))
1202 result->flags |= DIGRAPH;
1203 ACCEPT_CHAR (CPP_CLOSE_SQUARE);
1207 case '~': result->type = CPP_COMPL; break;
1208 case ',': result->type = CPP_COMMA; break;
1209 case '(': result->type = CPP_OPEN_PAREN; break;
1210 case ')': result->type = CPP_CLOSE_PAREN; break;
1211 case '[': result->type = CPP_OPEN_SQUARE; break;
1212 case ']': result->type = CPP_CLOSE_SQUARE; break;
1213 case '{': result->type = CPP_OPEN_BRACE; break;
1214 case '}': result->type = CPP_CLOSE_BRACE; break;
1215 case ';': result->type = CPP_SEMICOLON; break;
1218 if (CPP_OPTION (pfile, objc))
1220 /* In Objective C, '@' may begin keywords or strings, like
1221 @keyword or @"string". It would be nice to call
1222 get_effective_char here and test the result. However, we
1223 would then need to pass 2 characters to parse_identifier,
1224 making it ugly and slowing down its main loop. Instead,
1225 we assume we have an identifier, and recover if not. */
1226 result->type = CPP_NAME;
1227 result->val.node = parse_identifier (pfile, c);
1228 if (result->val.node->length != 1)
1231 /* OK, so it wasn't an identifier. Maybe a string? */
1232 if (buffer->read_ahead == '"')
1235 ACCEPT_CHAR (CPP_OSTRING);
1243 result->type = CPP_OTHER;
1244 result->val.aux = c;
1249 /* An upper bound on the number of bytes needed to spell a token,
1250 including preceding whitespace. */
1252 cpp_token_len (token)
1253 const cpp_token *token;
1257 switch (TOKEN_SPELL (token))
1259 default: len = 0; break;
1260 case SPELL_STRING: len = token->val.str.len; break;
1261 case SPELL_IDENT: len = token->val.node->length; break;
1263 /* 1 for whitespace, 4 for comment delimeters. */
1267 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
1268 already contain the enough space to hold the token's spelling.
1269 Returns a pointer to the character after the last character
1272 cpp_spell_token (pfile, token, buffer)
1273 cpp_reader *pfile; /* Would be nice to be rid of this... */
1274 const cpp_token *token;
1275 unsigned char *buffer;
1277 switch (TOKEN_SPELL (token))
1279 case SPELL_OPERATOR:
1281 const unsigned char *spelling;
1284 if (token->flags & DIGRAPH)
1285 spelling = digraph_spellings[token->type - CPP_FIRST_DIGRAPH];
1286 else if (token->flags & NAMED_OP)
1289 spelling = TOKEN_NAME (token);
1291 while ((c = *spelling++) != '\0')
1298 memcpy (buffer, token->val.node->name, token->val.node->length);
1299 buffer += token->val.node->length;
1304 int left, right, tag;
1305 switch (token->type)
1307 case CPP_STRING: left = '"'; right = '"'; tag = '\0'; break;
1308 case CPP_WSTRING: left = '"'; right = '"'; tag = 'L'; break;
1309 case CPP_OSTRING: left = '"'; right = '"'; tag = '@'; break;
1310 case CPP_CHAR: left = '\''; right = '\''; tag = '\0'; break;
1311 case CPP_WCHAR: left = '\''; right = '\''; tag = 'L'; break;
1312 case CPP_HEADER_NAME: left = '<'; right = '>'; tag = '\0'; break;
1313 default: left = '\0'; right = '\0'; tag = '\0'; break;
1315 if (tag) *buffer++ = tag;
1316 if (left) *buffer++ = left;
1317 memcpy (buffer, token->val.str.text, token->val.str.len);
1318 buffer += token->val.str.len;
1319 if (right) *buffer++ = right;
1324 *buffer++ = token->val.aux;
1328 cpp_ice (pfile, "Unspellable token %s", TOKEN_NAME (token));
1335 /* Returns a token as a null-terminated string. The string is
1336 temporary, and automatically freed later. Useful for diagnostics. */
1338 cpp_token_as_text (pfile, token)
1340 const cpp_token *token;
1342 unsigned int len = cpp_token_len (token);
1343 unsigned char *start = _cpp_pool_alloc (&pfile->temp_string_pool, len), *end;
1345 end = cpp_spell_token (pfile, token, start);
1351 /* Used by C front ends. Should really move to using cpp_token_as_text. */
1353 cpp_type2name (type)
1354 enum cpp_ttype type;
1356 return (const char *) token_spellings[type].name;
1359 /* Writes the spelling of token to FP. Separate from cpp_spell_token
1360 for efficiency - to avoid double-buffering. Also, outputs a space
1361 if PREV_WHITE is flagged. */
1363 cpp_output_token (token, fp)
1364 const cpp_token *token;
1367 if (token->flags & PREV_WHITE)
1370 switch (TOKEN_SPELL (token))
1372 case SPELL_OPERATOR:
1374 const unsigned char *spelling;
1376 if (token->flags & DIGRAPH)
1377 spelling = digraph_spellings[token->type - CPP_FIRST_DIGRAPH];
1378 else if (token->flags & NAMED_OP)
1381 spelling = TOKEN_NAME (token);
1383 ufputs (spelling, fp);
1389 ufputs (token->val.node->name, fp);
1394 int left, right, tag;
1395 switch (token->type)
1397 case CPP_STRING: left = '"'; right = '"'; tag = '\0'; break;
1398 case CPP_WSTRING: left = '"'; right = '"'; tag = 'L'; break;
1399 case CPP_OSTRING: left = '"'; right = '"'; tag = '@'; break;
1400 case CPP_CHAR: left = '\''; right = '\''; tag = '\0'; break;
1401 case CPP_WCHAR: left = '\''; right = '\''; tag = 'L'; break;
1402 case CPP_HEADER_NAME: left = '<'; right = '>'; tag = '\0'; break;
1403 default: left = '\0'; right = '\0'; tag = '\0'; break;
1405 if (tag) putc (tag, fp);
1406 if (left) putc (left, fp);
1407 fwrite (token->val.str.text, 1, token->val.str.len, fp);
1408 if (right) putc (right, fp);
1413 putc (token->val.aux, fp);
1417 /* An error, most probably. */
1422 /* Compare two tokens. */
1424 _cpp_equiv_tokens (a, b)
1425 const cpp_token *a, *b;
1427 if (a->type == b->type && a->flags == b->flags)
1428 switch (TOKEN_SPELL (a))
1430 default: /* Keep compiler happy. */
1431 case SPELL_OPERATOR:
1434 return a->val.aux == b->val.aux; /* Character. */
1436 return (a->type != CPP_MACRO_ARG || a->val.aux == b->val.aux);
1438 return a->val.node == b->val.node;
1440 return (a->val.str.len == b->val.str.len
1441 && !memcmp (a->val.str.text, b->val.str.text,
1449 /* Compare two token lists. */
1451 _cpp_equiv_toklists (a, b)
1452 const struct toklist *a, *b;
1454 unsigned int i, count;
1456 count = a->limit - a->first;
1457 if (count != (b->limit - b->first))
1460 for (i = 0; i < count; i++)
1461 if (! _cpp_equiv_tokens (&a->first[i], &b->first[i]))
1468 /* Determine whether two tokens can be pasted together, and if so,
1469 what the resulting token is. Returns CPP_EOF if the tokens cannot
1470 be pasted, or the appropriate type for the merged token if they
1473 cpp_can_paste (pfile, token1, token2, digraph)
1475 const cpp_token *token1, *token2;
1478 enum cpp_ttype a = token1->type, b = token2->type;
1479 int cxx = CPP_OPTION (pfile, cplusplus);
1481 /* Treat named operators as if they were ordinary NAMEs. */
1482 if (token1->flags & NAMED_OP)
1484 if (token2->flags & NAMED_OP)
1487 if (a <= CPP_LAST_EQ && b == CPP_EQ)
1488 return a + (CPP_EQ_EQ - CPP_EQ);
1493 if (b == a) return CPP_RSHIFT;
1494 if (b == CPP_QUERY && cxx) return CPP_MAX;
1495 if (b == CPP_GREATER_EQ) return CPP_RSHIFT_EQ;
1498 if (b == a) return CPP_LSHIFT;
1499 if (b == CPP_QUERY && cxx) return CPP_MIN;
1500 if (b == CPP_LESS_EQ) return CPP_LSHIFT_EQ;
1501 if (CPP_OPTION (pfile, digraphs))
1504 {*digraph = 1; return CPP_OPEN_SQUARE;} /* <: digraph */
1506 {*digraph = 1; return CPP_OPEN_BRACE;} /* <% digraph */
1510 case CPP_PLUS: if (b == a) return CPP_PLUS_PLUS; break;
1511 case CPP_AND: if (b == a) return CPP_AND_AND; break;
1512 case CPP_OR: if (b == a) return CPP_OR_OR; break;
1515 if (b == a) return CPP_MINUS_MINUS;
1516 if (b == CPP_GREATER) return CPP_DEREF;
1519 if (b == a && cxx) return CPP_SCOPE;
1520 if (b == CPP_GREATER && CPP_OPTION (pfile, digraphs))
1521 {*digraph = 1; return CPP_CLOSE_SQUARE;} /* :> digraph */
1525 if (CPP_OPTION (pfile, digraphs))
1527 if (b == CPP_GREATER)
1528 {*digraph = 1; return CPP_CLOSE_BRACE;} /* %> digraph */
1530 {*digraph = 1; return CPP_HASH;} /* %: digraph */
1534 if (b == CPP_MULT && cxx) return CPP_DEREF_STAR;
1537 if (b == CPP_MULT && cxx) return CPP_DOT_STAR;
1538 if (b == CPP_NUMBER) return CPP_NUMBER;
1542 if (b == a && (token1->flags & DIGRAPH) == (token2->flags & DIGRAPH))
1544 {*digraph = (token1->flags & DIGRAPH); return CPP_PASTE;}
1548 if (b == CPP_NAME) return CPP_NAME;
1550 && name_p (pfile, &token2->val.str)) return CPP_NAME;
1552 && token1->val.node == pfile->spec_nodes.n_L) return CPP_WCHAR;
1554 && token1->val.node == pfile->spec_nodes.n_L) return CPP_WSTRING;
1558 if (b == CPP_NUMBER) return CPP_NUMBER;
1559 if (b == CPP_NAME) return CPP_NUMBER;
1560 if (b == CPP_DOT) return CPP_NUMBER;
1561 /* Numbers cannot have length zero, so this is safe. */
1562 if ((b == CPP_PLUS || b == CPP_MINUS)
1563 && VALID_SIGN ('+', token1->val.str.text[token1->val.str.len - 1]))
1568 if (CPP_OPTION (pfile, objc) && token1->val.aux == '@')
1570 if (b == CPP_NAME) return CPP_NAME;
1571 if (b == CPP_STRING) return CPP_OSTRING;
1581 /* Returns nonzero if a space should be inserted to avoid an
1582 accidental token paste for output. For simplicity, it is
1583 conservative, and occasionally advises a space where one is not
1584 needed, e.g. "." and ".2". */
1587 cpp_avoid_paste (pfile, token1, token2)
1589 const cpp_token *token1, *token2;
1591 enum cpp_ttype a = token1->type, b = token2->type;
1594 if (token1->flags & NAMED_OP)
1596 if (token2->flags & NAMED_OP)
1600 if (token2->flags & DIGRAPH)
1601 c = digraph_spellings[b - CPP_FIRST_DIGRAPH][0];
1602 else if (token_spellings[b].category == SPELL_OPERATOR)
1603 c = token_spellings[b].name[0];
1605 /* Quickly get everything that can paste with an '='. */
1606 if (a <= CPP_LAST_EQ && c == '=')
1611 case CPP_GREATER: return c == '>' || c == '?';
1612 case CPP_LESS: return c == '<' || c == '?' || c == '%' || c == ':';
1613 case CPP_PLUS: return c == '+';
1614 case CPP_MINUS: return c == '-' || c == '>';
1615 case CPP_DIV: return c == '/' || c == '*'; /* Comments. */
1616 case CPP_MOD: return c == ':' || c == '>';
1617 case CPP_AND: return c == '&';
1618 case CPP_OR: return c == '|';
1619 case CPP_COLON: return c == ':' || c == '>';
1620 case CPP_DEREF: return c == '*';
1621 case CPP_DOT: return c == '.' || c == '%';
1622 case CPP_HASH: return c == '#' || c == '%'; /* Digraph form. */
1623 case CPP_NAME: return ((b == CPP_NUMBER
1624 && name_p (pfile, &token2->val.str))
1626 || b == CPP_CHAR || b == CPP_STRING); /* L */
1627 case CPP_NUMBER: return (b == CPP_NUMBER || b == CPP_NAME
1628 || c == '.' || c == '+' || c == '-');
1629 case CPP_OTHER: return (CPP_OPTION (pfile, objc)
1630 && token1->val.aux == '@'
1631 && (b == CPP_NAME || b == CPP_STRING));
1638 /* Output all the remaining tokens on the current line, and a newline
1639 character, to FP. Leading whitespace is removed. */
1641 cpp_output_line (pfile, fp)
1647 _cpp_get_token (pfile, &token);
1648 token.flags &= ~PREV_WHITE;
1649 while (token.type != CPP_EOF)
1651 cpp_output_token (&token, fp);
1652 _cpp_get_token (pfile, &token);
1670 #define DEFAULT_ALIGNMENT (offsetof (struct dummy, u))
1673 chunk_suitable (pool, chunk, size)
1678 /* Being at least twice SIZE means we can use memcpy in
1679 _cpp_next_chunk rather than memmove. Besides, it's a good idea
1681 return (chunk && pool->locked != chunk
1682 && (unsigned int) (chunk->limit - chunk->base) >= size * 2);
1685 /* Returns the end of the new pool. PTR points to a char in the old
1686 pool, and is updated to point to the same char in the new pool. */
1688 _cpp_next_chunk (pool, len, ptr)
1691 unsigned char **ptr;
1693 cpp_chunk *chunk = pool->cur->next;
1695 /* LEN is the minimum size we want in the new pool. */
1696 len += POOL_ROOM (pool);
1697 if (! chunk_suitable (pool, chunk, len))
1699 chunk = new_chunk (POOL_SIZE (pool) * 2 + len);
1701 chunk->next = pool->cur->next;
1702 pool->cur->next = chunk;
1705 /* Update the pointer before changing chunk's front. */
1707 *ptr += chunk->base - POOL_FRONT (pool);
1709 memcpy (chunk->base, POOL_FRONT (pool), POOL_ROOM (pool));
1710 chunk->front = chunk->base;
1713 return POOL_LIMIT (pool);
1720 unsigned char *base;
1723 size = ALIGN (size, DEFAULT_ALIGNMENT);
1724 base = (unsigned char *) xmalloc (size + sizeof (cpp_chunk));
1725 /* Put the chunk descriptor at the end. Then chunk overruns will
1726 cause obvious chaos. */
1727 result = (cpp_chunk *) (base + size);
1728 result->base = base;
1729 result->front = base;
1730 result->limit = base + size;
1737 _cpp_init_pool (pool, size, align, temp)
1739 unsigned int size, align, temp;
1742 align = DEFAULT_ALIGNMENT;
1743 if (align & (align - 1))
1745 pool->align = align;
1746 pool->cur = new_chunk (size);
1750 pool->cur->next = pool->cur;
1754 _cpp_lock_pool (pool)
1757 if (pool->locks++ == 0)
1758 pool->locked = pool->cur;
1762 _cpp_unlock_pool (pool)
1765 if (--pool->locks == 0)
1770 _cpp_free_pool (pool)
1773 cpp_chunk *chunk = pool->cur, *next;
1781 while (chunk && chunk != pool->cur);
1784 /* Reserve LEN bytes from a memory pool. */
1786 _cpp_pool_reserve (pool, len)
1790 len = ALIGN (len, pool->align);
1791 if (len > (unsigned int) POOL_ROOM (pool))
1792 _cpp_next_chunk (pool, len, 0);
1794 return POOL_FRONT (pool);
1797 /* Allocate LEN bytes from a memory pool. */
1799 _cpp_pool_alloc (pool, len)
1803 unsigned char *result = _cpp_pool_reserve (pool, len);
1805 POOL_COMMIT (pool, len);