#ifdef NEW_LEXER
-static void expand_comment_space PARAMS ((cpp_toklist *));
void init_trigraph_map PARAMS ((void));
static unsigned char* trigraph_replace PARAMS ((cpp_reader *, unsigned char *,
unsigned char *));
static void parse_string2 PARAMS ((cpp_reader *, cpp_toklist *, cpp_name *,
unsigned int, int));
static int trigraph_ok PARAMS ((cpp_reader *, const unsigned char *));
-static void save_comment PARAMS ((cpp_toklist *, cpp_token *, unsigned char *,
+static void save_comment PARAMS ((cpp_toklist *, cpp_token *,
+ const unsigned char *,
unsigned int, unsigned int));
void _cpp_lex_line PARAMS ((cpp_reader *, cpp_toklist *));
/* An upper bound on the number of bytes needed to spell a token,
including preceding whitespace. */
-#define TOKEN_LEN(token) (5 + (token_spellings[token->type].type > \
- SPELL_NONE ? token->val.name.len: 0))
+#define TOKEN_LEN(token) (5 + (token_spellings[(token)->type].type > \
+ SPELL_NONE ? (token)->val.name.len: 0))
#endif
/*
* The tokenizer's main loop. Returns a token list, representing a
- * logical line in the input file, terminated with a CPP_VSPACE
- * token. On EOF, a token list containing the single CPP_EOF token
- * is returned.
+ * logical line in the input file. On EOF after some tokens have
+ * been processed, we return immediately. Then in next call, or if
+ * EOF occurred at the beginning of a logical line, a single CPP_EOF
+ * token is placed in the list.
*
* Implementation relies almost entirely on lookback, rather than
* looking forwards. This means that tokenization requires just
unsigned char flags = 0;
unsigned int first_token = list->tokens_used;
+ list->line = CPP_BUF_LINE (buffer);
pfile->col_adjust = 0;
expanded:
token_limit = list->tokens + list->tokens_cap;
buffer->cur = cur;
cpp_warning (pfile, "backslash and newline separated by space");
}
- PUSH_TOKEN (CPP_VSPACE);
- goto out;
+ /* Skip vertical space until we have at least one token to
+ return. */
+ if (cur_token != &list->tokens[first_token])
+ goto out;
+ list->line = CPP_BUF_LINE (buffer);
+ break;
case '-':
if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_MINUS)
goto expanded;
}
- cur_token->type = CPP_EOF;
cur_token->flags = flags;
-
- if (cur_token != &list->tokens[first_token])
+ if (cur_token == &list->tokens[first_token])
{
- /* Next call back will get just a CPP_EOF. */
- buffer->cur = cur;
- cpp_warning (pfile, "no newline at end of file");
- PUSH_TOKEN (CPP_VSPACE);
+ /* FIXME: move this warning to callers who care. */
+ if (cur > buffer->buf && !IS_NEWLINE (cur[-1]))
+ cpp_warning (pfile, "no newline at end of file");
+ cur_token++->type = CPP_EOF;
}
out:
+ list->tokens[first_token].flags |= BOL;
buffer->cur = cur;
-
list->tokens_used = cur_token - list->tokens;
-
- /* FIXME: take this check out and put it in the caller.
- list->directive == 0 indicates an unknown directive (but null
- directive is OK). This is the first time we can be sure the
- directive is invalid, and thus warn about it, because it might
- have been split by escaped newlines. Also, don't complain about
- invalid directives in assembly source, we don't know where the
- comments are, and # may introduce assembler pseudo-ops. */
-
- if (IS_DIRECTIVE (list) && list->dirno == -1
- && list->tokens[1].type != CPP_VSPACE
- && !CPP_OPTION (pfile, lang_asm))
- cpp_error_with_line (pfile, list->line, list->tokens[1].col,
- "invalid preprocessing directive");
}
/* Write the spelling of a token TOKEN to BUFFER. The buffer must
cpp_reader *pfile;
cpp_toklist *list;
{
- cpp_token *token;
+ unsigned int i;
- token = &list->tokens[0];
- do
+ for (i = 0; i < list->tokens_used; i++)
{
- CPP_RESERVE (pfile, TOKEN_LEN (token));
- pfile->limit = spell_token (pfile, token, pfile->limit, 1);
+ CPP_RESERVE (pfile, TOKEN_LEN (&list->tokens[i]));
+ pfile->limit = spell_token (pfile, &list->tokens[i], pfile->limit, 1);
}
- while (token++->type != CPP_VSPACE);
}
#endif
I(CPP_COMMENT, 0) /* Only if output comments. */ \
N(CPP_MACRO_ARG, 0) /* Macro argument. */ \
N(CPP_SUBLIST, 0) /* Sublist. */ \
- T(CPP_VSPACE, "\n") /* End of line. */ \
N(CPP_EOF, 0) /* End of file. */ \
N(CPP_HEADER_NAME, 0) /* <stdio.h> in #include */ \
\
/* Obsolete - will be removed when no code uses them still. */ \
+ T(CPP_VSPACE, "\n") /* End of line. */ \
N(CPP_HSPACE, 0) /* Horizontal white space. */ \
N(CPP_DIRECTIVE, 0) /* #define and the like */ \
N(CPP_MACRO, 0) /* Like a NAME, but expanded. */
/* Flags for the cpp_token structure. */
#define PREV_WHITESPACE 1 /* If whitespace before this token. */
-#define DIGRAPH 2 /* If it was a digraph. */
-#define UNSIGNED_INT 4 /* If int preprocessing token unsigned. */
+#define BOL 2 /* Beginning of line. */
+#define DIGRAPH 4 /* If it was a digraph. */
+#define UNSIGNED_INT 8 /* If int preprocessing token unsigned. */
/* A preprocessing token. This has been carefully packed and should
occupy 16 bytes on both 32- and 64-bit hosts. */