From: Nathan Sidwell Date: Mon, 28 Oct 2019 12:35:39 +0000 (+0000) Subject: [C++ PATCH] simplify deferred parsing lexer X-Git-Tag: upstream/12.2.0~20861 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=14c835a01ceac44e685589489b46ffaabd034177;p=platform%2Fupstream%2Fgcc.git [C++ PATCH] simplify deferred parsing lexer https://gcc.gnu.org/ml/gcc-patches/2019-10/msg01962.html We use an eof_token global variable as a sentinel on a deferred parse (such as in-class function definitions, or default args). This complicates retrieving the next token in certain places. As such deferred parses always nest properly and completely before resuming the outer lexer, we can simply morph the token after the deferred buffer into a CPP_EOF token and restore it afterwards. I finally got around to implementing it with this patch. One complication is that we have to change the discriminator for when the token's value is a tree. We can't look at the token's type because it might have been overwritten. I add a bool flag to the token (there's several spare bits), and use that. This does simplify the discriminator because we just check a single bit, rather than a set of token types. * parser.h (struct cp_token): Drop {ENUM,BOOL}_BITFIELD C-ism. Add tree_check_p flag, use as nested union discriminator. (struct cp_lexer): Add saved_type & saved_keyword fields. * parser.c (eof_token): Delete. (cp_lexer_new_main): Always init last_token to last token of buffer. (cp_lexer_new_from_tokens): Overlay EOF token at end of range. (cp_lexer_destroy): Restore token under the EOF. (cp_lexer_previous_token_position): No check for eof_token here. (cp_lexer_get_preprocessor_token): Clear tree_check_p. (cp_lexer_peek_nth_token): Check CPP_EOF not eof_token. (cp_lexer_consume_token): Assert not CPP_EOF, no check for eof_token. (cp_lexer_purge_token): Likewise. (cp_lexer_purge_tokens_after): No check for EOF token. (cp_parser_nested_name_specifier, cp_parser_decltype) (cp_parser_template_id): Set tree_check_p. From-SVN: r277514 --- diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog index 78bbfb5..44c07f1 100644 --- a/gcc/cp/ChangeLog +++ b/gcc/cp/ChangeLog @@ -1,3 +1,23 @@ +2019-10-28 Nathan Sidwell + + * parser.h (struct cp_token): Drop {ENUM,BOOL}_BITFIELD C-ism. + Add tree_check_p flag, use as nested union discriminator. + (struct cp_lexer): Add saved_type & saved_keyword fields. + * parser.c (eof_token): Delete. + (cp_lexer_new_main): Always init last_token to last token of + buffer. + (cp_lexer_new_from_tokens): Overlay EOF token at end of range. + (cp_lexer_destroy): Restore token under the EOF. + (cp_lexer_previous_token_position): No check for eof_token here. + (cp_lexer_get_preprocessor_token): Clear tree_check_p. + (cp_lexer_peek_nth_token): Check CPP_EOF not eof_token. + (cp_lexer_consume_token): Assert not CPP_EOF, no check for + eof_token. + (cp_lexer_purge_token): Likewise. + (cp_lexer_purge_tokens_after): No check for EOF token. + (cp_parser_nested_name_specifier, cp_parser_decltype) + (cp_parser_template_id): Set tree_check_p. + 2019-10-24 Jakub Jelinek * decl2.c (cplus_decl_attributes): Add "omp declare target block" diff --git a/gcc/cp/parser.c b/gcc/cp/parser.c index 3857fe4..b394e2e 100644 --- a/gcc/cp/parser.c +++ b/gcc/cp/parser.c @@ -52,11 +52,6 @@ along with GCC; see the file COPYING3. If not see /* The cp_lexer_* routines mediate between the lexer proper (in libcpp and c-lex.c) and the C++ parser. */ -static cp_token eof_token = -{ - CPP_EOF, RID_MAX, 0, false, false, false, 0, { NULL } -}; - /* The various kinds of non integral constant we encounter. */ enum non_integral_constant { NIC_NONE, @@ -660,12 +655,10 @@ cp_lexer_new_main (void) vec_safe_push (lexer->buffer, token); } - lexer->last_token = lexer->buffer->address () + lexer->next_token = lexer->buffer->address (); + lexer->last_token = lexer->next_token + lexer->buffer->length () - 1; - lexer->next_token = lexer->buffer->length () - ? lexer->buffer->address () - : &eof_token; /* Subsequent preprocessor diagnostics should use compiler diagnostic functions to get the compiler source location. */ @@ -687,7 +680,14 @@ cp_lexer_new_from_tokens (cp_token_cache *cache) /* We do not own the buffer. */ lexer->buffer = NULL; - lexer->next_token = first == last ? &eof_token : first; + + /* Insert an EOF token. */ + lexer->saved_type = last->type; + lexer->saved_keyword = last->keyword; + last->type = CPP_EOF; + last->keyword = RID_MAX; + + lexer->next_token = first; lexer->last_token = last; lexer->saved_tokens.create (CP_SAVED_TOKEN_STACK); @@ -704,7 +704,14 @@ cp_lexer_new_from_tokens (cp_token_cache *cache) static void cp_lexer_destroy (cp_lexer *lexer) { - vec_free (lexer->buffer); + if (lexer->buffer) + vec_free (lexer->buffer); + else + { + /* Restore the token we overwrite with EOF. */ + lexer->last_token->type = lexer->saved_type; + lexer->last_token->keyword = lexer->saved_keyword; + } lexer->saved_tokens.release (); ggc_free (lexer); } @@ -731,8 +738,6 @@ cp_lexer_debugging_p (cp_lexer *lexer) static inline cp_token_position cp_lexer_token_position (cp_lexer *lexer, bool previous_p) { - gcc_assert (!previous_p || lexer->next_token != &eof_token); - return lexer->next_token - previous_p; } @@ -751,10 +756,7 @@ cp_lexer_set_token_position (cp_lexer *lexer, cp_token_position pos) static inline cp_token_position cp_lexer_previous_token_position (cp_lexer *lexer) { - if (lexer->next_token == &eof_token) - return lexer->last_token - 1; - else - return cp_lexer_token_position (lexer, true); + return cp_lexer_token_position (lexer, true); } static inline cp_token * @@ -807,6 +809,7 @@ cp_lexer_get_preprocessor_token (cp_lexer *lexer, cp_token *token) token->keyword = RID_MAX; token->purged_p = false; token->error_reported = false; + token->tree_check_p = false; /* On some systems, some header files are surrounded by an implicit extern "C" block. Set a flag in the token if it @@ -1082,16 +1085,9 @@ cp_lexer_peek_nth_token (cp_lexer* lexer, size_t n) --n; token = lexer->next_token; - gcc_assert (!n || token != &eof_token); - while (n != 0) + while (n && token->type != CPP_EOF) { ++token; - if (token == lexer->last_token) - { - token = &eof_token; - break; - } - if (!token->purged_p) --n; } @@ -1113,18 +1109,12 @@ cp_lexer_consume_token (cp_lexer* lexer) { cp_token *token = lexer->next_token; - gcc_assert (token != &eof_token); gcc_assert (!lexer->in_pragma || token->type != CPP_PRAGMA_EOL); do { + gcc_assert (token->type != CPP_EOF); lexer->next_token++; - if (lexer->next_token == lexer->last_token) - { - lexer->next_token = &eof_token; - break; - } - } while (lexer->next_token->purged_p); @@ -1150,21 +1140,14 @@ cp_lexer_purge_token (cp_lexer *lexer) { cp_token *tok = lexer->next_token; - gcc_assert (tok != &eof_token); + gcc_assert (tok->type != CPP_EOF); tok->purged_p = true; tok->location = UNKNOWN_LOCATION; tok->u.value = NULL_TREE; tok->keyword = RID_MAX; do - { - tok++; - if (tok == lexer->last_token) - { - tok = &eof_token; - break; - } - } + tok++; while (tok->purged_p); lexer->next_token = tok; } @@ -1178,12 +1161,9 @@ cp_lexer_purge_tokens_after (cp_lexer *lexer, cp_token *tok) { cp_token *peek = lexer->next_token; - if (peek == &eof_token) - peek = lexer->last_token; - gcc_assert (tok < peek); - for ( tok += 1; tok != peek; tok += 1) + for (tok++; tok != peek; tok++) { tok->purged_p = true; tok->location = UNKNOWN_LOCATION; @@ -6616,6 +6596,7 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser, /* Retrieve any deferred checks. Do not pop this access checks yet so the memory will not be reclaimed during token replacing below. */ token->u.tree_check_value = ggc_cleared_alloc (); + token->tree_check_p = true; token->u.tree_check_value->value = parser->scope; token->u.tree_check_value->checks = get_deferred_access_checks (); token->u.tree_check_value->qualifying_scope = @@ -14801,6 +14782,7 @@ cp_parser_decltype (cp_parser *parser) it again. */ start_token->type = CPP_DECLTYPE; start_token->u.tree_check_value = ggc_cleared_alloc (); + start_token->tree_check_p = true; start_token->u.tree_check_value->value = expr; start_token->u.tree_check_value->checks = get_deferred_access_checks (); start_token->keyword = RID_MAX; @@ -16588,6 +16570,7 @@ cp_parser_template_id (cp_parser *parser, /* Retrieve any deferred checks. Do not pop this access checks yet so the memory will not be reclaimed during token replacing below. */ token->u.tree_check_value = ggc_cleared_alloc (); + token->tree_check_p = true; token->u.tree_check_value->value = template_id; token->u.tree_check_value->checks = get_deferred_access_checks (); token->keyword = RID_MAX; diff --git a/gcc/cp/parser.h b/gcc/cp/parser.h index 2004982..7fee244 100644 --- a/gcc/cp/parser.h +++ b/gcc/cp/parser.h @@ -41,34 +41,34 @@ struct GTY(()) tree_check { struct GTY (()) cp_token { /* The kind of token. */ - ENUM_BITFIELD (cpp_ttype) type : 8; + enum cpp_ttype type : 8; /* If this token is a keyword, this value indicates which keyword. Otherwise, this value is RID_MAX. */ - ENUM_BITFIELD (rid) keyword : 8; + enum rid keyword : 8; /* Token flags. */ unsigned char flags; /* True if this token is from a context where it is implicitly extern "C" */ - BOOL_BITFIELD implicit_extern_c : 1; + bool implicit_extern_c : 1; /* True if an error has already been reported for this token, such as a CPP_NAME token that is not a keyword (i.e., for which KEYWORD is RID_MAX) iff this name was looked up and found to be ambiguous. */ - BOOL_BITFIELD error_reported : 1; + bool error_reported : 1; /* True for a token that has been purged. If a token is purged, it is no longer a valid token and it should be considered deleted. */ - BOOL_BITFIELD purged_p : 1; - /* 5 unused bits. */ + bool purged_p : 1; + bool tree_check_p : 1; + /* 4 unused bits. */ + /* The location at which this token was found. */ location_t location; /* The value associated with this token, if any. */ union cp_token_value { /* Used for compound tokens such as CPP_NESTED_NAME_SPECIFIER. */ - struct tree_check* GTY((tag ("1"))) tree_check_value; + struct tree_check* GTY((tag ("true"))) tree_check_value; /* Use for all other tokens. */ - tree GTY((tag ("0"))) value; - } GTY((desc ("(%1.type == CPP_TEMPLATE_ID)" - "|| (%1.type == CPP_NESTED_NAME_SPECIFIER)" - "|| (%1.type == CPP_DECLTYPE)"))) u; + tree GTY((tag ("false"))) value; + } GTY((desc ("%1.tree_check_p"))) u; }; @@ -99,6 +99,10 @@ struct GTY (()) cp_lexer { tokens. */ vec GTY ((skip)) saved_tokens; + /* Saved pieces of end token we replaced with the eof token. */ + enum cpp_ttype saved_type : 8; + enum rid saved_keyword : 8; + /* The next lexer in a linked list of lexers. */ struct cp_lexer *next;