From af3b3256270ca41eb0ab5e8d0af1ba2e0d698689 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Thu, 18 May 2017 19:21:48 +0000 Subject: [PATCH] Fix the location of "missing ';'" suggestions after annotation tokens. We were incorrectly setting PrevTokLocation to the first token in the annotation token instead of the last when consuming it. To fix this without adding a complex switch to the hot path through ConsumeToken, we now have a ConsumeAnnotationToken function for consuming annotation tokens in addition to the other Consume*Token special case functions. llvm-svn: 303372 --- clang/include/clang/Parse/Parser.h | 17 +++++++-- clang/lib/Parse/ParseCXXInlineMethods.cpp | 15 +------- clang/lib/Parse/ParseDecl.cpp | 19 ++++++---- clang/lib/Parse/ParseDeclCXX.cpp | 61 ++++++++++++++++--------------- clang/lib/Parse/ParseExpr.cpp | 4 +- clang/lib/Parse/ParseExprCXX.cpp | 23 +++++------- clang/lib/Parse/ParseOpenMP.cpp | 14 +++---- clang/lib/Parse/ParsePragma.cpp | 44 +++++++++++----------- clang/lib/Parse/ParseStmt.cpp | 4 +- clang/lib/Parse/ParseTemplate.cpp | 4 +- clang/lib/Parse/ParseTentative.cpp | 19 ++++++---- clang/lib/Parse/Parser.cpp | 22 ++++------- clang/test/Parser/cxx0x-decl.cpp | 16 ++++++++ 13 files changed, 137 insertions(+), 125 deletions(-) diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h index f5a7e02..c97b4059 100644 --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -304,8 +304,9 @@ public: } /// ConsumeToken - Consume the current 'peek token' and lex the next one. - /// This does not work with special tokens: string literals, code completion - /// and balanced tokens must be handled using the specific consume methods. + /// This does not work with special tokens: string literals, code completion, + /// annotation tokens and balanced tokens must be handled using the specific + /// consume methods. /// Returns the location of the consumed token. SourceLocation ConsumeToken() { assert(!isTokenSpecial() && @@ -366,7 +367,7 @@ private: /// isTokenSpecial - True if this token requires special consumption methods. bool isTokenSpecial() const { return isTokenStringLiteral() || isTokenParen() || isTokenBracket() || - isTokenBrace() || Tok.is(tok::code_completion); + isTokenBrace() || Tok.is(tok::code_completion) || Tok.isAnnotation(); } /// \brief Returns true if the current token is '=' or is a type of '='. @@ -397,9 +398,19 @@ private: if (Tok.is(tok::code_completion)) return ConsumeCodeCompletionTok ? ConsumeCodeCompletionToken() : handleUnexpectedCodeCompletionToken(); + if (Tok.isAnnotation()) + return ConsumeAnnotationToken(); return ConsumeToken(); } + SourceLocation ConsumeAnnotationToken() { + assert(Tok.isAnnotation() && "wrong consume method"); + SourceLocation Loc = Tok.getLocation(); + PrevTokLocation = Tok.getAnnotationEndLoc(); + PP.Lex(Tok); + return Loc; + } + /// ConsumeParen - This consume method keeps the paren count up-to-date. /// SourceLocation ConsumeParen() { diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp index 172b0ed..cc1e885 100644 --- a/clang/lib/Parse/ParseCXXInlineMethods.cpp +++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -731,19 +731,6 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2, ConsumeBrace(); break; - case tok::code_completion: - Toks.push_back(Tok); - ConsumeCodeCompletionToken(); - break; - - case tok::string_literal: - case tok::wide_string_literal: - case tok::utf8_string_literal: - case tok::utf16_string_literal: - case tok::utf32_string_literal: - Toks.push_back(Tok); - ConsumeStringToken(); - break; case tok::semi: if (StopAtSemi) return false; @@ -751,7 +738,7 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2, default: // consume this token. Toks.push_back(Tok); - ConsumeToken(); + ConsumeAnyToken(/*ConsumeCodeCompletionTok*/true); break; } isFirstTokenConsumed = false; diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 4ccee74..69e1af0 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -2989,7 +2989,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, } DS.getTypeSpecScope() = SS; - ConsumeToken(); // The C++ scope. + ConsumeAnnotationToken(); // The C++ scope. assert(Tok.is(tok::annot_template_id) && "ParseOptionalCXXScopeSpecifier not working"); AnnotateTemplateIdTokenAsType(); @@ -2998,7 +2998,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, if (Next.is(tok::annot_typename)) { DS.getTypeSpecScope() = SS; - ConsumeToken(); // The C++ scope. + ConsumeAnnotationToken(); // The C++ scope. if (Tok.getAnnotationValue()) { ParsedType T = getTypeAnnotation(Tok); isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, @@ -3010,7 +3010,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, else DS.SetTypeSpecError(); DS.SetRangeEnd(Tok.getAnnotationEndLoc()); - ConsumeToken(); // The typename + ConsumeAnnotationToken(); // The typename } if (Next.isNot(tok::identifier)) @@ -3037,7 +3037,8 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, // C++ doesn't have implicit int. Diagnose it as a typo w.r.t. to the // typename. if (!TypeRep) { - ConsumeToken(); // Eat the scope spec so the identifier is current. + // Eat the scope spec so the identifier is current. + ConsumeAnnotationToken(); ParsedAttributesWithRange Attrs(AttrFactory); if (ParseImplicitInt(DS, &SS, TemplateInfo, AS, DSContext, Attrs)) { if (!Attrs.empty()) { @@ -3050,7 +3051,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, } DS.getTypeSpecScope() = SS; - ConsumeToken(); // The C++ scope. + ConsumeAnnotationToken(); // The C++ scope. isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID, TypeRep, Policy); @@ -3080,7 +3081,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, break; DS.SetRangeEnd(Tok.getAnnotationEndLoc()); - ConsumeToken(); // The typename + ConsumeAnnotationToken(); // The typename continue; } @@ -4836,10 +4837,12 @@ bool Parser::isConstructorDeclarator(bool IsUnqualified, bool DeductionGuide) { } // Parse the constructor name. - if (Tok.isOneOf(tok::identifier, tok::annot_template_id)) { + if (Tok.is(tok::identifier)) { // We already know that we have a constructor name; just consume // the token. ConsumeToken(); + } else if (Tok.is(tok::annot_template_id)) { + ConsumeAnnotationToken(); } else { TPA.Revert(); return false; @@ -4895,7 +4898,7 @@ bool Parser::isConstructorDeclarator(bool IsUnqualified, bool DeductionGuide) { // be a constructor declaration with an invalid argument type. Keep // looking. if (Tok.is(tok::annot_cxxscope)) - ConsumeToken(); + ConsumeAnnotationToken(); ConsumeToken(); // If this is not a constructor, we must be parsing a declarator, diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index e6cf65e..527d45b 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -901,7 +901,7 @@ SourceLocation Parser::ParseDecltypeSpecifier(DeclSpec &DS) { if (Tok.is(tok::annot_decltype)) { Result = getExprAnnotation(Tok); EndLoc = Tok.getAnnotationEndLoc(); - ConsumeToken(); + ConsumeAnnotationToken(); if (Result.isInvalid()) { DS.SetTypeSpecError(); return EndLoc; @@ -1105,7 +1105,7 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc, assert(Tok.is(tok::annot_typename) && "template-id -> type failed"); ParsedType Type = getTypeAnnotation(Tok); EndLocation = Tok.getAnnotationEndLoc(); - ConsumeToken(); + ConsumeAnnotationToken(); if (Type) return Type; @@ -1162,7 +1162,7 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc, // return. EndLocation = Tok.getAnnotationEndLoc(); ParsedType Type = getTypeAnnotation(Tok); - ConsumeToken(); + ConsumeAnnotationToken(); return Type; } @@ -1565,7 +1565,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind, } } else if (Tok.is(tok::annot_template_id)) { TemplateId = takeTemplateIdAnnotation(Tok); - NameLoc = ConsumeToken(); + NameLoc = ConsumeAnnotationToken(); if (TemplateId->Kind != TNK_Type_template && TemplateId->Kind != TNK_Dependent_template_name) { @@ -3405,39 +3405,42 @@ MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) { // parse '::'[opt] nested-name-specifier[opt] CXXScopeSpec SS; ParseOptionalCXXScopeSpecifier(SS, nullptr, /*EnteringContext=*/false); - ParsedType TemplateTypeTy; - if (Tok.is(tok::annot_template_id)) { - TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok); - if (TemplateId->Kind == TNK_Type_template || - TemplateId->Kind == TNK_Dependent_template_name) { - AnnotateTemplateIdTokenAsType(/*IsClassName*/true); - assert(Tok.is(tok::annot_typename) && "template-id -> type failed"); - TemplateTypeTy = getTypeAnnotation(Tok); - } - } - // Uses of decltype will already have been converted to annot_decltype by - // ParseOptionalCXXScopeSpecifier at this point. - if (!TemplateTypeTy && Tok.isNot(tok::identifier) - && Tok.isNot(tok::annot_decltype)) { - Diag(Tok, diag::err_expected_member_or_base_name); - return true; - } + // : identifier IdentifierInfo *II = nullptr; - DeclSpec DS(AttrFactory); SourceLocation IdLoc = Tok.getLocation(); - if (Tok.is(tok::annot_decltype)) { + // : declype(...) + DeclSpec DS(AttrFactory); + // : template_name<...> + ParsedType TemplateTypeTy; + + if (Tok.is(tok::identifier)) { + // Get the identifier. This may be a member name or a class name, + // but we'll let the semantic analysis determine which it is. + II = Tok.getIdentifierInfo(); + ConsumeToken(); + } else if (Tok.is(tok::annot_decltype)) { // Get the decltype expression, if there is one. + // Uses of decltype will already have been converted to annot_decltype by + // ParseOptionalCXXScopeSpecifier at this point. + // FIXME: Can we get here with a scope specifier? ParseDecltypeSpecifier(DS); } else { - if (Tok.is(tok::identifier)) - // Get the identifier. This may be a member name or a class name, - // but we'll let the semantic analysis determine which it is. - II = Tok.getIdentifierInfo(); - ConsumeToken(); + TemplateIdAnnotation *TemplateId = Tok.is(tok::annot_template_id) + ? takeTemplateIdAnnotation(Tok) + : nullptr; + if (TemplateId && (TemplateId->Kind == TNK_Type_template || + TemplateId->Kind == TNK_Dependent_template_name)) { + AnnotateTemplateIdTokenAsType(/*IsClassName*/true); + assert(Tok.is(tok::annot_typename) && "template-id -> type failed"); + TemplateTypeTy = getTypeAnnotation(Tok); + ConsumeAnnotationToken(); + } else { + Diag(Tok, diag::err_expected_member_or_base_name); + return true; + } } - // Parse the '('. if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace)) { Diag(Tok, diag::warn_cxx98_compat_generalized_initializer_lists); diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index 727fd35..bcdc72d 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -804,7 +804,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression, case tok::annot_primary_expr: assert(Res.get() == nullptr && "Stray primary-expression annotation?"); Res = getExprAnnotation(Tok); - ConsumeToken(); + ConsumeAnnotationToken(); break; case tok::kw___super: @@ -1199,7 +1199,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression, if (Ty.isInvalid()) break; - ConsumeToken(); + ConsumeAnnotationToken(); Res = ParseObjCMessageExpressionBody(SourceLocation(), SourceLocation(), Ty.get(), nullptr); break; diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 56093f6..0b210e0 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -160,7 +160,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS, Actions.RestoreNestedNameSpecifierAnnotation(Tok.getAnnotationValue(), Tok.getAnnotationRange(), SS); - ConsumeToken(); + ConsumeAnnotationToken(); return false; } @@ -346,7 +346,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS, *LastII = TemplateId->Name; // Consume the template-id token. - ConsumeToken(); + ConsumeAnnotationToken(); assert(Tok.is(tok::coloncolon) && "NextToken() not working properly!"); SourceLocation CCLoc = ConsumeToken(); @@ -920,7 +920,7 @@ Optional Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro, PP.AnnotateCachedTokens(Tok); // Consume the annotated initializer. - ConsumeToken(); + ConsumeAnnotationToken(); } } } else @@ -1528,7 +1528,7 @@ Parser::ParseCXXPseudoDestructor(Expr *Base, SourceLocation OpLoc, // store it in the pseudo-dtor node (to be used when instantiating it). FirstTypeName.setTemplateId( (TemplateIdAnnotation *)Tok.getAnnotationValue()); - ConsumeToken(); + ConsumeAnnotationToken(); assert(Tok.is(tok::coloncolon) &&"ParseOptionalCXXScopeSpecifier fail"); CCLoc = ConsumeToken(); } else { @@ -1882,7 +1882,7 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) { DS.SetTypeSpecError(); DS.SetRangeEnd(Tok.getAnnotationEndLoc()); - ConsumeToken(); + ConsumeAnnotationToken(); DS.Finish(Actions, Policy); return; @@ -1951,11 +1951,8 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) { DS.Finish(Actions, Policy); return; } - if (Tok.is(tok::annot_typename)) - DS.SetRangeEnd(Tok.getAnnotationEndLoc()); - else - DS.SetRangeEnd(Tok.getLocation()); - ConsumeToken(); + ConsumeAnyToken(); + DS.SetRangeEnd(PrevTokLocation); DS.Finish(Actions, Policy); } @@ -2529,12 +2526,12 @@ bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, bool EnteringContext, /*NontrivialTypeSourceInfo=*/true); Result.setConstructorName(Ty, TemplateId->TemplateNameLoc, TemplateId->RAngleLoc); - ConsumeToken(); + ConsumeAnnotationToken(); return false; } Result.setConstructorTemplateId(TemplateId); - ConsumeToken(); + ConsumeAnnotationToken(); return false; } @@ -2542,7 +2539,7 @@ bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, bool EnteringContext, // our unqualified-id. Result.setTemplateId(TemplateId); TemplateKWLoc = TemplateId->TemplateKWLoc; - ConsumeToken(); + ConsumeAnnotationToken(); return false; } diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp index 86ac035..b925dd7 100644 --- a/clang/lib/Parse/ParseOpenMP.cpp +++ b/clang/lib/Parse/ParseOpenMP.cpp @@ -532,7 +532,7 @@ Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr, ConsumeAnyToken(); } // Skip the last annot_pragma_openmp_end. - SourceLocation EndLoc = ConsumeToken(); + SourceLocation EndLoc = ConsumeAnnotationToken(); if (!IsError) { return Actions.ActOnOpenMPDeclareSimdDirective( Ptr, BS, Simdlen.get(), Uniforms, Aligneds, Alignments, Linears, @@ -562,7 +562,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl( assert(Tok.is(tok::annot_pragma_openmp) && "Not an OpenMP directive!"); ParenBraceBracketBalancer BalancerRAIIObj(*this); - SourceLocation Loc = ConsumeToken(); + SourceLocation Loc = ConsumeAnnotationToken(); auto DKind = ParseOpenMPDirectiveKind(*this); switch (DKind) { @@ -578,7 +578,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl( SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch); } // Skip the last annot_pragma_openmp_end. - ConsumeToken(); + ConsumeAnnotationToken(); return Actions.ActOnOpenMPThreadprivateDirective(Loc, Helper.getIdentifiers()); } @@ -596,7 +596,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl( ConsumeAnyToken(); } // Skip the last annot_pragma_openmp_end. - ConsumeToken(); + ConsumeAnnotationToken(); return Res; } break; @@ -686,7 +686,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl( ParseExternalDeclaration(attrs); if (Tok.isAnnotation() && Tok.is(tok::annot_pragma_openmp)) { TentativeParsingAction TPA(*this); - ConsumeToken(); + ConsumeAnnotationToken(); DKind = ParseOpenMPDirectiveKind(*this); if (DKind != OMPD_end_declare_target) TPA.Revert(); @@ -814,7 +814,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective( FirstClauses(OMPC_unknown + 1); unsigned ScopeFlags = Scope::FnScope | Scope::DeclScope | Scope::OpenMPDirectiveScope; - SourceLocation Loc = ConsumeToken(), EndLoc; + SourceLocation Loc = ConsumeAnnotationToken(), EndLoc; auto DKind = ParseOpenMPDirectiveKind(*this); OpenMPDirectiveKind CancelRegion = OMPD_unknown; // Name of critical directive. @@ -973,7 +973,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective( // End location of the directive. EndLoc = Tok.getLocation(); // Consume final annot_pragma_openmp_end. - ConsumeToken(); + ConsumeAnnotationToken(); // OpenMP [2.13.8, ordered Construct, Syntax] // If the depend clause is specified, the ordered construct is a stand-alone diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp index c34cd09..18aebe6 100644 --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -382,7 +382,7 @@ void Parser::resetPragmaHandlers() { /// annot_pragma_unused 'x' annot_pragma_unused 'y' void Parser::HandlePragmaUnused() { assert(Tok.is(tok::annot_pragma_unused)); - SourceLocation UnusedLoc = ConsumeToken(); + SourceLocation UnusedLoc = ConsumeAnnotationToken(); Actions.ActOnPragmaUnused(Tok, getCurScope(), UnusedLoc); ConsumeToken(); // The argument token. } @@ -391,7 +391,7 @@ void Parser::HandlePragmaVisibility() { assert(Tok.is(tok::annot_pragma_vis)); const IdentifierInfo *VisType = static_cast(Tok.getAnnotationValue()); - SourceLocation VisLoc = ConsumeToken(); + SourceLocation VisLoc = ConsumeAnnotationToken(); Actions.ActOnPragmaVisibility(VisType, VisLoc); } @@ -407,7 +407,7 @@ void Parser::HandlePragmaPack() { assert(Tok.is(tok::annot_pragma_pack)); PragmaPackInfo *Info = static_cast(Tok.getAnnotationValue()); - SourceLocation PragmaLoc = ConsumeToken(); + SourceLocation PragmaLoc = ConsumeAnnotationToken(); ExprResult Alignment; if (Info->Alignment.is(tok::numeric_constant)) { Alignment = Actions.ActOnNumericConstant(Info->Alignment); @@ -423,7 +423,7 @@ void Parser::HandlePragmaMSStruct() { PragmaMSStructKind Kind = static_cast( reinterpret_cast(Tok.getAnnotationValue())); Actions.ActOnPragmaMSStruct(Kind); - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); } void Parser::HandlePragmaAlign() { @@ -431,7 +431,7 @@ void Parser::HandlePragmaAlign() { Sema::PragmaOptionsAlignKind Kind = static_cast( reinterpret_cast(Tok.getAnnotationValue())); - SourceLocation PragmaLoc = ConsumeToken(); + SourceLocation PragmaLoc = ConsumeAnnotationToken(); Actions.ActOnPragmaOptionsAlign(Kind, PragmaLoc); } @@ -440,12 +440,12 @@ void Parser::HandlePragmaDump() { IdentifierInfo *II = reinterpret_cast(Tok.getAnnotationValue()); Actions.ActOnPragmaDump(getCurScope(), Tok.getLocation(), II); - ConsumeToken(); + ConsumeAnnotationToken(); } void Parser::HandlePragmaWeak() { assert(Tok.is(tok::annot_pragma_weak)); - SourceLocation PragmaLoc = ConsumeToken(); + SourceLocation PragmaLoc = ConsumeAnnotationToken(); Actions.ActOnPragmaWeakID(Tok.getIdentifierInfo(), PragmaLoc, Tok.getLocation()); ConsumeToken(); // The weak name. @@ -453,7 +453,7 @@ void Parser::HandlePragmaWeak() { void Parser::HandlePragmaWeakAlias() { assert(Tok.is(tok::annot_pragma_weakalias)); - SourceLocation PragmaLoc = ConsumeToken(); + SourceLocation PragmaLoc = ConsumeAnnotationToken(); IdentifierInfo *WeakName = Tok.getIdentifierInfo(); SourceLocation WeakNameLoc = Tok.getLocation(); ConsumeToken(); @@ -467,7 +467,7 @@ void Parser::HandlePragmaWeakAlias() { void Parser::HandlePragmaRedefineExtname() { assert(Tok.is(tok::annot_pragma_redefine_extname)); - SourceLocation RedefLoc = ConsumeToken(); + SourceLocation RedefLoc = ConsumeAnnotationToken(); IdentifierInfo *RedefName = Tok.getIdentifierInfo(); SourceLocation RedefNameLoc = Tok.getLocation(); ConsumeToken(); @@ -498,13 +498,13 @@ void Parser::HandlePragmaFPContract() { } Actions.ActOnPragmaFPContract(FPC); - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); } StmtResult Parser::HandlePragmaCaptured() { assert(Tok.is(tok::annot_pragma_captured)); - ConsumeToken(); + ConsumeAnnotationToken(); if (Tok.isNot(tok::l_brace)) { PP.Diag(Tok, diag::err_expected) << tok::l_brace; @@ -541,7 +541,7 @@ void Parser::HandlePragmaOpenCLExtension() { auto State = Data->second; auto Ident = Data->first; SourceLocation NameLoc = Tok.getLocation(); - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); auto &Opt = Actions.getOpenCLOptions(); auto Name = Ident->getName(); @@ -580,7 +580,7 @@ void Parser::HandlePragmaMSPointersToMembers() { LangOptions::PragmaMSPointersToMembersKind RepresentationMethod = static_cast( reinterpret_cast(Tok.getAnnotationValue())); - SourceLocation PragmaLoc = ConsumeToken(); // The annotation token. + SourceLocation PragmaLoc = ConsumeAnnotationToken(); Actions.ActOnPragmaMSPointersToMembers(RepresentationMethod, PragmaLoc); } @@ -590,7 +590,7 @@ void Parser::HandlePragmaMSVtorDisp() { Sema::PragmaMsStackAction Action = static_cast((Value >> 16) & 0xFFFF); MSVtorDispAttr::Mode Mode = MSVtorDispAttr::Mode(Value & 0xFFFF); - SourceLocation PragmaLoc = ConsumeToken(); // The annotation token. + SourceLocation PragmaLoc = ConsumeAnnotationToken(); Actions.ActOnPragmaMSVtorDisp(Action, PragmaLoc, Mode); } @@ -600,7 +600,7 @@ void Parser::HandlePragmaMSPragma() { auto TheTokens = (std::pair, size_t> *)Tok.getAnnotationValue(); PP.EnterTokenStream(std::move(TheTokens->first), TheTokens->second, true); - SourceLocation PragmaLocation = ConsumeToken(); // The annotation token. + SourceLocation PragmaLocation = ConsumeAnnotationToken(); assert(Tok.isAnyIdentifier()); StringRef PragmaName = Tok.getIdentifierInfo()->getName(); PP.Lex(Tok); // pragma kind @@ -896,7 +896,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) { bool PragmaUnroll = PragmaNameInfo->getName() == "unroll"; bool PragmaNoUnroll = PragmaNameInfo->getName() == "nounroll"; if (Toks.empty() && (PragmaUnroll || PragmaNoUnroll)) { - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); Hint.Range = Info->PragmaName.getLocation(); return true; } @@ -923,7 +923,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) { bool AssumeSafetyArg = !OptionUnroll && !OptionDistribute; // Verify loop hint has an argument. if (Toks[0].is(tok::eof)) { - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); Diag(Toks[0].getLocation(), diag::err_pragma_loop_missing_argument) << /*StateArgument=*/StateOption << /*FullKeyword=*/OptionUnroll << /*AssumeSafetyKeyword=*/AssumeSafetyArg; @@ -932,7 +932,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) { // Validate the argument. if (StateOption) { - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); SourceLocation StateLoc = Toks[0].getLocation(); IdentifierInfo *StateInfo = Toks[0].getIdentifierInfo(); @@ -955,7 +955,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) { } else { // Enter constant expression including eof terminator into token stream. PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false); - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); ExprResult R = ParseConstantExpression(); @@ -1241,7 +1241,7 @@ void Parser::HandlePragmaAttribute() { SourceLocation PragmaLoc = Tok.getLocation(); auto *Info = static_cast(Tok.getAnnotationValue()); if (Info->Action == PragmaAttributeInfo::Pop) { - ConsumeToken(); + ConsumeAnnotationToken(); Actions.ActOnPragmaAttributePop(PragmaLoc); return; } @@ -1249,7 +1249,7 @@ void Parser::HandlePragmaAttribute() { assert(Info->Action == PragmaAttributeInfo::Push && "Unexpected #pragma attribute command"); PP.EnterTokenStream(Info->Tokens, /*DisableMacroExpansion=*/false); - ConsumeToken(); + ConsumeAnnotationToken(); ParsedAttributes &Attrs = Info->Attributes; Attrs.clearListOnly(); @@ -2526,7 +2526,7 @@ void Parser::HandlePragmaFP() { } Actions.ActOnPragmaFPContract(FPC); - ConsumeToken(); // The annotation token. + ConsumeAnnotationToken(); } /// \brief Parses loop or unroll pragma hint value and fills in Info. diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp index 7d78046..d147ab0 100644 --- a/clang/lib/Parse/ParseStmt.cpp +++ b/clang/lib/Parse/ParseStmt.cpp @@ -338,13 +338,13 @@ Retry: case tok::annot_pragma_fp_contract: ProhibitAttributes(Attrs); Diag(Tok, diag::err_pragma_fp_contract_scope); - ConsumeToken(); + ConsumeAnnotationToken(); return StmtError(); case tok::annot_pragma_fp: ProhibitAttributes(Attrs); Diag(Tok, diag::err_pragma_fp_scope); - ConsumeToken(); + ConsumeAnnotationToken(); return StmtError(); case tok::annot_pragma_opencl_extension: diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp index 6a81e14..28ae113a 100644 --- a/clang/lib/Parse/ParseTemplate.cpp +++ b/clang/lib/Parse/ParseTemplate.cpp @@ -1234,7 +1234,7 @@ bool Parser::IsTemplateArgumentList(unsigned Skip) { } Tentative(*this); while (Skip) { - ConsumeToken(); + ConsumeAnyToken(); --Skip; } @@ -1248,7 +1248,7 @@ bool Parser::IsTemplateArgumentList(unsigned Skip) { // See whether we have declaration specifiers, which indicate a type. while (isCXXDeclarationSpecifier() == TPResult::True) - ConsumeToken(); + ConsumeAnyToken(); // If we have a '>' or a ',' then this is a template argument list. return Tok.isOneOf(tok::greater, tok::comma); diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp index 0ea3f8d..8b076b6 100644 --- a/clang/lib/Parse/ParseTentative.cpp +++ b/clang/lib/Parse/ParseTentative.cpp @@ -208,17 +208,20 @@ Parser::TPResult Parser::TryConsumeDeclarationSpecifier() { TryAnnotateCXXScopeToken()) return TPResult::Error; if (Tok.is(tok::annot_cxxscope)) + ConsumeAnnotationToken(); + if (Tok.is(tok::identifier)) ConsumeToken(); - if (Tok.isNot(tok::identifier) && Tok.isNot(tok::annot_template_id)) + else if (Tok.is(tok::annot_template_id)) + ConsumeAnnotationToken(); + else return TPResult::Error; - ConsumeToken(); break; case tok::annot_cxxscope: - ConsumeToken(); + ConsumeAnnotationToken(); // Fall through. default: - ConsumeToken(); + ConsumeAnyToken(); if (getLangOpts().ObjC1 && Tok.is(tok::less)) return TryParseProtocolQualifiers(); @@ -706,7 +709,7 @@ Parser::TPResult Parser::TryParsePtrOperatorSeq() { if (Tok.isOneOf(tok::star, tok::amp, tok::caret, tok::ampamp) || (Tok.is(tok::annot_cxxscope) && NextToken().is(tok::star))) { // ptr-operator - ConsumeToken(); + ConsumeAnyToken(); while (Tok.isOneOf(tok::kw_const, tok::kw_volatile, tok::kw_restrict, tok::kw__Nonnull, tok::kw__Nullable, tok::kw__Null_unspecified)) @@ -883,7 +886,7 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract, mayHaveIdentifier) { // declarator-id if (Tok.is(tok::annot_cxxscope)) - ConsumeToken(); + ConsumeAnnotationToken(); else if (Tok.is(tok::identifier)) TentativelyDeclaredIdentifiers.push_back(Tok.getIdentifierInfo()); if (Tok.is(tok::kw_operator)) { @@ -1399,7 +1402,7 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult, SS); if (SS.getScopeRep() && SS.getScopeRep()->isDependent()) { RevertingTentativeParsingAction PA(*this); - ConsumeToken(); + ConsumeAnnotationToken(); ConsumeToken(); bool isIdentifier = Tok.is(tok::identifier); TPResult TPR = TPResult::False; @@ -1471,7 +1474,7 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult, if (getLangOpts().ObjC1 && NextToken().is(tok::less)) { // Tentatively parse the protocol qualifiers. RevertingTentativeParsingAction PA(*this); - ConsumeToken(); // The type token + ConsumeAnyToken(); // The type token TPResult TPR = TryParseProtocolQualifiers(); bool isFollowedByParen = Tok.is(tok::l_paren); diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 38476ce..4fe038d 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -337,21 +337,13 @@ bool Parser::SkipUntil(ArrayRef Toks, SkipUntilFlags Flags) { ConsumeBrace(); break; - case tok::string_literal: - case tok::wide_string_literal: - case tok::utf8_string_literal: - case tok::utf16_string_literal: - case tok::utf32_string_literal: - ConsumeStringToken(); - break; - case tok::semi: if (HasFlagsSet(Flags, StopAtSemi)) return false; // FALL THROUGH. default: // Skip this token. - ConsumeToken(); + ConsumeAnyToken(); break; } isFirstTokenSkipped = false; @@ -578,19 +570,19 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result) { Actions.ActOnModuleInclude(Tok.getLocation(), reinterpret_cast( Tok.getAnnotationValue())); - ConsumeToken(); + ConsumeAnnotationToken(); return false; case tok::annot_module_begin: Actions.ActOnModuleBegin(Tok.getLocation(), reinterpret_cast( Tok.getAnnotationValue())); - ConsumeToken(); + ConsumeAnnotationToken(); return false; case tok::annot_module_end: Actions.ActOnModuleEnd(Tok.getLocation(), reinterpret_cast( Tok.getAnnotationValue())); - ConsumeToken(); + ConsumeAnnotationToken(); return false; case tok::annot_pragma_attribute: @@ -2169,7 +2161,7 @@ bool Parser::parseMisplacedModuleImport() { Actions.ActOnModuleEnd(Tok.getLocation(), reinterpret_cast( Tok.getAnnotationValue())); - ConsumeToken(); + ConsumeAnnotationToken(); continue; } // Inform caller that recovery failed, the error must be handled at upper @@ -2181,7 +2173,7 @@ bool Parser::parseMisplacedModuleImport() { Actions.ActOnModuleBegin(Tok.getLocation(), reinterpret_cast( Tok.getAnnotationValue())); - ConsumeToken(); + ConsumeAnnotationToken(); ++MisplacedModuleBeginCount; continue; case tok::annot_module_include: @@ -2190,7 +2182,7 @@ bool Parser::parseMisplacedModuleImport() { Actions.ActOnModuleInclude(Tok.getLocation(), reinterpret_cast( Tok.getAnnotationValue())); - ConsumeToken(); + ConsumeAnnotationToken(); // If there is another module import, process it. continue; default: diff --git a/clang/test/Parser/cxx0x-decl.cpp b/clang/test/Parser/cxx0x-decl.cpp index d912e19..ba322a5 100644 --- a/clang/test/Parser/cxx0x-decl.cpp +++ b/clang/test/Parser/cxx0x-decl.cpp @@ -123,6 +123,22 @@ namespace ColonColonDecltype { ::decltype(S())::T invalid; // expected-error {{expected unqualified-id}} } +namespace AliasDeclEndLocation { + template struct A {}; + // Ensure that we correctly determine the end of this declaration to be the + // end of the annotation token, not the beginning. + using B = AliasDeclEndLocation::A // expected-error {{expected ';' after alias declaration}} + +; + // FIXME: After splitting this >> into two > tokens, we incorrectly determine + // the end of the template-id to be after the *second* '>'. + // Perhaps we could synthesize an expansion FileID containing '> >' to fix this? + using C = AliasDeclEndLocation::A\ +> // expected-error {{expected ';' after alias declaration}} + ; +} + struct Base { virtual void f() = 0; virtual void g() = 0; virtual void h() = 0; }; struct MemberComponentOrder : Base { void f() override __asm__("foobar") __attribute__(( )) {} -- 2.7.4