From f43ff34ae67a6557fe9fd1ba0e992da18bd102f7 Mon Sep 17 00:00:00 2001 From: Kadir Cetinkaya Date: Fri, 12 Mar 2021 09:36:06 +0100 Subject: [PATCH] [clang] Mark re-injected tokens appropriately during pragma handling This hides such tokens from TokenWatcher, preventing crashes in clients trying to match spelled and expanded tokens. Fixes https://github.com/clangd/clangd/issues/712 Differential Revision: https://reviews.llvm.org/D98483 --- clang/lib/Parse/ParsePragma.cpp | 9 +++++++++ clang/unittests/Tooling/Syntax/TokensTest.cpp | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp index b9cdcf4..4ce8e4c4 100644 --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -14,11 +14,13 @@ #include "clang/Basic/PragmaKinds.h" #include "clang/Basic/TargetInfo.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/Token.h" #include "clang/Parse/LoopHint.h" #include "clang/Parse/ParseDiagnostic.h" #include "clang/Parse/Parser.h" #include "clang/Parse/RAIIObjectsForParser.h" #include "clang/Sema/Scope.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringSwitch.h" using namespace clang; @@ -292,6 +294,10 @@ struct PragmaMaxTokensTotalHandler : public PragmaHandler { Token &FirstToken) override; }; +void markAsReinjectedForRelexing(llvm::MutableArrayRef Toks) { + for (auto &T : Toks) + T.setFlag(clang::Token::IsReinjected); +} } // end namespace void Parser::initializePragmaHandlers() { @@ -2619,6 +2625,7 @@ void PragmaMSPragma::HandlePragma(Preprocessor &PP, TokenVector.push_back(EoF); // We must allocate this array with new because EnterTokenStream is going to // delete it later. + markAsReinjectedForRelexing(TokenVector); auto TokenArray = std::make_unique(TokenVector.size()); std::copy(TokenVector.begin(), TokenVector.end(), TokenArray.get()); auto Value = new (PP.getPreprocessorAllocator()) @@ -3176,6 +3183,7 @@ static bool ParseLoopHintValue(Preprocessor &PP, Token &Tok, Token PragmaName, EOFTok.setLocation(Tok.getLocation()); ValueList.push_back(EOFTok); // Terminates expression for parsing. + markAsReinjectedForRelexing(ValueList); Info.Toks = llvm::makeArrayRef(ValueList).copy(PP.getPreprocessorAllocator()); Info.PragmaName = PragmaName; @@ -3632,6 +3640,7 @@ void PragmaAttributeHandler::HandlePragma(Preprocessor &PP, EOFTok.setLocation(EndLoc); AttributeTokens.push_back(EOFTok); + markAsReinjectedForRelexing(AttributeTokens); Info->Tokens = llvm::makeArrayRef(AttributeTokens).copy(PP.getPreprocessorAllocator()); } diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp index 7d5943b..6a21be6 100644 --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -1037,4 +1037,13 @@ TEST_F(TokenBufferTest, ExpandedBySpelled) { IsEmpty()); } +TEST_F(TokenCollectorTest, Pragmas) { + // Tokens coming from concatenations. + recordTokens(R"cpp( + void foo() { + #pragma unroll 4 + for(int i=0;i<4;++i); + } + )cpp"); +} } // namespace -- 2.7.4