From: Argyrios Kyrtzidis Date: Thu, 9 Jun 2022 05:00:58 +0000 (-0700) Subject: [Lex] Keep track of skipped preprocessor blocks and advance the lexer directly if... X-Git-Tag: upstream/15.0.7~4851 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f7e19a59284208712314a2d0702b48c445909130;p=platform%2Fupstream%2Fllvm.git [Lex] Keep track of skipped preprocessor blocks and advance the lexer directly if they are revisited This speeds up preprocessing, specifically for preprocessing the clang sources time is reduced by about -36%, using measurements on M1Pro with a release+thinLTO build. Differential Revision: https://reviews.llvm.org/D127379 --- diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 81d1481..9bd4b40 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -952,6 +952,18 @@ private: /// of that list. MacroInfoChain *MIChainHead = nullptr; + /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running. + /// This is used to guard against calling this function recursively. + /// + /// See comments at the use-site for more context about why it is needed. + bool SkippingExcludedConditionalBlock = false; + + /// Keeps track of skipped range mappings that were recorded while skipping + /// excluded conditional directives. It maps the source buffer pointer at + /// the beginning of a skipped block, to the number of bytes that should be + /// skipped. + llvm::DenseMap RecordedSkippedRanges; + void updateOutOfDateIdentifier(IdentifierInfo &II) const; public: diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 1356dc0..70d001f 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -33,15 +33,16 @@ #include "clang/Lex/Token.h" #include "clang/Lex/VariadicMacroSupport.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/Support/AlignOf.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Path.h" +#include "llvm/Support/SaveAndRestore.h" #include #include #include @@ -481,6 +482,19 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc) { + // In SkippingRangeStateTy we are depending on SkipExcludedConditionalBlock() + // not getting called recursively by storing the RecordedSkippedRanges + // DenseMap lookup pointer (field SkipRangePtr). SkippingRangeStateTy expects + // that RecordedSkippedRanges won't get modified and SkipRangePtr won't be + // invalidated. If this changes and there is a need to call + // SkipExcludedConditionalBlock() recursively, SkippingRangeStateTy should + // change to do a second lookup in endLexPass function instead of reusing the + // lookup pointer. + assert(!SkippingExcludedConditionalBlock && + "calling SkipExcludedConditionalBlock recursively"); + llvm::SaveAndRestore SARSkipping(SkippingExcludedConditionalBlock, + true); + ++NumSkipped; assert(!CurTokenLexer && CurPPLexer && "Lexing a macro, not a file?"); @@ -495,10 +509,53 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, CurPPLexer->LexingRawMode = true; Token Tok; SourceLocation endLoc; + + /// Keeps track and caches skipped ranges and also retrieves a prior skipped + /// range if the same block is re-visited. + struct SkippingRangeStateTy { + Preprocessor &PP; + + const char *BeginPtr = nullptr; + unsigned *SkipRangePtr = nullptr; + + SkippingRangeStateTy(Preprocessor &PP) : PP(PP) {} + + void beginLexPass() { + if (BeginPtr) + return; // continue skipping a block. + + // Initiate a skipping block and adjust the lexer if we already skipped it + // before. + BeginPtr = PP.CurLexer->getBufferLocation(); + SkipRangePtr = &PP.RecordedSkippedRanges[BeginPtr]; + if (*SkipRangePtr) { + PP.CurLexer->seek(PP.CurLexer->getCurrentBufferOffset() + *SkipRangePtr, + /*IsAtStartOfLine*/ true); + } + } + + void endLexPass(const char *Hashptr) { + if (!BeginPtr) { + // Not doing normal lexing. + assert(PP.CurLexer->isDependencyDirectivesLexer()); + return; + } + + // Finished skipping a block, record the range if it's first time visited. + if (!*SkipRangePtr) { + *SkipRangePtr = Hashptr - BeginPtr; + } + assert(*SkipRangePtr == Hashptr - BeginPtr); + BeginPtr = nullptr; + SkipRangePtr = nullptr; + } + } SkippingRangeState(*this); + while (true) { if (CurLexer->isDependencyDirectivesLexer()) { CurLexer->LexDependencyDirectiveTokenWhileSkipping(Tok); } else { + SkippingRangeState.beginLexPass(); while (true) { CurLexer->Lex(Tok); @@ -537,6 +594,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, CurPPLexer->ParsingPreprocessorDirective = true; if (CurLexer) CurLexer->SetKeepWhitespaceMode(false); + assert(Tok.is(tok::hash)); + const char *Hashptr = CurLexer->getBufferLocation() - Tok.getLength(); + assert(CurLexer->getSourceLocation(Hashptr) == Tok.getLocation()); // Read the next token, the directive flavor. LexUnexpandedToken(Tok); @@ -611,6 +671,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, // If we popped the outermost skipping block, we're done skipping! if (!CondInfo.WasSkipping) { + SkippingRangeState.endLexPass(Hashptr); // Restore the value of LexingRawMode so that trailing comments // are handled correctly, if we've reached the outermost block. CurPPLexer->LexingRawMode = false; @@ -628,6 +689,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, // as a non-skipping conditional. PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel(); + if (!CondInfo.WasSkipping) + SkippingRangeState.endLexPass(Hashptr); + // If this is a #else with a #else before it, report the error. if (CondInfo.FoundElse) Diag(Tok, diag::pp_err_else_after_else); @@ -653,6 +717,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, } else if (Sub == "lif") { // "elif". PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel(); + if (!CondInfo.WasSkipping) + SkippingRangeState.endLexPass(Hashptr); + // If this is a #elif with a #else before it, report the error. if (CondInfo.FoundElse) Diag(Tok, diag::pp_err_elif_after_else) << PED_Elif; @@ -695,6 +762,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel(); Token DirectiveToken = Tok; + if (!CondInfo.WasSkipping) + SkippingRangeState.endLexPass(Hashptr); + // Warn if using `#elifdef` & `#elifndef` in not C2x & C++2b mode even // if this branch is in a skipping block. unsigned DiagID;