From cb1beee76f520857b985db2af6117ad97aa45c4e Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Sat, 4 May 2019 06:46:18 +0000 Subject: [PATCH] [c++20] Implement tweaked __VA_OPT__ rules from P1042R1: * __VA_OPT__ is expanded if the *expanded* __VA_ARGS__ is non-empty, not if the original argument contained no tokens. * Placemarkers at the start and end of __VA_OPT__ are retained just long enough to paste them with adjacent ## operators. We never paste "across" a discarded placemarker. llvm-svn: 359964 --- clang/include/clang/Lex/MacroArgs.h | 13 ++++--- clang/include/clang/Lex/VariadicMacroSupport.h | 26 ++++++++++++- clang/lib/Lex/MacroArgs.cpp | 9 ++--- clang/lib/Lex/TokenLexer.cpp | 52 ++++++++++++++++++++++--- clang/test/Preprocessor/macro_vaopt_expand.cpp | 8 ++-- clang/test/Preprocessor/macro_vaopt_p1042r1.cpp | 30 ++++++++++++++ clang/www/cxx_status.html | 2 +- 7 files changed, 116 insertions(+), 24 deletions(-) create mode 100644 clang/test/Preprocessor/macro_vaopt_p1042r1.cpp diff --git a/clang/include/clang/Lex/MacroArgs.h b/clang/include/clang/Lex/MacroArgs.h index c2ba4eb..8806f2d 100644 --- a/clang/include/clang/Lex/MacroArgs.h +++ b/clang/include/clang/Lex/MacroArgs.h @@ -112,18 +112,19 @@ public: bool isVarargsElidedUse() const { return VarargsElided; } /// Returns true if the macro was defined with a variadic (ellipsis) parameter - /// AND was invoked with at least one token supplied as a variadic argument. + /// AND was invoked with at least one token supplied as a variadic argument + /// (after pre-expansion). /// /// \code /// #define F(a) a /// #define V(a, ...) __VA_OPT__(a) - /// F() <-- returns false on this invocation. - /// V(,a) <-- returns true on this invocation. - /// V(,) <-- returns false on this invocation. + /// F() <-- returns false on this invocation. + /// V(,a) <-- returns true on this invocation. + /// V(,) <-- returns false on this invocation. + /// V(,F()) <-- returns false on this invocation. /// \endcode /// - - bool invokedWithVariadicArgument(const MacroInfo *const MI) const; + bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP); /// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of /// tokens into the literal string token that should be produced by the C # diff --git a/clang/include/clang/Lex/VariadicMacroSupport.h b/clang/include/clang/Lex/VariadicMacroSupport.h index 4274a4d..989e0ac 100644 --- a/clang/include/clang/Lex/VariadicMacroSupport.h +++ b/clang/include/clang/Lex/VariadicMacroSupport.h @@ -113,6 +113,8 @@ namespace clang { UnmatchedOpeningParens.push_back(LParenLoc); } + /// Are we at the top level within the __VA_OPT__? + bool isAtTopLevel() const { return UnmatchedOpeningParens.size() == 1; } }; /// A class for tracking whether we're inside a VA_OPT during a @@ -135,7 +137,8 @@ namespace clang { unsigned StringifyBefore : 1; unsigned CharifyBefore : 1; - + unsigned BeginsWithPlaceholder : 1; + unsigned EndsWithPlaceholder : 1; bool hasStringifyBefore() const { assert(!isReset() && @@ -151,7 +154,8 @@ namespace clang { public: VAOptExpansionContext(Preprocessor &PP) : VAOptDefinitionContext(PP), LeadingSpaceForStringifiedToken(false), - StringifyBefore(false), CharifyBefore(false) { + StringifyBefore(false), CharifyBefore(false), + BeginsWithPlaceholder(false), EndsWithPlaceholder(false) { SyntheticEOFToken.startToken(); SyntheticEOFToken.setKind(tok::eof); } @@ -162,6 +166,8 @@ namespace clang { LeadingSpaceForStringifiedToken = false; StringifyBefore = false; CharifyBefore = false; + BeginsWithPlaceholder = false; + EndsWithPlaceholder = false; } const Token &getEOFTok() const { return SyntheticEOFToken; } @@ -174,7 +180,23 @@ namespace clang { LeadingSpaceForStringifiedToken = HasLeadingSpace; } + void hasPlaceholderAfterHashhashAtStart() { BeginsWithPlaceholder = true; } + void hasPlaceholderBeforeRParen() { + if (isAtTopLevel()) + EndsWithPlaceholder = true; + } + + bool beginsWithPlaceholder() const { + assert(!isReset() && + "Must only be called if the state has not been reset"); + return BeginsWithPlaceholder; + } + bool endsWithPlaceholder() const { + assert(!isReset() && + "Must only be called if the state has not been reset"); + return EndsWithPlaceholder; + } bool hasCharifyBefore() const { assert(!isReset() && diff --git a/clang/lib/Lex/MacroArgs.cpp b/clang/lib/Lex/MacroArgs.cpp index dd0b79c..06e3add 100644 --- a/clang/lib/Lex/MacroArgs.cpp +++ b/clang/lib/Lex/MacroArgs.cpp @@ -135,15 +135,12 @@ const Token *MacroArgs::getUnexpArgument(unsigned Arg) const { return Result; } -// This function assumes that the variadic arguments are the tokens -// corresponding to the last parameter (ellipsis) - and since tokens are -// separated by the 'eof' token, if that is the only token corresponding to that -// last parameter, we know no variadic arguments were supplied. -bool MacroArgs::invokedWithVariadicArgument(const MacroInfo *const MI) const { +bool MacroArgs::invokedWithVariadicArgument(const MacroInfo *const MI, + Preprocessor &PP) { if (!MI->isVariadic()) return false; const int VariadicArgIndex = getNumMacroArguments() - 1; - return getUnexpArgument(VariadicArgIndex)->isNot(tok::eof); + return getPreExpArgument(VariadicArgIndex, PP).front().isNot(tok::eof); } /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp index b0d5286..9d132a5 100644 --- a/clang/lib/Lex/TokenLexer.cpp +++ b/clang/lib/Lex/TokenLexer.cpp @@ -243,8 +243,7 @@ void TokenLexer::ExpandFunctionArguments() { // we install the newly expanded sequence as the new 'Tokens' list. bool MadeChange = false; - const bool CalledWithVariadicArguments = - ActualArgs->invokedWithVariadicArgument(Macro); + Optional CalledWithVariadicArguments; VAOptExpansionContext VCtx(PP); @@ -291,7 +290,12 @@ void TokenLexer::ExpandFunctionArguments() { // this token. Note sawClosingParen() returns true only if the r_paren matches // the closing r_paren of the __VA_OPT__. if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) { - if (!CalledWithVariadicArguments) { + // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__. + if (!CalledWithVariadicArguments.hasValue()) { + CalledWithVariadicArguments = + ActualArgs->invokedWithVariadicArgument(Macro, PP); + } + if (!*CalledWithVariadicArguments) { // Skip this token. continue; } @@ -314,8 +318,8 @@ void TokenLexer::ExpandFunctionArguments() { stringifyVAOPTContents(ResultToks, VCtx, /*ClosingParenLoc*/ Tokens[I].getLocation()); - } else if (/*No tokens within VAOPT*/ !( - ResultToks.size() - VCtx.getNumberOfTokensPriorToVAOpt())) { + } else if (/*No tokens within VAOPT*/ + ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) { // Treat VAOPT as a placemarker token. Eat either the '##' before the // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that // hashhash was not a placemarker) or the '##' @@ -326,6 +330,26 @@ void TokenLexer::ExpandFunctionArguments() { } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) { ++I; // Skip the following hashhash. } + } else { + // If there's a ## before the __VA_OPT__, we might have discovered + // that the __VA_OPT__ begins with a placeholder. We delay action on + // that to now to avoid messing up our stashed count of tokens before + // __VA_OPT__. + if (VCtx.beginsWithPlaceholder()) { + assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 && + ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() && + ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is( + tok::hashhash) && + "no token paste before __VA_OPT__"); + ResultToks.erase(ResultToks.begin() + + VCtx.getNumberOfTokensPriorToVAOpt() - 1); + } + // If the expansion of __VA_OPT__ ends with a placeholder, eat any + // following '##' token. + if (VCtx.endsWithPlaceholder() && I + 1 != E && + Tokens[I + 1].is(tok::hashhash)) { + ++I; + } } VCtx.reset(); // We processed __VA_OPT__'s closing paren (and the exit out of @@ -386,6 +410,7 @@ void TokenLexer::ExpandFunctionArguments() { !ResultToks.empty() && ResultToks.back().is(tok::hashhash); bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash); bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash); + bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren); assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) && "unexpected ## in ResultToks"); @@ -470,6 +495,18 @@ void TokenLexer::ExpandFunctionArguments() { NextTokGetsSpace); ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false); NextTokGetsSpace = false; + } else { + // We're creating a placeholder token. Usually this doesn't matter, + // but it can affect paste behavior when at the start or end of a + // __VA_OPT__. + if (NonEmptyPasteBefore) { + // We're imagining a placeholder token is inserted here. If this is + // the first token in a __VA_OPT__ after a ##, delete the ##. + assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__"); + VCtx.hasPlaceholderAfterHashhashAtStart(); + } + if (RParenAfter) + VCtx.hasPlaceholderBeforeRParen(); } continue; } @@ -534,6 +571,9 @@ void TokenLexer::ExpandFunctionArguments() { continue; } + if (RParenAfter) + VCtx.hasPlaceholderBeforeRParen(); + // If this is on the RHS of a paste operator, we've already copied the // paste operator to the ResultToks list, unless the LHS was empty too. // Remove it. @@ -547,6 +587,8 @@ void TokenLexer::ExpandFunctionArguments() { if (!VCtx.isInVAOpt() || ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()) ResultToks.pop_back(); + else + VCtx.hasPlaceholderAfterHashhashAtStart(); } // If this is the __VA_ARGS__ token, and if the argument wasn't provided, diff --git a/clang/test/Preprocessor/macro_vaopt_expand.cpp b/clang/test/Preprocessor/macro_vaopt_expand.cpp index 52f18af..7ec4f61 100644 --- a/clang/test/Preprocessor/macro_vaopt_expand.cpp +++ b/clang/test/Preprocessor/macro_vaopt_expand.cpp @@ -129,8 +129,8 @@ #define G(a,...) __VA_OPT__(B a) ## 1 26: F(,1) 26_1: G(,1) -// CHECK: 26: B1 -// CHECK: 26_1: B1 +// CHECK: 26: B 1 +// CHECK: 26_1: B 1 #undef F #undef G @@ -140,9 +140,9 @@ 27: F(,1) 27_1: F(A0,1) 28: G(,1) -// CHECK: 27: B11 +// CHECK: 27: B 11 // CHECK: 27_1: BexpandedA0 11 -// CHECK: 28: B11 +// CHECK: 28: B 11 #undef F #undef G diff --git a/clang/test/Preprocessor/macro_vaopt_p1042r1.cpp b/clang/test/Preprocessor/macro_vaopt_p1042r1.cpp new file mode 100644 index 0000000..f12dd20 --- /dev/null +++ b/clang/test/Preprocessor/macro_vaopt_p1042r1.cpp @@ -0,0 +1,30 @@ + RUN: %clang_cc1 -E %s -pedantic -std=c++2a | FileCheck -strict-whitespace %s + +#define LPAREN() ( +#define G(Q) 42 +#define F1(R, X, ...) __VA_OPT__(G R X) ) +1: int x = F1(LPAREN(), 0, <:-); +// CHECK: 1: int x = 42; + +#define F2(...) f(0 __VA_OPT__(,) __VA_ARGS__) +#define EMP +2: F2(EMP) +// CHECK: 2: f(0 ) + +#define H3(X, ...) #__VA_OPT__(X##X X##X) +3: H3(, 0) +// CHECK: 3: "" + +#define H4(X, ...) __VA_OPT__(a X ## X) ## b +4: H4(, 1) +// CHECK: 4: a b + +#define H4B(X, ...) a ## __VA_OPT__(X ## X b) +4B: H4B(, 1) +// CHECK: 4B: a b + +#define H5A(...) __VA_OPT__()/**/__VA_OPT__() +#define H5B(X) a ## X ## b +#define H5C(X) H5B(X) +5: H5C(H5A()) +// CHECK: 5: ab diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 5306c3b..2c06f0e 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -858,7 +858,7 @@ as the draft C++2a standard evolves. P1042R1 - Partial + SVN Designated initializers -- 2.7.4