From b05dc1b8766a47482cae432011fd2faa04c83a3e Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Fri, 10 Feb 2023 08:50:49 -0800 Subject: [PATCH] [clang-format] Add a space between an overloaded operator and '>' The token annotator doesn't annotate the template opener and closer as such if they enclose an overloaded operator. This causes the space between the operator and the closer to be removed, resulting in invalid C++ code. Fixes #58602. Differential Revision: https://reviews.llvm.org/D143755 --- clang/lib/Format/FormatToken.h | 4 +- clang/lib/Format/FormatTokenLexer.cpp | 33 ++++++++++++-- clang/lib/Format/FormatTokenLexer.h | 1 + clang/lib/Format/TokenAnnotator.cpp | 32 ++++++++----- clang/unittests/Format/FormatTest.cpp | 8 ++++ clang/unittests/Format/TokenAnnotatorTest.cpp | 65 +++++++++++++++++++++++++++ 6 files changed, 126 insertions(+), 17 deletions(-) diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index f3349a4..d5a2a39 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -743,8 +743,8 @@ public: } /// Returns the next token ignoring comments. - [[nodiscard]] const FormatToken *getNextNonComment() const { - const FormatToken *Tok = Next; + [[nodiscard]] FormatToken *getNextNonComment() const { + FormatToken *Tok = Next; while (Tok && Tok->is(tok::comment)) Tok = Tok->Next; return Tok; diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp index f06f9fb..ae54de9 100644 --- a/clang/lib/Format/FormatTokenLexer.cpp +++ b/clang/lib/Format/FormatTokenLexer.cpp @@ -103,6 +103,8 @@ void FormatTokenLexer::tryMergePreviousTokens() { return; if (tryMergeLessLess()) return; + if (tryMergeGreaterGreater()) + return; if (tryMergeForEach()) return; if (Style.isCpp() && tryTransformTryUsageForC()) @@ -460,12 +462,11 @@ bool FormatTokenLexer::tryMergeLessLess() { return false; auto X = Tokens.size() > 3 ? First[-1] : nullptr; - auto Y = First[2]; - if ((X && X->is(tok::less)) || Y->is(tok::less)) + if (X && X->is(tok::less)) return false; - // Do not remove a whitespace between the two "<" e.g. "operator< <>". - if (X && X->is(tok::kw_operator) && Y->is(tok::greater)) + auto Y = First[2]; + if ((!X || X->isNot(tok::kw_operator)) && Y->is(tok::less)) return false; First[0]->Tok.setKind(tok::lessless); @@ -475,6 +476,30 @@ bool FormatTokenLexer::tryMergeLessLess() { return true; } +bool FormatTokenLexer::tryMergeGreaterGreater() { + // Merge kw_operator,greater,greater into kw_operator,greatergreater. + if (Tokens.size() < 2) + return false; + + auto First = Tokens.end() - 2; + if (First[0]->isNot(tok::greater) || First[1]->isNot(tok::greater)) + return false; + + // Only merge if there currently is no whitespace between the first two ">". + if (First[1]->hasWhitespaceBefore()) + return false; + + auto Tok = Tokens.size() > 2 ? First[-1] : nullptr; + if (Tok && Tok->isNot(tok::kw_operator)) + return false; + + First[0]->Tok.setKind(tok::greatergreater); + First[0]->TokenText = ">>"; + First[0]->ColumnWidth += 1; + Tokens.erase(Tokens.end() - 1); + return true; +} + bool FormatTokenLexer::tryMergeTokens(ArrayRef Kinds, TokenType NewType) { if (Tokens.size() < Kinds.size()) diff --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h index 950305a..0a8123f 100644 --- a/clang/lib/Format/FormatTokenLexer.h +++ b/clang/lib/Format/FormatTokenLexer.h @@ -51,6 +51,7 @@ private: void tryMergePreviousTokens(); bool tryMergeLessLess(); + bool tryMergeGreaterGreater(); bool tryMergeNSStringLiteral(); bool tryMergeJSPrivateIdentifier(); bool tryMergeCSharpStringLiteral(); diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 2bafa48..34c51c2 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -1219,19 +1219,25 @@ private: !CurrentToken->isOneOf(tok::l_paren, tok::semi, tok::r_paren)) { if (CurrentToken->isOneOf(tok::star, tok::amp)) CurrentToken->setType(TT_PointerOrReference); - consumeToken(); - if (!CurrentToken) - continue; - if (CurrentToken->is(tok::comma) && - CurrentToken->Previous->isNot(tok::kw_operator)) { + auto Next = CurrentToken->getNextNonComment(); + if (!Next) break; - } - if (CurrentToken->Previous->isOneOf(TT_BinaryOperator, TT_UnaryOperator, - tok::comma, tok::star, tok::arrow, - tok::amp, tok::ampamp) || + if (Next->is(tok::less)) + next(); + else + consumeToken(); + assert(CurrentToken); + auto Previous = CurrentToken->getPreviousNonComment(); + assert(Previous); + if (CurrentToken->is(tok::comma) && Previous->isNot(tok::kw_operator)) + break; + if (Previous->isOneOf(TT_BinaryOperator, TT_UnaryOperator, tok::comma, + tok::star, tok::arrow, tok::amp, tok::ampamp) || // User defined literal. - CurrentToken->Previous->TokenText.startswith("\"\"")) { - CurrentToken->Previous->setType(TT_OverloadedOperator); + Previous->TokenText.startswith("\"\"")) { + Previous->setType(TT_OverloadedOperator); + if (CurrentToken->isOneOf(tok::less, tok::greater)) + break; } } if (CurrentToken && CurrentToken->is(tok::l_paren)) @@ -3893,6 +3899,10 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, return true; if (Style.isCpp()) { + if (Left.is(TT_OverloadedOperator) && + Right.isOneOf(TT_TemplateOpener, TT_TemplateCloser)) { + return true; + } // Space between UDL and dot: auto b = 4s .count(); if (Right.is(tok::period) && Left.is(tok::numeric_constant)) return true; diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 69736ee..cefc4a5 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -10663,6 +10663,14 @@ TEST_F(FormatTest, UnderstandsOverloadedOperators) { verifyFormat("foo() { ::operator new(n * sizeof(foo)); }"); } +TEST_F(FormatTest, SpaceBeforeTemplateCloser) { + verifyFormat("C<&operator- > minus;"); + verifyFormat("C<&operator> > gt;"); + verifyFormat("C<&operator>= > ge;"); + verifyFormat("C<&operator<= > le;"); + verifyFormat("C<&operator< > lt;"); +} + TEST_F(FormatTest, UnderstandsFunctionRefQualification) { verifyFormat("void A::b() && {}"); verifyFormat("void A::b() && noexcept {}"); diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index 87d86ec..b138abc 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -574,6 +574,71 @@ TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) { EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen); } +TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) { + struct { + const char *Text; + tok::TokenKind Kind; + } Operators[] = {{"+", tok::plus}, + {"-", tok::minus}, + // FIXME: + // {"*", tok::star}, + {"/", tok::slash}, + {"%", tok::percent}, + {"^", tok::caret}, + // FIXME: + // {"&", tok::amp}, + {"|", tok::pipe}, + {"~", tok::tilde}, + {"!", tok::exclaim}, + {"=", tok::equal}, + // FIXME: + // {"<", tok::less}, + {">", tok::greater}, + {"+=", tok::plusequal}, + {"-=", tok::minusequal}, + {"*=", tok::starequal}, + {"/=", tok::slashequal}, + {"%=", tok::percentequal}, + {"^=", tok::caretequal}, + {"&=", tok::ampequal}, + {"|=", tok::pipeequal}, + {"<<", tok::lessless}, + {">>", tok::greatergreater}, + {">>=", tok::greatergreaterequal}, + {"<<=", tok::lesslessequal}, + {"==", tok::equalequal}, + {"!=", tok::exclaimequal}, + {"<=", tok::lessequal}, + {">=", tok::greaterequal}, + {"<=>", tok::spaceship}, + {"&&", tok::ampamp}, + {"||", tok::pipepipe}, + {"++", tok::plusplus}, + {"--", tok::minusminus}, + {",", tok::comma}, + {"->*", tok::arrowstar}, + {"->", tok::arrow}}; + + for (const auto &Operator : Operators) { + std::string Input("C<&operator"); + Input += Operator.Text; + Input += " > a;"; + auto Tokens = annotate(std::string(Input)); + ASSERT_EQ(Tokens.size(), 9u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener); + EXPECT_TOKEN(Tokens[4], Operator.Kind, TT_OverloadedOperator); + EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser); + } + + auto Tokens = annotate("C<&operator< > lt;"); + ASSERT_EQ(Tokens.size(), 12u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener); + EXPECT_TOKEN(Tokens[4], tok::less, TT_OverloadedOperator); + EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener); + EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser); + EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser); +} + TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) { auto Tokens = annotate("template \n" "concept C = (Foo && Bar) && (Bar && Baz);"); -- 2.7.4