From 87e0cb4f1ad299c87c3e26676a9b31b3caf58921 Mon Sep 17 00:00:00 2001 From: Ilya Biryukov Date: Tue, 5 Nov 2019 19:06:12 +0100 Subject: [PATCH] [clangd] Implement semantic highlightings via findExplicitReferences Summary: To keep the logic of finding locations of interesting AST nodes in one place. The advantage is better coverage of various AST nodes, both now and in the future: as new nodes get added to `findExplicitReferences`, semantic highlighting will automatically pick them up. The drawback of this change is that we have to traverse declarations inside our file twice in order to highlight dependent names, 'auto' and 'decltype'. Hopefully, this should not affect the actual latency too much, most time should be spent in building the AST and not traversing it. Reviewers: hokein Reviewed By: hokein Subscribers: nridge, merge_guards_bot, MaskRay, jkorous, arphaman, kadircet, usaxena95, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D69673 --- clang-tools-extra/clangd/FindTarget.cpp | 4 + clang-tools-extra/clangd/FindTarget.h | 3 + clang-tools-extra/clangd/SemanticHighlighting.cpp | 248 ++++++++------------- .../clangd/unittests/SemanticHighlightingTests.cpp | 9 + 4 files changed, 113 insertions(+), 151 deletions(-) diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp index 4e61d22..c536cbf 100644 --- a/clang-tools-extra/clangd/FindTarget.cpp +++ b/clang-tools-extra/clangd/FindTarget.cpp @@ -732,6 +732,10 @@ void findExplicitReferences(const Decl *D, assert(D); ExplicitReferenceColletor(Out).TraverseDecl(const_cast(D)); } +void findExplicitReferences(const ASTContext &AST, + llvm::function_ref Out) { + ExplicitReferenceColletor(Out).TraverseAST(const_cast(AST)); +} llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, DeclRelation R) { switch (R) { diff --git a/clang-tools-extra/clangd/FindTarget.h b/clang-tools-extra/clangd/FindTarget.h index 9ddd24a..bd5d769 100644 --- a/clang-tools-extra/clangd/FindTarget.h +++ b/clang-tools-extra/clangd/FindTarget.h @@ -22,6 +22,7 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_FINDTARGET_H #define LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_FINDTARGET_H +#include "clang/AST/ASTContext.h" #include "clang/AST/ASTTypeTraits.h" #include "clang/AST/NestedNameSpecifier.h" #include "clang/AST/Stmt.h" @@ -107,6 +108,8 @@ void findExplicitReferences(const Stmt *S, llvm::function_ref Out); void findExplicitReferences(const Decl *D, llvm::function_ref Out); +void findExplicitReferences(const ASTContext &AST, + llvm::function_ref Out); /// Similar to targetDecl(), however instead of applying a filter, all possible /// decls are returned along with their DeclRelationSets. diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp index 62d3a16..9838600 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "SemanticHighlighting.h" +#include "FindTarget.h" #include "Logger.h" #include "ParsedAST.h" #include "Protocol.h" @@ -15,10 +16,16 @@ #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclarationName.h" +#include "clang/AST/ExprCXX.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/AST/Type.h" #include "clang/AST/TypeLoc.h" +#include "clang/Basic/LangOptions.h" #include "clang/Basic/SourceLocation.h" +#include "clang/Basic/SourceManager.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/Support/Casting.h" #include namespace clang { @@ -103,12 +110,12 @@ llvm::Optional kindForType(const Type *TP) { return kindForDecl(TD); return llvm::None; } -// Given a set of candidate declarations, if the declarations all have the same -// highlighting kind, return that highlighting kind, otherwise return None. -template -llvm::Optional kindForCandidateDecls(IteratorRange Decls) { + +llvm::Optional kindForReference(const ReferenceLoc &R) { llvm::Optional Result; - for (NamedDecl *Decl : Decls) { + for (const NamedDecl *Decl : R.Targets) { + if (!canHighlightName(Decl->getDeclName())) + return llvm::None; auto Kind = kindForDecl(Decl); if (!Kind || (Result && Kind != Result)) return llvm::None; @@ -117,27 +124,49 @@ llvm::Optional kindForCandidateDecls(IteratorRange Decls) { return Result; } -// Collects all semantic tokens in an ASTContext. -class HighlightingTokenCollector - : public RecursiveASTVisitor { - std::vector Tokens; - ParsedAST &AST; - +/// Consumes source locations and maps them to text ranges for highlightings. +class HighlightingsBuilder { public: - HighlightingTokenCollector(ParsedAST &AST) : AST(AST) {} - - std::vector collectTokens() { - Tokens.clear(); - TraverseAST(AST.getASTContext()); - // Add highlightings for macro expansions as they are not traversed by the - // visitor. - for (const auto &M : AST.getMacros().Ranges) - Tokens.push_back({HighlightingKind::Macro, M}); + HighlightingsBuilder(const SourceManager &SourceMgr, + const LangOptions &LangOpts) + : SourceMgr(SourceMgr), LangOpts(LangOpts) {} + + void addToken(HighlightingToken T) { Tokens.push_back(T); } + + void addToken(SourceLocation Loc, HighlightingKind Kind) { + if (Loc.isInvalid()) + return; + if (Loc.isMacroID()) { + // Only intereseted in highlighting arguments in macros (DEF_X(arg)). + if (!SourceMgr.isMacroArgExpansion(Loc)) + return; + Loc = SourceMgr.getSpellingLoc(Loc); + } + + // Non top level decls that are included from a header are not filtered by + // topLevelDecls. (example: method declarations being included from + // another file for a class from another file). + // There are also cases with macros where the spelling loc will not be in + // the main file and the highlighting would be incorrect. + if (!isInsideMainFile(Loc, SourceMgr)) + return; + + auto Range = getTokenRange(SourceMgr, LangOpts, Loc); + if (!Range) { + // R should always have a value, if it doesn't something is very wrong. + elog("Tried to add semantic token with an invalid range"); + return; + } + Tokens.push_back(HighlightingToken{Kind, *Range}); + } + + std::vector collect() && { // Initializer lists can give duplicates of tokens, therefore all tokens // must be deduplicated. llvm::sort(Tokens); auto Last = std::unique(Tokens.begin(), Tokens.end()); Tokens.erase(Last, Tokens.end()); + // Macros can give tokens that have the same source range but conflicting // kinds. In this case all tokens sharing this source range should be // removed. @@ -161,155 +190,59 @@ public: return NonConflicting; } - bool VisitNamespaceAliasDecl(NamespaceAliasDecl *NAD) { - // The target namespace of an alias can not be found in any other way. - addToken(NAD->getTargetNameLoc(), NAD->getAliasedNamespace()); - return true; - } - - bool VisitMemberExpr(MemberExpr *ME) { - if (canHighlightName(ME->getMemberNameInfo().getName())) - addToken(ME->getMemberLoc(), ME->getMemberDecl()); - return true; - } - - bool VisitOverloadExpr(OverloadExpr *E) { - if (canHighlightName(E->getName())) - addToken(E->getNameLoc(), - kindForCandidateDecls(E->decls()) - .getValueOr(HighlightingKind::DependentName)); - return true; - } - - bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) { - if (canHighlightName(E->getDeclName())) - addToken(E->getLocation(), HighlightingKind::DependentName); - return true; - } - - bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) { - if (canHighlightName(E->getMember())) - addToken(E->getMemberLoc(), HighlightingKind::DependentName); - return true; - } - - bool VisitNamedDecl(NamedDecl *ND) { - if (canHighlightName(ND->getDeclName())) - addToken(ND->getLocation(), ND); - return true; - } +private: + const SourceManager &SourceMgr; + const LangOptions &LangOpts; + std::vector Tokens; +}; - bool VisitUsingDecl(UsingDecl *UD) { - if (auto K = kindForCandidateDecls(UD->shadows())) - addToken(UD->getLocation(), *K); - return true; - } +/// Produces highlightings, which are not captured by findExplicitReferences, +/// e.g. highlights dependent names and 'auto' as the underlying type. +class CollectExtraHighlightings + : public RecursiveASTVisitor { +public: + CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {} - bool VisitDeclRefExpr(DeclRefExpr *Ref) { - if (canHighlightName(Ref->getNameInfo().getName())) - addToken(Ref->getLocation(), Ref->getDecl()); + bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) { + if (auto K = kindForType(L.getTypePtr())) + H.addToken(L.getBeginLoc(), *K); return true; } - bool VisitTypedefTypeLoc(TypedefTypeLoc TL) { - addToken(TL.getBeginLoc(), TL.getTypedefNameDecl()); + bool VisitDeclaratorDecl(DeclaratorDecl *D) { + auto *AT = D->getType()->getContainedAutoType(); + if (!AT) + return true; + if (auto K = kindForType(AT->getDeducedType().getTypePtrOrNull())) + H.addToken(D->getTypeSpecStartLoc(), *K); return true; } - bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc TL) { - if (const TemplateDecl *TD = - TL.getTypePtr()->getTemplateName().getAsTemplateDecl()) - addToken(TL.getBeginLoc(), TD); + bool VisitOverloadExpr(OverloadExpr *E) { + if (!E->decls().empty()) + return true; // handled by findExplicitReferences. + H.addToken(E->getNameLoc(), HighlightingKind::DependentName); return true; } - bool VisitTagTypeLoc(TagTypeLoc L) { - if (L.isDefinition()) - return true; // Definition will be highligthed by VisitNamedDecl. - if (auto K = kindForType(L.getTypePtr())) - addToken(L.getBeginLoc(), *K); + bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) { + H.addToken(E->getMemberNameInfo().getLoc(), + HighlightingKind::DependentName); return true; } - bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) { - if (auto K = kindForType(L.getTypePtr())) - addToken(L.getBeginLoc(), *K); + bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) { + H.addToken(E->getNameInfo().getLoc(), HighlightingKind::DependentName); return true; } bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) { - addToken(L.getNameLoc(), HighlightingKind::DependentType); - return true; - } - - bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc TL) { - addToken(TL.getBeginLoc(), HighlightingKind::TemplateParameter); - return true; - } - - bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSLoc) { - if (auto *NNS = NNSLoc.getNestedNameSpecifier()) { - if (NNS->getKind() == NestedNameSpecifier::Namespace || - NNS->getKind() == NestedNameSpecifier::NamespaceAlias) - addToken(NNSLoc.getLocalBeginLoc(), HighlightingKind::Namespace); - } - return RecursiveASTVisitor< - HighlightingTokenCollector>::TraverseNestedNameSpecifierLoc(NNSLoc); - } - - bool TraverseConstructorInitializer(CXXCtorInitializer *CI) { - if (const FieldDecl *FD = CI->getMember()) - addToken(CI->getSourceLocation(), FD); - return RecursiveASTVisitor< - HighlightingTokenCollector>::TraverseConstructorInitializer(CI); - } - - bool VisitDeclaratorDecl(DeclaratorDecl *D) { - // Highlight 'auto' with its underlying type. - auto *AT = D->getType()->getContainedAutoType(); - if (!AT) - return true; - auto K = kindForType(AT->getDeducedType().getTypePtrOrNull()); - if (!K) - return true; - addToken(D->getTypeSpecStartLoc(), *K); + H.addToken(L.getNameLoc(), HighlightingKind::DependentType); return true; } private: - void addToken(SourceLocation Loc, HighlightingKind Kind) { - if (Loc.isInvalid()) - return; - const auto &SM = AST.getSourceManager(); - if (Loc.isMacroID()) { - // Only intereseted in highlighting arguments in macros (DEF_X(arg)). - if (!SM.isMacroArgExpansion(Loc)) - return; - Loc = SM.getSpellingLoc(Loc); - } - - // Non top level decls that are included from a header are not filtered by - // topLevelDecls. (example: method declarations being included from - // another file for a class from another file). - // There are also cases with macros where the spelling loc will not be in - // the main file and the highlighting would be incorrect. - if (!isInsideMainFile(Loc, SM)) - return; - - auto R = getTokenRange(SM, AST.getASTContext().getLangOpts(), Loc); - if (!R) { - // R should always have a value, if it doesn't something is very wrong. - elog("Tried to add semantic token with an invalid range"); - return; - } - - Tokens.push_back({Kind, R.getValue()}); - } - - void addToken(SourceLocation Loc, const NamedDecl *D) { - if (auto K = kindForDecl(D)) - addToken(Loc, *K); - } + HighlightingsBuilder &H; }; // Encode binary data into base64. @@ -367,6 +300,23 @@ takeLine(ArrayRef AllTokens, } } // namespace +std::vector getSemanticHighlightings(ParsedAST &AST) { + auto &C = AST.getASTContext(); + // Add highlightings for AST nodes. + HighlightingsBuilder Builder(AST.getSourceManager(), C.getLangOpts()); + // Highlight 'decltype' and 'auto' as their underlying types. + CollectExtraHighlightings(Builder).TraverseAST(C); + // Highlight all decls and references coming from the AST. + findExplicitReferences(C, [&](ReferenceLoc R) { + if (auto Kind = kindForReference(R)) + Builder.addToken(R.NameLoc, *Kind); + }); + // Add highlightings for macro expansions. + for (const auto &M : AST.getMacros().Ranges) + Builder.addToken({HighlightingKind::Macro, M}); + return std::move(Builder).collect(); +} + llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) { switch (K) { case HighlightingKind::Variable: @@ -466,10 +416,6 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R) { return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens); } -std::vector getSemanticHighlightings(ParsedAST &AST) { - return HighlightingTokenCollector(AST).collectTokens(); -} - std::vector toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { if (Tokens.size() == 0) diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp index 139773b..36237e9 100644 --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -589,6 +589,15 @@ TEST(SemanticHighlighting, GetsCorrectTokens) { R"cpp( void $Function[[foo]](); using ::$Function[[foo]]; + )cpp", + // Highlighting of template template arguments. + R"cpp( + template