clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

   1 //==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "Annotations.h"
  10 #include "Config.h"
  11 #include "Protocol.h"
  12 #include "SemanticHighlighting.h"
  13 #include "SourceCode.h"
  14 #include "TestTU.h"
  15 #include "llvm/ADT/ArrayRef.h"
  16 #include "llvm/ADT/STLExtras.h"
  17 #include "llvm/ADT/StringRef.h"
  18 #include "llvm/Support/Error.h"
  19 #include "llvm/Support/ScopedPrinter.h"
  20 #include "llvm/Support/raw_ostream.h"
  21 #include "gmock/gmock.h"
  22 #include <algorithm>
  23
  24 namespace clang {
  25 namespace clangd {
  26 namespace {
  27
  28 using testing::IsEmpty;
  29 using testing::SizeIs;
  30
  31 /// Annotates the input code with provided semantic highlightings. Results look
  32 /// something like:
  33 ///   class $Class[[X]] {
  34 ///     $Primitive[[int]] $Field[[a]] = 0;
  35 ///   };
  36 std::string annotate(llvm::StringRef Input,
  37                      llvm::ArrayRef<HighlightingToken> Tokens) {
  38   assert(llvm::is_sorted(
  39       Tokens, [](const HighlightingToken &L, const HighlightingToken &R) {
  40         return L.R.start < R.R.start;
  41       }));
  42
  43   std::string Buf;
  44   llvm::raw_string_ostream OS(Buf);
  45   unsigned NextChar = 0;
  46   for (auto &T : Tokens) {
  47     unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
  48     unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end));
  49     assert(StartOffset <= EndOffset);
  50     assert(NextChar <= StartOffset);
  51
  52     bool hasDef =
  53         T.Modifiers & (1 << uint32_t(HighlightingModifier::Definition));
  54     bool hasDecl =
  55         T.Modifiers & (1 << uint32_t(HighlightingModifier::Declaration));
  56     EXPECT_TRUE(!hasDef || hasDecl);
  57
  58     OS << Input.substr(NextChar, StartOffset - NextChar);
  59     OS << '$' << T.Kind;
  60     for (unsigned I = 0;
  61          I <= static_cast<uint32_t>(HighlightingModifier::LastModifier); ++I) {
  62       if (T.Modifiers & (1 << I)) {
  63         // _decl_def is common and redundant, just print _def instead.
  64         if (I != uint32_t(HighlightingModifier::Declaration) || !hasDef)
  65           OS << '_' << static_cast<HighlightingModifier>(I);
  66       }
  67     }
  68     OS << "[[" << Input.substr(StartOffset, EndOffset - StartOffset) << "]]";
  69     NextChar = EndOffset;
  70   }
  71   OS << Input.substr(NextChar);
  72   return std::move(OS.str());
  73 }
  74
  75 void checkHighlightings(llvm::StringRef Code,
  76                         std::vector<std::pair</*FileName*/ llvm::StringRef,
  77                                               /*FileContent*/ llvm::StringRef>>
  78                             AdditionalFiles = {},
  79                         uint32_t ModifierMask = -1,
  80                         std::vector<std::string> AdditionalArgs = {}) {
  81   Annotations Test(Code);
  82   TestTU TU;
  83   TU.Code = std::string(Test.code());
  84
  85   TU.ExtraArgs.push_back("-std=c++20");
  86   TU.ExtraArgs.push_back("-xobjective-c++");
  87   TU.ExtraArgs.insert(std::end(TU.ExtraArgs), std::begin(AdditionalArgs),
  88                       std::end(AdditionalArgs));
  89
  90   for (auto File : AdditionalFiles)
  91     TU.AdditionalFiles.insert({File.first, std::string(File.second)});
  92   auto AST = TU.build();
  93   auto Actual =
  94       getSemanticHighlightings(AST, /*IncludeInactiveRegionTokens=*/true);
  95   for (auto &Token : Actual)
  96     Token.Modifiers &= ModifierMask;
  97
  98   EXPECT_EQ(Code, annotate(Test.code(), Actual));
  99 }
 100
 101 constexpr static uint32_t ScopeModifierMask =
 102     1 << unsigned(HighlightingModifier::FunctionScope) |
 103     1 << unsigned(HighlightingModifier::ClassScope) |
 104     1 << unsigned(HighlightingModifier::FileScope) |
 105     1 << unsigned(HighlightingModifier::GlobalScope);
 106
 107 TEST(SemanticHighlighting, GetsCorrectTokens) {
 108   const char *TestCases[] = {
 109       R"cpp(
 110       struct $Class_def[[AS]] {
 111         double $Field_decl[[SomeMember]];
 112       };
 113       struct {
 114       } $Variable_def[[S]];
 115       void $Function_def[[foo]](int $Parameter_def[[A]], $Class[[AS]] $Parameter_def[[As]]) {
 116         $Primitive_deduced_defaultLibrary[[auto]] $LocalVariable_def[[VeryLongVariableName]] = 12312;
 117         $Class[[AS]]     $LocalVariable_def[[AA]];
 118         $Primitive_deduced_defaultLibrary[[auto]] $LocalVariable_def[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] $Operator[[+]] $Parameter[[A]];
 119         auto $LocalVariable_def[[FN]] = [ $LocalVariable[[AA]]](int $Parameter_def[[A]]) -> void {};
 120         $LocalVariable[[FN]]$Operator_userDefined[[(]]12312$Operator_userDefined[[)]];
 121       }
 122     )cpp",
 123       R"cpp(
 124       void $Function_decl[[foo]](int);
 125       void $Function_decl[[Gah]]();
 126       void $Function_def[[foo]]() {
 127         auto $LocalVariable_def[[Bou]] = $Function[[Gah]];
 128       }
 129       struct $Class_def[[A]] {
 130         void $Method_decl[[abc]]();
 131       };
 132     )cpp",
 133       R"cpp(
 134       namespace $Namespace_decl[[abc]] {
 135         template$Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 136         struct $Class_def[[A]] {
 137           $TemplateParameter[[T]] $Field_decl[[t]];
 138         };
 139       }
 140       template$Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 141       struct $Class_def[[C]] : $Namespace[[abc]]::$Class[[A]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]] {
 142         typename $TemplateParameter[[T]]::$Type_dependentName[[A]]* $Field_decl[[D]];
 143       };
 144       $Namespace[[abc]]::$Class[[A]]$Bracket[[<]]int$Bracket[[>]] $Variable_def[[AA]];
 145       typedef $Namespace[[abc]]::$Class[[A]]$Bracket[[<]]int$Bracket[[>]] $Class_decl[[AAA]];
 146       struct $Class_def[[B]] {
 147         $Class_decl_constrDestr[[B]]();
 148         ~$Class_decl_constrDestr[[B]]();
 149         void operator$Operator_decl[[<<]]($Class[[B]]);
 150         $Class[[AAA]] $Field_decl[[AA]];
 151       };
 152       $Class[[B]]::$Class_def_constrDestr[[B]]() {}
 153       $Class[[B]]::~$Class_def_constrDestr[[B]]() {}
 154       void $Function_def[[f]] () {
 155         $Class[[B]] $LocalVariable_def[[BB]] = $Class[[B]]();
 156         $LocalVariable[[BB]].~$Class_constrDestr[[B]]();
 157         $Class[[B]]();
 158       }
 159     )cpp",
 160       R"cpp(
 161       enum class $Enum_decl[[E]] {
 162         $EnumConstant_decl_readonly[[A]],
 163         $EnumConstant_decl_readonly[[B]],
 164       };
 165       enum $Enum_decl[[EE]] {
 166         $EnumConstant_decl_readonly[[Hi]],
 167       };
 168       struct $Class_def[[A]] {
 169         $Enum[[E]] $Field_decl[[EEE]];
 170         $Enum[[EE]] $Field_decl[[EEEE]];
 171       };
 172       int $Variable_def[[I]] = $EnumConstant_readonly[[Hi]];
 173       $Enum[[E]] $Variable_def[[L]] = $Enum[[E]]::$EnumConstant_readonly[[B]];
 174     )cpp",
 175       R"cpp(
 176       namespace $Namespace_decl[[abc]] {
 177         namespace {}
 178         namespace $Namespace_decl[[bcd]] {
 179           struct $Class_def[[A]] {};
 180           namespace $Namespace_decl[[cde]] {
 181             struct $Class_def[[A]] {
 182               enum class $Enum_decl[[B]] {
 183                 $EnumConstant_decl_readonly[[Hi]],
 184               };
 185             };
 186           }
 187         }
 188       }
 189       using namespace $Namespace[[abc]]::$Namespace[[bcd]];
 190       namespace $Namespace_decl[[vwz]] =
 191             $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
 192       $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_def[[AA]];
 193       $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable_def[[AAA]] =
 194             $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant_readonly[[Hi]];
 195       ::$Namespace[[vwz]]::$Class[[A]] $Variable_def[[B]];
 196       ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_def[[BB]];
 197     )cpp",
 198       R"cpp(
 199       struct $Class_def[[D]] {
 200         double $Field_decl[[C]];
 201       };
 202       struct $Class_def[[A]] {
 203         double $Field_decl[[B]];
 204         $Class[[D]] $Field_decl[[E]];
 205         static double $StaticField_decl_static[[S]];
 206         static void $StaticMethod_def_static[[bar]]() {}
 207         void $Method_def[[foo]]() {
 208           $Field[[B]] $Operator[[=]] 123;
 209           this->$Field[[B]] $Operator[[=]] 156;
 210           this->$Method[[foo]]();
 211           $Method[[foo]]();
 212           $StaticMethod_static[[bar]]();
 213           $StaticField_static[[S]] $Operator[[=]] 90.1;
 214         }
 215       };
 216       void $Function_def[[foo]]() {
 217         $Class[[A]] $LocalVariable_def[[AA]];
 218         $LocalVariable[[AA]].$Field[[B]] $Operator[[+=]] 2;
 219         $LocalVariable[[AA]].$Method[[foo]]();
 220         $LocalVariable[[AA]].$Field[[E]].$Field[[C]];
 221         $Class[[A]]::$StaticField_static[[S]] $Operator[[=]] 90;
 222       }
 223     )cpp",
 224       R"cpp(
 225       struct $Class_def[[AA]] {
 226         int $Field_decl[[A]];
 227       };
 228       int $Variable_def[[B]];
 229       $Class[[AA]] $Variable_def[[A]]{$Variable[[B]]};
 230     )cpp",
 231       R"cpp(
 232       namespace $Namespace_decl[[a]] {
 233         struct $Class_def[[A]] {};
 234         typedef char $Primitive_decl[[C]];
 235       }
 236       typedef $Namespace[[a]]::$Class[[A]] $Class_decl[[B]];
 237       using $Class_decl[[BB]] = $Namespace[[a]]::$Class[[A]];
 238       enum class $Enum_decl[[E]] {};
 239       typedef $Enum[[E]] $Enum_decl[[C]];
 240       typedef $Enum[[C]] $Enum_decl[[CC]];
 241       using $Enum_decl[[CD]] = $Enum[[CC]];
 242       $Enum[[CC]] $Function_decl[[f]]($Class[[B]]);
 243       $Enum[[CD]] $Function_decl[[f]]($Class[[BB]]);
 244       typedef $Namespace[[a]]::$Primitive[[C]] $Primitive_decl[[PC]];
 245       typedef float $Primitive_decl[[F]];
 246     )cpp",
 247       R"cpp(
 248       template$Bracket[[<]]typename $TemplateParameter_def[[T]], typename = void$Bracket[[>]]
 249       class $Class_def[[A]] {
 250         $TemplateParameter[[T]] $Field_decl[[AA]];
 251         $TemplateParameter[[T]] $Method_decl[[foo]]();
 252       };
 253       template$Bracket[[<]]class $TemplateParameter_def[[TT]]$Bracket[[>]]
 254       class $Class_def[[B]] {
 255         $Class[[A]]$Bracket[[<]]$TemplateParameter[[TT]]$Bracket[[>]] $Field_decl[[AA]];
 256       };
 257       template$Bracket[[<]]class $TemplateParameter_def[[TT]], class $TemplateParameter_def[[GG]]$Bracket[[>]]
 258       class $Class_def[[BB]] {};
 259       template$Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 260       class $Class_def[[BB]]$Bracket[[<]]$TemplateParameter[[T]], int$Bracket[[>]] {};
 261       template$Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 262       class $Class_def[[BB]]$Bracket[[<]]$TemplateParameter[[T]], $TemplateParameter[[T]]*$Bracket[[>]] {};
 263
 264       template$Bracket[[<]]template$Bracket[[<]]class$Bracket[[>]] class $TemplateParameter_def[[T]], class $TemplateParameter_def[[C]]$Bracket[[>]]
 265       $TemplateParameter[[T]]$Bracket[[<]]$TemplateParameter[[C]]$Bracket[[>]] $Function_decl[[f]]();
 266
 267       template$Bracket[[<]]typename$Bracket[[>]]
 268       class $Class_def[[Foo]] {};
 269
 270       template$Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 271       void $Function_decl[[foo]]($TemplateParameter[[T]] ...);
 272     )cpp",
 273       R"cpp(
 274       template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 275       struct $Class_def[[Tmpl]] {$TemplateParameter[[T]] $Field_decl[[x]] = 0;};
 276       extern template struct $Class_def[[Tmpl]]$Bracket[[<]]float$Bracket[[>]];
 277       template struct $Class_def[[Tmpl]]$Bracket[[<]]double$Bracket[[>]];
 278     )cpp",
 279       // This test is to guard against highlightings disappearing when using
 280       // conversion operators as their behaviour in the clang AST differ from
 281       // other CXXMethodDecls.
 282       R"cpp(
 283       class $Class_def[[Foo]] {};
 284       struct $Class_def[[Bar]] {
 285         explicit operator $Class[[Foo]]*() const;
 286         explicit operator int() const;
 287         operator $Class[[Foo]]();
 288       };
 289       void $Function_def[[f]]() {
 290         $Class[[Bar]] $LocalVariable_def[[B]];
 291         $Class[[Foo]] $LocalVariable_def[[F]] = $LocalVariable[[B]];
 292         $Class[[Foo]] *$LocalVariable_def[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
 293         int $LocalVariable_def[[I]] = (int)$LocalVariable[[B]];
 294       }
 295     )cpp",
 296       R"cpp(
 297       struct $Class_def[[B]] {};
 298       struct $Class_def[[A]] {
 299         $Class[[B]] $Field_decl[[BB]];
 300         $Class[[A]] &operator$Operator_decl[[=]]($Class[[A]] &&$Parameter_def[[O]]);
 301       };
 302
 303       $Class[[A]] &$Class[[A]]::operator$Operator_def[[=]]($Class[[A]] &&$Parameter_def[[O]]) = default;
 304     )cpp",
 305       R"cpp(
 306       enum $Enum_decl[[En]] {
 307         $EnumConstant_decl_readonly[[EC]],
 308       };
 309       class $Class_def[[Foo]] {};
 310       class $Class_def[[Bar]] {
 311       public:
 312         $Class[[Foo]] $Field_decl[[Fo]];
 313         $Enum[[En]] $Field_decl[[E]];
 314         int $Field_decl[[I]];
 315         $Class_def_constrDestr[[Bar]] ($Class[[Foo]] $Parameter_def[[F]],
 316                 $Enum[[En]] $Parameter_def[[E]])
 317         : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
 318           $Field[[I]] (123) {}
 319       };
 320       class $Class_def[[Bar2]] : public $Class[[Bar]] {
 321         $Class_def_constrDestr[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant_readonly[[EC]]) {}
 322       };
 323     )cpp",
 324       R"cpp(
 325       enum $Enum_decl[[E]] {
 326         $EnumConstant_decl_readonly[[E]],
 327       };
 328       class $Class_def[[Foo]] {};
 329       $Enum_deduced[[auto]] $Variable_def[[AE]] = $Enum[[E]]::$EnumConstant_readonly[[E]];
 330       $Class_deduced[[auto]] $Variable_def[[AF]] = $Class[[Foo]]();
 331       $Class_deduced[[decltype]](auto) $Variable_def[[AF2]] = $Class[[Foo]]();
 332       $Class_deduced[[auto]] *$Variable_def[[AFP]] = $Operator[[&]]$Variable[[AF]];
 333       $Enum_deduced[[auto]] &$Variable_def[[AER]] = $Variable[[AE]];
 334       $Primitive_deduced_defaultLibrary[[auto]] $Variable_def[[Form]] = 10.2 $Operator[[+]] 2 $Operator[[*]] 4;
 335       $Primitive_deduced_defaultLibrary[[decltype]]($Variable[[Form]]) $Variable_def[[F]] = 10;
 336       auto $Variable_def[[Fun]] = []()->void{};
 337     )cpp",
 338       R"cpp(
 339       class $Class_def[[G]] {};
 340       template$Bracket[[<]]$Class[[G]] *$TemplateParameter_def_readonly[[U]]$Bracket[[>]]
 341       class $Class_def[[GP]] {};
 342       template$Bracket[[<]]$Class[[G]] &$TemplateParameter_def_readonly[[U]]$Bracket[[>]]
 343       class $Class_def[[GR]] {};
 344       template$Bracket[[<]]int *$TemplateParameter_def_readonly[[U]]$Bracket[[>]]
 345       class $Class_def[[IP]] {
 346         void $Method_def[[f]]() {
 347           $Operator[[*]]$TemplateParameter_readonly[[U]] $Operator[[+=]] 5;
 348         }
 349       };
 350       template$Bracket[[<]]unsigned $TemplateParameter_def_readonly[[U]] = 2$Bracket[[>]]
 351       class $Class_def[[Foo]] {
 352         void $Method_def[[f]]() {
 353           for(int $LocalVariable_def[[I]] = 0;
 354             $LocalVariable[[I]] $Operator[[<]] $TemplateParameter_readonly[[U]];) {}
 355         }
 356       };
 357
 358       $Class[[G]] $Variable_def[[L]];
 359       void $Function_def[[f]]() {
 360         $Class[[Foo]]$Bracket[[<]]123$Bracket[[>]] $LocalVariable_def[[F]];
 361         $Class[[GP]]$Bracket[[<]]$Operator[[&]]$Variable[[L]]$Bracket[[>]] $LocalVariable_def[[LL]];
 362         $Class[[GR]]$Bracket[[<]]$Variable[[L]]$Bracket[[>]] $LocalVariable_def[[LLL]];
 363       }
 364     )cpp",
 365       R"cpp(
 366       template$Bracket[[<]]typename $TemplateParameter_def[[T]],
 367         void ($TemplateParameter[[T]]::*$TemplateParameter_def_readonly[[method]])(int)$Bracket[[>]]
 368       struct $Class_def[[G]] {
 369         void $Method_def[[foo]](
 370             $TemplateParameter[[T]] *$Parameter_def[[O]]) {
 371           ($Parameter[[O]]$Operator_userDefined[[->*]]$TemplateParameter_readonly[[method]])(10);
 372
 373         }
 374       };
 375       struct $Class_def[[F]] {
 376         void $Method_decl[[f]](int);
 377       };
 378       template$Bracket[[<]]void (*$TemplateParameter_def_readonly[[Func]])()$Bracket[[>]]
 379       struct $Class_def[[A]] {
 380         void $Method_def[[f]]() {
 381           ($Operator[[*]]$TemplateParameter_readonly[[Func]])();
 382         }
 383       };
 384
 385       void $Function_def[[foo]]() {
 386         $Class[[F]] $LocalVariable_def[[FF]];
 387         $Class[[G]]$Bracket[[<]]$Class[[F]], $Operator[[&]]$Class[[F]]::$Method[[f]]$Bracket[[>]] $LocalVariable_def[[GG]];
 388         $LocalVariable[[GG]].$Method[[foo]]($Operator[[&]]$LocalVariable_usedAsMutablePointer[[FF]]);
 389         $Class[[A]]$Bracket[[<]]$Function[[foo]]$Bracket[[>]] $LocalVariable_def[[AA]];
 390       }
 391     )cpp",
 392       // Tokens that share a source range but have conflicting Kinds are not
 393       // highlighted.
 394       R"cpp(
 395       #define $Macro_decl[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
 396       #define $Macro_decl[[DEF_CLASS]](T) class T {};
 397       // Preamble ends.
 398       $Macro[[DEF_MULTIPLE]](XYZ);
 399       $Macro[[DEF_MULTIPLE]](XYZW);
 400       $Macro[[DEF_CLASS]]($Class_def[[A]])
 401       #define $Macro_decl[[MACRO_CONCAT]](X, V, T) T foo##X = V
 402       #define $Macro_decl[[DEF_VAR]](X, V) int X = V
 403       #define $Macro_decl[[DEF_VAR_T]](T, X, V) T X = V
 404       #define $Macro_decl[[DEF_VAR_REV]](V, X) $Macro[[DEF_VAR]](X, V)
 405       #define $Macro_decl[[CPY]](X) X
 406       #define $Macro_decl[[DEF_VAR_TYPE]](X, Y) X Y
 407       #define $Macro_decl[[SOME_NAME]] variable
 408       #define $Macro_decl[[SOME_NAME_SET]] variable2 = 123
 409       #define $Macro_decl[[INC_VAR]](X) X += 2
 410       void $Function_def[[foo]]() {
 411         $Macro[[DEF_VAR]]($LocalVariable_def[[X]],  123);
 412         $Macro[[DEF_VAR_REV]](908, $LocalVariable_def[[XY]]);
 413         int $Macro[[CPY]]( $LocalVariable_def[[XX]] );
 414         $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable_def[[AA]]);
 415         double $Macro[[SOME_NAME]];
 416         int $Macro[[SOME_NAME_SET]];
 417         $LocalVariable[[variable]] $Operator[[=]] 20.1;
 418         $Macro[[MACRO_CONCAT]](var, 2, float);
 419         $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
 420               $Macro[[CPY]]($LocalVariable_def[[Nested]])),
 421             $Macro[[CPY]]($Class[[A]]()));
 422         $Macro[[INC_VAR]]($LocalVariable[[variable]]);
 423       }
 424       void $Macro[[SOME_NAME]]();
 425       $Macro[[DEF_VAR]]($Variable_def[[MMMMM]], 567);
 426       $Macro[[DEF_VAR_REV]](756, $Variable_def[[AB]]);
 427
 428       #define $Macro_decl[[CALL_FN]](F) F();
 429       #define $Macro_decl[[DEF_FN]](F) void F ()
 430       $Macro[[DEF_FN]]($Function_def[[g]]) {
 431         $Macro[[CALL_FN]]($Function[[foo]]);
 432       }
 433     )cpp",
 434       R"cpp(
 435       #define $Macro_decl[[fail]](expr) expr
 436       #define $Macro_decl[[assert]](COND) if (!(COND)) { $Macro[[fail]]("assertion failed" #COND); }
 437       // Preamble ends.
 438       int $Variable_def[[x]];
 439       int $Variable_def[[y]];
 440       int $Function_decl[[f]]();
 441       void $Function_def[[foo]]() {
 442         $Macro[[assert]]($Variable[[x]] $Operator[[!=]] $Variable[[y]]);
 443         $Macro[[assert]]($Variable[[x]] $Operator[[!=]] $Function[[f]]());
 444       }
 445     )cpp",
 446       // highlighting all macro references
 447       R"cpp(
 448       #ifndef $Macro[[name]]
 449       #define $Macro_decl[[name]]
 450       #endif
 451
 452       #define $Macro_decl[[test]]
 453       #undef $Macro[[test]]
 454       #ifdef $Macro[[test]]
 455       #endif
 456
 457       #if defined($Macro[[test]])
 458       #endif
 459     )cpp",
 460       R"cpp(
 461       struct $Class_def[[S]] {
 462         float $Field_decl[[Value]];
 463         $Class[[S]] *$Field_decl[[Next]];
 464       };
 465       $Class[[S]] $Variable_def[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
 466       auto [$Variable_decl[[G1]], $Variable_decl[[G2]]] = $Variable[[Global]];
 467       void $Function_def[[f]]($Class[[S]] $Parameter_def[[P]]) {
 468         int $LocalVariable_def[[A]][2] = {1,2};
 469         auto [$LocalVariable_decl[[B1]], $LocalVariable_decl[[B2]]] = $LocalVariable[[A]];
 470         auto [$LocalVariable_decl[[G1]], $LocalVariable_decl[[G2]]] = $Variable[[Global]];
 471         $Class_deduced[[auto]] [$LocalVariable_decl[[P1]], $LocalVariable_decl[[P2]]] = $Parameter[[P]];
 472         // Highlights references to BindingDecls.
 473         $LocalVariable[[B1]]$Operator[[++]];
 474       }
 475     )cpp",
 476       R"cpp(
 477       template$Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 478       class $Class_def[[A]] {
 479         using $TemplateParameter_decl[[TemplateParam1]] = $TemplateParameter[[T]];
 480         typedef $TemplateParameter[[T]] $TemplateParameter_decl[[TemplateParam2]];
 481         using $Primitive_decl[[IntType]] = int;
 482
 483         using $Typedef_decl[[Pointer]] = $TemplateParameter[[T]] *;
 484         using $Typedef_decl[[LVReference]] = $TemplateParameter[[T]] &;
 485         using $Typedef_decl[[RVReference]] = $TemplateParameter[[T]]&&;
 486         using $Typedef_decl[[Array]] = $TemplateParameter[[T]]*[3];
 487         using $Typedef_decl[[MemberPointer]] = int ($Class[[A]]::*)(int);
 488
 489         // Use various previously defined typedefs in a function type.
 490         void $Method_decl[[func]](
 491           $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
 492           $Typedef[[Array]], $Typedef[[MemberPointer]]);
 493       };
 494     )cpp",
 495       R"cpp(
 496       template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 497       void $Function_decl[[phase1]]($TemplateParameter[[T]]);
 498       template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 499       void $Function_def[[foo]]($TemplateParameter[[T]] $Parameter_def[[P]]) {
 500         $Function[[phase1]]($Parameter[[P]]);
 501         $Unknown_dependentName[[phase2]]($Parameter[[P]]);
 502       }
 503     )cpp",
 504       R"cpp(
 505       class $Class_def[[A]] {
 506         template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 507         void $Method_decl[[bar]]($TemplateParameter[[T]]);
 508       };
 509
 510       template $Bracket[[<]]class $TemplateParameter_def[[U]]$Bracket[[>]]
 511       void $Function_def[[foo]]($TemplateParameter[[U]] $Parameter_def[[P]]) {
 512         $Class[[A]]().$Method[[bar]]($Parameter[[P]]);
 513       }
 514     )cpp",
 515       R"cpp(
 516       struct $Class_def[[A]] {
 517         template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 518         static void $StaticMethod_decl_static[[foo]]($TemplateParameter[[T]]);
 519       };
 520
 521       template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 522       struct $Class_def[[B]] {
 523         void $Method_def[[bar]]() {
 524           $Class[[A]]::$StaticMethod_static[[foo]]($TemplateParameter[[T]]());
 525         }
 526       };
 527     )cpp",
 528       R"cpp(
 529       template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 530       void $Function_decl[[foo]](typename $TemplateParameter[[T]]::$Type_dependentName[[Type]]
 531                                             = $TemplateParameter[[T]]::$Unknown_dependentName[[val]]);
 532     )cpp",
 533       R"cpp(
 534       template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 535       void $Function_def[[foo]]($TemplateParameter[[T]] $Parameter_def[[P]]) {
 536         $Parameter[[P]].$Unknown_dependentName[[Field]];
 537       }
 538     )cpp",
 539       R"cpp(
 540       template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 541       class $Class_def[[A]] {
 542         int $Method_def[[foo]]() {
 543           return $TemplateParameter[[T]]::$Unknown_dependentName[[Field]];
 544         }
 545       };
 546     )cpp",
 547       // Highlighting the using decl as the underlying using shadow decl.
 548       R"cpp(
 549       void $Function_decl[[foo]]();
 550       using ::$Function[[foo]];
 551     )cpp",
 552       // Highlighting of template template arguments.
 553       R"cpp(
 554       template $Bracket[[<]]template $Bracket[[<]]class$Bracket[[>]] class $TemplateParameter_def[[TT]],
 555                 template $Bracket[[<]]class$Bracket[[>]] class ...$TemplateParameter_def[[TTs]]$Bracket[[>]]
 556       struct $Class_def[[Foo]] {
 557         $Class[[Foo]]$Bracket[[<]]$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...$Bracket[[>]]
 558           *$Field_decl[[t]];
 559       };
 560     )cpp",
 561       // Inactive code highlighting
 562       R"cpp(
 563       // Code in the preamble.
 564       // Inactive lines get an empty InactiveCode token at the beginning.
 565       #ifdef $Macro[[test]]
 566 $InactiveCode[[int Inactive1;]]
 567       #endif
 568
 569       // A declaration to cause the preamble to end.
 570       int $Variable_def[[EndPreamble]];
 571
 572       // Code after the preamble.
 573       // Code inside inactive blocks does not get regular highlightings
 574       // because it's not part of the AST.
 575       #define $Macro_decl[[test2]]
 576       #if defined($Macro[[test]])
 577 $InactiveCode[[int Inactive2;]]
 578       #elif defined($Macro[[test2]])
 579       int $Variable_def[[Active1]];
 580       #else
 581 $InactiveCode[[int Inactive3;]]
 582       #endif
 583
 584       #ifndef $Macro[[test]]
 585       int $Variable_def[[Active2]];
 586       #endif
 587
 588       #ifdef $Macro[[test]]
 589 $InactiveCode[[int Inactive4;]]
 590       #else
 591       int $Variable_def[[Active3]];
 592       #endif
 593     )cpp",
 594       // Argument to 'sizeof...'
 595       R"cpp(
 596       template $Bracket[[<]]typename... $TemplateParameter_def[[Elements]]$Bracket[[>]]
 597       struct $Class_def[[TupleSize]] {
 598         static const int $StaticField_decl_readonly_static[[size]] =
 599 sizeof...($TemplateParameter[[Elements]]);
 600       };
 601     )cpp",
 602       // More dependent types
 603       R"cpp(
 604       template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 605       struct $Class_def[[Waldo]] {
 606         using $Typedef_decl[[Location1]] = typename $TemplateParameter[[T]]
 607             ::$Type_dependentName[[Resolver]]::$Type_dependentName[[Location]];
 608         using $Typedef_decl[[Location2]] = typename $TemplateParameter[[T]]
 609             ::template $Type_dependentName[[Resolver]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]
 610             ::$Type_dependentName[[Location]];
 611         using $Typedef_decl[[Location3]] = typename $TemplateParameter[[T]]
 612             ::$Type_dependentName[[Resolver]]
 613             ::template $Type_dependentName[[Location]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]];
 614         static const int $StaticField_decl_readonly_static[[Value]] = $TemplateParameter[[T]]
 615             ::$Type_dependentName[[Resolver]]::$Unknown_dependentName[[Value]];
 616       };
 617     )cpp",
 618       // Dependent name with heuristic target
 619       R"cpp(
 620       template $Bracket[[<]]typename$Bracket[[>]]
 621       struct $Class_def[[Foo]] {
 622         int $Field_decl[[Waldo]];
 623         void $Method_def[[bar]]() {
 624           $Class[[Foo]]().$Field_dependentName[[Waldo]];
 625         }
 626         template $Bracket[[<]]typename $TemplateParameter_def[[U]]$Bracket[[>]]
 627         void $Method_def[[bar1]]() {
 628           $Class[[Foo]]$Bracket[[<]]$TemplateParameter[[U]]$Bracket[[>]]().$Field_dependentName[[Waldo]];
 629         }
 630
 631         void $Method_decl[[Overload]]();
 632         void $Method_decl_readonly[[Overload]]() const;
 633       };
 634       template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 635       void $Function_def[[baz]]($Class[[Foo]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]] $Parameter_def[[o]]) {
 636         $Parameter[[o]].$Method_readonly_dependentName[[Overload]]();
 637       }
 638     )cpp",
 639       // Concepts
 640       R"cpp(
 641       template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 642       concept $Concept_decl[[Fooable]] =
 643           requires($TemplateParameter[[T]] $Parameter_def[[F]]) {
 644             $Parameter[[F]].$Unknown_dependentName[[foo]]();
 645           };
 646       template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 647           requires $Concept[[Fooable]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]
 648       void $Function_def[[bar]]($TemplateParameter[[T]] $Parameter_def[[F]]) {
 649         $Parameter[[F]].$Unknown_dependentName[[foo]]();
 650       }
 651     )cpp",
 652       // Dependent template name
 653       R"cpp(
 654       template $Bracket[[<]]template $Bracket[[<]]typename$Bracket[[>]] class$Bracket[[>]] struct $Class_def[[A]] {};
 655       template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 656       using $Typedef_decl[[W]] = $Class[[A]]$Bracket[[<]]
 657         $TemplateParameter[[T]]::template $Class_dependentName[[Waldo]]
 658       $Bracket[[>]];
 659     )cpp",
 660       R"cpp(
 661       class $Class_def_abstract[[Abstract]] {
 662       public:
 663         virtual void $Method_decl_abstract_virtual[[pure]]() = 0;
 664         virtual void $Method_decl_virtual[[impl]]();
 665       };
 666       void $Function_def[[foo]]($Class_abstract[[Abstract]]* $Parameter_def[[A]]) {
 667           $Parameter[[A]]->$Method_abstract_virtual[[pure]]();
 668           $Parameter[[A]]->$Method_virtual[[impl]]();
 669       }
 670       )cpp",
 671       R"cpp(
 672       <:[deprecated]:> int $Variable_def_deprecated[[x]];
 673       )cpp",
 674       R"cpp(
 675         // ObjC: Classes and methods
 676         @class $Class_decl[[Forward]];
 677
 678         @interface $Class_def[[Foo]]
 679         @end
 680         @interface $Class_def[[Bar]] : $Class[[Foo]]
 681         -(id) $Method_decl[[x]]:(int)$Parameter_def[[a]] $Method_decl[[y]]:(int)$Parameter_def[[b]];
 682         +(instancetype)$StaticMethod_decl_static[[sharedInstance]];
 683         +(void) $StaticMethod_decl_static[[explode]];
 684         @end
 685         @implementation $Class_def[[Bar]]
 686         -(id) $Method_def[[x]]:(int)$Parameter_def[[a]] $Method_def[[y]]:(int)$Parameter_def[[b]] {
 687           return self;
 688         }
 689         +(instancetype)$StaticMethod_def_static[[sharedInstance]] { return 0; }
 690         +(void) $StaticMethod_def_static[[explode]] {}
 691         @end
 692
 693         void $Function_def[[m]]($Class[[Bar]] *$Parameter_def[[b]]) {
 694           [$Parameter[[b]] $Method[[x]]:1 $Method[[y]]:2];
 695           [$Class[[Bar]] $StaticMethod_static[[explode]]];
 696         }
 697       )cpp",
 698       R"cpp(
 699         // ObjC: Protocols
 700         @protocol $Interface_def[[Protocol]]
 701         @end
 702         @protocol $Interface_def[[Protocol2]] <$Interface[[Protocol]]>
 703         @end
 704         @interface $Class_def[[Klass]] <$Interface[[Protocol]]>
 705         @end
 706         id<$Interface[[Protocol]]> $Variable_def[[x]];
 707       )cpp",
 708       R"cpp(
 709         // ObjC: Categories
 710         @interface $Class_def[[Foo]]
 711         @end
 712         @interface $Class[[Foo]]($Namespace_def[[Bar]])
 713         @end
 714         @implementation $Class[[Foo]]($Namespace_def[[Bar]])
 715         @end
 716       )cpp",
 717       R"cpp(
 718         // ObjC: Properties and Ivars.
 719         @interface $Class_def[[Foo]] {
 720           int $Field_decl[[_someProperty]];
 721         }
 722         @property(nonatomic, assign) int $Field_decl[[someProperty]];
 723         @property(readonly, class) $Class[[Foo]] *$Field_decl_readonly_static[[sharedInstance]];
 724         @end
 725         @implementation $Class_def[[Foo]]
 726         @synthesize someProperty = _someProperty;
 727         - (int)$Method_def[[otherMethod]] {
 728           return 0;
 729         }
 730         - (int)$Method_def[[doSomething]] {
 731           $Class[[Foo]].$Field_static[[sharedInstance]].$Field[[someProperty]] $Operator[[=]] 1;
 732           self.$Field[[someProperty]] $Operator[[=]] self.$Field[[someProperty]] $Operator[[+]] self.$Field[[otherMethod]] $Operator[[+]] 1;
 733           self->$Field[[_someProperty]] $Operator[[=]] $Field[[_someProperty]] $Operator[[+]] 1;
 734         }
 735         @end
 736       )cpp",
 737       // Member imported from dependent base
 738       R"cpp(
 739         template $Bracket[[<]]typename$Bracket[[>]] struct $Class_def[[Base]] {
 740           int $Field_decl[[member]];
 741         };
 742         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 743         struct $Class_def[[Derived]] : $Class[[Base]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]] {
 744           using $Class[[Base]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]::$Field_dependentName[[member]];
 745
 746           void $Method_def[[method]]() {
 747             (void)$Field_dependentName[[member]];
 748           }
 749         };
 750       )cpp",
 751       // Modifier for variables passed as non-const references
 752       R"cpp(
 753         struct $Class_def[[ClassWithOp]] {
 754             void operator$Operator_decl[[(]]$Operator_decl[[)]](int);
 755             void operator$Operator_decl[[(]]$Operator_decl[[)]](int, int &);
 756             void operator$Operator_decl[[(]]$Operator_decl[[)]](int, int, const int &);
 757             int &operator$Operator_decl[[[]]$Operator_decl[[]]](int &);
 758             int operator$Operator_decl[[[]]$Operator_decl[[]]](int) const;
 759         };
 760         struct $Class_def[[ClassWithStaticMember]] {
 761             static inline int $StaticField_def_static[[j]] = 0;
 762         };
 763         struct $Class_def[[ClassWithRefMembers]] {
 764           $Class_def_constrDestr[[ClassWithRefMembers]](int $Parameter_def[[i]])
 765             : $Field[[i1]]($Parameter[[i]]),
 766               $Field_readonly[[i2]]($Parameter[[i]]),
 767               $Field[[i3]]($Parameter_usedAsMutableReference[[i]]),
 768               $Field_readonly[[i4]]($Class[[ClassWithStaticMember]]::$StaticField_static[[j]]),
 769               $Field[[i5]]($Class[[ClassWithStaticMember]]::$StaticField_static_usedAsMutableReference[[j]])
 770           {}
 771           int $Field_decl[[i1]];
 772           const int &$Field_decl_readonly[[i2]];
 773           int &$Field_decl[[i3]];
 774           const int &$Field_decl_readonly[[i4]];
 775           int &$Field_decl[[i5]];
 776         };
 777         void $Function_def[[fun]](int, const int,
 778                                    int*, const int*,
 779                                    int&, const int&,
 780                                    int*&, const int*&, const int* const &,
 781                                    int**, int**&, int** const &,
 782                                    int = 123) {
 783           int $LocalVariable_def[[val]];
 784           int* $LocalVariable_def[[ptr]];
 785           const int* $LocalVariable_def_readonly[[constPtr]];
 786           int** $LocalVariable_def[[array]];
 787           $Function[[fun]]($LocalVariable[[val]], $LocalVariable[[val]],
 788                            $LocalVariable_usedAsMutablePointer[[ptr]], $LocalVariable_readonly[[constPtr]],
 789                            $LocalVariable_usedAsMutableReference[[val]], $LocalVariable[[val]],
 790
 791                            $LocalVariable_usedAsMutableReference[[ptr]],
 792                            $LocalVariable_readonly_usedAsMutableReference[[constPtr]],
 793                            $LocalVariable_readonly[[constPtr]],
 794
 795                            $LocalVariable_usedAsMutablePointer[[array]], $LocalVariable_usedAsMutableReference[[array]],
 796                            $LocalVariable[[array]]
 797                            );
 798           [](int){}$Operator_userDefined[[(]]$LocalVariable[[val]]$Operator_userDefined[[)]];
 799           [](int&){}$Operator_userDefined[[(]]$LocalVariable_usedAsMutableReference[[val]]$Operator_userDefined[[)]];
 800           [](const int&){}$Operator_userDefined[[(]]$LocalVariable[[val]]$Operator_userDefined[[)]];
 801           $Class[[ClassWithOp]] $LocalVariable_def[[c]];
 802           const $Class[[ClassWithOp]] $LocalVariable_def_readonly[[c2]];
 803           $LocalVariable[[c]]$Operator_userDefined[[(]]$LocalVariable[[val]]$Operator_userDefined[[)]];
 804           $LocalVariable[[c]]$Operator_userDefined[[(]]0, $LocalVariable_usedAsMutableReference[[val]]$Operator_userDefined[[)]];
 805           $LocalVariable[[c]]$Operator_userDefined[[(]]0, 0, $LocalVariable[[val]]$Operator_userDefined[[)]];
 806           $LocalVariable[[c]]$Operator_userDefined[[[]]$LocalVariable_usedAsMutableReference[[val]]$Operator_userDefined[[]]];
 807           $LocalVariable_readonly[[c2]]$Operator_userDefined[[[]]$LocalVariable[[val]]$Operator_userDefined[[]]];
 808         }
 809         struct $Class_def[[S]] {
 810           $Class_def_constrDestr[[S]](int&) {
 811             $Class[[S]] $LocalVariable_def[[s1]]($Field_usedAsMutableReference[[field]]);
 812             $Class[[S]] $LocalVariable_def[[s2]]($LocalVariable[[s1]].$Field_usedAsMutableReference[[field]]);
 813
 814             $Class[[S]] $LocalVariable_def[[s3]]($StaticField_static_usedAsMutableReference[[staticField]]);
 815             $Class[[S]] $LocalVariable_def[[s4]]($Class[[S]]::$StaticField_static_usedAsMutableReference[[staticField]]);
 816           }
 817           int $Field_decl[[field]];
 818           static int $StaticField_decl_static[[staticField]];
 819         };
 820         template $Bracket[[<]]typename $TemplateParameter_def[[X]]$Bracket[[>]]
 821         void $Function_def[[foo]]($TemplateParameter[[X]]& $Parameter_def[[x]]) {
 822           // We do not support dependent types, so this one should *not* get the modifier.
 823           $Function[[foo]]($Parameter[[x]]);
 824         }
 825       )cpp",
 826       // init-captures
 827       R"cpp(
 828         void $Function_def[[foo]]() {
 829           int $LocalVariable_def[[a]], $LocalVariable_def[[b]];
 830           [ $LocalVariable_def[[c]] = $LocalVariable[[a]],
 831             $LocalVariable_def[[d]]($LocalVariable[[b]]) ]() {}$Operator_userDefined[[(]]$Operator_userDefined[[)]];
 832         }
 833       )cpp",
 834       // Enum base specifier
 835       R"cpp(
 836         using $Primitive_decl[[MyTypedef]] = int;
 837         enum $Enum_decl[[MyEnum]] : $Primitive[[MyTypedef]] {};
 838       )cpp",
 839       // Enum base specifier
 840       R"cpp(
 841         typedef int $Primitive_decl[[MyTypedef]];
 842         enum $Enum_decl[[MyEnum]] : $Primitive[[MyTypedef]] {};
 843       )cpp",
 844       // Using enum
 845       R"cpp(
 846       enum class $Enum_decl[[Color]] { $EnumConstant_decl_readonly[[Black]] };
 847       namespace $Namespace_decl[[ns]] {
 848         using enum $Enum[[Color]];
 849         $Enum[[Color]] $Variable_def[[ModelT]] = $EnumConstant[[Black]];
 850       }
 851       )cpp",
 852       // Issue 1096
 853       R"cpp(
 854         void $Function_decl[[Foo]]();
 855         // Use <: :> digraphs for deprecated attribute to avoid conflict with annotation syntax
 856         <:<:deprecated:>:> void $Function_decl_deprecated[[Foo]](int* $Parameter_def[[x]]);
 857         void $Function_decl[[Foo]](int $Parameter_def[[x]]);
 858         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 859         void $Function_def[[Bar]]($TemplateParameter[[T]] $Parameter_def[[x]]) {
 860             $Function_deprecated[[Foo]]($Parameter[[x]]);
 861             $Function_deprecated[[Foo]]($Parameter[[x]]);
 862             $Function_deprecated[[Foo]]($Parameter[[x]]);
 863         }
 864       )cpp",
 865       // Predefined identifiers
 866       R"cpp(
 867         void $Function_def[[Foo]]() {
 868             const char *$LocalVariable_def_readonly[[s]] = $LocalVariable_readonly_static[[__func__]];
 869         }
 870       )cpp",
 871       // override and final
 872       R"cpp(
 873         class $Class_def_abstract[[Base]] { virtual void $Method_decl_abstract_virtual[[m]]() = 0; };
 874         class $Class_def[[override]] : public $Class_abstract[[Base]] { void $Method_decl_virtual[[m]]() $Modifier[[override]]; };
 875         class $Class_def[[final]] : public $Class[[override]] { void $Method_decl_virtual[[m]]() $Modifier[[override]] $Modifier[[final]]; };
 876       )cpp",
 877       // Issue 1222: readonly modifier for generic parameter
 878       R"cpp(
 879         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 880         auto $Function_def[[foo]](const $TemplateParameter[[T]] $Parameter_def_readonly[[template_type]],
 881                                   const $TemplateParameter[[auto]] $Parameter_def_readonly[[auto_type]],
 882                                   const int $Parameter_def_readonly[[explicit_type]]) {
 883             return $Parameter_readonly[[template_type]]
 884                  $Operator_userDefined[[+]] $Parameter_readonly[[auto_type]]
 885                  $Operator_userDefined[[+]] $Parameter_readonly[[explicit_type]];
 886         }
 887       )cpp",
 888       // Explicit template specialization
 889       R"cpp(
 890         struct $Class_def[[Base]]{};
 891         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 892         struct $Class_def[[S]] : public $Class[[Base]] {};
 893         template $Bracket[[<]]$Bracket[[>]]
 894         struct $Class_def[[S]]$Bracket[[<]]void$Bracket[[>]] : public $Class[[Base]] {};
 895
 896         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 897         $TemplateParameter[[T]] $Variable_def[[x]] = {};
 898         template $Bracket[[<]]$Bracket[[>]]
 899         int $Variable_def[[x]]$Bracket[[<]]int$Bracket[[>]] = (int)sizeof($Class[[Base]]);
 900       )cpp",
 901       // operator calls in template
 902       R"cpp(
 903         template$Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]] class $Class_def[[C]] {
 904             bool $Method_def[[compare]]($TemplateParameter[[T]] $Parameter_def[[t1]], $TemplateParameter[[T]] $Parameter_def[[t2]]) { return $Parameter[[t1]] $Operator_userDefined[[==]] $Parameter[[t2]]; }
 905             $TemplateParameter[[T]] $Method_def[[deref]]($TemplateParameter[[T]] *$Parameter_def[[t]]) { return $Operator_userDefined[[*]]$Parameter[[t]]; }
 906         };
 907       )cpp",
 908       // new and delete
 909       R"cpp(
 910         struct $Class_def[[S]] { int *$Field_decl[[a]]; };
 911         void $Function_def[[f]]() {
 912           $Class[[S]] *$LocalVariable_def[[s]] = $Operator[[new]] $Class[[S]];
 913           $LocalVariable[[s]]->$Field[[a]] $Operator[[=]] $Operator[[new]] int[10];
 914           $Operator[[delete]][] $LocalVariable[[s]]->$Field[[a]];
 915           $Operator[[delete]] $LocalVariable[[s]];
 916         }
 917       )cpp",
 918       // explicit operator invocation
 919       R"cpp(
 920         struct $Class_def[[S]] {
 921             $Class[[S]] operator$Operator_decl[[+]](const $Class[[S]] &$Parameter_def_readonly[[other]]);
 922         };
 923         void $Function_def[[f]]() {
 924             $Class[[S]] $LocalVariable_def[[s]];
 925             $Class[[S]] $LocalVariable_def[[s2]] = $LocalVariable[[s]].operator$Operator_userDefined[[+]]($LocalVariable[[s]]);
 926         }
 927       )cpp",
 928       R"cpp(
 929         // Brackets support: C++ casts
 930         void $Function_def[[f]]() {
 931           struct $Class_def[[B]] { virtual ~$Class_decl_constrDestr[[B]](); };
 932           struct $Class_def[[D]] : public $Class[[B]] {};
 933           $Class[[B]] $LocalVariable_def[[b]];
 934           int $LocalVariable_def[[i]] = static_cast$Bracket[[<]]int$Bracket[[>]](3.5);
 935           void *$LocalVariable_def[[p]] = reinterpret_cast$Bracket[[<]]void *$Bracket[[>]](0);
 936           $Class[[D]] &$LocalVariable_def[[d]] = dynamic_cast$Bracket[[<]]$Class[[D]] &$Bracket[[>]]($LocalVariable[[b]]);
 937         }
 938       )cpp",
 939       // Brackets support: Nested template instantiations.
 940       R"cpp(
 941         template$Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]] struct $Class_def[[S]] {};
 942         void $Function_def[[f]]() {
 943             $Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]int$Bracket[[>]]$Bracket[[>]] $LocalVariable_def[[s1]];
 944             $Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]int$Bracket[[>]]\
 945 $Bracket[[>]] $LocalVariable_def[[s2]];
 946             $Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]int$Bracket[[>]]$Bracket[[>]]$Bracket[[>]] $LocalVariable_def[[s3]];
 947             $Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]int$Bracket[[>]]\
 948 $Bracket[[>]]$Bracket[[>]] $LocalVariable_def[[s4]];
 949             $Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]int$Bracket[[>]]\
 950   $Bracket[[>]]$Bracket[[>]] $LocalVariable_def[[s5]];
 951             $Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]int$Bracket[[>]]\
 952 $Bracket[[>]]$Bracket[[>]] $LocalVariable_def[[s6]];
 953             $Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]$Class[[S]]$Bracket[[<]]int$Bracket[[>]]$Bracket[[>]]\
 954   $Bracket[[>]] $LocalVariable_def[[s7]];
 955       }
 956       )cpp",
 957       // Brackets support: One of the brackets is a macro.
 958       R"cpp(
 959         #define $Macro_decl[[LESS]] <
 960         template $Macro[[LESS]] typename $TemplateParameter_def[[T]] > class $Class_def[[A]] {};
 961       )cpp",
 962       // Brackets support: Specializations
 963       R"cpp(
 964         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]] class $Class_def[[S]] {
 965         public:
 966           template $Bracket[[<]]typename $TemplateParameter_def[[U]]$Bracket[[>]] class $Class_decl[[Nested]];
 967         };
 968         template $Bracket[[<]]$Bracket[[>]] class $Class_def[[S]]$Bracket[[<]]int$Bracket[[>]] {};
 969         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 970         class $Class_def[[S]]$Bracket[[<]]$TemplateParameter[[T]]*$Bracket[[>]] {};
 971         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 972         template $Bracket[[<]]typename $TemplateParameter_def[[U]]$Bracket[[>]]
 973         class $Class[[S]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]::$Class_def[[Nested]] {};
 974         template $Bracket[[<]]$Bracket[[>]]
 975         template $Bracket[[<]]$Bracket[[>]]
 976         class $Class[[S]]$Bracket[[<]]float$Bracket[[>]]::$Class_def[[Nested]]$Bracket[[<]]float$Bracket[[>]] {};
 977         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]] void $Function_decl[[foo]]();
 978         void $Function_def[[bar]]() {
 979           $Function[[foo]]$Bracket[[<]]int$Bracket[[>]]();
 980         }
 981         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]] constexpr int $Variable_def_readonly[[V]] = 42;
 982         constexpr int $Variable_def_readonly[[Y]] = $Variable_readonly[[V]]$Bracket[[<]]char$Bracket[[>]];
 983         template $Bracket[[<]]$Bracket[[>]]
 984         constexpr int $Variable_def_readonly[[V]]$Bracket[[<]]int$Bracket[[>]] = 5;
 985         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 986         constexpr int $Variable_def_readonly[[V]]$Bracket[[<]]$TemplateParameter[[T]]*$Bracket[[>]] = 6;
 987         template $Bracket[[<]]typename$Bracket[[>]]
 988         class $Class_def[[A]] {
 989           enum class $Enum_decl[[E]];
 990         };
 991         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
 992         enum class $Class[[A]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]::$Enum_decl[[E]] {$EnumConstant_decl_readonly[[X]], $EnumConstant_decl_readonly[[Y]], $EnumConstant_decl_readonly[[Z]]};
 993         template $Bracket[[<]]class $TemplateParameter_def[[T]]$Bracket[[>]]
 994         class $Class_def[[B]] {
 995           template $Bracket[[<]]class $TemplateParameter_def[[U]]$Bracket[[>]] void $Method_def[[foo]]($TemplateParameter[[U]]) { }
 996           template$Bracket[[<]]$Bracket[[>]] void $Method_def[[foo]]$Bracket[[<]]int$Bracket[[>]](int) { }
 997           friend void $Function_decl[[foo]]$Bracket[[<]]$Bracket[[>]]($TemplateParameter[[T]]);
 998         };
 999       )cpp",
1000       // Brackets support: Function calls
1001       R"cpp(
1002         template $Bracket[[<]]typename$Bracket[[>]] void $Function_decl[[free]]();
1003         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
1004           struct $Class_def[[A]] {
1005             template $Bracket[[<]]typename$Bracket[[>]] void $Method_decl[[mem]]();
1006         };
1007         void $Function_def[[foo]]() {
1008           $Class[[A]]$Bracket[[<]]int$Bracket[[>]] $LocalVariable_def[[a]];
1009           $LocalVariable[[a]].$Method[[mem]]$Bracket[[<]]int$Bracket[[>]]();
1010         }
1011         template$Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
1012         void $Function_def[[bar]]() {
1013           $Function[[free]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]();
1014           $Class[[A]]$Bracket[[<]]int$Bracket[[>]] $LocalVariable_def[[a]];
1015           $LocalVariable[[a]].$Method[[mem]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]();
1016           $Class[[A]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]] $LocalVariable_def[[b]];
1017           $LocalVariable[[b]].template $Method_dependentName[[mem]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]();
1018         }
1019       )cpp",
1020       // Brackets support: Concepts
1021       R"cpp(
1022         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
1023         concept $Concept_decl[[C]] = true;
1024         template $Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]] requires $Concept[[C]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]
1025         class $Class_def[[Z]] {};
1026         template $Bracket[[<]]typename, typename$Bracket[[>]]
1027         concept $Concept_decl[[C2]] = true;
1028         template $Bracket[[<]]$Concept[[C2]]$Bracket[[<]]int$Bracket[[>]] $TemplateParameter_def[[A]]$Bracket[[>]]
1029         class $Class_def[[B]] {};
1030       )cpp",
1031       // no crash
1032       R"cpp(
1033         struct $Class_def[[Foo]] {
1034           void $Method_decl[[foo]]();
1035         };
1036
1037         void $Function_def[[s]]($Class[[Foo]] $Parameter_def[[f]]) {
1038           auto $LocalVariable_def[[k]] = $Operator[[&]]$Class[[Foo]]::$Method[[foo]];
1039           ($Parameter[[f]]$Operator[[.*]]$LocalVariable[[k]])(); // no crash on VisitCXXMemberCallExpr
1040         }
1041       )cpp",
1042       R"cpp(
1043         template$Bracket[[<]]typename$Bracket[[>]]
1044         class $Class_def[[Foo]] {};
1045
1046         template$Bracket[[<]]typename $TemplateParameter_def[[T]]$Bracket[[>]]
1047         void $Function_def[[k]]() {
1048           auto $LocalVariable_def[[s]] = $Operator[[new]] $Class[[Foo]]$Bracket[[<]]$TemplateParameter[[T]]$Bracket[[>]]();
1049           $Operator[[delete]] $LocalVariable[[s]];
1050         }
1051       )cpp",
1052       // Recursive UsingValueDecl
1053       R"cpp(
1054         template $Bracket[[<]]int$Bracket[[>]] class $Class_def[[X]] {
1055           template $Bracket[[<]]int$Bracket[[>]] class $Class_def[[Y]] {
1056             using $Class[[Y]]$Bracket[[<]]0$Bracket[[>]]::$Unknown_dependentName[[xxx]];
1057           };
1058         };
1059     )cpp"};
1060   for (const auto &TestCase : TestCases)
1061     // Mask off scope modifiers to keep the tests manageable.
1062     // They're tested separately.
1063     checkHighlightings(TestCase, {}, ~ScopeModifierMask);
1064
1065   checkHighlightings(R"cpp(
1066     class $Class_def[[A]] {
1067       #include "imp.h"
1068     };
1069   )cpp",
1070                      {{"imp.h", R"cpp(
1071     int someMethod();
1072     void otherMethod();
1073   )cpp"}},
1074                      ~ScopeModifierMask);
1075
1076   // A separate test for macros in headers.
1077   checkHighlightings(R"cpp(
1078     #include "imp.h"
1079     $Macro[[DEFINE_Y]]
1080     $Macro[[DXYZ_Y]](A);
1081   )cpp",
1082                      {{"imp.h", R"cpp(
1083     #define DXYZ(X) class X {};
1084     #define DXYZ_Y(Y) DXYZ(x##Y)
1085     #define DEFINE(X) int X;
1086     #define DEFINE_Y DEFINE(Y)
1087   )cpp"}},
1088                      ~ScopeModifierMask);
1089
1090   checkHighlightings(R"cpp(
1091     #include "SYSObject.h"
1092     @interface $Class_defaultLibrary[[SYSObject]] ($Namespace_def[[UserCategory]])
1093     @property(nonatomic, readonly) int $Field_decl_readonly[[user_property]];
1094     @end
1095     int $Function_def[[somethingUsingSystemSymbols]]() {
1096       $Class_defaultLibrary[[SYSObject]] *$LocalVariable_def[[obj]] = [$Class_defaultLibrary[[SYSObject]] $StaticMethod_static_defaultLibrary[[new]]];
1097       return $LocalVariable[[obj]].$Field_defaultLibrary[[value]] $Operator[[+]] $LocalVariable[[obj]].$Field_readonly[[user_property]];
1098     }
1099   )cpp",
1100                      {{"SystemSDK/SYSObject.h", R"cpp(
1101     @interface SYSObject
1102     @property(nonatomic, assign) int value;
1103     + (instancetype)new;
1104     @end
1105   )cpp"}},
1106                      ~ScopeModifierMask, {"-isystemSystemSDK/"});
1107 }
1108
1109 TEST(SemanticHighlighting, ScopeModifiers) {
1110   const char *TestCases[] = {
1111       R"cpp(
1112         static int $Variable_fileScope[[x]];
1113         namespace $Namespace_globalScope[[ns]] {
1114           class $Class_globalScope[[x]];
1115         }
1116         namespace {
1117           void $Function_fileScope[[foo]]();
1118         }
1119       )cpp",
1120       R"cpp(
1121         void $Function_globalScope[[foo]](int $Parameter_functionScope[[y]]) {
1122           int $LocalVariable_functionScope[[z]];
1123         }
1124       )cpp",
1125       R"cpp(
1126         // Lambdas are considered functions, not classes.
1127         auto $Variable_fileScope[[x]] = [$LocalVariable_functionScope[[m]](42)] {
1128           return $LocalVariable_functionScope[[m]];
1129         };
1130       )cpp",
1131       R"cpp(
1132         // Classes in functions are classes.
1133         void $Function_globalScope[[foo]]() {
1134           class $Class_functionScope[[X]] {
1135             int $Field_classScope[[x]];
1136           };
1137         };
1138       )cpp",
1139       R"cpp(
1140         template $Bracket[[<]]int $TemplateParameter_classScope[[T]]$Bracket[[>]]
1141         class $Class_globalScope[[X]] {
1142         };
1143       )cpp",
1144       R"cpp(
1145         // No useful scope for template parameters of variable templates.
1146         template $Bracket[[<]]typename $TemplateParameter[[A]]$Bracket[[>]]
1147         unsigned $Variable_globalScope[[X]] =
1148           $TemplateParameter[[A]]::$Unknown_classScope[[x]];
1149       )cpp",
1150       R"cpp(
1151         #define $Macro_globalScope[[X]] 1
1152         int $Variable_globalScope[[Y]] = $Macro_globalScope[[X]];
1153       )cpp",
1154   };
1155
1156   for (const char *Test : TestCases)
1157     checkHighlightings(Test, {}, ScopeModifierMask);
1158 }
1159
1160 // Ranges are highlighted as variables, unless highlighted as $Function etc.
1161 std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
1162   Annotations A(MarkedText);
1163   std::vector<HighlightingToken> Results;
1164   for (const Range& R : A.ranges())
1165     Results.push_back({HighlightingKind::Variable, 0, R});
1166   for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
1167     HighlightingKind Kind = static_cast<HighlightingKind>(I);
1168     for (const Range& R : A.ranges(llvm::to_string(Kind)))
1169       Results.push_back({Kind, 0, R});
1170   }
1171   llvm::sort(Results);
1172   return Results;
1173 }
1174
1175 TEST(SemanticHighlighting, toSemanticTokens) {
1176   auto Tokens = tokens(R"(
1177  [[blah]]
1178
1179     $Function[[big]] [[bang]]
1180   )");
1181   Tokens.front().Modifiers |= unsigned(HighlightingModifier::Declaration);
1182   Tokens.front().Modifiers |= unsigned(HighlightingModifier::Readonly);
1183   auto Results = toSemanticTokens(Tokens, /*Code=*/"");
1184
1185   ASSERT_THAT(Results, SizeIs(3));
1186   EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
1187   EXPECT_EQ(Results[0].tokenModifiers,
1188             unsigned(HighlightingModifier::Declaration) |
1189                 unsigned(HighlightingModifier::Readonly));
1190   EXPECT_EQ(Results[0].deltaLine, 1u);
1191   EXPECT_EQ(Results[0].deltaStart, 1u);
1192   EXPECT_EQ(Results[0].length, 4u);
1193
1194   EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
1195   EXPECT_EQ(Results[1].tokenModifiers, 0u);
1196   EXPECT_EQ(Results[1].deltaLine, 2u);
1197   EXPECT_EQ(Results[1].deltaStart, 4u);
1198   EXPECT_EQ(Results[1].length, 3u);
1199
1200   EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
1201   EXPECT_EQ(Results[1].tokenModifiers, 0u);
1202   EXPECT_EQ(Results[2].deltaLine, 0u);
1203   EXPECT_EQ(Results[2].deltaStart, 4u);
1204   EXPECT_EQ(Results[2].length, 4u);
1205 }
1206
1207 TEST(SemanticHighlighting, diffSemanticTokens) {
1208   auto Before = toSemanticTokens(tokens(R"(
1209     [[foo]] [[bar]] [[baz]]
1210     [[one]] [[two]] [[three]]
1211   )"),
1212                                  /*Code=*/"");
1213   EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
1214
1215   auto After = toSemanticTokens(tokens(R"(
1216     [[foo]] [[hello]] [[world]] [[baz]]
1217     [[one]] [[two]] [[three]]
1218   )"),
1219                                 /*Code=*/"");
1220
1221   // Replace [bar, baz] with [hello, world, baz]
1222   auto Diff = diffTokens(Before, After);
1223   ASSERT_THAT(Diff, SizeIs(1));
1224   EXPECT_EQ(1u, Diff.front().startToken);
1225   EXPECT_EQ(2u, Diff.front().deleteTokens);
1226   ASSERT_THAT(Diff.front().tokens, SizeIs(3));
1227   // hello
1228   EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
1229   EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
1230   EXPECT_EQ(5u, Diff.front().tokens[0].length);
1231   // world
1232   EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
1233   EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
1234   EXPECT_EQ(5u, Diff.front().tokens[1].length);
1235   // baz
1236   EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
1237   EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
1238   EXPECT_EQ(3u, Diff.front().tokens[2].length);
1239 }
1240
1241 TEST(SemanticHighlighting, MultilineTokens) {
1242   llvm::StringRef AnnotatedCode = R"cpp(
1243   [[fo
1244 o
1245 o]] [[bar]])cpp";
1246   auto Toks = toSemanticTokens(tokens(AnnotatedCode),
1247                                Annotations(AnnotatedCode).code());
1248   ASSERT_THAT(Toks, SizeIs(4));
1249   // foo
1250   EXPECT_EQ(Toks[0].deltaLine, 1u);
1251   EXPECT_EQ(Toks[0].deltaStart, 2u);
1252   EXPECT_EQ(Toks[0].length, 2u);
1253   EXPECT_EQ(Toks[1].deltaLine, 1u);
1254   EXPECT_EQ(Toks[1].deltaStart, 0u);
1255   EXPECT_EQ(Toks[1].length, 1u);
1256   EXPECT_EQ(Toks[2].deltaLine, 1u);
1257   EXPECT_EQ(Toks[2].deltaStart, 0u);
1258   EXPECT_EQ(Toks[2].length, 1u);
1259
1260   // bar
1261   EXPECT_EQ(Toks[3].deltaLine, 0u);
1262   EXPECT_EQ(Toks[3].deltaStart, 2u);
1263   EXPECT_EQ(Toks[3].length, 3u);
1264 }
1265
1266 TEST(SemanticHighlighting, WithHighlightingFilter) {
1267   llvm::StringRef AnnotatedCode = R"cpp(
1268 int *$Variable[[x]] = new int;
1269 )cpp";
1270   Config Cfg;
1271   Cfg.SemanticTokens.DisabledKinds = {"Operator"};
1272   Cfg.SemanticTokens.DisabledModifiers = {"Declaration", "Definition"};
1273   WithContextValue WithCfg(Config::Key, std::move(Cfg));
1274   checkHighlightings(AnnotatedCode, {}, ~ScopeModifierMask);
1275 }
1276 } // namespace
1277 } // namespace clangd
1278 } // namespace clang