1 //==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Protocol.h"
12 #include "SemanticHighlighting.h"
13 #include "SourceCode.h"
14 #include "TestFS.h"
15 #include "TestTU.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/Support/Error.h"
20 #include "llvm/Support/ScopedPrinter.h"
21 #include "gmock/gmock.h"
22 #include <algorithm>
23
24 namespace clang {
25 namespace clangd {
26 namespace {
27
28 using testing::IsEmpty;
29 using testing::SizeIs;
30
31 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
32 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
33
34 std::vector<HighlightingToken>
makeHighlightingTokens(llvm::ArrayRef<Range> Ranges,HighlightingKind Kind)35 makeHighlightingTokens(llvm::ArrayRef<Range> Ranges, HighlightingKind Kind) {
36 std::vector<HighlightingToken> Tokens(Ranges.size());
37 for (int I = 0, End = Ranges.size(); I < End; ++I) {
38 Tokens[I].R = Ranges[I];
39 Tokens[I].Kind = Kind;
40 }
41
42 return Tokens;
43 }
44
getExpectedTokens(Annotations & Test)45 std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
46 static const std::map<HighlightingKind, std::string> KindToString{
47 {HighlightingKind::Variable, "Variable"},
48 {HighlightingKind::LocalVariable, "LocalVariable"},
49 {HighlightingKind::Parameter, "Parameter"},
50 {HighlightingKind::Function, "Function"},
51 {HighlightingKind::Class, "Class"},
52 {HighlightingKind::Enum, "Enum"},
53 {HighlightingKind::Namespace, "Namespace"},
54 {HighlightingKind::EnumConstant, "EnumConstant"},
55 {HighlightingKind::Field, "Field"},
56 {HighlightingKind::StaticField, "StaticField"},
57 {HighlightingKind::Method, "Method"},
58 {HighlightingKind::StaticMethod, "StaticMethod"},
59 {HighlightingKind::Typedef, "Typedef"},
60 {HighlightingKind::DependentType, "DependentType"},
61 {HighlightingKind::DependentName, "DependentName"},
62 {HighlightingKind::TemplateParameter, "TemplateParameter"},
63 {HighlightingKind::Concept, "Concept"},
64 {HighlightingKind::Primitive, "Primitive"},
65 {HighlightingKind::Macro, "Macro"}};
66 std::vector<HighlightingToken> ExpectedTokens;
67 for (const auto &KindString : KindToString) {
68 std::vector<HighlightingToken> Toks = makeHighlightingTokens(
69 Test.ranges(KindString.second), KindString.first);
70 ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
71 }
72 llvm::sort(ExpectedTokens);
73 return ExpectedTokens;
74 }
75
76 /// Annotates the input code with provided semantic highlightings. Results look
77 /// something like:
78 /// class $Class[[X]] {
79 /// $Primitive[[int]] $Field[[a]] = 0;
80 /// };
annotate(llvm::StringRef Input,llvm::ArrayRef<HighlightingToken> Tokens)81 std::string annotate(llvm::StringRef Input,
82 llvm::ArrayRef<HighlightingToken> Tokens) {
83 assert(std::is_sorted(
84 Tokens.begin(), Tokens.end(),
85 [](const HighlightingToken &L, const HighlightingToken &R) {
86 return L.R.start < R.R.start;
87 }));
88
89 std::string Result;
90 unsigned NextChar = 0;
91 for (auto &T : Tokens) {
92 unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
93 unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end));
94 assert(StartOffset <= EndOffset);
95 assert(NextChar <= StartOffset);
96
97 Result += Input.substr(NextChar, StartOffset - NextChar);
98 Result += std::string(
99 llvm::formatv("${0}[[{1}]]", T.Kind,
100 Input.substr(StartOffset, EndOffset - StartOffset)));
101 NextChar = EndOffset;
102 }
103 Result += Input.substr(NextChar);
104 return Result;
105 }
106
checkHighlightings(llvm::StringRef Code,std::vector<std::pair<llvm::StringRef,llvm::StringRef>> AdditionalFiles={})107 void checkHighlightings(llvm::StringRef Code,
108 std::vector<std::pair</*FileName*/ llvm::StringRef,
109 /*FileContent*/ llvm::StringRef>>
110 AdditionalFiles = {}) {
111 Annotations Test(Code);
112 TestTU TU;
113 TU.Code = std::string(Test.code());
114
115 // FIXME: Auto-completion in a template requires disabling delayed template
116 // parsing.
117 TU.ExtraArgs.push_back("-fno-delayed-template-parsing");
118 TU.ExtraArgs.push_back("-std=c++20");
119
120 for (auto File : AdditionalFiles)
121 TU.AdditionalFiles.insert({File.first, std::string(File.second)});
122 auto AST = TU.build();
123
124 EXPECT_EQ(Code, annotate(Test.code(), getSemanticHighlightings(AST)));
125 }
126
127 // Any annotations in OldCode and NewCode are converted into their corresponding
128 // HighlightingToken. The tokens are diffed against each other. Any lines where
129 // the tokens should diff must be marked with a ^ somewhere on that line in
130 // NewCode. If there are diffs that aren't marked with ^ the test fails. The
131 // test also fails if there are lines marked with ^ that don't differ.
checkDiffedHighlights(llvm::StringRef OldCode,llvm::StringRef NewCode)132 void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
133 Annotations OldTest(OldCode);
134 Annotations NewTest(NewCode);
135 std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
136 std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
137
138 llvm::DenseMap<int, std::vector<HighlightingToken>> ExpectedLines;
139 for (const Position &Point : NewTest.points()) {
140 ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
141 // are inserted on these lines later.
142 }
143 std::vector<LineHighlightings> ExpectedLinePairHighlighting;
144 for (const HighlightingToken &Token : NewTokens) {
145 auto It = ExpectedLines.find(Token.R.start.line);
146 if (It != ExpectedLines.end())
147 It->second.push_back(Token);
148 }
149 for (auto &LineTokens : ExpectedLines)
150 ExpectedLinePairHighlighting.push_back(
151 {LineTokens.first, LineTokens.second, /*IsInactive = */ false});
152
153 std::vector<LineHighlightings> ActualDiffed =
154 diffHighlightings(NewTokens, OldTokens);
155 EXPECT_THAT(ActualDiffed,
156 testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting))
157 << OldCode;
158 }
159
TEST(SemanticHighlighting,GetsCorrectTokens)160 TEST(SemanticHighlighting, GetsCorrectTokens) {
161 const char *TestCases[] = {
162 R"cpp(
163 struct $Class[[AS]] {
164 double $Field[[SomeMember]];
165 };
166 struct {
167 } $Variable[[S]];
168 void $Function[[foo]](int $Parameter[[A]], $Class[[AS]] $Parameter[[As]]) {
169 $Primitive[[auto]] $LocalVariable[[VeryLongVariableName]] = 12312;
170 $Class[[AS]] $LocalVariable[[AA]];
171 $Primitive[[auto]] $LocalVariable[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
172 auto $LocalVariable[[FN]] = [ $LocalVariable[[AA]]](int $Parameter[[A]]) -> void {};
173 $LocalVariable[[FN]](12312);
174 }
175 )cpp",
176 R"cpp(
177 void $Function[[foo]](int);
178 void $Function[[Gah]]();
179 void $Function[[foo]]() {
180 auto $LocalVariable[[Bou]] = $Function[[Gah]];
181 }
182 struct $Class[[A]] {
183 void $Method[[abc]]();
184 };
185 )cpp",
186 R"cpp(
187 namespace $Namespace[[abc]] {
188 template<typename $TemplateParameter[[T]]>
189 struct $Class[[A]] {
190 $TemplateParameter[[T]] $Field[[t]];
191 };
192 }
193 template<typename $TemplateParameter[[T]]>
194 struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
195 typename $TemplateParameter[[T]]::$DependentType[[A]]* $Field[[D]];
196 };
197 $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
198 typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
199 struct $Class[[B]] {
200 $Class[[B]]();
201 ~$Class[[B]]();
202 void operator<<($Class[[B]]);
203 $Class[[AAA]] $Field[[AA]];
204 };
205 $Class[[B]]::$Class[[B]]() {}
206 $Class[[B]]::~$Class[[B]]() {}
207 void $Function[[f]] () {
208 $Class[[B]] $LocalVariable[[BB]] = $Class[[B]]();
209 $LocalVariable[[BB]].~$Class[[B]]();
210 $Class[[B]]();
211 }
212 )cpp",
213 R"cpp(
214 enum class $Enum[[E]] {
215 $EnumConstant[[A]],
216 $EnumConstant[[B]],
217 };
218 enum $Enum[[EE]] {
219 $EnumConstant[[Hi]],
220 };
221 struct $Class[[A]] {
222 $Enum[[E]] $Field[[EEE]];
223 $Enum[[EE]] $Field[[EEEE]];
224 };
225 int $Variable[[I]] = $EnumConstant[[Hi]];
226 $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
227 )cpp",
228 R"cpp(
229 namespace $Namespace[[abc]] {
230 namespace {}
231 namespace $Namespace[[bcd]] {
232 struct $Class[[A]] {};
233 namespace $Namespace[[cde]] {
234 struct $Class[[A]] {
235 enum class $Enum[[B]] {
236 $EnumConstant[[Hi]],
237 };
238 };
239 }
240 }
241 }
242 using namespace $Namespace[[abc]]::$Namespace[[bcd]];
243 namespace $Namespace[[vwz]] =
244 $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
245 $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
246 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
247 $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
248 ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
249 ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
250 )cpp",
251 R"cpp(
252 struct $Class[[D]] {
253 double $Field[[C]];
254 };
255 struct $Class[[A]] {
256 double $Field[[B]];
257 $Class[[D]] $Field[[E]];
258 static double $StaticField[[S]];
259 static void $StaticMethod[[bar]]() {}
260 void $Method[[foo]]() {
261 $Field[[B]] = 123;
262 this->$Field[[B]] = 156;
263 this->$Method[[foo]]();
264 $Method[[foo]]();
265 $StaticMethod[[bar]]();
266 $StaticField[[S]] = 90.1;
267 }
268 };
269 void $Function[[foo]]() {
270 $Class[[A]] $LocalVariable[[AA]];
271 $LocalVariable[[AA]].$Field[[B]] += 2;
272 $LocalVariable[[AA]].$Method[[foo]]();
273 $LocalVariable[[AA]].$Field[[E]].$Field[[C]];
274 $Class[[A]]::$StaticField[[S]] = 90;
275 }
276 )cpp",
277 R"cpp(
278 struct $Class[[AA]] {
279 int $Field[[A]];
280 };
281 int $Variable[[B]];
282 $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
283 )cpp",
284 R"cpp(
285 namespace $Namespace[[a]] {
286 struct $Class[[A]] {};
287 typedef char $Primitive[[C]];
288 }
289 typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
290 using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
291 enum class $Enum[[E]] {};
292 typedef $Enum[[E]] $Enum[[C]];
293 typedef $Enum[[C]] $Enum[[CC]];
294 using $Enum[[CD]] = $Enum[[CC]];
295 $Enum[[CC]] $Function[[f]]($Class[[B]]);
296 $Enum[[CD]] $Function[[f]]($Class[[BB]]);
297 typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
298 typedef float $Primitive[[F]];
299 )cpp",
300 R"cpp(
301 template<typename $TemplateParameter[[T]], typename = void>
302 class $Class[[A]] {
303 $TemplateParameter[[T]] $Field[[AA]];
304 $TemplateParameter[[T]] $Method[[foo]]();
305 };
306 template<class $TemplateParameter[[TT]]>
307 class $Class[[B]] {
308 $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
309 };
310 template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
311 class $Class[[BB]] {};
312 template<class $TemplateParameter[[T]]>
313 class $Class[[BB]]<$TemplateParameter[[T]], int> {};
314 template<class $TemplateParameter[[T]]>
315 class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
316
317 template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
318 $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
319
320 template<typename>
321 class $Class[[Foo]] {};
322
323 template<typename $TemplateParameter[[T]]>
324 void $Function[[foo]]($TemplateParameter[[T]] ...);
325 )cpp",
326 R"cpp(
327 template <class $TemplateParameter[[T]]>
328 struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
329 extern template struct $Class[[Tmpl]]<float>;
330 template struct $Class[[Tmpl]]<double>;
331 )cpp",
332 // This test is to guard against highlightings disappearing when using
333 // conversion operators as their behaviour in the clang AST differ from
334 // other CXXMethodDecls.
335 R"cpp(
336 class $Class[[Foo]] {};
337 struct $Class[[Bar]] {
338 explicit operator $Class[[Foo]]*() const;
339 explicit operator int() const;
340 operator $Class[[Foo]]();
341 };
342 void $Function[[f]]() {
343 $Class[[Bar]] $LocalVariable[[B]];
344 $Class[[Foo]] $LocalVariable[[F]] = $LocalVariable[[B]];
345 $Class[[Foo]] *$LocalVariable[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
346 int $LocalVariable[[I]] = (int)$LocalVariable[[B]];
347 }
348 )cpp",
349 R"cpp(
350 struct $Class[[B]] {};
351 struct $Class[[A]] {
352 $Class[[B]] $Field[[BB]];
353 $Class[[A]] &operator=($Class[[A]] &&$Parameter[[O]]);
354 };
355
356 $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter[[O]]) = default;
357 )cpp",
358 R"cpp(
359 enum $Enum[[En]] {
360 $EnumConstant[[EC]],
361 };
362 class $Class[[Foo]] {};
363 class $Class[[Bar]] {
364 public:
365 $Class[[Foo]] $Field[[Fo]];
366 $Enum[[En]] $Field[[E]];
367 int $Field[[I]];
368 $Class[[Bar]] ($Class[[Foo]] $Parameter[[F]],
369 $Enum[[En]] $Parameter[[E]])
370 : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
371 $Field[[I]] (123) {}
372 };
373 class $Class[[Bar2]] : public $Class[[Bar]] {
374 $Class[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant[[EC]]) {}
375 };
376 )cpp",
377 R"cpp(
378 enum $Enum[[E]] {
379 $EnumConstant[[E]],
380 };
381 class $Class[[Foo]] {};
382 $Enum[[auto]] $Variable[[AE]] = $Enum[[E]]::$EnumConstant[[E]];
383 $Class[[auto]] $Variable[[AF]] = $Class[[Foo]]();
384 $Class[[decltype]](auto) $Variable[[AF2]] = $Class[[Foo]]();
385 $Class[[auto]] *$Variable[[AFP]] = &$Variable[[AF]];
386 $Enum[[auto]] &$Variable[[AER]] = $Variable[[AE]];
387 $Primitive[[auto]] $Variable[[Form]] = 10.2 + 2 * 4;
388 $Primitive[[decltype]]($Variable[[Form]]) $Variable[[F]] = 10;
389 auto $Variable[[Fun]] = []()->void{};
390 )cpp",
391 R"cpp(
392 class $Class[[G]] {};
393 template<$Class[[G]] *$TemplateParameter[[U]]>
394 class $Class[[GP]] {};
395 template<$Class[[G]] &$TemplateParameter[[U]]>
396 class $Class[[GR]] {};
397 template<int *$TemplateParameter[[U]]>
398 class $Class[[IP]] {
399 void $Method[[f]]() {
400 *$TemplateParameter[[U]] += 5;
401 }
402 };
403 template<unsigned $TemplateParameter[[U]] = 2>
404 class $Class[[Foo]] {
405 void $Method[[f]]() {
406 for(int $LocalVariable[[I]] = 0;
407 $LocalVariable[[I]] < $TemplateParameter[[U]];) {}
408 }
409 };
410
411 $Class[[G]] $Variable[[L]];
412 void $Function[[f]]() {
413 $Class[[Foo]]<123> $LocalVariable[[F]];
414 $Class[[GP]]<&$Variable[[L]]> $LocalVariable[[LL]];
415 $Class[[GR]]<$Variable[[L]]> $LocalVariable[[LLL]];
416 }
417 )cpp",
418 R"cpp(
419 template<typename $TemplateParameter[[T]],
420 void ($TemplateParameter[[T]]::*$TemplateParameter[[method]])(int)>
421 struct $Class[[G]] {
422 void $Method[[foo]](
423 $TemplateParameter[[T]] *$Parameter[[O]]) {
424 ($Parameter[[O]]->*$TemplateParameter[[method]])(10);
425 }
426 };
427 struct $Class[[F]] {
428 void $Method[[f]](int);
429 };
430 template<void (*$TemplateParameter[[Func]])()>
431 struct $Class[[A]] {
432 void $Method[[f]]() {
433 (*$TemplateParameter[[Func]])();
434 }
435 };
436
437 void $Function[[foo]]() {
438 $Class[[F]] $LocalVariable[[FF]];
439 $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable[[GG]];
440 $LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
441 $Class[[A]]<$Function[[foo]]> $LocalVariable[[AA]];
442 }
443 )cpp",
444 // Tokens that share a source range but have conflicting Kinds are not
445 // highlighted.
446 R"cpp(
447 #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
448 #define $Macro[[DEF_CLASS]](T) class T {};
449 // Preamble ends.
450 $Macro[[DEF_MULTIPLE]](XYZ);
451 $Macro[[DEF_MULTIPLE]](XYZW);
452 $Macro[[DEF_CLASS]]($Class[[A]])
453 #define $Macro[[MACRO_CONCAT]](X, V, T) T foo##X = V
454 #define $Macro[[DEF_VAR]](X, V) int X = V
455 #define $Macro[[DEF_VAR_T]](T, X, V) T X = V
456 #define $Macro[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
457 #define $Macro[[CPY]](X) X
458 #define $Macro[[DEF_VAR_TYPE]](X, Y) X Y
459 #define $Macro[[SOME_NAME]] variable
460 #define $Macro[[SOME_NAME_SET]] variable2 = 123
461 #define $Macro[[INC_VAR]](X) X += 2
462 void $Function[[foo]]() {
463 $Macro[[DEF_VAR]]($LocalVariable[[X]], 123);
464 $Macro[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
465 int $Macro[[CPY]]( $LocalVariable[[XX]] );
466 $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
467 double $Macro[[SOME_NAME]];
468 int $Macro[[SOME_NAME_SET]];
469 $LocalVariable[[variable]] = 20.1;
470 $Macro[[MACRO_CONCAT]](var, 2, float);
471 $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
472 $Macro[[CPY]]($LocalVariable[[Nested]])),
473 $Macro[[CPY]]($Class[[A]]()));
474 $Macro[[INC_VAR]]($LocalVariable[[variable]]);
475 }
476 void $Macro[[SOME_NAME]]();
477 $Macro[[DEF_VAR]]($Variable[[MMMMM]], 567);
478 $Macro[[DEF_VAR_REV]](756, $Variable[[AB]]);
479
480 #define $Macro[[CALL_FN]](F) F();
481 #define $Macro[[DEF_FN]](F) void F ()
482 $Macro[[DEF_FN]]($Function[[g]]) {
483 $Macro[[CALL_FN]]($Function[[foo]]);
484 }
485 )cpp",
486 R"cpp(
487 #define $Macro[[fail]](expr) expr
488 #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
489 // Preamble ends.
490 int $Variable[[x]];
491 int $Variable[[y]];
492 int $Function[[f]]();
493 void $Function[[foo]]() {
494 $Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
495 $Macro[[assert]]($Variable[[x]] != $Function[[f]]());
496 }
497 )cpp",
498 // highlighting all macro references
499 R"cpp(
500 #ifndef $Macro[[name]]
501 #define $Macro[[name]]
502 #endif
503
504 #define $Macro[[test]]
505 #undef $Macro[[test]]
506 $InactiveCode[[#ifdef test]]
507 $InactiveCode[[#endif]]
508
509 $InactiveCode[[#if defined(test)]]
510 $InactiveCode[[#endif]]
511 )cpp",
512 R"cpp(
513 struct $Class[[S]] {
514 float $Field[[Value]];
515 $Class[[S]] *$Field[[Next]];
516 };
517 $Class[[S]] $Variable[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
518 auto [$Variable[[G1]], $Variable[[G2]]] = $Variable[[Global]];
519 void $Function[[f]]($Class[[S]] $Parameter[[P]]) {
520 int $LocalVariable[[A]][2] = {1,2};
521 auto [$LocalVariable[[B1]], $LocalVariable[[B2]]] = $LocalVariable[[A]];
522 auto [$LocalVariable[[G1]], $LocalVariable[[G2]]] = $Variable[[Global]];
523 $Class[[auto]] [$LocalVariable[[P1]], $LocalVariable[[P2]]] = $Parameter[[P]];
524 // Highlights references to BindingDecls.
525 $LocalVariable[[B1]]++;
526 }
527 )cpp",
528 R"cpp(
529 template<class $TemplateParameter[[T]]>
530 class $Class[[A]] {
531 using $TemplateParameter[[TemplateParam1]] = $TemplateParameter[[T]];
532 typedef $TemplateParameter[[T]] $TemplateParameter[[TemplateParam2]];
533 using $Primitive[[IntType]] = int;
534
535 using $Typedef[[Pointer]] = $TemplateParameter[[T]] *;
536 using $Typedef[[LVReference]] = $TemplateParameter[[T]] &;
537 using $Typedef[[RVReference]] = $TemplateParameter[[T]]&&;
538 using $Typedef[[Array]] = $TemplateParameter[[T]]*[3];
539 using $Typedef[[MemberPointer]] = int ($Class[[A]]::*)(int);
540
541 // Use various previously defined typedefs in a function type.
542 void $Method[[func]](
543 $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
544 $Typedef[[Array]], $Typedef[[MemberPointer]]);
545 };
546 )cpp",
547 R"cpp(
548 template <class $TemplateParameter[[T]]>
549 void $Function[[phase1]]($TemplateParameter[[T]]);
550 template <class $TemplateParameter[[T]]>
551 void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
552 $Function[[phase1]]($Parameter[[P]]);
553 $DependentName[[phase2]]($Parameter[[P]]);
554 }
555 )cpp",
556 R"cpp(
557 class $Class[[A]] {
558 template <class $TemplateParameter[[T]]>
559 void $Method[[bar]]($TemplateParameter[[T]]);
560 };
561
562 template <class $TemplateParameter[[U]]>
563 void $Function[[foo]]($TemplateParameter[[U]] $Parameter[[P]]) {
564 $Class[[A]]().$Method[[bar]]($Parameter[[P]]);
565 }
566 )cpp",
567 R"cpp(
568 struct $Class[[A]] {
569 template <class $TemplateParameter[[T]]>
570 static void $StaticMethod[[foo]]($TemplateParameter[[T]]);
571 };
572
573 template <class $TemplateParameter[[T]]>
574 struct $Class[[B]] {
575 void $Method[[bar]]() {
576 $Class[[A]]::$StaticMethod[[foo]]($TemplateParameter[[T]]());
577 }
578 };
579 )cpp",
580 R"cpp(
581 template <class $TemplateParameter[[T]]>
582 void $Function[[foo]](typename $TemplateParameter[[T]]::$DependentType[[Type]]
583 = $TemplateParameter[[T]]::$DependentName[[val]]);
584 )cpp",
585 R"cpp(
586 template <class $TemplateParameter[[T]]>
587 void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
588 $Parameter[[P]].$DependentName[[Field]];
589 }
590 )cpp",
591 R"cpp(
592 template <class $TemplateParameter[[T]]>
593 class $Class[[A]] {
594 int $Method[[foo]]() {
595 return $TemplateParameter[[T]]::$DependentName[[Field]];
596 }
597 };
598 )cpp",
599 // Highlighting the using decl as the underlying using shadow decl.
600 R"cpp(
601 void $Function[[foo]]();
602 using ::$Function[[foo]];
603 )cpp",
604 // Highlighting of template template arguments.
605 R"cpp(
606 template <template <class> class $TemplateParameter[[TT]],
607 template <class> class ...$TemplateParameter[[TTs]]>
608 struct $Class[[Foo]] {
609 $Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
610 *$Field[[t]];
611 };
612 )cpp",
613 // Inactive code highlighting
614 R"cpp(
615 // Code in the preamble.
616 // Inactive lines get an empty InactiveCode token at the beginning.
617 $InactiveCode[[#ifdef test]]
618 $InactiveCode[[#endif]]
619
620 // A declaration to cause the preamble to end.
621 int $Variable[[EndPreamble]];
622
623 // Code after the preamble.
624 // Code inside inactive blocks does not get regular highlightings
625 // because it's not part of the AST.
626 #define $Macro[[test2]]
627 $InactiveCode[[#if defined(test)]]
628 $InactiveCode[[int Inactive2;]]
629 $InactiveCode[[#elif defined(test2)]]
630 int $Variable[[Active1]];
631 $InactiveCode[[#else]]
632 $InactiveCode[[int Inactive3;]]
633 $InactiveCode[[#endif]]
634
635 #ifndef $Macro[[test]]
636 int $Variable[[Active2]];
637 #endif
638
639 $InactiveCode[[#ifdef test]]
640 $InactiveCode[[int Inactive4;]]
641 $InactiveCode[[#else]]
642 int $Variable[[Active3]];
643 #endif
644 )cpp",
645 // Argument to 'sizeof...'
646 R"cpp(
647 template <typename... $TemplateParameter[[Elements]]>
648 struct $Class[[TupleSize]] {
649 static const int $StaticField[[size]] =
650 sizeof...($TemplateParameter[[Elements]]);
651 };
652 )cpp",
653 // More dependent types
654 R"cpp(
655 template <typename $TemplateParameter[[T]]>
656 struct $Class[[Waldo]] {
657 using $Typedef[[Location1]] = typename $TemplateParameter[[T]]
658 ::$DependentType[[Resolver]]::$DependentType[[Location]];
659 using $Typedef[[Location2]] = typename $TemplateParameter[[T]]
660 ::template $DependentType[[Resolver]]<$TemplateParameter[[T]]>
661 ::$DependentType[[Location]];
662 using $Typedef[[Location3]] = typename $TemplateParameter[[T]]
663 ::$DependentType[[Resolver]]
664 ::template $DependentType[[Location]]<$TemplateParameter[[T]]>;
665 static const int $StaticField[[Value]] = $TemplateParameter[[T]]
666 ::$DependentType[[Resolver]]::$DependentName[[Value]];
667 };
668 )cpp",
669 // Dependent name with heuristic target
670 R"cpp(
671 template <typename>
672 struct $Class[[Foo]] {
673 int $Field[[Waldo]];
674 void $Method[[bar]]() {
675 $Class[[Foo]]().$Field[[Waldo]];
676 }
677 template <typename $TemplateParameter[[U]]>
678 void $Method[[bar1]]() {
679 $Class[[Foo]]<$TemplateParameter[[U]]>().$Field[[Waldo]];
680 }
681 };
682 )cpp",
683 // Concepts
684 R"cpp(
685 template <typename $TemplateParameter[[T]]>
686 concept $Concept[[Fooable]] =
687 requires($TemplateParameter[[T]] $Parameter[[F]]) {
688 $Parameter[[F]].$DependentName[[foo]]();
689 };
690 template <typename $TemplateParameter[[T]]>
691 requires $Concept[[Fooable]]<$TemplateParameter[[T]]>
692 void $Function[[bar]]($TemplateParameter[[T]] $Parameter[[F]]) {
693 $Parameter[[F]].$DependentName[[foo]]();
694 }
695 )cpp",
696 // Dependent template name
697 R"cpp(
698 template <template <typename> class> struct $Class[[A]] {};
699 template <typename $TemplateParameter[[T]]>
700 using $Typedef[[W]] = $Class[[A]]<
701 $TemplateParameter[[T]]::template $DependentType[[Waldo]]
702 >;
703 )cpp"};
704 for (const auto &TestCase : TestCases) {
705 checkHighlightings(TestCase);
706 }
707
708 checkHighlightings(R"cpp(
709 class $Class[[A]] {
710 #include "imp.h"
711 };
712 )cpp",
713 {{"imp.h", R"cpp(
714 int someMethod();
715 void otherMethod();
716 )cpp"}});
717
718 // A separate test for macros in headers.
719 checkHighlightings(R"cpp(
720 #include "imp.h"
721 $Macro[[DEFINE_Y]]
722 $Macro[[DXYZ_Y]](A);
723 )cpp",
724 {{"imp.h", R"cpp(
725 #define DXYZ(X) class X {};
726 #define DXYZ_Y(Y) DXYZ(x##Y)
727 #define DEFINE(X) int X;
728 #define DEFINE_Y DEFINE(Y)
729 )cpp"}});
730 }
731
TEST(SemanticHighlighting,GeneratesHighlightsWhenFileChange)732 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
733 class HighlightingsCounter : public ClangdServer::Callbacks {
734 public:
735 std::atomic<int> Count = {0};
736
737 void onHighlightingsReady(
738 PathRef File, llvm::StringRef Version,
739 std::vector<HighlightingToken> Highlightings) override {
740 ++Count;
741 }
742 };
743
744 auto FooCpp = testPath("foo.cpp");
745 MockFS FS;
746 FS.Files[FooCpp] = "";
747
748 MockCompilationDatabase MCD;
749 HighlightingsCounter Counter;
750 ClangdServer Server(MCD, FS, ClangdServer::optsForTest(), &Counter);
751 Server.addDocument(FooCpp, "int a;");
752 ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for server";
753 ASSERT_EQ(Counter.Count, 1);
754 }
755
756 // Ranges are highlighted as variables, unless highlighted as $Function etc.
tokens(llvm::StringRef MarkedText)757 std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
758 Annotations A(MarkedText);
759 std::vector<HighlightingToken> Results;
760 for (const Range& R : A.ranges())
761 Results.push_back({HighlightingKind::Variable, R});
762 for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
763 HighlightingKind Kind = static_cast<HighlightingKind>(I);
764 for (const Range& R : A.ranges(llvm::to_string(Kind)))
765 Results.push_back({Kind, R});
766 }
767 llvm::sort(Results);
768 return Results;
769 }
770
TEST(SemanticHighlighting,toSemanticTokens)771 TEST(SemanticHighlighting, toSemanticTokens) {
772 auto Results = toSemanticTokens(tokens(R"(
773 [[blah]]
774
775 $Function[[big]] [[bang]]
776 )"));
777
778 ASSERT_THAT(Results, SizeIs(3));
779 EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
780 EXPECT_EQ(Results[0].deltaLine, 1u);
781 EXPECT_EQ(Results[0].deltaStart, 1u);
782 EXPECT_EQ(Results[0].length, 4u);
783
784 EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
785 EXPECT_EQ(Results[1].deltaLine, 2u);
786 EXPECT_EQ(Results[1].deltaStart, 4u);
787 EXPECT_EQ(Results[1].length, 3u);
788
789 EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
790 EXPECT_EQ(Results[2].deltaLine, 0u);
791 EXPECT_EQ(Results[2].deltaStart, 4u);
792 EXPECT_EQ(Results[2].length, 4u);
793 }
794
TEST(SemanticHighlighting,diffSemanticTokens)795 TEST(SemanticHighlighting, diffSemanticTokens) {
796 auto Before = toSemanticTokens(tokens(R"(
797 [[foo]] [[bar]] [[baz]]
798 [[one]] [[two]] [[three]]
799 )"));
800 EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
801
802 auto After = toSemanticTokens(tokens(R"(
803 [[foo]] [[hello]] [[world]] [[baz]]
804 [[one]] [[two]] [[three]]
805 )"));
806
807 // Replace [bar, baz] with [hello, world, baz]
808 auto Diff = diffTokens(Before, After);
809 ASSERT_THAT(Diff, SizeIs(1));
810 EXPECT_EQ(1u, Diff.front().startToken);
811 EXPECT_EQ(2u, Diff.front().deleteTokens);
812 ASSERT_THAT(Diff.front().tokens, SizeIs(3));
813 // hello
814 EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
815 EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
816 EXPECT_EQ(5u, Diff.front().tokens[0].length);
817 // world
818 EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
819 EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
820 EXPECT_EQ(5u, Diff.front().tokens[1].length);
821 // baz
822 EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
823 EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
824 EXPECT_EQ(3u, Diff.front().tokens[2].length);
825 }
826
TEST(SemanticHighlighting,toTheiaSemanticHighlightingInformation)827 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
828 auto CreatePosition = [](int Line, int Character) -> Position {
829 Position Pos;
830 Pos.line = Line;
831 Pos.character = Character;
832 return Pos;
833 };
834
835 std::vector<LineHighlightings> Tokens{
836 {3,
837 {{HighlightingKind::Variable,
838 Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
839 {HighlightingKind::Function,
840 Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
841 /* IsInactive = */ false},
842 {1,
843 {{HighlightingKind::Variable,
844 Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
845 /* IsInactive = */ true}};
846 std::vector<TheiaSemanticHighlightingInformation> ActualResults =
847 toTheiaSemanticHighlightingInformation(Tokens);
848 std::vector<TheiaSemanticHighlightingInformation> ExpectedResults = {
849 {3, "AAAACAAEAAAAAAAEAAMAAw=="}, {1, "AAAAAQAEAAA="}};
850 EXPECT_EQ(ActualResults, ExpectedResults);
851 }
852
TEST(SemanticHighlighting,HighlightingDiffer)853 TEST(SemanticHighlighting, HighlightingDiffer) {
854 struct {
855 llvm::StringRef OldCode;
856 llvm::StringRef NewCode;
857 } TestCases[]{{
858 R"(
859 $Variable[[A]]
860 $Class[[B]]
861 $Function[[C]]
862 )",
863 R"(
864 $Variable[[A]]
865 $Class[[D]]
866 $Function[[C]]
867 )"},
868 {
869 R"(
870 $Class[[C]]
871 $Field[[F]]
872 $Variable[[V]]
873 $Class[[C]] $Variable[[V]] $Field[[F]]
874 )",
875 R"(
876 $Class[[C]]
877 $Field[[F]]
878 ^$Function[[F]]
879 $Class[[C]] $Variable[[V]] $Field[[F]]
880 )"},
881 {
882 R"(
883
884 $Class[[A]]
885 $Variable[[A]]
886 )",
887 R"(
888
889 ^
890 ^$Class[[A]]
891 ^$Variable[[A]]
892 )"},
893 {
894 R"(
895 $Class[[C]]
896 $Field[[F]]
897 $Variable[[V]]
898 $Class[[C]] $Variable[[V]] $Field[[F]]
899 )",
900 R"(
901 $Class[[C]]
902 ^
903 ^
904 $Class[[C]] $Variable[[V]] $Field[[F]]
905 )"},
906 {
907 R"(
908 $Class[[A]]
909 $Variable[[A]]
910 $Variable[[A]]
911 )",
912 R"(
913 $Class[[A]]
914 ^$Variable[[AA]]
915 $Variable[[A]]
916 )"},
917 {
918 R"(
919 $Class[[A]]
920 $Variable[[A]]
921 )",
922 R"(
923 $Class[[A]]
924 $Variable[[A]]
925 ^$Class[[A]]
926 ^$Variable[[A]]
927 )"},
928 {
929 R"(
930 $Variable[[A]]
931 $Variable[[A]]
932 $Variable[[A]]
933 )",
934 R"(
935 ^$Class[[A]]
936 ^$Class[[A]]
937 ^$Class[[A]]
938 )"}};
939
940 for (const auto &Test : TestCases)
941 checkDiffedHighlights(Test.OldCode, Test.NewCode);
942 }
943
TEST(SemanticHighlighting,DiffBeyondTheEndOfFile)944 TEST(SemanticHighlighting, DiffBeyondTheEndOfFile) {
945 llvm::StringRef OldCode =
946 R"(
947 $Class[[A]]
948 $Variable[[A]]
949 $Class[[A]]
950 $Variable[[A]]
951 )";
952 llvm::StringRef NewCode =
953 R"(
954 $Class[[A]] // line 1
955 $Variable[[A]] // line 2
956 )";
957
958 Annotations OldTest(OldCode);
959 Annotations NewTest(NewCode);
960 std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
961 std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
962
963 auto ActualDiff = diffHighlightings(NewTokens, OldTokens);
964 EXPECT_THAT(ActualDiff,
965 testing::UnorderedElementsAre(
966 testing::AllOf(LineNumber(3), EmptyHighlightings()),
967 testing::AllOf(LineNumber(4), EmptyHighlightings())));
968 }
969
970 } // namespace
971 } // namespace clangd
972 } // namespace clang
973