1 //===--- CommentLexer.cpp -------------------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "clang/AST/CommentLexer.h"
11 #include "clang/AST/CommentCommandTraits.h"
12 #include "clang/AST/CommentDiagnostic.h"
13 #include "clang/Basic/CharInfo.h"
14 #include "llvm/ADT/StringExtras.h"
15 #include "llvm/ADT/StringSwitch.h"
16 #include "llvm/Support/ConvertUTF.h"
17 #include "llvm/Support/ErrorHandling.h"
18
19 namespace clang {
20 namespace comments {
21
dump(const Lexer & L,const SourceManager & SM) const22 void Token::dump(const Lexer &L, const SourceManager &SM) const {
23 llvm::errs() << "comments::Token Kind=" << Kind << " ";
24 Loc.dump(SM);
25 llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
26 }
27
isHTMLNamedCharacterReferenceCharacter(char C)28 static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
29 return isLetter(C);
30 }
31
isHTMLDecimalCharacterReferenceCharacter(char C)32 static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
33 return isDigit(C);
34 }
35
isHTMLHexCharacterReferenceCharacter(char C)36 static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
37 return isHexDigit(C);
38 }
39
convertCodePointToUTF8(llvm::BumpPtrAllocator & Allocator,unsigned CodePoint)40 static inline StringRef convertCodePointToUTF8(
41 llvm::BumpPtrAllocator &Allocator,
42 unsigned CodePoint) {
43 char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
44 char *ResolvedPtr = Resolved;
45 if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
46 return StringRef(Resolved, ResolvedPtr - Resolved);
47 else
48 return StringRef();
49 }
50
51 namespace {
52
53 #include "clang/AST/CommentHTMLTags.inc"
54 #include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
55
56 } // end anonymous namespace
57
resolveHTMLNamedCharacterReference(StringRef Name) const58 StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
59 // Fast path, first check a few most widely used named character references.
60 return llvm::StringSwitch<StringRef>(Name)
61 .Case("amp", "&")
62 .Case("lt", "<")
63 .Case("gt", ">")
64 .Case("quot", "\"")
65 .Case("apos", "\'")
66 // Slow path.
67 .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
68 }
69
resolveHTMLDecimalCharacterReference(StringRef Name) const70 StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
71 unsigned CodePoint = 0;
72 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
73 assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
74 CodePoint *= 10;
75 CodePoint += Name[i] - '0';
76 }
77 return convertCodePointToUTF8(Allocator, CodePoint);
78 }
79
resolveHTMLHexCharacterReference(StringRef Name) const80 StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
81 unsigned CodePoint = 0;
82 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
83 CodePoint *= 16;
84 const char C = Name[i];
85 assert(isHTMLHexCharacterReferenceCharacter(C));
86 CodePoint += llvm::hexDigitValue(C);
87 }
88 return convertCodePointToUTF8(Allocator, CodePoint);
89 }
90
skipLineStartingDecorations()91 void Lexer::skipLineStartingDecorations() {
92 // This function should be called only for C comments
93 assert(CommentState == LCS_InsideCComment);
94
95 if (BufferPtr == CommentEnd)
96 return;
97
98 switch (*BufferPtr) {
99 case ' ':
100 case '\t':
101 case '\f':
102 case '\v': {
103 const char *NewBufferPtr = BufferPtr;
104 NewBufferPtr++;
105 if (NewBufferPtr == CommentEnd)
106 return;
107
108 char C = *NewBufferPtr;
109 while (isHorizontalWhitespace(C)) {
110 NewBufferPtr++;
111 if (NewBufferPtr == CommentEnd)
112 return;
113 C = *NewBufferPtr;
114 }
115 if (C == '*')
116 BufferPtr = NewBufferPtr + 1;
117 break;
118 }
119 case '*':
120 BufferPtr++;
121 break;
122 }
123 }
124
125 namespace {
126 /// Returns pointer to the first newline character in the string.
findNewline(const char * BufferPtr,const char * BufferEnd)127 const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
128 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
129 if (isVerticalWhitespace(*BufferPtr))
130 return BufferPtr;
131 }
132 return BufferEnd;
133 }
134
skipNewline(const char * BufferPtr,const char * BufferEnd)135 const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
136 if (BufferPtr == BufferEnd)
137 return BufferPtr;
138
139 if (*BufferPtr == '\n')
140 BufferPtr++;
141 else {
142 assert(*BufferPtr == '\r');
143 BufferPtr++;
144 if (BufferPtr != BufferEnd && *BufferPtr == '\n')
145 BufferPtr++;
146 }
147 return BufferPtr;
148 }
149
skipNamedCharacterReference(const char * BufferPtr,const char * BufferEnd)150 const char *skipNamedCharacterReference(const char *BufferPtr,
151 const char *BufferEnd) {
152 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
153 if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
154 return BufferPtr;
155 }
156 return BufferEnd;
157 }
158
skipDecimalCharacterReference(const char * BufferPtr,const char * BufferEnd)159 const char *skipDecimalCharacterReference(const char *BufferPtr,
160 const char *BufferEnd) {
161 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
162 if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
163 return BufferPtr;
164 }
165 return BufferEnd;
166 }
167
skipHexCharacterReference(const char * BufferPtr,const char * BufferEnd)168 const char *skipHexCharacterReference(const char *BufferPtr,
169 const char *BufferEnd) {
170 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
171 if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
172 return BufferPtr;
173 }
174 return BufferEnd;
175 }
176
isHTMLIdentifierStartingCharacter(char C)177 bool isHTMLIdentifierStartingCharacter(char C) {
178 return isLetter(C);
179 }
180
isHTMLIdentifierCharacter(char C)181 bool isHTMLIdentifierCharacter(char C) {
182 return isAlphanumeric(C);
183 }
184
skipHTMLIdentifier(const char * BufferPtr,const char * BufferEnd)185 const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
186 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
187 if (!isHTMLIdentifierCharacter(*BufferPtr))
188 return BufferPtr;
189 }
190 return BufferEnd;
191 }
192
193 /// Skip HTML string quoted in single or double quotes. Escaping quotes inside
194 /// string allowed.
195 ///
196 /// Returns pointer to closing quote.
skipHTMLQuotedString(const char * BufferPtr,const char * BufferEnd)197 const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
198 {
199 const char Quote = *BufferPtr;
200 assert(Quote == '\"' || Quote == '\'');
201
202 BufferPtr++;
203 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
204 const char C = *BufferPtr;
205 if (C == Quote && BufferPtr[-1] != '\\')
206 return BufferPtr;
207 }
208 return BufferEnd;
209 }
210
skipWhitespace(const char * BufferPtr,const char * BufferEnd)211 const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
212 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
213 if (!isWhitespace(*BufferPtr))
214 return BufferPtr;
215 }
216 return BufferEnd;
217 }
218
isWhitespace(const char * BufferPtr,const char * BufferEnd)219 bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
220 return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
221 }
222
isCommandNameStartCharacter(char C)223 bool isCommandNameStartCharacter(char C) {
224 return isLetter(C);
225 }
226
isCommandNameCharacter(char C)227 bool isCommandNameCharacter(char C) {
228 return isAlphanumeric(C);
229 }
230
skipCommandName(const char * BufferPtr,const char * BufferEnd)231 const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
232 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
233 if (!isCommandNameCharacter(*BufferPtr))
234 return BufferPtr;
235 }
236 return BufferEnd;
237 }
238
239 /// Return the one past end pointer for BCPL comments.
240 /// Handles newlines escaped with backslash or trigraph for backslahs.
findBCPLCommentEnd(const char * BufferPtr,const char * BufferEnd)241 const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
242 const char *CurPtr = BufferPtr;
243 while (CurPtr != BufferEnd) {
244 while (!isVerticalWhitespace(*CurPtr)) {
245 CurPtr++;
246 if (CurPtr == BufferEnd)
247 return BufferEnd;
248 }
249 // We found a newline, check if it is escaped.
250 const char *EscapePtr = CurPtr - 1;
251 while(isHorizontalWhitespace(*EscapePtr))
252 EscapePtr--;
253
254 if (*EscapePtr == '\\' ||
255 (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
256 EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
257 // We found an escaped newline.
258 CurPtr = skipNewline(CurPtr, BufferEnd);
259 } else
260 return CurPtr; // Not an escaped newline.
261 }
262 return BufferEnd;
263 }
264
265 /// Return the one past end pointer for C comments.
266 /// Very dumb, does not handle escaped newlines or trigraphs.
findCCommentEnd(const char * BufferPtr,const char * BufferEnd)267 const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
268 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
269 if (*BufferPtr == '*') {
270 assert(BufferPtr + 1 != BufferEnd);
271 if (*(BufferPtr + 1) == '/')
272 return BufferPtr;
273 }
274 }
275 llvm_unreachable("buffer end hit before '*/' was seen");
276 }
277
278 } // end anonymous namespace
279
formTokenWithChars(Token & Result,const char * TokEnd,tok::TokenKind Kind)280 void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
281 tok::TokenKind Kind) {
282 const unsigned TokLen = TokEnd - BufferPtr;
283 Result.setLocation(getSourceLocation(BufferPtr));
284 Result.setKind(Kind);
285 Result.setLength(TokLen);
286 #ifndef NDEBUG
287 Result.TextPtr = "<UNSET>";
288 Result.IntVal = 7;
289 #endif
290 BufferPtr = TokEnd;
291 }
292
lexCommentText(Token & T)293 void Lexer::lexCommentText(Token &T) {
294 assert(CommentState == LCS_InsideBCPLComment ||
295 CommentState == LCS_InsideCComment);
296
297 switch (State) {
298 case LS_Normal:
299 break;
300 case LS_VerbatimBlockFirstLine:
301 lexVerbatimBlockFirstLine(T);
302 return;
303 case LS_VerbatimBlockBody:
304 lexVerbatimBlockBody(T);
305 return;
306 case LS_VerbatimLineText:
307 lexVerbatimLineText(T);
308 return;
309 case LS_HTMLStartTag:
310 lexHTMLStartTag(T);
311 return;
312 case LS_HTMLEndTag:
313 lexHTMLEndTag(T);
314 return;
315 }
316
317 assert(State == LS_Normal);
318
319 const char *TokenPtr = BufferPtr;
320 assert(TokenPtr < CommentEnd);
321 while (TokenPtr != CommentEnd) {
322 switch(*TokenPtr) {
323 case '\\':
324 case '@': {
325 // Commands that start with a backslash and commands that start with
326 // 'at' have equivalent semantics. But we keep information about the
327 // exact syntax in AST for comments.
328 tok::TokenKind CommandKind =
329 (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
330 TokenPtr++;
331 if (TokenPtr == CommentEnd) {
332 formTextToken(T, TokenPtr);
333 return;
334 }
335 char C = *TokenPtr;
336 switch (C) {
337 default:
338 break;
339
340 case '\\': case '@': case '&': case '$':
341 case '#': case '<': case '>': case '%':
342 case '\"': case '.': case ':':
343 // This is one of \\ \@ \& \$ etc escape sequences.
344 TokenPtr++;
345 if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
346 // This is the \:: escape sequence.
347 TokenPtr++;
348 }
349 StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
350 formTokenWithChars(T, TokenPtr, tok::text);
351 T.setText(UnescapedText);
352 return;
353 }
354
355 // Don't make zero-length commands.
356 if (!isCommandNameStartCharacter(*TokenPtr)) {
357 formTextToken(T, TokenPtr);
358 return;
359 }
360
361 TokenPtr = skipCommandName(TokenPtr, CommentEnd);
362 unsigned Length = TokenPtr - (BufferPtr + 1);
363
364 // Hardcoded support for lexing LaTeX formula commands
365 // \f$ \f[ \f] \f{ \f} as a single command.
366 if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
367 C = *TokenPtr;
368 if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
369 TokenPtr++;
370 Length++;
371 }
372 }
373
374 StringRef CommandName(BufferPtr + 1, Length);
375
376 const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
377 if (!Info) {
378 if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
379 StringRef CorrectedName = Info->Name;
380 SourceLocation Loc = getSourceLocation(BufferPtr);
381 SourceRange CommandRange(Loc.getLocWithOffset(1),
382 getSourceLocation(TokenPtr));
383 Diag(Loc, diag::warn_correct_comment_command_name)
384 << CommandName << CorrectedName
385 << FixItHint::CreateReplacement(CommandRange, CorrectedName);
386 } else {
387 formTokenWithChars(T, TokenPtr, tok::unknown_command);
388 T.setUnknownCommandName(CommandName);
389 Diag(T.getLocation(), diag::warn_unknown_comment_command_name);
390 return;
391 }
392 }
393 if (Info->IsVerbatimBlockCommand) {
394 setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
395 return;
396 }
397 if (Info->IsVerbatimLineCommand) {
398 setupAndLexVerbatimLine(T, TokenPtr, Info);
399 return;
400 }
401 formTokenWithChars(T, TokenPtr, CommandKind);
402 T.setCommandID(Info->getID());
403 return;
404 }
405
406 case '&':
407 lexHTMLCharacterReference(T);
408 return;
409
410 case '<': {
411 TokenPtr++;
412 if (TokenPtr == CommentEnd) {
413 formTextToken(T, TokenPtr);
414 return;
415 }
416 const char C = *TokenPtr;
417 if (isHTMLIdentifierStartingCharacter(C))
418 setupAndLexHTMLStartTag(T);
419 else if (C == '/')
420 setupAndLexHTMLEndTag(T);
421 else
422 formTextToken(T, TokenPtr);
423 return;
424 }
425
426 case '\n':
427 case '\r':
428 TokenPtr = skipNewline(TokenPtr, CommentEnd);
429 formTokenWithChars(T, TokenPtr, tok::newline);
430
431 if (CommentState == LCS_InsideCComment)
432 skipLineStartingDecorations();
433 return;
434
435 default: {
436 size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr).
437 find_first_of("\n\r\\@&<");
438 if (End != StringRef::npos)
439 TokenPtr += End;
440 else
441 TokenPtr = CommentEnd;
442 formTextToken(T, TokenPtr);
443 return;
444 }
445 }
446 }
447 }
448
setupAndLexVerbatimBlock(Token & T,const char * TextBegin,char Marker,const CommandInfo * Info)449 void Lexer::setupAndLexVerbatimBlock(Token &T,
450 const char *TextBegin,
451 char Marker, const CommandInfo *Info) {
452 assert(Info->IsVerbatimBlockCommand);
453
454 VerbatimBlockEndCommandName.clear();
455 VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
456 VerbatimBlockEndCommandName.append(Info->EndCommandName);
457
458 formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
459 T.setVerbatimBlockID(Info->getID());
460
461 // If there is a newline following the verbatim opening command, skip the
462 // newline so that we don't create an tok::verbatim_block_line with empty
463 // text content.
464 if (BufferPtr != CommentEnd &&
465 isVerticalWhitespace(*BufferPtr)) {
466 BufferPtr = skipNewline(BufferPtr, CommentEnd);
467 State = LS_VerbatimBlockBody;
468 return;
469 }
470
471 State = LS_VerbatimBlockFirstLine;
472 }
473
lexVerbatimBlockFirstLine(Token & T)474 void Lexer::lexVerbatimBlockFirstLine(Token &T) {
475 again:
476 assert(BufferPtr < CommentEnd);
477
478 // FIXME: It would be better to scan the text once, finding either the block
479 // end command or newline.
480 //
481 // Extract current line.
482 const char *Newline = findNewline(BufferPtr, CommentEnd);
483 StringRef Line(BufferPtr, Newline - BufferPtr);
484
485 // Look for end command in current line.
486 size_t Pos = Line.find(VerbatimBlockEndCommandName);
487 const char *TextEnd;
488 const char *NextLine;
489 if (Pos == StringRef::npos) {
490 // Current line is completely verbatim.
491 TextEnd = Newline;
492 NextLine = skipNewline(Newline, CommentEnd);
493 } else if (Pos == 0) {
494 // Current line contains just an end command.
495 const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
496 StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
497 formTokenWithChars(T, End, tok::verbatim_block_end);
498 T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
499 State = LS_Normal;
500 return;
501 } else {
502 // There is some text, followed by end command. Extract text first.
503 TextEnd = BufferPtr + Pos;
504 NextLine = TextEnd;
505 // If there is only whitespace before end command, skip whitespace.
506 if (isWhitespace(BufferPtr, TextEnd)) {
507 BufferPtr = TextEnd;
508 goto again;
509 }
510 }
511
512 StringRef Text(BufferPtr, TextEnd - BufferPtr);
513 formTokenWithChars(T, NextLine, tok::verbatim_block_line);
514 T.setVerbatimBlockText(Text);
515
516 State = LS_VerbatimBlockBody;
517 }
518
lexVerbatimBlockBody(Token & T)519 void Lexer::lexVerbatimBlockBody(Token &T) {
520 assert(State == LS_VerbatimBlockBody);
521
522 if (CommentState == LCS_InsideCComment)
523 skipLineStartingDecorations();
524
525 if (BufferPtr == CommentEnd) {
526 formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
527 T.setVerbatimBlockText("");
528 return;
529 }
530
531 lexVerbatimBlockFirstLine(T);
532 }
533
setupAndLexVerbatimLine(Token & T,const char * TextBegin,const CommandInfo * Info)534 void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
535 const CommandInfo *Info) {
536 assert(Info->IsVerbatimLineCommand);
537 formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
538 T.setVerbatimLineID(Info->getID());
539
540 State = LS_VerbatimLineText;
541 }
542
lexVerbatimLineText(Token & T)543 void Lexer::lexVerbatimLineText(Token &T) {
544 assert(State == LS_VerbatimLineText);
545
546 // Extract current line.
547 const char *Newline = findNewline(BufferPtr, CommentEnd);
548 StringRef Text(BufferPtr, Newline - BufferPtr);
549 formTokenWithChars(T, Newline, tok::verbatim_line_text);
550 T.setVerbatimLineText(Text);
551
552 State = LS_Normal;
553 }
554
lexHTMLCharacterReference(Token & T)555 void Lexer::lexHTMLCharacterReference(Token &T) {
556 const char *TokenPtr = BufferPtr;
557 assert(*TokenPtr == '&');
558 TokenPtr++;
559 if (TokenPtr == CommentEnd) {
560 formTextToken(T, TokenPtr);
561 return;
562 }
563 const char *NamePtr;
564 bool isNamed = false;
565 bool isDecimal = false;
566 char C = *TokenPtr;
567 if (isHTMLNamedCharacterReferenceCharacter(C)) {
568 NamePtr = TokenPtr;
569 TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
570 isNamed = true;
571 } else if (C == '#') {
572 TokenPtr++;
573 if (TokenPtr == CommentEnd) {
574 formTextToken(T, TokenPtr);
575 return;
576 }
577 C = *TokenPtr;
578 if (isHTMLDecimalCharacterReferenceCharacter(C)) {
579 NamePtr = TokenPtr;
580 TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
581 isDecimal = true;
582 } else if (C == 'x' || C == 'X') {
583 TokenPtr++;
584 NamePtr = TokenPtr;
585 TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
586 } else {
587 formTextToken(T, TokenPtr);
588 return;
589 }
590 } else {
591 formTextToken(T, TokenPtr);
592 return;
593 }
594 if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
595 *TokenPtr != ';') {
596 formTextToken(T, TokenPtr);
597 return;
598 }
599 StringRef Name(NamePtr, TokenPtr - NamePtr);
600 TokenPtr++; // Skip semicolon.
601 StringRef Resolved;
602 if (isNamed)
603 Resolved = resolveHTMLNamedCharacterReference(Name);
604 else if (isDecimal)
605 Resolved = resolveHTMLDecimalCharacterReference(Name);
606 else
607 Resolved = resolveHTMLHexCharacterReference(Name);
608
609 if (Resolved.empty()) {
610 formTextToken(T, TokenPtr);
611 return;
612 }
613 formTokenWithChars(T, TokenPtr, tok::text);
614 T.setText(Resolved);
615 }
616
setupAndLexHTMLStartTag(Token & T)617 void Lexer::setupAndLexHTMLStartTag(Token &T) {
618 assert(BufferPtr[0] == '<' &&
619 isHTMLIdentifierStartingCharacter(BufferPtr[1]));
620 const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
621 StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
622 if (!isHTMLTagName(Name)) {
623 formTextToken(T, TagNameEnd);
624 return;
625 }
626
627 formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
628 T.setHTMLTagStartName(Name);
629
630 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
631
632 const char C = *BufferPtr;
633 if (BufferPtr != CommentEnd &&
634 (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
635 State = LS_HTMLStartTag;
636 }
637
lexHTMLStartTag(Token & T)638 void Lexer::lexHTMLStartTag(Token &T) {
639 assert(State == LS_HTMLStartTag);
640
641 const char *TokenPtr = BufferPtr;
642 char C = *TokenPtr;
643 if (isHTMLIdentifierCharacter(C)) {
644 TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
645 StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
646 formTokenWithChars(T, TokenPtr, tok::html_ident);
647 T.setHTMLIdent(Ident);
648 } else {
649 switch (C) {
650 case '=':
651 TokenPtr++;
652 formTokenWithChars(T, TokenPtr, tok::html_equals);
653 break;
654 case '\"':
655 case '\'': {
656 const char *OpenQuote = TokenPtr;
657 TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
658 const char *ClosingQuote = TokenPtr;
659 if (TokenPtr != CommentEnd) // Skip closing quote.
660 TokenPtr++;
661 formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
662 T.setHTMLQuotedString(StringRef(OpenQuote + 1,
663 ClosingQuote - (OpenQuote + 1)));
664 break;
665 }
666 case '>':
667 TokenPtr++;
668 formTokenWithChars(T, TokenPtr, tok::html_greater);
669 State = LS_Normal;
670 return;
671 case '/':
672 TokenPtr++;
673 if (TokenPtr != CommentEnd && *TokenPtr == '>') {
674 TokenPtr++;
675 formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
676 } else
677 formTextToken(T, TokenPtr);
678
679 State = LS_Normal;
680 return;
681 }
682 }
683
684 // Now look ahead and return to normal state if we don't see any HTML tokens
685 // ahead.
686 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
687 if (BufferPtr == CommentEnd) {
688 State = LS_Normal;
689 return;
690 }
691
692 C = *BufferPtr;
693 if (!isHTMLIdentifierStartingCharacter(C) &&
694 C != '=' && C != '\"' && C != '\'' && C != '>') {
695 State = LS_Normal;
696 return;
697 }
698 }
699
setupAndLexHTMLEndTag(Token & T)700 void Lexer::setupAndLexHTMLEndTag(Token &T) {
701 assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
702
703 const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
704 const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
705 StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
706 if (!isHTMLTagName(Name)) {
707 formTextToken(T, TagNameEnd);
708 return;
709 }
710
711 const char *End = skipWhitespace(TagNameEnd, CommentEnd);
712
713 formTokenWithChars(T, End, tok::html_end_tag);
714 T.setHTMLTagEndName(Name);
715
716 if (BufferPtr != CommentEnd && *BufferPtr == '>')
717 State = LS_HTMLEndTag;
718 }
719
lexHTMLEndTag(Token & T)720 void Lexer::lexHTMLEndTag(Token &T) {
721 assert(BufferPtr != CommentEnd && *BufferPtr == '>');
722
723 formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
724 State = LS_Normal;
725 }
726
Lexer(llvm::BumpPtrAllocator & Allocator,DiagnosticsEngine & Diags,const CommandTraits & Traits,SourceLocation FileLoc,const char * BufferStart,const char * BufferEnd)727 Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
728 const CommandTraits &Traits,
729 SourceLocation FileLoc,
730 const char *BufferStart, const char *BufferEnd):
731 Allocator(Allocator), Diags(Diags), Traits(Traits),
732 BufferStart(BufferStart), BufferEnd(BufferEnd),
733 FileLoc(FileLoc), BufferPtr(BufferStart),
734 CommentState(LCS_BeforeComment), State(LS_Normal) {
735 }
736
lex(Token & T)737 void Lexer::lex(Token &T) {
738 again:
739 switch (CommentState) {
740 case LCS_BeforeComment:
741 if (BufferPtr == BufferEnd) {
742 formTokenWithChars(T, BufferPtr, tok::eof);
743 return;
744 }
745
746 assert(*BufferPtr == '/');
747 BufferPtr++; // Skip first slash.
748 switch(*BufferPtr) {
749 case '/': { // BCPL comment.
750 BufferPtr++; // Skip second slash.
751
752 if (BufferPtr != BufferEnd) {
753 // Skip Doxygen magic marker, if it is present.
754 // It might be missing because of a typo //< or /*<, or because we
755 // merged this non-Doxygen comment into a bunch of Doxygen comments
756 // around it: /** ... */ /* ... */ /** ... */
757 const char C = *BufferPtr;
758 if (C == '/' || C == '!')
759 BufferPtr++;
760 }
761
762 // Skip less-than symbol that marks trailing comments.
763 // Skip it even if the comment is not a Doxygen one, because //< and /*<
764 // are frequent typos.
765 if (BufferPtr != BufferEnd && *BufferPtr == '<')
766 BufferPtr++;
767
768 CommentState = LCS_InsideBCPLComment;
769 if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
770 State = LS_Normal;
771 CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
772 goto again;
773 }
774 case '*': { // C comment.
775 BufferPtr++; // Skip star.
776
777 // Skip Doxygen magic marker.
778 const char C = *BufferPtr;
779 if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
780 BufferPtr++;
781
782 // Skip less-than symbol that marks trailing comments.
783 if (BufferPtr != BufferEnd && *BufferPtr == '<')
784 BufferPtr++;
785
786 CommentState = LCS_InsideCComment;
787 State = LS_Normal;
788 CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
789 goto again;
790 }
791 default:
792 llvm_unreachable("second character of comment should be '/' or '*'");
793 }
794
795 case LCS_BetweenComments: {
796 // Consecutive comments are extracted only if there is only whitespace
797 // between them. So we can search for the start of the next comment.
798 const char *EndWhitespace = BufferPtr;
799 while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
800 EndWhitespace++;
801
802 // Turn any whitespace between comments (and there is only whitespace
803 // between them -- guaranteed by comment extraction) into a newline. We
804 // have two newlines between C comments in total (first one was synthesized
805 // after a comment).
806 formTokenWithChars(T, EndWhitespace, tok::newline);
807
808 CommentState = LCS_BeforeComment;
809 break;
810 }
811
812 case LCS_InsideBCPLComment:
813 case LCS_InsideCComment:
814 if (BufferPtr != CommentEnd) {
815 lexCommentText(T);
816 break;
817 } else {
818 // Skip C comment closing sequence.
819 if (CommentState == LCS_InsideCComment) {
820 assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
821 BufferPtr += 2;
822 assert(BufferPtr <= BufferEnd);
823
824 // Synthenize newline just after the C comment, regardless if there is
825 // actually a newline.
826 formTokenWithChars(T, BufferPtr, tok::newline);
827
828 CommentState = LCS_BetweenComments;
829 break;
830 } else {
831 // Don't synthesized a newline after BCPL comment.
832 CommentState = LCS_BetweenComments;
833 goto again;
834 }
835 }
836 }
837 }
838
getSpelling(const Token & Tok,const SourceManager & SourceMgr,bool * Invalid) const839 StringRef Lexer::getSpelling(const Token &Tok,
840 const SourceManager &SourceMgr,
841 bool *Invalid) const {
842 SourceLocation Loc = Tok.getLocation();
843 std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
844
845 bool InvalidTemp = false;
846 StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
847 if (InvalidTemp) {
848 *Invalid = true;
849 return StringRef();
850 }
851
852 const char *Begin = File.data() + LocInfo.second;
853 return StringRef(Begin, Tok.getLength());
854 }
855
856 } // end namespace comments
857 } // end namespace clang
858