• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===--- CommentLexer.cpp -------------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "clang/AST/CommentLexer.h"
11 #include "clang/AST/CommentCommandTraits.h"
12 #include "clang/AST/CommentDiagnostic.h"
13 #include "clang/Basic/CharInfo.h"
14 #include "llvm/ADT/StringExtras.h"
15 #include "llvm/ADT/StringSwitch.h"
16 #include "llvm/Support/ConvertUTF.h"
17 #include "llvm/Support/ErrorHandling.h"
18 
19 namespace clang {
20 namespace comments {
21 
dump(const Lexer & L,const SourceManager & SM) const22 void Token::dump(const Lexer &L, const SourceManager &SM) const {
23   llvm::errs() << "comments::Token Kind=" << Kind << " ";
24   Loc.dump(SM);
25   llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
26 }
27 
isHTMLNamedCharacterReferenceCharacter(char C)28 static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
29   return isLetter(C);
30 }
31 
isHTMLDecimalCharacterReferenceCharacter(char C)32 static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
33   return isDigit(C);
34 }
35 
isHTMLHexCharacterReferenceCharacter(char C)36 static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
37   return isHexDigit(C);
38 }
39 
convertCodePointToUTF8(llvm::BumpPtrAllocator & Allocator,unsigned CodePoint)40 static inline StringRef convertCodePointToUTF8(
41                                       llvm::BumpPtrAllocator &Allocator,
42                                       unsigned CodePoint) {
43   char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
44   char *ResolvedPtr = Resolved;
45   if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
46     return StringRef(Resolved, ResolvedPtr - Resolved);
47   else
48     return StringRef();
49 }
50 
51 namespace {
52 
53 #include "clang/AST/CommentHTMLTags.inc"
54 #include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
55 
56 } // end anonymous namespace
57 
resolveHTMLNamedCharacterReference(StringRef Name) const58 StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
59   // Fast path, first check a few most widely used named character references.
60   return llvm::StringSwitch<StringRef>(Name)
61       .Case("amp", "&")
62       .Case("lt", "<")
63       .Case("gt", ">")
64       .Case("quot", "\"")
65       .Case("apos", "\'")
66       // Slow path.
67       .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
68 }
69 
resolveHTMLDecimalCharacterReference(StringRef Name) const70 StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
71   unsigned CodePoint = 0;
72   for (unsigned i = 0, e = Name.size(); i != e; ++i) {
73     assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
74     CodePoint *= 10;
75     CodePoint += Name[i] - '0';
76   }
77   return convertCodePointToUTF8(Allocator, CodePoint);
78 }
79 
resolveHTMLHexCharacterReference(StringRef Name) const80 StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
81   unsigned CodePoint = 0;
82   for (unsigned i = 0, e = Name.size(); i != e; ++i) {
83     CodePoint *= 16;
84     const char C = Name[i];
85     assert(isHTMLHexCharacterReferenceCharacter(C));
86     CodePoint += llvm::hexDigitValue(C);
87   }
88   return convertCodePointToUTF8(Allocator, CodePoint);
89 }
90 
skipLineStartingDecorations()91 void Lexer::skipLineStartingDecorations() {
92   // This function should be called only for C comments
93   assert(CommentState == LCS_InsideCComment);
94 
95   if (BufferPtr == CommentEnd)
96     return;
97 
98   switch (*BufferPtr) {
99   case ' ':
100   case '\t':
101   case '\f':
102   case '\v': {
103     const char *NewBufferPtr = BufferPtr;
104     NewBufferPtr++;
105     if (NewBufferPtr == CommentEnd)
106       return;
107 
108     char C = *NewBufferPtr;
109     while (isHorizontalWhitespace(C)) {
110       NewBufferPtr++;
111       if (NewBufferPtr == CommentEnd)
112         return;
113       C = *NewBufferPtr;
114     }
115     if (C == '*')
116       BufferPtr = NewBufferPtr + 1;
117     break;
118   }
119   case '*':
120     BufferPtr++;
121     break;
122   }
123 }
124 
125 namespace {
126 /// Returns pointer to the first newline character in the string.
findNewline(const char * BufferPtr,const char * BufferEnd)127 const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
128   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
129     if (isVerticalWhitespace(*BufferPtr))
130       return BufferPtr;
131   }
132   return BufferEnd;
133 }
134 
skipNewline(const char * BufferPtr,const char * BufferEnd)135 const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
136   if (BufferPtr == BufferEnd)
137     return BufferPtr;
138 
139   if (*BufferPtr == '\n')
140     BufferPtr++;
141   else {
142     assert(*BufferPtr == '\r');
143     BufferPtr++;
144     if (BufferPtr != BufferEnd && *BufferPtr == '\n')
145       BufferPtr++;
146   }
147   return BufferPtr;
148 }
149 
skipNamedCharacterReference(const char * BufferPtr,const char * BufferEnd)150 const char *skipNamedCharacterReference(const char *BufferPtr,
151                                         const char *BufferEnd) {
152   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
153     if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
154       return BufferPtr;
155   }
156   return BufferEnd;
157 }
158 
skipDecimalCharacterReference(const char * BufferPtr,const char * BufferEnd)159 const char *skipDecimalCharacterReference(const char *BufferPtr,
160                                           const char *BufferEnd) {
161   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
162     if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
163       return BufferPtr;
164   }
165   return BufferEnd;
166 }
167 
skipHexCharacterReference(const char * BufferPtr,const char * BufferEnd)168 const char *skipHexCharacterReference(const char *BufferPtr,
169                                       const char *BufferEnd) {
170   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
171     if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
172       return BufferPtr;
173   }
174   return BufferEnd;
175 }
176 
isHTMLIdentifierStartingCharacter(char C)177 bool isHTMLIdentifierStartingCharacter(char C) {
178   return isLetter(C);
179 }
180 
isHTMLIdentifierCharacter(char C)181 bool isHTMLIdentifierCharacter(char C) {
182   return isAlphanumeric(C);
183 }
184 
skipHTMLIdentifier(const char * BufferPtr,const char * BufferEnd)185 const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
186   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
187     if (!isHTMLIdentifierCharacter(*BufferPtr))
188       return BufferPtr;
189   }
190   return BufferEnd;
191 }
192 
193 /// Skip HTML string quoted in single or double quotes.  Escaping quotes inside
194 /// string allowed.
195 ///
196 /// Returns pointer to closing quote.
skipHTMLQuotedString(const char * BufferPtr,const char * BufferEnd)197 const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
198 {
199   const char Quote = *BufferPtr;
200   assert(Quote == '\"' || Quote == '\'');
201 
202   BufferPtr++;
203   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
204     const char C = *BufferPtr;
205     if (C == Quote && BufferPtr[-1] != '\\')
206       return BufferPtr;
207   }
208   return BufferEnd;
209 }
210 
skipWhitespace(const char * BufferPtr,const char * BufferEnd)211 const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
212   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
213     if (!isWhitespace(*BufferPtr))
214       return BufferPtr;
215   }
216   return BufferEnd;
217 }
218 
isWhitespace(const char * BufferPtr,const char * BufferEnd)219 bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
220   return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
221 }
222 
isCommandNameStartCharacter(char C)223 bool isCommandNameStartCharacter(char C) {
224   return isLetter(C);
225 }
226 
isCommandNameCharacter(char C)227 bool isCommandNameCharacter(char C) {
228   return isAlphanumeric(C);
229 }
230 
skipCommandName(const char * BufferPtr,const char * BufferEnd)231 const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
232   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
233     if (!isCommandNameCharacter(*BufferPtr))
234       return BufferPtr;
235   }
236   return BufferEnd;
237 }
238 
239 /// Return the one past end pointer for BCPL comments.
240 /// Handles newlines escaped with backslash or trigraph for backslahs.
findBCPLCommentEnd(const char * BufferPtr,const char * BufferEnd)241 const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
242   const char *CurPtr = BufferPtr;
243   while (CurPtr != BufferEnd) {
244     while (!isVerticalWhitespace(*CurPtr)) {
245       CurPtr++;
246       if (CurPtr == BufferEnd)
247         return BufferEnd;
248     }
249     // We found a newline, check if it is escaped.
250     const char *EscapePtr = CurPtr - 1;
251     while(isHorizontalWhitespace(*EscapePtr))
252       EscapePtr--;
253 
254     if (*EscapePtr == '\\' ||
255         (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
256          EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
257       // We found an escaped newline.
258       CurPtr = skipNewline(CurPtr, BufferEnd);
259     } else
260       return CurPtr; // Not an escaped newline.
261   }
262   return BufferEnd;
263 }
264 
265 /// Return the one past end pointer for C comments.
266 /// Very dumb, does not handle escaped newlines or trigraphs.
findCCommentEnd(const char * BufferPtr,const char * BufferEnd)267 const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
268   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
269     if (*BufferPtr == '*') {
270       assert(BufferPtr + 1 != BufferEnd);
271       if (*(BufferPtr + 1) == '/')
272         return BufferPtr;
273     }
274   }
275   llvm_unreachable("buffer end hit before '*/' was seen");
276 }
277 
278 } // end anonymous namespace
279 
formTokenWithChars(Token & Result,const char * TokEnd,tok::TokenKind Kind)280 void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
281                                tok::TokenKind Kind) {
282   const unsigned TokLen = TokEnd - BufferPtr;
283   Result.setLocation(getSourceLocation(BufferPtr));
284   Result.setKind(Kind);
285   Result.setLength(TokLen);
286 #ifndef NDEBUG
287   Result.TextPtr = "<UNSET>";
288   Result.IntVal = 7;
289 #endif
290   BufferPtr = TokEnd;
291 }
292 
lexCommentText(Token & T)293 void Lexer::lexCommentText(Token &T) {
294   assert(CommentState == LCS_InsideBCPLComment ||
295          CommentState == LCS_InsideCComment);
296 
297   switch (State) {
298   case LS_Normal:
299     break;
300   case LS_VerbatimBlockFirstLine:
301     lexVerbatimBlockFirstLine(T);
302     return;
303   case LS_VerbatimBlockBody:
304     lexVerbatimBlockBody(T);
305     return;
306   case LS_VerbatimLineText:
307     lexVerbatimLineText(T);
308     return;
309   case LS_HTMLStartTag:
310     lexHTMLStartTag(T);
311     return;
312   case LS_HTMLEndTag:
313     lexHTMLEndTag(T);
314     return;
315   }
316 
317   assert(State == LS_Normal);
318 
319   const char *TokenPtr = BufferPtr;
320   assert(TokenPtr < CommentEnd);
321   while (TokenPtr != CommentEnd) {
322     switch(*TokenPtr) {
323       case '\\':
324       case '@': {
325         // Commands that start with a backslash and commands that start with
326         // 'at' have equivalent semantics.  But we keep information about the
327         // exact syntax in AST for comments.
328         tok::TokenKind CommandKind =
329             (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
330         TokenPtr++;
331         if (TokenPtr == CommentEnd) {
332           formTextToken(T, TokenPtr);
333           return;
334         }
335         char C = *TokenPtr;
336         switch (C) {
337         default:
338           break;
339 
340         case '\\': case '@': case '&': case '$':
341         case '#':  case '<': case '>': case '%':
342         case '\"': case '.': case ':':
343           // This is one of \\ \@ \& \$ etc escape sequences.
344           TokenPtr++;
345           if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
346             // This is the \:: escape sequence.
347             TokenPtr++;
348           }
349           StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
350           formTokenWithChars(T, TokenPtr, tok::text);
351           T.setText(UnescapedText);
352           return;
353         }
354 
355         // Don't make zero-length commands.
356         if (!isCommandNameStartCharacter(*TokenPtr)) {
357           formTextToken(T, TokenPtr);
358           return;
359         }
360 
361         TokenPtr = skipCommandName(TokenPtr, CommentEnd);
362         unsigned Length = TokenPtr - (BufferPtr + 1);
363 
364         // Hardcoded support for lexing LaTeX formula commands
365         // \f$ \f[ \f] \f{ \f} as a single command.
366         if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
367           C = *TokenPtr;
368           if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
369             TokenPtr++;
370             Length++;
371           }
372         }
373 
374         StringRef CommandName(BufferPtr + 1, Length);
375 
376         const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
377         if (!Info) {
378           if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
379             StringRef CorrectedName = Info->Name;
380             SourceLocation Loc = getSourceLocation(BufferPtr);
381             SourceRange CommandRange(Loc.getLocWithOffset(1),
382                                      getSourceLocation(TokenPtr));
383             Diag(Loc, diag::warn_correct_comment_command_name)
384               << CommandName << CorrectedName
385               << FixItHint::CreateReplacement(CommandRange, CorrectedName);
386           } else {
387             formTokenWithChars(T, TokenPtr, tok::unknown_command);
388             T.setUnknownCommandName(CommandName);
389             Diag(T.getLocation(), diag::warn_unknown_comment_command_name);
390             return;
391           }
392         }
393         if (Info->IsVerbatimBlockCommand) {
394           setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
395           return;
396         }
397         if (Info->IsVerbatimLineCommand) {
398           setupAndLexVerbatimLine(T, TokenPtr, Info);
399           return;
400         }
401         formTokenWithChars(T, TokenPtr, CommandKind);
402         T.setCommandID(Info->getID());
403         return;
404       }
405 
406       case '&':
407         lexHTMLCharacterReference(T);
408         return;
409 
410       case '<': {
411         TokenPtr++;
412         if (TokenPtr == CommentEnd) {
413           formTextToken(T, TokenPtr);
414           return;
415         }
416         const char C = *TokenPtr;
417         if (isHTMLIdentifierStartingCharacter(C))
418           setupAndLexHTMLStartTag(T);
419         else if (C == '/')
420           setupAndLexHTMLEndTag(T);
421         else
422           formTextToken(T, TokenPtr);
423         return;
424       }
425 
426       case '\n':
427       case '\r':
428         TokenPtr = skipNewline(TokenPtr, CommentEnd);
429         formTokenWithChars(T, TokenPtr, tok::newline);
430 
431         if (CommentState == LCS_InsideCComment)
432           skipLineStartingDecorations();
433         return;
434 
435       default: {
436         size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr).
437                          find_first_of("\n\r\\@&<");
438         if (End != StringRef::npos)
439           TokenPtr += End;
440         else
441           TokenPtr = CommentEnd;
442         formTextToken(T, TokenPtr);
443         return;
444       }
445     }
446   }
447 }
448 
setupAndLexVerbatimBlock(Token & T,const char * TextBegin,char Marker,const CommandInfo * Info)449 void Lexer::setupAndLexVerbatimBlock(Token &T,
450                                      const char *TextBegin,
451                                      char Marker, const CommandInfo *Info) {
452   assert(Info->IsVerbatimBlockCommand);
453 
454   VerbatimBlockEndCommandName.clear();
455   VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
456   VerbatimBlockEndCommandName.append(Info->EndCommandName);
457 
458   formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
459   T.setVerbatimBlockID(Info->getID());
460 
461   // If there is a newline following the verbatim opening command, skip the
462   // newline so that we don't create an tok::verbatim_block_line with empty
463   // text content.
464   if (BufferPtr != CommentEnd &&
465       isVerticalWhitespace(*BufferPtr)) {
466     BufferPtr = skipNewline(BufferPtr, CommentEnd);
467     State = LS_VerbatimBlockBody;
468     return;
469   }
470 
471   State = LS_VerbatimBlockFirstLine;
472 }
473 
lexVerbatimBlockFirstLine(Token & T)474 void Lexer::lexVerbatimBlockFirstLine(Token &T) {
475 again:
476   assert(BufferPtr < CommentEnd);
477 
478   // FIXME: It would be better to scan the text once, finding either the block
479   // end command or newline.
480   //
481   // Extract current line.
482   const char *Newline = findNewline(BufferPtr, CommentEnd);
483   StringRef Line(BufferPtr, Newline - BufferPtr);
484 
485   // Look for end command in current line.
486   size_t Pos = Line.find(VerbatimBlockEndCommandName);
487   const char *TextEnd;
488   const char *NextLine;
489   if (Pos == StringRef::npos) {
490     // Current line is completely verbatim.
491     TextEnd = Newline;
492     NextLine = skipNewline(Newline, CommentEnd);
493   } else if (Pos == 0) {
494     // Current line contains just an end command.
495     const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
496     StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
497     formTokenWithChars(T, End, tok::verbatim_block_end);
498     T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
499     State = LS_Normal;
500     return;
501   } else {
502     // There is some text, followed by end command.  Extract text first.
503     TextEnd = BufferPtr + Pos;
504     NextLine = TextEnd;
505     // If there is only whitespace before end command, skip whitespace.
506     if (isWhitespace(BufferPtr, TextEnd)) {
507       BufferPtr = TextEnd;
508       goto again;
509     }
510   }
511 
512   StringRef Text(BufferPtr, TextEnd - BufferPtr);
513   formTokenWithChars(T, NextLine, tok::verbatim_block_line);
514   T.setVerbatimBlockText(Text);
515 
516   State = LS_VerbatimBlockBody;
517 }
518 
lexVerbatimBlockBody(Token & T)519 void Lexer::lexVerbatimBlockBody(Token &T) {
520   assert(State == LS_VerbatimBlockBody);
521 
522   if (CommentState == LCS_InsideCComment)
523     skipLineStartingDecorations();
524 
525   if (BufferPtr == CommentEnd) {
526     formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
527     T.setVerbatimBlockText("");
528     return;
529   }
530 
531   lexVerbatimBlockFirstLine(T);
532 }
533 
setupAndLexVerbatimLine(Token & T,const char * TextBegin,const CommandInfo * Info)534 void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
535                                     const CommandInfo *Info) {
536   assert(Info->IsVerbatimLineCommand);
537   formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
538   T.setVerbatimLineID(Info->getID());
539 
540   State = LS_VerbatimLineText;
541 }
542 
lexVerbatimLineText(Token & T)543 void Lexer::lexVerbatimLineText(Token &T) {
544   assert(State == LS_VerbatimLineText);
545 
546   // Extract current line.
547   const char *Newline = findNewline(BufferPtr, CommentEnd);
548   StringRef Text(BufferPtr, Newline - BufferPtr);
549   formTokenWithChars(T, Newline, tok::verbatim_line_text);
550   T.setVerbatimLineText(Text);
551 
552   State = LS_Normal;
553 }
554 
lexHTMLCharacterReference(Token & T)555 void Lexer::lexHTMLCharacterReference(Token &T) {
556   const char *TokenPtr = BufferPtr;
557   assert(*TokenPtr == '&');
558   TokenPtr++;
559   if (TokenPtr == CommentEnd) {
560     formTextToken(T, TokenPtr);
561     return;
562   }
563   const char *NamePtr;
564   bool isNamed = false;
565   bool isDecimal = false;
566   char C = *TokenPtr;
567   if (isHTMLNamedCharacterReferenceCharacter(C)) {
568     NamePtr = TokenPtr;
569     TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
570     isNamed = true;
571   } else if (C == '#') {
572     TokenPtr++;
573     if (TokenPtr == CommentEnd) {
574       formTextToken(T, TokenPtr);
575       return;
576     }
577     C = *TokenPtr;
578     if (isHTMLDecimalCharacterReferenceCharacter(C)) {
579       NamePtr = TokenPtr;
580       TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
581       isDecimal = true;
582     } else if (C == 'x' || C == 'X') {
583       TokenPtr++;
584       NamePtr = TokenPtr;
585       TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
586     } else {
587       formTextToken(T, TokenPtr);
588       return;
589     }
590   } else {
591     formTextToken(T, TokenPtr);
592     return;
593   }
594   if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
595       *TokenPtr != ';') {
596     formTextToken(T, TokenPtr);
597     return;
598   }
599   StringRef Name(NamePtr, TokenPtr - NamePtr);
600   TokenPtr++; // Skip semicolon.
601   StringRef Resolved;
602   if (isNamed)
603     Resolved = resolveHTMLNamedCharacterReference(Name);
604   else if (isDecimal)
605     Resolved = resolveHTMLDecimalCharacterReference(Name);
606   else
607     Resolved = resolveHTMLHexCharacterReference(Name);
608 
609   if (Resolved.empty()) {
610     formTextToken(T, TokenPtr);
611     return;
612   }
613   formTokenWithChars(T, TokenPtr, tok::text);
614   T.setText(Resolved);
615 }
616 
setupAndLexHTMLStartTag(Token & T)617 void Lexer::setupAndLexHTMLStartTag(Token &T) {
618   assert(BufferPtr[0] == '<' &&
619          isHTMLIdentifierStartingCharacter(BufferPtr[1]));
620   const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
621   StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
622   if (!isHTMLTagName(Name)) {
623     formTextToken(T, TagNameEnd);
624     return;
625   }
626 
627   formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
628   T.setHTMLTagStartName(Name);
629 
630   BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
631 
632   const char C = *BufferPtr;
633   if (BufferPtr != CommentEnd &&
634       (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
635     State = LS_HTMLStartTag;
636 }
637 
lexHTMLStartTag(Token & T)638 void Lexer::lexHTMLStartTag(Token &T) {
639   assert(State == LS_HTMLStartTag);
640 
641   const char *TokenPtr = BufferPtr;
642   char C = *TokenPtr;
643   if (isHTMLIdentifierCharacter(C)) {
644     TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
645     StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
646     formTokenWithChars(T, TokenPtr, tok::html_ident);
647     T.setHTMLIdent(Ident);
648   } else {
649     switch (C) {
650     case '=':
651       TokenPtr++;
652       formTokenWithChars(T, TokenPtr, tok::html_equals);
653       break;
654     case '\"':
655     case '\'': {
656       const char *OpenQuote = TokenPtr;
657       TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
658       const char *ClosingQuote = TokenPtr;
659       if (TokenPtr != CommentEnd) // Skip closing quote.
660         TokenPtr++;
661       formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
662       T.setHTMLQuotedString(StringRef(OpenQuote + 1,
663                                       ClosingQuote - (OpenQuote + 1)));
664       break;
665     }
666     case '>':
667       TokenPtr++;
668       formTokenWithChars(T, TokenPtr, tok::html_greater);
669       State = LS_Normal;
670       return;
671     case '/':
672       TokenPtr++;
673       if (TokenPtr != CommentEnd && *TokenPtr == '>') {
674         TokenPtr++;
675         formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
676       } else
677         formTextToken(T, TokenPtr);
678 
679       State = LS_Normal;
680       return;
681     }
682   }
683 
684   // Now look ahead and return to normal state if we don't see any HTML tokens
685   // ahead.
686   BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
687   if (BufferPtr == CommentEnd) {
688     State = LS_Normal;
689     return;
690   }
691 
692   C = *BufferPtr;
693   if (!isHTMLIdentifierStartingCharacter(C) &&
694       C != '=' && C != '\"' && C != '\'' && C != '>') {
695     State = LS_Normal;
696     return;
697   }
698 }
699 
setupAndLexHTMLEndTag(Token & T)700 void Lexer::setupAndLexHTMLEndTag(Token &T) {
701   assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
702 
703   const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
704   const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
705   StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
706   if (!isHTMLTagName(Name)) {
707     formTextToken(T, TagNameEnd);
708     return;
709   }
710 
711   const char *End = skipWhitespace(TagNameEnd, CommentEnd);
712 
713   formTokenWithChars(T, End, tok::html_end_tag);
714   T.setHTMLTagEndName(Name);
715 
716   if (BufferPtr != CommentEnd && *BufferPtr == '>')
717     State = LS_HTMLEndTag;
718 }
719 
lexHTMLEndTag(Token & T)720 void Lexer::lexHTMLEndTag(Token &T) {
721   assert(BufferPtr != CommentEnd && *BufferPtr == '>');
722 
723   formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
724   State = LS_Normal;
725 }
726 
Lexer(llvm::BumpPtrAllocator & Allocator,DiagnosticsEngine & Diags,const CommandTraits & Traits,SourceLocation FileLoc,const char * BufferStart,const char * BufferEnd)727 Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
728              const CommandTraits &Traits,
729              SourceLocation FileLoc,
730              const char *BufferStart, const char *BufferEnd):
731     Allocator(Allocator), Diags(Diags), Traits(Traits),
732     BufferStart(BufferStart), BufferEnd(BufferEnd),
733     FileLoc(FileLoc), BufferPtr(BufferStart),
734     CommentState(LCS_BeforeComment), State(LS_Normal) {
735 }
736 
lex(Token & T)737 void Lexer::lex(Token &T) {
738 again:
739   switch (CommentState) {
740   case LCS_BeforeComment:
741     if (BufferPtr == BufferEnd) {
742       formTokenWithChars(T, BufferPtr, tok::eof);
743       return;
744     }
745 
746     assert(*BufferPtr == '/');
747     BufferPtr++; // Skip first slash.
748     switch(*BufferPtr) {
749     case '/': { // BCPL comment.
750       BufferPtr++; // Skip second slash.
751 
752       if (BufferPtr != BufferEnd) {
753         // Skip Doxygen magic marker, if it is present.
754         // It might be missing because of a typo //< or /*<, or because we
755         // merged this non-Doxygen comment into a bunch of Doxygen comments
756         // around it: /** ... */ /* ... */ /** ... */
757         const char C = *BufferPtr;
758         if (C == '/' || C == '!')
759           BufferPtr++;
760       }
761 
762       // Skip less-than symbol that marks trailing comments.
763       // Skip it even if the comment is not a Doxygen one, because //< and /*<
764       // are frequent typos.
765       if (BufferPtr != BufferEnd && *BufferPtr == '<')
766         BufferPtr++;
767 
768       CommentState = LCS_InsideBCPLComment;
769       if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
770         State = LS_Normal;
771       CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
772       goto again;
773     }
774     case '*': { // C comment.
775       BufferPtr++; // Skip star.
776 
777       // Skip Doxygen magic marker.
778       const char C = *BufferPtr;
779       if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
780         BufferPtr++;
781 
782       // Skip less-than symbol that marks trailing comments.
783       if (BufferPtr != BufferEnd && *BufferPtr == '<')
784         BufferPtr++;
785 
786       CommentState = LCS_InsideCComment;
787       State = LS_Normal;
788       CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
789       goto again;
790     }
791     default:
792       llvm_unreachable("second character of comment should be '/' or '*'");
793     }
794 
795   case LCS_BetweenComments: {
796     // Consecutive comments are extracted only if there is only whitespace
797     // between them.  So we can search for the start of the next comment.
798     const char *EndWhitespace = BufferPtr;
799     while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
800       EndWhitespace++;
801 
802     // Turn any whitespace between comments (and there is only whitespace
803     // between them -- guaranteed by comment extraction) into a newline.  We
804     // have two newlines between C comments in total (first one was synthesized
805     // after a comment).
806     formTokenWithChars(T, EndWhitespace, tok::newline);
807 
808     CommentState = LCS_BeforeComment;
809     break;
810   }
811 
812   case LCS_InsideBCPLComment:
813   case LCS_InsideCComment:
814     if (BufferPtr != CommentEnd) {
815       lexCommentText(T);
816       break;
817     } else {
818       // Skip C comment closing sequence.
819       if (CommentState == LCS_InsideCComment) {
820         assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
821         BufferPtr += 2;
822         assert(BufferPtr <= BufferEnd);
823 
824         // Synthenize newline just after the C comment, regardless if there is
825         // actually a newline.
826         formTokenWithChars(T, BufferPtr, tok::newline);
827 
828         CommentState = LCS_BetweenComments;
829         break;
830       } else {
831         // Don't synthesized a newline after BCPL comment.
832         CommentState = LCS_BetweenComments;
833         goto again;
834       }
835     }
836   }
837 }
838 
getSpelling(const Token & Tok,const SourceManager & SourceMgr,bool * Invalid) const839 StringRef Lexer::getSpelling(const Token &Tok,
840                              const SourceManager &SourceMgr,
841                              bool *Invalid) const {
842   SourceLocation Loc = Tok.getLocation();
843   std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
844 
845   bool InvalidTemp = false;
846   StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
847   if (InvalidTemp) {
848     *Invalid = true;
849     return StringRef();
850   }
851 
852   const char *Begin = File.data() + LocInfo.second;
853   return StringRef(Begin, Tok.getLength());
854 }
855 
856 } // end namespace comments
857 } // end namespace clang
858