• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements pieces of the Preprocessor interface that manage the
11 // current lexer stack.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Lex/Preprocessor.h"
16 #include "clang/Basic/FileManager.h"
17 #include "clang/Basic/SourceManager.h"
18 #include "clang/Lex/HeaderSearch.h"
19 #include "clang/Lex/LexDiagnostic.h"
20 #include "clang/Lex/MacroInfo.h"
21 #include "clang/Lex/PTHManager.h"
22 #include "llvm/ADT/StringSwitch.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/MemoryBuffer.h"
25 #include "llvm/Support/Path.h"
26 using namespace clang;
27 
~PPCallbacks()28 PPCallbacks::~PPCallbacks() {}
29 
30 //===----------------------------------------------------------------------===//
31 // Miscellaneous Methods.
32 //===----------------------------------------------------------------------===//
33 
34 /// isInPrimaryFile - Return true if we're in the top-level file, not in a
35 /// \#include.  This looks through macro expansions and active _Pragma lexers.
isInPrimaryFile() const36 bool Preprocessor::isInPrimaryFile() const {
37   if (IsFileLexer())
38     return IncludeMacroStack.empty();
39 
40   // If there are any stacked lexers, we're in a #include.
41   assert(IsFileLexer(IncludeMacroStack[0]) &&
42          "Top level include stack isn't our primary lexer?");
43   for (unsigned i = 1, e = IncludeMacroStack.size(); i != e; ++i)
44     if (IsFileLexer(IncludeMacroStack[i]))
45       return false;
46   return true;
47 }
48 
49 /// getCurrentLexer - Return the current file lexer being lexed from.  Note
50 /// that this ignores any potentially active macro expansions and _Pragma
51 /// expansions going on at the time.
getCurrentFileLexer() const52 PreprocessorLexer *Preprocessor::getCurrentFileLexer() const {
53   if (IsFileLexer())
54     return CurPPLexer;
55 
56   // Look for a stacked lexer.
57   for (unsigned i = IncludeMacroStack.size(); i != 0; --i) {
58     const IncludeStackInfo& ISI = IncludeMacroStack[i-1];
59     if (IsFileLexer(ISI))
60       return ISI.ThePPLexer;
61   }
62   return nullptr;
63 }
64 
65 
66 //===----------------------------------------------------------------------===//
67 // Methods for Entering and Callbacks for leaving various contexts
68 //===----------------------------------------------------------------------===//
69 
70 /// EnterSourceFile - Add a source file to the top of the include stack and
71 /// start lexing tokens from it instead of the current buffer.
EnterSourceFile(FileID FID,const DirectoryLookup * CurDir,SourceLocation Loc)72 bool Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir,
73                                    SourceLocation Loc) {
74   assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
75   ++NumEnteredSourceFiles;
76 
77   if (MaxIncludeStackDepth < IncludeMacroStack.size())
78     MaxIncludeStackDepth = IncludeMacroStack.size();
79 
80   if (PTH) {
81     if (PTHLexer *PL = PTH->CreateLexer(FID)) {
82       EnterSourceFileWithPTH(PL, CurDir);
83       return false;
84     }
85   }
86 
87   // Get the MemoryBuffer for this FID, if it fails, we fail.
88   bool Invalid = false;
89   const llvm::MemoryBuffer *InputFile =
90     getSourceManager().getBuffer(FID, Loc, &Invalid);
91   if (Invalid) {
92     SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
93     Diag(Loc, diag::err_pp_error_opening_file)
94       << std::string(SourceMgr.getBufferName(FileStart)) << "";
95     return true;
96   }
97 
98   if (isCodeCompletionEnabled() &&
99       SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
100     CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
101     CodeCompletionLoc =
102         CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
103   }
104 
105   EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
106   return false;
107 }
108 
109 /// EnterSourceFileWithLexer - Add a source file to the top of the include stack
110 ///  and start lexing tokens from it instead of the current buffer.
EnterSourceFileWithLexer(Lexer * TheLexer,const DirectoryLookup * CurDir)111 void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
112                                             const DirectoryLookup *CurDir) {
113 
114   // Add the current lexer to the include stack.
115   if (CurPPLexer || CurTokenLexer)
116     PushIncludeMacroStack();
117 
118   CurLexer.reset(TheLexer);
119   CurPPLexer = TheLexer;
120   CurDirLookup = CurDir;
121   CurSubmodule = nullptr;
122   if (CurLexerKind != CLK_LexAfterModuleImport)
123     CurLexerKind = CLK_Lexer;
124 
125   // Notify the client, if desired, that we are in a new source file.
126   if (Callbacks && !CurLexer->Is_PragmaLexer) {
127     SrcMgr::CharacteristicKind FileType =
128        SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
129 
130     Callbacks->FileChanged(CurLexer->getFileLoc(),
131                            PPCallbacks::EnterFile, FileType);
132   }
133 }
134 
135 /// EnterSourceFileWithPTH - Add a source file to the top of the include stack
136 /// and start getting tokens from it using the PTH cache.
EnterSourceFileWithPTH(PTHLexer * PL,const DirectoryLookup * CurDir)137 void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
138                                           const DirectoryLookup *CurDir) {
139 
140   if (CurPPLexer || CurTokenLexer)
141     PushIncludeMacroStack();
142 
143   CurDirLookup = CurDir;
144   CurPTHLexer.reset(PL);
145   CurPPLexer = CurPTHLexer.get();
146   CurSubmodule = nullptr;
147   if (CurLexerKind != CLK_LexAfterModuleImport)
148     CurLexerKind = CLK_PTHLexer;
149 
150   // Notify the client, if desired, that we are in a new source file.
151   if (Callbacks) {
152     FileID FID = CurPPLexer->getFileID();
153     SourceLocation EnterLoc = SourceMgr.getLocForStartOfFile(FID);
154     SrcMgr::CharacteristicKind FileType =
155       SourceMgr.getFileCharacteristic(EnterLoc);
156     Callbacks->FileChanged(EnterLoc, PPCallbacks::EnterFile, FileType);
157   }
158 }
159 
160 /// EnterMacro - Add a Macro to the top of the include stack and start lexing
161 /// tokens from it instead of the current buffer.
EnterMacro(Token & Tok,SourceLocation ILEnd,MacroInfo * Macro,MacroArgs * Args)162 void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
163                               MacroInfo *Macro, MacroArgs *Args) {
164   std::unique_ptr<TokenLexer> TokLexer;
165   if (NumCachedTokenLexers == 0) {
166     TokLexer = llvm::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
167   } else {
168     TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
169     TokLexer->Init(Tok, ILEnd, Macro, Args);
170   }
171 
172   PushIncludeMacroStack();
173   CurDirLookup = nullptr;
174   CurTokenLexer = std::move(TokLexer);
175   if (CurLexerKind != CLK_LexAfterModuleImport)
176     CurLexerKind = CLK_TokenLexer;
177 }
178 
179 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
180 /// which will cause the lexer to start returning the specified tokens.
181 ///
182 /// If DisableMacroExpansion is true, tokens lexed from the token stream will
183 /// not be subject to further macro expansion.  Otherwise, these tokens will
184 /// be re-macro-expanded when/if expansion is enabled.
185 ///
186 /// If OwnsTokens is false, this method assumes that the specified stream of
187 /// tokens has a permanent owner somewhere, so they do not need to be copied.
188 /// If it is true, it assumes the array of tokens is allocated with new[] and
189 /// must be freed.
190 ///
EnterTokenStream(const Token * Toks,unsigned NumToks,bool DisableMacroExpansion,bool OwnsTokens)191 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
192                                     bool DisableMacroExpansion,
193                                     bool OwnsTokens) {
194   if (CurLexerKind == CLK_CachingLexer) {
195     if (CachedLexPos < CachedTokens.size()) {
196       // We're entering tokens into the middle of our cached token stream. We
197       // can't represent that, so just insert the tokens into the buffer.
198       CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
199                           Toks, Toks + NumToks);
200       if (OwnsTokens)
201         delete [] Toks;
202       return;
203     }
204 
205     // New tokens are at the end of the cached token sequnece; insert the
206     // token stream underneath the caching lexer.
207     ExitCachingLexMode();
208     EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
209     EnterCachingLexMode();
210     return;
211   }
212 
213   // Create a macro expander to expand from the specified token stream.
214   std::unique_ptr<TokenLexer> TokLexer;
215   if (NumCachedTokenLexers == 0) {
216     TokLexer = llvm::make_unique<TokenLexer>(
217         Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
218   } else {
219     TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
220     TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
221   }
222 
223   // Save our current state.
224   PushIncludeMacroStack();
225   CurDirLookup = nullptr;
226   CurTokenLexer = std::move(TokLexer);
227   if (CurLexerKind != CLK_LexAfterModuleImport)
228     CurLexerKind = CLK_TokenLexer;
229 }
230 
231 /// \brief Compute the relative path that names the given file relative to
232 /// the given directory.
computeRelativePath(FileManager & FM,const DirectoryEntry * Dir,const FileEntry * File,SmallString<128> & Result)233 static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
234                                 const FileEntry *File,
235                                 SmallString<128> &Result) {
236   Result.clear();
237 
238   StringRef FilePath = File->getDir()->getName();
239   StringRef Path = FilePath;
240   while (!Path.empty()) {
241     if (const DirectoryEntry *CurDir = FM.getDirectory(Path)) {
242       if (CurDir == Dir) {
243         Result = FilePath.substr(Path.size());
244         llvm::sys::path::append(Result,
245                                 llvm::sys::path::filename(File->getName()));
246         return;
247       }
248     }
249 
250     Path = llvm::sys::path::parent_path(Path);
251   }
252 
253   Result = File->getName();
254 }
255 
PropagateLineStartLeadingSpaceInfo(Token & Result)256 void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
257   if (CurTokenLexer) {
258     CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
259     return;
260   }
261   if (CurLexer) {
262     CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
263     return;
264   }
265   // FIXME: Handle other kinds of lexers?  It generally shouldn't matter,
266   // but it might if they're empty?
267 }
268 
269 /// \brief Determine the location to use as the end of the buffer for a lexer.
270 ///
271 /// If the file ends with a newline, form the EOF token on the newline itself,
272 /// rather than "on the line following it", which doesn't exist.  This makes
273 /// diagnostics relating to the end of file include the last file that the user
274 /// actually typed, which is goodness.
getCurLexerEndPos()275 const char *Preprocessor::getCurLexerEndPos() {
276   const char *EndPos = CurLexer->BufferEnd;
277   if (EndPos != CurLexer->BufferStart &&
278       (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
279     --EndPos;
280 
281     // Handle \n\r and \r\n:
282     if (EndPos != CurLexer->BufferStart &&
283         (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
284         EndPos[-1] != EndPos[0])
285       --EndPos;
286   }
287 
288   return EndPos;
289 }
290 
291 
292 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
293 /// the current file.  This either returns the EOF token or pops a level off
294 /// the include stack and keeps going.
HandleEndOfFile(Token & Result,bool isEndOfMacro)295 bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
296   assert(!CurTokenLexer &&
297          "Ending a file when currently in a macro!");
298 
299   // See if this file had a controlling macro.
300   if (CurPPLexer) {  // Not ending a macro, ignore it.
301     if (const IdentifierInfo *ControllingMacro =
302           CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
303       // Okay, this has a controlling macro, remember in HeaderFileInfo.
304       if (const FileEntry *FE = CurPPLexer->getFileEntry()) {
305         HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
306         if (MacroInfo *MI =
307               getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro))) {
308           MI->UsedForHeaderGuard = true;
309         }
310         if (const IdentifierInfo *DefinedMacro =
311               CurPPLexer->MIOpt.GetDefinedMacro()) {
312           if (!isMacroDefined(ControllingMacro) &&
313               DefinedMacro != ControllingMacro &&
314               HeaderInfo.FirstTimeLexingFile(FE)) {
315 
316             // If the edit distance between the two macros is more than 50%,
317             // DefinedMacro may not be header guard, or can be header guard of
318             // another header file. Therefore, it maybe defining something
319             // completely different. This can be observed in the wild when
320             // handling feature macros or header guards in different files.
321 
322             const StringRef ControllingMacroName = ControllingMacro->getName();
323             const StringRef DefinedMacroName = DefinedMacro->getName();
324             const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
325                                                   DefinedMacroName.size()) / 2;
326             const unsigned ED = ControllingMacroName.edit_distance(
327                 DefinedMacroName, true, MaxHalfLength);
328             if (ED <= MaxHalfLength) {
329               // Emit a warning for a bad header guard.
330               Diag(CurPPLexer->MIOpt.GetMacroLocation(),
331                    diag::warn_header_guard)
332                   << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
333               Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
334                    diag::note_header_guard)
335                   << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
336                   << ControllingMacro
337                   << FixItHint::CreateReplacement(
338                          CurPPLexer->MIOpt.GetDefinedLocation(),
339                          ControllingMacro->getName());
340             }
341           }
342         }
343       }
344     }
345   }
346 
347   // Complain about reaching a true EOF within arc_cf_code_audited.
348   // We don't want to complain about reaching the end of a macro
349   // instantiation or a _Pragma.
350   if (PragmaARCCFCodeAuditedLoc.isValid() &&
351       !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
352     Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
353 
354     // Recover by leaving immediately.
355     PragmaARCCFCodeAuditedLoc = SourceLocation();
356   }
357 
358   // Complain about reaching a true EOF within assume_nonnull.
359   // We don't want to complain about reaching the end of a macro
360   // instantiation or a _Pragma.
361   if (PragmaAssumeNonNullLoc.isValid() &&
362       !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
363     Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
364 
365     // Recover by leaving immediately.
366     PragmaAssumeNonNullLoc = SourceLocation();
367   }
368 
369   // If this is a #include'd file, pop it off the include stack and continue
370   // lexing the #includer file.
371   if (!IncludeMacroStack.empty()) {
372 
373     // If we lexed the code-completion file, act as if we reached EOF.
374     if (isCodeCompletionEnabled() && CurPPLexer &&
375         SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
376             CodeCompletionFileLoc) {
377       if (CurLexer) {
378         Result.startToken();
379         CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
380         CurLexer.reset();
381       } else {
382         assert(CurPTHLexer && "Got EOF but no current lexer set!");
383         CurPTHLexer->getEOF(Result);
384         CurPTHLexer.reset();
385       }
386 
387       CurPPLexer = nullptr;
388       return true;
389     }
390 
391     if (!isEndOfMacro && CurPPLexer &&
392         SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
393       // Notify SourceManager to record the number of FileIDs that were created
394       // during lexing of the #include'd file.
395       unsigned NumFIDs =
396           SourceMgr.local_sloc_entry_size() -
397           CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
398       SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
399     }
400 
401     FileID ExitedFID;
402     if (Callbacks && !isEndOfMacro && CurPPLexer)
403       ExitedFID = CurPPLexer->getFileID();
404 
405     bool LeavingSubmodule = CurSubmodule && CurLexer;
406     if (LeavingSubmodule) {
407       // Notify the parser that we've left the module.
408       const char *EndPos = getCurLexerEndPos();
409       Result.startToken();
410       CurLexer->BufferPtr = EndPos;
411       CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
412       Result.setAnnotationEndLoc(Result.getLocation());
413       Result.setAnnotationValue(CurSubmodule);
414 
415       // We're done with this submodule.
416       LeaveSubmodule();
417     }
418 
419     // We're done with the #included file.
420     RemoveTopOfLexerStack();
421 
422     // Propagate info about start-of-line/leading white-space/etc.
423     PropagateLineStartLeadingSpaceInfo(Result);
424 
425     // Notify the client, if desired, that we are in a new source file.
426     if (Callbacks && !isEndOfMacro && CurPPLexer) {
427       SrcMgr::CharacteristicKind FileType =
428         SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
429       Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
430                              PPCallbacks::ExitFile, FileType, ExitedFID);
431     }
432 
433     // Client should lex another token unless we generated an EOM.
434     return LeavingSubmodule;
435   }
436 
437   // If this is the end of the main file, form an EOF token.
438   if (CurLexer) {
439     const char *EndPos = getCurLexerEndPos();
440     Result.startToken();
441     CurLexer->BufferPtr = EndPos;
442     CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
443 
444     if (isCodeCompletionEnabled()) {
445       // Inserting the code-completion point increases the source buffer by 1,
446       // but the main FileID was created before inserting the point.
447       // Compensate by reducing the EOF location by 1, otherwise the location
448       // will point to the next FileID.
449       // FIXME: This is hacky, the code-completion point should probably be
450       // inserted before the main FileID is created.
451       if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
452         Result.setLocation(Result.getLocation().getLocWithOffset(-1));
453     }
454 
455     if (!isIncrementalProcessingEnabled())
456       // We're done with lexing.
457       CurLexer.reset();
458   } else {
459     assert(CurPTHLexer && "Got EOF but no current lexer set!");
460     CurPTHLexer->getEOF(Result);
461     CurPTHLexer.reset();
462   }
463 
464   if (!isIncrementalProcessingEnabled())
465     CurPPLexer = nullptr;
466 
467   if (TUKind == TU_Complete) {
468     // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
469     // collected all macro locations that we need to warn because they are not
470     // used.
471     for (WarnUnusedMacroLocsTy::iterator
472            I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
473            I!=E; ++I)
474       Diag(*I, diag::pp_macro_not_used);
475   }
476 
477   // If we are building a module that has an umbrella header, make sure that
478   // each of the headers within the directory covered by the umbrella header
479   // was actually included by the umbrella header.
480   if (Module *Mod = getCurrentModule()) {
481     if (Mod->getUmbrellaHeader()) {
482       SourceLocation StartLoc
483         = SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
484 
485       if (!getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
486                                       StartLoc)) {
487         ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
488         const DirectoryEntry *Dir = Mod->getUmbrellaDir().Entry;
489         vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem();
490         std::error_code EC;
491         for (vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC), End;
492              Entry != End && !EC; Entry.increment(EC)) {
493           using llvm::StringSwitch;
494 
495           // Check whether this entry has an extension typically associated with
496           // headers.
497           if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->getName()))
498                  .Cases(".h", ".H", ".hh", ".hpp", true)
499                  .Default(false))
500             continue;
501 
502           if (const FileEntry *Header =
503                   getFileManager().getFile(Entry->getName()))
504             if (!getSourceManager().hasFileInfo(Header)) {
505               if (!ModMap.isHeaderInUnavailableModule(Header)) {
506                 // Find the relative path that would access this header.
507                 SmallString<128> RelativePath;
508                 computeRelativePath(FileMgr, Dir, Header, RelativePath);
509                 Diag(StartLoc, diag::warn_uncovered_module_header)
510                   << Mod->getFullModuleName() << RelativePath;
511               }
512             }
513         }
514       }
515     }
516   }
517 
518   return true;
519 }
520 
521 /// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
522 /// hits the end of its token stream.
HandleEndOfTokenLexer(Token & Result)523 bool Preprocessor::HandleEndOfTokenLexer(Token &Result) {
524   assert(CurTokenLexer && !CurPPLexer &&
525          "Ending a macro when currently in a #include file!");
526 
527   if (!MacroExpandingLexersStack.empty() &&
528       MacroExpandingLexersStack.back().first == CurTokenLexer.get())
529     removeCachedMacroExpandedTokensOfLastLexer();
530 
531   // Delete or cache the now-dead macro expander.
532   if (NumCachedTokenLexers == TokenLexerCacheSize)
533     CurTokenLexer.reset();
534   else
535     TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
536 
537   // Handle this like a #include file being popped off the stack.
538   return HandleEndOfFile(Result, true);
539 }
540 
541 /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
542 /// lexer stack.  This should only be used in situations where the current
543 /// state of the top-of-stack lexer is unknown.
RemoveTopOfLexerStack()544 void Preprocessor::RemoveTopOfLexerStack() {
545   assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
546 
547   if (CurTokenLexer) {
548     // Delete or cache the now-dead macro expander.
549     if (NumCachedTokenLexers == TokenLexerCacheSize)
550       CurTokenLexer.reset();
551     else
552       TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
553   }
554 
555   PopIncludeMacroStack();
556 }
557 
558 /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
559 /// comment (/##/) in microsoft mode, this method handles updating the current
560 /// state, returning the token on the next source line.
HandleMicrosoftCommentPaste(Token & Tok)561 void Preprocessor::HandleMicrosoftCommentPaste(Token &Tok) {
562   assert(CurTokenLexer && !CurPPLexer &&
563          "Pasted comment can only be formed from macro");
564   // We handle this by scanning for the closest real lexer, switching it to
565   // raw mode and preprocessor mode.  This will cause it to return \n as an
566   // explicit EOD token.
567   PreprocessorLexer *FoundLexer = nullptr;
568   bool LexerWasInPPMode = false;
569   for (unsigned i = 0, e = IncludeMacroStack.size(); i != e; ++i) {
570     IncludeStackInfo &ISI = *(IncludeMacroStack.end()-i-1);
571     if (ISI.ThePPLexer == nullptr) continue;  // Scan for a real lexer.
572 
573     // Once we find a real lexer, mark it as raw mode (disabling macro
574     // expansions) and preprocessor mode (return EOD).  We know that the lexer
575     // was *not* in raw mode before, because the macro that the comment came
576     // from was expanded.  However, it could have already been in preprocessor
577     // mode (#if COMMENT) in which case we have to return it to that mode and
578     // return EOD.
579     FoundLexer = ISI.ThePPLexer;
580     FoundLexer->LexingRawMode = true;
581     LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
582     FoundLexer->ParsingPreprocessorDirective = true;
583     break;
584   }
585 
586   // Okay, we either found and switched over the lexer, or we didn't find a
587   // lexer.  In either case, finish off the macro the comment came from, getting
588   // the next token.
589   if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
590 
591   // Discarding comments as long as we don't have EOF or EOD.  This 'comments
592   // out' the rest of the line, including any tokens that came from other macros
593   // that were active, as in:
594   //  #define submacro a COMMENT b
595   //    submacro c
596   // which should lex to 'a' only: 'b' and 'c' should be removed.
597   while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
598     Lex(Tok);
599 
600   // If we got an eod token, then we successfully found the end of the line.
601   if (Tok.is(tok::eod)) {
602     assert(FoundLexer && "Can't get end of line without an active lexer");
603     // Restore the lexer back to normal mode instead of raw mode.
604     FoundLexer->LexingRawMode = false;
605 
606     // If the lexer was already in preprocessor mode, just return the EOD token
607     // to finish the preprocessor line.
608     if (LexerWasInPPMode) return;
609 
610     // Otherwise, switch out of PP mode and return the next lexed token.
611     FoundLexer->ParsingPreprocessorDirective = false;
612     return Lex(Tok);
613   }
614 
615   // If we got an EOF token, then we reached the end of the token stream but
616   // didn't find an explicit \n.  This can only happen if there was no lexer
617   // active (an active lexer would return EOD at EOF if there was no \n in
618   // preprocessor directive mode), so just return EOF as our token.
619   assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
620 }
621 
EnterSubmodule(Module * M,SourceLocation ImportLoc)622 void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc) {
623   if (!getLangOpts().ModulesLocalVisibility) {
624     // Just track that we entered this submodule.
625     BuildingSubmoduleStack.push_back(BuildingSubmoduleInfo(
626         M, ImportLoc, CurSubmoduleState, PendingModuleMacroNames.size()));
627     return;
628   }
629 
630   // Resolve as much of the module definition as we can now, before we enter
631   // one of its headers.
632   // FIXME: Can we enable Complain here?
633   // FIXME: Can we do this when local visibility is disabled?
634   ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
635   ModMap.resolveExports(M, /*Complain=*/false);
636   ModMap.resolveUses(M, /*Complain=*/false);
637   ModMap.resolveConflicts(M, /*Complain=*/false);
638 
639   // If this is the first time we've entered this module, set up its state.
640   auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
641   auto &State = R.first->second;
642   bool FirstTime = R.second;
643   if (FirstTime) {
644     // Determine the set of starting macros for this submodule; take these
645     // from the "null" module (the predefines buffer).
646     //
647     // FIXME: If we have local visibility but not modules enabled, the
648     // NullSubmoduleState is polluted by #defines in the top-level source
649     // file.
650     auto &StartingMacros = NullSubmoduleState.Macros;
651 
652     // Restore to the starting state.
653     // FIXME: Do this lazily, when each macro name is first referenced.
654     for (auto &Macro : StartingMacros) {
655       // Skip uninteresting macros.
656       if (!Macro.second.getLatest() &&
657           Macro.second.getOverriddenMacros().empty())
658         continue;
659 
660       MacroState MS(Macro.second.getLatest());
661       MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
662       State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
663     }
664   }
665 
666   // Track that we entered this module.
667   BuildingSubmoduleStack.push_back(BuildingSubmoduleInfo(
668       M, ImportLoc, CurSubmoduleState, PendingModuleMacroNames.size()));
669 
670   // Switch to this submodule as the current submodule.
671   CurSubmoduleState = &State;
672 
673   // This module is visible to itself.
674   if (FirstTime)
675     makeModuleVisible(M, ImportLoc);
676 }
677 
needModuleMacros() const678 bool Preprocessor::needModuleMacros() const {
679   // If we're not within a submodule, we never need to create ModuleMacros.
680   if (BuildingSubmoduleStack.empty())
681     return false;
682   // If we are tracking module macro visibility even for textually-included
683   // headers, we need ModuleMacros.
684   if (getLangOpts().ModulesLocalVisibility)
685     return true;
686   // Otherwise, we only need module macros if we're actually compiling a module
687   // interface.
688   return getLangOpts().CompilingModule;
689 }
690 
LeaveSubmodule()691 void Preprocessor::LeaveSubmodule() {
692   auto &Info = BuildingSubmoduleStack.back();
693 
694   Module *LeavingMod = Info.M;
695   SourceLocation ImportLoc = Info.ImportLoc;
696 
697   if (!needModuleMacros() ||
698       (!getLangOpts().ModulesLocalVisibility &&
699        LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) {
700     // If we don't need module macros, or this is not a module for which we
701     // are tracking macro visibility, don't build any, and preserve the list
702     // of pending names for the surrounding submodule.
703     BuildingSubmoduleStack.pop_back();
704     makeModuleVisible(LeavingMod, ImportLoc);
705     return;
706   }
707 
708   // Create ModuleMacros for any macros defined in this submodule.
709   llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
710   for (unsigned I = Info.OuterPendingModuleMacroNames;
711        I != PendingModuleMacroNames.size(); ++I) {
712     auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]);
713     if (!VisitedMacros.insert(II).second)
714       continue;
715 
716     auto MacroIt = CurSubmoduleState->Macros.find(II);
717     if (MacroIt == CurSubmoduleState->Macros.end())
718       continue;
719     auto &Macro = MacroIt->second;
720 
721     // Find the starting point for the MacroDirective chain in this submodule.
722     MacroDirective *OldMD = nullptr;
723     auto *OldState = Info.OuterSubmoduleState;
724     if (getLangOpts().ModulesLocalVisibility)
725       OldState = &NullSubmoduleState;
726     if (OldState && OldState != CurSubmoduleState) {
727       // FIXME: It'd be better to start at the state from when we most recently
728       // entered this submodule, but it doesn't really matter.
729       auto &OldMacros = OldState->Macros;
730       auto OldMacroIt = OldMacros.find(II);
731       if (OldMacroIt == OldMacros.end())
732         OldMD = nullptr;
733       else
734         OldMD = OldMacroIt->second.getLatest();
735     }
736 
737     // This module may have exported a new macro. If so, create a ModuleMacro
738     // representing that fact.
739     bool ExplicitlyPublic = false;
740     for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) {
741       assert(MD && "broken macro directive chain");
742 
743       // Stop on macros defined in other submodules of this module that we
744       // #included along the way. There's no point doing this if we're
745       // tracking local submodule visibility, since there can be no such
746       // directives in our list.
747       if (!getLangOpts().ModulesLocalVisibility) {
748         Module *Mod = getModuleContainingLocation(MD->getLocation());
749         if (Mod != LeavingMod &&
750             Mod->getTopLevelModule() == LeavingMod->getTopLevelModule())
751           break;
752       }
753 
754       if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
755         // The latest visibility directive for a name in a submodule affects
756         // all the directives that come before it.
757         if (VisMD->isPublic())
758           ExplicitlyPublic = true;
759         else if (!ExplicitlyPublic)
760           // Private with no following public directive: not exported.
761           break;
762       } else {
763         MacroInfo *Def = nullptr;
764         if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
765           Def = DefMD->getInfo();
766 
767         // FIXME: Issue a warning if multiple headers for the same submodule
768         // define a macro, rather than silently ignoring all but the first.
769         bool IsNew;
770         // Don't bother creating a module macro if it would represent a #undef
771         // that doesn't override anything.
772         if (Def || !Macro.getOverriddenMacros().empty())
773           addModuleMacro(LeavingMod, II, Def,
774                          Macro.getOverriddenMacros(), IsNew);
775         break;
776       }
777     }
778   }
779   PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
780 
781   // FIXME: Before we leave this submodule, we should parse all the other
782   // headers within it. Otherwise, we're left with an inconsistent state
783   // where we've made the module visible but don't yet have its complete
784   // contents.
785 
786   // Put back the outer module's state, if we're tracking it.
787   if (getLangOpts().ModulesLocalVisibility)
788     CurSubmoduleState = Info.OuterSubmoduleState;
789 
790   BuildingSubmoduleStack.pop_back();
791 
792   // A nested #include makes the included submodule visible.
793   makeModuleVisible(LeavingMod, ImportLoc);
794 }
795