• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 //  This file implements the Preprocessor interface.
10 //
11 //===----------------------------------------------------------------------===//
12 //
13 // Options to support:
14 //   -H       - Print the name of each header file used.
15 //   -d[DNI] - Dump various things.
16 //   -fworking-directory - #line's with preprocessor's working dir.
17 //   -fpreprocessed
18 //   -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19 //   -W*
20 //   -w
21 //
22 // Messages to emit:
23 //   "Multiple include guards may be useful for:\n"
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "clang/Lex/Preprocessor.h"
28 #include "clang/Basic/Builtins.h"
29 #include "clang/Basic/FileManager.h"
30 #include "clang/Basic/FileSystemStatCache.h"
31 #include "clang/Basic/IdentifierTable.h"
32 #include "clang/Basic/LLVM.h"
33 #include "clang/Basic/LangOptions.h"
34 #include "clang/Basic/Module.h"
35 #include "clang/Basic/SourceLocation.h"
36 #include "clang/Basic/SourceManager.h"
37 #include "clang/Basic/TargetInfo.h"
38 #include "clang/Lex/CodeCompletionHandler.h"
39 #include "clang/Lex/ExternalPreprocessorSource.h"
40 #include "clang/Lex/HeaderSearch.h"
41 #include "clang/Lex/LexDiagnostic.h"
42 #include "clang/Lex/Lexer.h"
43 #include "clang/Lex/LiteralSupport.h"
44 #include "clang/Lex/MacroArgs.h"
45 #include "clang/Lex/MacroInfo.h"
46 #include "clang/Lex/ModuleLoader.h"
47 #include "clang/Lex/Pragma.h"
48 #include "clang/Lex/PreprocessingRecord.h"
49 #include "clang/Lex/PreprocessorLexer.h"
50 #include "clang/Lex/PreprocessorOptions.h"
51 #include "clang/Lex/ScratchBuffer.h"
52 #include "clang/Lex/Token.h"
53 #include "clang/Lex/TokenLexer.h"
54 #include "llvm/ADT/APInt.h"
55 #include "llvm/ADT/ArrayRef.h"
56 #include "llvm/ADT/DenseMap.h"
57 #include "llvm/ADT/STLExtras.h"
58 #include "llvm/ADT/SmallString.h"
59 #include "llvm/ADT/SmallVector.h"
60 #include "llvm/ADT/StringRef.h"
61 #include "llvm/ADT/StringSwitch.h"
62 #include "llvm/Support/Capacity.h"
63 #include "llvm/Support/ErrorHandling.h"
64 #include "llvm/Support/MemoryBuffer.h"
65 #include "llvm/Support/raw_ostream.h"
66 #include <algorithm>
67 #include <cassert>
68 #include <memory>
69 #include <string>
70 #include <utility>
71 #include <vector>
72 
73 using namespace clang;
74 
75 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
76 
77 ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
78 
Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,DiagnosticsEngine & diags,LangOptions & opts,SourceManager & SM,HeaderSearch & Headers,ModuleLoader & TheModuleLoader,IdentifierInfoLookup * IILookup,bool OwnsHeaders,TranslationUnitKind TUKind)79 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
80                            DiagnosticsEngine &diags, LangOptions &opts,
81                            SourceManager &SM, HeaderSearch &Headers,
82                            ModuleLoader &TheModuleLoader,
83                            IdentifierInfoLookup *IILookup, bool OwnsHeaders,
84                            TranslationUnitKind TUKind)
85     : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86       FileMgr(Headers.getFileMgr()), SourceMgr(SM),
87       ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88       TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
89       // As the language options may have not been loaded yet (when
90       // deserializing an ASTUnit), adding keywords to the identifier table is
91       // deferred to Preprocessor::Initialize().
92       Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
93       TUKind(TUKind), SkipMainFilePreamble(0, true),
94       CurSubmoduleState(&NullSubmoduleState) {
95   OwnsHeaderSearch = OwnsHeaders;
96 
97   // Default to discarding comments.
98   KeepComments = false;
99   KeepMacroComments = false;
100   SuppressIncludeNotFoundError = false;
101 
102   // Macro expansion is enabled.
103   DisableMacroExpansion = false;
104   MacroExpansionInDirectivesOverride = false;
105   InMacroArgs = false;
106   ArgMacro = nullptr;
107   InMacroArgPreExpansion = false;
108   NumCachedTokenLexers = 0;
109   PragmasEnabled = true;
110   ParsingIfOrElifDirective = false;
111   PreprocessedOutput = false;
112 
113   // We haven't read anything from the external source.
114   ReadMacrosFromExternalSource = false;
115 
116   BuiltinInfo = std::make_unique<Builtin::Context>();
117 
118   // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
119   // a macro. They get unpoisoned where it is allowed.
120   (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
121   SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
122   if (getLangOpts().CPlusPlus20) {
123     (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
124     SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
125   } else {
126     Ident__VA_OPT__ = nullptr;
127   }
128 
129   // Initialize the pragma handlers.
130   RegisterBuiltinPragmas();
131 
132   // Initialize builtin macros like __LINE__ and friends.
133   RegisterBuiltinMacros();
134 
135   if(LangOpts.Borland) {
136     Ident__exception_info        = getIdentifierInfo("_exception_info");
137     Ident___exception_info       = getIdentifierInfo("__exception_info");
138     Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
139     Ident__exception_code        = getIdentifierInfo("_exception_code");
140     Ident___exception_code       = getIdentifierInfo("__exception_code");
141     Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
142     Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
143     Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
144     Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
145   } else {
146     Ident__exception_info = Ident__exception_code = nullptr;
147     Ident__abnormal_termination = Ident___exception_info = nullptr;
148     Ident___exception_code = Ident___abnormal_termination = nullptr;
149     Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
150     Ident_AbnormalTermination = nullptr;
151   }
152 
153   // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
154   if (usingPCHWithPragmaHdrStop())
155     SkippingUntilPragmaHdrStop = true;
156 
157   // If using a PCH with a through header, start skipping tokens.
158   if (!this->PPOpts->PCHThroughHeader.empty() &&
159       !this->PPOpts->ImplicitPCHInclude.empty())
160     SkippingUntilPCHThroughHeader = true;
161 
162   if (this->PPOpts->GeneratePreamble)
163     PreambleConditionalStack.startRecording();
164 
165   ExcludedConditionalDirectiveSkipMappings =
166       this->PPOpts->ExcludedConditionalDirectiveSkipMappings;
167   if (ExcludedConditionalDirectiveSkipMappings)
168     ExcludedConditionalDirectiveSkipMappings->clear();
169 
170   MaxTokens = LangOpts.MaxTokens;
171 }
172 
~Preprocessor()173 Preprocessor::~Preprocessor() {
174   assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
175 
176   IncludeMacroStack.clear();
177 
178   // Destroy any macro definitions.
179   while (MacroInfoChain *I = MIChainHead) {
180     MIChainHead = I->Next;
181     I->~MacroInfoChain();
182   }
183 
184   // Free any cached macro expanders.
185   // This populates MacroArgCache, so all TokenLexers need to be destroyed
186   // before the code below that frees up the MacroArgCache list.
187   std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
188   CurTokenLexer.reset();
189 
190   // Free any cached MacroArgs.
191   for (MacroArgs *ArgList = MacroArgCache; ArgList;)
192     ArgList = ArgList->deallocate();
193 
194   // Delete the header search info, if we own it.
195   if (OwnsHeaderSearch)
196     delete &HeaderInfo;
197 }
198 
Initialize(const TargetInfo & Target,const TargetInfo * AuxTarget)199 void Preprocessor::Initialize(const TargetInfo &Target,
200                               const TargetInfo *AuxTarget) {
201   assert((!this->Target || this->Target == &Target) &&
202          "Invalid override of target information");
203   this->Target = &Target;
204 
205   assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
206          "Invalid override of aux target information.");
207   this->AuxTarget = AuxTarget;
208 
209   // Initialize information about built-ins.
210   BuiltinInfo->InitializeTarget(Target, AuxTarget);
211   HeaderInfo.setTarget(Target);
212 
213   // Populate the identifier table with info about keywords for the current language.
214   Identifiers.AddKeywords(LangOpts);
215 }
216 
InitializeForModelFile()217 void Preprocessor::InitializeForModelFile() {
218   NumEnteredSourceFiles = 0;
219 
220   // Reset pragmas
221   PragmaHandlersBackup = std::move(PragmaHandlers);
222   PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
223   RegisterBuiltinPragmas();
224 
225   // Reset PredefinesFileID
226   PredefinesFileID = FileID();
227 }
228 
FinalizeForModelFile()229 void Preprocessor::FinalizeForModelFile() {
230   NumEnteredSourceFiles = 1;
231 
232   PragmaHandlers = std::move(PragmaHandlersBackup);
233 }
234 
DumpToken(const Token & Tok,bool DumpFlags) const235 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
236   llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
237                << getSpelling(Tok) << "'";
238 
239   if (!DumpFlags) return;
240 
241   llvm::errs() << "\t";
242   if (Tok.isAtStartOfLine())
243     llvm::errs() << " [StartOfLine]";
244   if (Tok.hasLeadingSpace())
245     llvm::errs() << " [LeadingSpace]";
246   if (Tok.isExpandDisabled())
247     llvm::errs() << " [ExpandDisabled]";
248   if (Tok.needsCleaning()) {
249     const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
250     llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
251                  << "']";
252   }
253 
254   llvm::errs() << "\tLoc=<";
255   DumpLocation(Tok.getLocation());
256   llvm::errs() << ">";
257 }
258 
DumpLocation(SourceLocation Loc) const259 void Preprocessor::DumpLocation(SourceLocation Loc) const {
260   Loc.print(llvm::errs(), SourceMgr);
261 }
262 
DumpMacro(const MacroInfo & MI) const263 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
264   llvm::errs() << "MACRO: ";
265   for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
266     DumpToken(MI.getReplacementToken(i));
267     llvm::errs() << "  ";
268   }
269   llvm::errs() << "\n";
270 }
271 
PrintStats()272 void Preprocessor::PrintStats() {
273   llvm::errs() << "\n*** Preprocessor Stats:\n";
274   llvm::errs() << NumDirectives << " directives found:\n";
275   llvm::errs() << "  " << NumDefined << " #define.\n";
276   llvm::errs() << "  " << NumUndefined << " #undef.\n";
277   llvm::errs() << "  #include/#include_next/#import:\n";
278   llvm::errs() << "    " << NumEnteredSourceFiles << " source files entered.\n";
279   llvm::errs() << "    " << MaxIncludeStackDepth << " max include stack depth\n";
280   llvm::errs() << "  " << NumIf << " #if/#ifndef/#ifdef.\n";
281   llvm::errs() << "  " << NumElse << " #else/#elif.\n";
282   llvm::errs() << "  " << NumEndif << " #endif.\n";
283   llvm::errs() << "  " << NumPragma << " #pragma.\n";
284   llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
285 
286   llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
287              << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
288              << NumFastMacroExpanded << " on the fast path.\n";
289   llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
290              << " token paste (##) operations performed, "
291              << NumFastTokenPaste << " on the fast path.\n";
292 
293   llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
294 
295   llvm::errs() << "\n  BumpPtr: " << BP.getTotalMemory();
296   llvm::errs() << "\n  Macro Expanded Tokens: "
297                << llvm::capacity_in_bytes(MacroExpandedTokens);
298   llvm::errs() << "\n  Predefines Buffer: " << Predefines.capacity();
299   // FIXME: List information for all submodules.
300   llvm::errs() << "\n  Macros: "
301                << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
302   llvm::errs() << "\n  #pragma push_macro Info: "
303                << llvm::capacity_in_bytes(PragmaPushMacroInfo);
304   llvm::errs() << "\n  Poison Reasons: "
305                << llvm::capacity_in_bytes(PoisonReasons);
306   llvm::errs() << "\n  Comment Handlers: "
307                << llvm::capacity_in_bytes(CommentHandlers) << "\n";
308 }
309 
310 Preprocessor::macro_iterator
macro_begin(bool IncludeExternalMacros) const311 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
312   if (IncludeExternalMacros && ExternalSource &&
313       !ReadMacrosFromExternalSource) {
314     ReadMacrosFromExternalSource = true;
315     ExternalSource->ReadDefinedMacros();
316   }
317 
318   // Make sure we cover all macros in visible modules.
319   for (const ModuleMacro &Macro : ModuleMacros)
320     CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
321 
322   return CurSubmoduleState->Macros.begin();
323 }
324 
getTotalMemory() const325 size_t Preprocessor::getTotalMemory() const {
326   return BP.getTotalMemory()
327     + llvm::capacity_in_bytes(MacroExpandedTokens)
328     + Predefines.capacity() /* Predefines buffer. */
329     // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
330     // and ModuleMacros.
331     + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
332     + llvm::capacity_in_bytes(PragmaPushMacroInfo)
333     + llvm::capacity_in_bytes(PoisonReasons)
334     + llvm::capacity_in_bytes(CommentHandlers);
335 }
336 
337 Preprocessor::macro_iterator
macro_end(bool IncludeExternalMacros) const338 Preprocessor::macro_end(bool IncludeExternalMacros) const {
339   if (IncludeExternalMacros && ExternalSource &&
340       !ReadMacrosFromExternalSource) {
341     ReadMacrosFromExternalSource = true;
342     ExternalSource->ReadDefinedMacros();
343   }
344 
345   return CurSubmoduleState->Macros.end();
346 }
347 
348 /// Compares macro tokens with a specified token value sequence.
MacroDefinitionEquals(const MacroInfo * MI,ArrayRef<TokenValue> Tokens)349 static bool MacroDefinitionEquals(const MacroInfo *MI,
350                                   ArrayRef<TokenValue> Tokens) {
351   return Tokens.size() == MI->getNumTokens() &&
352       std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
353 }
354 
getLastMacroWithSpelling(SourceLocation Loc,ArrayRef<TokenValue> Tokens) const355 StringRef Preprocessor::getLastMacroWithSpelling(
356                                     SourceLocation Loc,
357                                     ArrayRef<TokenValue> Tokens) const {
358   SourceLocation BestLocation;
359   StringRef BestSpelling;
360   for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
361        I != E; ++I) {
362     const MacroDirective::DefInfo
363       Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
364     if (!Def || !Def.getMacroInfo())
365       continue;
366     if (!Def.getMacroInfo()->isObjectLike())
367       continue;
368     if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
369       continue;
370     SourceLocation Location = Def.getLocation();
371     // Choose the macro defined latest.
372     if (BestLocation.isInvalid() ||
373         (Location.isValid() &&
374          SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
375       BestLocation = Location;
376       BestSpelling = I->first->getName();
377     }
378   }
379   return BestSpelling;
380 }
381 
recomputeCurLexerKind()382 void Preprocessor::recomputeCurLexerKind() {
383   if (CurLexer)
384     CurLexerKind = CLK_Lexer;
385   else if (CurTokenLexer)
386     CurLexerKind = CLK_TokenLexer;
387   else
388     CurLexerKind = CLK_CachingLexer;
389 }
390 
SetCodeCompletionPoint(const FileEntry * File,unsigned CompleteLine,unsigned CompleteColumn)391 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
392                                           unsigned CompleteLine,
393                                           unsigned CompleteColumn) {
394   assert(File);
395   assert(CompleteLine && CompleteColumn && "Starts from 1:1");
396   assert(!CodeCompletionFile && "Already set");
397 
398   // Load the actual file's contents.
399   Optional<llvm::MemoryBufferRef> Buffer =
400       SourceMgr.getMemoryBufferForFileOrNone(File);
401   if (!Buffer)
402     return true;
403 
404   // Find the byte position of the truncation point.
405   const char *Position = Buffer->getBufferStart();
406   for (unsigned Line = 1; Line < CompleteLine; ++Line) {
407     for (; *Position; ++Position) {
408       if (*Position != '\r' && *Position != '\n')
409         continue;
410 
411       // Eat \r\n or \n\r as a single line.
412       if ((Position[1] == '\r' || Position[1] == '\n') &&
413           Position[0] != Position[1])
414         ++Position;
415       ++Position;
416       break;
417     }
418   }
419 
420   Position += CompleteColumn - 1;
421 
422   // If pointing inside the preamble, adjust the position at the beginning of
423   // the file after the preamble.
424   if (SkipMainFilePreamble.first &&
425       SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
426     if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
427       Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
428   }
429 
430   if (Position > Buffer->getBufferEnd())
431     Position = Buffer->getBufferEnd();
432 
433   CodeCompletionFile = File;
434   CodeCompletionOffset = Position - Buffer->getBufferStart();
435 
436   auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
437       Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
438   char *NewBuf = NewBuffer->getBufferStart();
439   char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
440   *NewPos = '\0';
441   std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
442   SourceMgr.overrideFileContents(File, std::move(NewBuffer));
443 
444   return false;
445 }
446 
CodeCompleteIncludedFile(llvm::StringRef Dir,bool IsAngled)447 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
448                                             bool IsAngled) {
449   if (CodeComplete)
450     CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
451   setCodeCompletionReached();
452 }
453 
CodeCompleteNaturalLanguage()454 void Preprocessor::CodeCompleteNaturalLanguage() {
455   if (CodeComplete)
456     CodeComplete->CodeCompleteNaturalLanguage();
457   setCodeCompletionReached();
458 }
459 
460 /// getSpelling - This method is used to get the spelling of a token into a
461 /// SmallVector. Note that the returned StringRef may not point to the
462 /// supplied buffer if a copy can be avoided.
getSpelling(const Token & Tok,SmallVectorImpl<char> & Buffer,bool * Invalid) const463 StringRef Preprocessor::getSpelling(const Token &Tok,
464                                           SmallVectorImpl<char> &Buffer,
465                                           bool *Invalid) const {
466   // NOTE: this has to be checked *before* testing for an IdentifierInfo.
467   if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
468     // Try the fast path.
469     if (const IdentifierInfo *II = Tok.getIdentifierInfo())
470       return II->getName();
471   }
472 
473   // Resize the buffer if we need to copy into it.
474   if (Tok.needsCleaning())
475     Buffer.resize(Tok.getLength());
476 
477   const char *Ptr = Buffer.data();
478   unsigned Len = getSpelling(Tok, Ptr, Invalid);
479   return StringRef(Ptr, Len);
480 }
481 
482 /// CreateString - Plop the specified string into a scratch buffer and return a
483 /// location for it.  If specified, the source location provides a source
484 /// location for the token.
CreateString(StringRef Str,Token & Tok,SourceLocation ExpansionLocStart,SourceLocation ExpansionLocEnd)485 void Preprocessor::CreateString(StringRef Str, Token &Tok,
486                                 SourceLocation ExpansionLocStart,
487                                 SourceLocation ExpansionLocEnd) {
488   Tok.setLength(Str.size());
489 
490   const char *DestPtr;
491   SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
492 
493   if (ExpansionLocStart.isValid())
494     Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
495                                        ExpansionLocEnd, Str.size());
496   Tok.setLocation(Loc);
497 
498   // If this is a raw identifier or a literal token, set the pointer data.
499   if (Tok.is(tok::raw_identifier))
500     Tok.setRawIdentifierData(DestPtr);
501   else if (Tok.isLiteral())
502     Tok.setLiteralData(DestPtr);
503 }
504 
SplitToken(SourceLocation Loc,unsigned Length)505 SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
506   auto &SM = getSourceManager();
507   SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
508   std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
509   bool Invalid = false;
510   StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
511   if (Invalid)
512     return SourceLocation();
513 
514   // FIXME: We could consider re-using spelling for tokens we see repeatedly.
515   const char *DestPtr;
516   SourceLocation Spelling =
517       ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
518   return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
519 }
520 
getCurrentModule()521 Module *Preprocessor::getCurrentModule() {
522   if (!getLangOpts().isCompilingModule())
523     return nullptr;
524 
525   return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
526 }
527 
528 //===----------------------------------------------------------------------===//
529 // Preprocessor Initialization Methods
530 //===----------------------------------------------------------------------===//
531 
532 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
533 /// which implicitly adds the builtin defines etc.
EnterMainSourceFile()534 void Preprocessor::EnterMainSourceFile() {
535   // We do not allow the preprocessor to reenter the main file.  Doing so will
536   // cause FileID's to accumulate information from both runs (e.g. #line
537   // information) and predefined macros aren't guaranteed to be set properly.
538   assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
539   FileID MainFileID = SourceMgr.getMainFileID();
540 
541   // If MainFileID is loaded it means we loaded an AST file, no need to enter
542   // a main file.
543   if (!SourceMgr.isLoadedFileID(MainFileID)) {
544     // Enter the main file source buffer.
545     EnterSourceFile(MainFileID, nullptr, SourceLocation());
546 
547     // If we've been asked to skip bytes in the main file (e.g., as part of a
548     // precompiled preamble), do so now.
549     if (SkipMainFilePreamble.first > 0)
550       CurLexer->SetByteOffset(SkipMainFilePreamble.first,
551                               SkipMainFilePreamble.second);
552 
553     // Tell the header info that the main file was entered.  If the file is later
554     // #imported, it won't be re-entered.
555     if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
556       HeaderInfo.IncrementIncludeCount(FE);
557   }
558 
559   // Preprocess Predefines to populate the initial preprocessor state.
560   std::unique_ptr<llvm::MemoryBuffer> SB =
561     llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
562   assert(SB && "Cannot create predefined source buffer");
563   FileID FID = SourceMgr.createFileID(std::move(SB));
564   assert(FID.isValid() && "Could not create FileID for predefines?");
565   setPredefinesFileID(FID);
566 
567   // Start parsing the predefines.
568   EnterSourceFile(FID, nullptr, SourceLocation());
569 
570   if (!PPOpts->PCHThroughHeader.empty()) {
571     // Lookup and save the FileID for the through header. If it isn't found
572     // in the search path, it's a fatal error.
573     const DirectoryLookup *CurDir;
574     Optional<FileEntryRef> File = LookupFile(
575         SourceLocation(), PPOpts->PCHThroughHeader,
576         /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, CurDir,
577         /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
578         /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
579         /*IsFrameworkFound=*/nullptr);
580     if (!File) {
581       Diag(SourceLocation(), diag::err_pp_through_header_not_found)
582           << PPOpts->PCHThroughHeader;
583       return;
584     }
585     setPCHThroughHeaderFileID(
586         SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User));
587   }
588 
589   // Skip tokens from the Predefines and if needed the main file.
590   if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
591       (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
592     SkipTokensWhileUsingPCH();
593 }
594 
setPCHThroughHeaderFileID(FileID FID)595 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
596   assert(PCHThroughHeaderFileID.isInvalid() &&
597          "PCHThroughHeaderFileID already set!");
598   PCHThroughHeaderFileID = FID;
599 }
600 
isPCHThroughHeader(const FileEntry * FE)601 bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
602   assert(PCHThroughHeaderFileID.isValid() &&
603          "Invalid PCH through header FileID");
604   return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
605 }
606 
creatingPCHWithThroughHeader()607 bool Preprocessor::creatingPCHWithThroughHeader() {
608   return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
609          PCHThroughHeaderFileID.isValid();
610 }
611 
usingPCHWithThroughHeader()612 bool Preprocessor::usingPCHWithThroughHeader() {
613   return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
614          PCHThroughHeaderFileID.isValid();
615 }
616 
creatingPCHWithPragmaHdrStop()617 bool Preprocessor::creatingPCHWithPragmaHdrStop() {
618   return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
619 }
620 
usingPCHWithPragmaHdrStop()621 bool Preprocessor::usingPCHWithPragmaHdrStop() {
622   return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
623 }
624 
625 /// Skip tokens until after the #include of the through header or
626 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
627 /// and the main file may be skipped. If the end of the predefines file
628 /// is reached, skipping continues into the main file. If the end of the
629 /// main file is reached, it's a fatal error.
SkipTokensWhileUsingPCH()630 void Preprocessor::SkipTokensWhileUsingPCH() {
631   bool ReachedMainFileEOF = false;
632   bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
633   bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
634   Token Tok;
635   while (true) {
636     bool InPredefines =
637         (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
638     switch (CurLexerKind) {
639     case CLK_Lexer:
640       CurLexer->Lex(Tok);
641      break;
642     case CLK_TokenLexer:
643       CurTokenLexer->Lex(Tok);
644       break;
645     case CLK_CachingLexer:
646       CachingLex(Tok);
647       break;
648     case CLK_LexAfterModuleImport:
649       LexAfterModuleImport(Tok);
650       break;
651     }
652     if (Tok.is(tok::eof) && !InPredefines) {
653       ReachedMainFileEOF = true;
654       break;
655     }
656     if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
657       break;
658     if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
659       break;
660   }
661   if (ReachedMainFileEOF) {
662     if (UsingPCHThroughHeader)
663       Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
664           << PPOpts->PCHThroughHeader << 1;
665     else if (!PPOpts->PCHWithHdrStopCreate)
666       Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
667   }
668 }
669 
replayPreambleConditionalStack()670 void Preprocessor::replayPreambleConditionalStack() {
671   // Restore the conditional stack from the preamble, if there is one.
672   if (PreambleConditionalStack.isReplaying()) {
673     assert(CurPPLexer &&
674            "CurPPLexer is null when calling replayPreambleConditionalStack.");
675     CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
676     PreambleConditionalStack.doneReplaying();
677     if (PreambleConditionalStack.reachedEOFWhileSkipping())
678       SkipExcludedConditionalBlock(
679           PreambleConditionalStack.SkipInfo->HashTokenLoc,
680           PreambleConditionalStack.SkipInfo->IfTokenLoc,
681           PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
682           PreambleConditionalStack.SkipInfo->FoundElse,
683           PreambleConditionalStack.SkipInfo->ElseLoc);
684   }
685 }
686 
EndSourceFile()687 void Preprocessor::EndSourceFile() {
688   // Notify the client that we reached the end of the source file.
689   if (Callbacks)
690     Callbacks->EndOfMainFile();
691 }
692 
693 //===----------------------------------------------------------------------===//
694 // Lexer Event Handling.
695 //===----------------------------------------------------------------------===//
696 
697 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
698 /// identifier information for the token and install it into the token,
699 /// updating the token kind accordingly.
LookUpIdentifierInfo(Token & Identifier) const700 IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
701   assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
702 
703   // Look up this token, see if it is a macro, or if it is a language keyword.
704   IdentifierInfo *II;
705   if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
706     // No cleaning needed, just use the characters from the lexed buffer.
707     II = getIdentifierInfo(Identifier.getRawIdentifier());
708   } else {
709     // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
710     SmallString<64> IdentifierBuffer;
711     StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
712 
713     if (Identifier.hasUCN()) {
714       SmallString<64> UCNIdentifierBuffer;
715       expandUCNs(UCNIdentifierBuffer, CleanedStr);
716       II = getIdentifierInfo(UCNIdentifierBuffer);
717     } else {
718       II = getIdentifierInfo(CleanedStr);
719     }
720   }
721 
722   // Update the token info (identifier info and appropriate token kind).
723   Identifier.setIdentifierInfo(II);
724   if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
725       getSourceManager().isInSystemHeader(Identifier.getLocation()))
726     Identifier.setKind(tok::identifier);
727   else
728     Identifier.setKind(II->getTokenID());
729 
730   return II;
731 }
732 
SetPoisonReason(IdentifierInfo * II,unsigned DiagID)733 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
734   PoisonReasons[II] = DiagID;
735 }
736 
PoisonSEHIdentifiers(bool Poison)737 void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
738   assert(Ident__exception_code && Ident__exception_info);
739   assert(Ident___exception_code && Ident___exception_info);
740   Ident__exception_code->setIsPoisoned(Poison);
741   Ident___exception_code->setIsPoisoned(Poison);
742   Ident_GetExceptionCode->setIsPoisoned(Poison);
743   Ident__exception_info->setIsPoisoned(Poison);
744   Ident___exception_info->setIsPoisoned(Poison);
745   Ident_GetExceptionInfo->setIsPoisoned(Poison);
746   Ident__abnormal_termination->setIsPoisoned(Poison);
747   Ident___abnormal_termination->setIsPoisoned(Poison);
748   Ident_AbnormalTermination->setIsPoisoned(Poison);
749 }
750 
HandlePoisonedIdentifier(Token & Identifier)751 void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
752   assert(Identifier.getIdentifierInfo() &&
753          "Can't handle identifiers without identifier info!");
754   llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
755     PoisonReasons.find(Identifier.getIdentifierInfo());
756   if(it == PoisonReasons.end())
757     Diag(Identifier, diag::err_pp_used_poisoned_id);
758   else
759     Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
760 }
761 
762 /// Returns a diagnostic message kind for reporting a future keyword as
763 /// appropriate for the identifier and specified language.
getFutureCompatDiagKind(const IdentifierInfo & II,const LangOptions & LangOpts)764 static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II,
765                                           const LangOptions &LangOpts) {
766   assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
767 
768   if (LangOpts.CPlusPlus)
769     return llvm::StringSwitch<diag::kind>(II.getName())
770 #define CXX11_KEYWORD(NAME, FLAGS)                                             \
771         .Case(#NAME, diag::warn_cxx11_keyword)
772 #define CXX20_KEYWORD(NAME, FLAGS)                                             \
773         .Case(#NAME, diag::warn_cxx20_keyword)
774 #include "clang/Basic/TokenKinds.def"
775         // char8_t is not modeled as a CXX20_KEYWORD because it's not
776         // unconditionally enabled in C++20 mode. (It can be disabled
777         // by -fno-char8_t.)
778         .Case("char8_t", diag::warn_cxx20_keyword)
779         ;
780 
781   llvm_unreachable(
782       "Keyword not known to come from a newer Standard or proposed Standard");
783 }
784 
updateOutOfDateIdentifier(IdentifierInfo & II) const785 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
786   assert(II.isOutOfDate() && "not out of date");
787   getExternalSource()->updateOutOfDateIdentifier(II);
788 }
789 
790 /// HandleIdentifier - This callback is invoked when the lexer reads an
791 /// identifier.  This callback looks up the identifier in the map and/or
792 /// potentially macro expands it or turns it into a named token (like 'for').
793 ///
794 /// Note that callers of this method are guarded by checking the
795 /// IdentifierInfo's 'isHandleIdentifierCase' bit.  If this method changes, the
796 /// IdentifierInfo methods that compute these properties will need to change to
797 /// match.
HandleIdentifier(Token & Identifier)798 bool Preprocessor::HandleIdentifier(Token &Identifier) {
799   assert(Identifier.getIdentifierInfo() &&
800          "Can't handle identifiers without identifier info!");
801 
802   IdentifierInfo &II = *Identifier.getIdentifierInfo();
803 
804   // If the information about this identifier is out of date, update it from
805   // the external source.
806   // We have to treat __VA_ARGS__ in a special way, since it gets
807   // serialized with isPoisoned = true, but our preprocessor may have
808   // unpoisoned it if we're defining a C99 macro.
809   if (II.isOutOfDate()) {
810     bool CurrentIsPoisoned = false;
811     const bool IsSpecialVariadicMacro =
812         &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
813     if (IsSpecialVariadicMacro)
814       CurrentIsPoisoned = II.isPoisoned();
815 
816     updateOutOfDateIdentifier(II);
817     Identifier.setKind(II.getTokenID());
818 
819     if (IsSpecialVariadicMacro)
820       II.setIsPoisoned(CurrentIsPoisoned);
821   }
822 
823   // If this identifier was poisoned, and if it was not produced from a macro
824   // expansion, emit an error.
825   if (II.isPoisoned() && CurPPLexer) {
826     HandlePoisonedIdentifier(Identifier);
827   }
828 
829   // If this is a macro to be expanded, do it.
830   if (MacroDefinition MD = getMacroDefinition(&II)) {
831     auto *MI = MD.getMacroInfo();
832     assert(MI && "macro definition with no macro info?");
833     if (!DisableMacroExpansion) {
834       if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
835         // C99 6.10.3p10: If the preprocessing token immediately after the
836         // macro name isn't a '(', this macro should not be expanded.
837         if (!MI->isFunctionLike() || isNextPPTokenLParen())
838           return HandleMacroExpandedIdentifier(Identifier, MD);
839       } else {
840         // C99 6.10.3.4p2 says that a disabled macro may never again be
841         // expanded, even if it's in a context where it could be expanded in the
842         // future.
843         Identifier.setFlag(Token::DisableExpand);
844         if (MI->isObjectLike() || isNextPPTokenLParen())
845           Diag(Identifier, diag::pp_disabled_macro_expansion);
846       }
847     }
848   }
849 
850   // If this identifier is a keyword in a newer Standard or proposed Standard,
851   // produce a warning. Don't warn if we're not considering macro expansion,
852   // since this identifier might be the name of a macro.
853   // FIXME: This warning is disabled in cases where it shouldn't be, like
854   //   "#define constexpr constexpr", "int constexpr;"
855   if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
856     Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
857         << II.getName();
858     // Don't diagnose this keyword again in this translation unit.
859     II.setIsFutureCompatKeyword(false);
860   }
861 
862   // If this is an extension token, diagnose its use.
863   // We avoid diagnosing tokens that originate from macro definitions.
864   // FIXME: This warning is disabled in cases where it shouldn't be,
865   // like "#define TY typeof", "TY(1) x".
866   if (II.isExtensionToken() && !DisableMacroExpansion)
867     Diag(Identifier, diag::ext_token_used);
868 
869   // If this is the 'import' contextual keyword following an '@', note
870   // that the next token indicates a module name.
871   //
872   // Note that we do not treat 'import' as a contextual
873   // keyword when we're in a caching lexer, because caching lexers only get
874   // used in contexts where import declarations are disallowed.
875   //
876   // Likewise if this is the C++ Modules TS import keyword.
877   if (((LastTokenWasAt && II.isModulesImport()) ||
878        Identifier.is(tok::kw_import)) &&
879       !InMacroArgs && !DisableMacroExpansion &&
880       (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
881       CurLexerKind != CLK_CachingLexer) {
882     ModuleImportLoc = Identifier.getLocation();
883     ModuleImportPath.clear();
884     ModuleImportExpectsIdentifier = true;
885     CurLexerKind = CLK_LexAfterModuleImport;
886   }
887   return true;
888 }
889 
Lex(Token & Result)890 void Preprocessor::Lex(Token &Result) {
891   ++LexLevel;
892 
893   // We loop here until a lex function returns a token; this avoids recursion.
894   bool ReturnedToken;
895   do {
896     switch (CurLexerKind) {
897     case CLK_Lexer:
898       ReturnedToken = CurLexer->Lex(Result);
899       break;
900     case CLK_TokenLexer:
901       ReturnedToken = CurTokenLexer->Lex(Result);
902       break;
903     case CLK_CachingLexer:
904       CachingLex(Result);
905       ReturnedToken = true;
906       break;
907     case CLK_LexAfterModuleImport:
908       ReturnedToken = LexAfterModuleImport(Result);
909       break;
910     }
911   } while (!ReturnedToken);
912 
913   if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure)
914     return;
915 
916   if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
917     // Remember the identifier before code completion token.
918     setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
919     setCodeCompletionTokenRange(Result.getLocation(), Result.getEndLoc());
920     // Set IdenfitierInfo to null to avoid confusing code that handles both
921     // identifiers and completion tokens.
922     Result.setIdentifierInfo(nullptr);
923   }
924 
925   // Update ImportSeqState to track our position within a C++20 import-seq
926   // if this token is being produced as a result of phase 4 of translation.
927   if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
928       !Result.getFlag(Token::IsReinjected)) {
929     switch (Result.getKind()) {
930     case tok::l_paren: case tok::l_square: case tok::l_brace:
931       ImportSeqState.handleOpenBracket();
932       break;
933     case tok::r_paren: case tok::r_square:
934       ImportSeqState.handleCloseBracket();
935       break;
936     case tok::r_brace:
937       ImportSeqState.handleCloseBrace();
938       break;
939     case tok::semi:
940       ImportSeqState.handleSemi();
941       break;
942     case tok::header_name:
943     case tok::annot_header_unit:
944       ImportSeqState.handleHeaderName();
945       break;
946     case tok::kw_export:
947       ImportSeqState.handleExport();
948       break;
949     case tok::identifier:
950       if (Result.getIdentifierInfo()->isModulesImport()) {
951         ImportSeqState.handleImport();
952         if (ImportSeqState.afterImportSeq()) {
953           ModuleImportLoc = Result.getLocation();
954           ModuleImportPath.clear();
955           ModuleImportExpectsIdentifier = true;
956           CurLexerKind = CLK_LexAfterModuleImport;
957         }
958         break;
959       }
960       LLVM_FALLTHROUGH;
961     default:
962       ImportSeqState.handleMisc();
963       break;
964     }
965   }
966 
967   LastTokenWasAt = Result.is(tok::at);
968   --LexLevel;
969 
970   if ((LexLevel == 0 || PreprocessToken) &&
971       !Result.getFlag(Token::IsReinjected)) {
972     if (LexLevel == 0)
973       ++TokenCount;
974     if (OnToken)
975       OnToken(Result);
976   }
977 }
978 
979 /// Lex a header-name token (including one formed from header-name-tokens if
980 /// \p AllowConcatenation is \c true).
981 ///
982 /// \param FilenameTok Filled in with the next token. On success, this will
983 ///        be either a header_name token. On failure, it will be whatever other
984 ///        token was found instead.
985 /// \param AllowMacroExpansion If \c true, allow the header name to be formed
986 ///        by macro expansion (concatenating tokens as necessary if the first
987 ///        token is a '<').
988 /// \return \c true if we reached EOD or EOF while looking for a > token in
989 ///         a concatenated header name and diagnosed it. \c false otherwise.
LexHeaderName(Token & FilenameTok,bool AllowMacroExpansion)990 bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
991   // Lex using header-name tokenization rules if tokens are being lexed from
992   // a file. Just grab a token normally if we're in a macro expansion.
993   if (CurPPLexer)
994     CurPPLexer->LexIncludeFilename(FilenameTok);
995   else
996     Lex(FilenameTok);
997 
998   // This could be a <foo/bar.h> file coming from a macro expansion.  In this
999   // case, glue the tokens together into an angle_string_literal token.
1000   SmallString<128> FilenameBuffer;
1001   if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
1002     bool StartOfLine = FilenameTok.isAtStartOfLine();
1003     bool LeadingSpace = FilenameTok.hasLeadingSpace();
1004     bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1005 
1006     SourceLocation Start = FilenameTok.getLocation();
1007     SourceLocation End;
1008     FilenameBuffer.push_back('<');
1009 
1010     // Consume tokens until we find a '>'.
1011     // FIXME: A header-name could be formed starting or ending with an
1012     // alternative token. It's not clear whether that's ill-formed in all
1013     // cases.
1014     while (FilenameTok.isNot(tok::greater)) {
1015       Lex(FilenameTok);
1016       if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
1017         Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1018         Diag(Start, diag::note_matching) << tok::less;
1019         return true;
1020       }
1021 
1022       End = FilenameTok.getLocation();
1023 
1024       // FIXME: Provide code completion for #includes.
1025       if (FilenameTok.is(tok::code_completion)) {
1026         setCodeCompletionReached();
1027         Lex(FilenameTok);
1028         continue;
1029       }
1030 
1031       // Append the spelling of this token to the buffer. If there was a space
1032       // before it, add it now.
1033       if (FilenameTok.hasLeadingSpace())
1034         FilenameBuffer.push_back(' ');
1035 
1036       // Get the spelling of the token, directly into FilenameBuffer if
1037       // possible.
1038       size_t PreAppendSize = FilenameBuffer.size();
1039       FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1040 
1041       const char *BufPtr = &FilenameBuffer[PreAppendSize];
1042       unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1043 
1044       // If the token was spelled somewhere else, copy it into FilenameBuffer.
1045       if (BufPtr != &FilenameBuffer[PreAppendSize])
1046         memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1047 
1048       // Resize FilenameBuffer to the correct size.
1049       if (FilenameTok.getLength() != ActualLen)
1050         FilenameBuffer.resize(PreAppendSize + ActualLen);
1051     }
1052 
1053     FilenameTok.startToken();
1054     FilenameTok.setKind(tok::header_name);
1055     FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1056     FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1057     FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1058     CreateString(FilenameBuffer, FilenameTok, Start, End);
1059   } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
1060     // Convert a string-literal token of the form " h-char-sequence "
1061     // (produced by macro expansion) into a header-name token.
1062     //
1063     // The rules for header-names don't quite match the rules for
1064     // string-literals, but all the places where they differ result in
1065     // undefined behavior, so we can and do treat them the same.
1066     //
1067     // A string-literal with a prefix or suffix is not translated into a
1068     // header-name. This could theoretically be observable via the C++20
1069     // context-sensitive header-name formation rules.
1070     StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1071     if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1072       FilenameTok.setKind(tok::header_name);
1073   }
1074 
1075   return false;
1076 }
1077 
1078 /// Collect the tokens of a C++20 pp-import-suffix.
CollectPpImportSuffix(SmallVectorImpl<Token> & Toks)1079 void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) {
1080   // FIXME: For error recovery, consider recognizing attribute syntax here
1081   // and terminating / diagnosing a missing semicolon if we find anything
1082   // else? (Can we leave that to the parser?)
1083   unsigned BracketDepth = 0;
1084   while (true) {
1085     Toks.emplace_back();
1086     Lex(Toks.back());
1087 
1088     switch (Toks.back().getKind()) {
1089     case tok::l_paren: case tok::l_square: case tok::l_brace:
1090       ++BracketDepth;
1091       break;
1092 
1093     case tok::r_paren: case tok::r_square: case tok::r_brace:
1094       if (BracketDepth == 0)
1095         return;
1096       --BracketDepth;
1097       break;
1098 
1099     case tok::semi:
1100       if (BracketDepth == 0)
1101         return;
1102     break;
1103 
1104     case tok::eof:
1105       return;
1106 
1107     default:
1108       break;
1109     }
1110   }
1111 }
1112 
1113 
1114 /// Lex a token following the 'import' contextual keyword.
1115 ///
1116 ///     pp-import: [C++20]
1117 ///           import header-name pp-import-suffix[opt] ;
1118 ///           import header-name-tokens pp-import-suffix[opt] ;
1119 /// [ObjC]    @ import module-name ;
1120 /// [Clang]   import module-name ;
1121 ///
1122 ///     header-name-tokens:
1123 ///           string-literal
1124 ///           < [any sequence of preprocessing-tokens other than >] >
1125 ///
1126 ///     module-name:
1127 ///           module-name-qualifier[opt] identifier
1128 ///
1129 ///     module-name-qualifier
1130 ///           module-name-qualifier[opt] identifier .
1131 ///
1132 /// We respond to a pp-import by importing macros from the named module.
LexAfterModuleImport(Token & Result)1133 bool Preprocessor::LexAfterModuleImport(Token &Result) {
1134   // Figure out what kind of lexer we actually have.
1135   recomputeCurLexerKind();
1136 
1137   // Lex the next token. The header-name lexing rules are used at the start of
1138   // a pp-import.
1139   //
1140   // For now, we only support header-name imports in C++20 mode.
1141   // FIXME: Should we allow this in all language modes that support an import
1142   // declaration as an extension?
1143   if (ModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1144     if (LexHeaderName(Result))
1145       return true;
1146   } else {
1147     Lex(Result);
1148   }
1149 
1150   // Allocate a holding buffer for a sequence of tokens and introduce it into
1151   // the token stream.
1152   auto EnterTokens = [this](ArrayRef<Token> Toks) {
1153     auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1154     std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1155     EnterTokenStream(std::move(ToksCopy), Toks.size(),
1156                      /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1157   };
1158 
1159   // Check for a header-name.
1160   SmallVector<Token, 32> Suffix;
1161   if (Result.is(tok::header_name)) {
1162     // Enter the header-name token into the token stream; a Lex action cannot
1163     // both return a token and cache tokens (doing so would corrupt the token
1164     // cache if the call to Lex comes from CachingLex / PeekAhead).
1165     Suffix.push_back(Result);
1166 
1167     // Consume the pp-import-suffix and expand any macros in it now. We'll add
1168     // it back into the token stream later.
1169     CollectPpImportSuffix(Suffix);
1170     if (Suffix.back().isNot(tok::semi)) {
1171       // This is not a pp-import after all.
1172       EnterTokens(Suffix);
1173       return false;
1174     }
1175 
1176     // C++2a [cpp.module]p1:
1177     //   The ';' preprocessing-token terminating a pp-import shall not have
1178     //   been produced by macro replacement.
1179     SourceLocation SemiLoc = Suffix.back().getLocation();
1180     if (SemiLoc.isMacroID())
1181       Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1182 
1183     // Reconstitute the import token.
1184     Token ImportTok;
1185     ImportTok.startToken();
1186     ImportTok.setKind(tok::kw_import);
1187     ImportTok.setLocation(ModuleImportLoc);
1188     ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1189     ImportTok.setLength(6);
1190 
1191     auto Action = HandleHeaderIncludeOrImport(
1192         /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1193     switch (Action.Kind) {
1194     case ImportAction::None:
1195       break;
1196 
1197     case ImportAction::ModuleBegin:
1198       // Let the parser know we're textually entering the module.
1199       Suffix.emplace_back();
1200       Suffix.back().startToken();
1201       Suffix.back().setKind(tok::annot_module_begin);
1202       Suffix.back().setLocation(SemiLoc);
1203       Suffix.back().setAnnotationEndLoc(SemiLoc);
1204       Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1205       LLVM_FALLTHROUGH;
1206 
1207     case ImportAction::ModuleImport:
1208     case ImportAction::SkippedModuleImport:
1209       // We chose to import (or textually enter) the file. Convert the
1210       // header-name token into a header unit annotation token.
1211       Suffix[0].setKind(tok::annot_header_unit);
1212       Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1213       Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1214       // FIXME: Call the moduleImport callback?
1215       break;
1216     case ImportAction::Failure:
1217       assert(TheModuleLoader.HadFatalFailure &&
1218              "This should be an early exit only to a fatal error");
1219       Result.setKind(tok::eof);
1220       CurLexer->cutOffLexing();
1221       EnterTokens(Suffix);
1222       return true;
1223     }
1224 
1225     EnterTokens(Suffix);
1226     return false;
1227   }
1228 
1229   // The token sequence
1230   //
1231   //   import identifier (. identifier)*
1232   //
1233   // indicates a module import directive. We already saw the 'import'
1234   // contextual keyword, so now we're looking for the identifiers.
1235   if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1236     // We expected to see an identifier here, and we did; continue handling
1237     // identifiers.
1238     ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1239                                               Result.getLocation()));
1240     ModuleImportExpectsIdentifier = false;
1241     CurLexerKind = CLK_LexAfterModuleImport;
1242     return true;
1243   }
1244 
1245   // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1246   // see the next identifier. (We can also see a '[[' that begins an
1247   // attribute-specifier-seq here under the C++ Modules TS.)
1248   if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1249     ModuleImportExpectsIdentifier = true;
1250     CurLexerKind = CLK_LexAfterModuleImport;
1251     return true;
1252   }
1253 
1254   // If we didn't recognize a module name at all, this is not a (valid) import.
1255   if (ModuleImportPath.empty() || Result.is(tok::eof))
1256     return true;
1257 
1258   // Consume the pp-import-suffix and expand any macros in it now, if we're not
1259   // at the semicolon already.
1260   SourceLocation SemiLoc = Result.getLocation();
1261   if (Result.isNot(tok::semi)) {
1262     Suffix.push_back(Result);
1263     CollectPpImportSuffix(Suffix);
1264     if (Suffix.back().isNot(tok::semi)) {
1265       // This is not an import after all.
1266       EnterTokens(Suffix);
1267       return false;
1268     }
1269     SemiLoc = Suffix.back().getLocation();
1270   }
1271 
1272   // Under the Modules TS, the dot is just part of the module name, and not
1273   // a real hierarchy separator. Flatten such module names now.
1274   //
1275   // FIXME: Is this the right level to be performing this transformation?
1276   std::string FlatModuleName;
1277   if (getLangOpts().ModulesTS || getLangOpts().CPlusPlusModules) {
1278     for (auto &Piece : ModuleImportPath) {
1279       if (!FlatModuleName.empty())
1280         FlatModuleName += ".";
1281       FlatModuleName += Piece.first->getName();
1282     }
1283     SourceLocation FirstPathLoc = ModuleImportPath[0].second;
1284     ModuleImportPath.clear();
1285     ModuleImportPath.push_back(
1286         std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1287   }
1288 
1289   Module *Imported = nullptr;
1290   if (getLangOpts().Modules) {
1291     Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1292                                           ModuleImportPath,
1293                                           Module::Hidden,
1294                                           /*IsInclusionDirective=*/false);
1295     if (Imported)
1296       makeModuleVisible(Imported, SemiLoc);
1297   }
1298   if (Callbacks)
1299     Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1300 
1301   if (!Suffix.empty()) {
1302     EnterTokens(Suffix);
1303     return false;
1304   }
1305   return true;
1306 }
1307 
makeModuleVisible(Module * M,SourceLocation Loc)1308 void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) {
1309   CurSubmoduleState->VisibleModules.setVisible(
1310       M, Loc, [](Module *) {},
1311       [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1312         // FIXME: Include the path in the diagnostic.
1313         // FIXME: Include the import location for the conflicting module.
1314         Diag(ModuleImportLoc, diag::warn_module_conflict)
1315             << Path[0]->getFullModuleName()
1316             << Conflict->getFullModuleName()
1317             << Message;
1318       });
1319 
1320   // Add this module to the imports list of the currently-built submodule.
1321   if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1322     BuildingSubmoduleStack.back().M->Imports.insert(M);
1323 }
1324 
FinishLexStringLiteral(Token & Result,std::string & String,const char * DiagnosticTag,bool AllowMacroExpansion)1325 bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1326                                           const char *DiagnosticTag,
1327                                           bool AllowMacroExpansion) {
1328   // We need at least one string literal.
1329   if (Result.isNot(tok::string_literal)) {
1330     Diag(Result, diag::err_expected_string_literal)
1331       << /*Source='in...'*/0 << DiagnosticTag;
1332     return false;
1333   }
1334 
1335   // Lex string literal tokens, optionally with macro expansion.
1336   SmallVector<Token, 4> StrToks;
1337   do {
1338     StrToks.push_back(Result);
1339 
1340     if (Result.hasUDSuffix())
1341       Diag(Result, diag::err_invalid_string_udl);
1342 
1343     if (AllowMacroExpansion)
1344       Lex(Result);
1345     else
1346       LexUnexpandedToken(Result);
1347   } while (Result.is(tok::string_literal));
1348 
1349   // Concatenate and parse the strings.
1350   StringLiteralParser Literal(StrToks, *this);
1351   assert(Literal.isAscii() && "Didn't allow wide strings in");
1352 
1353   if (Literal.hadError)
1354     return false;
1355 
1356   if (Literal.Pascal) {
1357     Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1358       << /*Source='in...'*/0 << DiagnosticTag;
1359     return false;
1360   }
1361 
1362   String = std::string(Literal.GetString());
1363   return true;
1364 }
1365 
parseSimpleIntegerLiteral(Token & Tok,uint64_t & Value)1366 bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1367   assert(Tok.is(tok::numeric_constant));
1368   SmallString<8> IntegerBuffer;
1369   bool NumberInvalid = false;
1370   StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1371   if (NumberInvalid)
1372     return false;
1373   NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1374                                getLangOpts(), getTargetInfo(),
1375                                getDiagnostics());
1376   if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1377     return false;
1378   llvm::APInt APVal(64, 0);
1379   if (Literal.GetIntegerValue(APVal))
1380     return false;
1381   Lex(Tok);
1382   Value = APVal.getLimitedValue();
1383   return true;
1384 }
1385 
addCommentHandler(CommentHandler * Handler)1386 void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1387   assert(Handler && "NULL comment handler");
1388   assert(llvm::find(CommentHandlers, Handler) == CommentHandlers.end() &&
1389          "Comment handler already registered");
1390   CommentHandlers.push_back(Handler);
1391 }
1392 
removeCommentHandler(CommentHandler * Handler)1393 void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1394   std::vector<CommentHandler *>::iterator Pos =
1395       llvm::find(CommentHandlers, Handler);
1396   assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1397   CommentHandlers.erase(Pos);
1398 }
1399 
HandleComment(Token & result,SourceRange Comment)1400 bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1401   bool AnyPendingTokens = false;
1402   for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1403        HEnd = CommentHandlers.end();
1404        H != HEnd; ++H) {
1405     if ((*H)->HandleComment(*this, Comment))
1406       AnyPendingTokens = true;
1407   }
1408   if (!AnyPendingTokens || getCommentRetentionState())
1409     return false;
1410   Lex(result);
1411   return true;
1412 }
1413 
1414 ModuleLoader::~ModuleLoader() = default;
1415 
1416 CommentHandler::~CommentHandler() = default;
1417 
1418 EmptylineHandler::~EmptylineHandler() = default;
1419 
1420 CodeCompletionHandler::~CodeCompletionHandler() = default;
1421 
createPreprocessingRecord()1422 void Preprocessor::createPreprocessingRecord() {
1423   if (Record)
1424     return;
1425 
1426   Record = new PreprocessingRecord(getSourceManager());
1427   addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1428 }
1429