1 //===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the CrossTranslationUnit interface.
10 //
11 //===----------------------------------------------------------------------===//
12 #include "clang/CrossTU/CrossTranslationUnit.h"
13 #include "clang/AST/ASTImporter.h"
14 #include "clang/AST/Decl.h"
15 #include "clang/AST/ParentMapContext.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/CrossTU/CrossTUDiagnostic.h"
18 #include "clang/Frontend/ASTUnit.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/TextDiagnosticPrinter.h"
21 #include "clang/Index/USRGeneration.h"
22 #include "llvm/ADT/Optional.h"
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/ADT/Triple.h"
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/ErrorHandling.h"
27 #include "llvm/Support/ManagedStatic.h"
28 #include "llvm/Support/Path.h"
29 #include "llvm/Support/YAMLParser.h"
30 #include "llvm/Support/raw_ostream.h"
31 #include <algorithm>
32 #include <fstream>
33 #include <sstream>
34 #include <tuple>
35
36 namespace clang {
37 namespace cross_tu {
38
39 namespace {
40
41 #define DEBUG_TYPE "CrossTranslationUnit"
42 STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
43 STATISTIC(
44 NumNotInOtherTU,
45 "The # of getCTUDefinition called but the function is not in any other TU");
46 STATISTIC(NumGetCTUSuccess,
47 "The # of getCTUDefinition successfully returned the "
48 "requested function's body");
49 STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
50 "encountered an unsupported AST Node");
51 STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
52 "encountered an ODR error");
53 STATISTIC(NumTripleMismatch, "The # of triple mismatches");
54 STATISTIC(NumLangMismatch, "The # of language mismatches");
55 STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
56 STATISTIC(NumASTLoadThresholdReached,
57 "The # of ASTs not loaded because of threshold");
58
59 // Same as Triple's equality operator, but we check a field only if that is
60 // known in both instances.
hasEqualKnownFields(const llvm::Triple & Lhs,const llvm::Triple & Rhs)61 bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
62 using llvm::Triple;
63 if (Lhs.getArch() != Triple::UnknownArch &&
64 Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
65 return false;
66 if (Lhs.getSubArch() != Triple::NoSubArch &&
67 Rhs.getSubArch() != Triple::NoSubArch &&
68 Lhs.getSubArch() != Rhs.getSubArch())
69 return false;
70 if (Lhs.getVendor() != Triple::UnknownVendor &&
71 Rhs.getVendor() != Triple::UnknownVendor &&
72 Lhs.getVendor() != Rhs.getVendor())
73 return false;
74 if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
75 Lhs.getOS() != Rhs.getOS())
76 return false;
77 if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
78 Rhs.getEnvironment() != Triple::UnknownEnvironment &&
79 Lhs.getEnvironment() != Rhs.getEnvironment())
80 return false;
81 if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
82 Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
83 Lhs.getObjectFormat() != Rhs.getObjectFormat())
84 return false;
85 return true;
86 }
87
88 // FIXME: This class is will be removed after the transition to llvm::Error.
89 class IndexErrorCategory : public std::error_category {
90 public:
name() const91 const char *name() const noexcept override { return "clang.index"; }
92
message(int Condition) const93 std::string message(int Condition) const override {
94 switch (static_cast<index_error_code>(Condition)) {
95 case index_error_code::unspecified:
96 return "An unknown error has occurred.";
97 case index_error_code::missing_index_file:
98 return "The index file is missing.";
99 case index_error_code::invalid_index_format:
100 return "Invalid index file format.";
101 case index_error_code::multiple_definitions:
102 return "Multiple definitions in the index file.";
103 case index_error_code::missing_definition:
104 return "Missing definition from the index file.";
105 case index_error_code::failed_import:
106 return "Failed to import the definition.";
107 case index_error_code::failed_to_get_external_ast:
108 return "Failed to load external AST source.";
109 case index_error_code::failed_to_generate_usr:
110 return "Failed to generate USR.";
111 case index_error_code::triple_mismatch:
112 return "Triple mismatch";
113 case index_error_code::lang_mismatch:
114 return "Language mismatch";
115 case index_error_code::lang_dialect_mismatch:
116 return "Language dialect mismatch";
117 case index_error_code::load_threshold_reached:
118 return "Load threshold reached";
119 case index_error_code::invocation_list_ambiguous:
120 return "Invocation list file contains multiple references to the same "
121 "source file.";
122 case index_error_code::invocation_list_file_not_found:
123 return "Invocation list file is not found.";
124 case index_error_code::invocation_list_empty:
125 return "Invocation list file is empty.";
126 case index_error_code::invocation_list_wrong_format:
127 return "Invocation list file is in wrong format.";
128 case index_error_code::invocation_list_lookup_unsuccessful:
129 return "Invocation list file does not contain the requested source file.";
130 }
131 llvm_unreachable("Unrecognized index_error_code.");
132 }
133 };
134
135 static llvm::ManagedStatic<IndexErrorCategory> Category;
136 } // end anonymous namespace
137
138 char IndexError::ID;
139
log(raw_ostream & OS) const140 void IndexError::log(raw_ostream &OS) const {
141 OS << Category->message(static_cast<int>(Code)) << '\n';
142 }
143
convertToErrorCode() const144 std::error_code IndexError::convertToErrorCode() const {
145 return std::error_code(static_cast<int>(Code), *Category);
146 }
147
148 llvm::Expected<llvm::StringMap<std::string>>
parseCrossTUIndex(StringRef IndexPath)149 parseCrossTUIndex(StringRef IndexPath) {
150 std::ifstream ExternalMapFile{std::string(IndexPath)};
151 if (!ExternalMapFile)
152 return llvm::make_error<IndexError>(index_error_code::missing_index_file,
153 IndexPath.str());
154
155 llvm::StringMap<std::string> Result;
156 std::string Line;
157 unsigned LineNo = 1;
158 while (std::getline(ExternalMapFile, Line)) {
159 StringRef LineRef{Line};
160 const size_t Delimiter = LineRef.find(" ");
161 if (Delimiter > 0 && Delimiter != std::string::npos) {
162 StringRef LookupName = LineRef.substr(0, Delimiter);
163
164 // Store paths with posix-style directory separator.
165 SmallString<32> FilePath(LineRef.substr(Delimiter + 1));
166 llvm::sys::path::native(FilePath, llvm::sys::path::Style::posix);
167
168 bool InsertionOccured;
169 std::tie(std::ignore, InsertionOccured) =
170 Result.try_emplace(LookupName, FilePath.begin(), FilePath.end());
171 if (!InsertionOccured)
172 return llvm::make_error<IndexError>(
173 index_error_code::multiple_definitions, IndexPath.str(), LineNo);
174 } else
175 return llvm::make_error<IndexError>(
176 index_error_code::invalid_index_format, IndexPath.str(), LineNo);
177 ++LineNo;
178 }
179 return Result;
180 }
181
182 std::string
createCrossTUIndexString(const llvm::StringMap<std::string> & Index)183 createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
184 std::ostringstream Result;
185 for (const auto &E : Index)
186 Result << E.getKey().str() << " " << E.getValue() << '\n';
187 return Result.str();
188 }
189
containsConst(const VarDecl * VD,const ASTContext & ACtx)190 bool containsConst(const VarDecl *VD, const ASTContext &ACtx) {
191 CanQualType CT = ACtx.getCanonicalType(VD->getType());
192 if (!CT.isConstQualified()) {
193 const RecordType *RTy = CT->getAs<RecordType>();
194 if (!RTy || !RTy->hasConstFields())
195 return false;
196 }
197 return true;
198 }
199
hasBodyOrInit(const FunctionDecl * D,const FunctionDecl * & DefD)200 static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
201 return D->hasBody(DefD);
202 }
hasBodyOrInit(const VarDecl * D,const VarDecl * & DefD)203 static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
204 return D->getAnyInitializer(DefD);
205 }
hasBodyOrInit(const T * D)206 template <typename T> static bool hasBodyOrInit(const T *D) {
207 const T *Unused;
208 return hasBodyOrInit(D, Unused);
209 }
210
CrossTranslationUnitContext(CompilerInstance & CI)211 CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
212 : Context(CI.getASTContext()), ASTStorage(CI) {}
213
~CrossTranslationUnitContext()214 CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
215
216 llvm::Optional<std::string>
getLookupName(const NamedDecl * ND)217 CrossTranslationUnitContext::getLookupName(const NamedDecl *ND) {
218 SmallString<128> DeclUSR;
219 bool Ret = index::generateUSRForDecl(ND, DeclUSR);
220 if (Ret)
221 return {};
222 return std::string(DeclUSR.str());
223 }
224
225 /// Recursively visits the decls of a DeclContext, and returns one with the
226 /// given USR.
227 template <typename T>
228 const T *
findDefInDeclContext(const DeclContext * DC,StringRef LookupName)229 CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
230 StringRef LookupName) {
231 assert(DC && "Declaration Context must not be null");
232 for (const Decl *D : DC->decls()) {
233 const auto *SubDC = dyn_cast<DeclContext>(D);
234 if (SubDC)
235 if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
236 return ND;
237
238 const auto *ND = dyn_cast<T>(D);
239 const T *ResultDecl;
240 if (!ND || !hasBodyOrInit(ND, ResultDecl))
241 continue;
242 llvm::Optional<std::string> ResultLookupName = getLookupName(ResultDecl);
243 if (!ResultLookupName || *ResultLookupName != LookupName)
244 continue;
245 return ResultDecl;
246 }
247 return nullptr;
248 }
249
250 template <typename T>
getCrossTUDefinitionImpl(const T * D,StringRef CrossTUDir,StringRef IndexName,bool DisplayCTUProgress)251 llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
252 const T *D, StringRef CrossTUDir, StringRef IndexName,
253 bool DisplayCTUProgress) {
254 assert(D && "D is missing, bad call to this function!");
255 assert(!hasBodyOrInit(D) &&
256 "D has a body or init in current translation unit!");
257 ++NumGetCTUCalled;
258 const llvm::Optional<std::string> LookupName = getLookupName(D);
259 if (!LookupName)
260 return llvm::make_error<IndexError>(
261 index_error_code::failed_to_generate_usr);
262 llvm::Expected<ASTUnit *> ASTUnitOrError =
263 loadExternalAST(*LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
264 if (!ASTUnitOrError)
265 return ASTUnitOrError.takeError();
266 ASTUnit *Unit = *ASTUnitOrError;
267 assert(&Unit->getFileManager() ==
268 &Unit->getASTContext().getSourceManager().getFileManager());
269
270 const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
271 const llvm::Triple &TripleFrom =
272 Unit->getASTContext().getTargetInfo().getTriple();
273 // The imported AST had been generated for a different target.
274 // Some parts of the triple in the loaded ASTContext can be unknown while the
275 // very same parts in the target ASTContext are known. Thus we check for the
276 // known parts only.
277 if (!hasEqualKnownFields(TripleTo, TripleFrom)) {
278 // TODO: Pass the SourceLocation of the CallExpression for more precise
279 // diagnostics.
280 ++NumTripleMismatch;
281 return llvm::make_error<IndexError>(index_error_code::triple_mismatch,
282 std::string(Unit->getMainFileName()),
283 TripleTo.str(), TripleFrom.str());
284 }
285
286 const auto &LangTo = Context.getLangOpts();
287 const auto &LangFrom = Unit->getASTContext().getLangOpts();
288
289 // FIXME: Currenty we do not support CTU across C++ and C and across
290 // different dialects of C++.
291 if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
292 ++NumLangMismatch;
293 return llvm::make_error<IndexError>(index_error_code::lang_mismatch);
294 }
295
296 // If CPP dialects are different then return with error.
297 //
298 // Consider this STL code:
299 // template<typename _Alloc>
300 // struct __alloc_traits
301 // #if __cplusplus >= 201103L
302 // : std::allocator_traits<_Alloc>
303 // #endif
304 // { // ...
305 // };
306 // This class template would create ODR errors during merging the two units,
307 // since in one translation unit the class template has a base class, however
308 // in the other unit it has none.
309 if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
310 LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
311 LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
312 LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
313 ++NumLangDialectMismatch;
314 return llvm::make_error<IndexError>(
315 index_error_code::lang_dialect_mismatch);
316 }
317
318 TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
319 if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
320 return importDefinition(ResultDecl, Unit);
321 return llvm::make_error<IndexError>(index_error_code::failed_import);
322 }
323
324 llvm::Expected<const FunctionDecl *>
getCrossTUDefinition(const FunctionDecl * FD,StringRef CrossTUDir,StringRef IndexName,bool DisplayCTUProgress)325 CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
326 StringRef CrossTUDir,
327 StringRef IndexName,
328 bool DisplayCTUProgress) {
329 return getCrossTUDefinitionImpl(FD, CrossTUDir, IndexName,
330 DisplayCTUProgress);
331 }
332
333 llvm::Expected<const VarDecl *>
getCrossTUDefinition(const VarDecl * VD,StringRef CrossTUDir,StringRef IndexName,bool DisplayCTUProgress)334 CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
335 StringRef CrossTUDir,
336 StringRef IndexName,
337 bool DisplayCTUProgress) {
338 return getCrossTUDefinitionImpl(VD, CrossTUDir, IndexName,
339 DisplayCTUProgress);
340 }
341
emitCrossTUDiagnostics(const IndexError & IE)342 void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE) {
343 switch (IE.getCode()) {
344 case index_error_code::missing_index_file:
345 Context.getDiagnostics().Report(diag::err_ctu_error_opening)
346 << IE.getFileName();
347 break;
348 case index_error_code::invalid_index_format:
349 Context.getDiagnostics().Report(diag::err_extdefmap_parsing)
350 << IE.getFileName() << IE.getLineNum();
351 break;
352 case index_error_code::multiple_definitions:
353 Context.getDiagnostics().Report(diag::err_multiple_def_index)
354 << IE.getLineNum();
355 break;
356 case index_error_code::triple_mismatch:
357 Context.getDiagnostics().Report(diag::warn_ctu_incompat_triple)
358 << IE.getFileName() << IE.getTripleToName() << IE.getTripleFromName();
359 break;
360 default:
361 break;
362 }
363 }
364
ASTUnitStorage(CompilerInstance & CI)365 CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
366 CompilerInstance &CI)
367 : Loader(CI, CI.getAnalyzerOpts()->CTUDir,
368 CI.getAnalyzerOpts()->CTUInvocationList),
369 LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
370 ? CI.getAnalyzerOpts()->CTUImportCppThreshold
371 : CI.getAnalyzerOpts()->CTUImportThreshold) {}
372
373 llvm::Expected<ASTUnit *>
getASTUnitForFile(StringRef FileName,bool DisplayCTUProgress)374 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
375 StringRef FileName, bool DisplayCTUProgress) {
376 // Try the cache first.
377 auto ASTCacheEntry = FileASTUnitMap.find(FileName);
378 if (ASTCacheEntry == FileASTUnitMap.end()) {
379
380 // Do not load if the limit is reached.
381 if (!LoadGuard) {
382 ++NumASTLoadThresholdReached;
383 return llvm::make_error<IndexError>(
384 index_error_code::load_threshold_reached);
385 }
386
387 auto LoadAttempt = Loader.load(FileName);
388
389 if (!LoadAttempt)
390 return LoadAttempt.takeError();
391
392 std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
393
394 // Need the raw pointer and the unique_ptr as well.
395 ASTUnit *Unit = LoadedUnit.get();
396
397 // Update the cache.
398 FileASTUnitMap[FileName] = std::move(LoadedUnit);
399
400 LoadGuard.indicateLoadSuccess();
401
402 if (DisplayCTUProgress)
403 llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
404
405 return Unit;
406
407 } else {
408 // Found in the cache.
409 return ASTCacheEntry->second.get();
410 }
411 }
412
413 llvm::Expected<ASTUnit *>
getASTUnitForFunction(StringRef FunctionName,StringRef CrossTUDir,StringRef IndexName,bool DisplayCTUProgress)414 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
415 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
416 bool DisplayCTUProgress) {
417 // Try the cache first.
418 auto ASTCacheEntry = NameASTUnitMap.find(FunctionName);
419 if (ASTCacheEntry == NameASTUnitMap.end()) {
420 // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
421
422 // Ensure that the Index is loaded, as we need to search in it.
423 if (llvm::Error IndexLoadError =
424 ensureCTUIndexLoaded(CrossTUDir, IndexName))
425 return std::move(IndexLoadError);
426
427 // Check if there is and entry in the index for the function.
428 if (!NameFileMap.count(FunctionName)) {
429 ++NumNotInOtherTU;
430 return llvm::make_error<IndexError>(index_error_code::missing_definition);
431 }
432
433 // Search in the index for the filename where the definition of FuncitonName
434 // resides.
435 if (llvm::Expected<ASTUnit *> FoundForFile =
436 getASTUnitForFile(NameFileMap[FunctionName], DisplayCTUProgress)) {
437
438 // Update the cache.
439 NameASTUnitMap[FunctionName] = *FoundForFile;
440 return *FoundForFile;
441
442 } else {
443 return FoundForFile.takeError();
444 }
445 } else {
446 // Found in the cache.
447 return ASTCacheEntry->second;
448 }
449 }
450
451 llvm::Expected<std::string>
getFileForFunction(StringRef FunctionName,StringRef CrossTUDir,StringRef IndexName)452 CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
453 StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
454 if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
455 return std::move(IndexLoadError);
456 return NameFileMap[FunctionName];
457 }
458
ensureCTUIndexLoaded(StringRef CrossTUDir,StringRef IndexName)459 llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
460 StringRef CrossTUDir, StringRef IndexName) {
461 // Dont initialize if the map is filled.
462 if (!NameFileMap.empty())
463 return llvm::Error::success();
464
465 // Get the absolute path to the index file.
466 SmallString<256> IndexFile = CrossTUDir;
467 if (llvm::sys::path::is_absolute(IndexName))
468 IndexFile = IndexName;
469 else
470 llvm::sys::path::append(IndexFile, IndexName);
471
472 if (auto IndexMapping = parseCrossTUIndex(IndexFile)) {
473 // Initialize member map.
474 NameFileMap = *IndexMapping;
475 return llvm::Error::success();
476 } else {
477 // Error while parsing CrossTU index file.
478 return IndexMapping.takeError();
479 };
480 }
481
loadExternalAST(StringRef LookupName,StringRef CrossTUDir,StringRef IndexName,bool DisplayCTUProgress)482 llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
483 StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
484 bool DisplayCTUProgress) {
485 // FIXME: The current implementation only supports loading decls with
486 // a lookup name from a single translation unit. If multiple
487 // translation units contains decls with the same lookup name an
488 // error will be returned.
489
490 // Try to get the value from the heavily cached storage.
491 llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
492 LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
493
494 if (!Unit)
495 return Unit.takeError();
496
497 // Check whether the backing pointer of the Expected is a nullptr.
498 if (!*Unit)
499 return llvm::make_error<IndexError>(
500 index_error_code::failed_to_get_external_ast);
501
502 return Unit;
503 }
504
ASTLoader(CompilerInstance & CI,StringRef CTUDir,StringRef InvocationListFilePath)505 CrossTranslationUnitContext::ASTLoader::ASTLoader(
506 CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
507 : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
508
509 CrossTranslationUnitContext::LoadResultTy
load(StringRef Identifier)510 CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
511 llvm::SmallString<256> Path;
512 if (llvm::sys::path::is_absolute(Identifier, PathStyle)) {
513 Path = Identifier;
514 } else {
515 Path = CTUDir;
516 llvm::sys::path::append(Path, PathStyle, Identifier);
517 }
518
519 // The path is stored in the InvocationList member in posix style. To
520 // successfully lookup an entry based on filepath, it must be converted.
521 llvm::sys::path::native(Path, PathStyle);
522
523 // Normalize by removing relative path components.
524 llvm::sys::path::remove_dots(Path, /*remove_dot_dot*/ true, PathStyle);
525
526 if (Path.endswith(".ast"))
527 return loadFromDump(Path);
528 else
529 return loadFromSource(Path);
530 }
531
532 CrossTranslationUnitContext::LoadResultTy
loadFromDump(StringRef ASTDumpPath)533 CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
534 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
535 TextDiagnosticPrinter *DiagClient =
536 new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
537 IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
538 IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
539 new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
540 return ASTUnit::LoadFromASTFile(
541 std::string(ASTDumpPath.str()),
542 CI.getPCHContainerOperations()->getRawReader(), ASTUnit::LoadEverything,
543 Diags, CI.getFileSystemOpts());
544 }
545
546 /// Load the AST from a source-file, which is supposed to be located inside the
547 /// YAML formatted invocation list file under the filesystem path specified by
548 /// \p InvocationList. The invocation list should contain absolute paths.
549 /// \p SourceFilePath is the absolute path of the source file that contains the
550 /// function definition the analysis is looking for. The Index is built by the
551 /// \p clang-extdef-mapping tool, which is also supposed to be generating
552 /// absolute paths.
553 ///
554 /// Proper diagnostic emission requires absolute paths, so even if a future
555 /// change introduces the handling of relative paths, this must be taken into
556 /// consideration.
557 CrossTranslationUnitContext::LoadResultTy
loadFromSource(StringRef SourceFilePath)558 CrossTranslationUnitContext::ASTLoader::loadFromSource(
559 StringRef SourceFilePath) {
560
561 if (llvm::Error InitError = lazyInitInvocationList())
562 return std::move(InitError);
563 assert(InvocationList);
564
565 auto Invocation = InvocationList->find(SourceFilePath);
566 if (Invocation == InvocationList->end())
567 return llvm::make_error<IndexError>(
568 index_error_code::invocation_list_lookup_unsuccessful);
569
570 const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
571
572 SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
573 std::transform(InvocationCommand.begin(), InvocationCommand.end(),
574 CommandLineArgs.begin(),
575 [](auto &&CmdPart) { return CmdPart.c_str(); });
576
577 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts{&CI.getDiagnosticOpts()};
578 auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
579 IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
580 CI.getDiagnostics().getDiagnosticIDs()};
581 IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
582 new DiagnosticsEngine{DiagID, &*DiagOpts, DiagClient});
583
584 return std::unique_ptr<ASTUnit>(ASTUnit::LoadFromCommandLine(
585 CommandLineArgs.begin(), (CommandLineArgs.end()),
586 CI.getPCHContainerOperations(), Diags,
587 CI.getHeaderSearchOpts().ResourceDir));
588 }
589
590 llvm::Expected<InvocationListTy>
parseInvocationList(StringRef FileContent,llvm::sys::path::Style PathStyle)591 parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle) {
592 InvocationListTy InvocationList;
593
594 /// LLVM YAML parser is used to extract information from invocation list file.
595 llvm::SourceMgr SM;
596 llvm::yaml::Stream InvocationFile(FileContent, SM);
597
598 /// Only the first document is processed.
599 llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
600
601 /// There has to be at least one document available.
602 if (FirstInvocationFile == InvocationFile.end())
603 return llvm::make_error<IndexError>(
604 index_error_code::invocation_list_empty);
605
606 llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
607 if (!DocumentRoot)
608 return llvm::make_error<IndexError>(
609 index_error_code::invocation_list_wrong_format);
610
611 /// According to the format specified the document must be a mapping, where
612 /// the keys are paths to source files, and values are sequences of invocation
613 /// parts.
614 auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(DocumentRoot);
615 if (!Mappings)
616 return llvm::make_error<IndexError>(
617 index_error_code::invocation_list_wrong_format);
618
619 for (auto &NextMapping : *Mappings) {
620 /// The keys should be strings, which represent a source-file path.
621 auto *Key = dyn_cast<llvm::yaml::ScalarNode>(NextMapping.getKey());
622 if (!Key)
623 return llvm::make_error<IndexError>(
624 index_error_code::invocation_list_wrong_format);
625
626 SmallString<32> ValueStorage;
627 StringRef SourcePath = Key->getValue(ValueStorage);
628
629 // Store paths with PathStyle directory separator.
630 SmallString<32> NativeSourcePath(SourcePath);
631 llvm::sys::path::native(NativeSourcePath, PathStyle);
632
633 StringRef InvocationKey(NativeSourcePath);
634
635 if (InvocationList.find(InvocationKey) != InvocationList.end())
636 return llvm::make_error<IndexError>(
637 index_error_code::invocation_list_ambiguous);
638
639 /// The values should be sequences of strings, each representing a part of
640 /// the invocation.
641 auto *Args = dyn_cast<llvm::yaml::SequenceNode>(NextMapping.getValue());
642 if (!Args)
643 return llvm::make_error<IndexError>(
644 index_error_code::invocation_list_wrong_format);
645
646 for (auto &Arg : *Args) {
647 auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(&Arg);
648 if (!CmdString)
649 return llvm::make_error<IndexError>(
650 index_error_code::invocation_list_wrong_format);
651 /// Every conversion starts with an empty working storage, as it is not
652 /// clear if this is a requirement of the YAML parser.
653 ValueStorage.clear();
654 InvocationList[InvocationKey].emplace_back(
655 CmdString->getValue(ValueStorage));
656 }
657
658 if (InvocationList[InvocationKey].empty())
659 return llvm::make_error<IndexError>(
660 index_error_code::invocation_list_wrong_format);
661 }
662
663 return InvocationList;
664 }
665
lazyInitInvocationList()666 llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
667 /// Lazily initialize the invocation list member used for on-demand parsing.
668 if (InvocationList)
669 return llvm::Error::success();
670
671 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
672 llvm::MemoryBuffer::getFile(InvocationListFilePath);
673 if (!FileContent)
674 return llvm::make_error<IndexError>(
675 index_error_code::invocation_list_file_not_found);
676 std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
677 assert(ContentBuffer && "If no error was produced after loading, the pointer "
678 "should not be nullptr.");
679
680 llvm::Expected<InvocationListTy> ExpectedInvocationList =
681 parseInvocationList(ContentBuffer->getBuffer(), PathStyle);
682
683 if (!ExpectedInvocationList)
684 return ExpectedInvocationList.takeError();
685
686 InvocationList = *ExpectedInvocationList;
687
688 return llvm::Error::success();
689 }
690
691 template <typename T>
692 llvm::Expected<const T *>
importDefinitionImpl(const T * D,ASTUnit * Unit)693 CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
694 assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
695
696 assert(&D->getASTContext() == &Unit->getASTContext() &&
697 "ASTContext of Decl and the unit should match.");
698 ASTImporter &Importer = getOrCreateASTImporter(Unit);
699
700 auto ToDeclOrError = Importer.Import(D);
701 if (!ToDeclOrError) {
702 handleAllErrors(ToDeclOrError.takeError(),
703 [&](const ImportError &IE) {
704 switch (IE.Error) {
705 case ImportError::NameConflict:
706 ++NumNameConflicts;
707 break;
708 case ImportError::UnsupportedConstruct:
709 ++NumUnsupportedNodeFound;
710 break;
711 case ImportError::Unknown:
712 llvm_unreachable("Unknown import error happened.");
713 break;
714 }
715 });
716 return llvm::make_error<IndexError>(index_error_code::failed_import);
717 }
718 auto *ToDecl = cast<T>(*ToDeclOrError);
719 assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
720 ++NumGetCTUSuccess;
721
722 // Parent map is invalidated after changing the AST.
723 ToDecl->getASTContext().getParentMapContext().clear();
724
725 return ToDecl;
726 }
727
728 llvm::Expected<const FunctionDecl *>
importDefinition(const FunctionDecl * FD,ASTUnit * Unit)729 CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
730 ASTUnit *Unit) {
731 return importDefinitionImpl(FD, Unit);
732 }
733
734 llvm::Expected<const VarDecl *>
importDefinition(const VarDecl * VD,ASTUnit * Unit)735 CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
736 ASTUnit *Unit) {
737 return importDefinitionImpl(VD, Unit);
738 }
739
lazyInitImporterSharedSt(TranslationUnitDecl * ToTU)740 void CrossTranslationUnitContext::lazyInitImporterSharedSt(
741 TranslationUnitDecl *ToTU) {
742 if (!ImporterSharedSt)
743 ImporterSharedSt = std::make_shared<ASTImporterSharedState>(*ToTU);
744 }
745
746 ASTImporter &
getOrCreateASTImporter(ASTUnit * Unit)747 CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
748 ASTContext &From = Unit->getASTContext();
749
750 auto I = ASTUnitImporterMap.find(From.getTranslationUnitDecl());
751 if (I != ASTUnitImporterMap.end())
752 return *I->second;
753 lazyInitImporterSharedSt(Context.getTranslationUnitDecl());
754 ASTImporter *NewImporter = new ASTImporter(
755 Context, Context.getSourceManager().getFileManager(), From,
756 From.getSourceManager().getFileManager(), false, ImporterSharedSt);
757 NewImporter->setFileIDImportHandler([this, Unit](FileID ToID, FileID FromID) {
758 assert(ImportedFileIDs.find(ToID) == ImportedFileIDs.end() &&
759 "FileID already imported, should not happen.");
760 ImportedFileIDs[ToID] = std::make_pair(FromID, Unit);
761 });
762 ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(NewImporter);
763 return *NewImporter;
764 }
765
766 llvm::Optional<std::pair<SourceLocation, ASTUnit *>>
getImportedFromSourceLocation(const clang::SourceLocation & ToLoc) const767 CrossTranslationUnitContext::getImportedFromSourceLocation(
768 const clang::SourceLocation &ToLoc) const {
769 const SourceManager &SM = Context.getSourceManager();
770 auto DecToLoc = SM.getDecomposedLoc(ToLoc);
771
772 auto I = ImportedFileIDs.find(DecToLoc.first);
773 if (I == ImportedFileIDs.end())
774 return {};
775
776 FileID FromID = I->second.first;
777 clang::ASTUnit *Unit = I->second.second;
778 SourceLocation FromLoc =
779 Unit->getSourceManager().getComposedLoc(FromID, DecToLoc.second);
780
781 return std::make_pair(FromLoc, Unit);
782 }
783
784 } // namespace cross_tu
785 } // namespace clang
786