• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===--- GlobalCompilationDatabase.cpp ---------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "GlobalCompilationDatabase.h"
10 #include "FS.h"
11 #include "support/Logger.h"
12 #include "support/Path.h"
13 #include "clang/Frontend/CompilerInvocation.h"
14 #include "clang/Tooling/ArgumentsAdjusters.h"
15 #include "clang/Tooling/CompilationDatabase.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/ScopeExit.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/Support/FileSystem.h"
22 #include "llvm/Support/FileUtilities.h"
23 #include "llvm/Support/Path.h"
24 #include "llvm/Support/Program.h"
25 #include <chrono>
26 #include <string>
27 #include <tuple>
28 #include <vector>
29 
30 namespace clang {
31 namespace clangd {
32 namespace {
33 
34 // Runs the given action on all parent directories of filename, starting from
35 // deepest directory and going up to root. Stops whenever action succeeds.
actOnAllParentDirectories(PathRef FileName,llvm::function_ref<bool (PathRef)> Action)36 void actOnAllParentDirectories(PathRef FileName,
37                                llvm::function_ref<bool(PathRef)> Action) {
38   for (auto Path = llvm::sys::path::parent_path(FileName);
39        !Path.empty() && !Action(Path);
40        Path = llvm::sys::path::parent_path(Path))
41     ;
42 }
43 
44 } // namespace
45 
46 tooling::CompileCommand
getFallbackCommand(PathRef File) const47 GlobalCompilationDatabase::getFallbackCommand(PathRef File) const {
48   std::vector<std::string> Argv = {"clang"};
49   // Clang treats .h files as C by default and files without extension as linker
50   // input, resulting in unhelpful diagnostics.
51   // Parsing as Objective C++ is friendly to more cases.
52   auto FileExtension = llvm::sys::path::extension(File);
53   if (FileExtension.empty() || FileExtension == ".h")
54     Argv.push_back("-xobjective-c++-header");
55   Argv.push_back(std::string(File));
56   tooling::CompileCommand Cmd(llvm::sys::path::parent_path(File),
57                               llvm::sys::path::filename(File), std::move(Argv),
58                               /*Output=*/"");
59   Cmd.Heuristic = "clangd fallback";
60   return Cmd;
61 }
62 
63 // Loads and caches the CDB from a single directory.
64 //
65 // This class is threadsafe, which is to say we have independent locks for each
66 // directory we're searching for a CDB.
67 // Loading is deferred until first access.
68 //
69 // The DirectoryBasedCDB keeps a map from path => DirectoryCache.
70 // Typical usage is to:
71 //  - 1) determine all the paths that might be searched
72 //  - 2) acquire the map lock and get-or-create all the DirectoryCache entries
73 //  - 3) release the map lock and query the caches as desired
74 //
75 // FIXME: this should revalidate the cache sometimes
76 // FIXME: IO should go through a VFS
77 class DirectoryBasedGlobalCompilationDatabase::DirectoryCache {
78   // Absolute canonical path that we're the cache for. (Not case-folded).
79   const std::string Path;
80 
81   // True if we've looked for a CDB here and found none.
82   // (This makes it possible for get() to return without taking a lock)
83   // FIXME: this should have an expiry time instead of lasting forever.
84   std::atomic<bool> FinalizedNoCDB = {false};
85 
86   // Guards following cache state.
87   std::mutex Mu;
88   // Has cache been filled from disk? FIXME: this should be an expiry time.
89   bool CachePopulated = false;
90   // Whether a new CDB has been loaded but not broadcast yet.
91   bool NeedsBroadcast = false;
92   // Last loaded CDB, meaningful if CachePopulated is set.
93   // shared_ptr so we can overwrite this when callers are still using the CDB.
94   std::shared_ptr<tooling::CompilationDatabase> CDB;
95 
96 public:
DirectoryCache(llvm::StringRef Path)97   DirectoryCache(llvm::StringRef Path) : Path(Path) {
98     assert(llvm::sys::path::is_absolute(Path));
99   }
100 
101   // Get the CDB associated with this directory.
102   // ShouldBroadcast:
103   //  - as input, signals whether the caller is willing to broadcast a
104   //    newly-discovered CDB. (e.g. to trigger background indexing)
105   //  - as output, signals whether the caller should do so.
106   // (If a new CDB is discovered and ShouldBroadcast is false, we mark the
107   // CDB as needing broadcast, and broadcast it next time we can).
108   std::shared_ptr<const tooling::CompilationDatabase>
get(bool & ShouldBroadcast)109   get(bool &ShouldBroadcast) {
110     // Fast path for common case without taking lock.
111     if (FinalizedNoCDB.load()) {
112       ShouldBroadcast = false;
113       return nullptr;
114     }
115     std::lock_guard<std::mutex> Lock(Mu);
116     auto RequestBroadcast = llvm::make_scope_exit([&, OldCDB(CDB.get())] {
117       // If we loaded a new CDB, it should be broadcast at some point.
118       if (CDB != nullptr && CDB.get() != OldCDB)
119         NeedsBroadcast = true;
120       else if (CDB == nullptr) // nothing to broadcast anymore!
121         NeedsBroadcast = false;
122       // If we have something to broadcast, then do so iff allowed.
123       if (!ShouldBroadcast)
124         return;
125       ShouldBroadcast = NeedsBroadcast;
126       NeedsBroadcast = false;
127     });
128 
129     // For now, we never actually attempt to revalidate a populated cache.
130     if (CachePopulated)
131       return CDB;
132     assert(CDB == nullptr);
133 
134     load();
135     CachePopulated = true;
136 
137     if (!CDB)
138       FinalizedNoCDB.store(true);
139     return CDB;
140   }
141 
path() const142   llvm::StringRef path() const { return Path; }
143 
144 private:
145   // Updates `CDB` from disk state.
load()146   void load() {
147     std::string Error; // ignored, because it's often "didn't find anything".
148     CDB = tooling::CompilationDatabase::loadFromDirectory(Path, Error);
149     if (!CDB) {
150       // Fallback: check for $src/build, the conventional CMake build root.
151       // Probe existence first to avoid each plugin doing IO if it doesn't
152       // exist.
153       llvm::SmallString<256> BuildDir(Path);
154       llvm::sys::path::append(BuildDir, "build");
155       if (llvm::sys::fs::is_directory(BuildDir)) {
156         vlog("Found candidate build directory {0}", BuildDir);
157         CDB = tooling::CompilationDatabase::loadFromDirectory(BuildDir, Error);
158       }
159     }
160     if (CDB) {
161       log("Loaded compilation database from {0}", Path);
162     } else {
163       vlog("No compilation database at {0}", Path);
164     }
165   }
166 };
167 
168 DirectoryBasedGlobalCompilationDatabase::
DirectoryBasedGlobalCompilationDatabase(llvm::Optional<Path> CompileCommandsDir)169     DirectoryBasedGlobalCompilationDatabase(
170         llvm::Optional<Path> CompileCommandsDir) {
171   if (CompileCommandsDir)
172     OnlyDirCache = std::make_unique<DirectoryCache>(*CompileCommandsDir);
173 }
174 
175 DirectoryBasedGlobalCompilationDatabase::
176     ~DirectoryBasedGlobalCompilationDatabase() = default;
177 
178 llvm::Optional<tooling::CompileCommand>
getCompileCommand(PathRef File) const179 DirectoryBasedGlobalCompilationDatabase::getCompileCommand(PathRef File) const {
180   CDBLookupRequest Req;
181   Req.FileName = File;
182   Req.ShouldBroadcast = true;
183 
184   auto Res = lookupCDB(Req);
185   if (!Res) {
186     log("Failed to find compilation database for {0}", File);
187     return llvm::None;
188   }
189 
190   auto Candidates = Res->CDB->getCompileCommands(File);
191   if (!Candidates.empty())
192     return std::move(Candidates.front());
193 
194   return None;
195 }
196 
197 // For platforms where paths are case-insensitive (but case-preserving),
198 // we need to do case-insensitive comparisons and use lowercase keys.
199 // FIXME: Make Path a real class with desired semantics instead.
200 //        This class is not the only place this problem exists.
201 // FIXME: Mac filesystems default to case-insensitive, but may be sensitive.
202 
maybeCaseFoldPath(PathRef Path)203 static std::string maybeCaseFoldPath(PathRef Path) {
204 #if defined(_WIN32) || defined(__APPLE__)
205   return Path.lower();
206 #else
207   return std::string(Path);
208 #endif
209 }
210 
pathEqual(PathRef A,PathRef B)211 static bool pathEqual(PathRef A, PathRef B) {
212 #if defined(_WIN32) || defined(__APPLE__)
213   return A.equals_lower(B);
214 #else
215   return A == B;
216 #endif
217 }
218 
219 std::vector<DirectoryBasedGlobalCompilationDatabase::DirectoryCache *>
getDirectoryCaches(llvm::ArrayRef<llvm::StringRef> Dirs) const220 DirectoryBasedGlobalCompilationDatabase::getDirectoryCaches(
221     llvm::ArrayRef<llvm::StringRef> Dirs) const {
222   std::vector<std::string> FoldedDirs;
223   FoldedDirs.reserve(Dirs.size());
224   for (const auto &Dir : Dirs)
225     FoldedDirs.push_back(maybeCaseFoldPath(Dir));
226 
227   std::vector<DirectoryCache *> Ret;
228   Ret.reserve(Dirs.size());
229 
230   std::lock_guard<std::mutex> Lock(DirCachesMutex);
231   for (unsigned I = 0; I < Dirs.size(); ++I)
232     Ret.push_back(&DirCaches.try_emplace(FoldedDirs[I], Dirs[I]).first->second);
233   return Ret;
234 }
235 
236 llvm::Optional<DirectoryBasedGlobalCompilationDatabase::CDBLookupResult>
lookupCDB(CDBLookupRequest Request) const237 DirectoryBasedGlobalCompilationDatabase::lookupCDB(
238     CDBLookupRequest Request) const {
239   assert(llvm::sys::path::is_absolute(Request.FileName) &&
240          "path must be absolute");
241 
242   bool ShouldBroadcast = false;
243   DirectoryCache *DirCache = nullptr;
244   std::shared_ptr<const tooling::CompilationDatabase> CDB = nullptr;
245   if (OnlyDirCache) {
246     DirCache = OnlyDirCache.get();
247     ShouldBroadcast = Request.ShouldBroadcast;
248     CDB = DirCache->get(ShouldBroadcast);
249   } else {
250     // Traverse the canonical version to prevent false positives. i.e.:
251     // src/build/../a.cc can detect a CDB in /src/build if not canonicalized.
252     std::string CanonicalPath = removeDots(Request.FileName);
253     std::vector<llvm::StringRef> SearchDirs;
254     actOnAllParentDirectories(CanonicalPath, [&](PathRef Path) {
255       SearchDirs.push_back(Path);
256       return false;
257     });
258     for (DirectoryCache *Candidate : getDirectoryCaches(SearchDirs)) {
259       bool CandidateShouldBroadcast = Request.ShouldBroadcast;
260       if ((CDB = Candidate->get(CandidateShouldBroadcast))) {
261         DirCache = Candidate;
262         ShouldBroadcast = CandidateShouldBroadcast;
263         break;
264       }
265     }
266   }
267 
268   if (!CDB)
269     return llvm::None;
270 
271   CDBLookupResult Result;
272   Result.CDB = std::move(CDB);
273   Result.PI.SourceRoot = DirCache->path().str();
274 
275   // FIXME: Maybe make the following part async, since this can block
276   // retrieval of compile commands.
277   if (ShouldBroadcast)
278     broadcastCDB(Result);
279   return Result;
280 }
281 
broadcastCDB(CDBLookupResult Result) const282 void DirectoryBasedGlobalCompilationDatabase::broadcastCDB(
283     CDBLookupResult Result) const {
284   assert(Result.CDB && "Trying to broadcast an invalid CDB!");
285 
286   std::vector<std::string> AllFiles = Result.CDB->getAllFiles();
287   // We assume CDB in CompileCommandsDir owns all of its entries, since we don't
288   // perform any search in parent paths whenever it is set.
289   if (OnlyDirCache) {
290     assert(OnlyDirCache->path() == Result.PI.SourceRoot &&
291            "Trying to broadcast a CDB outside of CompileCommandsDir!");
292     OnCommandChanged.broadcast(std::move(AllFiles));
293     return;
294   }
295 
296   // Uniquify all parent directories of all files.
297   llvm::StringMap<bool> DirectoryHasCDB;
298   std::vector<llvm::StringRef> FileAncestors;
299   for (llvm::StringRef File : AllFiles) {
300     actOnAllParentDirectories(File, [&](PathRef Path) {
301       auto It = DirectoryHasCDB.try_emplace(Path);
302       // Already seen this path, and all of its parents.
303       if (!It.second)
304         return true;
305 
306       FileAncestors.push_back(It.first->getKey());
307       return pathEqual(Path, Result.PI.SourceRoot);
308     });
309   }
310   // Work out which ones have CDBs in them.
311   for (DirectoryCache *Dir : getDirectoryCaches(FileAncestors)) {
312     bool ShouldBroadcast = false;
313     if (Dir->get(ShouldBroadcast))
314       DirectoryHasCDB.find(Dir->path())->setValue(true);
315   }
316 
317   std::vector<std::string> GovernedFiles;
318   for (llvm::StringRef File : AllFiles) {
319     // A file is governed by this CDB if lookup for the file would find it.
320     // Independent of whether it has an entry for that file or not.
321     actOnAllParentDirectories(File, [&](PathRef Path) {
322       if (DirectoryHasCDB.lookup(Path)) {
323         if (pathEqual(Path, Result.PI.SourceRoot))
324           // Make sure listeners always get a canonical path for the file.
325           GovernedFiles.push_back(removeDots(File));
326         // Stop as soon as we hit a CDB.
327         return true;
328       }
329       return false;
330     });
331   }
332 
333   OnCommandChanged.broadcast(std::move(GovernedFiles));
334 }
335 
336 llvm::Optional<ProjectInfo>
getProjectInfo(PathRef File) const337 DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const {
338   CDBLookupRequest Req;
339   Req.FileName = File;
340   Req.ShouldBroadcast = false;
341   auto Res = lookupCDB(Req);
342   if (!Res)
343     return llvm::None;
344   return Res->PI;
345 }
346 
OverlayCDB(const GlobalCompilationDatabase * Base,std::vector<std::string> FallbackFlags,tooling::ArgumentsAdjuster Adjuster)347 OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base,
348                        std::vector<std::string> FallbackFlags,
349                        tooling::ArgumentsAdjuster Adjuster)
350     : Base(Base), ArgsAdjuster(std::move(Adjuster)),
351       FallbackFlags(std::move(FallbackFlags)) {
352   if (Base)
353     BaseChanged = Base->watch([this](const std::vector<std::string> Changes) {
354       OnCommandChanged.broadcast(Changes);
355     });
356 }
357 
358 llvm::Optional<tooling::CompileCommand>
getCompileCommand(PathRef File) const359 OverlayCDB::getCompileCommand(PathRef File) const {
360   llvm::Optional<tooling::CompileCommand> Cmd;
361   {
362     std::lock_guard<std::mutex> Lock(Mutex);
363     auto It = Commands.find(removeDots(File));
364     if (It != Commands.end())
365       Cmd = It->second;
366   }
367   if (!Cmd && Base)
368     Cmd = Base->getCompileCommand(File);
369   if (!Cmd)
370     return llvm::None;
371   if (ArgsAdjuster)
372     Cmd->CommandLine = ArgsAdjuster(Cmd->CommandLine, Cmd->Filename);
373   return Cmd;
374 }
375 
getFallbackCommand(PathRef File) const376 tooling::CompileCommand OverlayCDB::getFallbackCommand(PathRef File) const {
377   auto Cmd = Base ? Base->getFallbackCommand(File)
378                   : GlobalCompilationDatabase::getFallbackCommand(File);
379   std::lock_guard<std::mutex> Lock(Mutex);
380   Cmd.CommandLine.insert(Cmd.CommandLine.end(), FallbackFlags.begin(),
381                          FallbackFlags.end());
382   if (ArgsAdjuster)
383     Cmd.CommandLine = ArgsAdjuster(Cmd.CommandLine, Cmd.Filename);
384   return Cmd;
385 }
386 
setCompileCommand(PathRef File,llvm::Optional<tooling::CompileCommand> Cmd)387 void OverlayCDB::setCompileCommand(
388     PathRef File, llvm::Optional<tooling::CompileCommand> Cmd) {
389   // We store a canonical version internally to prevent mismatches between set
390   // and get compile commands. Also it assures clients listening to broadcasts
391   // doesn't receive different names for the same file.
392   std::string CanonPath = removeDots(File);
393   {
394     std::unique_lock<std::mutex> Lock(Mutex);
395     if (Cmd)
396       Commands[CanonPath] = std::move(*Cmd);
397     else
398       Commands.erase(CanonPath);
399   }
400   OnCommandChanged.broadcast({CanonPath});
401 }
402 
getProjectInfo(PathRef File) const403 llvm::Optional<ProjectInfo> OverlayCDB::getProjectInfo(PathRef File) const {
404   // It wouldn't make much sense to treat files with overridden commands
405   // specially when we can't do the same for the (unknown) local headers they
406   // include or changing behavior mid-air after receiving an override.
407   if (Base)
408     return Base->getProjectInfo(File);
409   return llvm::None;
410 }
411 } // namespace clangd
412 } // namespace clang
413