• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/Support/FileCollector.h"
10 #include "llvm/ADT/SmallString.h"
11 #include "llvm/Support/FileSystem.h"
12 #include "llvm/Support/Path.h"
13 #include "llvm/Support/Process.h"
14 
15 using namespace llvm;
16 
isCaseSensitivePath(StringRef Path)17 static bool isCaseSensitivePath(StringRef Path) {
18   SmallString<256> TmpDest = Path, UpperDest, RealDest;
19 
20   // Remove component traversals, links, etc.
21   if (!sys::fs::real_path(Path, TmpDest))
22     return true; // Current default value in vfs.yaml
23   Path = TmpDest;
24 
25   // Change path to all upper case and ask for its real path, if the latter
26   // exists and is equal to path, it's not case sensitive. Default to case
27   // sensitive in the absence of real_path, since this is the YAMLVFSWriter
28   // default.
29   UpperDest = Path.upper();
30   if (sys::fs::real_path(UpperDest, RealDest) && Path.equals(RealDest))
31     return false;
32   return true;
33 }
34 
FileCollector(std::string Root,std::string OverlayRoot)35 FileCollector::FileCollector(std::string Root, std::string OverlayRoot)
36     : Root(std::move(Root)), OverlayRoot(std::move(OverlayRoot)) {
37   sys::fs::create_directories(this->Root, true);
38 }
39 
getRealPath(StringRef SrcPath,SmallVectorImpl<char> & Result)40 bool FileCollector::getRealPath(StringRef SrcPath,
41                                 SmallVectorImpl<char> &Result) {
42   SmallString<256> RealPath;
43   StringRef FileName = sys::path::filename(SrcPath);
44   std::string Directory = sys::path::parent_path(SrcPath).str();
45   auto DirWithSymlink = SymlinkMap.find(Directory);
46 
47   // Use real_path to fix any symbolic link component present in a path.
48   // Computing the real path is expensive, cache the search through the parent
49   // path Directory.
50   if (DirWithSymlink == SymlinkMap.end()) {
51     auto EC = sys::fs::real_path(Directory, RealPath);
52     if (EC)
53       return false;
54     SymlinkMap[Directory] = RealPath.str();
55   } else {
56     RealPath = DirWithSymlink->second;
57   }
58 
59   sys::path::append(RealPath, FileName);
60   Result.swap(RealPath);
61   return true;
62 }
63 
addFile(const Twine & file)64 void FileCollector::addFile(const Twine &file) {
65   std::lock_guard<std::mutex> lock(Mutex);
66   std::string FileStr = file.str();
67   if (markAsSeen(FileStr))
68     addFileImpl(FileStr);
69 }
70 
addFileImpl(StringRef SrcPath)71 void FileCollector::addFileImpl(StringRef SrcPath) {
72   // We need an absolute src path to append to the root.
73   SmallString<256> AbsoluteSrc = SrcPath;
74   sys::fs::make_absolute(AbsoluteSrc);
75 
76   // Canonicalize src to a native path to avoid mixed separator styles.
77   sys::path::native(AbsoluteSrc);
78 
79   // Remove redundant leading "./" pieces and consecutive separators.
80   AbsoluteSrc = sys::path::remove_leading_dotslash(AbsoluteSrc);
81 
82   // Canonicalize the source path by removing "..", "." components.
83   SmallString<256> VirtualPath = AbsoluteSrc;
84   sys::path::remove_dots(VirtualPath, /*remove_dot_dot=*/true);
85 
86   // If a ".." component is present after a symlink component, remove_dots may
87   // lead to the wrong real destination path. Let the source be canonicalized
88   // like that but make sure we always use the real path for the destination.
89   SmallString<256> CopyFrom;
90   if (!getRealPath(AbsoluteSrc, CopyFrom))
91     CopyFrom = VirtualPath;
92 
93   SmallString<256> DstPath = StringRef(Root);
94   sys::path::append(DstPath, sys::path::relative_path(CopyFrom));
95 
96   // Always map a canonical src path to its real path into the YAML, by doing
97   // this we map different virtual src paths to the same entry in the VFS
98   // overlay, which is a way to emulate symlink inside the VFS; this is also
99   // needed for correctness, not doing that can lead to module redefinition
100   // errors.
101   addFileToMapping(VirtualPath, DstPath);
102 }
103 
104 /// Set the access and modification time for the given file from the given
105 /// status object.
106 static std::error_code
copyAccessAndModificationTime(StringRef Filename,const sys::fs::file_status & Stat)107 copyAccessAndModificationTime(StringRef Filename,
108                               const sys::fs::file_status &Stat) {
109   int FD;
110 
111   if (auto EC =
112           sys::fs::openFileForWrite(Filename, FD, sys::fs::CD_OpenExisting))
113     return EC;
114 
115   if (auto EC = sys::fs::setLastAccessAndModificationTime(
116           FD, Stat.getLastAccessedTime(), Stat.getLastModificationTime()))
117     return EC;
118 
119   if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
120     return EC;
121 
122   return {};
123 }
124 
copyFiles(bool StopOnError)125 std::error_code FileCollector::copyFiles(bool StopOnError) {
126   for (auto &entry : VFSWriter.getMappings()) {
127     // Create directory tree.
128     if (std::error_code EC =
129             sys::fs::create_directories(sys::path::parent_path(entry.RPath),
130                                         /*IgnoreExisting=*/true)) {
131       if (StopOnError)
132         return EC;
133     }
134 
135     // Get the status of the original file/directory.
136     sys::fs::file_status Stat;
137     if (std::error_code EC = sys::fs::status(entry.VPath, Stat)) {
138       if (StopOnError)
139         return EC;
140       continue;
141     }
142 
143     if (Stat.type() == sys::fs::file_type::directory_file) {
144       // Construct a directory when it's just a directory entry.
145       if (std::error_code EC =
146               sys::fs::create_directories(entry.RPath,
147                                           /*IgnoreExisting=*/true)) {
148         if (StopOnError)
149           return EC;
150       }
151       continue;
152     }
153 
154     // Copy file over.
155     if (std::error_code EC = sys::fs::copy_file(entry.VPath, entry.RPath)) {
156       if (StopOnError)
157         return EC;
158     }
159 
160     // Copy over permissions.
161     if (auto perms = sys::fs::getPermissions(entry.VPath)) {
162       if (std::error_code EC = sys::fs::setPermissions(entry.RPath, *perms)) {
163         if (StopOnError)
164           return EC;
165       }
166     }
167 
168     // Copy over modification time.
169     copyAccessAndModificationTime(entry.RPath, Stat);
170   }
171   return {};
172 }
173 
writeMapping(StringRef mapping_file)174 std::error_code FileCollector::writeMapping(StringRef mapping_file) {
175   std::lock_guard<std::mutex> lock(Mutex);
176 
177   VFSWriter.setOverlayDir(OverlayRoot);
178   VFSWriter.setCaseSensitivity(isCaseSensitivePath(OverlayRoot));
179   VFSWriter.setUseExternalNames(false);
180 
181   std::error_code EC;
182   raw_fd_ostream os(mapping_file, EC, sys::fs::OF_Text);
183   if (EC)
184     return EC;
185 
186   VFSWriter.write(os);
187 
188   return {};
189 }
190 
191 namespace {
192 
193 class FileCollectorFileSystem : public vfs::FileSystem {
194 public:
FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,std::shared_ptr<FileCollector> Collector)195   explicit FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,
196                                    std::shared_ptr<FileCollector> Collector)
197       : FS(std::move(FS)), Collector(std::move(Collector)) {}
198 
status(const Twine & Path)199   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override {
200     auto Result = FS->status(Path);
201     if (Result && Result->exists())
202       Collector->addFile(Path);
203     return Result;
204   }
205 
206   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
openFileForRead(const Twine & Path)207   openFileForRead(const Twine &Path) override {
208     auto Result = FS->openFileForRead(Path);
209     if (Result && *Result)
210       Collector->addFile(Path);
211     return Result;
212   }
213 
dir_begin(const llvm::Twine & Dir,std::error_code & EC)214   llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
215                                           std::error_code &EC) override {
216     auto It = FS->dir_begin(Dir, EC);
217     if (EC)
218       return It;
219     // Collect everything that's listed in case the user needs it.
220     Collector->addFile(Dir);
221     for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
222       if (It->type() == sys::fs::file_type::regular_file ||
223           It->type() == sys::fs::file_type::directory_file ||
224           It->type() == sys::fs::file_type::symlink_file) {
225         Collector->addFile(It->path());
226       }
227     }
228     if (EC)
229       return It;
230     // Return a new iterator.
231     return FS->dir_begin(Dir, EC);
232   }
233 
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const234   std::error_code getRealPath(const Twine &Path,
235                               SmallVectorImpl<char> &Output) const override {
236     auto EC = FS->getRealPath(Path, Output);
237     if (!EC) {
238       Collector->addFile(Path);
239       if (Output.size() > 0)
240         Collector->addFile(Output);
241     }
242     return EC;
243   }
244 
isLocal(const Twine & Path,bool & Result)245   std::error_code isLocal(const Twine &Path, bool &Result) override {
246     return FS->isLocal(Path, Result);
247   }
248 
getCurrentWorkingDirectory() const249   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
250     return FS->getCurrentWorkingDirectory();
251   }
252 
setCurrentWorkingDirectory(const llvm::Twine & Path)253   std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
254     return FS->setCurrentWorkingDirectory(Path);
255   }
256 
257 private:
258   IntrusiveRefCntPtr<vfs::FileSystem> FS;
259   std::shared_ptr<FileCollector> Collector;
260 };
261 
262 } // end anonymous namespace
263 
264 IntrusiveRefCntPtr<vfs::FileSystem>
createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,std::shared_ptr<FileCollector> Collector)265 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
266                                   std::shared_ptr<FileCollector> Collector) {
267   return new FileCollectorFileSystem(std::move(BaseFS), std::move(Collector));
268 }
269