1 //===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Support/FileCollector.h"
10 #include "llvm/ADT/SmallString.h"
11 #include "llvm/Support/FileSystem.h"
12 #include "llvm/Support/Path.h"
13 #include "llvm/Support/Process.h"
14
15 using namespace llvm;
16
isCaseSensitivePath(StringRef Path)17 static bool isCaseSensitivePath(StringRef Path) {
18 SmallString<256> TmpDest = Path, UpperDest, RealDest;
19
20 // Remove component traversals, links, etc.
21 if (!sys::fs::real_path(Path, TmpDest))
22 return true; // Current default value in vfs.yaml
23 Path = TmpDest;
24
25 // Change path to all upper case and ask for its real path, if the latter
26 // exists and is equal to path, it's not case sensitive. Default to case
27 // sensitive in the absence of real_path, since this is the YAMLVFSWriter
28 // default.
29 UpperDest = Path.upper();
30 if (sys::fs::real_path(UpperDest, RealDest) && Path.equals(RealDest))
31 return false;
32 return true;
33 }
34
FileCollector(std::string Root,std::string OverlayRoot)35 FileCollector::FileCollector(std::string Root, std::string OverlayRoot)
36 : Root(std::move(Root)), OverlayRoot(std::move(OverlayRoot)) {
37 sys::fs::create_directories(this->Root, true);
38 }
39
getRealPath(StringRef SrcPath,SmallVectorImpl<char> & Result)40 bool FileCollector::getRealPath(StringRef SrcPath,
41 SmallVectorImpl<char> &Result) {
42 SmallString<256> RealPath;
43 StringRef FileName = sys::path::filename(SrcPath);
44 std::string Directory = sys::path::parent_path(SrcPath).str();
45 auto DirWithSymlink = SymlinkMap.find(Directory);
46
47 // Use real_path to fix any symbolic link component present in a path.
48 // Computing the real path is expensive, cache the search through the parent
49 // path Directory.
50 if (DirWithSymlink == SymlinkMap.end()) {
51 auto EC = sys::fs::real_path(Directory, RealPath);
52 if (EC)
53 return false;
54 SymlinkMap[Directory] = RealPath.str();
55 } else {
56 RealPath = DirWithSymlink->second;
57 }
58
59 sys::path::append(RealPath, FileName);
60 Result.swap(RealPath);
61 return true;
62 }
63
addFile(const Twine & file)64 void FileCollector::addFile(const Twine &file) {
65 std::lock_guard<std::mutex> lock(Mutex);
66 std::string FileStr = file.str();
67 if (markAsSeen(FileStr))
68 addFileImpl(FileStr);
69 }
70
addFileImpl(StringRef SrcPath)71 void FileCollector::addFileImpl(StringRef SrcPath) {
72 // We need an absolute src path to append to the root.
73 SmallString<256> AbsoluteSrc = SrcPath;
74 sys::fs::make_absolute(AbsoluteSrc);
75
76 // Canonicalize src to a native path to avoid mixed separator styles.
77 sys::path::native(AbsoluteSrc);
78
79 // Remove redundant leading "./" pieces and consecutive separators.
80 AbsoluteSrc = sys::path::remove_leading_dotslash(AbsoluteSrc);
81
82 // Canonicalize the source path by removing "..", "." components.
83 SmallString<256> VirtualPath = AbsoluteSrc;
84 sys::path::remove_dots(VirtualPath, /*remove_dot_dot=*/true);
85
86 // If a ".." component is present after a symlink component, remove_dots may
87 // lead to the wrong real destination path. Let the source be canonicalized
88 // like that but make sure we always use the real path for the destination.
89 SmallString<256> CopyFrom;
90 if (!getRealPath(AbsoluteSrc, CopyFrom))
91 CopyFrom = VirtualPath;
92
93 SmallString<256> DstPath = StringRef(Root);
94 sys::path::append(DstPath, sys::path::relative_path(CopyFrom));
95
96 // Always map a canonical src path to its real path into the YAML, by doing
97 // this we map different virtual src paths to the same entry in the VFS
98 // overlay, which is a way to emulate symlink inside the VFS; this is also
99 // needed for correctness, not doing that can lead to module redefinition
100 // errors.
101 addFileToMapping(VirtualPath, DstPath);
102 }
103
104 /// Set the access and modification time for the given file from the given
105 /// status object.
106 static std::error_code
copyAccessAndModificationTime(StringRef Filename,const sys::fs::file_status & Stat)107 copyAccessAndModificationTime(StringRef Filename,
108 const sys::fs::file_status &Stat) {
109 int FD;
110
111 if (auto EC =
112 sys::fs::openFileForWrite(Filename, FD, sys::fs::CD_OpenExisting))
113 return EC;
114
115 if (auto EC = sys::fs::setLastAccessAndModificationTime(
116 FD, Stat.getLastAccessedTime(), Stat.getLastModificationTime()))
117 return EC;
118
119 if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
120 return EC;
121
122 return {};
123 }
124
copyFiles(bool StopOnError)125 std::error_code FileCollector::copyFiles(bool StopOnError) {
126 for (auto &entry : VFSWriter.getMappings()) {
127 // Create directory tree.
128 if (std::error_code EC =
129 sys::fs::create_directories(sys::path::parent_path(entry.RPath),
130 /*IgnoreExisting=*/true)) {
131 if (StopOnError)
132 return EC;
133 }
134
135 // Get the status of the original file/directory.
136 sys::fs::file_status Stat;
137 if (std::error_code EC = sys::fs::status(entry.VPath, Stat)) {
138 if (StopOnError)
139 return EC;
140 continue;
141 }
142
143 if (Stat.type() == sys::fs::file_type::directory_file) {
144 // Construct a directory when it's just a directory entry.
145 if (std::error_code EC =
146 sys::fs::create_directories(entry.RPath,
147 /*IgnoreExisting=*/true)) {
148 if (StopOnError)
149 return EC;
150 }
151 continue;
152 }
153
154 // Copy file over.
155 if (std::error_code EC = sys::fs::copy_file(entry.VPath, entry.RPath)) {
156 if (StopOnError)
157 return EC;
158 }
159
160 // Copy over permissions.
161 if (auto perms = sys::fs::getPermissions(entry.VPath)) {
162 if (std::error_code EC = sys::fs::setPermissions(entry.RPath, *perms)) {
163 if (StopOnError)
164 return EC;
165 }
166 }
167
168 // Copy over modification time.
169 copyAccessAndModificationTime(entry.RPath, Stat);
170 }
171 return {};
172 }
173
writeMapping(StringRef mapping_file)174 std::error_code FileCollector::writeMapping(StringRef mapping_file) {
175 std::lock_guard<std::mutex> lock(Mutex);
176
177 VFSWriter.setOverlayDir(OverlayRoot);
178 VFSWriter.setCaseSensitivity(isCaseSensitivePath(OverlayRoot));
179 VFSWriter.setUseExternalNames(false);
180
181 std::error_code EC;
182 raw_fd_ostream os(mapping_file, EC, sys::fs::OF_Text);
183 if (EC)
184 return EC;
185
186 VFSWriter.write(os);
187
188 return {};
189 }
190
191 namespace {
192
193 class FileCollectorFileSystem : public vfs::FileSystem {
194 public:
FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,std::shared_ptr<FileCollector> Collector)195 explicit FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,
196 std::shared_ptr<FileCollector> Collector)
197 : FS(std::move(FS)), Collector(std::move(Collector)) {}
198
status(const Twine & Path)199 llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override {
200 auto Result = FS->status(Path);
201 if (Result && Result->exists())
202 Collector->addFile(Path);
203 return Result;
204 }
205
206 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
openFileForRead(const Twine & Path)207 openFileForRead(const Twine &Path) override {
208 auto Result = FS->openFileForRead(Path);
209 if (Result && *Result)
210 Collector->addFile(Path);
211 return Result;
212 }
213
dir_begin(const llvm::Twine & Dir,std::error_code & EC)214 llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
215 std::error_code &EC) override {
216 auto It = FS->dir_begin(Dir, EC);
217 if (EC)
218 return It;
219 // Collect everything that's listed in case the user needs it.
220 Collector->addFile(Dir);
221 for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
222 if (It->type() == sys::fs::file_type::regular_file ||
223 It->type() == sys::fs::file_type::directory_file ||
224 It->type() == sys::fs::file_type::symlink_file) {
225 Collector->addFile(It->path());
226 }
227 }
228 if (EC)
229 return It;
230 // Return a new iterator.
231 return FS->dir_begin(Dir, EC);
232 }
233
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const234 std::error_code getRealPath(const Twine &Path,
235 SmallVectorImpl<char> &Output) const override {
236 auto EC = FS->getRealPath(Path, Output);
237 if (!EC) {
238 Collector->addFile(Path);
239 if (Output.size() > 0)
240 Collector->addFile(Output);
241 }
242 return EC;
243 }
244
isLocal(const Twine & Path,bool & Result)245 std::error_code isLocal(const Twine &Path, bool &Result) override {
246 return FS->isLocal(Path, Result);
247 }
248
getCurrentWorkingDirectory() const249 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
250 return FS->getCurrentWorkingDirectory();
251 }
252
setCurrentWorkingDirectory(const llvm::Twine & Path)253 std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
254 return FS->setCurrentWorkingDirectory(Path);
255 }
256
257 private:
258 IntrusiveRefCntPtr<vfs::FileSystem> FS;
259 std::shared_ptr<FileCollector> Collector;
260 };
261
262 } // end anonymous namespace
263
264 IntrusiveRefCntPtr<vfs::FileSystem>
createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,std::shared_ptr<FileCollector> Collector)265 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
266 std::shared_ptr<FileCollector> Collector) {
267 return new FileCollectorFileSystem(std::move(BaseFS), std::move(Collector));
268 }
269