1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "incfs"
18
19 #include "incfs.h"
20
21 #include <IncrementalProperties.sysprop.h>
22 #include <android-base/file.h>
23 #include <android-base/logging.h>
24 #include <android-base/no_destructor.h>
25 #include <android-base/parsebool.h>
26 #include <android-base/properties.h>
27 #include <android-base/stringprintf.h>
28 #include <android-base/strings.h>
29 #include <android-base/unique_fd.h>
30 #include <dirent.h>
31 #include <errno.h>
32 #include <fcntl.h>
33 #include <libgen.h>
34 #include <openssl/sha.h>
35 #include <selinux/android.h>
36 #include <selinux/selinux.h>
37 #include <sys/inotify.h>
38 #include <sys/mount.h>
39 #include <sys/poll.h>
40 #include <sys/stat.h>
41 #include <sys/syscall.h>
42 #include <sys/types.h>
43 #include <sys/vfs.h>
44 #include <sys/xattr.h>
45 #include <unistd.h>
46
47 #include <charconv>
48 #include <chrono>
49 #include <iterator>
50 #include <mutex>
51 #include <optional>
52 #include <string_view>
53
54 #include "MountRegistry.h"
55 #include "path.h"
56
57 using namespace std::literals;
58 using namespace android::incfs;
59 using namespace android::sysprop;
60 namespace ab = android::base;
61
62 struct IncFsControl final {
63 IncFsFd cmd;
64 IncFsFd pendingReads;
65 IncFsFd logs;
66 IncFsFd blocksWritten;
IncFsControlIncFsControl67 constexpr IncFsControl(IncFsFd cmd, IncFsFd pendingReads, IncFsFd logs, IncFsFd blocksWritten)
68 : cmd(cmd), pendingReads(pendingReads), logs(logs), blocksWritten(blocksWritten) {}
69 };
70
registry()71 static MountRegistry& registry() {
72 static ab::NoDestructor<MountRegistry> instance{};
73 return *instance;
74 }
75
openRaw(std::string_view file)76 static ab::unique_fd openRaw(std::string_view file) {
77 auto fd = ab::unique_fd(::open(details::c_str(file), O_RDONLY | O_CLOEXEC));
78 if (fd < 0) {
79 return ab::unique_fd{-errno};
80 }
81 return fd;
82 }
83
openRaw(std::string_view dir,std::string_view name)84 static ab::unique_fd openRaw(std::string_view dir, std::string_view name) {
85 return openRaw(path::join(dir, name));
86 }
87
indexPath(std::string_view root,IncFsFileId fileId)88 static std::string indexPath(std::string_view root, IncFsFileId fileId) {
89 return path::join(root, INCFS_INDEX_NAME, toString(fileId));
90 }
91
rootForCmd(int fd)92 static std::string rootForCmd(int fd) {
93 auto cmdFile = path::fromFd(fd);
94 if (cmdFile.empty()) {
95 LOG(INFO) << __func__ << "(): name empty for " << fd;
96 return {};
97 }
98 auto res = path::dirName(cmdFile);
99 if (res.empty()) {
100 LOG(INFO) << __func__ << "(): dirname empty for " << cmdFile;
101 return {};
102 }
103 if (!path::endsWith(cmdFile, INCFS_PENDING_READS_FILENAME)) {
104 LOG(INFO) << __func__ << "(): invalid file name " << cmdFile;
105 return {};
106 }
107 if (cmdFile.data() == res.data() || cmdFile.starts_with(res)) {
108 cmdFile.resize(res.size());
109 return cmdFile;
110 }
111 return std::string(res);
112 }
113
isFsAvailable()114 static bool isFsAvailable() {
115 static const char kProcFilesystems[] = "/proc/filesystems";
116 std::string filesystems;
117 if (!ab::ReadFileToString(kProcFilesystems, &filesystems)) {
118 return false;
119 }
120 const auto result = filesystems.find("\t" INCFS_NAME "\n") != std::string::npos;
121 LOG(INFO) << "isFsAvailable: " << (result ? "true" : "false");
122 return result;
123 }
124
getFirstApiLevel()125 static int getFirstApiLevel() {
126 uint64_t api_level = android::base::GetUintProperty<uint64_t>("ro.product.first_api_level", 0);
127 LOG(INFO) << "Initial API level of the device: " << api_level;
128 return api_level;
129 }
130
incFsPropertyValue()131 static std::string_view incFsPropertyValue() {
132 constexpr const int R_API = 30;
133 static const auto kDefaultValue{getFirstApiLevel() > R_API ? "on" : ""};
134 static const ab::NoDestructor<std::string> kValue{
135 IncrementalProperties::enable().value_or(kDefaultValue)};
136 LOG(INFO) << "ro.incremental.enable: " << *kValue;
137 return *kValue;
138 }
139
parseProperty(std::string_view property)140 static std::pair<bool, std::string_view> parseProperty(std::string_view property) {
141 auto boolVal = ab::ParseBool(property);
142 if (boolVal == ab::ParseBoolResult::kTrue) {
143 return {isFsAvailable(), {}};
144 }
145 if (boolVal == ab::ParseBoolResult::kFalse) {
146 return {false, {}};
147 }
148
149 // Don't load the module at once, but instead only check if it is loadable.
150 static const auto kModulePrefix = "module:"sv;
151 if (property.starts_with(kModulePrefix)) {
152 const auto modulePath = property.substr(kModulePrefix.size());
153 return {::access(details::c_str(modulePath), R_OK | X_OK), modulePath};
154 }
155 return {false, {}};
156 }
157
158 template <class Callback>
forEachFileIn(std::string_view dirPath,Callback cb)159 static IncFsErrorCode forEachFileIn(std::string_view dirPath, Callback cb) {
160 auto dir = path::openDir(details::c_str(dirPath));
161 if (!dir) {
162 return -EINVAL;
163 }
164
165 int res = 0;
166 while (auto entry = (errno = 0, ::readdir(dir.get()))) {
167 if (entry->d_type != DT_REG) {
168 continue;
169 }
170 ++res;
171 if (!cb(entry->d_name)) {
172 break;
173 }
174 }
175 if (errno) {
176 return -errno;
177 }
178 return res;
179 }
180
181 namespace {
182
183 class IncFsInit {
184 public:
IncFsInit()185 IncFsInit() {
186 auto [featureEnabled, moduleName] = parseProperty(incFsPropertyValue());
187 featureEnabled_ = featureEnabled;
188 moduleName_ = moduleName;
189 loaded_ = featureEnabled_ && isFsAvailable();
190 }
191
192 constexpr ~IncFsInit() = default;
193
enabled() const194 bool enabled() const { return featureEnabled_; }
enabledAndReady() const195 bool enabledAndReady() const {
196 if (!featureEnabled_) {
197 return false;
198 }
199 if (moduleName_.empty()) {
200 return true;
201 }
202 if (loaded_) {
203 return true;
204 }
205 std::call_once(loadedFlag_, [this] {
206 if (isFsAvailable()) {
207 // Loaded from a different process, I suppose.
208 loaded_ = true;
209 LOG(INFO) << "IncFS is already available, skipped loading";
210 return;
211 }
212 const ab::unique_fd fd(TEMP_FAILURE_RETRY(
213 ::open(details::c_str(moduleName_), O_RDONLY | O_NOFOLLOW | O_CLOEXEC)));
214 if (fd < 0) {
215 PLOG(ERROR) << "could not open IncFs kernel module \"" << moduleName_ << '"';
216 return;
217 }
218
219 const auto rc = syscall(__NR_finit_module, fd.get(), "", 0);
220 if (rc < 0) {
221 PLOG(ERROR) << "finit_module for IncFs \"" << moduleName_ << "\" failed";
222 return;
223 }
224 if (!isFsAvailable()) {
225 LOG(ERROR) << "loaded IncFs kernel module \"" << moduleName_
226 << "\" but incremental-fs is still not available";
227 }
228 loaded_ = true;
229 LOG(INFO) << "successfully loaded IncFs kernel module \"" << moduleName_ << '"';
230 });
231 return loaded_;
232 }
233
234 private:
235 bool featureEnabled_;
236 std::string_view moduleName_;
237 mutable std::once_flag loadedFlag_;
238 mutable bool loaded_;
239 };
240
241 } // namespace
242
init()243 static IncFsInit& init() {
244 static IncFsInit initer;
245 return initer;
246 }
247
IncFs_IsEnabled()248 bool IncFs_IsEnabled() {
249 return init().enabled();
250 }
251
readIncFsFeatures()252 static Features readIncFsFeatures() {
253 init().enabledAndReady();
254
255 static const char kSysfsFeaturesDir[] = "/sys/fs/" INCFS_NAME "/features";
256 const auto dir = path::openDir(kSysfsFeaturesDir);
257 if (!dir) {
258 PLOG(ERROR) << "IncFs_Features: failed to open features dir, assuming v1/none.";
259 return Features::none;
260 }
261
262 int res = Features::none;
263 while (auto entry = ::readdir(dir.get())) {
264 if (entry->d_type != DT_REG) {
265 continue;
266 }
267 if (entry->d_name == "corefs"sv) {
268 res |= Features::core;
269 } else if (entry->d_name == "v2"sv || entry->d_name == "report_uid"sv) {
270 res |= Features::v2;
271 }
272 }
273
274 LOG(INFO) << "IncFs_Features: " << ((res & Features::v2) ? "v2" : "v1");
275
276 return Features(res);
277 }
278
IncFs_Features()279 IncFsFeatures IncFs_Features() {
280 static const auto features = IncFsFeatures(readIncFsFeatures());
281 return features;
282 }
283
isIncFsFdImpl(int fd)284 bool isIncFsFdImpl(int fd) {
285 struct statfs fs = {};
286 if (::fstatfs(fd, &fs) != 0) {
287 PLOG(WARNING) << __func__ << "(): could not fstatfs fd " << fd;
288 return false;
289 }
290
291 return fs.f_type == (decltype(fs.f_type))INCFS_MAGIC_NUMBER;
292 }
293
isIncFsPathImpl(const char * path)294 bool isIncFsPathImpl(const char* path) {
295 struct statfs fs = {};
296 if (::statfs(path, &fs) != 0) {
297 PLOG(WARNING) << __func__ << "(): could not statfs " << path;
298 return false;
299 }
300
301 return fs.f_type == (decltype(fs.f_type))INCFS_MAGIC_NUMBER;
302 }
303
isDir(const char * path)304 static int isDir(const char* path) {
305 struct stat st;
306 if (::stat(path, &st) != 0) {
307 return -errno;
308 }
309 if (!S_ISDIR(st.st_mode)) {
310 return -ENOTDIR;
311 }
312 return 0;
313 }
314
isAbsolute(const char * path)315 static bool isAbsolute(const char* path) {
316 return path && path[0] == '/';
317 }
318
isValidMountTarget(const char * path)319 static int isValidMountTarget(const char* path) {
320 if (!isAbsolute(path)) {
321 return -EINVAL;
322 }
323 if (isIncFsPath(path)) {
324 LOG(ERROR) << "[incfs] mounting over existing incfs mount is not allowed";
325 return -EINVAL;
326 }
327 if (const auto err = isDir(path); err != 0) {
328 return err;
329 }
330 if (const auto err = path::isEmptyDir(path); err != 0) {
331 return err;
332 }
333 return 0;
334 }
335
rmDirContent(const char * path)336 static int rmDirContent(const char* path) {
337 auto dir = path::openDir(path);
338 if (!dir) {
339 return -EINVAL;
340 }
341 while (auto entry = ::readdir(dir.get())) {
342 if (entry->d_name == "."sv || entry->d_name == ".."sv) {
343 continue;
344 }
345 auto fullPath = ab::StringPrintf("%s/%s", path, entry->d_name);
346 if (entry->d_type == DT_DIR) {
347 if (const auto err = rmDirContent(fullPath.c_str()); err != 0) {
348 return err;
349 }
350 if (const auto err = ::rmdir(fullPath.c_str()); err != 0) {
351 return err;
352 }
353 } else {
354 if (const auto err = ::unlink(fullPath.c_str()); err != 0) {
355 return err;
356 }
357 }
358 }
359 return 0;
360 }
361
makeMountOptionsString(IncFsMountOptions options)362 static std::string makeMountOptionsString(IncFsMountOptions options) {
363 auto opts = ab::StringPrintf("read_timeout_ms=%u,readahead=0,rlog_pages=%u,rlog_wakeup_cnt=1,",
364 unsigned(options.defaultReadTimeoutMs),
365 unsigned(options.readLogBufferPages < 0
366 ? INCFS_DEFAULT_PAGE_READ_BUFFER_PAGES
367 : options.readLogBufferPages));
368 if (features() & Features::v2) {
369 ab::StringAppendF(&opts, "report_uid,");
370 if (options.sysfsName && *options.sysfsName) {
371 ab::StringAppendF(&opts, "sysfs_name=%s,", options.sysfsName);
372 }
373 }
374 return opts;
375 }
376
makeControl(const char * root)377 static IncFsControl* makeControl(const char* root) {
378 auto cmd = openRaw(root, INCFS_PENDING_READS_FILENAME);
379 if (!cmd.ok()) {
380 return nullptr;
381 }
382 ab::unique_fd pendingReads(fcntl(cmd.get(), F_DUPFD_CLOEXEC, cmd.get()));
383 if (!pendingReads.ok()) {
384 return nullptr;
385 }
386 auto logs = openRaw(root, INCFS_LOG_FILENAME);
387 if (!logs.ok()) {
388 return nullptr;
389 }
390 ab::unique_fd blocksWritten;
391 if (features() & Features::v2) {
392 blocksWritten = openRaw(root, INCFS_BLOCKS_WRITTEN_FILENAME);
393 if (!blocksWritten.ok()) {
394 return nullptr;
395 }
396 }
397 auto control =
398 IncFs_CreateControl(cmd.get(), pendingReads.get(), logs.get(), blocksWritten.get());
399 if (control) {
400 (void)cmd.release();
401 (void)pendingReads.release();
402 (void)logs.release();
403 (void)blocksWritten.release();
404 } else {
405 errno = ENOMEM;
406 }
407 return control;
408 }
409
makeCommandPath(std::string_view root,std::string_view item)410 static std::string makeCommandPath(std::string_view root, std::string_view item) {
411 auto [itemRoot, subpath] = registry().rootAndSubpathFor(item);
412 if (itemRoot != root) {
413 return {};
414 }
415 // TODO: add "/.cmd/" if we decide to use a separate control tree.
416 return path::join(itemRoot, subpath);
417 }
418
toString(IncFsFileId id,char * out)419 static void toString(IncFsFileId id, char* out) {
420 // Make sure this function matches the one in the kernel (e.g. same case for a-f digits).
421 static constexpr char kHexChar[] = "0123456789abcdef";
422
423 for (auto item = std::begin(id.data); item != std::end(id.data); ++item, out += 2) {
424 out[0] = kHexChar[(*item & 0xf0) >> 4];
425 out[1] = kHexChar[(*item & 0x0f)];
426 }
427 }
428
toStringImpl(IncFsFileId id)429 static std::string toStringImpl(IncFsFileId id) {
430 std::string res(kIncFsFileIdStringLength, '\0');
431 toString(id, res.data());
432 return res;
433 }
434
toFileIdImpl(std::string_view str)435 static IncFsFileId toFileIdImpl(std::string_view str) {
436 if (str.size() != kIncFsFileIdStringLength) {
437 return kIncFsInvalidFileId;
438 }
439
440 IncFsFileId res;
441 auto out = (char*)&res;
442 for (auto it = str.begin(); it != str.end(); it += 2, ++out) {
443 static const auto fromChar = [](char src) -> int {
444 if (src >= '0' && src <= '9') {
445 return src - '0';
446 }
447 if (src >= 'a' && src <= 'f') {
448 return src - 'a' + 10;
449 }
450 return -1;
451 };
452
453 const int c[2] = {fromChar(it[0]), fromChar(it[1])};
454 if (c[0] == -1 || c[1] == -1) {
455 errno = EINVAL;
456 return kIncFsInvalidFileId;
457 }
458 *out = (c[0] << 4) | c[1];
459 }
460 return res;
461 }
462
IncFs_FileIdToString(IncFsFileId id,char * out)463 int IncFs_FileIdToString(IncFsFileId id, char* out) {
464 if (!out) {
465 return -EINVAL;
466 }
467 toString(id, out);
468 return 0;
469 }
470
IncFs_FileIdFromString(const char * in)471 IncFsFileId IncFs_FileIdFromString(const char* in) {
472 return toFileIdImpl({in, kIncFsFileIdStringLength});
473 }
474
IncFs_FileIdFromMetadata(IncFsSpan metadata)475 IncFsFileId IncFs_FileIdFromMetadata(IncFsSpan metadata) {
476 IncFsFileId id = {};
477 if (size_t(metadata.size) <= sizeof(id)) {
478 memcpy(&id, metadata.data, metadata.size);
479 } else {
480 uint8_t buffer[SHA_DIGEST_LENGTH];
481 static_assert(sizeof(buffer) >= sizeof(id));
482
483 SHA_CTX ctx;
484 SHA1_Init(&ctx);
485 SHA1_Update(&ctx, metadata.data, metadata.size);
486 SHA1_Final(buffer, &ctx);
487 memcpy(&id, buffer, sizeof(id));
488 }
489 return id;
490 }
491
restoreconControlFiles(std::string_view targetDir)492 static bool restoreconControlFiles(std::string_view targetDir) {
493 static constexpr auto restorecon = [](const char* name) {
494 if (const auto err = selinux_android_restorecon(name, SELINUX_ANDROID_RESTORECON_FORCE);
495 err != 0) {
496 errno = -err;
497 PLOG(ERROR) << "[incfs] Failed to restorecon: " << name;
498 return false;
499 }
500 return true;
501 };
502 if (!restorecon(path::join(targetDir, INCFS_PENDING_READS_FILENAME).c_str())) {
503 return false;
504 }
505 if (!restorecon(path::join(targetDir, INCFS_LOG_FILENAME).c_str())) {
506 return false;
507 }
508 if ((features() & Features::v2) &&
509 !restorecon(path::join(targetDir, INCFS_BLOCKS_WRITTEN_FILENAME).c_str())) {
510 return false;
511 }
512 return true;
513 }
514
IncFs_Mount(const char * backingPath,const char * targetDir,IncFsMountOptions options)515 IncFsControl* IncFs_Mount(const char* backingPath, const char* targetDir,
516 IncFsMountOptions options) {
517 if (!init().enabledAndReady()) {
518 LOG(WARNING) << "[incfs] Feature is not enabled";
519 errno = ENOTSUP;
520 return nullptr;
521 }
522
523 if (auto err = isValidMountTarget(targetDir); err != 0) {
524 errno = -err;
525 return nullptr;
526 }
527 if (!isAbsolute(backingPath)) {
528 errno = EINVAL;
529 return nullptr;
530 }
531
532 if (options.flags & createOnly) {
533 if (const auto err = path::isEmptyDir(backingPath); err != 0) {
534 errno = -err;
535 return nullptr;
536 }
537 } else if (options.flags & android::incfs::truncate) {
538 if (const auto err = rmDirContent(backingPath); err != 0) {
539 errno = -err;
540 return nullptr;
541 }
542 }
543
544 const auto opts = makeMountOptionsString(options);
545 if (::mount(backingPath, targetDir, INCFS_NAME, MS_NOSUID | MS_NODEV | MS_NOATIME,
546 opts.c_str())) {
547 PLOG(ERROR) << "[incfs] Failed to mount IncFS filesystem: " << targetDir;
548 return nullptr;
549 }
550
551 if (!restoreconControlFiles(targetDir)) {
552 (void)IncFs_Unmount(targetDir);
553 return nullptr;
554 }
555
556 auto control = makeControl(targetDir);
557 if (control == nullptr) {
558 (void)IncFs_Unmount(targetDir);
559 return nullptr;
560 }
561 return control;
562 }
563
IncFs_Open(const char * dir)564 IncFsControl* IncFs_Open(const char* dir) {
565 auto root = registry().rootFor(dir);
566 if (root.empty()) {
567 errno = EINVAL;
568 return nullptr;
569 }
570 return makeControl(details::c_str(root));
571 }
572
IncFs_GetControlFd(const IncFsControl * control,IncFsFdType type)573 IncFsFd IncFs_GetControlFd(const IncFsControl* control, IncFsFdType type) {
574 if (!control) {
575 return -EINVAL;
576 }
577 switch (type) {
578 case CMD:
579 return control->cmd;
580 case PENDING_READS:
581 return control->pendingReads;
582 case LOGS:
583 return control->logs;
584 case BLOCKS_WRITTEN:
585 return control->blocksWritten;
586 default:
587 return -EINVAL;
588 }
589 }
590
IncFs_ReleaseControlFds(IncFsControl * control,IncFsFd out[],IncFsSize outSize)591 IncFsSize IncFs_ReleaseControlFds(IncFsControl* control, IncFsFd out[], IncFsSize outSize) {
592 if (!control || !out) {
593 return -EINVAL;
594 }
595 if (outSize < IncFsFdType::FDS_COUNT) {
596 return -ERANGE;
597 }
598 out[CMD] = std::exchange(control->cmd, -1);
599 out[PENDING_READS] = std::exchange(control->pendingReads, -1);
600 out[LOGS] = std::exchange(control->logs, -1);
601 out[BLOCKS_WRITTEN] = std::exchange(control->blocksWritten, -1);
602 return IncFsFdType::FDS_COUNT;
603 }
604
IncFs_CreateControl(IncFsFd cmd,IncFsFd pendingReads,IncFsFd logs,IncFsFd blocksWritten)605 IncFsControl* IncFs_CreateControl(IncFsFd cmd, IncFsFd pendingReads, IncFsFd logs,
606 IncFsFd blocksWritten) {
607 return new IncFsControl(cmd, pendingReads, logs, blocksWritten);
608 }
609
IncFs_DeleteControl(IncFsControl * control)610 void IncFs_DeleteControl(IncFsControl* control) {
611 if (control) {
612 if (control->cmd >= 0) {
613 close(control->cmd);
614 }
615 if (control->pendingReads >= 0) {
616 close(control->pendingReads);
617 }
618 if (control->logs >= 0) {
619 close(control->logs);
620 }
621 if (control->blocksWritten >= 0) {
622 close(control->blocksWritten);
623 }
624 delete control;
625 }
626 }
627
IncFs_SetOptions(const IncFsControl * control,IncFsMountOptions options)628 IncFsErrorCode IncFs_SetOptions(const IncFsControl* control, IncFsMountOptions options) {
629 if (!control) {
630 return -EINVAL;
631 }
632 auto root = rootForCmd(control->cmd);
633 if (root.empty()) {
634 return -EINVAL;
635 }
636 auto opts = makeMountOptionsString(options);
637 if (::mount(nullptr, root.c_str(), nullptr, MS_REMOUNT | MS_NOSUID | MS_NODEV | MS_NOATIME,
638 opts.c_str()) != 0) {
639 const auto error = errno;
640 PLOG(ERROR) << "[incfs] Failed to remount IncFS filesystem: " << root;
641 return -error;
642 }
643 return 0;
644 }
645
IncFs_Root(const IncFsControl * control,char buffer[],size_t * bufferSize)646 IncFsErrorCode IncFs_Root(const IncFsControl* control, char buffer[], size_t* bufferSize) {
647 if (!control) {
648 return -EINVAL;
649 }
650 std::string result = rootForCmd(control->cmd);
651 if (*bufferSize <= result.size()) {
652 *bufferSize = result.size() + 1;
653 return -EOVERFLOW;
654 }
655 result.copy(buffer, result.size());
656 buffer[result.size()] = '\0';
657 *bufferSize = result.size();
658 return 0;
659 }
660
661 template <class T>
read(IncFsSpan & data)662 std::optional<T> read(IncFsSpan& data) {
663 if (data.size < (int32_t)sizeof(T)) {
664 return {};
665 }
666 T res;
667 memcpy(&res, data.data, sizeof(res));
668 data.data += sizeof(res);
669 data.size -= sizeof(res);
670 return res;
671 }
672
validateSignatureFormat(IncFsSpan signature)673 static IncFsErrorCode validateSignatureFormat(IncFsSpan signature) {
674 if (signature.data == nullptr && signature.size == 0) {
675 return 0; // it's fine to have unverified files too
676 }
677 if ((signature.data == nullptr) != (signature.size == 0)) {
678 return -EINVAL;
679 }
680
681 // These structs are here purely for checking the minimum size. Maybe will use them for
682 // parsing later.
683 struct __attribute__((packed)) Hashing {
684 int32_t size;
685 int32_t algorithm;
686 int8_t log2_blocksize;
687 int32_t salt_size;
688 int32_t raw_root_hash_size;
689 };
690 struct __attribute__((packed)) Signing {
691 int32_t size;
692 int32_t apk_digest_size;
693 int32_t certificate_size;
694 int32_t addl_data_size;
695 int32_t public_key_size;
696 int32_t algorithm;
697 int32_t signature_size;
698 };
699 struct __attribute__((packed)) MinSignature {
700 int32_t version;
701 Hashing hashing_info;
702 Signing signing_info;
703 };
704
705 if (signature.size < (int32_t)sizeof(MinSignature)) {
706 return -ERANGE;
707 }
708 if (signature.size > INCFS_MAX_SIGNATURE_SIZE) {
709 return -ERANGE;
710 }
711
712 auto version = read<int32_t>(signature);
713 if (version.value_or(-1) != INCFS_SIGNATURE_VERSION) {
714 return -EINVAL;
715 }
716 auto hashSize = read<int32_t>(signature);
717 if (!hashSize || signature.size < *hashSize) {
718 return -EINVAL;
719 }
720 auto hashAlgo = read<int32_t>(signature);
721 if (hashAlgo.value_or(-1) != INCFS_HASH_TREE_SHA256) {
722 return -EINVAL;
723 }
724 auto logBlockSize = read<int8_t>(signature);
725 if (logBlockSize.value_or(-1) != 12 /* 2^12 == 4096 */) {
726 return -EINVAL;
727 }
728 auto saltSize = read<int32_t>(signature);
729 if (saltSize.value_or(-1) != 0) {
730 return -EINVAL;
731 }
732 auto rootHashSize = read<int32_t>(signature);
733 if (rootHashSize.value_or(-1) != INCFS_MAX_HASH_SIZE) {
734 return -EINVAL;
735 }
736 if (signature.size < *rootHashSize) {
737 return -EINVAL;
738 }
739 signature.data += *rootHashSize;
740 signature.size -= *rootHashSize;
741 auto signingSize = read<int32_t>(signature);
742 // everything remaining has to be in the signing info
743 if (signingSize.value_or(-1) != signature.size) {
744 return -EINVAL;
745 }
746
747 // TODO: validate the signature part too.
748 return 0;
749 }
750
IncFs_MakeFile(const IncFsControl * control,const char * path,int32_t mode,IncFsFileId id,IncFsNewFileParams params)751 IncFsErrorCode IncFs_MakeFile(const IncFsControl* control, const char* path, int32_t mode,
752 IncFsFileId id, IncFsNewFileParams params) {
753 if (!control) {
754 return -EINVAL;
755 }
756
757 auto [root, subpath] = registry().rootAndSubpathFor(path);
758 if (root.empty()) {
759 PLOG(WARNING) << "[incfs] makeFile failed for path " << path << ", root is empty.";
760 return -EINVAL;
761 }
762 if (params.size < 0) {
763 LOG(WARNING) << "[incfs] makeFile failed for path " << path
764 << ", size is invalid: " << params.size;
765 return -ERANGE;
766 }
767
768 const auto [subdir, name] = path::splitDirBase(subpath);
769 incfs_new_file_args args = {
770 .size = (uint64_t)params.size,
771 .mode = (uint16_t)mode,
772 .directory_path = (uint64_t)subdir.data(),
773 .file_name = (uint64_t)name.data(),
774 .file_attr = (uint64_t)params.metadata.data,
775 .file_attr_len = (uint32_t)params.metadata.size,
776 };
777 static_assert(sizeof(args.file_id.bytes) == sizeof(id.data));
778 memcpy(args.file_id.bytes, id.data, sizeof(args.file_id.bytes));
779
780 if (auto err = validateSignatureFormat(params.signature)) {
781 return err;
782 }
783 args.signature_info = (uint64_t)(uintptr_t)params.signature.data;
784 args.signature_size = (uint64_t)params.signature.size;
785
786 if (::ioctl(control->cmd, INCFS_IOC_CREATE_FILE, &args)) {
787 PLOG(WARNING) << "[incfs] makeFile failed for " << root << " / " << subdir << " / " << name
788 << " of " << params.size << " bytes";
789 return -errno;
790 }
791 if (::chmod(path::join(root, subpath).c_str(), mode)) {
792 PLOG(WARNING) << "[incfs] couldn't change file mode to 0" << std::oct << mode;
793 }
794
795 return 0;
796 }
797
IncFs_MakeMappedFile(const IncFsControl * control,const char * path,int32_t mode,IncFsNewMappedFileParams params)798 IncFsErrorCode IncFs_MakeMappedFile(const IncFsControl* control, const char* path, int32_t mode,
799 IncFsNewMappedFileParams params) {
800 if (!control) {
801 return -EINVAL;
802 }
803
804 auto [root, subpath] = registry().rootAndSubpathFor(path);
805 if (root.empty()) {
806 PLOG(WARNING) << "[incfs] makeMappedFile failed for path " << path << ", root is empty.";
807 return -EINVAL;
808 }
809 if (params.size < 0) {
810 LOG(WARNING) << "[incfs] makeMappedFile failed for path " << path
811 << ", size is invalid: " << params.size;
812 return -ERANGE;
813 }
814
815 const auto [subdir, name] = path::splitDirBase(subpath);
816 incfs_create_mapped_file_args args = {
817 .size = (uint64_t)params.size,
818 .mode = (uint16_t)mode,
819 .directory_path = (uint64_t)subdir.data(),
820 .file_name = (uint64_t)name.data(),
821 .source_offset = (uint64_t)params.sourceOffset,
822 };
823 static_assert(sizeof(args.source_file_id.bytes) == sizeof(params.sourceId.data));
824 memcpy(args.source_file_id.bytes, params.sourceId.data, sizeof(args.source_file_id.bytes));
825
826 if (::ioctl(control->cmd, INCFS_IOC_CREATE_MAPPED_FILE, &args)) {
827 PLOG(WARNING) << "[incfs] makeMappedFile failed for " << root << " / " << subdir << " / "
828 << name << " of " << params.size << " bytes starting at "
829 << params.sourceOffset;
830 return -errno;
831 }
832 if (::chmod(path::join(root, subpath).c_str(), mode)) {
833 PLOG(WARNING) << "[incfs] makeMappedFile error: couldn't change file mode to 0" << std::oct
834 << mode;
835 }
836
837 return 0;
838 }
839
makeDir(const char * commandPath,int32_t mode,bool allowExisting)840 static IncFsErrorCode makeDir(const char* commandPath, int32_t mode, bool allowExisting) {
841 if (!::mkdir(commandPath, mode)) {
842 if (::chmod(commandPath, mode)) {
843 PLOG(WARNING) << "[incfs] couldn't change directory mode to 0" << std::oct << mode;
844 }
845 return 0;
846 }
847 // don't touch the existing dir's mode - mkdir(1) works that way.
848 return (allowExisting && errno == EEXIST) ? 0 : -errno;
849 }
850
makeDirs(std::string_view commandPath,std::string_view path,std::string_view root,int32_t mode)851 static IncFsErrorCode makeDirs(std::string_view commandPath, std::string_view path,
852 std::string_view root, int32_t mode) {
853 auto commandCPath = details::c_str(commandPath);
854 const auto mkdirRes = makeDir(commandCPath, mode, true);
855 if (!mkdirRes) {
856 return 0;
857 }
858 if (mkdirRes != -ENOENT) {
859 LOG(ERROR) << __func__ << "(): mkdir failed for " << path << " - " << mkdirRes;
860 return mkdirRes;
861 }
862
863 const auto parent = path::dirName(commandPath);
864 if (!path::startsWith(parent, root)) {
865 // went too far, already out of the root mount
866 return -EINVAL;
867 }
868
869 if (auto parentMkdirRes = makeDirs(parent, path::dirName(path), root, mode)) {
870 return parentMkdirRes;
871 }
872 return makeDir(commandCPath, mode, true);
873 }
874
IncFs_MakeDir(const IncFsControl * control,const char * path,int32_t mode)875 IncFsErrorCode IncFs_MakeDir(const IncFsControl* control, const char* path, int32_t mode) {
876 if (!control) {
877 return -EINVAL;
878 }
879 const auto root = rootForCmd(control->cmd);
880 if (root.empty()) {
881 LOG(ERROR) << __func__ << "(): root is empty for " << path;
882 return -EINVAL;
883 }
884 auto commandPath = makeCommandPath(root, path);
885 if (commandPath.empty()) {
886 LOG(ERROR) << __func__ << "(): commandPath is empty for " << path;
887 return -EINVAL;
888 }
889 if (auto res = makeDir(commandPath.c_str(), mode, false)) {
890 LOG(ERROR) << __func__ << "(): mkdir failed for " << commandPath << " - " << res;
891 return res;
892 }
893 return 0;
894 }
895
IncFs_MakeDirs(const IncFsControl * control,const char * path,int32_t mode)896 IncFsErrorCode IncFs_MakeDirs(const IncFsControl* control, const char* path, int32_t mode) {
897 if (!control) {
898 return -EINVAL;
899 }
900 const auto root = rootForCmd(control->cmd);
901 if (root.empty()) {
902 LOG(ERROR) << __func__ << "(): root is empty for " << path;
903 return -EINVAL;
904 }
905 auto commandPath = makeCommandPath(root, path);
906 if (commandPath.empty()) {
907 LOG(ERROR) << __func__ << "(): commandPath is empty for " << path;
908 return -EINVAL;
909 }
910 return makeDirs(commandPath, path, root, mode);
911 }
912
getMetadata(const char * path,char buffer[],size_t * bufferSize)913 static IncFsErrorCode getMetadata(const char* path, char buffer[], size_t* bufferSize) {
914 const auto res = ::getxattr(path, kMetadataAttrName, buffer, *bufferSize);
915 if (res < 0) {
916 if (errno == ERANGE) {
917 auto neededSize = ::getxattr(path, kMetadataAttrName, buffer, 0);
918 if (neededSize >= 0) {
919 *bufferSize = neededSize;
920 return 0;
921 }
922 }
923 return -errno;
924 }
925 *bufferSize = res;
926 return 0;
927 }
928
IncFs_GetMetadataById(const IncFsControl * control,IncFsFileId fileId,char buffer[],size_t * bufferSize)929 IncFsErrorCode IncFs_GetMetadataById(const IncFsControl* control, IncFsFileId fileId, char buffer[],
930 size_t* bufferSize) {
931 if (!control) {
932 return -EINVAL;
933 }
934
935 const auto root = rootForCmd(control->cmd);
936 if (root.empty()) {
937 return -EINVAL;
938 }
939 auto name = indexPath(root, fileId);
940 return getMetadata(details::c_str(name), buffer, bufferSize);
941 }
942
IncFs_GetMetadataByPath(const IncFsControl * control,const char * path,char buffer[],size_t * bufferSize)943 IncFsErrorCode IncFs_GetMetadataByPath(const IncFsControl* control, const char* path, char buffer[],
944 size_t* bufferSize) {
945 if (!control) {
946 return -EINVAL;
947 }
948 const auto pathRoot = registry().rootFor(path);
949 const auto root = rootForCmd(control->cmd);
950 if (root.empty() || root != pathRoot) {
951 return -EINVAL;
952 }
953
954 return getMetadata(path, buffer, bufferSize);
955 }
956
957 template <class GetterFunc, class Param>
getId(GetterFunc getter,Param param)958 static IncFsFileId getId(GetterFunc getter, Param param) {
959 char buffer[kIncFsFileIdStringLength];
960 const auto res = getter(param, kIdAttrName, buffer, sizeof(buffer));
961 if (res != sizeof(buffer)) {
962 return kIncFsInvalidFileId;
963 }
964 return toFileIdImpl({buffer, std::size(buffer)});
965 }
966
IncFs_GetId(const IncFsControl * control,const char * path)967 IncFsFileId IncFs_GetId(const IncFsControl* control, const char* path) {
968 if (!control) {
969 return kIncFsInvalidFileId;
970 }
971 const auto pathRoot = registry().rootFor(path);
972 const auto root = rootForCmd(control->cmd);
973 if (root.empty() || root != pathRoot) {
974 errno = EINVAL;
975 return kIncFsInvalidFileId;
976 }
977 return getId(::getxattr, path);
978 }
979
getSignature(int fd,char buffer[],size_t * bufferSize)980 static IncFsErrorCode getSignature(int fd, char buffer[], size_t* bufferSize) {
981 incfs_get_file_sig_args args = {
982 .file_signature = (uint64_t)buffer,
983 .file_signature_buf_size = (uint32_t)*bufferSize,
984 };
985
986 auto res = ::ioctl(fd, INCFS_IOC_READ_FILE_SIGNATURE, &args);
987 if (res < 0) {
988 if (errno == E2BIG) {
989 *bufferSize = INCFS_MAX_SIGNATURE_SIZE;
990 }
991 return -errno;
992 }
993 *bufferSize = args.file_signature_len_out;
994 return 0;
995 }
996
IncFs_GetSignatureById(const IncFsControl * control,IncFsFileId fileId,char buffer[],size_t * bufferSize)997 IncFsErrorCode IncFs_GetSignatureById(const IncFsControl* control, IncFsFileId fileId,
998 char buffer[], size_t* bufferSize) {
999 if (!control) {
1000 return -EINVAL;
1001 }
1002
1003 const auto root = rootForCmd(control->cmd);
1004 if (root.empty()) {
1005 return -EINVAL;
1006 }
1007 auto file = indexPath(root, fileId);
1008 auto fd = openRaw(file);
1009 if (fd < 0) {
1010 return fd.get();
1011 }
1012 return getSignature(fd, buffer, bufferSize);
1013 }
1014
IncFs_GetSignatureByPath(const IncFsControl * control,const char * path,char buffer[],size_t * bufferSize)1015 IncFsErrorCode IncFs_GetSignatureByPath(const IncFsControl* control, const char* path,
1016 char buffer[], size_t* bufferSize) {
1017 if (!control) {
1018 return -EINVAL;
1019 }
1020
1021 const auto pathRoot = registry().rootFor(path);
1022 const auto root = rootForCmd(control->cmd);
1023 if (root.empty() || root != pathRoot) {
1024 return -EINVAL;
1025 }
1026 return IncFs_UnsafeGetSignatureByPath(path, buffer, bufferSize);
1027 }
1028
IncFs_UnsafeGetSignatureByPath(const char * path,char buffer[],size_t * bufferSize)1029 IncFsErrorCode IncFs_UnsafeGetSignatureByPath(const char* path, char buffer[], size_t* bufferSize) {
1030 if (!isIncFsPath(path)) {
1031 return -EINVAL;
1032 }
1033 auto fd = openRaw(path);
1034 if (fd < 0) {
1035 return fd.get();
1036 }
1037 return getSignature(fd, buffer, bufferSize);
1038 }
1039
IncFs_Link(const IncFsControl * control,const char * fromPath,const char * wherePath)1040 IncFsErrorCode IncFs_Link(const IncFsControl* control, const char* fromPath,
1041 const char* wherePath) {
1042 if (!control) {
1043 return -EINVAL;
1044 }
1045
1046 auto root = rootForCmd(control->cmd);
1047 if (root.empty()) {
1048 return -EINVAL;
1049 }
1050 auto cmdFrom = makeCommandPath(root, fromPath);
1051 if (cmdFrom.empty()) {
1052 return -EINVAL;
1053 }
1054 auto cmdWhere = makeCommandPath(root, wherePath);
1055 if (cmdWhere.empty()) {
1056 return -EINVAL;
1057 }
1058 if (::link(cmdFrom.c_str(), cmdWhere.c_str())) {
1059 return -errno;
1060 }
1061 return 0;
1062 }
1063
IncFs_Unlink(const IncFsControl * control,const char * path)1064 IncFsErrorCode IncFs_Unlink(const IncFsControl* control, const char* path) {
1065 if (!control) {
1066 return -EINVAL;
1067 }
1068
1069 auto root = rootForCmd(control->cmd);
1070 if (root.empty()) {
1071 return -EINVAL;
1072 }
1073 auto cmdPath = makeCommandPath(root, path);
1074 if (cmdPath.empty()) {
1075 return -EINVAL;
1076 }
1077 if (::unlink(cmdPath.c_str())) {
1078 if (errno == EISDIR) {
1079 if (!::rmdir(cmdPath.c_str())) {
1080 return 0;
1081 }
1082 }
1083 return -errno;
1084 }
1085 return 0;
1086 }
1087
1088 template <class RawPendingRead>
waitForReadsImpl(int fd,int32_t timeoutMs,RawPendingRead pendingReadsBuffer[],size_t * pendingReadsBufferSize)1089 static int waitForReadsImpl(int fd, int32_t timeoutMs, RawPendingRead pendingReadsBuffer[],
1090 size_t* pendingReadsBufferSize) {
1091 using namespace std::chrono;
1092 auto hrTimeout = steady_clock::duration(milliseconds(timeoutMs));
1093
1094 while (hrTimeout > hrTimeout.zero() || (!pendingReadsBuffer && hrTimeout == hrTimeout.zero())) {
1095 const auto startTs = steady_clock::now();
1096
1097 pollfd pfd = {fd, POLLIN, 0};
1098 const auto res = ::poll(&pfd, 1, duration_cast<milliseconds>(hrTimeout).count());
1099 if (res > 0) {
1100 break;
1101 }
1102 if (res == 0) {
1103 if (pendingReadsBufferSize) {
1104 *pendingReadsBufferSize = 0;
1105 }
1106 return -ETIMEDOUT;
1107 }
1108 const auto error = errno;
1109 if (error != EINTR) {
1110 PLOG(ERROR) << "poll() failed";
1111 return -error;
1112 }
1113 hrTimeout -= steady_clock::now() - startTs;
1114 }
1115 if (!pendingReadsBuffer) {
1116 return hrTimeout < hrTimeout.zero() ? -ETIMEDOUT : 0;
1117 }
1118
1119 auto res =
1120 ::read(fd, pendingReadsBuffer, *pendingReadsBufferSize * sizeof(*pendingReadsBuffer));
1121 if (res < 0) {
1122 const auto error = errno;
1123 PLOG(ERROR) << "read() failed";
1124 return -error;
1125 }
1126 if (res == 0) {
1127 *pendingReadsBufferSize = 0;
1128 return -ETIMEDOUT;
1129 }
1130 if ((res % sizeof(*pendingReadsBuffer)) != 0) {
1131 PLOG(ERROR) << "read() returned half of a struct??";
1132 return -EFAULT;
1133 }
1134 *pendingReadsBufferSize = res / sizeof(*pendingReadsBuffer);
1135 return 0;
1136 }
1137
1138 template <class PublicPendingRead, class RawPendingRead>
convertRead(RawPendingRead rawRead)1139 PublicPendingRead convertRead(RawPendingRead rawRead) {
1140 PublicPendingRead res = {
1141 .bootClockTsUs = rawRead.timestamp_us,
1142 .block = (IncFsBlockIndex)rawRead.block_index,
1143 .serialNo = rawRead.serial_number,
1144 };
1145 memcpy(&res.id.data, rawRead.file_id.bytes, sizeof(res.id.data));
1146
1147 if constexpr (std::is_same_v<PublicPendingRead, IncFsReadInfoWithUid>) {
1148 if constexpr (std::is_same_v<RawPendingRead, incfs_pending_read_info2>) {
1149 res.uid = rawRead.uid;
1150 } else {
1151 res.uid = kIncFsNoUid;
1152 }
1153 }
1154 return res;
1155 }
1156
1157 template <class RawPendingRead, class PublicPendingRead>
waitForReads(IncFsFd readFd,int32_t timeoutMs,PublicPendingRead buffer[],size_t * bufferSize)1158 static int waitForReads(IncFsFd readFd, int32_t timeoutMs, PublicPendingRead buffer[],
1159 size_t* bufferSize) {
1160 std::vector<RawPendingRead> pendingReads(*bufferSize);
1161 if (const auto res = waitForReadsImpl(readFd, timeoutMs, pendingReads.data(), bufferSize)) {
1162 return res;
1163 }
1164 for (size_t i = 0; i != *bufferSize; ++i) {
1165 buffer[i] = convertRead<PublicPendingRead>(pendingReads[i]);
1166 }
1167 return 0;
1168 }
1169
1170 template <class PublicPendingRead>
waitForReads(IncFsFd readFd,int32_t timeoutMs,PublicPendingRead buffer[],size_t * bufferSize)1171 static int waitForReads(IncFsFd readFd, int32_t timeoutMs, PublicPendingRead buffer[],
1172 size_t* bufferSize) {
1173 if (features() & Features::v2) {
1174 return waitForReads<incfs_pending_read_info2>(readFd, timeoutMs, buffer, bufferSize);
1175 }
1176 return waitForReads<incfs_pending_read_info>(readFd, timeoutMs, buffer, bufferSize);
1177 }
1178
IncFs_WaitForPendingReads(const IncFsControl * control,int32_t timeoutMs,IncFsReadInfo buffer[],size_t * bufferSize)1179 IncFsErrorCode IncFs_WaitForPendingReads(const IncFsControl* control, int32_t timeoutMs,
1180 IncFsReadInfo buffer[], size_t* bufferSize) {
1181 if (!control || control->pendingReads < 0) {
1182 return -EINVAL;
1183 }
1184
1185 return waitForReads(control->pendingReads, timeoutMs, buffer, bufferSize);
1186 }
1187
IncFs_WaitForPendingReadsWithUid(const IncFsControl * control,int32_t timeoutMs,IncFsReadInfoWithUid buffer[],size_t * bufferSize)1188 IncFsErrorCode IncFs_WaitForPendingReadsWithUid(const IncFsControl* control, int32_t timeoutMs,
1189 IncFsReadInfoWithUid buffer[], size_t* bufferSize) {
1190 if (!control || control->pendingReads < 0) {
1191 return -EINVAL;
1192 }
1193
1194 return waitForReads(control->pendingReads, timeoutMs, buffer, bufferSize);
1195 }
1196
IncFs_WaitForPageReads(const IncFsControl * control,int32_t timeoutMs,IncFsReadInfo buffer[],size_t * bufferSize)1197 IncFsErrorCode IncFs_WaitForPageReads(const IncFsControl* control, int32_t timeoutMs,
1198 IncFsReadInfo buffer[], size_t* bufferSize) {
1199 if (!control || control->logs < 0) {
1200 return -EINVAL;
1201 }
1202
1203 return waitForReads(control->logs, timeoutMs, buffer, bufferSize);
1204 }
1205
IncFs_WaitForPageReadsWithUid(const IncFsControl * control,int32_t timeoutMs,IncFsReadInfoWithUid buffer[],size_t * bufferSize)1206 IncFsErrorCode IncFs_WaitForPageReadsWithUid(const IncFsControl* control, int32_t timeoutMs,
1207 IncFsReadInfoWithUid buffer[], size_t* bufferSize) {
1208 if (!control || control->logs < 0) {
1209 return -EINVAL;
1210 }
1211
1212 return waitForReads(control->logs, timeoutMs, buffer, bufferSize);
1213 }
1214
openForSpecialOps(int cmd,const char * path)1215 static IncFsFd openForSpecialOps(int cmd, const char* path) {
1216 ab::unique_fd fd(::open(path, O_RDONLY | O_CLOEXEC));
1217 if (fd < 0) {
1218 return -errno;
1219 }
1220 struct incfs_permit_fill args = {.file_descriptor = (uint32_t)fd.get()};
1221 auto err = ::ioctl(cmd, INCFS_IOC_PERMIT_FILL, &args);
1222 if (err < 0) {
1223 return -errno;
1224 }
1225 return fd.release();
1226 }
1227
IncFs_OpenForSpecialOpsByPath(const IncFsControl * control,const char * path)1228 IncFsFd IncFs_OpenForSpecialOpsByPath(const IncFsControl* control, const char* path) {
1229 if (!control) {
1230 return -EINVAL;
1231 }
1232
1233 const auto pathRoot = registry().rootFor(path);
1234 const auto cmd = control->cmd;
1235 const auto root = rootForCmd(cmd);
1236 if (root.empty() || root != pathRoot) {
1237 return -EINVAL;
1238 }
1239 return openForSpecialOps(cmd, makeCommandPath(root, path).c_str());
1240 }
1241
IncFs_OpenForSpecialOpsById(const IncFsControl * control,IncFsFileId id)1242 IncFsFd IncFs_OpenForSpecialOpsById(const IncFsControl* control, IncFsFileId id) {
1243 if (!control) {
1244 return -EINVAL;
1245 }
1246
1247 const auto cmd = control->cmd;
1248 const auto root = rootForCmd(cmd);
1249 if (root.empty()) {
1250 return -EINVAL;
1251 }
1252 auto name = indexPath(root, id);
1253 return openForSpecialOps(cmd, makeCommandPath(root, name).c_str());
1254 }
1255
writeBlocks(int fd,const incfs_fill_block blocks[],int blocksCount)1256 static int writeBlocks(int fd, const incfs_fill_block blocks[], int blocksCount) {
1257 if (fd < 0 || blocksCount == 0) {
1258 return 0;
1259 }
1260 if (blocksCount < 0) {
1261 return -EINVAL;
1262 }
1263
1264 auto ptr = blocks;
1265 const auto end = blocks + blocksCount;
1266 do {
1267 struct incfs_fill_blocks args = {.count = uint64_t(end - ptr),
1268 .fill_blocks = (uint64_t)(uintptr_t)ptr};
1269 const auto written = ::ioctl(fd, INCFS_IOC_FILL_BLOCKS, &args);
1270 if (written < 0) {
1271 if (errno == EINTR) {
1272 continue;
1273 }
1274 const auto error = errno;
1275 PLOG(WARNING) << "writing IncFS blocks failed";
1276 if (ptr == blocks) {
1277 return -error;
1278 }
1279 // something has been written, return a success here and let the
1280 // next call handle the error.
1281 break;
1282 }
1283 ptr += written;
1284 } while (ptr < end);
1285 return ptr - blocks;
1286 }
1287
IncFs_WriteBlocks(const IncFsDataBlock blocks[],size_t blocksCount)1288 IncFsErrorCode IncFs_WriteBlocks(const IncFsDataBlock blocks[], size_t blocksCount) {
1289 incfs_fill_block incfsBlocks[128];
1290 int writtenCount = 0;
1291 int incfsBlocksUsed = 0;
1292 int lastBlockFd = -1;
1293 for (size_t i = 0; i < blocksCount; ++i) {
1294 if (lastBlockFd != blocks[i].fileFd || incfsBlocksUsed == std::size(incfsBlocks)) {
1295 auto count = writeBlocks(lastBlockFd, incfsBlocks, incfsBlocksUsed);
1296 if (count > 0) {
1297 writtenCount += count;
1298 }
1299 if (count != incfsBlocksUsed) {
1300 return writtenCount ? writtenCount : count;
1301 }
1302 lastBlockFd = blocks[i].fileFd;
1303 incfsBlocksUsed = 0;
1304 }
1305 incfsBlocks[incfsBlocksUsed] = incfs_fill_block{
1306 .block_index = (uint32_t)blocks[i].pageIndex,
1307 .data_len = blocks[i].dataSize,
1308 .data = (uint64_t)blocks[i].data,
1309 .compression = (uint8_t)blocks[i].compression,
1310 .flags = uint8_t(blocks[i].kind == INCFS_BLOCK_KIND_HASH ? INCFS_BLOCK_FLAGS_HASH
1311 : 0),
1312 };
1313 ++incfsBlocksUsed;
1314 }
1315 auto count = writeBlocks(lastBlockFd, incfsBlocks, incfsBlocksUsed);
1316 if (count > 0) {
1317 writtenCount += count;
1318 }
1319 return writtenCount ? writtenCount : count;
1320 }
1321
IncFs_BindMount(const char * sourceDir,const char * targetDir)1322 IncFsErrorCode IncFs_BindMount(const char* sourceDir, const char* targetDir) {
1323 if (!enabled()) {
1324 return -ENOTSUP;
1325 }
1326
1327 auto [sourceRoot, subpath] = registry().rootAndSubpathFor(sourceDir);
1328 if (sourceRoot.empty()) {
1329 return -EINVAL;
1330 }
1331 if (subpath.empty()) {
1332 LOG(WARNING) << "[incfs] Binding the root mount '" << sourceRoot << "' is not allowed";
1333 return -EINVAL;
1334 }
1335
1336 if (auto err = isValidMountTarget(targetDir); err != 0) {
1337 return err;
1338 }
1339
1340 if (::mount(sourceDir, targetDir, nullptr, MS_BIND, nullptr)) {
1341 PLOG(ERROR) << "[incfs] Failed to bind mount '" << sourceDir << "' to '" << targetDir
1342 << '\'';
1343 return -errno;
1344 }
1345 return 0;
1346 }
1347
IncFs_Unmount(const char * dir)1348 IncFsErrorCode IncFs_Unmount(const char* dir) {
1349 if (!enabled()) {
1350 return -ENOTSUP;
1351 }
1352
1353 errno = 0;
1354 if (::umount2(dir, MNT_FORCE) == 0 || errno == EINVAL || errno == ENOENT) {
1355 // EINVAL - not a mount point, ENOENT - doesn't exist at all
1356 return -errno;
1357 }
1358 PLOG(WARNING) << __func__ << ": umount(force) failed, detaching '" << dir << '\'';
1359 errno = 0;
1360 if (!::umount2(dir, MNT_DETACH)) {
1361 return 0;
1362 }
1363 PLOG(WARNING) << __func__ << ": umount(detach) returned non-zero for '" << dir << '\'';
1364 return 0;
1365 }
1366
IncFs_IsIncFsFd(int fd)1367 bool IncFs_IsIncFsFd(int fd) {
1368 return isIncFsFdImpl(fd);
1369 }
1370
IncFs_IsIncFsPath(const char * path)1371 bool IncFs_IsIncFsPath(const char* path) {
1372 return isIncFsPathImpl(path);
1373 }
1374
IncFs_GetFilledRanges(int fd,IncFsSpan outBuffer,IncFsFilledRanges * filledRanges)1375 IncFsErrorCode IncFs_GetFilledRanges(int fd, IncFsSpan outBuffer, IncFsFilledRanges* filledRanges) {
1376 return IncFs_GetFilledRangesStartingFrom(fd, 0, outBuffer, filledRanges);
1377 }
1378
IncFs_GetFilledRangesStartingFrom(int fd,int startBlockIndex,IncFsSpan outBuffer,IncFsFilledRanges * filledRanges)1379 IncFsErrorCode IncFs_GetFilledRangesStartingFrom(int fd, int startBlockIndex, IncFsSpan outBuffer,
1380 IncFsFilledRanges* filledRanges) {
1381 if (fd < 0) {
1382 return -EBADF;
1383 }
1384 if (startBlockIndex < 0) {
1385 return -EINVAL;
1386 }
1387 if (!outBuffer.data && outBuffer.size > 0) {
1388 return -EINVAL;
1389 }
1390 if (!filledRanges) {
1391 return -EINVAL;
1392 }
1393 // Use this to optimize the incfs call and have the same buffer for both the incfs and the
1394 // public structs.
1395 static_assert(sizeof(IncFsBlockRange) == sizeof(incfs_filled_range));
1396
1397 *filledRanges = {};
1398
1399 auto outStart = (IncFsBlockRange*)outBuffer.data;
1400 auto outEnd = outStart + outBuffer.size / sizeof(*outStart);
1401
1402 auto outPtr = outStart;
1403 int error = 0;
1404 int dataBlocks;
1405 incfs_get_filled_blocks_args args = {};
1406 for (;;) {
1407 auto start = args.index_out ? args.index_out : startBlockIndex;
1408 args = incfs_get_filled_blocks_args{
1409 .range_buffer = (uint64_t)(uintptr_t)outPtr,
1410 .range_buffer_size = uint32_t((outEnd - outPtr) * sizeof(*outPtr)),
1411 .start_index = start,
1412 };
1413 errno = 0;
1414 auto res = ::ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &args);
1415 error = errno;
1416 if (res && error != EINTR && error != ERANGE) {
1417 return -error;
1418 }
1419
1420 dataBlocks = args.data_blocks_out;
1421 outPtr += args.range_buffer_size_out / sizeof(incfs_filled_range);
1422 if (!res || error == ERANGE) {
1423 break;
1424 }
1425 // in case of EINTR we want to continue calling the function
1426 }
1427
1428 if (outPtr > outEnd) {
1429 outPtr = outEnd;
1430 error = ERANGE;
1431 }
1432
1433 filledRanges->endIndex = args.index_out;
1434 auto hashStartPtr = outPtr;
1435 if (outPtr != outStart) {
1436 // figure out the ranges for data block and hash blocks in the output
1437 for (; hashStartPtr != outStart; --hashStartPtr) {
1438 if ((hashStartPtr - 1)->begin < dataBlocks) {
1439 break;
1440 }
1441 }
1442 auto lastDataPtr = hashStartPtr - 1;
1443 // here we go, this is the first block that's before or at the hashes
1444 if (lastDataPtr->end <= dataBlocks) {
1445 ; // we're good, the boundary is between the ranges - |hashStartPtr| is correct
1446 } else {
1447 // the hard part: split the |lastDataPtr| range into the data and the hash pieces
1448 if (outPtr == outEnd) {
1449 // the buffer turned out to be too small, even though it actually wasn't
1450 error = ERANGE;
1451 if (hashStartPtr == outEnd) {
1452 // this is even worse: there's no room to put even a single hash block into.
1453 filledRanges->endIndex = lastDataPtr->end = dataBlocks;
1454 } else {
1455 std::copy_backward(lastDataPtr, outPtr - 1, outPtr);
1456 lastDataPtr->end = hashStartPtr->begin = dataBlocks;
1457 filledRanges->endIndex = (outPtr - 1)->end;
1458 }
1459 } else {
1460 std::copy_backward(lastDataPtr, outPtr, outPtr + 1);
1461 lastDataPtr->end = hashStartPtr->begin = dataBlocks;
1462 ++outPtr;
1463 }
1464 }
1465 // now fix the indices of all hash blocks - no one should know they're simply past the
1466 // regular data blocks in the file!
1467 for (auto ptr = hashStartPtr; ptr != outPtr; ++ptr) {
1468 ptr->begin -= dataBlocks;
1469 ptr->end -= dataBlocks;
1470 }
1471 }
1472
1473 filledRanges->dataRanges = outStart;
1474 filledRanges->dataRangesCount = hashStartPtr - outStart;
1475 filledRanges->hashRanges = hashStartPtr;
1476 filledRanges->hashRangesCount = outPtr - hashStartPtr;
1477
1478 return -error;
1479 }
1480
isFullyLoadedV2(std::string_view root,IncFsFileId id)1481 static IncFsErrorCode isFullyLoadedV2(std::string_view root, IncFsFileId id) {
1482 if (::access(path::join(root, INCFS_INCOMPLETE_NAME, toStringImpl(id)).c_str(), F_OK)) {
1483 if (errno == ENOENT) {
1484 return 0; // no such incomplete file -> it's fully loaded.
1485 }
1486 return -errno;
1487 }
1488 return -ENODATA;
1489 }
1490
isFullyLoadedSlow(int fd)1491 static IncFsErrorCode isFullyLoadedSlow(int fd) {
1492 char buffer[2 * sizeof(IncFsBlockRange)];
1493 IncFsFilledRanges ranges;
1494 auto res = IncFs_GetFilledRanges(fd, IncFsSpan{.data = buffer, .size = std::size(buffer)},
1495 &ranges);
1496 if (res == -ERANGE) {
1497 // need room for more than two ranges - definitely not fully loaded
1498 return -ENODATA;
1499 }
1500 if (res != 0) {
1501 return res;
1502 }
1503 // empty file
1504 if (ranges.endIndex == 0) {
1505 return 0;
1506 }
1507 // file with no hash tree
1508 if (ranges.dataRangesCount == 1 && ranges.hashRangesCount == 0) {
1509 return (ranges.dataRanges[0].begin == 0 && ranges.dataRanges[0].end == ranges.endIndex)
1510 ? 0
1511 : -ENODATA;
1512 }
1513 // file with a hash tree
1514 if (ranges.dataRangesCount == 1 && ranges.hashRangesCount == 1) {
1515 // calculate the expected data size from the size of the hash range and |endIndex|, which is
1516 // the total number of blocks in the file, both data and hash blocks together.
1517 if (ranges.hashRanges[0].begin != 0) {
1518 return -ENODATA;
1519 }
1520 const auto expectedDataBlocks =
1521 ranges.endIndex - (ranges.hashRanges[0].end - ranges.hashRanges[0].begin);
1522 return (ranges.dataRanges[0].begin == 0 && ranges.dataRanges[0].end == expectedDataBlocks)
1523 ? 0
1524 : -ENODATA;
1525 }
1526 return -ENODATA;
1527 }
1528
IncFs_IsFullyLoaded(int fd)1529 IncFsErrorCode IncFs_IsFullyLoaded(int fd) {
1530 if (features() & Features::v2) {
1531 const auto fdPath = path::fromFd(fd);
1532 if (fdPath.empty()) {
1533 return errno ? -errno : -EINVAL;
1534 }
1535 const auto id = getId(::fgetxattr, fd);
1536 if (id == kIncFsInvalidFileId) {
1537 return -errno;
1538 }
1539 return isFullyLoadedV2(registry().rootFor(fdPath), id);
1540 }
1541 return isFullyLoadedSlow(fd);
1542 }
IncFs_IsFullyLoadedByPath(const IncFsControl * control,const char * path)1543 IncFsErrorCode IncFs_IsFullyLoadedByPath(const IncFsControl* control, const char* path) {
1544 if (!control || !path) {
1545 return -EINVAL;
1546 }
1547 const auto root = rootForCmd(control->cmd);
1548 if (root.empty()) {
1549 return -EINVAL;
1550 }
1551 const auto pathRoot = registry().rootFor(path);
1552 if (pathRoot != root) {
1553 return -EINVAL;
1554 }
1555 if (features() & Features::v2) {
1556 const auto id = getId(::getxattr, path);
1557 if (id == kIncFsInvalidFileId) {
1558 return -errno;
1559 }
1560 return isFullyLoadedV2(root, id);
1561 }
1562 return isFullyLoadedSlow(openForSpecialOps(control->cmd, makeCommandPath(root, path).c_str()));
1563 }
IncFs_IsFullyLoadedById(const IncFsControl * control,IncFsFileId fileId)1564 IncFsErrorCode IncFs_IsFullyLoadedById(const IncFsControl* control, IncFsFileId fileId) {
1565 if (!control) {
1566 return -EINVAL;
1567 }
1568 const auto root = rootForCmd(control->cmd);
1569 if (root.empty()) {
1570 return -EINVAL;
1571 }
1572 if (features() & Features::v2) {
1573 return isFullyLoadedV2(root, fileId);
1574 }
1575 return isFullyLoadedSlow(
1576 openForSpecialOps(control->cmd,
1577 makeCommandPath(root, indexPath(root, fileId)).c_str()));
1578 }
1579
isEverythingLoadedV2(const IncFsControl * control)1580 static IncFsErrorCode isEverythingLoadedV2(const IncFsControl* control) {
1581 const auto root = rootForCmd(control->cmd);
1582 if (root.empty()) {
1583 return -EINVAL;
1584 }
1585 auto res = forEachFileIn(path::join(root, INCFS_INCOMPLETE_NAME), [](auto) { return false; });
1586 return res < 0 ? res : res > 0 ? -ENODATA : 0;
1587 }
1588
isEverythingLoadedSlow(const IncFsControl * control)1589 static IncFsErrorCode isEverythingLoadedSlow(const IncFsControl* control) {
1590 const auto root = rootForCmd(control->cmd);
1591 if (root.empty()) {
1592 return -EINVAL;
1593 }
1594 // No special API for this version of the driver, need to recurse and check each file
1595 // separately. Can at least speed it up by iterating over the .index/ dir and not dealing with
1596 // the directory tree.
1597 const auto indexPath = path::join(root, INCFS_INDEX_NAME);
1598 const auto dir = path::openDir(indexPath.c_str());
1599 if (!dir) {
1600 return -EINVAL;
1601 }
1602 while (const auto entry = ::readdir(dir.get())) {
1603 if (entry->d_type != DT_REG) {
1604 continue;
1605 }
1606 const auto name = path::join(indexPath, entry->d_name);
1607 auto fd =
1608 ab::unique_fd(openForSpecialOps(control->cmd, makeCommandPath(root, name).c_str()));
1609 if (fd.get() < 0) {
1610 PLOG(WARNING) << __func__ << "(): can't open " << entry->d_name << " for special ops";
1611 return fd.release();
1612 }
1613 const auto checkFullyLoaded = IncFs_IsFullyLoaded(fd.get());
1614 if (checkFullyLoaded == 0 || checkFullyLoaded == -EOPNOTSUPP ||
1615 checkFullyLoaded == -ENOTSUP || checkFullyLoaded == -ENOENT) {
1616 // special kinds of files may return an error here, but it still means
1617 // _this_ file is OK - you simply need to check the rest. E.g. can't query
1618 // a mapped file, instead need to check its parent.
1619 continue;
1620 }
1621 return checkFullyLoaded;
1622 }
1623 return 0;
1624 }
1625
IncFs_IsEverythingFullyLoaded(const IncFsControl * control)1626 IncFsErrorCode IncFs_IsEverythingFullyLoaded(const IncFsControl* control) {
1627 if (!control) {
1628 return -EINVAL;
1629 }
1630 if (features() & Features::v2) {
1631 return isEverythingLoadedV2(control);
1632 }
1633 return isEverythingLoadedSlow(control);
1634 }
1635
IncFs_SetUidReadTimeouts(const IncFsControl * control,const IncFsUidReadTimeouts timeouts[],size_t count)1636 IncFsErrorCode IncFs_SetUidReadTimeouts(const IncFsControl* control,
1637 const IncFsUidReadTimeouts timeouts[], size_t count) {
1638 if (!control) {
1639 return -EINVAL;
1640 }
1641 if (!(features() & Features::v2)) {
1642 return -ENOTSUP;
1643 }
1644
1645 std::vector<incfs_per_uid_read_timeouts> argTimeouts(count);
1646 for (size_t i = 0; i != count; ++i) {
1647 argTimeouts[i] = incfs_per_uid_read_timeouts{
1648 .uid = (uint32_t)timeouts[i].uid,
1649 .min_time_us = timeouts[i].minTimeUs,
1650 .min_pending_time_us = timeouts[i].minPendingTimeUs,
1651 .max_pending_time_us = timeouts[i].maxPendingTimeUs,
1652 };
1653 }
1654 incfs_set_read_timeouts_args args = {.timeouts_array = (uint64_t)(uintptr_t)argTimeouts.data(),
1655 .timeouts_array_size = uint32_t(
1656 argTimeouts.size() * sizeof(*argTimeouts.data()))};
1657 if (::ioctl(control->cmd, INCFS_IOC_SET_READ_TIMEOUTS, &args)) {
1658 PLOG(WARNING) << "[incfs] setUidReadTimeouts failed";
1659 return -errno;
1660 }
1661 return 0;
1662 }
1663
IncFs_GetUidReadTimeouts(const IncFsControl * control,IncFsUidReadTimeouts timeouts[],size_t * bufferSize)1664 IncFsErrorCode IncFs_GetUidReadTimeouts(const IncFsControl* control,
1665 IncFsUidReadTimeouts timeouts[], size_t* bufferSize) {
1666 if (!control || !bufferSize) {
1667 return -EINVAL;
1668 }
1669 if (!(features() & Features::v2)) {
1670 return -ENOTSUP;
1671 }
1672
1673 std::vector<incfs_per_uid_read_timeouts> argTimeouts(*bufferSize);
1674 incfs_get_read_timeouts_args args = {.timeouts_array = (uint64_t)(uintptr_t)argTimeouts.data(),
1675 .timeouts_array_size = uint32_t(
1676 argTimeouts.size() * sizeof(*argTimeouts.data())),
1677 .timeouts_array_size_out = args.timeouts_array_size};
1678 if (::ioctl(control->cmd, INCFS_IOC_GET_READ_TIMEOUTS, &args)) {
1679 if (errno == E2BIG) {
1680 *bufferSize = args.timeouts_array_size_out / sizeof(*argTimeouts.data());
1681 }
1682 return -errno;
1683 }
1684
1685 *bufferSize = args.timeouts_array_size_out / sizeof(*argTimeouts.data());
1686 for (size_t i = 0; i != *bufferSize; ++i) {
1687 timeouts[i].uid = argTimeouts[i].uid;
1688 timeouts[i].minTimeUs = argTimeouts[i].min_time_us;
1689 timeouts[i].minPendingTimeUs = argTimeouts[i].min_pending_time_us;
1690 timeouts[i].maxPendingTimeUs = argTimeouts[i].max_pending_time_us;
1691 }
1692 return 0;
1693 }
1694
getFileBlockCount(int fd,IncFsBlockCounts * blockCount)1695 static IncFsErrorCode getFileBlockCount(int fd, IncFsBlockCounts* blockCount) {
1696 incfs_get_block_count_args args = {};
1697 auto res = ::ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &args);
1698 if (res < 0) {
1699 return -errno;
1700 }
1701 *blockCount = IncFsBlockCounts{
1702 .totalDataBlocks = args.total_data_blocks_out,
1703 .filledDataBlocks = args.filled_data_blocks_out,
1704 .totalHashBlocks = args.total_hash_blocks_out,
1705 .filledHashBlocks = args.filled_hash_blocks_out,
1706 };
1707 return 0;
1708 }
1709
IncFs_GetFileBlockCountById(const IncFsControl * control,IncFsFileId id,IncFsBlockCounts * blockCount)1710 IncFsErrorCode IncFs_GetFileBlockCountById(const IncFsControl* control, IncFsFileId id,
1711 IncFsBlockCounts* blockCount) {
1712 if (!control) {
1713 return -EINVAL;
1714 }
1715 if (!(features() & Features::v2)) {
1716 return -ENOTSUP;
1717 }
1718 const auto root = rootForCmd(control->cmd);
1719 if (root.empty()) {
1720 return -EINVAL;
1721 }
1722 auto name = indexPath(root, id);
1723 auto fd = openRaw(name);
1724 if (fd < 0) {
1725 return fd.get();
1726 }
1727 return getFileBlockCount(fd, blockCount);
1728 }
1729
IncFs_GetFileBlockCountByPath(const IncFsControl * control,const char * path,IncFsBlockCounts * blockCount)1730 IncFsErrorCode IncFs_GetFileBlockCountByPath(const IncFsControl* control, const char* path,
1731 IncFsBlockCounts* blockCount) {
1732 if (!control) {
1733 return -EINVAL;
1734 }
1735 if (!(features() & Features::v2)) {
1736 return -ENOTSUP;
1737 }
1738 const auto pathRoot = registry().rootFor(path);
1739 const auto root = rootForCmd(control->cmd);
1740 if (root.empty() || root != pathRoot) {
1741 return -EINVAL;
1742 }
1743 auto fd = openRaw(path);
1744 if (fd < 0) {
1745 return fd.get();
1746 }
1747 return getFileBlockCount(fd, blockCount);
1748 }
1749
IncFs_ListIncompleteFiles(const IncFsControl * control,IncFsFileId ids[],size_t * bufferSize)1750 IncFsErrorCode IncFs_ListIncompleteFiles(const IncFsControl* control, IncFsFileId ids[],
1751 size_t* bufferSize) {
1752 if (!control || !bufferSize) {
1753 return -EINVAL;
1754 }
1755 if (!(features() & Features::v2)) {
1756 return -ENOTSUP;
1757 }
1758 const auto root = rootForCmd(control->cmd);
1759 if (root.empty()) {
1760 return -EINVAL;
1761 }
1762 size_t index = 0;
1763 int error = 0;
1764 const auto res = forEachFileIn(path::join(root, INCFS_INCOMPLETE_NAME), [&](const char* name) {
1765 if (index >= *bufferSize) {
1766 error = -E2BIG;
1767 } else {
1768 ids[index] = IncFs_FileIdFromString(name);
1769 }
1770 ++index;
1771 return true;
1772 });
1773 if (res < 0) {
1774 return res;
1775 }
1776 *bufferSize = index;
1777 return error ? error : 0;
1778 }
1779
IncFs_ForEachFile(const IncFsControl * control,void * context,FileCallback cb)1780 IncFsErrorCode IncFs_ForEachFile(const IncFsControl* control, void* context, FileCallback cb) {
1781 if (!control || !cb) {
1782 return -EINVAL;
1783 }
1784 const auto root = rootForCmd(control->cmd);
1785 if (root.empty()) {
1786 return -EINVAL;
1787 }
1788 return forEachFileIn(path::join(root, INCFS_INDEX_NAME), [&](const char* name) {
1789 return cb(context, control, IncFs_FileIdFromString(name));
1790 });
1791 }
1792
IncFs_ForEachIncompleteFile(const IncFsControl * control,void * context,FileCallback cb)1793 IncFsErrorCode IncFs_ForEachIncompleteFile(const IncFsControl* control, void* context,
1794 FileCallback cb) {
1795 if (!control || !cb) {
1796 return -EINVAL;
1797 }
1798 if (!(features() & Features::v2)) {
1799 return -ENOTSUP;
1800 }
1801 const auto root = rootForCmd(control->cmd);
1802 if (root.empty()) {
1803 return -EINVAL;
1804 }
1805 return forEachFileIn(path::join(root, INCFS_INCOMPLETE_NAME), [&](const char* name) {
1806 return cb(context, control, IncFs_FileIdFromString(name));
1807 });
1808 }
1809
IncFs_WaitForLoadingComplete(const IncFsControl * control,int32_t timeoutMs)1810 IncFsErrorCode IncFs_WaitForLoadingComplete(const IncFsControl* control, int32_t timeoutMs) {
1811 if (!control) {
1812 return -EINVAL;
1813 }
1814 if (!(features() & Features::v2)) {
1815 return -ENOTSUP;
1816 }
1817
1818 using namespace std::chrono;
1819 auto hrTimeout = steady_clock::duration(milliseconds(timeoutMs));
1820
1821 const auto root = rootForCmd(control->cmd);
1822 if (root.empty()) {
1823 return -EINVAL;
1824 }
1825
1826 ab::unique_fd fd(inotify_init1(IN_NONBLOCK | IN_CLOEXEC));
1827 if (!fd.ok()) {
1828 return -EFAULT;
1829 }
1830
1831 // first create all the watches, and only then list existing files to prevent races
1832 auto dirPath = path::join(root, INCFS_INCOMPLETE_NAME);
1833 int watchFd = inotify_add_watch(fd.get(), dirPath.c_str(), IN_DELETE);
1834 if (watchFd < 0) {
1835 return -errno;
1836 }
1837
1838 size_t count = 0;
1839 auto res = IncFs_ListIncompleteFiles(control, nullptr, &count);
1840 if (!res) {
1841 return 0;
1842 }
1843 if (res != -E2BIG) {
1844 return res;
1845 }
1846
1847 while (hrTimeout > hrTimeout.zero()) {
1848 const auto startTs = steady_clock::now();
1849
1850 pollfd pfd = {fd.get(), POLLIN, 0};
1851 const auto res = ::poll(&pfd, 1, duration_cast<milliseconds>(hrTimeout).count());
1852 if (res == 0) {
1853 return -ETIMEDOUT;
1854 }
1855 if (res < 0) {
1856 const auto error = errno;
1857 if (error != EINTR) {
1858 PLOG(ERROR) << "poll() failed";
1859 return -error;
1860 }
1861 } else {
1862 // empty the inotify fd first to not miss any new deletions,
1863 // then check if the directory is empty.
1864 char buffer[sizeof(inotify_event) + NAME_MAX + 1];
1865 for (;;) {
1866 auto err = TEMP_FAILURE_RETRY(::read(fd.get(), buffer, sizeof(buffer)));
1867 if (err < 0) {
1868 if (errno == EAGAIN) { // no new events
1869 break;
1870 }
1871 return -errno;
1872 }
1873 }
1874
1875 size_t count = 0;
1876 auto res = IncFs_ListIncompleteFiles(control, nullptr, &count);
1877 if (!res) {
1878 return 0;
1879 }
1880 if (res != -E2BIG) {
1881 return res;
1882 }
1883 }
1884 hrTimeout -= steady_clock::now() - startTs;
1885 }
1886
1887 return -ETIMEDOUT;
1888 }
1889
IncFs_WaitForFsWrittenBlocksChange(const IncFsControl * control,int32_t timeoutMs,IncFsSize * count)1890 IncFsErrorCode IncFs_WaitForFsWrittenBlocksChange(const IncFsControl* control, int32_t timeoutMs,
1891 IncFsSize* count) {
1892 if (!control || !count) {
1893 return -EINVAL;
1894 }
1895 if (!(features() & Features::v2)) {
1896 return -ENOTSUP;
1897 }
1898
1899 using namespace std::chrono;
1900 auto hrTimeout = steady_clock::duration(milliseconds(timeoutMs));
1901
1902 while (hrTimeout > hrTimeout.zero()) {
1903 const auto startTs = steady_clock::now();
1904
1905 pollfd pfd = {control->blocksWritten, POLLIN, 0};
1906 const auto res = ::poll(&pfd, 1, duration_cast<milliseconds>(hrTimeout).count());
1907 if (res > 0) {
1908 break;
1909 }
1910 if (res == 0) {
1911 return -ETIMEDOUT;
1912 }
1913 const auto error = errno;
1914 if (error != EINTR) {
1915 PLOG(ERROR) << "poll() failed";
1916 return -error;
1917 }
1918 hrTimeout -= steady_clock::now() - startTs;
1919 }
1920
1921 char str[32];
1922 auto size = ::read(control->blocksWritten, str, sizeof(str));
1923 if (size < 0) {
1924 const auto error = errno;
1925 PLOG(ERROR) << "read() failed";
1926 return -error;
1927 }
1928 const auto res = std::from_chars(str, str + size, *count);
1929 if (res.ec != std::errc{}) {
1930 return res.ec == std::errc::invalid_argument ? -EINVAL : -ERANGE;
1931 }
1932
1933 return 0;
1934 }
1935
reserveSpace(const char * backingPath,IncFsSize size)1936 static IncFsErrorCode reserveSpace(const char* backingPath, IncFsSize size) {
1937 auto fd = ab::unique_fd(::open(backingPath, O_WRONLY | O_CLOEXEC));
1938 if (fd < 0) {
1939 return -errno;
1940 }
1941 struct stat st = {};
1942 if (::fstat(fd.get(), &st)) {
1943 return -errno;
1944 }
1945 if (size == kIncFsTrimReservedSpace) {
1946 if (::ftruncate(fd.get(), st.st_size)) {
1947 return -errno;
1948 }
1949 } else {
1950 // Add 1.5% of the size for the hash tree and the blockmap, and some more blocks
1951 // for fixed overhead.
1952 // hash tree is ~33 bytes / page, and blockmap is 10 bytes / page
1953 // no need to round to a page size as filesystems already do that.
1954 const auto backingSize = IncFsSize(size * 1.015) + INCFS_DATA_FILE_BLOCK_SIZE * 4;
1955 if (backingSize < st.st_size) {
1956 return -EPERM;
1957 }
1958 if (::fallocate(fd.get(), FALLOC_FL_KEEP_SIZE, 0, backingSize)) {
1959 return -errno;
1960 }
1961 }
1962 return 0;
1963 }
1964
IncFs_ReserveSpaceByPath(const IncFsControl * control,const char * path,IncFsSize size)1965 IncFsErrorCode IncFs_ReserveSpaceByPath(const IncFsControl* control, const char* path,
1966 IncFsSize size) {
1967 if (!control || (size != kIncFsTrimReservedSpace && size < 0)) {
1968 return -EINVAL;
1969 }
1970 const auto [pathRoot, backingRoot, subpath] = registry().detailsFor(path);
1971 const auto root = rootForCmd(control->cmd);
1972 if (root.empty() || root != pathRoot) {
1973 return -EINVAL;
1974 }
1975 return reserveSpace(path::join(backingRoot, subpath).c_str(), size);
1976 }
1977
IncFs_ReserveSpaceById(const IncFsControl * control,IncFsFileId id,IncFsSize size)1978 IncFsErrorCode IncFs_ReserveSpaceById(const IncFsControl* control, IncFsFileId id, IncFsSize size) {
1979 if (!control || (size != kIncFsTrimReservedSpace && size < 0)) {
1980 return -EINVAL;
1981 }
1982 const auto root = rootForCmd(control->cmd);
1983 if (root.empty()) {
1984 return -EINVAL;
1985 }
1986 auto path = indexPath(root, id);
1987 const auto [pathRoot, backingRoot, subpath] = registry().detailsFor(path);
1988 if (root != pathRoot) {
1989 return -EINVAL;
1990 }
1991 return reserveSpace(path::join(backingRoot, subpath).c_str(), size);
1992 }
1993
1994 template <class IntType>
readIntFromFile(std::string_view rootDir,std::string_view subPath,IntType & result)1995 static int readIntFromFile(std::string_view rootDir, std::string_view subPath, IntType& result) {
1996 std::string content;
1997 if (!ab::ReadFileToString(path::join(rootDir, subPath), &content)) {
1998 PLOG(ERROR) << "IncFs_GetMetrics: failed to read file: " << rootDir << "/" << subPath;
1999 return -errno;
2000 }
2001 const auto res = std::from_chars(content.data(), content.data() + content.size(), result);
2002 if (res.ec != std::errc()) {
2003 return -static_cast<int>(res.ec);
2004 }
2005 return 0;
2006 }
2007
IncFs_GetMetrics(const char * sysfsName,IncFsMetrics * metrics)2008 IncFsErrorCode IncFs_GetMetrics(const char* sysfsName, IncFsMetrics* metrics) {
2009 if (!sysfsName || !*sysfsName) {
2010 return -EINVAL;
2011 }
2012
2013 const auto kSysfsMetricsDir =
2014 ab::StringPrintf("/sys/fs/%s/instances/%s", INCFS_NAME, sysfsName);
2015
2016 int err;
2017 if (err = readIntFromFile(kSysfsMetricsDir, "reads_delayed_min", metrics->readsDelayedMin);
2018 err != 0) {
2019 return err;
2020 }
2021 if (err = readIntFromFile(kSysfsMetricsDir, "reads_delayed_min_us", metrics->readsDelayedMinUs);
2022 err != 0) {
2023 return err;
2024 }
2025 if (err = readIntFromFile(kSysfsMetricsDir, "reads_delayed_pending",
2026 metrics->readsDelayedPending);
2027 err != 0) {
2028 return err;
2029 }
2030 if (err = readIntFromFile(kSysfsMetricsDir, "reads_delayed_pending_us",
2031 metrics->readsDelayedPendingUs);
2032 err != 0) {
2033 return err;
2034 }
2035 if (err = readIntFromFile(kSysfsMetricsDir, "reads_failed_hash_verification",
2036 metrics->readsFailedHashVerification);
2037 err != 0) {
2038 return err;
2039 }
2040 if (err = readIntFromFile(kSysfsMetricsDir, "reads_failed_other", metrics->readsFailedOther);
2041 err != 0) {
2042 return err;
2043 }
2044 if (err = readIntFromFile(kSysfsMetricsDir, "reads_failed_timed_out",
2045 metrics->readsFailedTimedOut);
2046 err != 0) {
2047 return err;
2048 }
2049 return 0;
2050 }
2051
IncFs_GetLastReadError(const IncFsControl * control,IncFsLastReadError * lastReadError)2052 IncFsErrorCode IncFs_GetLastReadError(const IncFsControl* control,
2053 IncFsLastReadError* lastReadError) {
2054 if (!control) {
2055 return -EINVAL;
2056 }
2057 if (!(features() & Features::v2)) {
2058 return -ENOTSUP;
2059 }
2060 incfs_get_last_read_error_args args = {};
2061 auto res = ::ioctl(control->cmd, INCFS_IOC_GET_LAST_READ_ERROR, &args);
2062 if (res < 0) {
2063 PLOG(ERROR) << "[incfs] IncFs_GetLastReadError failed.";
2064 return -errno;
2065 }
2066 *lastReadError = IncFsLastReadError{
2067 .timestampUs = args.time_us_out,
2068 .block = static_cast<IncFsBlockIndex>(args.page_out),
2069 .errorNo = args.errno_out,
2070 .uid = static_cast<IncFsUid>(args.uid_out),
2071 };
2072 static_assert(sizeof(args.file_id_out.bytes) == sizeof(lastReadError->id.data));
2073 memcpy(lastReadError->id.data, args.file_id_out.bytes, sizeof(args.file_id_out.bytes));
2074 return 0;
2075 }
2076
defaultMountRegistry()2077 MountRegistry& android::incfs::defaultMountRegistry() {
2078 return registry();
2079 }
2080