1 /*
2 * Copyright (C) 2024 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "host/commands/process_sandboxer/sandbox_manager.h"
17
18 #include <poll.h>
19 #include <signal.h>
20 #include <stdlib.h>
21 #include <sys/eventfd.h>
22 #include <sys/resource.h>
23 #include <sys/signalfd.h>
24 #include <sys/socket.h>
25 #include <sys/syscall.h>
26 #include <sys/wait.h>
27 #include <unistd.h>
28
29 #include <algorithm>
30 #include <cerrno>
31 #include <cstddef>
32 #include <cstdint>
33 #include <functional>
34 #include <memory>
35 #include <optional>
36 #include <sstream>
37 #include <string>
38 #include <string_view>
39 #include <thread>
40 #include <utility>
41 #include <vector>
42
43 #include <absl/functional/bind_front.h>
44 #include <absl/log/log.h>
45 #include <absl/log/vlog_is_on.h>
46 #include <absl/memory/memory.h>
47 #include <absl/random/bit_gen_ref.h>
48 #include <absl/random/uniform_int_distribution.h>
49 #include <absl/status/status.h>
50 #include <absl/status/statusor.h>
51 #include <absl/strings/numbers.h>
52 #include <absl/strings/str_cat.h>
53 #include <absl/strings/str_format.h>
54 #include <absl/time/time.h>
55 #include <absl/types/span.h>
56 #pragma clang diagnostic push
57 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
58 #pragma clang diagnostic ignored "-Wunused-parameter"
59 #include <sandboxed_api/sandbox2/executor.h>
60 #include <sandboxed_api/sandbox2/policy.h>
61 #include <sandboxed_api/sandbox2/sandbox2.h>
62 #include <sandboxed_api/util/fileops.h>
63 #include <sandboxed_api/util/path.h>
64 #pragma clang diagnostic pop
65
66 #include "host/commands/process_sandboxer/credentialed_unix_server.h"
67 #include "host/commands/process_sandboxer/pidfd.h"
68 #include "host/commands/process_sandboxer/policies.h"
69 #include "host/commands/process_sandboxer/poll_callback.h"
70 #include "host/commands/process_sandboxer/proxy_common.h"
71 #include "host/commands/process_sandboxer/signal_fd.h"
72
73 namespace cuttlefish::process_sandboxer {
74
75 using sandbox2::Executor;
76 using sandbox2::Policy;
77 using sandbox2::Sandbox2;
78 using sapi::file::CleanPath;
79 using sapi::file::JoinPath;
80 using sapi::file_util::fileops::FDCloser;
81
82 namespace {
83
ServerSocketOutsidePath(std::string_view runtime_dir)84 std::string ServerSocketOutsidePath(std::string_view runtime_dir) {
85 return JoinPath(runtime_dir, "/", "server.sock");
86 }
87
88 } // namespace
89
90 class SandboxManager::ProcessNoSandbox : public SandboxManager::ManagedProcess {
91 public:
ProcessNoSandbox(int client_fd,PidFd pid_fd)92 ProcessNoSandbox(int client_fd, PidFd pid_fd)
93 : client_fd_(client_fd), pid_fd_(std::move(pid_fd)) {}
~ProcessNoSandbox()94 ~ProcessNoSandbox() {
95 auto halt = pid_fd_.HaltHierarchy();
96 if (!halt.ok()) {
97 LOG(ERROR) << "Failed to halt children: " << halt.ToString();
98 }
99 }
100
ClientFd() const101 std::optional<int> ClientFd() const override { return client_fd_; }
PollFd() const102 int PollFd() const override { return pid_fd_.Get(); }
103
ExitCode()104 absl::StatusOr<uintptr_t> ExitCode() override {
105 siginfo_t infop;
106 idtype_t id_type = (idtype_t)3; // P_PIDFD
107 if (waitid(id_type, pid_fd_.Get(), &infop, WEXITED | WNOWAIT) < 0) {
108 return absl::ErrnoToStatus(errno, "`waitid` failed");
109 }
110 switch (infop.si_code) {
111 case CLD_EXITED:
112 return infop.si_status;
113 case CLD_DUMPED:
114 case CLD_KILLED:
115 LOG(ERROR) << "Child killed by signal " << infop.si_code;
116 return 255;
117 default:
118 LOG(ERROR) << "Unexpected si_code: " << infop.si_code;
119 return 255;
120 }
121 }
122
123 private:
124 int client_fd_;
125 PidFd pid_fd_;
126 };
127
128 class SandboxManager::SandboxedProcess : public SandboxManager::ManagedProcess {
129 public:
SandboxedProcess(std::optional<int> client_fd,FDCloser event_fd,std::unique_ptr<Sandbox2> sandbox)130 SandboxedProcess(std::optional<int> client_fd, FDCloser event_fd,
131 std::unique_ptr<Sandbox2> sandbox)
132 : client_fd_(client_fd),
133 event_fd_(std::move(event_fd)),
134 sandbox_(std::move(sandbox)) {
135 waiter_thread_ = std::thread([this]() { WaitForExit(); });
136 }
~SandboxedProcess()137 ~SandboxedProcess() override {
138 sandbox_->Kill();
139 waiter_thread_.join();
140 auto res = sandbox_->AwaitResult().ToStatus();
141 if (!res.ok()) {
142 LOG(ERROR) << "Issue in closing sandbox: '" << res.ToString() << "'";
143 }
144 }
145
ClientFd() const146 std::optional<int> ClientFd() const override { return client_fd_; }
PollFd() const147 int PollFd() const override { return event_fd_.get(); }
148
ExitCode()149 absl::StatusOr<uintptr_t> ExitCode() override {
150 return sandbox_->AwaitResult().reason_code();
151 }
152
153 private:
WaitForExit()154 void WaitForExit() {
155 sandbox_->AwaitResult().IgnoreResult();
156 uint64_t buf = 1;
157 if (write(event_fd_.get(), &buf, sizeof(buf)) < 0) {
158 PLOG(ERROR) << "Failed to write to eventfd";
159 }
160 }
161
162 std::optional<int> client_fd_;
163 FDCloser event_fd_;
164 std::thread waiter_thread_;
165 std::unique_ptr<Sandbox2> sandbox_;
166 };
167
RandomString(absl::BitGenRef gen,std::size_t size)168 std::string RandomString(absl::BitGenRef gen, std::size_t size) {
169 std::stringstream output;
170 absl::uniform_int_distribution<char> distribution;
171 for (std::size_t i = 0; i < size; i++) {
172 output << distribution(gen);
173 }
174 return output.str();
175 }
176
177 class SandboxManager::SocketClient {
178 public:
SocketClient(SandboxManager & manager,FDCloser client_fd)179 SocketClient(SandboxManager& manager, FDCloser client_fd)
180 : manager_(manager), client_fd_(std::move(client_fd)) {}
181 SocketClient(SocketClient&) = delete;
182
ClientFd() const183 int ClientFd() const { return client_fd_.get(); }
184
HandleMessage()185 absl::Status HandleMessage() {
186 auto message_status = Message::RecvFrom(client_fd_.get());
187 if (!message_status.ok()) {
188 return message_status.status();
189 }
190 auto creds_status = UpdateCredentials(message_status->Credentials());
191 if (!creds_status.ok()) {
192 return creds_status;
193 }
194
195 /* This handshake process is to reliably build a `pidfd` based on the pid
196 * supplied in the process `ucreds`, through the following steps:
197 * 1. Proxy process opens a socket and sends an opening message.
198 * 2. Server receives opening message with a kernel-validated `ucreds`
199 * containing the outside-sandbox pid.
200 * 3. Server opens a pidfd matching this pid.
201 * 4. Server sends a message to the client with some unique data.
202 * 5. Client responds with the unique data.
203 * 6. Server validates the unique data and credentials match.
204 * 7. Server launches a possible sandboxed subprocess based on the pidfd and
205 * /proc/{pid}/
206 *
207 * Step 5 builds confidence that the pidfd opened in step 3 still
208 * corresponds to the client sending messages on the client socket. The
209 * pidfd and /proc/{pid} data provide everything necessary to launch the
210 * subprocess.
211 */
212 auto& message = message_status->Data();
213 switch (client_state_) {
214 case ClientState::kInitial: {
215 if (message != kHandshakeBegin) {
216 std::string err =
217 absl::StrFormat("'%v' != '%v'", kHandshakeBegin, message);
218 return absl::InternalError(err);
219 }
220 pingback_ = RandomString(manager_.bit_gen_, 32);
221 absl::StatusOr<std::size_t> stat =
222 SendStringMsg(client_fd_.get(), pingback_);
223 if (stat.ok()) {
224 client_state_ = ClientState::kIgnoredFd;
225 }
226 return stat.status();
227 }
228 case ClientState::kIgnoredFd:
229 if (!absl::SimpleAtoi(message, &ignored_fd_)) {
230 std::string error =
231 absl::StrFormat("Expected integer, got '%v'", message);
232 return absl::InternalError(error);
233 }
234 client_state_ = ClientState::kPingback;
235 return absl::OkStatus();
236 case ClientState::kPingback: {
237 if (message != pingback_) {
238 std::string err =
239 absl::StrFormat("Incorrect '%v' != '%v'", message, pingback_);
240 return absl::InternalError(err);
241 }
242 client_state_ = ClientState::kWaitingForExit;
243 return LaunchProcess();
244 }
245 case ClientState::kWaitingForExit:
246 return absl::InternalError("No messages allowed");
247 }
248 }
249
SendExitCode(int code)250 absl::Status SendExitCode(int code) {
251 auto send_exit_status = SendStringMsg(client_fd_.get(), "exit");
252 if (!send_exit_status.ok()) {
253 return send_exit_status.status();
254 }
255
256 return SendStringMsg(client_fd_.get(), std::to_string(code)).status();
257 }
258
259 private:
260 enum class ClientState { kInitial, kIgnoredFd, kPingback, kWaitingForExit };
261
UpdateCredentials(const std::optional<ucred> & credentials)262 absl::Status UpdateCredentials(const std::optional<ucred>& credentials) {
263 if (!credentials) {
264 return absl::InvalidArgumentError("no creds");
265 } else if (!credentials_) {
266 credentials_ = credentials;
267 } else if (credentials_->pid != credentials->pid) {
268 std::string err = absl::StrFormat("pid went from '%d' to '%d'",
269 credentials_->pid, credentials->pid);
270 return absl::PermissionDeniedError(err);
271 } else if (credentials_->uid != credentials->uid) {
272 return absl::PermissionDeniedError("uid changed");
273 } else if (credentials_->gid != credentials->gid) {
274 return absl::PermissionDeniedError("gid changed");
275 }
276 if (!pid_fd_) {
277 absl::StatusOr<PidFd> pid_fd =
278 PidFd::FromRunningProcess(credentials_->pid);
279 if (!pid_fd.ok()) {
280 return pid_fd.status();
281 }
282 pid_fd_ = std::move(*pid_fd);
283 }
284 return absl::OkStatus();
285 }
286
LaunchProcess()287 absl::Status LaunchProcess() {
288 if (!pid_fd_) {
289 return absl::InternalError("missing pid_fd_");
290 }
291 absl::StatusOr<std::vector<std::string>> argv = pid_fd_->Argv();
292 if (!argv.ok()) {
293 return argv.status();
294 }
295 if ((*argv)[0] == "openssl") {
296 (*argv)[0] = "/usr/bin/openssl";
297 }
298 absl::StatusOr<std::vector<std::pair<FDCloser, int>>> fds =
299 pid_fd_->AllFds();
300 if (!fds.ok()) {
301 return fds.status();
302 }
303 absl::StatusOr<std::vector<std::string>> env = pid_fd_->Env();
304 if (!env.ok()) {
305 return env.status();
306 }
307 fds->erase(
308 std::remove_if(fds->begin(), fds->end(),
309 [this](auto& arg) { return arg.second == ignored_fd_; }),
310 fds->end());
311 return manager_.RunProcess(client_fd_.get(), std::move(*argv),
312 std::move(*fds), *env);
313 }
314
315 SandboxManager& manager_;
316 FDCloser client_fd_;
317 std::optional<ucred> credentials_;
318 std::optional<PidFd> pid_fd_;
319
320 ClientState client_state_ = ClientState::kInitial;
321 std::string pingback_;
322 int ignored_fd_ = -1;
323 };
324
SandboxManager(HostInfo host_info,std::string runtime_dir,SignalFd signals,CredentialedUnixServer server)325 SandboxManager::SandboxManager(HostInfo host_info, std::string runtime_dir,
326 SignalFd signals, CredentialedUnixServer server)
327 : host_info_(std::move(host_info)),
328 runtime_dir_(std::move(runtime_dir)),
329 signals_(std::move(signals)),
330 server_(std::move(server)) {}
331
Create(HostInfo host_info)332 absl::StatusOr<std::unique_ptr<SandboxManager>> SandboxManager::Create(
333 HostInfo host_info) {
334 std::string runtime_dir =
335 absl::StrFormat("/tmp/sandbox_manager.%u.XXXXXX", getpid());
336 if (mkdtemp(runtime_dir.data()) == nullptr) {
337 return absl::ErrnoToStatus(errno, "mkdtemp failed");
338 }
339 VLOG(1) << "Created temporary directory '" << runtime_dir << "'";
340
341 absl::StatusOr<SignalFd> signals = SignalFd::AllExceptSigChld();
342 if (!signals.ok()) {
343 return signals.status();
344 }
345
346 absl::StatusOr<CredentialedUnixServer> server =
347 CredentialedUnixServer::Open(ServerSocketOutsidePath(runtime_dir));
348 if (!server.ok()) {
349 return server.status();
350 }
351
352 return absl::WrapUnique(
353 new SandboxManager(std::move(host_info), std::move(runtime_dir),
354 std::move(*signals), std::move(*server)));
355 }
356
~SandboxManager()357 SandboxManager::~SandboxManager() {
358 VLOG(1) << "Sandbox shutting down";
359 if (!runtime_dir_.empty()) {
360 if (unlink(ServerSocketOutsidePath(runtime_dir_).c_str()) < 0) {
361 PLOG(ERROR) << "`unlink` failed";
362 }
363 if (rmdir(runtime_dir_.c_str()) < 0) {
364 PLOG(ERROR) << "Failed to remove '" << runtime_dir_ << "'";
365 }
366 }
367 }
368
RunProcess(std::optional<int> client_fd,absl::Span<const std::string> argv,std::vector<std::pair<FDCloser,int>> fds,absl::Span<const std::string> env)369 absl::Status SandboxManager::RunProcess(
370 std::optional<int> client_fd, absl::Span<const std::string> argv,
371 std::vector<std::pair<FDCloser, int>> fds,
372 absl::Span<const std::string> env) {
373 if (argv.empty()) {
374 return absl::InvalidArgumentError("Not enough arguments");
375 }
376 bool stdio_mapped[3] = {false, false, false};
377 for (const auto& [input_fd, target_fd] : fds) {
378 if (0 <= target_fd && target_fd <= 2) {
379 stdio_mapped[target_fd] = true;
380 }
381 }
382 // If stdio is not filled in, file descriptors opened by the target process
383 // may occupy the standard stdio positions. This can cause unexpected
384 for (int i = 0; i <= 2; i++) {
385 if (stdio_mapped[i]) {
386 continue;
387 }
388 auto& [stdio_dup, stdio] = fds.emplace_back(dup(i), i);
389 if (stdio_dup.get() < 0) {
390 return absl::ErrnoToStatus(errno, "Failed to `dup` stdio descriptor");
391 }
392 }
393 std::string exe = CleanPath(argv[0]);
394 std::unique_ptr<Policy> policy = PolicyForExecutable(
395 host_info_, ServerSocketOutsidePath(runtime_dir_), exe);
396 if (policy) {
397 return RunSandboxedProcess(client_fd, argv, std::move(fds), env,
398 std::move(policy));
399 } else {
400 return RunProcessNoSandbox(client_fd, argv, std::move(fds), env);
401 }
402 }
403
RunSandboxedProcess(std::optional<int> client_fd,absl::Span<const std::string> argv,std::vector<std::pair<FDCloser,int>> fds,absl::Span<const std::string> env,std::unique_ptr<Policy> policy)404 absl::Status SandboxManager::RunSandboxedProcess(
405 std::optional<int> client_fd, absl::Span<const std::string> argv,
406 std::vector<std::pair<FDCloser, int>> fds,
407 absl::Span<const std::string> env, std::unique_ptr<Policy> policy) {
408 if (VLOG_IS_ON(1)) {
409 std::stringstream process_stream;
410 process_stream << "Launching executable with argv: [\n";
411 for (const auto& arg : argv) {
412 process_stream << "\t\"" << arg << "\",\n";
413 }
414 process_stream << "] with FD mapping: [\n";
415 for (const auto& [fd_in, fd_out] : fds) {
416 process_stream << '\t' << fd_in.get() << " -> " << fd_out << ",\n";
417 }
418 process_stream << "]\n";
419 VLOG(1) << process_stream.str();
420 }
421
422 std::string exe = CleanPath(argv[0]);
423 auto executor = std::make_unique<Executor>(exe, argv, env);
424 executor->set_cwd(host_info_.runtime_dir);
425
426 // https://cs.android.com/android/platform/superproject/main/+/main:external/sandboxed-api/sandboxed_api/sandbox2/limits.h;l=116;drc=d451478e26c0352ecd6912461e867a1ae64b17f5
427 // Default is 120 seconds
428 executor->limits()->set_walltime_limit(absl::InfiniteDuration());
429 // Default is 1024 seconds
430 executor->limits()->set_rlimit_cpu(RLIM64_INFINITY);
431
432 for (auto& [fd_outer, fd_inner] : fds) {
433 // Will close `fd_outer` in this process
434 executor->ipc()->MapFd(fd_outer.Release(), fd_inner);
435 }
436
437 FDCloser event_fd(eventfd(0, EFD_CLOEXEC));
438 if (event_fd.get() < 0) {
439 return absl::ErrnoToStatus(errno, "`eventfd` failed");
440 }
441
442 auto sbx = std::make_unique<Sandbox2>(std::move(executor), std::move(policy));
443 if (!sbx->RunAsync()) {
444 return sbx->AwaitResult().ToStatus();
445 }
446
447 // A pidfd over the sandbox is another option, but there are two problems:
448 //
449 // 1. There's a race between launching the sandbox and opening the pidfd. If
450 // the sandboxed process exits too quickly, the monitor thread in sandbox2
451 // could reap it and another process could reuse the pid before `pidfd_open`
452 // runs. Sandbox2 could produce a pidfd itself using `CLONE_PIDFD`, but it
453 // does not do this at the time of writing.
454 //
455 // 2. The sandbox could outlive its top-level process. It's not clear to me if
456 // sandbox2 allows this in practice, but `AwaitResult` could theoretically
457 // wait on subprocesses of the original sandboxed process as well.
458 //
459 // To deal with these concerns, we use another thread blocked on AwaitResult
460 // that signals the eventfd when sandbox2 says the sandboxed process has
461 // exited.
462
463 subprocesses_.emplace_back(
464 new SandboxedProcess(client_fd, std::move(event_fd), std::move(sbx)));
465
466 return absl::OkStatus();
467 }
468
RunProcessNoSandbox(std::optional<int> client_fd,absl::Span<const std::string> argv,std::vector<std::pair<FDCloser,int>> fds,absl::Span<const std::string> env)469 absl::Status SandboxManager::RunProcessNoSandbox(
470 std::optional<int> client_fd, absl::Span<const std::string> argv,
471 std::vector<std::pair<FDCloser, int>> fds,
472 absl::Span<const std::string> env) {
473 if (!client_fd) {
474 return absl::InvalidArgumentError("no client for unsandboxed process");
475 }
476
477 absl::StatusOr<PidFd> fd = PidFd::LaunchSubprocess(argv, std::move(fds), env);
478 if (!fd.ok()) {
479 return fd.status();
480 }
481 subprocesses_.emplace_back(new ProcessNoSandbox(*client_fd, std::move(*fd)));
482
483 return absl::OkStatus();
484 }
485
Running() const486 bool SandboxManager::Running() const { return running_; }
487
Iterate()488 absl::Status SandboxManager::Iterate() {
489 PollCallback poll_cb;
490
491 poll_cb.Add(signals_.Fd(),
492 absl::bind_front(&SandboxManager::Signalled, this));
493 poll_cb.Add(server_.Fd(), absl::bind_front(&SandboxManager::NewClient, this));
494
495 for (auto it = subprocesses_.begin(); it != subprocesses_.end(); it++) {
496 int fd = (*it)->PollFd();
497 poll_cb.Add(fd, absl::bind_front(&SandboxManager::ProcessExit, this, it));
498 }
499 for (auto it = clients_.begin(); it != clients_.end(); it++) {
500 int fd = (*it)->ClientFd();
501 poll_cb.Add(fd, absl::bind_front(&SandboxManager::ClientMessage, this, it));
502 }
503
504 return poll_cb.Poll();
505 }
506
Signalled(short revents)507 absl::Status SandboxManager::Signalled(short revents) {
508 if (revents != POLLIN) {
509 running_ = false;
510 return absl::InternalError("signalfd exited");
511 }
512
513 absl::StatusOr<signalfd_siginfo> info = signals_.ReadSignal();
514 if (!info.ok()) {
515 return info.status();
516 }
517 VLOG(1) << "Received signal with signo '" << info->ssi_signo << "'";
518
519 switch (info->ssi_signo) {
520 case SIGHUP:
521 case SIGINT:
522 case SIGTERM:
523 LOG(INFO) << "Received signal '" << info->ssi_signo << "', exiting";
524 running_ = false;
525 return absl::OkStatus();
526 default:
527 std::string err = absl::StrCat("Unexpected signal ", info->ssi_signo);
528 return absl::InternalError(err);
529 }
530 }
531
NewClient(short revents)532 absl::Status SandboxManager::NewClient(short revents) {
533 if (revents != POLLIN) {
534 running_ = false;
535 return absl::InternalError("server socket exited");
536 }
537 absl::StatusOr<FDCloser> client = server_.AcceptClient();
538 if (!client.ok()) {
539 return client.status();
540 }
541 clients_.emplace_back(new SocketClient(*this, std::move(*client)));
542 return absl::OkStatus();
543 }
544
ProcessExit(SandboxManager::SboxIter it,short revents)545 absl::Status SandboxManager::ProcessExit(SandboxManager::SboxIter it,
546 short revents) {
547 if ((*it)->ClientFd()) {
548 int client_fd = *(*it)->ClientFd();
549 for (auto& client : clients_) {
550 if (client->ClientFd() != client_fd) {
551 continue;
552 }
553 auto exit_code = (*it)->ExitCode();
554 if (!exit_code.ok()) {
555 LOG(ERROR) << exit_code.status();
556 }
557 // TODO(schuffelen): Forward more complete exit information
558 auto send_res = client->SendExitCode(exit_code.value_or(254));
559 if (!send_res.ok()) {
560 return send_res;
561 }
562 }
563 }
564 subprocesses_.erase(it);
565 if (subprocesses_.empty()) {
566 running_ = false;
567 }
568 static constexpr char kErr[] = "eventfd exited";
569 return revents == POLLIN ? absl::OkStatus() : absl::InternalError(kErr);
570 }
571
ClientMessage(SandboxManager::ClientIter it,short rev)572 absl::Status SandboxManager::ClientMessage(SandboxManager::ClientIter it,
573 short rev) {
574 if (rev == POLLIN) {
575 return (*it)->HandleMessage();
576 }
577 clients_.erase(it);
578 return absl::InternalError("client dropped file descriptor");
579 }
580
581 } // namespace cuttlefish::process_sandboxer
582