1 // Copyright 2017 The Chromium OS Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 use std;
6 use std::cmp::min;
7 use std::convert::TryFrom;
8 use std::error::Error as StdError;
9 use std::ffi::CStr;
10 use std::fmt::{self, Display};
11 use std::fs::{File, OpenOptions};
12 use std::io::{self, stdin, Read};
13 use std::net::Ipv4Addr;
14 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
15 use std::os::unix::net::UnixStream;
16 use std::path::{Path, PathBuf};
17 use std::str;
18 use std::sync::{Arc, Barrier};
19 use std::thread;
20 use std::thread::JoinHandle;
21 use std::time::{Duration, SystemTime, UNIX_EPOCH};
22
23 use libc::{self, c_int, gid_t, uid_t};
24
25 use audio_streams::DummyStreamSource;
26 use devices::virtio::{self, VirtioDevice};
27 use devices::{self, HostBackendDeviceProvider, PciDevice, VirtioPciDevice, XhciController};
28 use io_jail::{self, Minijail};
29 use kvm::*;
30 use libcras::CrasClient;
31 use msg_socket::{MsgError, MsgReceiver, MsgSender, MsgSocket};
32 use net_util::{Error as NetError, MacAddress, Tap};
33 use qcow::{self, ImageType, QcowFile};
34 use rand_ish::SimpleRng;
35 use remain::sorted;
36 use resources::{Alloc, SystemAllocator};
37 use sync::{Condvar, Mutex};
38 use sys_util::net::{UnixSeqpacket, UnixSeqpacketListener, UnlinkUnixSeqpacketListener};
39
40 use sys_util::{
41 self, block_signal, clear_signal, drop_capabilities, error, flock, get_blocked_signals,
42 get_group_id, get_user_id, getegid, geteuid, info, register_signal_handler, set_cpu_affinity,
43 validate_raw_fd, warn, EventFd, FlockOperation, GuestAddress, GuestMemory, Killable,
44 MemoryMapping, PollContext, PollToken, Protection, SignalFd, Terminal, TimerFd, WatchingEvents,
45 SIGRTMIN,
46 };
47 use vhost;
48 use vm_control::{
49 BalloonControlCommand, BalloonControlRequestSocket, BalloonControlResponseSocket,
50 DiskControlCommand, DiskControlRequestSocket, DiskControlResponseSocket, DiskControlResult,
51 UsbControlSocket, VmControlResponseSocket, VmMemoryControlRequestSocket,
52 VmMemoryControlResponseSocket, VmMemoryRequest, VmMemoryResponse, VmRunMode,
53 };
54
55 use crate::{Config, DiskOption, Executable, TouchDeviceOption};
56
57 use arch::{self, LinuxArch, RunnableLinuxVm, VirtioDeviceStub, VmComponents, VmImage};
58
59 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
60 use aarch64::AArch64 as Arch;
61 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
62 use x86_64::X8664arch as Arch;
63
64 #[cfg(feature = "gpu-forward")]
65 use render_node_forward::*;
66 #[cfg(not(feature = "gpu-forward"))]
67 type RenderNodeHost = ();
68
69 #[sorted]
70 #[derive(Debug)]
71 pub enum Error {
72 AddGpuDeviceMemory(sys_util::Error),
73 AddPmemDeviceMemory(sys_util::Error),
74 AllocateGpuDeviceAddress,
75 AllocatePmemDeviceAddress(resources::Error),
76 BalloonDeviceNew(virtio::BalloonError),
77 BlockDeviceNew(sys_util::Error),
78 BlockSignal(sys_util::signal::Error),
79 BuildVm(<Arch as LinuxArch>::Error),
80 ChownTpmStorage(sys_util::Error),
81 CloneEventFd(sys_util::Error),
82 CreateCrasClient(libcras::Error),
83 CreateEventFd(sys_util::Error),
84 CreatePollContext(sys_util::Error),
85 CreateSignalFd(sys_util::SignalFdError),
86 CreateSocket(io::Error),
87 CreateTapDevice(NetError),
88 CreateTimerFd(sys_util::Error),
89 CreateTpmStorage(PathBuf, io::Error),
90 CreateUsbProvider(devices::usb::host_backend::error::Error),
91 DetectImageType(qcow::Error),
92 DeviceJail(io_jail::Error),
93 DevicePivotRoot(io_jail::Error),
94 Disk(io::Error),
95 DiskImageLock(sys_util::Error),
96 DropCapabilities(sys_util::Error),
97 InputDeviceNew(virtio::InputError),
98 InputEventsOpen(std::io::Error),
99 InvalidFdPath,
100 InvalidWaylandPath,
101 IoJail(io_jail::Error),
102 LoadKernel(Box<dyn StdError>),
103 NetDeviceNew(virtio::NetError),
104 OpenAndroidFstab(PathBuf, io::Error),
105 OpenBios(PathBuf, io::Error),
106 OpenInitrd(PathBuf, io::Error),
107 OpenKernel(PathBuf, io::Error),
108 OpenVinput(PathBuf, io::Error),
109 P9DeviceNew(virtio::P9Error),
110 PivotRootDoesntExist(&'static str),
111 PmemDeviceImageTooBig,
112 PmemDeviceNew(sys_util::Error),
113 PollContextAdd(sys_util::Error),
114 PollContextDelete(sys_util::Error),
115 QcowDeviceCreate(qcow::Error),
116 ReadLowmemAvailable(io::Error),
117 ReadLowmemMargin(io::Error),
118 RegisterBalloon(arch::DeviceRegistrationError),
119 RegisterBlock(arch::DeviceRegistrationError),
120 RegisterGpu(arch::DeviceRegistrationError),
121 RegisterNet(arch::DeviceRegistrationError),
122 RegisterP9(arch::DeviceRegistrationError),
123 RegisterRng(arch::DeviceRegistrationError),
124 RegisterSignalHandler(sys_util::Error),
125 RegisterWayland(arch::DeviceRegistrationError),
126 ReserveGpuMemory(sys_util::MmapError),
127 ReserveMemory(sys_util::Error),
128 ReservePmemMemory(sys_util::MmapError),
129 ResetTimerFd(sys_util::Error),
130 RngDeviceNew(virtio::RngError),
131 SettingGidMap(io_jail::Error),
132 SettingUidMap(io_jail::Error),
133 SignalFd(sys_util::SignalFdError),
134 SpawnVcpu(io::Error),
135 TimerFd(sys_util::Error),
136 ValidateRawFd(sys_util::Error),
137 VhostNetDeviceNew(virtio::vhost::Error),
138 VhostVsockDeviceNew(virtio::vhost::Error),
139 VirtioPciDev(sys_util::Error),
140 WaylandDeviceNew(sys_util::Error),
141 }
142
143 impl Display for Error {
144 #[remain::check]
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result145 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
146 use self::Error::*;
147
148 #[sorted]
149 match self {
150 AddGpuDeviceMemory(e) => write!(f, "failed to add gpu device memory: {}", e),
151 AddPmemDeviceMemory(e) => write!(f, "failed to add pmem device memory: {}", e),
152 AllocateGpuDeviceAddress => write!(f, "failed to allocate gpu device guest address"),
153 AllocatePmemDeviceAddress(e) => {
154 write!(f, "failed to allocate memory for pmem device: {}", e)
155 }
156 BalloonDeviceNew(e) => write!(f, "failed to create balloon: {}", e),
157 BlockDeviceNew(e) => write!(f, "failed to create block device: {}", e),
158 BlockSignal(e) => write!(f, "failed to block signal: {}", e),
159 BuildVm(e) => write!(f, "The architecture failed to build the vm: {}", e),
160 ChownTpmStorage(e) => write!(f, "failed to chown tpm storage: {}", e),
161 CloneEventFd(e) => write!(f, "failed to clone eventfd: {}", e),
162 CreateCrasClient(e) => write!(f, "failed to create cras client: {}", e),
163 CreateEventFd(e) => write!(f, "failed to create eventfd: {}", e),
164 CreatePollContext(e) => write!(f, "failed to create poll context: {}", e),
165 CreateSignalFd(e) => write!(f, "failed to create signalfd: {}", e),
166 CreateSocket(e) => write!(f, "failed to create socket: {}", e),
167 CreateTapDevice(e) => write!(f, "failed to create tap device: {}", e),
168 CreateTimerFd(e) => write!(f, "failed to create timerfd: {}", e),
169 CreateTpmStorage(p, e) => {
170 write!(f, "failed to create tpm storage dir {}: {}", p.display(), e)
171 }
172 CreateUsbProvider(e) => write!(f, "failed to create usb provider: {}", e),
173 DetectImageType(e) => write!(f, "failed to detect disk image type: {}", e),
174 DeviceJail(e) => write!(f, "failed to jail device: {}", e),
175 DevicePivotRoot(e) => write!(f, "failed to pivot root device: {}", e),
176 Disk(e) => write!(f, "failed to load disk image: {}", e),
177 DiskImageLock(e) => write!(f, "failed to lock disk image: {}", e),
178 DropCapabilities(e) => write!(f, "failed to drop process capabilities: {}", e),
179 InputDeviceNew(e) => write!(f, "failed to set up input device: {}", e),
180 InputEventsOpen(e) => write!(f, "failed to open event device: {}", e),
181 InvalidFdPath => write!(f, "failed parsing a /proc/self/fd/*"),
182 InvalidWaylandPath => write!(f, "wayland socket path has no parent or file name"),
183 IoJail(e) => write!(f, "{}", e),
184 LoadKernel(e) => write!(f, "failed to load kernel: {}", e),
185 NetDeviceNew(e) => write!(f, "failed to set up virtio networking: {}", e),
186 OpenAndroidFstab(p, e) => write!(
187 f,
188 "failed to open android fstab file {}: {}",
189 p.display(),
190 e
191 ),
192 OpenBios(p, e) => write!(f, "failed to open bios {}: {}", p.display(), e),
193 OpenInitrd(p, e) => write!(f, "failed to open initrd {}: {}", p.display(), e),
194 OpenKernel(p, e) => write!(f, "failed to open kernel image {}: {}", p.display(), e),
195 OpenVinput(p, e) => write!(f, "failed to open vinput device {}: {}", p.display(), e),
196 P9DeviceNew(e) => write!(f, "failed to create 9p device: {}", e),
197 PivotRootDoesntExist(p) => write!(f, "{} doesn't exist, can't jail devices.", p),
198 PmemDeviceImageTooBig => {
199 write!(f, "failed to create pmem device: pmem device image too big")
200 }
201 PmemDeviceNew(e) => write!(f, "failed to create pmem device: {}", e),
202 PollContextAdd(e) => write!(f, "failed to add fd to poll context: {}", e),
203 PollContextDelete(e) => write!(f, "failed to remove fd from poll context: {}", e),
204 QcowDeviceCreate(e) => write!(f, "failed to read qcow formatted file {}", e),
205 ReadLowmemAvailable(e) => write!(
206 f,
207 "failed to read /sys/kernel/mm/chromeos-low_mem/available: {}",
208 e
209 ),
210 ReadLowmemMargin(e) => write!(
211 f,
212 "failed to read /sys/kernel/mm/chromeos-low_mem/margin: {}",
213 e
214 ),
215 RegisterBalloon(e) => write!(f, "error registering balloon device: {}", e),
216 RegisterBlock(e) => write!(f, "error registering block device: {}", e),
217 RegisterGpu(e) => write!(f, "error registering gpu device: {}", e),
218 RegisterNet(e) => write!(f, "error registering net device: {}", e),
219 RegisterP9(e) => write!(f, "error registering 9p device: {}", e),
220 RegisterRng(e) => write!(f, "error registering rng device: {}", e),
221 RegisterSignalHandler(e) => write!(f, "error registering signal handler: {}", e),
222 RegisterWayland(e) => write!(f, "error registering wayland device: {}", e),
223 ReserveGpuMemory(e) => write!(f, "failed to reserve gpu memory: {}", e),
224 ReserveMemory(e) => write!(f, "failed to reserve memory: {}", e),
225 ReservePmemMemory(e) => write!(f, "failed to reserve pmem memory: {}", e),
226 ResetTimerFd(e) => write!(f, "failed to reset timerfd: {}", e),
227 RngDeviceNew(e) => write!(f, "failed to set up rng: {}", e),
228 SettingGidMap(e) => write!(f, "error setting GID map: {}", e),
229 SettingUidMap(e) => write!(f, "error setting UID map: {}", e),
230 SignalFd(e) => write!(f, "failed to read signal fd: {}", e),
231 SpawnVcpu(e) => write!(f, "failed to spawn VCPU thread: {}", e),
232 TimerFd(e) => write!(f, "failed to read timer fd: {}", e),
233 ValidateRawFd(e) => write!(f, "failed to validate raw fd: {}", e),
234 VhostNetDeviceNew(e) => write!(f, "failed to set up vhost networking: {}", e),
235 VhostVsockDeviceNew(e) => write!(f, "failed to set up virtual socket device: {}", e),
236 VirtioPciDev(e) => write!(f, "failed to create virtio pci dev: {}", e),
237 WaylandDeviceNew(e) => write!(f, "failed to create wayland device: {}", e),
238 }
239 }
240 }
241
242 impl From<io_jail::Error> for Error {
from(err: io_jail::Error) -> Self243 fn from(err: io_jail::Error) -> Self {
244 Error::IoJail(err)
245 }
246 }
247
248 impl std::error::Error for Error {}
249
250 type Result<T> = std::result::Result<T, Error>;
251
252 enum TaggedControlSocket {
253 Vm(VmControlResponseSocket),
254 VmMemory(VmMemoryControlResponseSocket),
255 }
256
257 impl AsRef<UnixSeqpacket> for TaggedControlSocket {
as_ref(&self) -> &UnixSeqpacket258 fn as_ref(&self) -> &UnixSeqpacket {
259 use self::TaggedControlSocket::*;
260 match &self {
261 Vm(ref socket) => socket,
262 VmMemory(ref socket) => socket,
263 }
264 }
265 }
266
267 impl AsRawFd for TaggedControlSocket {
as_raw_fd(&self) -> RawFd268 fn as_raw_fd(&self) -> RawFd {
269 self.as_ref().as_raw_fd()
270 }
271 }
272
create_base_minijail(root: &Path, seccomp_policy: &Path) -> Result<Minijail>273 fn create_base_minijail(root: &Path, seccomp_policy: &Path) -> Result<Minijail> {
274 // All child jails run in a new user namespace without any users mapped,
275 // they run as nobody unless otherwise configured.
276 let mut j = Minijail::new().map_err(Error::DeviceJail)?;
277 j.namespace_pids();
278 j.namespace_user();
279 j.namespace_user_disable_setgroups();
280 // Don't need any capabilities.
281 j.use_caps(0);
282 // Create a new mount namespace with an empty root FS.
283 j.namespace_vfs();
284 j.enter_pivot_root(root).map_err(Error::DevicePivotRoot)?;
285 // Run in an empty network namespace.
286 j.namespace_net();
287 // Apply the block device seccomp policy.
288 j.no_new_privs();
289 // Use TSYNC only for the side effect of it using SECCOMP_RET_TRAP, which will correctly kill
290 // the entire device process if a worker thread commits a seccomp violation.
291 j.set_seccomp_filter_tsync();
292 #[cfg(debug_assertions)]
293 j.log_seccomp_filter_failures();
294 j.parse_seccomp_filters(seccomp_policy)
295 .map_err(Error::DeviceJail)?;
296 j.use_seccomp_filter();
297 // Don't do init setup.
298 j.run_as_init();
299 Ok(j)
300 }
301
simple_jail(cfg: &Config, policy: &str) -> Result<Option<Minijail>>302 fn simple_jail(cfg: &Config, policy: &str) -> Result<Option<Minijail>> {
303 if cfg.sandbox {
304 let pivot_root: &str = option_env!("DEFAULT_PIVOT_ROOT").unwrap_or("/var/empty");
305 // A directory for a jailed device's pivot root.
306 let root_path = Path::new(pivot_root);
307 if !root_path.exists() {
308 return Err(Error::PivotRootDoesntExist(pivot_root));
309 }
310 let policy_path: PathBuf = cfg.seccomp_policy_dir.join(policy);
311 Ok(Some(create_base_minijail(root_path, &policy_path)?))
312 } else {
313 Ok(None)
314 }
315 }
316
317 type DeviceResult<T = VirtioDeviceStub> = std::result::Result<T, Error>;
318
create_block_device( cfg: &Config, disk: &DiskOption, disk_device_socket: DiskControlResponseSocket, ) -> DeviceResult319 fn create_block_device(
320 cfg: &Config,
321 disk: &DiskOption,
322 disk_device_socket: DiskControlResponseSocket,
323 ) -> DeviceResult {
324 // Special case '/proc/self/fd/*' paths. The FD is already open, just use it.
325 let raw_image: File = if disk.path.parent() == Some(Path::new("/proc/self/fd")) {
326 // Safe because we will validate |raw_fd|.
327 unsafe { File::from_raw_fd(raw_fd_from_path(&disk.path)?) }
328 } else {
329 OpenOptions::new()
330 .read(true)
331 .write(!disk.read_only)
332 .open(&disk.path)
333 .map_err(Error::Disk)?
334 };
335 // Lock the disk image to prevent other crosvm instances from using it.
336 let lock_op = if disk.read_only {
337 FlockOperation::LockShared
338 } else {
339 FlockOperation::LockExclusive
340 };
341 flock(&raw_image, lock_op, true).map_err(Error::DiskImageLock)?;
342
343 let image_type = qcow::detect_image_type(&raw_image).map_err(Error::DetectImageType)?;
344 let dev = match image_type {
345 ImageType::Raw => {
346 // Access as a raw block device.
347 let dev = virtio::Block::new(raw_image, disk.read_only, Some(disk_device_socket))
348 .map_err(Error::BlockDeviceNew)?;
349 Box::new(dev) as Box<dyn VirtioDevice>
350 }
351 ImageType::Qcow2 => {
352 // Valid qcow header present
353 let qcow_image = QcowFile::from(raw_image).map_err(Error::QcowDeviceCreate)?;
354 let dev = virtio::Block::new(qcow_image, disk.read_only, Some(disk_device_socket))
355 .map_err(Error::BlockDeviceNew)?;
356 Box::new(dev) as Box<dyn VirtioDevice>
357 }
358 };
359
360 Ok(VirtioDeviceStub {
361 dev,
362 jail: simple_jail(&cfg, "block_device.policy")?,
363 })
364 }
365
create_rng_device(cfg: &Config) -> DeviceResult366 fn create_rng_device(cfg: &Config) -> DeviceResult {
367 let dev = virtio::Rng::new().map_err(Error::RngDeviceNew)?;
368
369 Ok(VirtioDeviceStub {
370 dev: Box::new(dev),
371 jail: simple_jail(&cfg, "rng_device.policy")?,
372 })
373 }
374
375 #[cfg(feature = "tpm")]
create_tpm_device(cfg: &Config) -> DeviceResult376 fn create_tpm_device(cfg: &Config) -> DeviceResult {
377 use std::ffi::CString;
378 use std::fs;
379 use std::process;
380 use sys_util::chown;
381
382 let tpm_storage: PathBuf;
383 let mut tpm_jail = simple_jail(&cfg, "tpm_device.policy")?;
384
385 match &mut tpm_jail {
386 Some(jail) => {
387 // Create a tmpfs in the device's root directory for tpm
388 // simulator storage. The size is 20*1024, or 20 KB.
389 jail.mount_with_data(
390 Path::new("none"),
391 Path::new("/"),
392 "tmpfs",
393 (libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
394 "size=20480",
395 )?;
396
397 let crosvm_ids = add_crosvm_user_to_jail(jail, "tpm")?;
398
399 let pid = process::id();
400 let tpm_pid_dir = format!("/run/vm/tpm.{}", pid);
401 tpm_storage = Path::new(&tpm_pid_dir).to_owned();
402 fs::create_dir_all(&tpm_storage)
403 .map_err(|e| Error::CreateTpmStorage(tpm_storage.to_owned(), e))?;
404 let tpm_pid_dir_c = CString::new(tpm_pid_dir).expect("no nul bytes");
405 chown(&tpm_pid_dir_c, crosvm_ids.uid, crosvm_ids.gid)
406 .map_err(Error::ChownTpmStorage)?;
407
408 jail.mount_bind(&tpm_storage, &tpm_storage, true)?;
409 }
410 None => {
411 // Path used inside cros_sdk which does not have /run/vm.
412 tpm_storage = Path::new("/tmp/tpm-simulator").to_owned();
413 }
414 }
415
416 let dev = virtio::Tpm::new(tpm_storage);
417
418 Ok(VirtioDeviceStub {
419 dev: Box::new(dev),
420 jail: tpm_jail,
421 })
422 }
423
create_single_touch_device(cfg: &Config, single_touch_spec: &TouchDeviceOption) -> DeviceResult424 fn create_single_touch_device(cfg: &Config, single_touch_spec: &TouchDeviceOption) -> DeviceResult {
425 let socket = create_input_socket(&single_touch_spec.path).map_err(|e| {
426 error!("failed configuring virtio single touch: {:?}", e);
427 e
428 })?;
429
430 let dev = virtio::new_single_touch(socket, single_touch_spec.width, single_touch_spec.height)
431 .map_err(Error::InputDeviceNew)?;
432 Ok(VirtioDeviceStub {
433 dev: Box::new(dev),
434 jail: simple_jail(&cfg, "input_device.policy")?,
435 })
436 }
437
create_trackpad_device(cfg: &Config, trackpad_spec: &TouchDeviceOption) -> DeviceResult438 fn create_trackpad_device(cfg: &Config, trackpad_spec: &TouchDeviceOption) -> DeviceResult {
439 let socket = create_input_socket(&trackpad_spec.path).map_err(|e| {
440 error!("failed configuring virtio trackpad: {}", e);
441 e
442 })?;
443
444 let dev = virtio::new_trackpad(socket, trackpad_spec.width, trackpad_spec.height)
445 .map_err(Error::InputDeviceNew)?;
446
447 Ok(VirtioDeviceStub {
448 dev: Box::new(dev),
449 jail: simple_jail(&cfg, "input_device.policy")?,
450 })
451 }
452
create_mouse_device(cfg: &Config, mouse_socket: &Path) -> DeviceResult453 fn create_mouse_device(cfg: &Config, mouse_socket: &Path) -> DeviceResult {
454 let socket = create_input_socket(&mouse_socket).map_err(|e| {
455 error!("failed configuring virtio mouse: {}", e);
456 e
457 })?;
458
459 let dev = virtio::new_mouse(socket).map_err(Error::InputDeviceNew)?;
460
461 Ok(VirtioDeviceStub {
462 dev: Box::new(dev),
463 jail: simple_jail(&cfg, "input_device.policy")?,
464 })
465 }
466
create_keyboard_device(cfg: &Config, keyboard_socket: &Path) -> DeviceResult467 fn create_keyboard_device(cfg: &Config, keyboard_socket: &Path) -> DeviceResult {
468 let socket = create_input_socket(&keyboard_socket).map_err(|e| {
469 error!("failed configuring virtio keyboard: {}", e);
470 e
471 })?;
472
473 let dev = virtio::new_keyboard(socket).map_err(Error::InputDeviceNew)?;
474
475 Ok(VirtioDeviceStub {
476 dev: Box::new(dev),
477 jail: simple_jail(&cfg, "input_device.policy")?,
478 })
479 }
480
create_vinput_device(cfg: &Config, dev_path: &Path) -> DeviceResult481 fn create_vinput_device(cfg: &Config, dev_path: &Path) -> DeviceResult {
482 let dev_file = OpenOptions::new()
483 .read(true)
484 .write(true)
485 .open(dev_path)
486 .map_err(|e| Error::OpenVinput(dev_path.to_owned(), e))?;
487
488 let dev = virtio::new_evdev(dev_file).map_err(Error::InputDeviceNew)?;
489
490 Ok(VirtioDeviceStub {
491 dev: Box::new(dev),
492 jail: simple_jail(&cfg, "input_device.policy")?,
493 })
494 }
495
create_balloon_device(cfg: &Config, socket: BalloonControlResponseSocket) -> DeviceResult496 fn create_balloon_device(cfg: &Config, socket: BalloonControlResponseSocket) -> DeviceResult {
497 let dev = virtio::Balloon::new(socket).map_err(Error::BalloonDeviceNew)?;
498
499 Ok(VirtioDeviceStub {
500 dev: Box::new(dev),
501 jail: simple_jail(&cfg, "balloon_device.policy")?,
502 })
503 }
504
create_tap_net_device(cfg: &Config, tap_fd: RawFd) -> DeviceResult505 fn create_tap_net_device(cfg: &Config, tap_fd: RawFd) -> DeviceResult {
506 // Safe because we ensure that we get a unique handle to the fd.
507 let tap = unsafe {
508 Tap::from_raw_fd(validate_raw_fd(tap_fd).map_err(Error::ValidateRawFd)?)
509 .map_err(Error::CreateTapDevice)?
510 };
511
512 let dev = virtio::Net::from(tap).map_err(Error::NetDeviceNew)?;
513
514 Ok(VirtioDeviceStub {
515 dev: Box::new(dev),
516 jail: simple_jail(&cfg, "net_device.policy")?,
517 })
518 }
519
create_net_device( cfg: &Config, host_ip: Ipv4Addr, netmask: Ipv4Addr, mac_address: MacAddress, mem: &GuestMemory, ) -> DeviceResult520 fn create_net_device(
521 cfg: &Config,
522 host_ip: Ipv4Addr,
523 netmask: Ipv4Addr,
524 mac_address: MacAddress,
525 mem: &GuestMemory,
526 ) -> DeviceResult {
527 let dev = if cfg.vhost_net {
528 let dev =
529 virtio::vhost::Net::<Tap, vhost::Net<Tap>>::new(host_ip, netmask, mac_address, mem)
530 .map_err(Error::VhostNetDeviceNew)?;
531 Box::new(dev) as Box<dyn VirtioDevice>
532 } else {
533 let dev =
534 virtio::Net::<Tap>::new(host_ip, netmask, mac_address).map_err(Error::NetDeviceNew)?;
535 Box::new(dev) as Box<dyn VirtioDevice>
536 };
537
538 let policy = if cfg.vhost_net {
539 "vhost_net_device.policy"
540 } else {
541 "net_device.policy"
542 };
543
544 Ok(VirtioDeviceStub {
545 dev,
546 jail: simple_jail(&cfg, policy)?,
547 })
548 }
549
550 #[cfg(feature = "gpu")]
create_gpu_device( cfg: &Config, exit_evt: &EventFd, gpu_device_socket: VmMemoryControlRequestSocket, gpu_socket: virtio::resource_bridge::ResourceResponseSocket, wayland_socket_path: &Path, ) -> DeviceResult551 fn create_gpu_device(
552 cfg: &Config,
553 exit_evt: &EventFd,
554 gpu_device_socket: VmMemoryControlRequestSocket,
555 gpu_socket: virtio::resource_bridge::ResourceResponseSocket,
556 wayland_socket_path: &Path,
557 ) -> DeviceResult {
558 let jailed_wayland_path = Path::new("/wayland-0");
559
560 let dev = virtio::Gpu::new(
561 exit_evt.try_clone().map_err(Error::CloneEventFd)?,
562 Some(gpu_device_socket),
563 Some(gpu_socket),
564 if cfg.sandbox {
565 &jailed_wayland_path
566 } else {
567 wayland_socket_path
568 },
569 );
570
571 let jail = match simple_jail(&cfg, "gpu_device.policy")? {
572 Some(mut jail) => {
573 // Create a tmpfs in the device's root directory so that we can bind mount the
574 // dri directory into it. The size=67108864 is size=64*1024*1024 or size=64MB.
575 jail.mount_with_data(
576 Path::new("none"),
577 Path::new("/"),
578 "tmpfs",
579 (libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
580 "size=67108864",
581 )?;
582
583 // Device nodes required for DRM.
584 let sys_dev_char_path = Path::new("/sys/dev/char");
585 jail.mount_bind(sys_dev_char_path, sys_dev_char_path, false)?;
586 let sys_devices_path = Path::new("/sys/devices");
587 jail.mount_bind(sys_devices_path, sys_devices_path, false)?;
588 let drm_dri_path = Path::new("/dev/dri");
589 jail.mount_bind(drm_dri_path, drm_dri_path, false)?;
590
591 // Libraries that are required when mesa drivers are dynamically loaded.
592 let lib_path = Path::new("/lib64");
593 jail.mount_bind(lib_path, lib_path, false)?;
594 let usr_lib_path = Path::new("/usr/lib64");
595 jail.mount_bind(usr_lib_path, usr_lib_path, false)?;
596
597 // Bind mount the wayland socket into jail's root. This is necessary since each
598 // new wayland context must open() the socket.
599 jail.mount_bind(wayland_socket_path, jailed_wayland_path, true)?;
600
601 add_crosvm_user_to_jail(&mut jail, "gpu")?;
602
603 Some(jail)
604 }
605 None => None,
606 };
607
608 Ok(VirtioDeviceStub {
609 dev: Box::new(dev),
610 jail,
611 })
612 }
613
create_wayland_device( cfg: &Config, socket_path: &Path, socket: VmMemoryControlRequestSocket, resource_bridge: Option<virtio::resource_bridge::ResourceRequestSocket>, ) -> DeviceResult614 fn create_wayland_device(
615 cfg: &Config,
616 socket_path: &Path,
617 socket: VmMemoryControlRequestSocket,
618 resource_bridge: Option<virtio::resource_bridge::ResourceRequestSocket>,
619 ) -> DeviceResult {
620 let wayland_socket_dir = socket_path.parent().ok_or(Error::InvalidWaylandPath)?;
621 let wayland_socket_name = socket_path.file_name().ok_or(Error::InvalidWaylandPath)?;
622 let jailed_wayland_dir = Path::new("/wayland");
623 let jailed_wayland_path = jailed_wayland_dir.join(wayland_socket_name);
624
625 let dev = virtio::Wl::new(
626 if cfg.sandbox {
627 &jailed_wayland_path
628 } else {
629 socket_path
630 },
631 socket,
632 resource_bridge,
633 )
634 .map_err(Error::WaylandDeviceNew)?;
635
636 let jail = match simple_jail(&cfg, "wl_device.policy")? {
637 Some(mut jail) => {
638 // Create a tmpfs in the device's root directory so that we can bind mount the wayland
639 // socket directory into it. The size=67108864 is size=64*1024*1024 or size=64MB.
640 jail.mount_with_data(
641 Path::new("none"),
642 Path::new("/"),
643 "tmpfs",
644 (libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
645 "size=67108864",
646 )?;
647
648 // Bind mount the wayland socket's directory into jail's root. This is necessary since
649 // each new wayland context must open() the socket. If the wayland socket is ever
650 // destroyed and remade in the same host directory, new connections will be possible
651 // without restarting the wayland device.
652 jail.mount_bind(wayland_socket_dir, jailed_wayland_dir, true)?;
653
654 add_crosvm_user_to_jail(&mut jail, "Wayland")?;
655
656 Some(jail)
657 }
658 None => None,
659 };
660
661 Ok(VirtioDeviceStub {
662 dev: Box::new(dev),
663 jail,
664 })
665 }
666
create_vhost_vsock_device(cfg: &Config, cid: u64, mem: &GuestMemory) -> DeviceResult667 fn create_vhost_vsock_device(cfg: &Config, cid: u64, mem: &GuestMemory) -> DeviceResult {
668 let dev = virtio::vhost::Vsock::new(cid, mem).map_err(Error::VhostVsockDeviceNew)?;
669
670 Ok(VirtioDeviceStub {
671 dev: Box::new(dev),
672 jail: simple_jail(&cfg, "vhost_vsock_device.policy")?,
673 })
674 }
675
create_9p_device(cfg: &Config, chronos: Ids, src: &Path, tag: &str) -> DeviceResult676 fn create_9p_device(cfg: &Config, chronos: Ids, src: &Path, tag: &str) -> DeviceResult {
677 let (jail, root) = match simple_jail(&cfg, "9p_device.policy")? {
678 Some(mut jail) => {
679 // The shared directory becomes the root of the device's file system.
680 let root = Path::new("/");
681 jail.mount_bind(src, root, true)?;
682
683 // Set the uid/gid for the jailed process, and give a basic id map. This
684 // is required for the above bind mount to work.
685 jail.change_uid(chronos.uid);
686 jail.change_gid(chronos.gid);
687 jail.uidmap(&format!("{0} {0} 1", chronos.uid))
688 .map_err(Error::SettingUidMap)?;
689 jail.gidmap(&format!("{0} {0} 1", chronos.gid))
690 .map_err(Error::SettingGidMap)?;
691
692 (Some(jail), root)
693 }
694 None => {
695 // There's no bind mount so we tell the server to treat the source directory as the
696 // root.
697 (None, src)
698 }
699 };
700
701 let dev = virtio::P9::new(root, tag).map_err(Error::P9DeviceNew)?;
702
703 Ok(VirtioDeviceStub {
704 dev: Box::new(dev),
705 jail,
706 })
707 }
708
create_pmem_device( cfg: &Config, vm: &mut Vm, resources: &mut SystemAllocator, disk: &DiskOption, index: usize, ) -> DeviceResult709 fn create_pmem_device(
710 cfg: &Config,
711 vm: &mut Vm,
712 resources: &mut SystemAllocator,
713 disk: &DiskOption,
714 index: usize,
715 ) -> DeviceResult {
716 let fd = OpenOptions::new()
717 .read(true)
718 .write(!disk.read_only)
719 .open(&disk.path)
720 .map_err(Error::Disk)?;
721
722 let image_size = {
723 let metadata = std::fs::metadata(&disk.path).map_err(Error::Disk)?;
724 metadata.len()
725 };
726
727 let protection = {
728 if disk.read_only {
729 Protection::read()
730 } else {
731 Protection::read_write()
732 }
733 };
734
735 let memory_mapping = {
736 // Conversion from u64 to usize may fail on 32bit system.
737 let image_size = usize::try_from(image_size).map_err(|_| Error::PmemDeviceImageTooBig)?;
738
739 MemoryMapping::from_fd_offset_protection(&fd, image_size, 0, protection)
740 .map_err(Error::ReservePmemMemory)?
741 };
742
743 let mapping_address = resources
744 .device_allocator()
745 .allocate_with_align(
746 image_size,
747 Alloc::PmemDevice(index),
748 format!("pmem_disk_image_{}", index),
749 // Linux kernel requires pmem namespaces to be 128 MiB aligned.
750 128 * 1024 * 1024, /* 128 MiB */
751 )
752 .map_err(Error::AllocatePmemDeviceAddress)?;
753
754 vm.add_device_memory(
755 GuestAddress(mapping_address),
756 memory_mapping,
757 /* read_only = */ disk.read_only,
758 /* log_dirty_pages = */ false,
759 )
760 .map_err(Error::AddPmemDeviceMemory)?;
761
762 let dev = virtio::Pmem::new(fd, GuestAddress(mapping_address), image_size)
763 .map_err(Error::PmemDeviceNew)?;
764
765 Ok(VirtioDeviceStub {
766 dev: Box::new(dev) as Box<dyn VirtioDevice>,
767 /// TODO(jstaron) Create separate device policy for pmem_device.
768 jail: simple_jail(&cfg, "block_device.policy")?,
769 })
770 }
771
772 // gpu_device_socket is not used when GPU support is disabled.
773 #[cfg_attr(not(feature = "gpu"), allow(unused_variables))]
create_virtio_devices( cfg: &Config, mem: &GuestMemory, vm: &mut Vm, resources: &mut SystemAllocator, _exit_evt: &EventFd, wayland_device_socket: VmMemoryControlRequestSocket, gpu_device_socket: VmMemoryControlRequestSocket, balloon_device_socket: BalloonControlResponseSocket, disk_device_sockets: &mut Vec<DiskControlResponseSocket>, ) -> DeviceResult<Vec<VirtioDeviceStub>>774 fn create_virtio_devices(
775 cfg: &Config,
776 mem: &GuestMemory,
777 vm: &mut Vm,
778 resources: &mut SystemAllocator,
779 _exit_evt: &EventFd,
780 wayland_device_socket: VmMemoryControlRequestSocket,
781 gpu_device_socket: VmMemoryControlRequestSocket,
782 balloon_device_socket: BalloonControlResponseSocket,
783 disk_device_sockets: &mut Vec<DiskControlResponseSocket>,
784 ) -> DeviceResult<Vec<VirtioDeviceStub>> {
785 let mut devs = Vec::new();
786
787 for disk in &cfg.disks {
788 let disk_device_socket = disk_device_sockets.remove(0);
789 devs.push(create_block_device(cfg, disk, disk_device_socket)?);
790 }
791
792 for (index, pmem_disk) in cfg.pmem_devices.iter().enumerate() {
793 devs.push(create_pmem_device(cfg, vm, resources, pmem_disk, index)?);
794 }
795
796 devs.push(create_rng_device(cfg)?);
797
798 #[cfg(feature = "tpm")]
799 {
800 if cfg.software_tpm {
801 devs.push(create_tpm_device(cfg)?);
802 }
803 }
804
805 if let Some(single_touch_spec) = &cfg.virtio_single_touch {
806 devs.push(create_single_touch_device(cfg, single_touch_spec)?);
807 }
808
809 if let Some(trackpad_spec) = &cfg.virtio_trackpad {
810 devs.push(create_trackpad_device(cfg, trackpad_spec)?);
811 }
812
813 if let Some(mouse_socket) = &cfg.virtio_mouse {
814 devs.push(create_mouse_device(cfg, mouse_socket)?);
815 }
816
817 if let Some(keyboard_socket) = &cfg.virtio_keyboard {
818 devs.push(create_keyboard_device(cfg, keyboard_socket)?);
819 }
820
821 for dev_path in &cfg.virtio_input_evdevs {
822 devs.push(create_vinput_device(cfg, dev_path)?);
823 }
824
825 devs.push(create_balloon_device(cfg, balloon_device_socket)?);
826
827 // We checked above that if the IP is defined, then the netmask is, too.
828 for tap_fd in &cfg.tap_fd {
829 devs.push(create_tap_net_device(cfg, *tap_fd)?);
830 }
831
832 if let (Some(host_ip), Some(netmask), Some(mac_address)) =
833 (cfg.host_ip, cfg.netmask, cfg.mac_address)
834 {
835 devs.push(create_net_device(cfg, host_ip, netmask, mac_address, mem)?);
836 }
837
838 #[cfg_attr(not(feature = "gpu"), allow(unused_mut))]
839 let mut resource_bridge_wl_socket = None::<virtio::resource_bridge::ResourceRequestSocket>;
840
841 #[cfg(feature = "gpu")]
842 {
843 if cfg.gpu {
844 if let Some(wayland_socket_path) = &cfg.wayland_socket_path {
845 let (wl_socket, gpu_socket) =
846 virtio::resource_bridge::pair().map_err(Error::CreateSocket)?;
847 resource_bridge_wl_socket = Some(wl_socket);
848
849 devs.push(create_gpu_device(
850 cfg,
851 _exit_evt,
852 gpu_device_socket,
853 gpu_socket,
854 wayland_socket_path,
855 )?);
856 }
857 }
858 }
859
860 if let Some(wayland_socket_path) = cfg.wayland_socket_path.as_ref() {
861 devs.push(create_wayland_device(
862 cfg,
863 wayland_socket_path,
864 wayland_device_socket,
865 resource_bridge_wl_socket,
866 )?);
867 }
868
869 if let Some(cid) = cfg.cid {
870 devs.push(create_vhost_vsock_device(cfg, cid, mem)?);
871 }
872
873 let chronos = get_chronos_ids();
874
875 for (src, tag) in &cfg.shared_dirs {
876 devs.push(create_9p_device(cfg, chronos, src, tag)?);
877 }
878
879 Ok(devs)
880 }
881
create_devices( cfg: &Config, mem: &GuestMemory, vm: &mut Vm, resources: &mut SystemAllocator, exit_evt: &EventFd, wayland_device_socket: VmMemoryControlRequestSocket, gpu_device_socket: VmMemoryControlRequestSocket, balloon_device_socket: BalloonControlResponseSocket, disk_device_sockets: &mut Vec<DiskControlResponseSocket>, usb_provider: HostBackendDeviceProvider, ) -> DeviceResult<Vec<(Box<dyn PciDevice>, Option<Minijail>)>>882 fn create_devices(
883 cfg: &Config,
884 mem: &GuestMemory,
885 vm: &mut Vm,
886 resources: &mut SystemAllocator,
887 exit_evt: &EventFd,
888 wayland_device_socket: VmMemoryControlRequestSocket,
889 gpu_device_socket: VmMemoryControlRequestSocket,
890 balloon_device_socket: BalloonControlResponseSocket,
891 disk_device_sockets: &mut Vec<DiskControlResponseSocket>,
892 usb_provider: HostBackendDeviceProvider,
893 ) -> DeviceResult<Vec<(Box<dyn PciDevice>, Option<Minijail>)>> {
894 let stubs = create_virtio_devices(
895 &cfg,
896 mem,
897 vm,
898 resources,
899 exit_evt,
900 wayland_device_socket,
901 gpu_device_socket,
902 balloon_device_socket,
903 disk_device_sockets,
904 )?;
905
906 let mut pci_devices = Vec::new();
907
908 for stub in stubs {
909 let dev = VirtioPciDevice::new(mem.clone(), stub.dev).map_err(Error::VirtioPciDev)?;
910 let dev = Box::new(dev) as Box<dyn PciDevice>;
911 pci_devices.push((dev, stub.jail));
912 }
913
914 if cfg.cras_audio {
915 let mut server = Box::new(CrasClient::new().map_err(Error::CreateCrasClient)?);
916 if cfg.cras_capture {
917 server.enable_cras_capture();
918 }
919 let cras_audio = devices::Ac97Dev::new(mem.clone(), server);
920
921 pci_devices.push((
922 Box::new(cras_audio),
923 simple_jail(&cfg, "cras_audio_device.policy")?,
924 ));
925 }
926
927 if cfg.null_audio {
928 let server = Box::new(DummyStreamSource::new());
929 let null_audio = devices::Ac97Dev::new(mem.clone(), server);
930
931 pci_devices.push((
932 Box::new(null_audio),
933 simple_jail(&cfg, "null_audio_device.policy")?,
934 ));
935 }
936 // Create xhci controller.
937 let usb_controller = Box::new(XhciController::new(mem.clone(), usb_provider));
938 pci_devices.push((usb_controller, simple_jail(&cfg, "xhci.policy")?));
939
940 Ok(pci_devices)
941 }
942
943 #[derive(Copy, Clone)]
944 struct Ids {
945 uid: uid_t,
946 gid: gid_t,
947 }
948
get_chronos_ids() -> Ids949 fn get_chronos_ids() -> Ids {
950 let chronos_user_group = CStr::from_bytes_with_nul(b"chronos\0").unwrap();
951
952 let chronos_uid = match get_user_id(&chronos_user_group) {
953 Ok(u) => u,
954 Err(e) => {
955 warn!("falling back to current user id for 9p: {}", e);
956 geteuid()
957 }
958 };
959
960 let chronos_gid = match get_group_id(&chronos_user_group) {
961 Ok(u) => u,
962 Err(e) => {
963 warn!("falling back to current group id for 9p: {}", e);
964 getegid()
965 }
966 };
967
968 Ids {
969 uid: chronos_uid,
970 gid: chronos_gid,
971 }
972 }
973
974 // Set the uid/gid for the jailed process and give a basic id map. This is
975 // required for bind mounts to work.
add_crosvm_user_to_jail(jail: &mut Minijail, feature: &str) -> Result<Ids>976 fn add_crosvm_user_to_jail(jail: &mut Minijail, feature: &str) -> Result<Ids> {
977 let crosvm_user_group = CStr::from_bytes_with_nul(b"crosvm\0").unwrap();
978
979 let crosvm_uid = match get_user_id(&crosvm_user_group) {
980 Ok(u) => u,
981 Err(e) => {
982 warn!("falling back to current user id for {}: {}", feature, e);
983 geteuid()
984 }
985 };
986
987 let crosvm_gid = match get_group_id(&crosvm_user_group) {
988 Ok(u) => u,
989 Err(e) => {
990 warn!("falling back to current group id for {}: {}", feature, e);
991 getegid()
992 }
993 };
994
995 jail.change_uid(crosvm_uid);
996 jail.change_gid(crosvm_gid);
997 jail.uidmap(&format!("{0} {0} 1", crosvm_uid))
998 .map_err(Error::SettingUidMap)?;
999 jail.gidmap(&format!("{0} {0} 1", crosvm_gid))
1000 .map_err(Error::SettingGidMap)?;
1001
1002 Ok(Ids {
1003 uid: crosvm_uid,
1004 gid: crosvm_gid,
1005 })
1006 }
1007
raw_fd_from_path(path: &Path) -> Result<RawFd>1008 fn raw_fd_from_path(path: &Path) -> Result<RawFd> {
1009 if !path.is_file() {
1010 return Err(Error::InvalidFdPath);
1011 }
1012 let raw_fd = path
1013 .file_name()
1014 .and_then(|fd_osstr| fd_osstr.to_str())
1015 .and_then(|fd_str| fd_str.parse::<c_int>().ok())
1016 .ok_or(Error::InvalidFdPath)?;
1017 validate_raw_fd(raw_fd).map_err(Error::ValidateRawFd)
1018 }
1019
create_input_socket(path: &Path) -> Result<UnixStream>1020 fn create_input_socket(path: &Path) -> Result<UnixStream> {
1021 if path.parent() == Some(Path::new("/proc/self/fd")) {
1022 // Safe because we will validate |raw_fd|.
1023 unsafe { Ok(UnixStream::from_raw_fd(raw_fd_from_path(path)?)) }
1024 } else {
1025 UnixStream::connect(path).map_err(Error::InputEventsOpen)
1026 }
1027 }
1028
setup_vcpu_signal_handler() -> Result<()>1029 fn setup_vcpu_signal_handler() -> Result<()> {
1030 unsafe {
1031 extern "C" fn handle_signal() {}
1032 // Our signal handler does nothing and is trivially async signal safe.
1033 register_signal_handler(SIGRTMIN() + 0, handle_signal)
1034 .map_err(Error::RegisterSignalHandler)?;
1035 }
1036 block_signal(SIGRTMIN() + 0).map_err(Error::BlockSignal)?;
1037 Ok(())
1038 }
1039
1040 #[derive(Default)]
1041 struct VcpuRunMode {
1042 mtx: Mutex<VmRunMode>,
1043 cvar: Condvar,
1044 }
1045
1046 impl VcpuRunMode {
set_and_notify(&self, new_mode: VmRunMode)1047 fn set_and_notify(&self, new_mode: VmRunMode) {
1048 *self.mtx.lock() = new_mode;
1049 self.cvar.notify_all();
1050 }
1051 }
1052
run_vcpu( vcpu: Vcpu, cpu_id: u32, vcpu_affinity: Vec<usize>, start_barrier: Arc<Barrier>, io_bus: devices::Bus, mmio_bus: devices::Bus, exit_evt: EventFd, requires_kvmclock_ctrl: bool, run_mode_arc: Arc<VcpuRunMode>, ) -> Result<JoinHandle<()>>1053 fn run_vcpu(
1054 vcpu: Vcpu,
1055 cpu_id: u32,
1056 vcpu_affinity: Vec<usize>,
1057 start_barrier: Arc<Barrier>,
1058 io_bus: devices::Bus,
1059 mmio_bus: devices::Bus,
1060 exit_evt: EventFd,
1061 requires_kvmclock_ctrl: bool,
1062 run_mode_arc: Arc<VcpuRunMode>,
1063 ) -> Result<JoinHandle<()>> {
1064 thread::Builder::new()
1065 .name(format!("crosvm_vcpu{}", cpu_id))
1066 .spawn(move || {
1067 if vcpu_affinity.len() != 0 {
1068 if let Err(e) = set_cpu_affinity(vcpu_affinity) {
1069 error!("Failed to set CPU affinity: {}", e);
1070 }
1071 }
1072
1073 let mut sig_ok = true;
1074 match get_blocked_signals() {
1075 Ok(mut v) => {
1076 v.retain(|&x| x != SIGRTMIN() + 0);
1077 if let Err(e) = vcpu.set_signal_mask(&v) {
1078 error!(
1079 "Failed to set the KVM_SIGNAL_MASK for vcpu {} : {}",
1080 cpu_id, e
1081 );
1082 sig_ok = false;
1083 }
1084 }
1085 Err(e) => {
1086 error!(
1087 "Failed to retrieve signal mask for vcpu {} : {}",
1088 cpu_id, e
1089 );
1090 sig_ok = false;
1091 }
1092 };
1093
1094 start_barrier.wait();
1095
1096 if sig_ok {
1097 'vcpu_loop: loop {
1098 let mut interrupted_by_signal = false;
1099 match vcpu.run() {
1100 Ok(VcpuExit::IoIn { port, mut size }) => {
1101 let mut data = [0; 8];
1102 if size > data.len() {
1103 error!("unsupported IoIn size of {} bytes", size);
1104 size = data.len();
1105 }
1106 io_bus.read(port as u64, &mut data[..size]);
1107 if let Err(e) = vcpu.set_data(&data[..size]) {
1108 error!("failed to set return data for IoIn: {}", e);
1109 }
1110 }
1111 Ok(VcpuExit::IoOut {
1112 port,
1113 mut size,
1114 data,
1115 }) => {
1116 if size > data.len() {
1117 error!("unsupported IoOut size of {} bytes", size);
1118 size = data.len();
1119 }
1120 io_bus.write(port as u64, &data[..size]);
1121 }
1122 Ok(VcpuExit::MmioRead { address, size }) => {
1123 let mut data = [0; 8];
1124 mmio_bus.read(address, &mut data[..size]);
1125 // Setting data for mmio can not fail.
1126 let _ = vcpu.set_data(&data[..size]);
1127 }
1128 Ok(VcpuExit::MmioWrite {
1129 address,
1130 size,
1131 data,
1132 }) => {
1133 mmio_bus.write(address, &data[..size]);
1134 }
1135 Ok(VcpuExit::Hlt) => break,
1136 Ok(VcpuExit::Shutdown) => break,
1137 Ok(VcpuExit::SystemEvent(_, _)) => break,
1138 Ok(r) => warn!("unexpected vcpu exit: {:?}", r),
1139 Err(e) => match e.errno() {
1140 libc::EINTR => interrupted_by_signal = true,
1141 libc::EAGAIN => {}
1142 _ => {
1143 error!("vcpu hit unknown error: {}", e);
1144 break;
1145 }
1146 },
1147 }
1148
1149 if interrupted_by_signal {
1150 // Try to clear the signal that we use to kick VCPU if it is pending before
1151 // attempting to handle pause requests.
1152 if let Err(e) = clear_signal(SIGRTMIN() + 0) {
1153 error!("failed to clear pending signal: {}", e);
1154 break;
1155 }
1156 let mut run_mode_lock = run_mode_arc.mtx.lock();
1157 loop {
1158 match *run_mode_lock {
1159 VmRunMode::Running => break,
1160 VmRunMode::Suspending => {
1161 // On KVM implementations that use a paravirtualized clock (e.g.
1162 // x86), a flag must be set to indicate to the guest kernel that
1163 // a VCPU was suspended. The guest kernel will use this flag to
1164 // prevent the soft lockup detection from triggering when this
1165 // VCPU resumes, which could happen days later in realtime.
1166 if requires_kvmclock_ctrl {
1167 if let Err(e) = vcpu.kvmclock_ctrl() {
1168 error!("failed to signal to kvm that vcpu {} is being suspended: {}", cpu_id, e);
1169 }
1170 }
1171 }
1172 VmRunMode::Exiting => break 'vcpu_loop,
1173 }
1174 // Give ownership of our exclusive lock to the condition variable that
1175 // will block. When the condition variable is notified, `wait` will
1176 // unblock and return a new exclusive lock.
1177 run_mode_lock = run_mode_arc.cvar.wait(run_mode_lock);
1178 }
1179 }
1180 }
1181 }
1182 exit_evt
1183 .write(1)
1184 .expect("failed to signal vcpu exit eventfd");
1185 })
1186 .map_err(Error::SpawnVcpu)
1187 }
1188
1189 // Reads the contents of a file and converts the space-separated fields into a Vec of u64s.
1190 // Returns an error if any of the fields fail to parse.
file_fields_to_u64<P: AsRef<Path>>(path: P) -> io::Result<Vec<u64>>1191 fn file_fields_to_u64<P: AsRef<Path>>(path: P) -> io::Result<Vec<u64>> {
1192 let mut file = File::open(path)?;
1193
1194 let mut buf = [0u8; 32];
1195 let count = file.read(&mut buf)?;
1196
1197 let content =
1198 str::from_utf8(&buf[..count]).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
1199 content
1200 .trim()
1201 .split_whitespace()
1202 .map(|x| {
1203 x.parse::<u64>()
1204 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
1205 })
1206 .collect()
1207 }
1208
1209 // Reads the contents of a file and converts them into a u64, and if there
1210 // are multiple fields it only returns the first one.
file_to_u64<P: AsRef<Path>>(path: P) -> io::Result<u64>1211 fn file_to_u64<P: AsRef<Path>>(path: P) -> io::Result<u64> {
1212 file_fields_to_u64(path)?
1213 .into_iter()
1214 .next()
1215 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "empty file"))
1216 }
1217
run_config(cfg: Config) -> Result<()>1218 pub fn run_config(cfg: Config) -> Result<()> {
1219 if cfg.sandbox {
1220 // Printing something to the syslog before entering minijail so that libc's syslogger has a
1221 // chance to open files necessary for its operation, like `/etc/localtime`. After jailing,
1222 // access to those files will not be possible.
1223 info!("crosvm entering multiprocess mode");
1224 }
1225
1226 let (usb_control_socket, usb_provider) =
1227 HostBackendDeviceProvider::new().map_err(Error::CreateUsbProvider)?;
1228 // Masking signals is inherently dangerous, since this can persist across clones/execs. Do this
1229 // before any jailed devices have been spawned, so that we can catch any of them that fail very
1230 // quickly.
1231 let sigchld_fd = SignalFd::new(libc::SIGCHLD).map_err(Error::CreateSignalFd)?;
1232
1233 let initrd_image = if let Some(initrd_path) = &cfg.initrd_path {
1234 Some(File::open(initrd_path).map_err(|e| Error::OpenInitrd(initrd_path.clone(), e))?)
1235 } else {
1236 None
1237 };
1238
1239 let vm_image = match cfg.executable_path {
1240 Some(Executable::Kernel(ref kernel_path)) => VmImage::Kernel(
1241 File::open(kernel_path).map_err(|e| Error::OpenKernel(kernel_path.to_path_buf(), e))?,
1242 ),
1243 Some(Executable::Bios(ref bios_path)) => VmImage::Bios(
1244 File::open(bios_path).map_err(|e| Error::OpenBios(bios_path.to_path_buf(), e))?,
1245 ),
1246 _ => panic!("Did not receive a bios or kernel, should be impossible."),
1247 };
1248
1249 let components = VmComponents {
1250 memory_size: (cfg.memory.unwrap_or(256) << 20) as u64,
1251 vcpu_count: cfg.vcpu_count.unwrap_or(1),
1252 vcpu_affinity: cfg.vcpu_affinity.clone(),
1253 vm_image,
1254 android_fstab: cfg
1255 .android_fstab
1256 .as_ref()
1257 .map(|x| File::open(x).map_err(|e| Error::OpenAndroidFstab(x.to_path_buf(), e)))
1258 .map_or(Ok(None), |v| v.map(Some))?,
1259 initrd_image,
1260 extra_kernel_params: cfg.params.clone(),
1261 wayland_dmabuf: cfg.wayland_dmabuf,
1262 };
1263
1264 let control_server_socket = match &cfg.socket_path {
1265 Some(path) => Some(UnlinkUnixSeqpacketListener(
1266 UnixSeqpacketListener::bind(path).map_err(Error::CreateSocket)?,
1267 )),
1268 None => None,
1269 };
1270
1271 let mut control_sockets = Vec::new();
1272 let (wayland_host_socket, wayland_device_socket) =
1273 msg_socket::pair::<VmMemoryResponse, VmMemoryRequest>().map_err(Error::CreateSocket)?;
1274 control_sockets.push(TaggedControlSocket::VmMemory(wayland_host_socket));
1275 // Balloon gets a special socket so balloon requests can be forwarded from the main process.
1276 let (balloon_host_socket, balloon_device_socket) =
1277 msg_socket::pair::<BalloonControlCommand, ()>().map_err(Error::CreateSocket)?;
1278
1279 // Create one control socket per disk.
1280 let mut disk_device_sockets = Vec::new();
1281 let mut disk_host_sockets = Vec::new();
1282 let disk_count = cfg.disks.len();
1283 for _ in 0..disk_count {
1284 let (disk_host_socket, disk_device_socket) =
1285 msg_socket::pair::<DiskControlCommand, DiskControlResult>()
1286 .map_err(Error::CreateSocket)?;
1287 disk_host_sockets.push(disk_host_socket);
1288 disk_device_sockets.push(disk_device_socket);
1289 }
1290
1291 let (gpu_host_socket, gpu_device_socket) =
1292 msg_socket::pair::<VmMemoryResponse, VmMemoryRequest>().map_err(Error::CreateSocket)?;
1293 control_sockets.push(TaggedControlSocket::VmMemory(gpu_host_socket));
1294
1295 let sandbox = cfg.sandbox;
1296 let linux = Arch::build_vm(
1297 components,
1298 cfg.split_irqchip,
1299 &cfg.serial_parameters,
1300 |mem, vm, sys_allocator, exit_evt| {
1301 create_devices(
1302 &cfg,
1303 mem,
1304 vm,
1305 sys_allocator,
1306 exit_evt,
1307 wayland_device_socket,
1308 gpu_device_socket,
1309 balloon_device_socket,
1310 &mut disk_device_sockets,
1311 usb_provider,
1312 )
1313 },
1314 )
1315 .map_err(Error::BuildVm)?;
1316
1317 let _render_node_host = ();
1318 #[cfg(feature = "gpu-forward")]
1319 let (_render_node_host, linux) = {
1320 // Rebinds linux as mutable.
1321 let mut linux = linux;
1322
1323 // Reserve memory range for GPU buffer allocation in advance to bypass region count
1324 // limitation. We use mremap/MAP_FIXED later to make sure GPU buffers fall into this range.
1325 let gpu_mmap =
1326 MemoryMapping::new_protection(RENDER_NODE_HOST_SIZE as usize, Protection::none())
1327 .map_err(Error::ReserveGpuMemory)?;
1328
1329 // Put the non-accessible memory map into device memory so that no other devices use that
1330 // guest address space.
1331 let gpu_addr = linux
1332 .resources
1333 .device_allocator()
1334 .allocate(
1335 RENDER_NODE_HOST_SIZE,
1336 Alloc::GpuRenderNode,
1337 "gpu_render_node".to_string(),
1338 )
1339 .map_err(|_| Error::AllocateGpuDeviceAddress)?;
1340
1341 let host = RenderNodeHost::start(&gpu_mmap, gpu_addr, linux.vm.get_memory().clone());
1342
1343 // Makes the gpu memory accessible at allocated address.
1344 linux
1345 .vm
1346 .add_device_memory(
1347 GuestAddress(gpu_addr),
1348 gpu_mmap,
1349 /* read_only = */ false,
1350 /* log_dirty_pages = */ false,
1351 )
1352 .map_err(Error::AddGpuDeviceMemory)?;
1353 (host, linux)
1354 };
1355
1356 run_control(
1357 linux,
1358 control_server_socket,
1359 control_sockets,
1360 balloon_host_socket,
1361 &disk_host_sockets,
1362 usb_control_socket,
1363 sigchld_fd,
1364 _render_node_host,
1365 sandbox,
1366 )
1367 }
1368
run_control( mut linux: RunnableLinuxVm, control_server_socket: Option<UnlinkUnixSeqpacketListener>, mut control_sockets: Vec<TaggedControlSocket>, balloon_host_socket: BalloonControlRequestSocket, disk_host_sockets: &[DiskControlRequestSocket], usb_control_socket: UsbControlSocket, sigchld_fd: SignalFd, _render_node_host: RenderNodeHost, sandbox: bool, ) -> Result<()>1369 fn run_control(
1370 mut linux: RunnableLinuxVm,
1371 control_server_socket: Option<UnlinkUnixSeqpacketListener>,
1372 mut control_sockets: Vec<TaggedControlSocket>,
1373 balloon_host_socket: BalloonControlRequestSocket,
1374 disk_host_sockets: &[DiskControlRequestSocket],
1375 usb_control_socket: UsbControlSocket,
1376 sigchld_fd: SignalFd,
1377 _render_node_host: RenderNodeHost,
1378 sandbox: bool,
1379 ) -> Result<()> {
1380 // Paths to get the currently available memory and the low memory threshold.
1381 const LOWMEM_MARGIN: &str = "/sys/kernel/mm/chromeos-low_mem/margin";
1382 const LOWMEM_AVAILABLE: &str = "/sys/kernel/mm/chromeos-low_mem/available";
1383
1384 // The amount of additional memory to claim back from the VM whenever the system is
1385 // low on memory.
1386 const ONE_GB: u64 = (1 << 30);
1387
1388 let max_balloon_memory = match linux.vm.get_memory().memory_size() {
1389 // If the VM has at least 1.5 GB, the balloon driver can consume all but the last 1 GB.
1390 n if n >= (ONE_GB / 2) * 3 => n - ONE_GB,
1391 // Otherwise, if the VM has at least 500MB the balloon driver will consume at most
1392 // half of it.
1393 n if n >= (ONE_GB / 2) => n / 2,
1394 // Otherwise, the VM is too small for us to take memory away from it.
1395 _ => 0,
1396 };
1397 let mut current_balloon_memory: u64 = 0;
1398 let balloon_memory_increment: u64 = max_balloon_memory / 16;
1399
1400 #[derive(PollToken)]
1401 enum Token {
1402 Exit,
1403 Stdin,
1404 ChildSignal,
1405 CheckAvailableMemory,
1406 LowMemory,
1407 LowmemTimer,
1408 VmControlServer,
1409 VmControl { index: usize },
1410 }
1411
1412 let stdin_handle = stdin();
1413 let stdin_lock = stdin_handle.lock();
1414 stdin_lock
1415 .set_raw_mode()
1416 .expect("failed to set terminal raw mode");
1417
1418 let poll_ctx = PollContext::new().map_err(Error::CreatePollContext)?;
1419 poll_ctx
1420 .add(&linux.exit_evt, Token::Exit)
1421 .map_err(Error::PollContextAdd)?;
1422 if let Err(e) = poll_ctx.add(&stdin_handle, Token::Stdin) {
1423 warn!("failed to add stdin to poll context: {}", e);
1424 }
1425 poll_ctx
1426 .add(&sigchld_fd, Token::ChildSignal)
1427 .map_err(Error::PollContextAdd)?;
1428
1429 if let Some(socket_server) = &control_server_socket {
1430 poll_ctx
1431 .add(socket_server, Token::VmControlServer)
1432 .map_err(Error::PollContextAdd)?;
1433 }
1434 for (index, socket) in control_sockets.iter().enumerate() {
1435 poll_ctx
1436 .add(socket.as_ref(), Token::VmControl { index })
1437 .map_err(Error::PollContextAdd)?;
1438 }
1439
1440 // Watch for low memory notifications and take memory back from the VM.
1441 let low_mem = File::open("/dev/chromeos-low-mem").ok();
1442 if let Some(low_mem) = &low_mem {
1443 poll_ctx
1444 .add(low_mem, Token::LowMemory)
1445 .map_err(Error::PollContextAdd)?;
1446 } else {
1447 warn!("Unable to open low mem indicator, maybe not a chrome os kernel");
1448 }
1449
1450 // Used to rate limit balloon requests.
1451 let mut lowmem_timer = TimerFd::new().map_err(Error::CreateTimerFd)?;
1452 poll_ctx
1453 .add(&lowmem_timer, Token::LowmemTimer)
1454 .map_err(Error::PollContextAdd)?;
1455
1456 // Used to check whether it's ok to start giving memory back to the VM.
1457 let mut freemem_timer = TimerFd::new().map_err(Error::CreateTimerFd)?;
1458 poll_ctx
1459 .add(&freemem_timer, Token::CheckAvailableMemory)
1460 .map_err(Error::PollContextAdd)?;
1461
1462 // Used to add jitter to timer values so that we don't have a thundering herd problem when
1463 // multiple VMs are running.
1464 let mut simple_rng = SimpleRng::new(
1465 SystemTime::now()
1466 .duration_since(UNIX_EPOCH)
1467 .expect("time went backwards")
1468 .subsec_nanos() as u64,
1469 );
1470
1471 if sandbox {
1472 // Before starting VCPUs, in case we started with some capabilities, drop them all.
1473 drop_capabilities().map_err(Error::DropCapabilities)?;
1474 }
1475
1476 let mut vcpu_handles = Vec::with_capacity(linux.vcpus.len());
1477 let vcpu_thread_barrier = Arc::new(Barrier::new(linux.vcpus.len() + 1));
1478 let run_mode_arc = Arc::new(VcpuRunMode::default());
1479 setup_vcpu_signal_handler()?;
1480 for (cpu_id, vcpu) in linux.vcpus.into_iter().enumerate() {
1481 let handle = run_vcpu(
1482 vcpu,
1483 cpu_id as u32,
1484 linux.vcpu_affinity.clone(),
1485 vcpu_thread_barrier.clone(),
1486 linux.io_bus.clone(),
1487 linux.mmio_bus.clone(),
1488 linux.exit_evt.try_clone().map_err(Error::CloneEventFd)?,
1489 linux.vm.check_extension(Cap::KvmclockCtrl),
1490 run_mode_arc.clone(),
1491 )?;
1492 vcpu_handles.push(handle);
1493 }
1494 vcpu_thread_barrier.wait();
1495
1496 'poll: loop {
1497 let events = {
1498 match poll_ctx.wait() {
1499 Ok(v) => v,
1500 Err(e) => {
1501 error!("failed to poll: {}", e);
1502 break;
1503 }
1504 }
1505 };
1506
1507 let mut vm_control_indices_to_remove = Vec::new();
1508 for event in events.iter_readable() {
1509 match event.token() {
1510 Token::Exit => {
1511 info!("vcpu requested shutdown");
1512 break 'poll;
1513 }
1514 Token::Stdin => {
1515 let mut out = [0u8; 64];
1516 match stdin_lock.read_raw(&mut out[..]) {
1517 Ok(0) => {
1518 // Zero-length read indicates EOF. Remove from pollables.
1519 let _ = poll_ctx.delete(&stdin_handle);
1520 }
1521 Err(e) => {
1522 warn!("error while reading stdin: {}", e);
1523 let _ = poll_ctx.delete(&stdin_handle);
1524 }
1525 Ok(count) => {
1526 if let Some(ref stdio_serial) = linux.stdio_serial {
1527 stdio_serial
1528 .lock()
1529 .queue_input_bytes(&out[..count])
1530 .expect("failed to queue bytes into serial port");
1531 }
1532 }
1533 }
1534 }
1535 Token::ChildSignal => {
1536 // Print all available siginfo structs, then exit the loop.
1537 while let Some(siginfo) = sigchld_fd.read().map_err(Error::SignalFd)? {
1538 let pid = siginfo.ssi_pid;
1539 let pid_label = match linux.pid_debug_label_map.get(&pid) {
1540 Some(label) => format!("{} (pid {})", label, pid),
1541 None => format!("pid {}", pid),
1542 };
1543 error!(
1544 "child {} died: signo {}, status {}, code {}",
1545 pid_label, siginfo.ssi_signo, siginfo.ssi_status, siginfo.ssi_code
1546 );
1547 }
1548 break 'poll;
1549 }
1550 Token::CheckAvailableMemory => {
1551 // Acknowledge the timer.
1552 freemem_timer.wait().map_err(Error::TimerFd)?;
1553 if current_balloon_memory == 0 {
1554 // Nothing to see here.
1555 if let Err(e) = freemem_timer.clear() {
1556 warn!("unable to clear available memory check timer: {}", e);
1557 }
1558 continue;
1559 }
1560
1561 // Otherwise see if we can free up some memory.
1562 let margin = file_to_u64(LOWMEM_MARGIN).map_err(Error::ReadLowmemMargin)?;
1563 let available =
1564 file_to_u64(LOWMEM_AVAILABLE).map_err(Error::ReadLowmemAvailable)?;
1565
1566 // `available` and `margin` are specified in MB while `balloon_memory_increment` is in
1567 // bytes. So to correctly compare them we need to turn the increment value into MB.
1568 if available >= margin + 2 * (balloon_memory_increment >> 20) {
1569 current_balloon_memory =
1570 if current_balloon_memory >= balloon_memory_increment {
1571 current_balloon_memory - balloon_memory_increment
1572 } else {
1573 0
1574 };
1575 let command = BalloonControlCommand::Adjust {
1576 num_bytes: current_balloon_memory,
1577 };
1578 if let Err(e) = balloon_host_socket.send(&command) {
1579 warn!("failed to send memory value to balloon device: {}", e);
1580 }
1581 }
1582 }
1583 Token::LowMemory => {
1584 if let Some(low_mem) = &low_mem {
1585 let old_balloon_memory = current_balloon_memory;
1586 current_balloon_memory = min(
1587 current_balloon_memory + balloon_memory_increment,
1588 max_balloon_memory,
1589 );
1590 if current_balloon_memory != old_balloon_memory {
1591 let command = BalloonControlCommand::Adjust {
1592 num_bytes: current_balloon_memory,
1593 };
1594 if let Err(e) = balloon_host_socket.send(&command) {
1595 warn!("failed to send memory value to balloon device: {}", e);
1596 }
1597 }
1598
1599 // Stop polling the lowmem device until the timer fires.
1600 poll_ctx.delete(low_mem).map_err(Error::PollContextDelete)?;
1601
1602 // Add some jitter to the timer so that if there are multiple VMs running
1603 // they don't all start ballooning at exactly the same time.
1604 let lowmem_dur = Duration::from_millis(1000 + simple_rng.rng() % 200);
1605 lowmem_timer
1606 .reset(lowmem_dur, None)
1607 .map_err(Error::ResetTimerFd)?;
1608
1609 // Also start a timer to check when we can start giving memory back. Do the
1610 // first check after a minute (with jitter) and subsequent checks after
1611 // every 30 seconds (with jitter).
1612 let freemem_dur = Duration::from_secs(60 + simple_rng.rng() % 12);
1613 let freemem_int = Duration::from_secs(30 + simple_rng.rng() % 6);
1614 freemem_timer
1615 .reset(freemem_dur, Some(freemem_int))
1616 .map_err(Error::ResetTimerFd)?;
1617 }
1618 }
1619 Token::LowmemTimer => {
1620 // Acknowledge the timer.
1621 lowmem_timer.wait().map_err(Error::TimerFd)?;
1622
1623 if let Some(low_mem) = &low_mem {
1624 // Start polling the lowmem device again.
1625 poll_ctx
1626 .add(low_mem, Token::LowMemory)
1627 .map_err(Error::PollContextAdd)?;
1628 }
1629 }
1630 Token::VmControlServer => {
1631 if let Some(socket_server) = &control_server_socket {
1632 match socket_server.accept() {
1633 Ok(socket) => {
1634 poll_ctx
1635 .add(
1636 &socket,
1637 Token::VmControl {
1638 index: control_sockets.len(),
1639 },
1640 )
1641 .map_err(Error::PollContextAdd)?;
1642 control_sockets
1643 .push(TaggedControlSocket::Vm(MsgSocket::new(socket)));
1644 }
1645 Err(e) => error!("failed to accept socket: {}", e),
1646 }
1647 }
1648 }
1649 Token::VmControl { index } => {
1650 if let Some(socket) = control_sockets.get(index) {
1651 match socket {
1652 TaggedControlSocket::Vm(socket) => match socket.recv() {
1653 Ok(request) => {
1654 let mut run_mode_opt = None;
1655 let response = request.execute(
1656 &mut run_mode_opt,
1657 &balloon_host_socket,
1658 disk_host_sockets,
1659 &usb_control_socket,
1660 );
1661 if let Err(e) = socket.send(&response) {
1662 error!("failed to send VmResponse: {}", e);
1663 }
1664 if let Some(run_mode) = run_mode_opt {
1665 info!("control socket changed run mode to {}", run_mode);
1666 match run_mode {
1667 VmRunMode::Exiting => {
1668 break 'poll;
1669 }
1670 other => {
1671 run_mode_arc.set_and_notify(other);
1672 for handle in &vcpu_handles {
1673 let _ = handle.kill(SIGRTMIN() + 0);
1674 }
1675 }
1676 }
1677 }
1678 }
1679 Err(e) => {
1680 if let MsgError::BadRecvSize { actual: 0, .. } = e {
1681 vm_control_indices_to_remove.push(index);
1682 } else {
1683 error!("failed to recv VmRequest: {}", e);
1684 }
1685 }
1686 },
1687 TaggedControlSocket::VmMemory(socket) => match socket.recv() {
1688 Ok(request) => {
1689 let response =
1690 request.execute(&mut linux.vm, &mut linux.resources);
1691 if let Err(e) = socket.send(&response) {
1692 error!("failed to send VmMemoryControlResponse: {}", e);
1693 }
1694 }
1695 Err(e) => {
1696 if let MsgError::BadRecvSize { actual: 0, .. } = e {
1697 vm_control_indices_to_remove.push(index);
1698 } else {
1699 error!("failed to recv VmMemoryControlRequest: {}", e);
1700 }
1701 }
1702 },
1703 }
1704 }
1705 }
1706 }
1707 }
1708
1709 for event in events.iter_hungup() {
1710 match event.token() {
1711 Token::Exit => {}
1712 Token::Stdin => {
1713 let _ = poll_ctx.delete(&stdin_handle);
1714 }
1715 Token::ChildSignal => {}
1716 Token::CheckAvailableMemory => {}
1717 Token::LowMemory => {}
1718 Token::LowmemTimer => {}
1719 Token::VmControlServer => {}
1720 Token::VmControl { index } => {
1721 // It's possible more data is readable and buffered while the socket is hungup,
1722 // so don't delete the socket from the poll context until we're sure all the
1723 // data is read.
1724 match control_sockets
1725 .get(index)
1726 .map(|s| s.as_ref().get_readable_bytes())
1727 {
1728 Some(Ok(0)) | Some(Err(_)) => vm_control_indices_to_remove.push(index),
1729 Some(Ok(x)) => info!("control index {} has {} bytes readable", index, x),
1730 _ => {}
1731 }
1732 }
1733 }
1734 }
1735
1736 // Sort in reverse so the highest indexes are removed first. This removal algorithm
1737 // preserved correct indexes as each element is removed.
1738 vm_control_indices_to_remove.sort_unstable_by(|a, b| b.cmp(a));
1739 vm_control_indices_to_remove.dedup();
1740 for index in vm_control_indices_to_remove {
1741 control_sockets.swap_remove(index);
1742 if let Some(socket) = control_sockets.get(index) {
1743 poll_ctx
1744 .modify(
1745 socket,
1746 WatchingEvents::empty().set_read(),
1747 Token::VmControl { index },
1748 )
1749 .map_err(Error::PollContextAdd)?;
1750 }
1751 }
1752 }
1753
1754 // VCPU threads MUST see the VmRunMode flag, otherwise they may re-enter the VM.
1755 run_mode_arc.set_and_notify(VmRunMode::Exiting);
1756 for handle in vcpu_handles {
1757 match handle.kill(SIGRTMIN() + 0) {
1758 Ok(_) => {
1759 if let Err(e) = handle.join() {
1760 error!("failed to join vcpu thread: {:?}", e);
1761 }
1762 }
1763 Err(e) => error!("failed to kill vcpu thread: {}", e),
1764 }
1765 }
1766
1767 stdin_lock
1768 .set_canon_mode()
1769 .expect("failed to restore canonical mode for terminal");
1770
1771 Ok(())
1772 }
1773