1 // Copyright 2017 The ChromiumOS Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //! Handles IPC for controlling the main VM process.
6 //!
7 //! The VM Control IPC protocol is synchronous, meaning that each `VmRequest` sent over a connection
8 //! will receive a `VmResponse` for that request next time data is received over that connection.
9 //!
10 //! The wire message format is a little-endian C-struct of fixed size, along with a file descriptor
11 //! if the request type expects one.
12
13 pub mod api;
14 #[cfg(feature = "gdb")]
15 pub mod gdb;
16 #[cfg(feature = "gpu")]
17 pub mod gpu;
18
19 #[cfg(any(target_os = "android", target_os = "linux"))]
20 use base::linux::MemoryMappingBuilderUnix;
21 #[cfg(windows)]
22 use base::MemoryMappingBuilderWindows;
23 use hypervisor::BalloonEvent;
24 use hypervisor::MemCacheType;
25 use hypervisor::MemRegion;
26
27 #[cfg(feature = "balloon")]
28 mod balloon_tube;
29 pub mod client;
30 mod snapshot_format;
31 pub mod sys;
32
33 #[cfg(target_arch = "x86_64")]
34 use std::arch::x86_64::_rdtsc;
35 use std::collections::BTreeMap;
36 use std::collections::BTreeSet;
37 use std::collections::HashMap;
38 use std::convert::TryInto;
39 use std::fmt;
40 use std::fmt::Display;
41 use std::fs::File;
42 use std::path::PathBuf;
43 use std::result::Result as StdResult;
44 use std::str::FromStr;
45 use std::sync::mpsc;
46 use std::sync::Arc;
47
48 use anyhow::bail;
49 use anyhow::Context;
50 use base::error;
51 use base::info;
52 use base::warn;
53 use base::with_as_descriptor;
54 use base::AsRawDescriptor;
55 use base::Descriptor;
56 use base::Error as SysError;
57 use base::Event;
58 use base::ExternalMapping;
59 use base::IntoRawDescriptor;
60 use base::MappedRegion;
61 use base::MemoryMappingBuilder;
62 use base::MmapError;
63 use base::Protection;
64 use base::Result;
65 use base::SafeDescriptor;
66 use base::SharedMemory;
67 use base::Tube;
68 use hypervisor::Datamatch;
69 use hypervisor::IoEventAddress;
70 use hypervisor::IrqRoute;
71 use hypervisor::IrqSource;
72 pub use hypervisor::MemSlot;
73 use hypervisor::Vm;
74 use hypervisor::VmCap;
75 use libc::EINVAL;
76 use libc::EIO;
77 use libc::ENODEV;
78 use libc::ENOTSUP;
79 use libc::ERANGE;
80 #[cfg(feature = "registered_events")]
81 use protos::registered_events;
82 use remain::sorted;
83 use resources::Alloc;
84 use resources::SystemAllocator;
85 use rutabaga_gfx::DeviceId;
86 use rutabaga_gfx::RutabagaDescriptor;
87 use rutabaga_gfx::RutabagaFromRawDescriptor;
88 use rutabaga_gfx::RutabagaGralloc;
89 use rutabaga_gfx::RutabagaHandle;
90 use rutabaga_gfx::RutabagaMappedRegion;
91 use rutabaga_gfx::VulkanInfo;
92 use serde::Deserialize;
93 use serde::Serialize;
94 pub use snapshot_format::*;
95 use swap::SwapStatus;
96 use sync::Mutex;
97 #[cfg(any(target_os = "android", target_os = "linux"))]
98 pub use sys::FsMappingRequest;
99 #[cfg(any(target_os = "android", target_os = "linux"))]
100 pub use sys::VmMsyncRequest;
101 #[cfg(any(target_os = "android", target_os = "linux"))]
102 pub use sys::VmMsyncResponse;
103 use thiserror::Error;
104 pub use vm_control_product::GpuSendToMain;
105 pub use vm_control_product::GpuSendToService;
106 pub use vm_control_product::ServiceSendToGpu;
107 use vm_memory::GuestAddress;
108
109 #[cfg(feature = "balloon")]
110 pub use crate::balloon_tube::*;
111 #[cfg(feature = "gdb")]
112 pub use crate::gdb::VcpuDebug;
113 #[cfg(feature = "gdb")]
114 pub use crate::gdb::VcpuDebugStatus;
115 #[cfg(feature = "gdb")]
116 pub use crate::gdb::VcpuDebugStatusMessage;
117 #[cfg(feature = "gpu")]
118 use crate::gpu::GpuControlCommand;
119 #[cfg(feature = "gpu")]
120 use crate::gpu::GpuControlResult;
121
122 /// Control the state of a particular VM CPU.
123 #[derive(Clone, Debug)]
124 pub enum VcpuControl {
125 #[cfg(feature = "gdb")]
126 Debug(VcpuDebug),
127 RunState(VmRunMode),
128 MakeRT,
129 // Request the current state of the vCPU. The result is sent back over the included channel.
130 GetStates(mpsc::Sender<VmRunMode>),
131 // Request the vcpu write a snapshot of itself to the writer, then send a `Result` back over
132 // the channel after completion/failure.
133 Snapshot(SnapshotWriter, mpsc::Sender<anyhow::Result<()>>),
134 Restore(VcpuRestoreRequest),
135 }
136
137 /// Request to restore a Vcpu from a given snapshot, and report the results
138 /// back via the provided channel.
139 #[derive(Clone, Debug)]
140 pub struct VcpuRestoreRequest {
141 pub result_sender: mpsc::Sender<anyhow::Result<()>>,
142 pub snapshot_reader: SnapshotReader,
143 #[cfg(target_arch = "x86_64")]
144 pub host_tsc_reference_moment: u64,
145 }
146
147 /// Mode of execution for the VM.
148 #[derive(Debug, Copy, Clone, Default, PartialEq, Eq)]
149 pub enum VmRunMode {
150 /// The default run mode indicating the VCPUs are running.
151 #[default]
152 Running,
153 /// Indicates that the VCPUs are suspending execution until the `Running` mode is set.
154 Suspending,
155 /// Indicates that the VM is exiting all processes.
156 Exiting,
157 /// Indicates that the VM is in a breakpoint waiting for the debugger to do continue.
158 Breakpoint,
159 }
160
161 impl Display for VmRunMode {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result162 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
163 use self::VmRunMode::*;
164
165 match self {
166 Running => write!(f, "running"),
167 Suspending => write!(f, "suspending"),
168 Exiting => write!(f, "exiting"),
169 Breakpoint => write!(f, "breakpoint"),
170 }
171 }
172 }
173
174 // Trait for devices that get notification on specific GPE trigger
175 pub trait GpeNotify: Send {
notify(&mut self)176 fn notify(&mut self) {}
177 }
178
179 // Trait for devices that get notification on specific PCI PME
180 pub trait PmeNotify: Send {
notify(&mut self, _requester_id: u16)181 fn notify(&mut self, _requester_id: u16) {}
182 }
183
184 pub trait PmResource {
pwrbtn_evt(&mut self)185 fn pwrbtn_evt(&mut self) {}
slpbtn_evt(&mut self)186 fn slpbtn_evt(&mut self) {}
rtc_evt(&mut self)187 fn rtc_evt(&mut self) {}
gpe_evt(&mut self, _gpe: u32)188 fn gpe_evt(&mut self, _gpe: u32) {}
pme_evt(&mut self, _requester_id: u16)189 fn pme_evt(&mut self, _requester_id: u16) {}
register_gpe_notify_dev(&mut self, _gpe: u32, _notify_dev: Arc<Mutex<dyn GpeNotify>>)190 fn register_gpe_notify_dev(&mut self, _gpe: u32, _notify_dev: Arc<Mutex<dyn GpeNotify>>) {}
register_pme_notify_dev(&mut self, _bus: u8, _notify_dev: Arc<Mutex<dyn PmeNotify>>)191 fn register_pme_notify_dev(&mut self, _bus: u8, _notify_dev: Arc<Mutex<dyn PmeNotify>>) {}
192 }
193
194 /// The maximum number of devices that can be listed in one `UsbControlCommand`.
195 ///
196 /// This value was set to be equal to `xhci_regs::MAX_PORTS` for convenience, but it is not
197 /// necessary for correctness. Importing that value directly would be overkill because it would
198 /// require adding a big dependency for a single const.
199 pub const USB_CONTROL_MAX_PORTS: usize = 16;
200
201 #[derive(Serialize, Deserialize, Debug)]
202 pub enum DiskControlCommand {
203 /// Resize a disk to `new_size` in bytes.
204 Resize { new_size: u64 },
205 }
206
207 impl Display for DiskControlCommand {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result208 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
209 use self::DiskControlCommand::*;
210
211 match self {
212 Resize { new_size } => write!(f, "disk_resize {}", new_size),
213 }
214 }
215 }
216
217 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
218 pub enum DiskControlResult {
219 Ok,
220 Err(SysError),
221 }
222
223 /// Net control commands for adding and removing tap devices.
224 #[cfg(feature = "pci-hotplug")]
225 #[derive(Serialize, Deserialize, Debug)]
226 pub enum NetControlCommand {
227 AddTap(String),
228 RemoveTap(u8),
229 }
230
231 #[derive(Serialize, Deserialize, Debug)]
232 pub enum UsbControlCommand {
233 AttachDevice {
234 #[serde(with = "with_as_descriptor")]
235 file: File,
236 },
237 AttachSecurityKey {
238 #[serde(with = "with_as_descriptor")]
239 file: File,
240 },
241 DetachDevice {
242 port: u8,
243 },
244 ListDevice {
245 ports: [u8; USB_CONTROL_MAX_PORTS],
246 },
247 }
248
249 #[derive(Serialize, Deserialize, Copy, Clone, Debug, Default)]
250 pub struct UsbControlAttachedDevice {
251 pub port: u8,
252 pub vendor_id: u16,
253 pub product_id: u16,
254 }
255
256 impl UsbControlAttachedDevice {
valid(self) -> bool257 pub fn valid(self) -> bool {
258 self.port != 0
259 }
260 }
261
262 #[cfg(feature = "pci-hotplug")]
263 #[derive(Serialize, Deserialize, Debug, Clone)]
264 #[must_use]
265 /// Result for hotplug and removal of PCI device.
266 pub enum PciControlResult {
267 AddOk { bus: u8 },
268 ErrString(String),
269 RemoveOk,
270 }
271
272 #[cfg(feature = "pci-hotplug")]
273 impl Display for PciControlResult {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result274 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
275 use self::PciControlResult::*;
276
277 match self {
278 AddOk { bus } => write!(f, "add_ok {}", bus),
279 ErrString(e) => write!(f, "error: {}", e),
280 RemoveOk => write!(f, "remove_ok"),
281 }
282 }
283 }
284
285 #[derive(Serialize, Deserialize, Debug, Clone)]
286 pub enum UsbControlResult {
287 Ok { port: u8 },
288 NoAvailablePort,
289 NoSuchDevice,
290 NoSuchPort,
291 FailedToOpenDevice,
292 Devices([UsbControlAttachedDevice; USB_CONTROL_MAX_PORTS]),
293 FailedToInitHostDevice,
294 }
295
296 impl Display for UsbControlResult {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result297 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
298 use self::UsbControlResult::*;
299
300 match self {
301 UsbControlResult::Ok { port } => write!(f, "ok {}", port),
302 NoAvailablePort => write!(f, "no_available_port"),
303 NoSuchDevice => write!(f, "no_such_device"),
304 NoSuchPort => write!(f, "no_such_port"),
305 FailedToOpenDevice => write!(f, "failed_to_open_device"),
306 Devices(devices) => {
307 write!(f, "devices")?;
308 for d in devices.iter().filter(|d| d.valid()) {
309 write!(f, " {} {:04x} {:04x}", d.port, d.vendor_id, d.product_id)?;
310 }
311 std::result::Result::Ok(())
312 }
313 FailedToInitHostDevice => write!(f, "failed_to_init_host_device"),
314 }
315 }
316 }
317
318 /// Commands for snapshot feature
319 #[derive(Serialize, Deserialize, Debug)]
320 pub enum SnapshotCommand {
321 Take {
322 snapshot_path: PathBuf,
323 compress_memory: bool,
324 encrypt: bool,
325 },
326 }
327
328 /// Commands for actions on devices and the devices control thread.
329 #[derive(Serialize, Deserialize, Debug)]
330 pub enum DeviceControlCommand {
331 SleepDevices,
332 WakeDevices,
333 SnapshotDevices {
334 snapshot_writer: SnapshotWriter,
335 compress_memory: bool,
336 },
337 RestoreDevices {
338 snapshot_reader: SnapshotReader,
339 },
340 GetDevicesState,
341 Exit,
342 }
343
344 /// Commands to control the IRQ handler thread.
345 #[derive(Serialize, Deserialize)]
346 pub enum IrqHandlerRequest {
347 /// No response is sent for this command.
348 AddIrqControlTubes(Vec<Tube>),
349 /// Refreshes the set of event tokens (Events) from the Irqchip that the IRQ
350 /// handler waits on to forward IRQs to their final destination (e.g. via
351 /// Irqchip::service_irq_event).
352 ///
353 /// If the set of tokens exposed by the Irqchip changes while the VM is
354 /// running (such as for snapshot restore), this command must be sent
355 /// otherwise the VM will not receive IRQs as expected.
356 RefreshIrqEventTokens,
357 WakeAndNotifyIteration,
358 /// No response is sent for this command.
359 Exit,
360 }
361
362 const EXPECTED_MAX_IRQ_FLUSH_ITERATIONS: usize = 100;
363
364 /// Response for [IrqHandlerRequest].
365 #[derive(Serialize, Deserialize, Debug)]
366 pub enum IrqHandlerResponse {
367 /// Sent when the IRQ event tokens have been refreshed.
368 IrqEventTokenRefreshComplete,
369 /// Specifies the number of tokens serviced in the requested iteration
370 /// (less the token for the `WakeAndNotifyIteration` request).
371 HandlerIterationComplete(usize),
372 }
373
374 /// Source of a `VmMemoryRequest::RegisterMemory` mapping.
375 #[derive(Serialize, Deserialize)]
376 pub enum VmMemorySource {
377 /// Register shared memory represented by the given descriptor.
378 /// On Windows, descriptor MUST be a mapping handle.
379 SharedMemory(SharedMemory),
380 /// Register a file mapping from the given descriptor.
381 Descriptor {
382 /// File descriptor to map.
383 descriptor: SafeDescriptor,
384 /// Offset within the file in bytes.
385 offset: u64,
386 /// Size of the mapping in bytes.
387 size: u64,
388 },
389 /// Register memory mapped by Vulkano.
390 Vulkan {
391 descriptor: SafeDescriptor,
392 handle_type: u32,
393 memory_idx: u32,
394 device_uuid: [u8; 16],
395 driver_uuid: [u8; 16],
396 size: u64,
397 },
398 /// Register the current rutabaga external mapping.
399 ExternalMapping { ptr: u64, size: u64 },
400 }
401
402 // The following are wrappers to avoid base dependencies in the rutabaga crate
to_rutabaga_desciptor(s: SafeDescriptor) -> RutabagaDescriptor403 fn to_rutabaga_desciptor(s: SafeDescriptor) -> RutabagaDescriptor {
404 // SAFETY:
405 // Safe because we own the SafeDescriptor at this point.
406 unsafe { RutabagaDescriptor::from_raw_descriptor(s.into_raw_descriptor()) }
407 }
408
409 struct RutabagaMemoryRegion {
410 region: Box<dyn RutabagaMappedRegion>,
411 }
412
413 impl RutabagaMemoryRegion {
new(region: Box<dyn RutabagaMappedRegion>) -> RutabagaMemoryRegion414 pub fn new(region: Box<dyn RutabagaMappedRegion>) -> RutabagaMemoryRegion {
415 RutabagaMemoryRegion { region }
416 }
417 }
418
419 // SAFETY:
420 //
421 // Self guarantees `ptr`..`ptr+size` is an mmaped region owned by this object that
422 // can't be unmapped during the `MappedRegion`'s lifetime.
423 unsafe impl MappedRegion for RutabagaMemoryRegion {
as_ptr(&self) -> *mut u8424 fn as_ptr(&self) -> *mut u8 {
425 self.region.as_ptr()
426 }
427
size(&self) -> usize428 fn size(&self) -> usize {
429 self.region.size()
430 }
431 }
432
433 impl Display for VmMemorySource {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result434 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
435 use self::VmMemorySource::*;
436
437 match self {
438 SharedMemory(..) => write!(f, "VmMemorySource::SharedMemory"),
439 Descriptor { .. } => write!(f, "VmMemorySource::Descriptor"),
440 Vulkan { .. } => write!(f, "VmMemorySource::Vulkan"),
441 ExternalMapping { .. } => write!(f, "VmMemorySource::ExternalMapping"),
442 }
443 }
444 }
445
446 impl VmMemorySource {
447 /// Map the resource and return its mapping and size in bytes.
map( self, gralloc: &mut RutabagaGralloc, prot: Protection, ) -> Result<(Box<dyn MappedRegion>, u64, Option<SafeDescriptor>)>448 pub fn map(
449 self,
450 gralloc: &mut RutabagaGralloc,
451 prot: Protection,
452 ) -> Result<(Box<dyn MappedRegion>, u64, Option<SafeDescriptor>)> {
453 let (mem_region, size, descriptor) = match self {
454 VmMemorySource::Descriptor {
455 descriptor,
456 offset,
457 size,
458 } => (
459 map_descriptor(&descriptor, offset, size, prot)?,
460 size,
461 Some(descriptor),
462 ),
463
464 VmMemorySource::SharedMemory(shm) => {
465 (map_descriptor(&shm, 0, shm.size(), prot)?, shm.size(), None)
466 }
467 VmMemorySource::Vulkan {
468 descriptor,
469 handle_type,
470 memory_idx,
471 device_uuid,
472 driver_uuid,
473 size,
474 } => {
475 let device_id = DeviceId {
476 device_uuid,
477 driver_uuid,
478 };
479 let mapped_region = match gralloc.import_and_map(
480 RutabagaHandle {
481 os_handle: to_rutabaga_desciptor(descriptor),
482 handle_type,
483 },
484 VulkanInfo {
485 memory_idx,
486 device_id,
487 },
488 size,
489 ) {
490 Ok(mapped_region) => {
491 let mapped_region: Box<dyn MappedRegion> =
492 Box::new(RutabagaMemoryRegion::new(mapped_region));
493 mapped_region
494 }
495 Err(e) => {
496 error!("gralloc failed to import and map: {}", e);
497 return Err(SysError::new(EINVAL));
498 }
499 };
500 (mapped_region, size, None)
501 }
502 VmMemorySource::ExternalMapping { ptr, size } => {
503 let mapped_region: Box<dyn MappedRegion> = Box::new(ExternalMapping {
504 ptr,
505 size: size as usize,
506 });
507 (mapped_region, size, None)
508 }
509 };
510 Ok((mem_region, size, descriptor))
511 }
512 }
513
514 /// Destination of a `VmMemoryRequest::RegisterMemory` mapping in guest address space.
515 #[derive(Serialize, Deserialize)]
516 pub enum VmMemoryDestination {
517 /// Map at an offset within an existing PCI BAR allocation.
518 ExistingAllocation { allocation: Alloc, offset: u64 },
519 /// Map at the specified guest physical address.
520 GuestPhysicalAddress(u64),
521 }
522
523 impl VmMemoryDestination {
524 /// Allocate and return the guest address of a memory mapping destination.
allocate(self, allocator: &mut SystemAllocator, size: u64) -> Result<GuestAddress>525 pub fn allocate(self, allocator: &mut SystemAllocator, size: u64) -> Result<GuestAddress> {
526 let addr = match self {
527 VmMemoryDestination::ExistingAllocation { allocation, offset } => allocator
528 .mmio_allocator_any()
529 .address_from_pci_offset(allocation, offset, size)
530 .map_err(|_e| SysError::new(EINVAL))?,
531 VmMemoryDestination::GuestPhysicalAddress(gpa) => gpa,
532 };
533 Ok(GuestAddress(addr))
534 }
535 }
536
537 /// Request to register or unregister an ioevent.
538 #[derive(Serialize, Deserialize)]
539 pub struct IoEventUpdateRequest {
540 pub event: Event,
541 pub addr: u64,
542 pub datamatch: Datamatch,
543 pub register: bool,
544 }
545
546 #[derive(Serialize, Deserialize)]
547 pub enum VmMemoryRequest {
548 /// Prepare a shared memory region to make later operations more efficient. This
549 /// may be a no-op depending on underlying platform support.
550 PrepareSharedMemoryRegion { alloc: Alloc, cache: MemCacheType },
551 RegisterMemory {
552 /// Source of the memory to register (mapped file descriptor, shared memory region, etc.)
553 source: VmMemorySource,
554 /// Where to map the memory in the guest.
555 dest: VmMemoryDestination,
556 /// Whether to map the memory read only (true) or read-write (false).
557 prot: Protection,
558 /// Cache attribute for guest memory setting
559 cache: MemCacheType,
560 },
561 /// Call hypervisor to free the given memory range.
562 DynamicallyFreeMemoryRange {
563 guest_address: GuestAddress,
564 size: u64,
565 },
566 /// Call hypervisor to reclaim a priorly freed memory range.
567 DynamicallyReclaimMemoryRange {
568 guest_address: GuestAddress,
569 size: u64,
570 },
571 /// Balloon allocation/deallocation target reached.
572 BalloonTargetReached { size: u64 },
573 /// Unregister the given memory slot that was previously registered with `RegisterMemory`.
574 UnregisterMemory(VmMemoryRegionId),
575 /// Register an ioeventfd by looking up using Alloc info.
576 IoEventWithAlloc {
577 evt: Event,
578 allocation: Alloc,
579 offset: u64,
580 datamatch: Datamatch,
581 register: bool,
582 },
583 /// Register an eventfd with raw guest memory address.
584 IoEventRaw(IoEventUpdateRequest),
585 }
586
587 /// Struct for managing `VmMemoryRequest`s IOMMU related state.
588 pub struct VmMemoryRequestIommuClient {
589 tube: Arc<Mutex<Tube>>,
590 gpu_memory: BTreeSet<MemSlot>,
591 }
592
593 impl VmMemoryRequestIommuClient {
594 /// Constructs `VmMemoryRequestIommuClient` from a tube for communication with the viommu.
new(tube: Arc<Mutex<Tube>>) -> Self595 pub fn new(tube: Arc<Mutex<Tube>>) -> Self {
596 Self {
597 tube,
598 gpu_memory: BTreeSet::new(),
599 }
600 }
601 }
602
603 pub struct VmMemoryRegionState {
604 // alloc -> (pfn, slot)
605 slot_map: HashMap<Alloc, (u64, MemSlot)>,
606 // id -> (slot, Option<offset, size>)
607 mapped_regions: BTreeMap<VmMemoryRegionId, (MemSlot, Option<(usize, usize)>)>,
608 }
609
610 impl VmMemoryRegionState {
new() -> VmMemoryRegionState611 pub fn new() -> VmMemoryRegionState {
612 Self {
613 slot_map: HashMap::new(),
614 mapped_regions: BTreeMap::new(),
615 }
616 }
617 }
618
619 impl Default for VmMemoryRegionState {
default() -> Self620 fn default() -> Self {
621 Self::new()
622 }
623 }
624
try_map_to_prepared_region( vm: &mut impl Vm, region_state: &mut VmMemoryRegionState, source: &VmMemorySource, dest: &VmMemoryDestination, prot: &Protection, ) -> Option<VmMemoryResponse>625 fn try_map_to_prepared_region(
626 vm: &mut impl Vm,
627 region_state: &mut VmMemoryRegionState,
628 source: &VmMemorySource,
629 dest: &VmMemoryDestination,
630 prot: &Protection,
631 ) -> Option<VmMemoryResponse> {
632 let VmMemoryDestination::ExistingAllocation { allocation, offset } = dest else {
633 return None;
634 };
635
636 let (pfn, slot) = region_state.slot_map.get(allocation)?;
637
638 let (descriptor, file_offset, size) = match source {
639 VmMemorySource::Descriptor {
640 descriptor,
641 offset,
642 size,
643 } => (
644 Descriptor(descriptor.as_raw_descriptor()),
645 *offset,
646 *size as usize,
647 ),
648 VmMemorySource::SharedMemory(shm) => {
649 let size = shm.size() as usize;
650 (Descriptor(shm.as_raw_descriptor()), 0, size)
651 }
652 _ => {
653 error!(
654 "source {} is not compatible with fixed mapping into prepared memory region",
655 source
656 );
657 return Some(VmMemoryResponse::Err(SysError::new(EINVAL)));
658 }
659 };
660 if let Err(err) = vm.add_fd_mapping(
661 *slot,
662 *offset as usize,
663 size,
664 &descriptor,
665 file_offset,
666 *prot,
667 ) {
668 return Some(VmMemoryResponse::Err(err));
669 }
670 let pfn = pfn + (offset >> 12);
671 region_state.mapped_regions.insert(
672 VmMemoryRegionId(pfn),
673 (*slot, Some((*offset as usize, size))),
674 );
675 Some(VmMemoryResponse::RegisterMemory(VmMemoryRegionId(pfn)))
676 }
677
678 impl VmMemoryRequest {
679 /// Executes this request on the given Vm.
680 ///
681 /// # Arguments
682 /// * `vm` - The `Vm` to perform the request on.
683 /// * `allocator` - Used to allocate addresses.
684 ///
685 /// This does not return a result, instead encapsulating the success or failure in a
686 /// `VmMemoryResponse` with the intended purpose of sending the response back over the socket
687 /// that received this `VmMemoryResponse`.
execute( self, vm: &mut impl Vm, sys_allocator: &mut SystemAllocator, gralloc: &mut RutabagaGralloc, iommu_client: Option<&mut VmMemoryRequestIommuClient>, region_state: &mut VmMemoryRegionState, ) -> VmMemoryResponse688 pub fn execute(
689 self,
690 vm: &mut impl Vm,
691 sys_allocator: &mut SystemAllocator,
692 gralloc: &mut RutabagaGralloc,
693 iommu_client: Option<&mut VmMemoryRequestIommuClient>,
694 region_state: &mut VmMemoryRegionState,
695 ) -> VmMemoryResponse {
696 use self::VmMemoryRequest::*;
697 match self {
698 PrepareSharedMemoryRegion { alloc, cache } => {
699 // Currently the iommu_client is only used by virtio-gpu when used alongside GPU
700 // pci-passthrough.
701 //
702 // TODO(b/323368701): Make compatible with iommu_client by ensuring that
703 // VirtioIOMMUVfioCommand::VfioDmabufMap is submitted for both dynamic mappings and
704 // fixed mappings (i.e. whether or not try_map_to_prepared_region succeeds in
705 // RegisterMemory case below).
706 assert!(iommu_client.is_none());
707
708 if !sys::should_prepare_memory_region() {
709 return VmMemoryResponse::Ok;
710 }
711
712 match sys::prepare_shared_memory_region(vm, sys_allocator, alloc, cache) {
713 Ok(info) => {
714 region_state.slot_map.insert(alloc, info);
715 VmMemoryResponse::Ok
716 }
717 Err(e) => VmMemoryResponse::Err(e),
718 }
719 }
720 RegisterMemory {
721 source,
722 dest,
723 prot,
724 cache,
725 } => {
726 if let Some(resp) =
727 try_map_to_prepared_region(vm, region_state, &source, &dest, &prot)
728 {
729 return resp;
730 }
731
732 // Correct on Windows because callers of this IPC guarantee descriptor is a mapping
733 // handle.
734 let (mapped_region, size, descriptor) = match source.map(gralloc, prot) {
735 Ok((region, size, descriptor)) => (region, size, descriptor),
736 Err(e) => return VmMemoryResponse::Err(e),
737 };
738
739 let guest_addr = match dest.allocate(sys_allocator, size) {
740 Ok(addr) => addr,
741 Err(e) => return VmMemoryResponse::Err(e),
742 };
743
744 let slot = match vm.add_memory_region(
745 guest_addr,
746 mapped_region,
747 prot == Protection::read(),
748 false,
749 cache,
750 ) {
751 Ok(slot) => slot,
752 Err(e) => return VmMemoryResponse::Err(e),
753 };
754
755 if let (Some(descriptor), Some(iommu_client)) = (descriptor, iommu_client) {
756 let request =
757 VirtioIOMMURequest::VfioCommand(VirtioIOMMUVfioCommand::VfioDmabufMap {
758 mem_slot: slot,
759 gfn: guest_addr.0 >> 12,
760 size,
761 dma_buf: descriptor,
762 });
763
764 match virtio_iommu_request(&iommu_client.tube.lock(), &request) {
765 Ok(VirtioIOMMUResponse::VfioResponse(VirtioIOMMUVfioResult::Ok)) => (),
766 resp => {
767 error!("Unexpected message response: {:?}", resp);
768 // Ignore the result because there is nothing we can do with a failure.
769 let _ = vm.remove_memory_region(slot);
770 return VmMemoryResponse::Err(SysError::new(EINVAL));
771 }
772 };
773
774 iommu_client.gpu_memory.insert(slot);
775 }
776
777 let pfn = guest_addr.0 >> 12;
778 region_state
779 .mapped_regions
780 .insert(VmMemoryRegionId(pfn), (slot, None));
781 VmMemoryResponse::RegisterMemory(VmMemoryRegionId(pfn))
782 }
783 UnregisterMemory(id) => match region_state.mapped_regions.remove(&id) {
784 Some((slot, None)) => match vm.remove_memory_region(slot) {
785 Ok(_) => {
786 if let Some(iommu_client) = iommu_client {
787 if iommu_client.gpu_memory.remove(&slot) {
788 let request = VirtioIOMMURequest::VfioCommand(
789 VirtioIOMMUVfioCommand::VfioDmabufUnmap(slot),
790 );
791
792 match virtio_iommu_request(&iommu_client.tube.lock(), &request) {
793 Ok(VirtioIOMMUResponse::VfioResponse(
794 VirtioIOMMUVfioResult::Ok,
795 )) => VmMemoryResponse::Ok,
796 resp => {
797 error!("Unexpected message response: {:?}", resp);
798 VmMemoryResponse::Err(SysError::new(EINVAL))
799 }
800 }
801 } else {
802 VmMemoryResponse::Ok
803 }
804 } else {
805 VmMemoryResponse::Ok
806 }
807 }
808 Err(e) => VmMemoryResponse::Err(e),
809 },
810 Some((slot, Some((offset, size)))) => match vm.remove_mapping(slot, offset, size) {
811 Ok(()) => VmMemoryResponse::Ok,
812 Err(e) => VmMemoryResponse::Err(e),
813 },
814 None => VmMemoryResponse::Err(SysError::new(EINVAL)),
815 },
816 DynamicallyFreeMemoryRange {
817 guest_address,
818 size,
819 } => match vm.handle_balloon_event(BalloonEvent::Inflate(MemRegion {
820 guest_address,
821 size,
822 })) {
823 Ok(_) => VmMemoryResponse::Ok,
824 Err(e) => VmMemoryResponse::Err(e),
825 },
826 DynamicallyReclaimMemoryRange {
827 guest_address,
828 size,
829 } => match vm.handle_balloon_event(BalloonEvent::Deflate(MemRegion {
830 guest_address,
831 size,
832 })) {
833 Ok(_) => VmMemoryResponse::Ok,
834 Err(e) => VmMemoryResponse::Err(e),
835 },
836 BalloonTargetReached { size } => {
837 match vm.handle_balloon_event(BalloonEvent::BalloonTargetReached(size)) {
838 Ok(_) => VmMemoryResponse::Ok,
839 Err(e) => VmMemoryResponse::Err(e),
840 }
841 }
842 IoEventWithAlloc {
843 evt,
844 allocation,
845 offset,
846 datamatch,
847 register,
848 } => {
849 let len = match datamatch {
850 Datamatch::AnyLength => 1,
851 Datamatch::U8(_) => 1,
852 Datamatch::U16(_) => 2,
853 Datamatch::U32(_) => 4,
854 Datamatch::U64(_) => 8,
855 };
856 let addr = match sys_allocator
857 .mmio_allocator_any()
858 .address_from_pci_offset(allocation, offset, len)
859 {
860 Ok(addr) => addr,
861 Err(e) => {
862 error!("error getting target address: {:#}", e);
863 return VmMemoryResponse::Err(SysError::new(EINVAL));
864 }
865 };
866 let res = if register {
867 vm.register_ioevent(&evt, IoEventAddress::Mmio(addr), datamatch)
868 } else {
869 vm.unregister_ioevent(&evt, IoEventAddress::Mmio(addr), datamatch)
870 };
871 match res {
872 Ok(_) => VmMemoryResponse::Ok,
873 Err(e) => VmMemoryResponse::Err(e),
874 }
875 }
876 IoEventRaw(request) => {
877 let res = if request.register {
878 vm.register_ioevent(
879 &request.event,
880 IoEventAddress::Mmio(request.addr),
881 request.datamatch,
882 )
883 } else {
884 vm.unregister_ioevent(
885 &request.event,
886 IoEventAddress::Mmio(request.addr),
887 request.datamatch,
888 )
889 };
890 match res {
891 Ok(_) => VmMemoryResponse::Ok,
892 Err(e) => VmMemoryResponse::Err(e),
893 }
894 }
895 }
896 }
897 }
898
899 #[derive(Serialize, Deserialize, Debug, PartialOrd, PartialEq, Eq, Ord, Clone, Copy)]
900 /// Identifer for registered memory regions. Globally unique.
901 // The current implementation uses pfn as the unique identifier.
902 pub struct VmMemoryRegionId(u64);
903
904 #[derive(Serialize, Deserialize, Debug)]
905 pub enum VmMemoryResponse {
906 /// The request to register memory into guest address space was successful.
907 RegisterMemory(VmMemoryRegionId),
908 Ok,
909 Err(SysError),
910 }
911
912 #[derive(Serialize, Deserialize, Debug)]
913 pub enum VmIrqRequest {
914 /// Allocate one gsi, and associate gsi to irqfd with register_irqfd()
915 AllocateOneMsi {
916 irqfd: Event,
917 device_id: u32,
918 queue_id: usize,
919 device_name: String,
920 },
921 /// Allocate a specific gsi to irqfd with register_irqfd(). This must only
922 /// be used when it is known that the gsi is free. Only the snapshot
923 /// subsystem can make this guarantee, and use of this request by any other
924 /// caller is strongly discouraged.
925 AllocateOneMsiAtGsi {
926 irqfd: Event,
927 gsi: u32,
928 device_id: u32,
929 queue_id: usize,
930 device_name: String,
931 },
932 /// Add one msi route entry into the IRQ chip.
933 AddMsiRoute {
934 gsi: u32,
935 msi_address: u64,
936 msi_data: u32,
937 },
938 // unregister_irqfs() and release gsi
939 ReleaseOneIrq {
940 gsi: u32,
941 irqfd: Event,
942 },
943 }
944
945 /// Data to set up an IRQ event or IRQ route on the IRQ chip.
946 /// VmIrqRequest::execute can't take an `IrqChip` argument, because of a dependency cycle between
947 /// devices and vm_control, so it takes a Fn that processes an `IrqSetup`.
948 pub enum IrqSetup<'a> {
949 Event(u32, &'a Event, u32, usize, String),
950 Route(IrqRoute),
951 UnRegister(u32, &'a Event),
952 }
953
954 impl VmIrqRequest {
955 /// Executes this request on the given Vm.
956 ///
957 /// # Arguments
958 /// * `set_up_irq` - A function that applies an `IrqSetup` to an IRQ chip.
959 ///
960 /// This does not return a result, instead encapsulating the success or failure in a
961 /// `VmIrqResponse` with the intended purpose of sending the response back over the socket
962 /// that received this `VmIrqResponse`.
execute<F>(&self, set_up_irq: F, sys_allocator: &mut SystemAllocator) -> VmIrqResponse where F: FnOnce(IrqSetup) -> Result<()>,963 pub fn execute<F>(&self, set_up_irq: F, sys_allocator: &mut SystemAllocator) -> VmIrqResponse
964 where
965 F: FnOnce(IrqSetup) -> Result<()>,
966 {
967 use self::VmIrqRequest::*;
968 match *self {
969 AllocateOneMsi {
970 ref irqfd,
971 device_id,
972 queue_id,
973 ref device_name,
974 } => {
975 if let Some(irq_num) = sys_allocator.allocate_irq() {
976 match set_up_irq(IrqSetup::Event(
977 irq_num,
978 irqfd,
979 device_id,
980 queue_id,
981 device_name.clone(),
982 )) {
983 Ok(_) => VmIrqResponse::AllocateOneMsi { gsi: irq_num },
984 Err(e) => VmIrqResponse::Err(e),
985 }
986 } else {
987 VmIrqResponse::Err(SysError::new(EINVAL))
988 }
989 }
990 AllocateOneMsiAtGsi {
991 ref irqfd,
992 gsi,
993 device_id,
994 queue_id,
995 ref device_name,
996 } => {
997 match set_up_irq(IrqSetup::Event(
998 gsi,
999 irqfd,
1000 device_id,
1001 queue_id,
1002 device_name.clone(),
1003 )) {
1004 Ok(_) => VmIrqResponse::Ok,
1005 Err(e) => VmIrqResponse::Err(e),
1006 }
1007 }
1008 AddMsiRoute {
1009 gsi,
1010 msi_address,
1011 msi_data,
1012 } => {
1013 let route = IrqRoute {
1014 gsi,
1015 source: IrqSource::Msi {
1016 address: msi_address,
1017 data: msi_data,
1018 },
1019 };
1020 match set_up_irq(IrqSetup::Route(route)) {
1021 Ok(_) => VmIrqResponse::Ok,
1022 Err(e) => VmIrqResponse::Err(e),
1023 }
1024 }
1025 ReleaseOneIrq { gsi, ref irqfd } => {
1026 let _ = set_up_irq(IrqSetup::UnRegister(gsi, irqfd));
1027 sys_allocator.release_irq(gsi);
1028 VmIrqResponse::Ok
1029 }
1030 }
1031 }
1032 }
1033
1034 #[derive(Serialize, Deserialize, Debug)]
1035 pub enum VmIrqResponse {
1036 AllocateOneMsi { gsi: u32 },
1037 Ok,
1038 Err(SysError),
1039 }
1040
1041 #[derive(Serialize, Deserialize, Debug, Clone)]
1042 pub enum DevicesState {
1043 Sleep,
1044 Wake,
1045 }
1046
1047 #[derive(Serialize, Deserialize, Debug, Clone)]
1048 pub enum BatControlResult {
1049 Ok,
1050 NoBatDevice,
1051 NoSuchHealth,
1052 NoSuchProperty,
1053 NoSuchStatus,
1054 NoSuchBatType,
1055 StringParseIntErr,
1056 }
1057
1058 impl Display for BatControlResult {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result1059 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1060 use self::BatControlResult::*;
1061
1062 match self {
1063 Ok => write!(f, "Setting battery property successfully"),
1064 NoBatDevice => write!(f, "No battery device created"),
1065 NoSuchHealth => write!(f, "Invalid Battery health setting. Only support: unknown/good/overheat/dead/overvoltage/unexpectedfailure/cold/watchdogtimerexpire/safetytimerexpire/overcurrent"),
1066 NoSuchProperty => write!(f, "Battery doesn't have such property. Only support: status/health/present/capacity/aconline"),
1067 NoSuchStatus => write!(f, "Invalid Battery status setting. Only support: unknown/charging/discharging/notcharging/full"),
1068 NoSuchBatType => write!(f, "Invalid Battery type setting. Only support: goldfish"),
1069 StringParseIntErr => write!(f, "Battery property target ParseInt error"),
1070 }
1071 }
1072 }
1073
1074 #[derive(Serialize, Deserialize, Copy, Clone, Debug, Default, PartialEq, Eq)]
1075 #[serde(rename_all = "kebab-case")]
1076 pub enum BatteryType {
1077 #[default]
1078 Goldfish,
1079 }
1080
1081 impl FromStr for BatteryType {
1082 type Err = BatControlResult;
1083
from_str(s: &str) -> StdResult<Self, Self::Err>1084 fn from_str(s: &str) -> StdResult<Self, Self::Err> {
1085 match s {
1086 "goldfish" => Ok(BatteryType::Goldfish),
1087 _ => Err(BatControlResult::NoSuchBatType),
1088 }
1089 }
1090 }
1091
1092 #[derive(Serialize, Deserialize, Debug)]
1093 pub enum BatProperty {
1094 Status,
1095 Health,
1096 Present,
1097 Capacity,
1098 ACOnline,
1099 }
1100
1101 impl FromStr for BatProperty {
1102 type Err = BatControlResult;
1103
from_str(s: &str) -> StdResult<Self, Self::Err>1104 fn from_str(s: &str) -> StdResult<Self, Self::Err> {
1105 match s {
1106 "status" => Ok(BatProperty::Status),
1107 "health" => Ok(BatProperty::Health),
1108 "present" => Ok(BatProperty::Present),
1109 "capacity" => Ok(BatProperty::Capacity),
1110 "aconline" => Ok(BatProperty::ACOnline),
1111 _ => Err(BatControlResult::NoSuchProperty),
1112 }
1113 }
1114 }
1115
1116 #[derive(Serialize, Deserialize, Debug)]
1117 pub enum BatStatus {
1118 Unknown,
1119 Charging,
1120 DisCharging,
1121 NotCharging,
1122 Full,
1123 }
1124
1125 impl BatStatus {
new(status: String) -> std::result::Result<Self, BatControlResult>1126 pub fn new(status: String) -> std::result::Result<Self, BatControlResult> {
1127 match status.as_str() {
1128 "unknown" => Ok(BatStatus::Unknown),
1129 "charging" => Ok(BatStatus::Charging),
1130 "discharging" => Ok(BatStatus::DisCharging),
1131 "notcharging" => Ok(BatStatus::NotCharging),
1132 "full" => Ok(BatStatus::Full),
1133 _ => Err(BatControlResult::NoSuchStatus),
1134 }
1135 }
1136 }
1137
1138 impl FromStr for BatStatus {
1139 type Err = BatControlResult;
1140
from_str(s: &str) -> StdResult<Self, Self::Err>1141 fn from_str(s: &str) -> StdResult<Self, Self::Err> {
1142 match s {
1143 "unknown" => Ok(BatStatus::Unknown),
1144 "charging" => Ok(BatStatus::Charging),
1145 "discharging" => Ok(BatStatus::DisCharging),
1146 "notcharging" => Ok(BatStatus::NotCharging),
1147 "full" => Ok(BatStatus::Full),
1148 _ => Err(BatControlResult::NoSuchStatus),
1149 }
1150 }
1151 }
1152
1153 impl From<BatStatus> for u32 {
from(status: BatStatus) -> Self1154 fn from(status: BatStatus) -> Self {
1155 status as u32
1156 }
1157 }
1158
1159 #[derive(Serialize, Deserialize, Debug)]
1160 pub enum BatHealth {
1161 Unknown,
1162 Good,
1163 Overheat,
1164 Dead,
1165 OverVoltage,
1166 UnexpectedFailure,
1167 Cold,
1168 WatchdogTimerExpire,
1169 SafetyTimerExpire,
1170 OverCurrent,
1171 }
1172
1173 impl FromStr for BatHealth {
1174 type Err = BatControlResult;
1175
from_str(s: &str) -> StdResult<Self, Self::Err>1176 fn from_str(s: &str) -> StdResult<Self, Self::Err> {
1177 match s {
1178 "unknown" => Ok(BatHealth::Unknown),
1179 "good" => Ok(BatHealth::Good),
1180 "overheat" => Ok(BatHealth::Overheat),
1181 "dead" => Ok(BatHealth::Dead),
1182 "overvoltage" => Ok(BatHealth::OverVoltage),
1183 "unexpectedfailure" => Ok(BatHealth::UnexpectedFailure),
1184 "cold" => Ok(BatHealth::Cold),
1185 "watchdogtimerexpire" => Ok(BatHealth::WatchdogTimerExpire),
1186 "safetytimerexpire" => Ok(BatHealth::SafetyTimerExpire),
1187 "overcurrent" => Ok(BatHealth::OverCurrent),
1188 _ => Err(BatControlResult::NoSuchHealth),
1189 }
1190 }
1191 }
1192
1193 impl From<BatHealth> for u32 {
from(status: BatHealth) -> Self1194 fn from(status: BatHealth) -> Self {
1195 status as u32
1196 }
1197 }
1198
1199 #[derive(Serialize, Deserialize, Debug)]
1200 pub enum BatControlCommand {
1201 SetStatus(BatStatus),
1202 SetHealth(BatHealth),
1203 SetPresent(u32),
1204 SetCapacity(u32),
1205 SetACOnline(u32),
1206 }
1207
1208 impl BatControlCommand {
new(property: String, target: String) -> std::result::Result<Self, BatControlResult>1209 pub fn new(property: String, target: String) -> std::result::Result<Self, BatControlResult> {
1210 let cmd = property.parse::<BatProperty>()?;
1211 match cmd {
1212 BatProperty::Status => Ok(BatControlCommand::SetStatus(target.parse::<BatStatus>()?)),
1213 BatProperty::Health => Ok(BatControlCommand::SetHealth(target.parse::<BatHealth>()?)),
1214 BatProperty::Present => Ok(BatControlCommand::SetPresent(
1215 target
1216 .parse::<u32>()
1217 .map_err(|_| BatControlResult::StringParseIntErr)?,
1218 )),
1219 BatProperty::Capacity => Ok(BatControlCommand::SetCapacity(
1220 target
1221 .parse::<u32>()
1222 .map_err(|_| BatControlResult::StringParseIntErr)?,
1223 )),
1224 BatProperty::ACOnline => Ok(BatControlCommand::SetACOnline(
1225 target
1226 .parse::<u32>()
1227 .map_err(|_| BatControlResult::StringParseIntErr)?,
1228 )),
1229 }
1230 }
1231 }
1232
1233 /// Used for VM to control battery properties.
1234 pub struct BatControl {
1235 pub type_: BatteryType,
1236 pub control_tube: Tube,
1237 }
1238
1239 // Used to mark hotplug pci device's device type
1240 #[derive(Serialize, Deserialize, Debug, Clone)]
1241 pub enum HotPlugDeviceType {
1242 UpstreamPort,
1243 DownstreamPort,
1244 EndPoint,
1245 }
1246
1247 // Used for VM to hotplug pci devices
1248 #[derive(Serialize, Deserialize, Debug, Clone)]
1249 pub struct HotPlugDeviceInfo {
1250 pub device_type: HotPlugDeviceType,
1251 pub path: PathBuf,
1252 pub hp_interrupt: bool,
1253 }
1254
1255 /// Message for communicating a suspend or resume to the virtio-pvclock device.
1256 #[derive(Serialize, Deserialize, Debug, Clone)]
1257 pub enum PvClockCommand {
1258 Suspend,
1259 Resume,
1260 }
1261
1262 /// Message used by virtio-pvclock to communicate command results.
1263 #[derive(Serialize, Deserialize, Debug)]
1264 pub enum PvClockCommandResponse {
1265 Ok,
1266 DeviceInactive,
1267 Err(SysError),
1268 }
1269
1270 /// Commands for vmm-swap feature
1271 #[derive(Serialize, Deserialize, Debug)]
1272 pub enum SwapCommand {
1273 Enable,
1274 Trim,
1275 SwapOut,
1276 Disable { slow_file_cleanup: bool },
1277 Status,
1278 }
1279
1280 ///
1281 /// A request to the main process to perform some operation on the VM.
1282 ///
1283 /// Unless otherwise noted, each request should expect a `VmResponse::Ok` to be received on success.
1284 #[derive(Serialize, Deserialize, Debug)]
1285 pub enum VmRequest {
1286 /// Break the VM's run loop and exit.
1287 Exit,
1288 /// Trigger a power button event in the guest.
1289 Powerbtn,
1290 /// Trigger a sleep button event in the guest.
1291 Sleepbtn,
1292 /// Trigger a RTC interrupt in the guest.
1293 Rtc,
1294 /// Suspend the VM's VCPUs until resume.
1295 SuspendVcpus,
1296 /// Swap the memory content into files on a disk
1297 Swap(SwapCommand),
1298 /// Resume the VM's VCPUs that were previously suspended.
1299 ResumeVcpus,
1300 /// Inject a general-purpose event.
1301 Gpe(u32),
1302 /// Inject a PCI PME
1303 PciPme(u16),
1304 /// Make the VM's RT VCPU real-time.
1305 MakeRT,
1306 /// Command for balloon driver.
1307 #[cfg(feature = "balloon")]
1308 BalloonCommand(BalloonControlCommand),
1309 /// Send a command to a disk chosen by `disk_index`.
1310 /// `disk_index` is a 0-based count of `--disk`, `--rwdisk`, and `-r` command-line options.
1311 DiskCommand {
1312 disk_index: usize,
1313 command: DiskControlCommand,
1314 },
1315 /// Command to use controller.
1316 UsbCommand(UsbControlCommand),
1317 /// Command to modify the gpu.
1318 #[cfg(feature = "gpu")]
1319 GpuCommand(GpuControlCommand),
1320 /// Command to set battery.
1321 BatCommand(BatteryType, BatControlCommand),
1322 /// Command to add/remove multiple vfio-pci devices
1323 HotPlugVfioCommand {
1324 device: HotPlugDeviceInfo,
1325 add: bool,
1326 },
1327 /// Command to add/remove network tap device as virtio-pci device
1328 #[cfg(feature = "pci-hotplug")]
1329 HotPlugNetCommand(NetControlCommand),
1330 /// Command to Snapshot devices
1331 Snapshot(SnapshotCommand),
1332 /// Register for event notification
1333 #[cfg(feature = "registered_events")]
1334 RegisterListener {
1335 socket_addr: String,
1336 event: RegisteredEvent,
1337 },
1338 /// Unregister for notifications for event
1339 #[cfg(feature = "registered_events")]
1340 UnregisterListener {
1341 socket_addr: String,
1342 event: RegisteredEvent,
1343 },
1344 /// Unregister for all event notification
1345 #[cfg(feature = "registered_events")]
1346 Unregister { socket_addr: String },
1347 /// Suspend VM VCPUs and Devices until resume.
1348 SuspendVm,
1349 /// Resume VM VCPUs and Devices.
1350 ResumeVm,
1351 }
1352
1353 /// NOTE: when making any changes to this enum please also update
1354 /// RegisteredEventFfi in crosvm_control/src/lib.rs
1355 #[cfg(feature = "registered_events")]
1356 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone, Copy)]
1357 pub enum RegisteredEvent {
1358 VirtioBalloonWsReport,
1359 VirtioBalloonResize,
1360 VirtioBalloonOOMDeflation,
1361 }
1362
1363 #[cfg(feature = "registered_events")]
1364 #[derive(Serialize, Deserialize, Debug)]
1365 pub enum RegisteredEventWithData {
1366 VirtioBalloonWsReport {
1367 ws_buckets: Vec<WSBucket>,
1368 balloon_actual: u64,
1369 },
1370 VirtioBalloonResize,
1371 VirtioBalloonOOMDeflation,
1372 }
1373
1374 #[cfg(feature = "registered_events")]
1375 impl RegisteredEventWithData {
into_event(&self) -> RegisteredEvent1376 pub fn into_event(&self) -> RegisteredEvent {
1377 match self {
1378 Self::VirtioBalloonWsReport { .. } => RegisteredEvent::VirtioBalloonWsReport,
1379 Self::VirtioBalloonResize => RegisteredEvent::VirtioBalloonResize,
1380 Self::VirtioBalloonOOMDeflation => RegisteredEvent::VirtioBalloonOOMDeflation,
1381 }
1382 }
1383
into_proto(&self) -> registered_events::RegisteredEvent1384 pub fn into_proto(&self) -> registered_events::RegisteredEvent {
1385 match self {
1386 Self::VirtioBalloonWsReport {
1387 ws_buckets,
1388 balloon_actual,
1389 } => {
1390 let mut report = registered_events::VirtioBalloonWsReport {
1391 balloon_actual: *balloon_actual,
1392 ..registered_events::VirtioBalloonWsReport::new()
1393 };
1394 for ws in ws_buckets {
1395 report.ws_buckets.push(registered_events::VirtioWsBucket {
1396 age: ws.age,
1397 file_bytes: ws.bytes[0],
1398 anon_bytes: ws.bytes[1],
1399 ..registered_events::VirtioWsBucket::new()
1400 });
1401 }
1402 let mut event = registered_events::RegisteredEvent::new();
1403 event.set_ws_report(report);
1404 event
1405 }
1406 Self::VirtioBalloonResize => {
1407 let mut event = registered_events::RegisteredEvent::new();
1408 event.set_resize(registered_events::VirtioBalloonResize::new());
1409 event
1410 }
1411 Self::VirtioBalloonOOMDeflation => {
1412 let mut event = registered_events::RegisteredEvent::new();
1413 event.set_oom_deflation(registered_events::VirtioBalloonOOMDeflation::new());
1414 event
1415 }
1416 }
1417 }
1418
from_ws(ws: &BalloonWS, balloon_actual: u64) -> Self1419 pub fn from_ws(ws: &BalloonWS, balloon_actual: u64) -> Self {
1420 RegisteredEventWithData::VirtioBalloonWsReport {
1421 ws_buckets: ws.ws.clone(),
1422 balloon_actual,
1423 }
1424 }
1425 }
1426
handle_disk_command(command: &DiskControlCommand, disk_host_tube: &Tube) -> VmResponse1427 pub fn handle_disk_command(command: &DiskControlCommand, disk_host_tube: &Tube) -> VmResponse {
1428 // Forward the request to the block device process via its control socket.
1429 if let Err(e) = disk_host_tube.send(command) {
1430 error!("disk socket send failed: {}", e);
1431 return VmResponse::Err(SysError::new(EINVAL));
1432 }
1433
1434 // Wait for the disk control command to be processed
1435 match disk_host_tube.recv() {
1436 Ok(DiskControlResult::Ok) => VmResponse::Ok,
1437 Ok(DiskControlResult::Err(e)) => VmResponse::Err(e),
1438 Err(e) => {
1439 error!("disk socket recv failed: {}", e);
1440 VmResponse::Err(SysError::new(EINVAL))
1441 }
1442 }
1443 }
1444
1445 /// WARNING: descriptor must be a mapping handle on Windows.
map_descriptor( descriptor: &dyn AsRawDescriptor, offset: u64, size: u64, prot: Protection, ) -> Result<Box<dyn MappedRegion>>1446 fn map_descriptor(
1447 descriptor: &dyn AsRawDescriptor,
1448 offset: u64,
1449 size: u64,
1450 prot: Protection,
1451 ) -> Result<Box<dyn MappedRegion>> {
1452 let size: usize = size.try_into().map_err(|_e| SysError::new(ERANGE))?;
1453 match MemoryMappingBuilder::new(size)
1454 .from_descriptor(descriptor)
1455 .offset(offset)
1456 .protection(prot)
1457 .build()
1458 {
1459 Ok(mmap) => Ok(Box::new(mmap)),
1460 Err(MmapError::SystemCallFailed(e)) => Err(e),
1461 _ => Err(SysError::new(EINVAL)),
1462 }
1463 }
1464
1465 // Get vCPU state. vCPUs are expected to all hold the same state.
1466 // In this function, there may be a time where vCPUs are not
get_vcpu_state(kick_vcpus: impl Fn(VcpuControl), vcpu_num: usize) -> anyhow::Result<VmRunMode>1467 fn get_vcpu_state(kick_vcpus: impl Fn(VcpuControl), vcpu_num: usize) -> anyhow::Result<VmRunMode> {
1468 let (send_chan, recv_chan) = mpsc::channel();
1469 kick_vcpus(VcpuControl::GetStates(send_chan));
1470 if vcpu_num == 0 {
1471 bail!("vcpu_num is zero");
1472 }
1473 let mut current_mode_vec: Vec<VmRunMode> = Vec::new();
1474 for _ in 0..vcpu_num {
1475 match recv_chan.recv() {
1476 Ok(state) => current_mode_vec.push(state),
1477 Err(e) => {
1478 bail!("Failed to get vCPU state: {}", e);
1479 }
1480 };
1481 }
1482 let first_state = current_mode_vec[0];
1483 if first_state == VmRunMode::Exiting {
1484 panic!("Attempt to snapshot while exiting.");
1485 }
1486 if current_mode_vec.iter().any(|x| *x != first_state) {
1487 // We do not panic here. It could be that vCPUs are transitioning from one mode to another.
1488 bail!("Unknown VM state: vCPUs hold different states.");
1489 }
1490 Ok(first_state)
1491 }
1492
1493 /// A guard to guarantee that all the vCPUs are suspended during the scope.
1494 ///
1495 /// When this guard is dropped, it rolls back the state of CPUs.
1496 pub struct VcpuSuspendGuard<'a> {
1497 saved_run_mode: VmRunMode,
1498 kick_vcpus: &'a dyn Fn(VcpuControl),
1499 }
1500
1501 impl<'a> VcpuSuspendGuard<'a> {
1502 /// Check the all vCPU state and suspend the vCPUs if they are running.
1503 ///
1504 /// This returns [VcpuSuspendGuard] to rollback the vcpu state.
1505 ///
1506 /// # Arguments
1507 ///
1508 /// * `kick_vcpus` - A funtion to send [VcpuControl] message to all the vCPUs and interrupt
1509 /// them.
1510 /// * `vcpu_num` - The number of vCPUs.
new(kick_vcpus: &'a impl Fn(VcpuControl), vcpu_num: usize) -> anyhow::Result<Self>1511 pub fn new(kick_vcpus: &'a impl Fn(VcpuControl), vcpu_num: usize) -> anyhow::Result<Self> {
1512 // get initial vcpu state
1513 let saved_run_mode = get_vcpu_state(kick_vcpus, vcpu_num)?;
1514 match saved_run_mode {
1515 VmRunMode::Running => {
1516 kick_vcpus(VcpuControl::RunState(VmRunMode::Suspending));
1517 // Blocking call, waiting for response to ensure vCPU state was updated.
1518 // In case of failure, where a vCPU still has the state running, start up vcpus and
1519 // abort operation.
1520 let current_mode = get_vcpu_state(kick_vcpus, vcpu_num)?;
1521 if current_mode != VmRunMode::Suspending {
1522 kick_vcpus(VcpuControl::RunState(saved_run_mode));
1523 bail!("vCPUs failed to all suspend. Kicking back all vCPUs to their previous state: {saved_run_mode}");
1524 }
1525 }
1526 VmRunMode::Suspending => {
1527 // do nothing. keep the state suspending.
1528 }
1529 other => {
1530 bail!("vcpus are not in running/suspending state, but {}", other);
1531 }
1532 };
1533 Ok(Self {
1534 saved_run_mode,
1535 kick_vcpus,
1536 })
1537 }
1538 }
1539
1540 impl Drop for VcpuSuspendGuard<'_> {
drop(&mut self)1541 fn drop(&mut self) {
1542 if self.saved_run_mode != VmRunMode::Suspending {
1543 (self.kick_vcpus)(VcpuControl::RunState(self.saved_run_mode));
1544 }
1545 }
1546 }
1547
1548 /// A guard to guarantee that all devices are sleeping during its scope.
1549 ///
1550 /// When this guard is dropped, it wakes the devices.
1551 pub struct DeviceSleepGuard<'a> {
1552 device_control_tube: &'a Tube,
1553 devices_state: DevicesState,
1554 }
1555
1556 impl<'a> DeviceSleepGuard<'a> {
new(device_control_tube: &'a Tube) -> anyhow::Result<Self>1557 fn new(device_control_tube: &'a Tube) -> anyhow::Result<Self> {
1558 device_control_tube
1559 .send(&DeviceControlCommand::GetDevicesState)
1560 .context("send command to devices control socket")?;
1561 let devices_state = match device_control_tube
1562 .recv()
1563 .context("receive from devices control socket")?
1564 {
1565 VmResponse::DevicesState(state) => state,
1566 resp => bail!("failed to get devices state. Unexpected behavior: {}", resp),
1567 };
1568 if let DevicesState::Wake = devices_state {
1569 device_control_tube
1570 .send(&DeviceControlCommand::SleepDevices)
1571 .context("send command to devices control socket")?;
1572 match device_control_tube
1573 .recv()
1574 .context("receive from devices control socket")?
1575 {
1576 VmResponse::Ok => (),
1577 resp => bail!("device sleep failed: {}", resp),
1578 }
1579 }
1580 Ok(Self {
1581 device_control_tube,
1582 devices_state,
1583 })
1584 }
1585 }
1586
1587 impl Drop for DeviceSleepGuard<'_> {
drop(&mut self)1588 fn drop(&mut self) {
1589 if let DevicesState::Wake = self.devices_state {
1590 if let Err(e) = self
1591 .device_control_tube
1592 .send(&DeviceControlCommand::WakeDevices)
1593 {
1594 panic!("failed to request device wake after snapshot: {}", e);
1595 }
1596 match self.device_control_tube.recv() {
1597 Ok(VmResponse::Ok) => (),
1598 Ok(resp) => panic!("unexpected response to device wake request: {}", resp),
1599 Err(e) => panic!("failed to get reply for device wake request: {}", e),
1600 }
1601 }
1602 }
1603 }
1604
1605 impl VmRequest {
1606 /// Executes this request on the given Vm and other mutable state.
1607 ///
1608 /// This does not return a result, instead encapsulating the success or failure in a
1609 /// `VmResponse` with the intended purpose of sending the response back over the socket that
1610 /// received this `VmRequest`.
execute( &self, vm: &impl Vm, run_mode: &mut Option<VmRunMode>, disk_host_tubes: &[Tube], pm: &mut Option<Arc<Mutex<dyn PmResource + Send>>>, gpu_control_tube: Option<&Tube>, usb_control_tube: Option<&Tube>, bat_control: &mut Option<BatControl>, kick_vcpus: impl Fn(VcpuControl), force_s2idle: bool, #[cfg(feature = "swap")] swap_controller: Option<&swap::SwapController>, device_control_tube: &Tube, vcpu_size: usize, irq_handler_control: &Tube, snapshot_irqchip: impl Fn() -> anyhow::Result<serde_json::Value>, ) -> VmResponse1611 pub fn execute(
1612 &self,
1613 vm: &impl Vm,
1614 run_mode: &mut Option<VmRunMode>,
1615 disk_host_tubes: &[Tube],
1616 pm: &mut Option<Arc<Mutex<dyn PmResource + Send>>>,
1617 gpu_control_tube: Option<&Tube>,
1618 usb_control_tube: Option<&Tube>,
1619 bat_control: &mut Option<BatControl>,
1620 kick_vcpus: impl Fn(VcpuControl),
1621 force_s2idle: bool,
1622 #[cfg(feature = "swap")] swap_controller: Option<&swap::SwapController>,
1623 device_control_tube: &Tube,
1624 vcpu_size: usize,
1625 irq_handler_control: &Tube,
1626 snapshot_irqchip: impl Fn() -> anyhow::Result<serde_json::Value>,
1627 ) -> VmResponse {
1628 match *self {
1629 VmRequest::Exit => {
1630 *run_mode = Some(VmRunMode::Exiting);
1631 VmResponse::Ok
1632 }
1633 VmRequest::Powerbtn => {
1634 if let Some(pm) = pm {
1635 pm.lock().pwrbtn_evt();
1636 VmResponse::Ok
1637 } else {
1638 error!("{:#?} not supported", *self);
1639 VmResponse::Err(SysError::new(ENOTSUP))
1640 }
1641 }
1642 VmRequest::Sleepbtn => {
1643 if let Some(pm) = pm {
1644 pm.lock().slpbtn_evt();
1645 VmResponse::Ok
1646 } else {
1647 error!("{:#?} not supported", *self);
1648 VmResponse::Err(SysError::new(ENOTSUP))
1649 }
1650 }
1651 VmRequest::Rtc => {
1652 if let Some(pm) = pm {
1653 pm.lock().rtc_evt();
1654 VmResponse::Ok
1655 } else {
1656 error!("{:#?} not supported", *self);
1657 VmResponse::Err(SysError::new(ENOTSUP))
1658 }
1659 }
1660 VmRequest::SuspendVcpus => {
1661 *run_mode = Some(VmRunMode::Suspending);
1662 VmResponse::Ok
1663 }
1664 VmRequest::ResumeVcpus => {
1665 if let Err(e) = device_control_tube.send(&DeviceControlCommand::GetDevicesState) {
1666 error!("failed to send GetDevicesState: {}", e);
1667 return VmResponse::Err(SysError::new(EIO));
1668 }
1669 let devices_state = match device_control_tube.recv() {
1670 Ok(VmResponse::DevicesState(state)) => state,
1671 Ok(resp) => {
1672 error!("failed to get devices state. Unexpected behavior: {}", resp);
1673 return VmResponse::Err(SysError::new(EINVAL));
1674 }
1675 Err(e) => {
1676 error!("failed to get devices state. Unexpected behavior: {}", e);
1677 return VmResponse::Err(SysError::new(EINVAL));
1678 }
1679 };
1680 if let DevicesState::Sleep = devices_state {
1681 error!("Trying to wake Vcpus while Devices are asleep. Did you mean to use `crosvm resume --full`?");
1682 return VmResponse::Err(SysError::new(EINVAL));
1683 }
1684 *run_mode = Some(VmRunMode::Running);
1685
1686 if force_s2idle {
1687 // During resume also emulate powerbtn event which will allow to wakeup fully
1688 // suspended guest.
1689 if let Some(pm) = pm {
1690 pm.lock().pwrbtn_evt();
1691 } else {
1692 error!("triggering power btn during resume not supported");
1693 return VmResponse::Err(SysError::new(ENOTSUP));
1694 }
1695 }
1696 VmResponse::Ok
1697 }
1698 VmRequest::Swap(SwapCommand::Enable) => {
1699 #[cfg(feature = "swap")]
1700 if let Some(swap_controller) = swap_controller {
1701 // Suspend all vcpus and devices while vmm-swap is enabling (move the guest
1702 // memory contents to the staging memory) to guarantee no processes other than
1703 // the swap monitor process access the guest memory.
1704 let _vcpu_guard = match VcpuSuspendGuard::new(&kick_vcpus, vcpu_size) {
1705 Ok(guard) => guard,
1706 Err(e) => {
1707 error!("failed to suspend vcpus: {:?}", e);
1708 return VmResponse::Err(SysError::new(EINVAL));
1709 }
1710 };
1711 // TODO(b/253386409): Use `devices::Suspendable::sleep()` instead of sending
1712 // `SIGSTOP` signal.
1713 let _devices_guard = match swap_controller.suspend_devices() {
1714 Ok(guard) => guard,
1715 Err(e) => {
1716 error!("failed to suspend devices: {:?}", e);
1717 return VmResponse::Err(SysError::new(EINVAL));
1718 }
1719 };
1720
1721 return match swap_controller.enable() {
1722 Ok(()) => VmResponse::Ok,
1723 Err(e) => {
1724 error!("swap enable failed: {}", e);
1725 VmResponse::Err(SysError::new(EINVAL))
1726 }
1727 };
1728 }
1729 VmResponse::Err(SysError::new(ENOTSUP))
1730 }
1731 VmRequest::Swap(SwapCommand::Trim) => {
1732 #[cfg(feature = "swap")]
1733 if let Some(swap_controller) = swap_controller {
1734 return match swap_controller.trim() {
1735 Ok(()) => VmResponse::Ok,
1736 Err(e) => {
1737 error!("swap trim failed: {}", e);
1738 VmResponse::Err(SysError::new(EINVAL))
1739 }
1740 };
1741 }
1742 VmResponse::Err(SysError::new(ENOTSUP))
1743 }
1744 VmRequest::Swap(SwapCommand::SwapOut) => {
1745 #[cfg(feature = "swap")]
1746 if let Some(swap_controller) = swap_controller {
1747 return match swap_controller.swap_out() {
1748 Ok(()) => VmResponse::Ok,
1749 Err(e) => {
1750 error!("swap out failed: {}", e);
1751 VmResponse::Err(SysError::new(EINVAL))
1752 }
1753 };
1754 }
1755 VmResponse::Err(SysError::new(ENOTSUP))
1756 }
1757 VmRequest::Swap(SwapCommand::Disable {
1758 #[cfg(feature = "swap")]
1759 slow_file_cleanup,
1760 ..
1761 }) => {
1762 #[cfg(feature = "swap")]
1763 if let Some(swap_controller) = swap_controller {
1764 return match swap_controller.disable(slow_file_cleanup) {
1765 Ok(()) => VmResponse::Ok,
1766 Err(e) => {
1767 error!("swap disable failed: {}", e);
1768 VmResponse::Err(SysError::new(EINVAL))
1769 }
1770 };
1771 }
1772 VmResponse::Err(SysError::new(ENOTSUP))
1773 }
1774 VmRequest::Swap(SwapCommand::Status) => {
1775 #[cfg(feature = "swap")]
1776 if let Some(swap_controller) = swap_controller {
1777 return match swap_controller.status() {
1778 Ok(status) => VmResponse::SwapStatus(status),
1779 Err(e) => {
1780 error!("swap status failed: {}", e);
1781 VmResponse::Err(SysError::new(EINVAL))
1782 }
1783 };
1784 }
1785 VmResponse::Err(SysError::new(ENOTSUP))
1786 }
1787 VmRequest::SuspendVm => {
1788 info!("Starting crosvm suspend");
1789 kick_vcpus(VcpuControl::RunState(VmRunMode::Suspending));
1790 let current_mode = match get_vcpu_state(kick_vcpus, vcpu_size) {
1791 Ok(state) => state,
1792 Err(e) => {
1793 error!("failed to get vcpu state: {e}");
1794 return VmResponse::Err(SysError::new(EIO));
1795 }
1796 };
1797 if current_mode != VmRunMode::Suspending {
1798 error!("vCPUs failed to all suspend.");
1799 return VmResponse::Err(SysError::new(EIO));
1800 }
1801 if let Err(e) = device_control_tube
1802 .send(&DeviceControlCommand::SleepDevices)
1803 .context("send command to devices control socket")
1804 {
1805 error!("{:?}", e);
1806 return VmResponse::Err(SysError::new(EIO));
1807 };
1808 match device_control_tube
1809 .recv()
1810 .context("receive from devices control socket")
1811 {
1812 Ok(VmResponse::Ok) => {
1813 info!("Finished crosvm suspend successfully");
1814 VmResponse::Ok
1815 }
1816 Ok(resp) => {
1817 error!("device sleep failed: {}", resp);
1818 VmResponse::Err(SysError::new(EIO))
1819 }
1820 Err(e) => {
1821 error!("receive from devices control socket: {:?}", e);
1822 VmResponse::Err(SysError::new(EIO))
1823 }
1824 }
1825 }
1826 VmRequest::ResumeVm => {
1827 info!("Starting crosvm resume");
1828 if let Err(e) = device_control_tube
1829 .send(&DeviceControlCommand::WakeDevices)
1830 .context("send command to devices control socket")
1831 {
1832 error!("{:?}", e);
1833 return VmResponse::Err(SysError::new(EIO));
1834 };
1835 match device_control_tube
1836 .recv()
1837 .context("receive from devices control socket")
1838 {
1839 Ok(VmResponse::Ok) => {
1840 info!("Finished crosvm resume successfully");
1841 }
1842 Ok(resp) => {
1843 error!("device wake failed: {}", resp);
1844 return VmResponse::Err(SysError::new(EIO));
1845 }
1846 Err(e) => {
1847 error!("receive from devices control socket: {:?}", e);
1848 return VmResponse::Err(SysError::new(EIO));
1849 }
1850 }
1851 kick_vcpus(VcpuControl::RunState(VmRunMode::Running));
1852 VmResponse::Ok
1853 }
1854 VmRequest::Gpe(gpe) => {
1855 if let Some(pm) = pm.as_ref() {
1856 pm.lock().gpe_evt(gpe);
1857 VmResponse::Ok
1858 } else {
1859 error!("{:#?} not supported", *self);
1860 VmResponse::Err(SysError::new(ENOTSUP))
1861 }
1862 }
1863 VmRequest::PciPme(requester_id) => {
1864 if let Some(pm) = pm.as_ref() {
1865 pm.lock().pme_evt(requester_id);
1866 VmResponse::Ok
1867 } else {
1868 error!("{:#?} not supported", *self);
1869 VmResponse::Err(SysError::new(ENOTSUP))
1870 }
1871 }
1872 VmRequest::MakeRT => {
1873 kick_vcpus(VcpuControl::MakeRT);
1874 VmResponse::Ok
1875 }
1876 #[cfg(feature = "balloon")]
1877 VmRequest::BalloonCommand(_) => unreachable!("Should be handled with BalloonTube"),
1878 VmRequest::DiskCommand {
1879 disk_index,
1880 ref command,
1881 } => match &disk_host_tubes.get(disk_index) {
1882 Some(tube) => handle_disk_command(command, tube),
1883 None => VmResponse::Err(SysError::new(ENODEV)),
1884 },
1885 #[cfg(feature = "gpu")]
1886 VmRequest::GpuCommand(ref cmd) => match gpu_control_tube {
1887 Some(gpu_control) => {
1888 let res = gpu_control.send(cmd);
1889 if let Err(e) = res {
1890 error!("fail to send command to gpu control socket: {}", e);
1891 return VmResponse::Err(SysError::new(EIO));
1892 }
1893 match gpu_control.recv() {
1894 Ok(response) => VmResponse::GpuResponse(response),
1895 Err(e) => {
1896 error!("fail to recv command from gpu control socket: {}", e);
1897 VmResponse::Err(SysError::new(EIO))
1898 }
1899 }
1900 }
1901 None => {
1902 error!("gpu control is not enabled in crosvm");
1903 VmResponse::Err(SysError::new(EIO))
1904 }
1905 },
1906 VmRequest::UsbCommand(ref cmd) => {
1907 let usb_control_tube = match usb_control_tube {
1908 Some(t) => t,
1909 None => {
1910 error!("attempted to execute USB request without control tube");
1911 return VmResponse::Err(SysError::new(ENODEV));
1912 }
1913 };
1914 let res = usb_control_tube.send(cmd);
1915 if let Err(e) = res {
1916 error!("fail to send command to usb control socket: {}", e);
1917 return VmResponse::Err(SysError::new(EIO));
1918 }
1919 match usb_control_tube.recv() {
1920 Ok(response) => VmResponse::UsbResponse(response),
1921 Err(e) => {
1922 error!("fail to recv command from usb control socket: {}", e);
1923 VmResponse::Err(SysError::new(EIO))
1924 }
1925 }
1926 }
1927 VmRequest::BatCommand(type_, ref cmd) => {
1928 match bat_control {
1929 Some(battery) => {
1930 if battery.type_ != type_ {
1931 error!("ignored battery command due to battery type: expected {:?}, got {:?}", battery.type_, type_);
1932 return VmResponse::Err(SysError::new(EINVAL));
1933 }
1934
1935 let res = battery.control_tube.send(cmd);
1936 if let Err(e) = res {
1937 error!("fail to send command to bat control socket: {}", e);
1938 return VmResponse::Err(SysError::new(EIO));
1939 }
1940
1941 match battery.control_tube.recv() {
1942 Ok(response) => VmResponse::BatResponse(response),
1943 Err(e) => {
1944 error!("fail to recv command from bat control socket: {}", e);
1945 VmResponse::Err(SysError::new(EIO))
1946 }
1947 }
1948 }
1949 None => VmResponse::BatResponse(BatControlResult::NoBatDevice),
1950 }
1951 }
1952 VmRequest::HotPlugVfioCommand { device: _, add: _ } => VmResponse::Ok,
1953 #[cfg(feature = "pci-hotplug")]
1954 VmRequest::HotPlugNetCommand(ref _net_cmd) => {
1955 VmResponse::ErrString("hot plug not supported".to_owned())
1956 }
1957 VmRequest::Snapshot(SnapshotCommand::Take {
1958 ref snapshot_path,
1959 compress_memory,
1960 encrypt,
1961 }) => {
1962 info!("Starting crosvm snapshot");
1963 match do_snapshot(
1964 snapshot_path.to_path_buf(),
1965 vm,
1966 kick_vcpus,
1967 irq_handler_control,
1968 device_control_tube,
1969 vcpu_size,
1970 snapshot_irqchip,
1971 compress_memory,
1972 encrypt,
1973 ) {
1974 Ok(()) => {
1975 info!("Finished crosvm snapshot successfully");
1976 VmResponse::Ok
1977 }
1978 Err(e) => {
1979 error!("failed to handle snapshot: {:?}", e);
1980 VmResponse::Err(SysError::new(EIO))
1981 }
1982 }
1983 }
1984 #[cfg(feature = "registered_events")]
1985 VmRequest::RegisterListener {
1986 socket_addr: _,
1987 event: _,
1988 } => VmResponse::Ok,
1989 #[cfg(feature = "registered_events")]
1990 VmRequest::UnregisterListener {
1991 socket_addr: _,
1992 event: _,
1993 } => VmResponse::Ok,
1994 #[cfg(feature = "registered_events")]
1995 VmRequest::Unregister { socket_addr: _ } => VmResponse::Ok,
1996 }
1997 }
1998 }
1999
2000 /// Snapshot the VM to file at `snapshot_path`
do_snapshot( snapshot_path: PathBuf, vm: &impl Vm, kick_vcpus: impl Fn(VcpuControl), irq_handler_control: &Tube, device_control_tube: &Tube, vcpu_size: usize, snapshot_irqchip: impl Fn() -> anyhow::Result<serde_json::Value>, compress_memory: bool, encrypt: bool, ) -> anyhow::Result<()>2001 fn do_snapshot(
2002 snapshot_path: PathBuf,
2003 vm: &impl Vm,
2004 kick_vcpus: impl Fn(VcpuControl),
2005 irq_handler_control: &Tube,
2006 device_control_tube: &Tube,
2007 vcpu_size: usize,
2008 snapshot_irqchip: impl Fn() -> anyhow::Result<serde_json::Value>,
2009 compress_memory: bool,
2010 encrypt: bool,
2011 ) -> anyhow::Result<()> {
2012 let _vcpu_guard = VcpuSuspendGuard::new(&kick_vcpus, vcpu_size)?;
2013 let _device_guard = DeviceSleepGuard::new(device_control_tube)?;
2014
2015 // We want to flush all pending IRQs to the LAPICs. There are two cases:
2016 //
2017 // MSIs: these are directly delivered to the LAPIC. We must verify the handler
2018 // thread cycles once to deliver these interrupts.
2019 //
2020 // Legacy interrupts: in the case of a split IRQ chip, these interrupts may
2021 // flow through the userspace IOAPIC. If the hypervisor does not support
2022 // irqfds (e.g. WHPX), a single iteration will only flush the IRQ to the
2023 // IOAPIC. The underlying MSI will be asserted at this point, but if the
2024 // IRQ handler doesn't run another iteration, it won't be delivered to the
2025 // LAPIC. This is why we cycle the handler thread twice (doing so ensures we
2026 // process the underlying MSI).
2027 //
2028 // We can handle both of these cases by iterating until there are no tokens
2029 // serviced on the requested iteration. Note that in the legacy case, this
2030 // ensures at least two iterations.
2031 //
2032 // Note: within CrosVM, *all* interrupts are eventually converted into the
2033 // same mechanicism that MSIs use. This is why we say "underlying" MSI for
2034 // a legacy IRQ.
2035 let mut flush_attempts = 0;
2036 loop {
2037 irq_handler_control
2038 .send(&IrqHandlerRequest::WakeAndNotifyIteration)
2039 .context("failed to send flush command to IRQ handler thread")?;
2040 let resp = irq_handler_control
2041 .recv()
2042 .context("failed to recv flush response from IRQ handler thread")?;
2043 match resp {
2044 IrqHandlerResponse::HandlerIterationComplete(tokens_serviced) => {
2045 if tokens_serviced == 0 {
2046 break;
2047 }
2048 }
2049 _ => bail!("received unexpected reply from IRQ handler: {:?}", resp),
2050 }
2051 flush_attempts += 1;
2052 if flush_attempts > EXPECTED_MAX_IRQ_FLUSH_ITERATIONS {
2053 warn!("flushing IRQs for snapshot may be stalled after iteration {}, expected <= {} iterations", flush_attempts, EXPECTED_MAX_IRQ_FLUSH_ITERATIONS);
2054 }
2055 }
2056 info!("flushed IRQs in {} iterations", flush_attempts);
2057
2058 let snapshot_writer = SnapshotWriter::new(snapshot_path, encrypt)?;
2059
2060 // Snapshot hypervisor's paravirtualized clock.
2061 let pvclock_snapshot = if vm.check_capability(VmCap::PvClock) {
2062 serde_json::to_value(vm.get_pvclock()?)?
2063 } else {
2064 serde_json::Value::Null
2065 };
2066 snapshot_writer.write_fragment("pvclock", &pvclock_snapshot)?;
2067
2068 // Snapshot Vcpus
2069 info!("VCPUs snapshotting...");
2070 let (send_chan, recv_chan) = mpsc::channel();
2071 kick_vcpus(VcpuControl::Snapshot(
2072 snapshot_writer.add_namespace("vcpu")?,
2073 send_chan,
2074 ));
2075 // Validate all Vcpus snapshot successfully
2076 for _ in 0..vcpu_size {
2077 recv_chan
2078 .recv()
2079 .context("Failed to recv Vcpu snapshot response")?
2080 .context("Failed to snapshot Vcpu")?;
2081 }
2082 info!("VCPUs snapshotted.");
2083
2084 // Snapshot irqchip
2085 info!("Snapshotting irqchip...");
2086 let irqchip_snap = snapshot_irqchip()?;
2087 snapshot_writer
2088 .write_fragment("irqchip", &irqchip_snap)
2089 .context("Failed to write irqchip state")?;
2090 info!("Snapshotted irqchip.");
2091
2092 // Snapshot devices
2093 info!("Devices snapshotting...");
2094 device_control_tube
2095 .send(&DeviceControlCommand::SnapshotDevices {
2096 snapshot_writer,
2097 compress_memory,
2098 })
2099 .context("send command to devices control socket")?;
2100 let resp: VmResponse = device_control_tube
2101 .recv()
2102 .context("receive from devices control socket")?;
2103 if !matches!(resp, VmResponse::Ok) {
2104 bail!("unexpected SnapshotDevices response: {resp}");
2105 }
2106 info!("Devices snapshotted.");
2107 Ok(())
2108 }
2109
2110 /// Restore the VM to the snapshot at `restore_path`.
2111 ///
2112 /// Same as `VmRequest::execute` with a `VmRequest::Restore`. Exposed as a separate function
2113 /// because not all the `VmRequest::execute` arguments are available in the "cold restore" flow.
do_restore( restore_path: PathBuf, vm: &impl Vm, kick_vcpus: impl Fn(VcpuControl), kick_vcpu: impl Fn(VcpuControl, usize), irq_handler_control: &Tube, device_control_tube: &Tube, vcpu_size: usize, mut restore_irqchip: impl FnMut(serde_json::Value) -> anyhow::Result<()>, require_encrypted: bool, ) -> anyhow::Result<()>2114 pub fn do_restore(
2115 restore_path: PathBuf,
2116 vm: &impl Vm,
2117 kick_vcpus: impl Fn(VcpuControl),
2118 kick_vcpu: impl Fn(VcpuControl, usize),
2119 irq_handler_control: &Tube,
2120 device_control_tube: &Tube,
2121 vcpu_size: usize,
2122 mut restore_irqchip: impl FnMut(serde_json::Value) -> anyhow::Result<()>,
2123 require_encrypted: bool,
2124 ) -> anyhow::Result<()> {
2125 let _guard = VcpuSuspendGuard::new(&kick_vcpus, vcpu_size);
2126 let _devices_guard = DeviceSleepGuard::new(device_control_tube)?;
2127
2128 let snapshot_reader = SnapshotReader::new(restore_path, require_encrypted)?;
2129
2130 // Restore hypervisor's paravirtualized clock.
2131 let pvclock_snapshot: serde_json::Value = snapshot_reader.read_fragment("pvclock")?;
2132 if vm.check_capability(VmCap::PvClock) {
2133 vm.set_pvclock(&serde_json::from_value(pvclock_snapshot)?)?;
2134 } else {
2135 anyhow::ensure!(pvclock_snapshot == serde_json::Value::Null);
2136 };
2137
2138 // Restore IrqChip
2139 let irq_snapshot: serde_json::Value = snapshot_reader.read_fragment("irqchip")?;
2140 restore_irqchip(irq_snapshot)?;
2141
2142 // Restore Vcpu(s)
2143 let vcpu_snapshot_reader = snapshot_reader.namespace("vcpu")?;
2144 let vcpu_snapshot_count = vcpu_snapshot_reader.list_fragments()?.len();
2145 if vcpu_snapshot_count != vcpu_size {
2146 bail!(
2147 "bad cpu count in snapshot: expected={} got={}",
2148 vcpu_size,
2149 vcpu_snapshot_count,
2150 );
2151 }
2152 #[cfg(target_arch = "x86_64")]
2153 let host_tsc_reference_moment = {
2154 // SAFETY: rdtsc takes no arguments.
2155 unsafe { _rdtsc() }
2156 };
2157 let (send_chan, recv_chan) = mpsc::channel();
2158 for vcpu_id in 0..vcpu_size {
2159 kick_vcpu(
2160 VcpuControl::Restore(VcpuRestoreRequest {
2161 result_sender: send_chan.clone(),
2162 snapshot_reader: vcpu_snapshot_reader.clone(),
2163 #[cfg(target_arch = "x86_64")]
2164 host_tsc_reference_moment,
2165 }),
2166 vcpu_id,
2167 );
2168 }
2169 for _ in 0..vcpu_size {
2170 recv_chan
2171 .recv()
2172 .context("Failed to recv restore response")?
2173 .context("Failed to restore vcpu")?;
2174 }
2175
2176 // Restore devices
2177 device_control_tube
2178 .send(&DeviceControlCommand::RestoreDevices { snapshot_reader })
2179 .context("send command to devices control socket")?;
2180 let resp: VmResponse = device_control_tube
2181 .recv()
2182 .context("receive from devices control socket")?;
2183 if !matches!(resp, VmResponse::Ok) {
2184 bail!("unexpected RestoreDevices response: {resp}");
2185 }
2186
2187 irq_handler_control
2188 .send(&IrqHandlerRequest::RefreshIrqEventTokens)
2189 .context("failed to send refresh irq event token command to IRQ handler thread")?;
2190 let resp: IrqHandlerResponse = irq_handler_control
2191 .recv()
2192 .context("failed to recv refresh response from IRQ handler thread")?;
2193 if !matches!(resp, IrqHandlerResponse::IrqEventTokenRefreshComplete) {
2194 bail!(
2195 "received unexpected reply from IRQ handler thread: {:?}",
2196 resp
2197 );
2198 }
2199 Ok(())
2200 }
2201
2202 /// Indication of success or failure of a `VmRequest`.
2203 ///
2204 /// Success is usually indicated `VmResponse::Ok` unless there is data associated with the response.
2205 #[derive(Serialize, Deserialize, Debug, Clone)]
2206 #[must_use]
2207 pub enum VmResponse {
2208 /// Indicates the request was executed successfully.
2209 Ok,
2210 /// Indicates the request encountered some error during execution.
2211 Err(SysError),
2212 /// Indicates the request encountered some error during execution.
2213 ErrString(String),
2214 /// The request to register memory into guest address space was successfully done at page frame
2215 /// number `pfn` and memory slot number `slot`.
2216 RegisterMemory { pfn: u64, slot: u32 },
2217 /// Results of balloon control commands.
2218 #[cfg(feature = "balloon")]
2219 BalloonStats {
2220 stats: BalloonStats,
2221 balloon_actual: u64,
2222 },
2223 /// Results of balloon WS-R command
2224 #[cfg(feature = "balloon")]
2225 BalloonWS { ws: BalloonWS, balloon_actual: u64 },
2226 /// Results of PCI hot plug
2227 #[cfg(feature = "pci-hotplug")]
2228 PciHotPlugResponse { bus: u8 },
2229 /// Results of usb control commands.
2230 UsbResponse(UsbControlResult),
2231 #[cfg(feature = "gpu")]
2232 /// Results of gpu control commands.
2233 GpuResponse(GpuControlResult),
2234 /// Results of battery control commands.
2235 BatResponse(BatControlResult),
2236 /// Results of swap status command.
2237 SwapStatus(SwapStatus),
2238 /// Gets the state of Devices (sleep/wake)
2239 DevicesState(DevicesState),
2240 }
2241
2242 impl Display for VmResponse {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result2243 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2244 use self::VmResponse::*;
2245
2246 match self {
2247 Ok => write!(f, "ok"),
2248 Err(e) => write!(f, "error: {}", e),
2249 ErrString(e) => write!(f, "error: {}", e),
2250 RegisterMemory { pfn, slot } => write!(
2251 f,
2252 "memory registered to page frame number {:#x} and memory slot {}",
2253 pfn, slot
2254 ),
2255 #[cfg(feature = "balloon")]
2256 VmResponse::BalloonStats {
2257 stats,
2258 balloon_actual,
2259 } => {
2260 write!(
2261 f,
2262 "stats: {}\nballoon_actual: {}",
2263 serde_json::to_string_pretty(&stats)
2264 .unwrap_or_else(|_| "invalid_response".to_string()),
2265 balloon_actual
2266 )
2267 }
2268 #[cfg(feature = "balloon")]
2269 VmResponse::BalloonWS { ws, balloon_actual } => {
2270 write!(
2271 f,
2272 "ws: {}, balloon_actual: {}",
2273 serde_json::to_string_pretty(&ws)
2274 .unwrap_or_else(|_| "invalid_response".to_string()),
2275 balloon_actual,
2276 )
2277 }
2278 UsbResponse(result) => write!(f, "usb control request get result {:?}", result),
2279 #[cfg(feature = "pci-hotplug")]
2280 PciHotPlugResponse { bus } => write!(f, "pci hotplug bus {:?}", bus),
2281 #[cfg(feature = "gpu")]
2282 GpuResponse(result) => write!(f, "gpu control request result {:?}", result),
2283 BatResponse(result) => write!(f, "{}", result),
2284 SwapStatus(status) => {
2285 write!(
2286 f,
2287 "{}",
2288 serde_json::to_string(&status)
2289 .unwrap_or_else(|_| "invalid_response".to_string()),
2290 )
2291 }
2292 DevicesState(status) => write!(f, "devices status: {:?}", status),
2293 }
2294 }
2295 }
2296
2297 /// Enum that allows remote control of a wait context (used between the Windows GpuDisplay & the
2298 /// GPU worker).
2299 #[derive(Serialize, Deserialize)]
2300 pub enum ModifyWaitContext {
2301 Add(#[serde(with = "with_as_descriptor")] Descriptor),
2302 }
2303
2304 #[sorted]
2305 #[derive(Error, Debug)]
2306 pub enum VirtioIOMMUVfioError {
2307 #[error("socket failed")]
2308 SocketFailed,
2309 #[error("unexpected response: {0}")]
2310 UnexpectedResponse(VirtioIOMMUResponse),
2311 #[error("unknown command: `{0}`")]
2312 UnknownCommand(String),
2313 #[error("{0}")]
2314 VfioControl(VirtioIOMMUVfioResult),
2315 }
2316
2317 #[derive(Serialize, Deserialize, Debug)]
2318 pub enum VirtioIOMMUVfioCommand {
2319 // Add the vfio device attached to virtio-iommu.
2320 VfioDeviceAdd {
2321 endpoint_addr: u32,
2322 wrapper_id: u32,
2323 #[serde(with = "with_as_descriptor")]
2324 container: File,
2325 },
2326 // Delete the vfio device attached to virtio-iommu.
2327 VfioDeviceDel {
2328 endpoint_addr: u32,
2329 },
2330 // Map a dma-buf into vfio iommu table
2331 VfioDmabufMap {
2332 mem_slot: MemSlot,
2333 gfn: u64,
2334 size: u64,
2335 dma_buf: SafeDescriptor,
2336 },
2337 // Unmap a dma-buf from vfio iommu table
2338 VfioDmabufUnmap(MemSlot),
2339 }
2340
2341 #[derive(Serialize, Deserialize, Debug)]
2342 pub enum VirtioIOMMUVfioResult {
2343 Ok,
2344 NotInPCIRanges,
2345 NoAvailableContainer,
2346 NoSuchDevice,
2347 NoSuchMappedDmabuf,
2348 InvalidParam,
2349 }
2350
2351 impl Display for VirtioIOMMUVfioResult {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result2352 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2353 use self::VirtioIOMMUVfioResult::*;
2354
2355 match self {
2356 Ok => write!(f, "successfully"),
2357 NotInPCIRanges => write!(f, "not in the pci ranges of virtio-iommu"),
2358 NoAvailableContainer => write!(f, "no available vfio container"),
2359 NoSuchDevice => write!(f, "no such a vfio device"),
2360 NoSuchMappedDmabuf => write!(f, "no such a mapped dmabuf"),
2361 InvalidParam => write!(f, "invalid parameters"),
2362 }
2363 }
2364 }
2365
2366 /// A request to the virtio-iommu process to perform some operations.
2367 ///
2368 /// Unless otherwise noted, each request should expect a `VirtioIOMMUResponse::Ok` to be received on
2369 /// success.
2370 #[derive(Serialize, Deserialize, Debug)]
2371 pub enum VirtioIOMMURequest {
2372 /// Command for vfio related operations.
2373 VfioCommand(VirtioIOMMUVfioCommand),
2374 }
2375
2376 /// Indication of success or failure of a `VirtioIOMMURequest`.
2377 ///
2378 /// Success is usually indicated `VirtioIOMMUResponse::Ok` unless there is data associated with the
2379 /// response.
2380 #[derive(Serialize, Deserialize, Debug)]
2381 pub enum VirtioIOMMUResponse {
2382 /// Indicates the request was executed successfully.
2383 Ok,
2384 /// Indicates the request encountered some error during execution.
2385 Err(SysError),
2386 /// Results for Vfio commands.
2387 VfioResponse(VirtioIOMMUVfioResult),
2388 }
2389
2390 impl Display for VirtioIOMMUResponse {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result2391 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2392 use self::VirtioIOMMUResponse::*;
2393 match self {
2394 Ok => write!(f, "ok"),
2395 Err(e) => write!(f, "error: {}", e),
2396 VfioResponse(result) => write!(
2397 f,
2398 "The vfio-related virtio-iommu request got result: {:?}",
2399 result
2400 ),
2401 }
2402 }
2403 }
2404
2405 /// Send VirtioIOMMURequest without waiting for the response
virtio_iommu_request_async( iommu_control_tube: &Tube, req: &VirtioIOMMURequest, ) -> VirtioIOMMUResponse2406 pub fn virtio_iommu_request_async(
2407 iommu_control_tube: &Tube,
2408 req: &VirtioIOMMURequest,
2409 ) -> VirtioIOMMUResponse {
2410 match iommu_control_tube.send(&req) {
2411 Ok(_) => VirtioIOMMUResponse::Ok,
2412 Err(e) => {
2413 error!("virtio-iommu socket send failed: {:?}", e);
2414 VirtioIOMMUResponse::Err(SysError::last())
2415 }
2416 }
2417 }
2418
2419 pub type VirtioIOMMURequestResult = std::result::Result<VirtioIOMMUResponse, ()>;
2420
2421 /// Send VirtioIOMMURequest and wait to get the response
virtio_iommu_request( iommu_control_tube: &Tube, req: &VirtioIOMMURequest, ) -> VirtioIOMMURequestResult2422 pub fn virtio_iommu_request(
2423 iommu_control_tube: &Tube,
2424 req: &VirtioIOMMURequest,
2425 ) -> VirtioIOMMURequestResult {
2426 let response = match virtio_iommu_request_async(iommu_control_tube, req) {
2427 VirtioIOMMUResponse::Ok => match iommu_control_tube.recv() {
2428 Ok(response) => response,
2429 Err(e) => {
2430 error!("virtio-iommu socket recv failed: {:?}", e);
2431 VirtioIOMMUResponse::Err(SysError::last())
2432 }
2433 },
2434 resp => resp,
2435 };
2436 Ok(response)
2437 }
2438