1 // Copyright 2017 The ChromiumOS Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #![cfg_attr(windows, allow(unused))]
6
7 //! Emulates virtual and hardware devices.
8
9 pub mod ac_adapter;
10 pub mod acpi;
11 pub mod bat;
12 mod bus;
13 #[cfg(feature = "stats")]
14 mod bus_stats;
15 pub mod cmos;
16 #[cfg(target_arch = "x86_64")]
17 mod debugcon;
18 mod fw_cfg;
19 mod i8042;
20 mod irq_event;
21 pub mod irqchip;
22 mod pci;
23 mod pflash;
24 pub mod pl030;
25 pub mod pmc_virt;
26 mod serial;
27 pub mod serial_device;
28 mod suspendable;
29 mod sys;
30 #[cfg(any(target_os = "android", target_os = "linux"))]
31 mod virtcpufreq;
32 #[cfg(any(target_os = "android", target_os = "linux"))]
33 mod virtcpufreq_v2;
34 pub mod virtio;
35 #[cfg(feature = "vtpm")]
36 mod vtpm_proxy;
37
38 cfg_if::cfg_if! {
39 if #[cfg(target_arch = "x86_64")] {
40 mod pit;
41 pub use self::pit::{Pit, PitError};
42 pub mod tsc;
43 }
44 }
45
46 use std::sync::Arc;
47 use std::time::Instant;
48
49 use anyhow::anyhow;
50 use anyhow::Context;
51 use base::debug;
52 use base::error;
53 use base::info;
54 use base::Tube;
55 use base::TubeError;
56 use cros_async::AsyncTube;
57 use cros_async::Executor;
58 use serde::Deserialize;
59 use serde::Serialize;
60 use vm_control::DeviceControlCommand;
61 use vm_control::DevicesState;
62 use vm_control::VmResponse;
63 use vm_memory::GuestMemory;
64
65 pub use self::acpi::ACPIPMFixedEvent;
66 pub use self::acpi::ACPIPMResource;
67 pub use self::bat::BatteryError;
68 pub use self::bat::GoldfishBattery;
69 pub use self::bus::Bus;
70 pub use self::bus::BusAccessInfo;
71 pub use self::bus::BusDevice;
72 pub use self::bus::BusDeviceObj;
73 pub use self::bus::BusDeviceSync;
74 pub use self::bus::BusRange;
75 pub use self::bus::BusResumeDevice;
76 pub use self::bus::BusType;
77 pub use self::bus::Error as BusError;
78 pub use self::bus::HotPlugBus;
79 pub use self::bus::HotPlugKey;
80 #[cfg(feature = "stats")]
81 pub use self::bus_stats::BusStatistics;
82 #[cfg(target_arch = "x86_64")]
83 pub use self::debugcon::Debugcon;
84 pub use self::fw_cfg::Error as FwCfgError;
85 pub use self::fw_cfg::FwCfgDevice;
86 pub use self::fw_cfg::FwCfgItemType;
87 pub use self::fw_cfg::FwCfgParameters;
88 pub use self::fw_cfg::FW_CFG_BASE_PORT;
89 pub use self::fw_cfg::FW_CFG_MAX_FILE_SLOTS;
90 pub use self::fw_cfg::FW_CFG_WIDTH;
91 pub use self::i8042::I8042Device;
92 pub use self::irq_event::IrqEdgeEvent;
93 pub use self::irq_event::IrqLevelEvent;
94 pub use self::irqchip::*;
95 pub use self::pci::BarRange;
96 pub use self::pci::CrosvmDeviceId;
97 pub use self::pci::GpeScope;
98 #[cfg(feature = "pci-hotplug")]
99 pub use self::pci::HotPluggable;
100 #[cfg(feature = "pci-hotplug")]
101 pub use self::pci::IntxParameter;
102 #[cfg(feature = "pci-hotplug")]
103 pub use self::pci::NetResourceCarrier;
104 pub use self::pci::PciAddress;
105 pub use self::pci::PciAddressError;
106 pub use self::pci::PciBarConfiguration;
107 pub use self::pci::PciBarIndex;
108 pub use self::pci::PciBus;
109 pub use self::pci::PciClassCode;
110 pub use self::pci::PciConfigIo;
111 pub use self::pci::PciConfigMmio;
112 pub use self::pci::PciDevice;
113 pub use self::pci::PciDeviceError;
114 pub use self::pci::PciInterruptPin;
115 pub use self::pci::PciMmioMapper;
116 pub use self::pci::PciRoot;
117 pub use self::pci::PciRootCommand;
118 pub use self::pci::PciVirtualConfigMmio;
119 pub use self::pci::PreferredIrq;
120 #[cfg(feature = "pci-hotplug")]
121 pub use self::pci::ResourceCarrier;
122 pub use self::pci::StubPciDevice;
123 pub use self::pci::StubPciParameters;
124 pub use self::pflash::Pflash;
125 pub use self::pflash::PflashParameters;
126 pub use self::pl030::Pl030;
127 pub use self::pmc_virt::VirtualPmc;
128 pub use self::serial::Serial;
129 pub use self::serial_device::Error as SerialError;
130 pub use self::serial_device::SerialDevice;
131 pub use self::serial_device::SerialHardware;
132 pub use self::serial_device::SerialParameters;
133 pub use self::serial_device::SerialType;
134 pub use self::suspendable::DeviceState;
135 pub use self::suspendable::Suspendable;
136 #[cfg(any(target_os = "android", target_os = "linux"))]
137 pub use self::virtcpufreq::VirtCpufreq;
138 #[cfg(any(target_os = "android", target_os = "linux"))]
139 pub use self::virtcpufreq_v2::VirtCpufreqV2;
140 pub use self::virtio::VirtioMmioDevice;
141 pub use self::virtio::VirtioPciDevice;
142 #[cfg(feature = "vtpm")]
143 pub use self::vtpm_proxy::VtpmProxy;
144
145 cfg_if::cfg_if! {
146 if #[cfg(any(target_os = "android", target_os = "linux"))] {
147 mod platform;
148 mod proxy;
149 pub mod vmwdt;
150 pub mod vfio;
151 #[cfg(feature = "usb")]
152 #[macro_use]
153 mod register_space;
154 #[cfg(feature = "usb")]
155 pub mod usb;
156 #[cfg(feature = "usb")]
157 mod utils;
158
159 pub use self::pci::{
160 CoIommuDev, CoIommuParameters, CoIommuUnpinPolicy, PciBridge, PcieDownstreamPort,
161 PcieHostPort, PcieRootPort, PcieUpstreamPort, PvPanicCode, PvPanicPciDevice,
162 VfioPciDevice,
163 };
164 pub use self::platform::VfioPlatformDevice;
165 pub use self::ac_adapter::AcAdapter;
166 pub use self::proxy::ChildProcIntf;
167 pub use self::proxy::Error as ProxyError;
168 pub use self::proxy::ProxyDevice;
169 #[cfg(feature = "usb")]
170 pub use self::usb::backend::device_provider::DeviceProvider;
171 #[cfg(feature = "usb")]
172 pub use self::usb::xhci::xhci_controller::XhciController;
173 pub use self::vfio::VfioContainer;
174 pub use self::vfio::VfioDevice;
175 pub use self::vfio::VfioDeviceType;
176 pub use self::virtio::vfio_wrapper;
177
178 } else if #[cfg(windows)] {
179 } else {
180 compile_error!("Unsupported platform");
181 }
182 }
183
184 /// Request CoIOMMU to unpin a specific range.
185 #[derive(Serialize, Deserialize, Debug)]
186 pub struct UnpinRequest {
187 /// The ranges presents (start gfn, count).
188 ranges: Vec<(u64, u64)>,
189 }
190
191 #[derive(Serialize, Deserialize, Debug)]
192 pub enum UnpinResponse {
193 Success,
194 Failed,
195 }
196
197 #[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
198 pub enum IommuDevType {
199 #[serde(rename = "off")]
200 #[default]
201 NoIommu,
202 #[serde(rename = "viommu")]
203 VirtioIommu,
204 #[serde(rename = "coiommu")]
205 CoIommu,
206 #[serde(rename = "pkvm-iommu")]
207 PkvmPviommu,
208 }
209
210 // Thread that handles commands sent to devices - such as snapshot, sleep, suspend
211 // Created when the VM is first created, and re-created on resumption of the VM.
create_devices_worker_thread( guest_memory: GuestMemory, io_bus: Arc<Bus>, mmio_bus: Arc<Bus>, device_ctrl_resp: Tube, ) -> std::io::Result<std::thread::JoinHandle<()>>212 pub fn create_devices_worker_thread(
213 guest_memory: GuestMemory,
214 io_bus: Arc<Bus>,
215 mmio_bus: Arc<Bus>,
216 device_ctrl_resp: Tube,
217 ) -> std::io::Result<std::thread::JoinHandle<()>> {
218 std::thread::Builder::new()
219 .name("device_control".to_string())
220 .spawn(move || {
221 let ex = Executor::new().expect("Failed to create an executor");
222
223 let async_control = AsyncTube::new(&ex, device_ctrl_resp).unwrap();
224 match ex.run_until(async move {
225 handle_command_tube(async_control, guest_memory, io_bus, mmio_bus).await
226 }) {
227 Ok(_) => {}
228 Err(e) => {
229 error!("Device control thread exited with error: {}", e);
230 }
231 };
232 })
233 }
234
sleep_buses(buses: &[&Bus]) -> anyhow::Result<()>235 fn sleep_buses(buses: &[&Bus]) -> anyhow::Result<()> {
236 for bus in buses {
237 bus.sleep_devices()
238 .with_context(|| format!("failed to sleep devices on {:?} bus", bus.get_bus_type()))?;
239 debug!("Devices slept successfully on {:?} bus", bus.get_bus_type());
240 }
241 Ok(())
242 }
243
wake_buses(buses: &[&Bus])244 fn wake_buses(buses: &[&Bus]) {
245 for bus in buses {
246 bus.wake_devices()
247 .with_context(|| format!("failed to wake devices on {:?} bus", bus.get_bus_type()))
248 // Some devices may have slept. Eternally.
249 // Recovery - impossible.
250 // Shut down VM.
251 .expect("VM panicked to avoid unexpected behavior");
252 debug!(
253 "Devices awoken successfully on {:?} Bus",
254 bus.get_bus_type()
255 );
256 }
257 }
258
259 // Use 64MB chunks when writing the memory snapshot (if encryption is used).
260 const MEMORY_SNAP_ENCRYPTED_CHUNK_SIZE_BYTES: usize = 1024 * 1024 * 64;
261
snapshot_handler( snapshot_writer: snapshot::SnapshotWriter, guest_memory: &GuestMemory, buses: &[&Bus], compress_memory: bool, ) -> anyhow::Result<()>262 async fn snapshot_handler(
263 snapshot_writer: snapshot::SnapshotWriter,
264 guest_memory: &GuestMemory,
265 buses: &[&Bus],
266 compress_memory: bool,
267 ) -> anyhow::Result<()> {
268 let mem_snap_start = Instant::now();
269 // SAFETY:
270 // VM & devices are stopped.
271 let guest_memory_metadata = unsafe {
272 guest_memory
273 .snapshot(
274 &mut snapshot_writer
275 .raw_fragment_with_chunk_size("mem", MEMORY_SNAP_ENCRYPTED_CHUNK_SIZE_BYTES)?,
276 compress_memory,
277 )
278 .context("failed to snapshot memory")?
279 };
280 snapshot_writer.write_fragment("mem_metadata", &guest_memory_metadata)?;
281
282 let mem_snap_duration_ms = mem_snap_start.elapsed().as_millis();
283 info!(
284 "snapshot: memory snapshotted {}MB in {}ms",
285 guest_memory.memory_size() / 1024 / 1024,
286 mem_snap_duration_ms
287 );
288 metrics::log_metric_with_details(
289 metrics::MetricEventType::SnapshotSaveMemoryLatency,
290 mem_snap_duration_ms as i64,
291 &metrics_events::RecordDetails {},
292 );
293
294 for (i, bus) in buses.iter().enumerate() {
295 bus.snapshot_devices(&snapshot_writer.add_namespace(&format!("bus{i}"))?)
296 .context("failed to snapshot bus devices")?;
297 debug!(
298 "Devices snapshot successfully for {:?} Bus",
299 bus.get_bus_type()
300 );
301 }
302 Ok(())
303 }
304
restore_handler( snapshot_reader: snapshot::SnapshotReader, guest_memory: &GuestMemory, buses: &[&Bus], ) -> anyhow::Result<()>305 async fn restore_handler(
306 snapshot_reader: snapshot::SnapshotReader,
307 guest_memory: &GuestMemory,
308 buses: &[&Bus],
309 ) -> anyhow::Result<()> {
310 let mem_restore_start = Instant::now();
311 let guest_memory_metadata = snapshot_reader.read_fragment("mem_metadata")?;
312 // SAFETY:
313 // VM & devices are stopped.
314 unsafe {
315 guest_memory.restore(
316 guest_memory_metadata,
317 &mut snapshot_reader.raw_fragment("mem")?,
318 )?
319 };
320 let mem_restore_duration_ms = mem_restore_start.elapsed().as_millis();
321 info!(
322 "snapshot: memory restored {}MB in {}ms",
323 guest_memory.memory_size() / 1024 / 1024,
324 mem_restore_duration_ms
325 );
326 metrics::log_metric_with_details(
327 metrics::MetricEventType::SnapshotRestoreMemoryLatency,
328 mem_restore_duration_ms as i64,
329 &metrics_events::RecordDetails {},
330 );
331
332 for (i, bus) in buses.iter().enumerate() {
333 bus.restore_devices(&snapshot_reader.namespace(&format!("bus{i}"))?)
334 .context("failed to restore bus devices")?;
335 debug!(
336 "Devices restore successfully for {:?} Bus",
337 bus.get_bus_type()
338 );
339 }
340 Ok(())
341 }
342
handle_command_tube( command_tube: AsyncTube, guest_memory: GuestMemory, io_bus: Arc<Bus>, mmio_bus: Arc<Bus>, ) -> anyhow::Result<()>343 async fn handle_command_tube(
344 command_tube: AsyncTube,
345 guest_memory: GuestMemory,
346 io_bus: Arc<Bus>,
347 mmio_bus: Arc<Bus>,
348 ) -> anyhow::Result<()> {
349 let buses = &[&*io_bus, &*mmio_bus];
350
351 // We assume devices are awake. This is safe because if the VM starts the
352 // sleeping state, run_control will ask us to sleep devices.
353 let mut devices_state = DevicesState::Wake;
354
355 loop {
356 match command_tube.next().await {
357 Ok(command) => {
358 match command {
359 DeviceControlCommand::SleepDevices => {
360 if let DevicesState::Wake = devices_state {
361 match sleep_buses(buses) {
362 Ok(()) => {
363 devices_state = DevicesState::Sleep;
364 }
365 Err(e) => {
366 error!("failed to sleep: {:#}", e);
367
368 // Failing to sleep could mean a single device failing to sleep.
369 // Wake up devices to resume functionality of the VM.
370 info!("Attempting to wake devices after failed sleep");
371 wake_buses(buses);
372
373 command_tube
374 .send(VmResponse::ErrString(e.to_string()))
375 .await
376 .context("failed to send response.")?;
377 continue;
378 }
379 }
380 }
381 command_tube
382 .send(VmResponse::Ok)
383 .await
384 .context("failed to reply to sleep command")?;
385 }
386 DeviceControlCommand::WakeDevices => {
387 if let DevicesState::Sleep = devices_state {
388 wake_buses(buses);
389 devices_state = DevicesState::Wake;
390 }
391 command_tube
392 .send(VmResponse::Ok)
393 .await
394 .context("failed to reply to wake devices request")?;
395 }
396 DeviceControlCommand::SnapshotDevices {
397 snapshot_writer,
398 compress_memory,
399 } => {
400 assert!(
401 matches!(devices_state, DevicesState::Sleep),
402 "devices must be sleeping to snapshot"
403 );
404 if let Err(e) =
405 snapshot_handler(snapshot_writer, &guest_memory, buses, compress_memory)
406 .await
407 {
408 error!("failed to snapshot: {:#}", e);
409 command_tube
410 .send(VmResponse::ErrString(e.to_string()))
411 .await
412 .context("Failed to send response")?;
413 continue;
414 }
415 command_tube
416 .send(VmResponse::Ok)
417 .await
418 .context("Failed to send response")?;
419 }
420 DeviceControlCommand::RestoreDevices { snapshot_reader } => {
421 assert!(
422 matches!(devices_state, DevicesState::Sleep),
423 "devices must be sleeping to restore"
424 );
425 if let Err(e) =
426 restore_handler(snapshot_reader, &guest_memory, &[&*io_bus, &*mmio_bus])
427 .await
428 {
429 error!("failed to restore: {:#}", e);
430 command_tube
431 .send(VmResponse::ErrString(e.to_string()))
432 .await
433 .context("Failed to send response")?;
434 continue;
435 }
436 command_tube
437 .send(VmResponse::Ok)
438 .await
439 .context("Failed to send response")?;
440 }
441 DeviceControlCommand::GetDevicesState => {
442 command_tube
443 .send(VmResponse::DevicesState(devices_state.clone()))
444 .await
445 .context("failed to send response")?;
446 }
447 DeviceControlCommand::Exit => {
448 return Ok(());
449 }
450 };
451 }
452 Err(e) => {
453 if matches!(e, TubeError::Disconnected) {
454 // Tube disconnected - shut down thread.
455 return Ok(());
456 }
457 return Err(anyhow!("Failed to receive: {}", e));
458 }
459 }
460 }
461 }
462