1 // Copyright 2018 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 use std::collections::BTreeMap; 6 use std::sync::Arc; 7 8 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] 9 use acpi_tables::sdt::SDT; 10 use anyhow::anyhow; 11 use anyhow::Context; 12 use base::error; 13 use base::info; 14 use base::AsRawDescriptor; 15 use base::AsRawDescriptors; 16 use base::Event; 17 use base::Protection; 18 use base::RawDescriptor; 19 use base::Result; 20 use base::Tube; 21 use data_model::Le32; 22 use hypervisor::Datamatch; 23 use libc::ERANGE; 24 use resources::Alloc; 25 use resources::AllocOptions; 26 use resources::SystemAllocator; 27 use serde::Deserialize; 28 use serde::Serialize; 29 use sync::Mutex; 30 use virtio_sys::virtio_config::VIRTIO_CONFIG_S_ACKNOWLEDGE; 31 use virtio_sys::virtio_config::VIRTIO_CONFIG_S_DRIVER; 32 use virtio_sys::virtio_config::VIRTIO_CONFIG_S_DRIVER_OK; 33 use virtio_sys::virtio_config::VIRTIO_CONFIG_S_FAILED; 34 use virtio_sys::virtio_config::VIRTIO_CONFIG_S_FEATURES_OK; 35 use virtio_sys::virtio_config::VIRTIO_CONFIG_S_NEEDS_RESET; 36 use vm_control::MemSlot; 37 use vm_control::VmMemoryDestination; 38 use vm_control::VmMemoryRequest; 39 use vm_control::VmMemoryResponse; 40 use vm_control::VmMemorySource; 41 use vm_memory::GuestAddress; 42 use vm_memory::GuestMemory; 43 use zerocopy::AsBytes; 44 use zerocopy::FromBytes; 45 46 use self::virtio_pci_common_config::VirtioPciCommonConfig; 47 use super::*; 48 use crate::pci::BarRange; 49 use crate::pci::MsixCap; 50 use crate::pci::MsixConfig; 51 use crate::pci::PciAddress; 52 use crate::pci::PciBarConfiguration; 53 use crate::pci::PciBarIndex; 54 use crate::pci::PciBarPrefetchable; 55 use crate::pci::PciBarRegionType; 56 use crate::pci::PciCapability; 57 use crate::pci::PciCapabilityID; 58 use crate::pci::PciClassCode; 59 use crate::pci::PciConfiguration; 60 use crate::pci::PciDevice; 61 use crate::pci::PciDeviceError; 62 use crate::pci::PciDisplaySubclass; 63 use crate::pci::PciHeaderType; 64 use crate::pci::PciId; 65 use crate::pci::PciInterruptPin; 66 use crate::pci::PciSubclass; 67 use crate::virtio::ipc_memory_mapper::IpcMemoryMapper; 68 use crate::IrqLevelEvent; 69 use crate::Suspendable; 70 71 #[repr(u8)] 72 #[derive(Debug, Copy, Clone, enumn::N)] 73 pub enum PciCapabilityType { 74 CommonConfig = 1, 75 NotifyConfig = 2, 76 IsrConfig = 3, 77 DeviceConfig = 4, 78 PciConfig = 5, 79 // Doorbell, Notification and SharedMemory are Virtio Vhost User related PCI 80 // capabilities. Specified in 5.7.7.4 here 81 // https://stefanha.github.io/virtio/vhost-user-slave.html#x1-2830007. 82 DoorbellConfig = 6, 83 NotificationConfig = 7, 84 SharedMemoryConfig = 8, 85 } 86 87 #[allow(dead_code)] 88 #[repr(C)] 89 #[derive(Clone, Copy, FromBytes, AsBytes)] 90 pub struct VirtioPciCap { 91 // cap_vndr and cap_next are autofilled based on id() in pci configuration 92 pub cap_vndr: u8, // Generic PCI field: PCI_CAP_ID_VNDR 93 pub cap_next: u8, // Generic PCI field: next ptr 94 pub cap_len: u8, // Generic PCI field: capability length 95 pub cfg_type: u8, // Identifies the structure. 96 pub bar: u8, // Where to find it. 97 id: u8, // Multiple capabilities of the same type 98 padding: [u8; 2], // Pad to full dword. 99 pub offset: Le32, // Offset within bar. 100 pub length: Le32, // Length of the structure, in bytes. 101 } 102 103 impl PciCapability for VirtioPciCap { bytes(&self) -> &[u8]104 fn bytes(&self) -> &[u8] { 105 self.as_bytes() 106 } 107 id(&self) -> PciCapabilityID108 fn id(&self) -> PciCapabilityID { 109 PciCapabilityID::VendorSpecific 110 } 111 writable_bits(&self) -> Vec<u32>112 fn writable_bits(&self) -> Vec<u32> { 113 vec![0u32; 4] 114 } 115 } 116 117 impl VirtioPciCap { new(cfg_type: PciCapabilityType, bar: u8, offset: u32, length: u32) -> Self118 pub fn new(cfg_type: PciCapabilityType, bar: u8, offset: u32, length: u32) -> Self { 119 VirtioPciCap { 120 cap_vndr: 0, 121 cap_next: 0, 122 cap_len: std::mem::size_of::<VirtioPciCap>() as u8, 123 cfg_type: cfg_type as u8, 124 bar, 125 id: 0, 126 padding: [0; 2], 127 offset: Le32::from(offset), 128 length: Le32::from(length), 129 } 130 } 131 set_cap_len(&mut self, cap_len: u8)132 pub fn set_cap_len(&mut self, cap_len: u8) { 133 self.cap_len = cap_len; 134 } 135 } 136 137 #[allow(dead_code)] 138 #[repr(C)] 139 #[derive(Clone, Copy, AsBytes, FromBytes)] 140 pub struct VirtioPciNotifyCap { 141 cap: VirtioPciCap, 142 notify_off_multiplier: Le32, 143 } 144 145 impl PciCapability for VirtioPciNotifyCap { bytes(&self) -> &[u8]146 fn bytes(&self) -> &[u8] { 147 self.as_bytes() 148 } 149 id(&self) -> PciCapabilityID150 fn id(&self) -> PciCapabilityID { 151 PciCapabilityID::VendorSpecific 152 } 153 writable_bits(&self) -> Vec<u32>154 fn writable_bits(&self) -> Vec<u32> { 155 vec![0u32; 5] 156 } 157 } 158 159 impl VirtioPciNotifyCap { new( cfg_type: PciCapabilityType, bar: u8, offset: u32, length: u32, multiplier: Le32, ) -> Self160 pub fn new( 161 cfg_type: PciCapabilityType, 162 bar: u8, 163 offset: u32, 164 length: u32, 165 multiplier: Le32, 166 ) -> Self { 167 VirtioPciNotifyCap { 168 cap: VirtioPciCap { 169 cap_vndr: 0, 170 cap_next: 0, 171 cap_len: std::mem::size_of::<VirtioPciNotifyCap>() as u8, 172 cfg_type: cfg_type as u8, 173 bar, 174 id: 0, 175 padding: [0; 2], 176 offset: Le32::from(offset), 177 length: Le32::from(length), 178 }, 179 notify_off_multiplier: multiplier, 180 } 181 } 182 } 183 184 #[repr(C)] 185 #[derive(Clone, Copy, AsBytes, FromBytes)] 186 pub struct VirtioPciShmCap { 187 cap: VirtioPciCap, 188 offset_hi: Le32, // Most sig 32 bits of offset 189 length_hi: Le32, // Most sig 32 bits of length 190 } 191 192 impl PciCapability for VirtioPciShmCap { bytes(&self) -> &[u8]193 fn bytes(&self) -> &[u8] { 194 self.as_bytes() 195 } 196 id(&self) -> PciCapabilityID197 fn id(&self) -> PciCapabilityID { 198 PciCapabilityID::VendorSpecific 199 } 200 writable_bits(&self) -> Vec<u32>201 fn writable_bits(&self) -> Vec<u32> { 202 vec![0u32; 6] 203 } 204 } 205 206 impl VirtioPciShmCap { new(cfg_type: PciCapabilityType, bar: u8, offset: u64, length: u64, shmid: u8) -> Self207 pub fn new(cfg_type: PciCapabilityType, bar: u8, offset: u64, length: u64, shmid: u8) -> Self { 208 VirtioPciShmCap { 209 cap: VirtioPciCap { 210 cap_vndr: 0, 211 cap_next: 0, 212 cap_len: std::mem::size_of::<VirtioPciShmCap>() as u8, 213 cfg_type: cfg_type as u8, 214 bar, 215 id: shmid, 216 padding: [0; 2], 217 offset: Le32::from(offset as u32), 218 length: Le32::from(length as u32), 219 }, 220 offset_hi: Le32::from((offset >> 32) as u32), 221 length_hi: Le32::from((length >> 32) as u32), 222 } 223 } 224 } 225 226 /// Subclasses for virtio. 227 #[allow(dead_code)] 228 #[derive(Copy, Clone)] 229 pub enum PciVirtioSubclass { 230 NonTransitionalBase = 0xff, 231 } 232 233 impl PciSubclass for PciVirtioSubclass { get_register_value(&self) -> u8234 fn get_register_value(&self) -> u8 { 235 *self as u8 236 } 237 } 238 239 // Allocate one bar for the structs pointed to by the capability structures. 240 const COMMON_CONFIG_BAR_OFFSET: u64 = 0x0000; 241 const COMMON_CONFIG_SIZE: u64 = 56; 242 const COMMON_CONFIG_LAST: u64 = COMMON_CONFIG_BAR_OFFSET + COMMON_CONFIG_SIZE - 1; 243 const ISR_CONFIG_BAR_OFFSET: u64 = 0x1000; 244 const ISR_CONFIG_SIZE: u64 = 1; 245 const ISR_CONFIG_LAST: u64 = ISR_CONFIG_BAR_OFFSET + ISR_CONFIG_SIZE - 1; 246 const DEVICE_CONFIG_BAR_OFFSET: u64 = 0x2000; 247 const DEVICE_CONFIG_SIZE: u64 = 0x1000; 248 const DEVICE_CONFIG_LAST: u64 = DEVICE_CONFIG_BAR_OFFSET + DEVICE_CONFIG_SIZE - 1; 249 const NOTIFICATION_BAR_OFFSET: u64 = 0x3000; 250 const NOTIFICATION_SIZE: u64 = 0x1000; 251 const NOTIFICATION_LAST: u64 = NOTIFICATION_BAR_OFFSET + NOTIFICATION_SIZE - 1; 252 const MSIX_TABLE_BAR_OFFSET: u64 = 0x6000; 253 const MSIX_TABLE_SIZE: u64 = 0x1000; 254 const MSIX_TABLE_LAST: u64 = MSIX_TABLE_BAR_OFFSET + MSIX_TABLE_SIZE - 1; 255 const MSIX_PBA_BAR_OFFSET: u64 = 0x7000; 256 const MSIX_PBA_SIZE: u64 = 0x1000; 257 const MSIX_PBA_LAST: u64 = MSIX_PBA_BAR_OFFSET + MSIX_PBA_SIZE - 1; 258 const CAPABILITY_BAR_SIZE: u64 = 0x8000; 259 260 const NOTIFY_OFF_MULTIPLIER: u32 = 4; // A dword per notification address. 261 262 const VIRTIO_PCI_VENDOR_ID: u16 = 0x1af4; 263 const VIRTIO_PCI_DEVICE_ID_BASE: u16 = 0x1040; // Add to device type to get device ID. 264 const VIRTIO_PCI_REVISION_ID: u8 = 1; 265 266 const CAPABILITIES_BAR_NUM: usize = 0; 267 const SHMEM_BAR_NUM: usize = 2; 268 269 /// Implements the 270 /// [PCI](http://docs.oasis-open.org/virtio/virtio/v1.0/cs04/virtio-v1.0-cs04.html#x1-650001) 271 /// transport for virtio devices. 272 pub struct VirtioPciDevice { 273 config_regs: PciConfiguration, 274 preferred_address: Option<PciAddress>, 275 pci_address: Option<PciAddress>, 276 277 device: Box<dyn VirtioDevice>, 278 device_activated: bool, 279 disable_intx: bool, 280 281 interrupt: Option<Interrupt>, 282 interrupt_evt: Option<IrqLevelEvent>, 283 queues: Vec<Queue>, 284 queue_evts: Vec<Event>, 285 mem: GuestMemory, 286 settings_bar: u8, 287 msix_config: Arc<Mutex<MsixConfig>>, 288 msix_cap_reg_idx: Option<usize>, 289 common_config: VirtioPciCommonConfig, 290 291 iommu: Option<Arc<Mutex<IpcMemoryMapper>>>, 292 293 // A tube that is present if the device has shared memory regions, and 294 // is used to map/unmap files into the shared memory region. 295 shared_memory_tube: Option<Tube>, 296 297 // Tube for request registeration of ioevents when PCI BAR reprogramming is detected. 298 ioevent_tube: Tube, 299 } 300 301 impl VirtioPciDevice { 302 /// Constructs a new PCI transport for the given virtio device. new( mem: GuestMemory, device: Box<dyn VirtioDevice>, msi_device_tube: Tube, disable_intx: bool, shared_memory_tube: Option<Tube>, ioevent_tube: Tube, ) -> Result<Self>303 pub fn new( 304 mem: GuestMemory, 305 device: Box<dyn VirtioDevice>, 306 msi_device_tube: Tube, 307 disable_intx: bool, 308 shared_memory_tube: Option<Tube>, 309 ioevent_tube: Tube, 310 ) -> Result<Self> { 311 // shared_memory_tube is required if there are shared memory regions. 312 assert_eq!( 313 device.get_shared_memory_region().is_none(), 314 shared_memory_tube.is_none() 315 ); 316 317 let mut queue_evts = Vec::new(); 318 for _ in device.queue_max_sizes() { 319 queue_evts.push(Event::new()?) 320 } 321 let queues: Vec<Queue> = device 322 .queue_max_sizes() 323 .iter() 324 .map(|&s| Queue::new(s)) 325 .collect(); 326 327 let pci_device_id = VIRTIO_PCI_DEVICE_ID_BASE + device.device_type() as u16; 328 329 let (pci_device_class, pci_device_subclass) = match device.device_type() { 330 DeviceType::Gpu => ( 331 PciClassCode::DisplayController, 332 &PciDisplaySubclass::Other as &dyn PciSubclass, 333 ), 334 _ => ( 335 PciClassCode::TooOld, 336 &PciVirtioSubclass::NonTransitionalBase as &dyn PciSubclass, 337 ), 338 }; 339 340 let num_interrupts = device.num_interrupts(); 341 342 // One MSI-X vector per queue plus one for configuration changes. 343 let msix_num = u16::try_from(num_interrupts + 1).map_err(|_| base::Error::new(ERANGE))?; 344 let msix_config = Arc::new(Mutex::new(MsixConfig::new( 345 msix_num, 346 msi_device_tube, 347 PciId::new(VIRTIO_PCI_VENDOR_ID, pci_device_id).into(), 348 device.debug_label(), 349 ))); 350 351 let config_regs = PciConfiguration::new( 352 VIRTIO_PCI_VENDOR_ID, 353 pci_device_id, 354 pci_device_class, 355 pci_device_subclass, 356 None, 357 PciHeaderType::Device, 358 VIRTIO_PCI_VENDOR_ID, 359 pci_device_id, 360 VIRTIO_PCI_REVISION_ID, 361 ); 362 363 Ok(VirtioPciDevice { 364 config_regs, 365 preferred_address: device.pci_address(), 366 pci_address: None, 367 device, 368 device_activated: false, 369 disable_intx, 370 interrupt: None, 371 interrupt_evt: None, 372 queues, 373 queue_evts, 374 mem, 375 settings_bar: 0, 376 msix_config, 377 msix_cap_reg_idx: None, 378 common_config: VirtioPciCommonConfig { 379 driver_status: 0, 380 config_generation: 0, 381 device_feature_select: 0, 382 driver_feature_select: 0, 383 queue_select: 0, 384 msix_config: VIRTIO_MSI_NO_VECTOR, 385 }, 386 iommu: None, 387 shared_memory_tube, 388 ioevent_tube, 389 }) 390 } 391 is_driver_ready(&self) -> bool392 fn is_driver_ready(&self) -> bool { 393 let ready_bits = (VIRTIO_CONFIG_S_ACKNOWLEDGE 394 | VIRTIO_CONFIG_S_DRIVER 395 | VIRTIO_CONFIG_S_DRIVER_OK 396 | VIRTIO_CONFIG_S_FEATURES_OK) as u8; 397 (self.common_config.driver_status & ready_bits) == ready_bits 398 && self.common_config.driver_status & VIRTIO_CONFIG_S_FAILED as u8 == 0 399 } 400 401 /// Determines if the driver has requested the device reset itself is_reset_requested(&self) -> bool402 fn is_reset_requested(&self) -> bool { 403 self.common_config.driver_status == DEVICE_RESET as u8 404 } 405 add_settings_pci_capabilities( &mut self, settings_bar: u8, ) -> std::result::Result<(), PciDeviceError>406 fn add_settings_pci_capabilities( 407 &mut self, 408 settings_bar: u8, 409 ) -> std::result::Result<(), PciDeviceError> { 410 // Add pointers to the different configuration structures from the PCI capabilities. 411 let common_cap = VirtioPciCap::new( 412 PciCapabilityType::CommonConfig, 413 settings_bar, 414 COMMON_CONFIG_BAR_OFFSET as u32, 415 COMMON_CONFIG_SIZE as u32, 416 ); 417 self.config_regs 418 .add_capability(&common_cap) 419 .map_err(PciDeviceError::CapabilitiesSetup)?; 420 421 let isr_cap = VirtioPciCap::new( 422 PciCapabilityType::IsrConfig, 423 settings_bar, 424 ISR_CONFIG_BAR_OFFSET as u32, 425 ISR_CONFIG_SIZE as u32, 426 ); 427 self.config_regs 428 .add_capability(&isr_cap) 429 .map_err(PciDeviceError::CapabilitiesSetup)?; 430 431 // TODO(dgreid) - set based on device's configuration size? 432 let device_cap = VirtioPciCap::new( 433 PciCapabilityType::DeviceConfig, 434 settings_bar, 435 DEVICE_CONFIG_BAR_OFFSET as u32, 436 DEVICE_CONFIG_SIZE as u32, 437 ); 438 self.config_regs 439 .add_capability(&device_cap) 440 .map_err(PciDeviceError::CapabilitiesSetup)?; 441 442 let notify_cap = VirtioPciNotifyCap::new( 443 PciCapabilityType::NotifyConfig, 444 settings_bar, 445 NOTIFICATION_BAR_OFFSET as u32, 446 NOTIFICATION_SIZE as u32, 447 Le32::from(NOTIFY_OFF_MULTIPLIER), 448 ); 449 self.config_regs 450 .add_capability(¬ify_cap) 451 .map_err(PciDeviceError::CapabilitiesSetup)?; 452 453 //TODO(dgreid) - How will the configuration_cap work? 454 let configuration_cap = VirtioPciCap::new(PciCapabilityType::PciConfig, 0, 0, 0); 455 self.config_regs 456 .add_capability(&configuration_cap) 457 .map_err(PciDeviceError::CapabilitiesSetup)?; 458 459 let msix_cap = MsixCap::new( 460 settings_bar, 461 self.msix_config.lock().num_vectors(), 462 MSIX_TABLE_BAR_OFFSET as u32, 463 settings_bar, 464 MSIX_PBA_BAR_OFFSET as u32, 465 ); 466 let msix_offset = self 467 .config_regs 468 .add_capability(&msix_cap) 469 .map_err(PciDeviceError::CapabilitiesSetup)?; 470 self.msix_cap_reg_idx = Some(msix_offset / 4); 471 472 self.settings_bar = settings_bar; 473 Ok(()) 474 } 475 476 /// Activates the underlying `VirtioDevice`. `assign_irq` has to be called first. activate(&mut self) -> anyhow::Result<()>477 fn activate(&mut self) -> anyhow::Result<()> { 478 let interrupt_evt = if let Some(ref evt) = self.interrupt_evt { 479 evt.try_clone() 480 .with_context(|| format!("{} failed to clone interrupt_evt", self.debug_label()))? 481 } else { 482 return Err(anyhow!("{} interrupt_evt is none", self.debug_label())); 483 }; 484 485 let mem = self.mem.clone(); 486 487 let interrupt = Interrupt::new( 488 interrupt_evt, 489 Some(self.msix_config.clone()), 490 self.common_config.msix_config, 491 ); 492 self.interrupt = Some(interrupt.clone()); 493 494 // Use ready queues and their events. 495 let queues = self 496 .queues 497 .iter_mut() 498 .zip(self.queue_evts.iter()) 499 .filter(|(q, _)| q.ready()) 500 .map(|(queue, evt)| { 501 Ok(( 502 queue.activate().context("failed to activate queue")?, 503 evt.try_clone().context("failed to clone queue_evt")?, 504 )) 505 }) 506 .collect::<anyhow::Result<Vec<(Queue, Event)>>>()?; 507 508 if let Some(iommu) = &self.iommu { 509 self.device.set_iommu(iommu); 510 } 511 512 if let Err(e) = self.device.activate(mem, interrupt, queues) { 513 error!("{} activate failed: {:#}", self.debug_label(), e); 514 self.common_config.driver_status |= VIRTIO_CONFIG_S_NEEDS_RESET as u8; 515 } else { 516 self.device_activated = true; 517 } 518 519 Ok(()) 520 } 521 } 522 523 impl PciDevice for VirtioPciDevice { supports_iommu(&self) -> bool524 fn supports_iommu(&self) -> bool { 525 self.device.supports_iommu() 526 } 527 debug_label(&self) -> String528 fn debug_label(&self) -> String { 529 format!("pci{}", self.device.debug_label()) 530 } 531 preferred_address(&self) -> Option<PciAddress>532 fn preferred_address(&self) -> Option<PciAddress> { 533 self.preferred_address 534 } 535 allocate_address( &mut self, resources: &mut SystemAllocator, ) -> std::result::Result<PciAddress, PciDeviceError>536 fn allocate_address( 537 &mut self, 538 resources: &mut SystemAllocator, 539 ) -> std::result::Result<PciAddress, PciDeviceError> { 540 if self.pci_address.is_none() { 541 if let Some(address) = self.preferred_address { 542 if !resources.reserve_pci( 543 Alloc::PciBar { 544 bus: address.bus, 545 dev: address.dev, 546 func: address.func, 547 bar: 0, 548 }, 549 self.debug_label(), 550 ) { 551 return Err(PciDeviceError::PciAllocationFailed); 552 } 553 self.pci_address = Some(address); 554 } else { 555 self.pci_address = match resources.allocate_pci(0, self.debug_label()) { 556 Some(Alloc::PciBar { 557 bus, 558 dev, 559 func, 560 bar: _, 561 }) => Some(PciAddress { bus, dev, func }), 562 _ => None, 563 } 564 } 565 } 566 self.pci_address.ok_or(PciDeviceError::PciAllocationFailed) 567 } 568 keep_rds(&self) -> Vec<RawDescriptor>569 fn keep_rds(&self) -> Vec<RawDescriptor> { 570 let mut rds = self.device.keep_rds(); 571 if let Some(interrupt_evt) = &self.interrupt_evt { 572 rds.extend(interrupt_evt.as_raw_descriptors()); 573 } 574 let descriptor = self.msix_config.lock().get_msi_socket(); 575 rds.push(descriptor); 576 if let Some(iommu) = &self.iommu { 577 rds.append(&mut iommu.lock().as_raw_descriptors()); 578 } 579 rds.push(self.ioevent_tube.as_raw_descriptor()); 580 rds 581 } 582 assign_irq(&mut self, irq_evt: IrqLevelEvent, pin: PciInterruptPin, irq_num: u32)583 fn assign_irq(&mut self, irq_evt: IrqLevelEvent, pin: PciInterruptPin, irq_num: u32) { 584 self.interrupt_evt = Some(irq_evt); 585 if !self.disable_intx { 586 self.config_regs.set_irq(irq_num as u8, pin); 587 } 588 } 589 allocate_io_bars( &mut self, resources: &mut SystemAllocator, ) -> std::result::Result<Vec<BarRange>, PciDeviceError>590 fn allocate_io_bars( 591 &mut self, 592 resources: &mut SystemAllocator, 593 ) -> std::result::Result<Vec<BarRange>, PciDeviceError> { 594 let address = self 595 .pci_address 596 .expect("allocaten_address must be called prior to allocate_io_bars"); 597 // Allocate one bar for the structures pointed to by the capability structures. 598 let mut ranges: Vec<BarRange> = Vec::new(); 599 let settings_config_addr = resources 600 .allocate_mmio( 601 CAPABILITY_BAR_SIZE, 602 Alloc::PciBar { 603 bus: address.bus, 604 dev: address.dev, 605 func: address.func, 606 bar: 0, 607 }, 608 format!("virtio-{}-cap_bar", self.device.device_type()), 609 AllocOptions::new() 610 .max_address(u32::MAX.into()) 611 .align(CAPABILITY_BAR_SIZE), 612 ) 613 .map_err(|e| PciDeviceError::IoAllocationFailed(CAPABILITY_BAR_SIZE, e))?; 614 let config = PciBarConfiguration::new( 615 CAPABILITIES_BAR_NUM, 616 CAPABILITY_BAR_SIZE, 617 PciBarRegionType::Memory32BitRegion, 618 PciBarPrefetchable::NotPrefetchable, 619 ) 620 .set_address(settings_config_addr); 621 let settings_bar = self 622 .config_regs 623 .add_pci_bar(config) 624 .map_err(|e| PciDeviceError::IoRegistrationFailed(settings_config_addr, e))? 625 as u8; 626 ranges.push(BarRange { 627 addr: settings_config_addr, 628 size: CAPABILITY_BAR_SIZE, 629 prefetchable: false, 630 }); 631 632 // Once the BARs are allocated, the capabilities can be added to the PCI configuration. 633 self.add_settings_pci_capabilities(settings_bar)?; 634 635 Ok(ranges) 636 } 637 allocate_device_bars( &mut self, resources: &mut SystemAllocator, ) -> std::result::Result<Vec<BarRange>, PciDeviceError>638 fn allocate_device_bars( 639 &mut self, 640 resources: &mut SystemAllocator, 641 ) -> std::result::Result<Vec<BarRange>, PciDeviceError> { 642 let address = self 643 .pci_address 644 .expect("allocaten_address must be called prior to allocate_device_bars"); 645 let mut ranges: Vec<BarRange> = Vec::new(); 646 647 let configs = self.device.get_device_bars(address); 648 let configs = if !configs.is_empty() { 649 configs 650 } else { 651 let region = match self.device.get_shared_memory_region() { 652 None => return Ok(Vec::new()), 653 Some(r) => r, 654 }; 655 let config = PciBarConfiguration::new( 656 SHMEM_BAR_NUM, 657 region 658 .length 659 .checked_next_power_of_two() 660 .expect("bar too large"), 661 PciBarRegionType::Memory64BitRegion, 662 PciBarPrefetchable::Prefetchable, 663 ); 664 665 let alloc = Alloc::PciBar { 666 bus: address.bus, 667 dev: address.dev, 668 func: address.func, 669 bar: config.bar_index() as u8, 670 }; 671 672 self.device 673 .set_shared_memory_mapper(Box::new(VmRequester::new( 674 self.shared_memory_tube 675 .take() 676 .expect("missing shared_memory_tube"), 677 alloc, 678 ))); 679 680 vec![config] 681 }; 682 683 for config in configs { 684 let device_addr = resources 685 .allocate_mmio( 686 config.size(), 687 Alloc::PciBar { 688 bus: address.bus, 689 dev: address.dev, 690 func: address.func, 691 bar: config.bar_index() as u8, 692 }, 693 format!("virtio-{}-custom_bar", self.device.device_type()), 694 AllocOptions::new() 695 .prefetchable(config.is_prefetchable()) 696 .align(config.size()), 697 ) 698 .map_err(|e| PciDeviceError::IoAllocationFailed(config.size(), e))?; 699 let config = config.set_address(device_addr); 700 let _device_bar = self 701 .config_regs 702 .add_pci_bar(config) 703 .map_err(|e| PciDeviceError::IoRegistrationFailed(device_addr, e))?; 704 ranges.push(BarRange { 705 addr: device_addr, 706 size: config.size(), 707 prefetchable: false, 708 }); 709 } 710 711 if self.device.get_shared_memory_region().is_some() { 712 self.device 713 .set_shared_memory_region_base(GuestAddress(ranges[0].addr)); 714 } 715 716 Ok(ranges) 717 } 718 get_bar_configuration(&self, bar_num: usize) -> Option<PciBarConfiguration>719 fn get_bar_configuration(&self, bar_num: usize) -> Option<PciBarConfiguration> { 720 self.config_regs.get_bar_configuration(bar_num) 721 } 722 register_device_capabilities(&mut self) -> std::result::Result<(), PciDeviceError>723 fn register_device_capabilities(&mut self) -> std::result::Result<(), PciDeviceError> { 724 let mut caps = self.device.get_device_caps(); 725 if let Some(region) = self.device.get_shared_memory_region() { 726 caps.push(Box::new(VirtioPciShmCap::new( 727 PciCapabilityType::SharedMemoryConfig, 728 SHMEM_BAR_NUM as u8, 729 0, 730 region.length, 731 region.id, 732 ))); 733 } 734 735 for cap in caps { 736 self.config_regs 737 .add_capability(&*cap) 738 .map_err(PciDeviceError::CapabilitiesSetup)?; 739 } 740 741 Ok(()) 742 } 743 ioevents(&self) -> Vec<(&Event, u64, Datamatch)>744 fn ioevents(&self) -> Vec<(&Event, u64, Datamatch)> { 745 let bar0 = self.config_regs.get_bar_addr(self.settings_bar as usize); 746 let notify_base = bar0 + NOTIFICATION_BAR_OFFSET; 747 self.queue_evts 748 .iter() 749 .enumerate() 750 .map(|(i, event)| { 751 ( 752 event, 753 notify_base + i as u64 * NOTIFY_OFF_MULTIPLIER as u64, 754 Datamatch::AnyLength, 755 ) 756 }) 757 .collect() 758 } 759 get_vm_memory_request_tube(&self) -> Option<&Tube>760 fn get_vm_memory_request_tube(&self) -> Option<&Tube> { 761 Some(&self.ioevent_tube) 762 } 763 read_config_register(&self, reg_idx: usize) -> u32764 fn read_config_register(&self, reg_idx: usize) -> u32 { 765 let mut data: u32 = self.config_regs.read_reg(reg_idx); 766 if reg_idx < 5 && self.debug_label() == "pcivirtio-gpu" { 767 info!("virtio gpu read config register {}", reg_idx); 768 } 769 if let Some(msix_cap_reg_idx) = self.msix_cap_reg_idx { 770 if msix_cap_reg_idx == reg_idx { 771 data = self.msix_config.lock().read_msix_capability(data); 772 } 773 } 774 775 data 776 } 777 write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8])778 fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]) { 779 if let Some(msix_cap_reg_idx) = self.msix_cap_reg_idx { 780 if msix_cap_reg_idx == reg_idx { 781 let behavior = self.msix_config.lock().write_msix_capability(offset, data); 782 self.device.control_notify(behavior); 783 } 784 } 785 786 self.config_regs.write_reg(reg_idx, offset, data) 787 } 788 read_bar(&mut self, addr: u64, data: &mut [u8])789 fn read_bar(&mut self, addr: u64, data: &mut [u8]) { 790 let bar = match self 791 .config_regs 792 .get_bars() 793 .find(|bar| bar.address_range().contains(&addr)) 794 { 795 Some(bar) => bar, 796 None => return, 797 }; 798 799 if bar.bar_index() == self.settings_bar as PciBarIndex { 800 let offset = addr - bar.address(); 801 match offset { 802 COMMON_CONFIG_BAR_OFFSET..=COMMON_CONFIG_LAST => self.common_config.read( 803 offset - COMMON_CONFIG_BAR_OFFSET, 804 data, 805 &mut self.queues, 806 self.device.as_mut(), 807 ), 808 ISR_CONFIG_BAR_OFFSET..=ISR_CONFIG_LAST => { 809 if let Some(v) = data.get_mut(0) { 810 // Reading this register resets it to 0. 811 *v = if let Some(interrupt) = &self.interrupt { 812 interrupt.read_and_reset_interrupt_status() 813 } else { 814 0 815 }; 816 } 817 } 818 DEVICE_CONFIG_BAR_OFFSET..=DEVICE_CONFIG_LAST => { 819 self.device 820 .read_config(offset - DEVICE_CONFIG_BAR_OFFSET, data); 821 } 822 NOTIFICATION_BAR_OFFSET..=NOTIFICATION_LAST => { 823 // Handled with ioevents. 824 } 825 MSIX_TABLE_BAR_OFFSET..=MSIX_TABLE_LAST => { 826 self.msix_config 827 .lock() 828 .read_msix_table(offset - MSIX_TABLE_BAR_OFFSET, data); 829 } 830 MSIX_PBA_BAR_OFFSET..=MSIX_PBA_LAST => { 831 self.msix_config 832 .lock() 833 .read_pba_entries(offset - MSIX_PBA_BAR_OFFSET, data); 834 } 835 _ => (), 836 } 837 } else { 838 self.device 839 .read_bar(bar.bar_index(), addr - bar.address(), data); 840 } 841 } 842 write_bar(&mut self, addr: u64, data: &[u8])843 fn write_bar(&mut self, addr: u64, data: &[u8]) { 844 let bar = match self 845 .config_regs 846 .get_bars() 847 .find(|bar| bar.address_range().contains(&addr)) 848 { 849 Some(bar) => bar, 850 None => return, 851 }; 852 853 if bar.bar_index() == self.settings_bar as PciBarIndex { 854 let offset = addr - bar.address(); 855 match offset { 856 COMMON_CONFIG_BAR_OFFSET..=COMMON_CONFIG_LAST => self.common_config.write( 857 offset - COMMON_CONFIG_BAR_OFFSET, 858 data, 859 &mut self.queues, 860 self.device.as_mut(), 861 ), 862 ISR_CONFIG_BAR_OFFSET..=ISR_CONFIG_LAST => { 863 if let Some(v) = data.first() { 864 if let Some(interrupt) = &self.interrupt { 865 interrupt.clear_interrupt_status_bits(*v); 866 } 867 } 868 } 869 DEVICE_CONFIG_BAR_OFFSET..=DEVICE_CONFIG_LAST => { 870 self.device 871 .write_config(offset - DEVICE_CONFIG_BAR_OFFSET, data); 872 } 873 NOTIFICATION_BAR_OFFSET..=NOTIFICATION_LAST => { 874 // Handled with ioevents. 875 } 876 MSIX_TABLE_BAR_OFFSET..=MSIX_TABLE_LAST => { 877 let behavior = self 878 .msix_config 879 .lock() 880 .write_msix_table(offset - MSIX_TABLE_BAR_OFFSET, data); 881 self.device.control_notify(behavior); 882 } 883 MSIX_PBA_BAR_OFFSET..=MSIX_PBA_LAST => { 884 self.msix_config 885 .lock() 886 .write_pba_entries(offset - MSIX_PBA_BAR_OFFSET, data); 887 } 888 _ => (), 889 } 890 } else { 891 self.device 892 .write_bar(bar.bar_index(), addr - bar.address(), data); 893 } 894 895 if !self.device_activated && self.is_driver_ready() { 896 if let Some(iommu) = &self.iommu { 897 for q in &mut self.queues { 898 q.set_iommu(Arc::clone(iommu)); 899 } 900 } 901 902 if let Err(e) = self.activate() { 903 error!("failed to activate device: {:#}", e); 904 } 905 } 906 907 // Device has been reset by the driver 908 if self.device_activated && self.is_reset_requested() && self.device.reset() { 909 self.device_activated = false; 910 // reset queues 911 self.queues.iter_mut().for_each(Queue::reset); 912 // select queue 0 by default 913 self.common_config.queue_select = 0; 914 } 915 } 916 on_device_sandboxed(&mut self)917 fn on_device_sandboxed(&mut self) { 918 self.device.on_device_sandboxed(); 919 } 920 921 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] generate_acpi(&mut self, sdts: Vec<SDT>) -> Option<Vec<SDT>>922 fn generate_acpi(&mut self, sdts: Vec<SDT>) -> Option<Vec<SDT>> { 923 self.device.generate_acpi(&self.pci_address, sdts) 924 } 925 set_iommu(&mut self, iommu: IpcMemoryMapper) -> anyhow::Result<()>926 fn set_iommu(&mut self, iommu: IpcMemoryMapper) -> anyhow::Result<()> { 927 assert!(self.supports_iommu()); 928 self.iommu = Some(Arc::new(Mutex::new(iommu))); 929 Ok(()) 930 } 931 } 932 933 #[derive(Serialize, Deserialize)] 934 struct VirtioPciDeviceSnapshot { 935 inner_device: serde_json::Value, 936 msix_config: serde_json::Value, 937 } 938 939 impl Suspendable for VirtioPciDevice { sleep(&mut self) -> anyhow::Result<()>940 fn sleep(&mut self) -> anyhow::Result<()> { 941 if let Some(state) = self.device.stop()? { 942 self.queues = state.queues; 943 } 944 Ok(()) 945 } 946 wake(&mut self) -> anyhow::Result<()>947 fn wake(&mut self) -> anyhow::Result<()> { 948 if self.device_activated { 949 self.activate()?; 950 } 951 Ok(()) 952 } 953 snapshot(&self) -> anyhow::Result<serde_json::Value>954 fn snapshot(&self) -> anyhow::Result<serde_json::Value> { 955 serde_json::to_value(VirtioPciDeviceSnapshot { 956 inner_device: self.device.snapshot()?, 957 msix_config: self.msix_config.lock().snapshot()?, 958 }) 959 .context("failed to serialize VirtioPciDeviceSnapshot") 960 } 961 restore(&mut self, data: serde_json::Value) -> anyhow::Result<()>962 fn restore(&mut self, data: serde_json::Value) -> anyhow::Result<()> { 963 let deser: VirtioPciDeviceSnapshot = serde_json::from_value(data)?; 964 self.msix_config.lock().restore(deser.msix_config)?; 965 self.device.restore(deser.inner_device) 966 } 967 } 968 969 struct VmRequester { 970 tube: Tube, 971 alloc: Alloc, 972 mappings: BTreeMap<u64, MemSlot>, 973 } 974 975 impl VmRequester { new(tube: Tube, alloc: Alloc) -> Self976 fn new(tube: Tube, alloc: Alloc) -> Self { 977 Self { 978 tube, 979 alloc, 980 mappings: BTreeMap::new(), 981 } 982 } 983 } 984 985 impl SharedMemoryMapper for VmRequester { add_mapping( &mut self, source: VmMemorySource, offset: u64, prot: Protection, ) -> anyhow::Result<()>986 fn add_mapping( 987 &mut self, 988 source: VmMemorySource, 989 offset: u64, 990 prot: Protection, 991 ) -> anyhow::Result<()> { 992 let request = VmMemoryRequest::RegisterMemory { 993 source, 994 dest: VmMemoryDestination::ExistingAllocation { 995 allocation: self.alloc, 996 offset, 997 }, 998 prot, 999 }; 1000 self.tube.send(&request).context("failed to send request")?; 1001 match self 1002 .tube 1003 .recv() 1004 .context("failed to recieve request response")? 1005 { 1006 VmMemoryResponse::RegisterMemory { pfn: _, slot } => { 1007 self.mappings.insert(offset, slot); 1008 Ok(()) 1009 } 1010 e => Err(anyhow!("unexpected response {:?}", e)), 1011 } 1012 } 1013 remove_mapping(&mut self, offset: u64) -> anyhow::Result<()>1014 fn remove_mapping(&mut self, offset: u64) -> anyhow::Result<()> { 1015 let slot = self.mappings.remove(&offset).context("invalid offset")?; 1016 self.tube 1017 .send(&VmMemoryRequest::UnregisterMemory(slot)) 1018 .context("failed to send request")?; 1019 match self 1020 .tube 1021 .recv() 1022 .context("failed to recieve request response")? 1023 { 1024 VmMemoryResponse::Ok => Ok(()), 1025 e => Err(anyhow!(format!("unexpected response {:?}", e))), 1026 } 1027 } 1028 as_raw_descriptor(&self) -> Option<RawDescriptor>1029 fn as_raw_descriptor(&self) -> Option<RawDescriptor> { 1030 Some(self.tube.as_raw_descriptor()) 1031 } 1032 } 1033