1 // Copyright 2021 The Chromium OS Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 use std::str::FromStr; 6 use std::sync::Arc; 7 use sync::Mutex; 8 9 use crate::bus::{HostHotPlugKey, HotPlugBus}; 10 use crate::pci::pci_configuration::PciCapabilityID; 11 use crate::pci::{MsixConfig, PciAddress, PciCapability, PciDeviceError}; 12 13 use crate::pci::pcie::pci_bridge::PciBridgeBusRange; 14 use crate::pci::pcie::pcie_device::{PciPmcCap, PcieCap, PcieDevice, PmcConfig}; 15 use crate::pci::pcie::pcie_host::PcieHostRootPort; 16 use crate::pci::pcie::*; 17 18 use anyhow::{anyhow, Result}; 19 use base::warn; 20 use data_model::DataInit; 21 use resources::{Alloc, SystemAllocator}; 22 use vm_control::GpeNotify; 23 24 // reserve 8MB memory window 25 const PCIE_RP_BR_MEM_SIZE: u64 = 0x80_0000; 26 // reserve 64MB prefetch window 27 const PCIE_RP_BR_PREF_MEM_SIZE: u64 = 0x400_0000; 28 29 const PCIE_RP_DID: u16 = 0x3420; 30 pub struct PcieRootPort { 31 pcie_cap_reg_idx: Option<usize>, 32 msix_config: Option<Arc<Mutex<MsixConfig>>>, 33 pmc_config: PmcConfig, 34 pmc_cap_reg_idx: Option<usize>, 35 pci_address: Option<PciAddress>, 36 slot_control: Option<u16>, 37 slot_status: u16, 38 root_control: u16, 39 root_status: u32, 40 hp_interrupt_pending: bool, 41 pme_pending_request_id: Option<PciAddress>, 42 bus_range: PciBridgeBusRange, 43 downstream_device: Option<(PciAddress, Option<HostHotPlugKey>)>, 44 removed_downstream: Option<PciAddress>, 45 pcie_host: Option<PcieHostRootPort>, 46 prepare_hotplug: bool, 47 } 48 49 impl PcieRootPort { 50 /// Constructs a new PCIE root port new(secondary_bus_num: u8, slot_implemented: bool) -> Self51 pub fn new(secondary_bus_num: u8, slot_implemented: bool) -> Self { 52 let bus_range = PciBridgeBusRange { 53 primary: 0, 54 secondary: secondary_bus_num, 55 subordinate: secondary_bus_num, 56 }; 57 PcieRootPort { 58 pcie_cap_reg_idx: None, 59 msix_config: None, 60 pmc_config: PmcConfig::new(), 61 pmc_cap_reg_idx: None, 62 pci_address: None, 63 slot_control: if slot_implemented { 64 Some(PCIE_SLTCTL_PIC_OFF | PCIE_SLTCTL_AIC_OFF) 65 } else { 66 None 67 }, 68 slot_status: 0, 69 root_control: 0, 70 root_status: 0, 71 hp_interrupt_pending: false, 72 pme_pending_request_id: None, 73 bus_range, 74 downstream_device: None, 75 removed_downstream: None, 76 pcie_host: None, 77 prepare_hotplug: false, 78 } 79 } 80 81 /// Constructs a new PCIE root port which associated with the host physical pcie RP new_from_host(pcie_host: PcieHostRootPort, slot_implemented: bool) -> Result<Self>82 pub fn new_from_host(pcie_host: PcieHostRootPort, slot_implemented: bool) -> Result<Self> { 83 let bus_range = pcie_host.get_bus_range(); 84 // if physical pcie root port isn't on bus 0, ignore this physical pcie root port. 85 if bus_range.primary != 0 { 86 return Err(anyhow!( 87 "physical pcie RP isn't on bus 0: {}", 88 bus_range.primary 89 )); 90 } 91 92 Ok(PcieRootPort { 93 pcie_cap_reg_idx: None, 94 msix_config: None, 95 pmc_config: PmcConfig::new(), 96 pmc_cap_reg_idx: None, 97 pci_address: None, 98 slot_control: if slot_implemented { 99 Some(PCIE_SLTCTL_PIC_OFF | PCIE_SLTCTL_AIC_OFF) 100 } else { 101 None 102 }, 103 slot_status: 0, 104 root_control: 0, 105 root_status: 0, 106 hp_interrupt_pending: false, 107 pme_pending_request_id: None, 108 bus_range, 109 downstream_device: None, 110 removed_downstream: None, 111 pcie_host: Some(pcie_host), 112 prepare_hotplug: false, 113 }) 114 } 115 get_slot_control(&self) -> u16116 fn get_slot_control(&self) -> u16 { 117 if let Some(slot_control) = self.slot_control { 118 return slot_control; 119 } 120 0 121 } 122 read_pcie_cap(&self, offset: usize, data: &mut u32)123 fn read_pcie_cap(&self, offset: usize, data: &mut u32) { 124 if offset == PCIE_SLTCTL_OFFSET { 125 *data = ((self.slot_status as u32) << 16) | (self.get_slot_control() as u32); 126 } else if offset == PCIE_ROOTCTL_OFFSET { 127 *data = self.root_control as u32; 128 } else if offset == PCIE_ROOTSTA_OFFSET { 129 *data = self.root_status; 130 } 131 } 132 write_pcie_cap(&mut self, offset: usize, data: &[u8])133 fn write_pcie_cap(&mut self, offset: usize, data: &[u8]) { 134 self.removed_downstream = None; 135 match offset { 136 PCIE_SLTCTL_OFFSET => { 137 let value = match u16::from_slice(data) { 138 Some(&v) => v, 139 None => { 140 warn!("write SLTCTL isn't word, len: {}", data.len()); 141 return; 142 } 143 }; 144 145 // if slot is populated, power indicator is off, 146 // it will detach devices 147 let old_control = self.get_slot_control(); 148 match self.slot_control.as_mut() { 149 Some(v) => *v = value, 150 None => return, 151 } 152 if (self.slot_status & PCIE_SLTSTA_PDS != 0) 153 && (value & PCIE_SLTCTL_PIC_OFF == PCIE_SLTCTL_PIC_OFF) 154 && (old_control & PCIE_SLTCTL_PIC_OFF != PCIE_SLTCTL_PIC_OFF) 155 { 156 if let Some((guest_pci_addr, _)) = self.downstream_device { 157 self.removed_downstream = Some(guest_pci_addr); 158 self.downstream_device = None; 159 } 160 self.slot_status &= !PCIE_SLTSTA_PDS; 161 self.slot_status |= PCIE_SLTSTA_PDC; 162 self.trigger_hp_interrupt(); 163 } 164 165 if old_control != value { 166 // send Command completed events 167 self.slot_status |= PCIE_SLTSTA_CC; 168 self.trigger_cc_interrupt(); 169 } 170 } 171 PCIE_SLTSTA_OFFSET => { 172 if self.slot_control.is_none() { 173 return; 174 } 175 let value = match u16::from_slice(data) { 176 Some(v) => *v, 177 None => { 178 warn!("write SLTSTA isn't word, len: {}", data.len()); 179 return; 180 } 181 }; 182 if value & PCIE_SLTSTA_ABP != 0 { 183 self.slot_status &= !PCIE_SLTSTA_ABP; 184 } 185 if value & PCIE_SLTSTA_PFD != 0 { 186 self.slot_status &= !PCIE_SLTSTA_PFD; 187 } 188 if value & PCIE_SLTSTA_PDC != 0 { 189 self.slot_status &= !PCIE_SLTSTA_PDC; 190 } 191 if value & PCIE_SLTSTA_CC != 0 { 192 self.slot_status &= !PCIE_SLTSTA_CC; 193 } 194 if value & PCIE_SLTSTA_DLLSC != 0 { 195 self.slot_status &= !PCIE_SLTSTA_DLLSC; 196 } 197 } 198 PCIE_ROOTCTL_OFFSET => match u16::from_slice(data) { 199 Some(v) => self.root_control = *v, 200 None => warn!("write root control isn't word, len: {}", data.len()), 201 }, 202 PCIE_ROOTSTA_OFFSET => match u32::from_slice(data) { 203 Some(v) => { 204 if *v & PCIE_ROOTSTA_PME_STATUS != 0 { 205 if let Some(request_id) = self.pme_pending_request_id { 206 self.root_status &= !PCIE_ROOTSTA_PME_PENDING; 207 let req_id = ((request_id.bus as u32) << 8) 208 | ((request_id.dev as u32) << 3) 209 | (request_id.func as u32); 210 self.root_status &= !PCIE_ROOTSTA_PME_REQ_ID_MASK; 211 self.root_status |= req_id; 212 self.root_status |= PCIE_ROOTSTA_PME_STATUS; 213 self.pme_pending_request_id = None; 214 self.trigger_pme_interrupt(); 215 } else { 216 self.root_status &= !PCIE_ROOTSTA_PME_STATUS; 217 if self.hp_interrupt_pending { 218 self.hp_interrupt_pending = false; 219 self.trigger_hp_interrupt(); 220 } 221 } 222 } 223 } 224 None => warn!("write root status isn't dword, len: {}", data.len()), 225 }, 226 _ => (), 227 } 228 } 229 trigger_interrupt(&self)230 fn trigger_interrupt(&self) { 231 if let Some(msix_config) = &self.msix_config { 232 let mut msix_config = msix_config.lock(); 233 if msix_config.enabled() { 234 msix_config.trigger(0) 235 } 236 } 237 } 238 trigger_cc_interrupt(&self)239 fn trigger_cc_interrupt(&self) { 240 if (self.get_slot_control() & PCIE_SLTCTL_CCIE) != 0 241 && (self.slot_status & PCIE_SLTSTA_CC) != 0 242 { 243 self.trigger_interrupt() 244 } 245 } 246 trigger_hp_interrupt(&self)247 fn trigger_hp_interrupt(&self) { 248 let slot_control = self.get_slot_control(); 249 if (slot_control & PCIE_SLTCTL_HPIE) != 0 250 && (self.slot_status & slot_control & (PCIE_SLTCTL_ABPE | PCIE_SLTCTL_PDCE)) != 0 251 { 252 self.trigger_interrupt() 253 } 254 } 255 trigger_pme_interrupt(&self)256 fn trigger_pme_interrupt(&self) { 257 if (self.root_control & PCIE_ROOTCTL_PME_ENABLE) != 0 258 && (self.root_status & PCIE_ROOTSTA_PME_STATUS) != 0 259 { 260 self.trigger_interrupt() 261 } 262 } 263 inject_pme(&mut self)264 fn inject_pme(&mut self) { 265 if (self.root_status & PCIE_ROOTSTA_PME_STATUS) != 0 { 266 self.root_status |= PCIE_ROOTSTA_PME_PENDING; 267 self.pme_pending_request_id = self.pci_address; 268 } else { 269 let request_id = self.pci_address.unwrap(); 270 let req_id = ((request_id.bus as u32) << 8) 271 | ((request_id.dev as u32) << 3) 272 | (request_id.func as u32); 273 self.root_status &= !PCIE_ROOTSTA_PME_REQ_ID_MASK; 274 self.root_status |= req_id; 275 self.pme_pending_request_id = None; 276 self.root_status |= PCIE_ROOTSTA_PME_STATUS; 277 self.trigger_pme_interrupt(); 278 } 279 } 280 281 // when RP is D3, HP interrupt is disabled by pcie driver, so inject a PME to wakeup 282 // RP first, then inject HP interrupt. trigger_hp_or_pme_interrupt(&mut self)283 fn trigger_hp_or_pme_interrupt(&mut self) { 284 if self.pmc_config.should_trigger_pme() { 285 self.hp_interrupt_pending = true; 286 self.inject_pme(); 287 } else { 288 self.trigger_hp_interrupt(); 289 } 290 } 291 } 292 293 impl PcieDevice for PcieRootPort { get_device_id(&self) -> u16294 fn get_device_id(&self) -> u16 { 295 match &self.pcie_host { 296 Some(host) => host.read_device_id(), 297 None => PCIE_RP_DID, 298 } 299 } 300 debug_label(&self) -> String301 fn debug_label(&self) -> String { 302 match &self.pcie_host { 303 Some(host) => host.host_name(), 304 None => "PcieRootPort".to_string(), 305 } 306 } 307 allocate_address( &mut self, resources: &mut SystemAllocator, ) -> std::result::Result<PciAddress, PciDeviceError>308 fn allocate_address( 309 &mut self, 310 resources: &mut SystemAllocator, 311 ) -> std::result::Result<PciAddress, PciDeviceError> { 312 if self.pci_address.is_none() { 313 match &self.pcie_host { 314 Some(host) => { 315 let address = PciAddress::from_str(&host.host_name()) 316 .map_err(|e| PciDeviceError::PciAddressParseFailure(host.host_name(), e))?; 317 if resources.reserve_pci( 318 Alloc::PciBar { 319 bus: address.bus, 320 dev: address.dev, 321 func: address.func, 322 bar: 0, 323 }, 324 host.host_name(), 325 ) { 326 self.pci_address = Some(address); 327 } else { 328 self.pci_address = None; 329 } 330 } 331 None => match resources.allocate_pci(self.bus_range.primary, self.debug_label()) { 332 Some(Alloc::PciBar { 333 bus, 334 dev, 335 func, 336 bar: _, 337 }) => self.pci_address = Some(PciAddress { bus, dev, func }), 338 _ => self.pci_address = None, 339 }, 340 } 341 } 342 self.pci_address.ok_or(PciDeviceError::PciAllocationFailed) 343 } 344 clone_interrupt(&mut self, msix_config: Arc<Mutex<MsixConfig>>)345 fn clone_interrupt(&mut self, msix_config: Arc<Mutex<MsixConfig>>) { 346 self.msix_config = Some(msix_config); 347 } 348 get_caps(&self) -> Vec<Box<dyn PciCapability>>349 fn get_caps(&self) -> Vec<Box<dyn PciCapability>> { 350 vec![ 351 Box::new(PcieCap::new( 352 PcieDevicePortType::RootPort, 353 self.slot_control.is_some(), 354 0, 355 )), 356 Box::new(PciPmcCap::new()), 357 ] 358 } 359 set_capability_reg_idx(&mut self, id: PciCapabilityID, reg_idx: usize)360 fn set_capability_reg_idx(&mut self, id: PciCapabilityID, reg_idx: usize) { 361 match id { 362 PciCapabilityID::PciExpress => self.pcie_cap_reg_idx = Some(reg_idx), 363 PciCapabilityID::PowerManagement => self.pmc_cap_reg_idx = Some(reg_idx), 364 _ => (), 365 } 366 } 367 read_config(&self, reg_idx: usize, data: &mut u32)368 fn read_config(&self, reg_idx: usize, data: &mut u32) { 369 if let Some(pcie_cap_reg_idx) = self.pcie_cap_reg_idx { 370 if reg_idx >= pcie_cap_reg_idx && reg_idx < pcie_cap_reg_idx + (PCIE_CAP_LEN / 4) { 371 let offset = (reg_idx - pcie_cap_reg_idx) * 4; 372 self.read_pcie_cap(offset, data); 373 } 374 } 375 if let Some(pmc_cap_reg_idx) = self.pmc_cap_reg_idx { 376 if reg_idx == pmc_cap_reg_idx + PMC_CAP_CONTROL_STATE_OFFSET { 377 self.pmc_config.read(data); 378 } 379 } 380 if let Some(host) = &self.pcie_host { 381 // pcie host may override some config registers 382 host.read_config(reg_idx, data); 383 } 384 } 385 write_config(&mut self, reg_idx: usize, offset: u64, data: &[u8])386 fn write_config(&mut self, reg_idx: usize, offset: u64, data: &[u8]) { 387 if let Some(pcie_cap_reg_idx) = self.pcie_cap_reg_idx { 388 if reg_idx >= pcie_cap_reg_idx && reg_idx < pcie_cap_reg_idx + (PCIE_CAP_LEN / 4) { 389 let delta = ((reg_idx - pcie_cap_reg_idx) * 4) + offset as usize; 390 self.write_pcie_cap(delta, data); 391 } 392 } 393 if let Some(pmc_cap_reg_idx) = self.pmc_cap_reg_idx { 394 if reg_idx == pmc_cap_reg_idx + PMC_CAP_CONTROL_STATE_OFFSET { 395 let old_status = self.pmc_config.get_power_status(); 396 self.pmc_config.write(offset, data); 397 let new_status = self.pmc_config.get_power_status(); 398 if old_status == PciDevicePower::D3 399 && new_status == PciDevicePower::D0 400 && self.prepare_hotplug 401 { 402 if let Some(host) = self.pcie_host.as_mut() { 403 host.hotplug_probe(); 404 self.prepare_hotplug = false; 405 } 406 } 407 } 408 } 409 if let Some(host) = self.pcie_host.as_mut() { 410 // device may write data to host, or do something at specific register write 411 host.write_config(reg_idx, offset, data); 412 } 413 } 414 get_bus_range(&self) -> Option<PciBridgeBusRange>415 fn get_bus_range(&self) -> Option<PciBridgeBusRange> { 416 Some(self.bus_range) 417 } 418 get_removed_devices(&self) -> Vec<PciAddress>419 fn get_removed_devices(&self) -> Vec<PciAddress> { 420 let mut removed_devices = Vec::new(); 421 if let Some(removed_downstream) = self.removed_downstream { 422 removed_devices.push(removed_downstream); 423 } 424 425 removed_devices 426 } 427 hotplug_implemented(&self) -> bool428 fn hotplug_implemented(&self) -> bool { 429 self.slot_control.is_some() 430 } 431 get_bridge_window_size(&self) -> (u64, u64)432 fn get_bridge_window_size(&self) -> (u64, u64) { 433 if let Some(host) = &self.pcie_host { 434 host.get_bridge_window_size() 435 } else { 436 (PCIE_RP_BR_MEM_SIZE, PCIE_RP_BR_PREF_MEM_SIZE) 437 } 438 } 439 } 440 441 impl HotPlugBus for PcieRootPort { hot_plug(&mut self, addr: PciAddress)442 fn hot_plug(&mut self, addr: PciAddress) { 443 match self.downstream_device { 444 Some((guest_addr, _)) => { 445 if guest_addr != addr { 446 return; 447 } 448 } 449 None => return, 450 } 451 452 self.slot_status = self.slot_status | PCIE_SLTSTA_PDS | PCIE_SLTSTA_PDC | PCIE_SLTSTA_ABP; 453 self.trigger_hp_or_pme_interrupt(); 454 } 455 hot_unplug(&mut self, addr: PciAddress)456 fn hot_unplug(&mut self, addr: PciAddress) { 457 match self.downstream_device { 458 Some((guest_addr, _)) => { 459 if guest_addr != addr { 460 return; 461 } 462 } 463 None => return, 464 } 465 466 self.slot_status = self.slot_status | PCIE_SLTSTA_PDC | PCIE_SLTSTA_ABP; 467 self.trigger_hp_or_pme_interrupt(); 468 469 if let Some(host) = self.pcie_host.as_mut() { 470 host.hot_unplug(); 471 } 472 } 473 is_match(&self, host_addr: PciAddress) -> Option<u8>474 fn is_match(&self, host_addr: PciAddress) -> Option<u8> { 475 let _ = self.slot_control?; 476 477 if self.downstream_device.is_none() 478 && ((host_addr.bus >= self.bus_range.secondary 479 && host_addr.bus <= self.bus_range.subordinate) 480 || self.pcie_host.is_none()) 481 { 482 Some(self.bus_range.secondary) 483 } else { 484 None 485 } 486 } 487 add_hotplug_device(&mut self, host_key: HostHotPlugKey, guest_addr: PciAddress)488 fn add_hotplug_device(&mut self, host_key: HostHotPlugKey, guest_addr: PciAddress) { 489 if self.slot_control.is_none() { 490 return; 491 } 492 493 self.downstream_device = Some((guest_addr, Some(host_key))) 494 } 495 get_hotplug_device(&self, host_key: HostHotPlugKey) -> Option<PciAddress>496 fn get_hotplug_device(&self, host_key: HostHotPlugKey) -> Option<PciAddress> { 497 if let Some((guest_address, Some(host_info))) = &self.downstream_device { 498 match host_info { 499 HostHotPlugKey::Vfio { host_addr } => { 500 let saved_addr = *host_addr; 501 match host_key { 502 HostHotPlugKey::Vfio { host_addr } => { 503 if host_addr == saved_addr { 504 return Some(*guest_address); 505 } 506 } 507 } 508 } 509 } 510 } 511 512 None 513 } 514 } 515 516 impl GpeNotify for PcieRootPort { notify(&mut self)517 fn notify(&mut self) { 518 if self.slot_control.is_none() { 519 return; 520 } 521 522 if self.pcie_host.is_some() { 523 self.prepare_hotplug = true; 524 } 525 526 if self.pmc_config.should_trigger_pme() { 527 self.inject_pme(); 528 } 529 } 530 } 531