1 // Copyright 2022 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #![cfg_attr(windows, allow(dead_code))] 6 7 use base::error; 8 use base::AsRawDescriptor; 9 use base::Event; 10 use base::RawDescriptor; 11 use base::Tube; 12 use bit_field::*; 13 use vm_control::VmIrqRequest; 14 use vm_control::VmIrqResponse; 15 use zerocopy::FromBytes; 16 use zerocopy::Immutable; 17 use zerocopy::IntoBytes; 18 use zerocopy::KnownLayout; 19 20 use crate::pci::pci_configuration::PciCapConfig; 21 use crate::pci::pci_configuration::PciCapConfigWriteResult; 22 use crate::pci::PciCapability; 23 use crate::pci::PciCapabilityID; 24 25 // MSI registers 26 pub const PCI_MSI_NEXT_POINTER: u32 = 0x1; // Next cap pointer 27 pub const PCI_MSI_FLAGS: u32 = 0x2; // Message Control 28 const PCI_MSI_FLAGS_ENABLE: u16 = 0x0001; // MSI feature enabled 29 pub const PCI_MSI_FLAGS_64BIT: u16 = 0x0080; // 64-bit addresses allowed 30 pub const PCI_MSI_FLAGS_MASKBIT: u16 = 0x0100; // Per-vector masking capable 31 const PCI_MSI_ADDRESS_LO: u32 = 0x4; // MSI address lower 32 bits 32 const PCI_MSI_ADDRESS_HI: u32 = 0x8; // MSI address upper 32 bits (if 64 bit allowed) 33 const PCI_MSI_DATA_32: u32 = 0x8; // 16 bits of data for 32-bit message address 34 const PCI_MSI_DATA_64: u32 = 0xC; // 16 bits of date for 64-bit message address 35 36 // MSI length 37 const MSI_LENGTH_32BIT_WITHOUT_MASK: u32 = 0xA; 38 const MSI_LENGTH_32BIT_WITH_MASK: u32 = 0x14; 39 const MSI_LENGTH_64BIT_WITHOUT_MASK: u32 = 0xE; 40 const MSI_LENGTH_64BIT_WITH_MASK: u32 = 0x18; 41 42 pub enum MsiStatus { 43 Enabled, 44 Disabled, 45 NothingToDo, 46 } 47 48 /// Wrapper over MSI Capability Structure 49 pub struct MsiConfig { 50 is_64bit: bool, 51 mask_cap: bool, 52 ctrl: u16, 53 address: u64, 54 data: u16, 55 vm_socket_irq: Tube, 56 irqfd: Option<Event>, 57 gsi: Option<u32>, 58 device_id: u32, 59 device_name: String, 60 } 61 62 impl MsiConfig { new( is_64bit: bool, mask_cap: bool, vm_socket_irq: Tube, device_id: u32, device_name: String, ) -> Self63 pub fn new( 64 is_64bit: bool, 65 mask_cap: bool, 66 vm_socket_irq: Tube, 67 device_id: u32, 68 device_name: String, 69 ) -> Self { 70 let mut ctrl: u16 = 0; 71 if is_64bit { 72 ctrl |= PCI_MSI_FLAGS_64BIT; 73 } 74 if mask_cap { 75 ctrl |= PCI_MSI_FLAGS_MASKBIT; 76 } 77 MsiConfig { 78 is_64bit, 79 mask_cap, 80 ctrl, 81 address: 0, 82 data: 0, 83 vm_socket_irq, 84 irqfd: None, 85 gsi: None, 86 device_id, 87 device_name, 88 } 89 } 90 len(&self) -> u3291 fn len(&self) -> u32 { 92 match (self.is_64bit, self.mask_cap) { 93 (true, true) => MSI_LENGTH_64BIT_WITH_MASK, 94 (true, false) => MSI_LENGTH_64BIT_WITHOUT_MASK, 95 (false, true) => MSI_LENGTH_32BIT_WITH_MASK, 96 (false, false) => MSI_LENGTH_32BIT_WITHOUT_MASK, 97 } 98 } 99 is_msi_reg(&self, offset: u32, index: u64, len: usize) -> bool100 pub fn is_msi_reg(&self, offset: u32, index: u64, len: usize) -> bool { 101 let msi_len = self.len(); 102 index >= offset as u64 103 && index + len as u64 <= (offset + msi_len) as u64 104 && len as u32 <= msi_len 105 } 106 read_msi_capability(&self, offset: u32, data: u32) -> u32107 pub fn read_msi_capability(&self, offset: u32, data: u32) -> u32 { 108 if offset == 0 { 109 (self.ctrl as u32) << 16 | (data & u16::MAX as u32) 110 } else { 111 data 112 } 113 } 114 write_msi_capability(&mut self, offset: u32, data: &[u8]) -> MsiStatus115 pub fn write_msi_capability(&mut self, offset: u32, data: &[u8]) -> MsiStatus { 116 let len = data.len(); 117 let mut ret = MsiStatus::NothingToDo; 118 let old_address = self.address; 119 let old_data = self.data; 120 121 // write msi ctl 122 if len == 2 && offset == PCI_MSI_FLAGS { 123 let was_enabled = self.is_msi_enabled(); 124 let value: [u8; 2] = [data[0], data[1]]; 125 self.ctrl = u16::from_le_bytes(value); 126 let is_enabled = self.is_msi_enabled(); 127 if !was_enabled && is_enabled { 128 self.enable(); 129 ret = MsiStatus::Enabled; 130 } else if was_enabled && !is_enabled { 131 ret = MsiStatus::Disabled; 132 } 133 } else if len == 4 && offset == PCI_MSI_ADDRESS_LO && !self.is_64bit { 134 //write 32 bit message address 135 let value: [u8; 8] = [data[0], data[1], data[2], data[3], 0, 0, 0, 0]; 136 self.address = u64::from_le_bytes(value); 137 } else if len == 4 && offset == PCI_MSI_ADDRESS_LO && self.is_64bit { 138 // write 64 bit message address low part 139 let value: [u8; 8] = [data[0], data[1], data[2], data[3], 0, 0, 0, 0]; 140 self.address &= !0xffffffff; 141 self.address |= u64::from_le_bytes(value); 142 } else if len == 4 && offset == PCI_MSI_ADDRESS_HI && self.is_64bit { 143 //write 64 bit message address high part 144 let value: [u8; 8] = [0, 0, 0, 0, data[0], data[1], data[2], data[3]]; 145 self.address &= 0xffffffff; 146 self.address |= u64::from_le_bytes(value); 147 } else if len == 8 && offset == PCI_MSI_ADDRESS_LO && self.is_64bit { 148 // write 64 bit message address 149 let value: [u8; 8] = [ 150 data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], 151 ]; 152 self.address = u64::from_le_bytes(value); 153 } else if len == 2 154 && ((offset == PCI_MSI_DATA_32 && !self.is_64bit) 155 || (offset == PCI_MSI_DATA_64 && self.is_64bit)) 156 { 157 // write message data 158 let value: [u8; 2] = [data[0], data[1]]; 159 self.data = u16::from_le_bytes(value); 160 } 161 162 if self.is_msi_enabled() && (old_address != self.address || old_data != self.data) { 163 self.add_msi_route(); 164 } 165 166 ret 167 } 168 is_msi_enabled(&self) -> bool169 pub fn is_msi_enabled(&self) -> bool { 170 self.ctrl & PCI_MSI_FLAGS_ENABLE == PCI_MSI_FLAGS_ENABLE 171 } 172 add_msi_route(&self)173 fn add_msi_route(&self) { 174 let gsi = match self.gsi { 175 Some(g) => g, 176 None => { 177 error!("Add msi route but gsi is none"); 178 return; 179 } 180 }; 181 if let Err(e) = self.vm_socket_irq.send(&VmIrqRequest::AddMsiRoute { 182 gsi, 183 msi_address: self.address, 184 msi_data: self.data.into(), 185 }) { 186 error!("failed to send AddMsiRoute request at {:?}", e); 187 return; 188 } 189 match self.vm_socket_irq.recv() { 190 Ok(VmIrqResponse::Err(e)) => error!("failed to call AddMsiRoute request {:?}", e), 191 Ok(_) => {} 192 Err(e) => error!("failed to receive AddMsiRoute response {:?}", e), 193 } 194 } 195 allocate_one_msi(&mut self)196 fn allocate_one_msi(&mut self) { 197 let irqfd = match self.irqfd.take() { 198 Some(e) => e, 199 None => match Event::new() { 200 Ok(e) => e, 201 Err(e) => { 202 error!("failed to create event: {:?}", e); 203 return; 204 } 205 }, 206 }; 207 208 let request = VmIrqRequest::AllocateOneMsi { 209 irqfd, 210 device_id: self.device_id, 211 queue_id: 0, 212 device_name: self.device_name.clone(), 213 }; 214 let request_result = self.vm_socket_irq.send(&request); 215 216 // Stash the irqfd in self immediately because we used take above. 217 self.irqfd = match request { 218 VmIrqRequest::AllocateOneMsi { irqfd, .. } => Some(irqfd), 219 _ => unreachable!(), 220 }; 221 222 if let Err(e) = request_result { 223 error!("failed to send AllocateOneMsi request: {:?}", e); 224 return; 225 } 226 227 match self.vm_socket_irq.recv() { 228 Ok(VmIrqResponse::AllocateOneMsi { gsi }) => self.gsi = Some(gsi), 229 _ => error!("failed to receive AllocateOneMsi Response"), 230 } 231 } 232 enable(&mut self)233 fn enable(&mut self) { 234 if self.gsi.is_none() || self.irqfd.is_none() { 235 self.allocate_one_msi(); 236 } 237 238 self.add_msi_route(); 239 } 240 get_irqfd(&self) -> Option<&Event>241 pub fn get_irqfd(&self) -> Option<&Event> { 242 self.irqfd.as_ref() 243 } 244 destroy(&mut self)245 pub fn destroy(&mut self) { 246 if let Some(gsi) = self.gsi { 247 if let Some(irqfd) = self.irqfd.take() { 248 let request = VmIrqRequest::ReleaseOneIrq { gsi, irqfd }; 249 if self.vm_socket_irq.send(&request).is_ok() { 250 let _ = self.vm_socket_irq.recv::<VmIrqResponse>(); 251 } 252 } 253 } 254 } 255 256 /// Return the raw descriptor of the MSI device socket get_msi_socket(&self) -> RawDescriptor257 pub fn get_msi_socket(&self) -> RawDescriptor { 258 self.vm_socket_irq.as_raw_descriptor() 259 } 260 trigger(&self)261 pub fn trigger(&self) { 262 if let Some(irqfd) = self.irqfd.as_ref() { 263 irqfd.signal().unwrap(); 264 } 265 } 266 } 267 268 #[bitfield] 269 #[derive(Copy, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)] 270 pub struct MsiCtrl { 271 enable: B1, 272 multi_msg_capable: B3, 273 multi_msg_enable: B3, 274 is_64bit: B1, 275 per_vector_masking: B1, 276 extended_msg_data_capable: B1, 277 extended_msg_data_enable: B1, 278 reserved: B5, 279 } 280 281 #[allow(dead_code)] 282 #[repr(C)] 283 #[derive(Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout)] 284 /// MSI Capability Structure 285 pub struct MsiCap { 286 // To make add_capability() happy 287 _cap_vndr: u8, 288 _cap_next: u8, 289 // Message Control Register 290 msg_ctl: MsiCtrl, 291 // Message Address 292 msg_addr: u32, 293 // Msi Vary structure 294 msi_vary: [u8; 16], 295 } 296 297 impl PciCapability for MsiCap { bytes(&self) -> &[u8]298 fn bytes(&self) -> &[u8] { 299 self.as_bytes() 300 } 301 id(&self) -> PciCapabilityID302 fn id(&self) -> PciCapabilityID { 303 PciCapabilityID::MessageSignalledInterrupts 304 } 305 writable_bits(&self) -> Vec<u32>306 fn writable_bits(&self) -> Vec<u32> { 307 // Check spec for detail 308 match ( 309 self.msg_ctl.get_is_64bit(), 310 self.msg_ctl.get_per_vector_masking(), 311 ) { 312 (0, 0) => vec![0x0471_0000, 0xffff_fffc, 0xffff_ffff], 313 (0, 1) => vec![0x0471_0000, 0xffff_fffc, 0xffff_ffff, 0xffff_ffff], 314 (1, 0) => vec![ 315 0x0471_0000, 316 0xffff_fffc, 317 0xffff_ffff, 318 0xffff_ffff, 319 0x0000_0000, 320 ], 321 (1, 1) => vec![ 322 0x0471_0000, 323 0xffff_fffc, 324 0xffff_ffff, 325 0xffff_ffff, 326 0xffff_ffff, 327 0x0000_0000, 328 ], 329 (_, _) => Vec::new(), 330 } 331 } 332 } 333 334 impl MsiCap { new(is_64bit: bool, mask_cap: bool) -> Self335 pub fn new(is_64bit: bool, mask_cap: bool) -> Self { 336 let mut msg_ctl = MsiCtrl::new(); 337 if is_64bit { 338 msg_ctl.set_is_64bit(1); 339 } 340 if mask_cap { 341 msg_ctl.set_per_vector_masking(1); 342 } 343 MsiCap { 344 _cap_vndr: 0, 345 _cap_next: 0, 346 msg_ctl, 347 msg_addr: 0, 348 msi_vary: [0; 16], 349 } 350 } 351 } 352 353 const MSI_CONFIG_READ_MASK: [u32; MSI_LENGTH_64BIT_WITH_MASK as usize / 4] = 354 [0xffff_0000, 0, 0, 0, 0, 0]; 355 356 impl PciCapConfig for MsiConfig { read_mask(&self) -> &'static [u32]357 fn read_mask(&self) -> &'static [u32] { 358 let num_regs = (self.len() + 3) / 4; 359 &MSI_CONFIG_READ_MASK[0..(num_regs as usize)] 360 } 361 read_reg(&self, reg_idx: usize) -> u32362 fn read_reg(&self, reg_idx: usize) -> u32 { 363 self.read_msi_capability(reg_idx as u32 * 4, 0) 364 } 365 write_reg( &mut self, reg_idx: usize, offset: u64, data: &[u8], ) -> Option<Box<dyn PciCapConfigWriteResult>>366 fn write_reg( 367 &mut self, 368 reg_idx: usize, 369 offset: u64, 370 data: &[u8], 371 ) -> Option<Box<dyn PciCapConfigWriteResult>> { 372 let offset = reg_idx as u32 * 4 + offset as u32; 373 self.write_msi_capability(offset, data); 374 None 375 } 376 } 377