1 // Copyright 2021 The Chromium OS Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 use crate::vfio::{VfioDevice, VfioError, VfioIrq}; 5 use crate::{BusAccessInfo, BusDevice, BusDeviceObj, IrqEdgeEvent, IrqLevelEvent}; 6 use anyhow::{bail, Context, Result}; 7 use base::{ 8 error, pagesize, AsRawDescriptor, AsRawDescriptors, Event, MappedRegion, MemoryMapping, 9 MemoryMappingBuilder, RawDescriptor, Tube, 10 }; 11 use resources::SystemAllocator; 12 use std::fs::File; 13 use std::sync::Arc; 14 use std::u32; 15 use vfio_sys::*; 16 use vm_control::{VmMemoryDestination, VmMemoryRequest, VmMemoryResponse, VmMemorySource}; 17 18 struct MmioInfo { 19 index: u32, 20 start: u64, 21 length: u64, 22 } 23 24 pub struct VfioPlatformDevice { 25 device: Arc<VfioDevice>, 26 interrupt_edge_evt: Vec<IrqEdgeEvent>, 27 interrupt_level_evt: Vec<IrqLevelEvent>, 28 mmio_regions: Vec<MmioInfo>, 29 vm_socket_mem: Tube, 30 // scratch MemoryMapping to avoid unmap beform vm exit 31 mem: Vec<MemoryMapping>, 32 } 33 34 impl BusDevice for VfioPlatformDevice { debug_label(&self) -> String35 fn debug_label(&self) -> String { 36 format!("vfio {} device", self.device.device_name()) 37 } 38 read(&mut self, info: BusAccessInfo, data: &mut [u8])39 fn read(&mut self, info: BusAccessInfo, data: &mut [u8]) { 40 self.read_mmio(info.address, data) 41 } 42 write(&mut self, info: BusAccessInfo, data: &[u8])43 fn write(&mut self, info: BusAccessInfo, data: &[u8]) { 44 self.write_mmio(info.address, data) 45 } 46 } 47 48 impl BusDeviceObj for VfioPlatformDevice { as_platform_device(&self) -> Option<&VfioPlatformDevice>49 fn as_platform_device(&self) -> Option<&VfioPlatformDevice> { 50 Some(self) 51 } as_platform_device_mut(&mut self) -> Option<&mut VfioPlatformDevice>52 fn as_platform_device_mut(&mut self) -> Option<&mut VfioPlatformDevice> { 53 Some(self) 54 } into_platform_device(self: Box<Self>) -> Option<Box<VfioPlatformDevice>>55 fn into_platform_device(self: Box<Self>) -> Option<Box<VfioPlatformDevice>> { 56 Some(self) 57 } 58 } 59 60 impl VfioPlatformDevice { 61 /// Constructs a new Vfio Platform device for the given Vfio device new(device: VfioDevice, vfio_device_socket_mem: Tube) -> Self62 pub fn new(device: VfioDevice, vfio_device_socket_mem: Tube) -> Self { 63 let dev = Arc::new(device); 64 VfioPlatformDevice { 65 device: dev, 66 interrupt_edge_evt: Vec::new(), 67 interrupt_level_evt: Vec::new(), 68 mmio_regions: Vec::new(), 69 vm_socket_mem: vfio_device_socket_mem, 70 mem: Vec::new(), 71 } 72 } 73 get_platform_irqs(&self) -> Result<Vec<VfioIrq>, VfioError>74 pub fn get_platform_irqs(&self) -> Result<Vec<VfioIrq>, VfioError> { 75 self.device.get_irqs() 76 } 77 irq_is_automask(&self, irq: &VfioIrq) -> bool78 pub fn irq_is_automask(&self, irq: &VfioIrq) -> bool { 79 irq.flags & VFIO_IRQ_INFO_AUTOMASKED != 0 80 } 81 setup_irq_resample(&mut self, resample_evt: &Event, index: u32) -> Result<()>82 fn setup_irq_resample(&mut self, resample_evt: &Event, index: u32) -> Result<()> { 83 self.device.irq_mask(index).context("Intx mask failed")?; 84 self.device 85 .resample_virq_enable(resample_evt, index) 86 .context("resample enable failed")?; 87 self.device 88 .irq_unmask(index) 89 .context("Intx unmask failed")?; 90 Ok(()) 91 } 92 assign_edge_platform_irq(&mut self, irq_evt: &IrqEdgeEvent, index: u32) -> Result<()>93 pub fn assign_edge_platform_irq(&mut self, irq_evt: &IrqEdgeEvent, index: u32) -> Result<()> { 94 let interrupt_evt = irq_evt.try_clone().context("failed to clone irq event")?; 95 self.device 96 .irq_enable(&[Some(interrupt_evt.get_trigger())], index, 0) 97 .context("platform irq enable failed")?; 98 self.interrupt_edge_evt.push(interrupt_evt); 99 Ok(()) 100 } 101 assign_level_platform_irq(&mut self, irq_evt: &IrqLevelEvent, index: u32) -> Result<()>102 pub fn assign_level_platform_irq(&mut self, irq_evt: &IrqLevelEvent, index: u32) -> Result<()> { 103 let interrupt_evt = irq_evt.try_clone().context("failed to clone irq event")?; 104 self.device 105 .irq_enable(&[Some(interrupt_evt.get_trigger())], index, 0) 106 .context("platform irq enable failed")?; 107 if let Err(e) = self.setup_irq_resample(interrupt_evt.get_resample(), index) { 108 self.disable_irqs(index); 109 bail!("failed to set up irq resampling: {}", e); 110 } 111 self.interrupt_level_evt.push(interrupt_evt); 112 Ok(()) 113 } 114 find_region(&self, addr: u64) -> Option<MmioInfo>115 fn find_region(&self, addr: u64) -> Option<MmioInfo> { 116 for mmio_info in self.mmio_regions.iter() { 117 if addr >= mmio_info.start && addr < mmio_info.start + mmio_info.length { 118 return Some(MmioInfo { 119 index: mmio_info.index, 120 start: mmio_info.start, 121 length: mmio_info.length, 122 }); 123 } 124 } 125 None 126 } 127 allocate_regions( &mut self, resources: &mut SystemAllocator, ) -> Result<Vec<(u64, u64)>, resources::Error>128 pub fn allocate_regions( 129 &mut self, 130 resources: &mut SystemAllocator, 131 ) -> Result<Vec<(u64, u64)>, resources::Error> { 132 let mut ranges = Vec::new(); 133 for i in 0..self.device.get_region_count() { 134 let size = self.device.get_region_size(i); 135 let alloc_id = resources.get_anon_alloc(); 136 let allocator = resources 137 .mmio_platform_allocator() 138 .ok_or(resources::Error::MissingPlatformMMIOAddresses)?; 139 let start_addr = allocator.allocate_with_align( 140 size, 141 alloc_id, 142 "vfio_mmio".to_string(), 143 pagesize() as u64, 144 )?; 145 ranges.push((start_addr, size)); 146 147 self.mmio_regions.push(MmioInfo { 148 index: i, 149 start: start_addr, 150 length: size, 151 }); 152 } 153 Ok(ranges) 154 } 155 region_mmap(&self, index: u32, start_addr: u64) -> Vec<MemoryMapping>156 fn region_mmap(&self, index: u32, start_addr: u64) -> Vec<MemoryMapping> { 157 let mut mem_map: Vec<MemoryMapping> = Vec::new(); 158 if self.device.get_region_flags(index) & VFIO_REGION_INFO_FLAG_MMAP != 0 { 159 let mmaps = self.device.get_region_mmap(index); 160 if mmaps.is_empty() { 161 return mem_map; 162 } 163 164 for mmap in mmaps.iter() { 165 let mmap_offset = mmap.offset; 166 let mmap_size = mmap.size; 167 let guest_map_start = start_addr + mmap_offset; 168 let region_offset = self.device.get_region_offset(index); 169 let offset = region_offset + mmap_offset; 170 let descriptor = match self.device.device_file().try_clone() { 171 Ok(device_file) => device_file.into(), 172 Err(_) => break, 173 }; 174 if self 175 .vm_socket_mem 176 .send(&VmMemoryRequest::RegisterMemory { 177 source: VmMemorySource::Descriptor { 178 descriptor, 179 offset, 180 size: mmap_size, 181 }, 182 dest: VmMemoryDestination::GuestPhysicalAddress(guest_map_start), 183 read_only: false, 184 }) 185 .is_err() 186 { 187 break; 188 } 189 190 let response: VmMemoryResponse = match self.vm_socket_mem.recv() { 191 Ok(res) => res, 192 Err(_) => break, 193 }; 194 match response { 195 VmMemoryResponse::Ok => { 196 // Even if vm has mapped this region, but it is in vm main process, 197 // device process doesn't has this mapping, but vfio_dma_map() need it 198 // in device process, so here map it again. 199 let mmap = match MemoryMappingBuilder::new(mmap_size as usize) 200 .from_file(self.device.device_file()) 201 .offset(offset) 202 .build() 203 { 204 Ok(v) => v, 205 Err(_e) => break, 206 }; 207 let host = (&mmap).as_ptr() as u64; 208 // Safe because the given guest_map_start is valid guest bar address. and 209 // the host pointer is correct and valid guaranteed by MemoryMapping interface. 210 match unsafe { 211 self.device 212 .vfio_dma_map(guest_map_start, mmap_size, host, true) 213 } { 214 Ok(_) => mem_map.push(mmap), 215 Err(e) => { 216 error!( 217 "{}, index: {}, start_addr:0x{:x}, host:0x{:x}", 218 e, index, start_addr, host 219 ); 220 break; 221 } 222 } 223 } 224 _ => break, 225 } 226 } 227 } 228 229 mem_map 230 } 231 regions_mmap(&mut self)232 fn regions_mmap(&mut self) { 233 for mmio_info in self.mmio_regions.iter() { 234 let mut mem_map = self.region_mmap(mmio_info.index, mmio_info.start); 235 self.mem.append(&mut mem_map); 236 } 237 } 238 disable_irqs(&mut self, index: u32)239 fn disable_irqs(&mut self, index: u32) { 240 if let Err(e) = self.device.irq_disable(index) { 241 error!("Platform irq disable failed: {}", e); 242 } 243 } 244 read_mmio(&mut self, addr: u64, data: &mut [u8])245 fn read_mmio(&mut self, addr: u64, data: &mut [u8]) { 246 if let Some(mmio_info) = self.find_region(addr) { 247 let offset = addr - mmio_info.start; 248 let index = mmio_info.index; 249 self.device.region_read(index, data, offset); 250 } 251 // We have no other way than wait for 1st access and then do the mmap, 252 // so that next accesses are dual-stage MMU accelerated. 253 self.regions_mmap(); 254 } 255 write_mmio(&mut self, addr: u64, data: &[u8])256 fn write_mmio(&mut self, addr: u64, data: &[u8]) { 257 if let Some(mmio_info) = self.find_region(addr) { 258 let offset = addr - mmio_info.start; 259 let index = mmio_info.index; 260 self.device.region_write(index, data, offset); 261 } 262 // We have no other way than wait for 1st access and then do the mmap, 263 // so that next accesses are dual-stage MMU accelerated. 264 self.regions_mmap(); 265 } 266 keep_rds(&self) -> Vec<RawDescriptor>267 pub fn keep_rds(&self) -> Vec<RawDescriptor> { 268 let mut rds = self.device.keep_rds(); 269 270 for irq_evt in self.interrupt_edge_evt.iter() { 271 rds.extend(irq_evt.as_raw_descriptors()); 272 } 273 274 for irq_evt in self.interrupt_level_evt.iter() { 275 rds.extend(irq_evt.as_raw_descriptors()); 276 } 277 278 rds.push(self.vm_socket_mem.as_raw_descriptor()); 279 rds 280 } 281 282 /// Gets the vfio device backing `File`. device_file(&self) -> &File283 pub fn device_file(&self) -> &File { 284 self.device.device_file() 285 } 286 } 287