1 // Copyright 2020 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 use std::collections::BTreeMap; 6 use std::convert::TryFrom; 7 use std::fmt::Debug; 8 9 use anyhow::Context; 10 use base::Error; 11 use base::Result; 12 use cros_fdt::Fdt; 13 use downcast_rs::impl_downcast; 14 use libc::EINVAL; 15 use serde::Deserialize; 16 use serde::Serialize; 17 use snapshot::AnySnapshot; 18 use vm_memory::GuestAddress; 19 20 use crate::Hypervisor; 21 use crate::IrqRoute; 22 use crate::IrqSource; 23 use crate::IrqSourceChip; 24 use crate::Vcpu; 25 use crate::Vm; 26 27 /// Represents a version of Power State Coordination Interface (PSCI). 28 #[derive(Eq, Ord, PartialEq, PartialOrd)] 29 pub struct PsciVersion { 30 pub major: u16, 31 pub minor: u16, 32 } 33 34 impl PsciVersion { new(major: u16, minor: u16) -> Result<Self>35 pub fn new(major: u16, minor: u16) -> Result<Self> { 36 if (major as i16) < 0 { 37 Err(Error::new(EINVAL)) 38 } else { 39 Ok(Self { major, minor }) 40 } 41 } 42 } 43 44 impl TryFrom<u32> for PsciVersion { 45 type Error = base::Error; 46 try_from(item: u32) -> Result<Self>47 fn try_from(item: u32) -> Result<Self> { 48 Self::new((item >> 16) as u16, item as u16) 49 } 50 } 51 52 // Aarch64 does not provide a concrete number as to how many registers exist. 53 // The list of registers available exceeds 600 registers 54 pub const AARCH64_MAX_REG_COUNT: usize = 1024; 55 56 pub const PSCI_0_2: PsciVersion = PsciVersion { major: 0, minor: 2 }; 57 pub const PSCI_1_0: PsciVersion = PsciVersion { major: 1, minor: 0 }; 58 59 /// AArch64 system register as used in MSR/MRS instructions. 60 #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)] 61 #[serde(transparent)] 62 pub struct AArch64SysRegId(u16); 63 64 impl AArch64SysRegId { 65 /// Construct a system register ID from Op0, Op1, CRn, CRm, Op2. 66 /// 67 /// The meanings of the arguments are described in the ARMv8 Architecture Reference Manual 68 /// "System instruction class encoding overview" section. new(op0: u8, op1: u8, crn: u8, crm: u8, op2: u8) -> Result<Self>69 pub fn new(op0: u8, op1: u8, crn: u8, crm: u8, op2: u8) -> Result<Self> { 70 if op0 > 0b11 || op1 > 0b111 || crn > 0b1111 || crm > 0b1111 || op2 > 0b111 { 71 return Err(Error::new(EINVAL)); 72 } 73 74 Ok(Self::new_unchecked(op0, op1, crn, crm, op2)) 75 } 76 77 /// Construct a system register ID from Op0, Op1, CRn, CRm, Op2. 78 /// 79 /// Out-of-range values will be silently truncated. new_unchecked(op0: u8, op1: u8, crn: u8, crm: u8, op2: u8) -> Self80 pub const fn new_unchecked(op0: u8, op1: u8, crn: u8, crm: u8, op2: u8) -> Self { 81 let op0 = (op0 as u16 & 0b11) << 14; 82 let op1 = (op1 as u16 & 0b111) << 11; 83 let crn = (crn as u16 & 0b1111) << 7; 84 let crm = (crm as u16 & 0b1111) << 3; 85 let op2 = op2 as u16 & 0b111; 86 Self(op0 | op1 | crn | crm | op2) 87 } 88 from_encoded(v: u16) -> Self89 pub fn from_encoded(v: u16) -> Self { 90 Self(v) 91 } 92 93 #[inline] op0(&self) -> u894 pub const fn op0(&self) -> u8 { 95 ((self.0 >> 14) & 0b11) as u8 96 } 97 98 #[inline] op1(&self) -> u899 pub const fn op1(&self) -> u8 { 100 ((self.0 >> 11) & 0b111) as u8 101 } 102 103 #[inline] crn(&self) -> u8104 pub const fn crn(&self) -> u8 { 105 ((self.0 >> 7) & 0b1111) as u8 106 } 107 108 #[inline] crm(&self) -> u8109 pub const fn crm(&self) -> u8 { 110 ((self.0 >> 3) & 0b1111) as u8 111 } 112 113 #[inline] op2(&self) -> u8114 pub const fn op2(&self) -> u8 { 115 (self.0 & 0b111) as u8 116 } 117 118 /// Returns the system register as encoded in bits 5-20 of MRS and MSR instructions. encoded(&self) -> u16119 pub const fn encoded(&self) -> u16 { 120 self.0 121 } 122 } 123 124 impl Debug for AArch64SysRegId { fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result125 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 126 f.debug_struct("AArch64SysRegId") 127 .field("Op0", &self.op0()) 128 .field("Op1", &self.op1()) 129 .field("CRn", &self.crn()) 130 .field("CRm", &self.crm()) 131 .field("Op2", &self.op2()) 132 .finish() 133 } 134 } 135 136 #[rustfmt::skip] 137 #[allow(non_upper_case_globals)] 138 impl AArch64SysRegId { 139 // Op0 Op1 CRn CRm Op2 140 pub const MPIDR_EL1: Self = Self::new_unchecked(0b11, 0b000, 0b0000, 0b0000, 0b101); 141 pub const CCSIDR_EL1: Self = Self::new_unchecked(0b11, 0b001, 0b0000, 0b0000, 0b000); 142 pub const CSSELR_EL1: Self = Self::new_unchecked(0b11, 0b010, 0b0000, 0b0000, 0b000); 143 pub const FPCR: Self = Self::new_unchecked(0b11, 0b011, 0b0100, 0b0100, 0b000); 144 pub const FPSR: Self = Self::new_unchecked(0b11, 0b011, 0b0100, 0b0100, 0b001); 145 pub const SPSR_EL1: Self = Self::new_unchecked(0b11, 0b000, 0b0100, 0b0000, 0b000); 146 pub const SPSR_irq: Self = Self::new_unchecked(0b11, 0b100, 0b0100, 0b0011, 0b000); 147 pub const SPSR_abt: Self = Self::new_unchecked(0b11, 0b100, 0b0100, 0b0011, 0b001); 148 pub const SPSR_und: Self = Self::new_unchecked(0b11, 0b100, 0b0100, 0b0011, 0b010); 149 pub const SPSR_fiq: Self = Self::new_unchecked(0b11, 0b100, 0b0100, 0b0011, 0b011); 150 pub const ELR_EL1: Self = Self::new_unchecked(0b11, 0b000, 0b0100, 0b0000, 0b001); 151 pub const SP_EL1: Self = Self::new_unchecked(0b11, 0b100, 0b0100, 0b0001, 0b000); 152 pub const CNTVCT_EL0: Self = Self::new_unchecked(0b11, 0b011, 0b1110, 0b0000, 0b010); 153 pub const CNTV_CVAL_EL0: Self = Self::new_unchecked(0b11, 0b011, 0b1110, 0b0011, 0b010); 154 } 155 156 #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] 157 pub enum VcpuRegAArch64 { 158 X(u8), 159 Sp, 160 Pc, 161 Pstate, 162 System(AArch64SysRegId), 163 } 164 165 /// A wrapper for using a VM on aarch64 and getting/setting its state. 166 pub trait VmAArch64: Vm { 167 /// Gets the `Hypervisor` that created this VM. get_hypervisor(&self) -> &dyn Hypervisor168 fn get_hypervisor(&self) -> &dyn Hypervisor; 169 170 /// Load pVM firmware for the VM, creating a memslot for it as needed. 171 /// 172 /// Only works on protected VMs (i.e. those that support `VmCap::Protected`). load_protected_vm_firmware(&mut self, fw_addr: GuestAddress, fw_max_size: u64) -> Result<()>173 fn load_protected_vm_firmware(&mut self, fw_addr: GuestAddress, fw_max_size: u64) 174 -> Result<()>; 175 176 /// Create a Vcpu with the specified Vcpu ID. create_vcpu(&self, id: usize) -> Result<Box<dyn VcpuAArch64>>177 fn create_vcpu(&self, id: usize) -> Result<Box<dyn VcpuAArch64>>; 178 179 /// Create DT configuration node for the hypervisor. 180 /// `fdt` - Fdt initialized at the root node. 181 /// `phandles` - Map of strings to a phandle. create_fdt(&self, fdt: &mut Fdt, phandles: &BTreeMap<&str, u32>) -> cros_fdt::Result<()>182 fn create_fdt(&self, fdt: &mut Fdt, phandles: &BTreeMap<&str, u32>) -> cros_fdt::Result<()>; 183 184 // Initialize a VM. Called after building the VM. Presently called before vCPUs are initialized. init_arch( &self, payload_entry_address: GuestAddress, fdt_address: GuestAddress, fdt_size: usize, ) -> Result<()>185 fn init_arch( 186 &self, 187 payload_entry_address: GuestAddress, 188 fdt_address: GuestAddress, 189 fdt_size: usize, 190 ) -> Result<()>; 191 192 /// Set an offset that describes a number of counter cycles that are subtracted from both 193 /// virtual and physical counter views. set_counter_offset(&self, _offset: u64) -> Result<()>194 fn set_counter_offset(&self, _offset: u64) -> Result<()> { 195 Err(Error::new(libc::ENOSYS)) 196 } 197 } 198 199 /// A wrapper around creating and using a VCPU on aarch64. 200 pub trait VcpuAArch64: Vcpu { 201 /// Does ARM-specific initialization of this VCPU. Inits the VCPU with the preferred target 202 /// VCPU type and the specified `features`, and resets the value of all registers to defaults. 203 /// All VCPUs should be created before calling this function. init(&self, features: &[VcpuFeature]) -> Result<()>204 fn init(&self, features: &[VcpuFeature]) -> Result<()>; 205 206 /// Initializes the ARM Performance Monitor Unit v3 on this VCPU, with overflow interrupt number 207 /// `irq`. init_pmu(&self, irq: u64) -> Result<()>208 fn init_pmu(&self, irq: u64) -> Result<()>; 209 210 /// Checks if ARM ParaVirtualized Time is supported on this VCPU has_pvtime_support(&self) -> bool211 fn has_pvtime_support(&self) -> bool; 212 213 /// Initializes the ARM ParaVirtualized Time on this VCPU, with base address of the stolen time 214 /// structure as `pvtime_ipa`. init_pvtime(&self, pvtime_ipa: u64) -> Result<()>215 fn init_pvtime(&self, pvtime_ipa: u64) -> Result<()>; 216 217 /// Sets the value of a register on this VCPU. set_one_reg(&self, reg_id: VcpuRegAArch64, data: u64) -> Result<()>218 fn set_one_reg(&self, reg_id: VcpuRegAArch64, data: u64) -> Result<()>; 219 220 /// Gets the value of a register on this VCPU. get_one_reg(&self, reg_id: VcpuRegAArch64) -> Result<u64>221 fn get_one_reg(&self, reg_id: VcpuRegAArch64) -> Result<u64>; 222 223 /// Sets the value of a Neon vector register (V0-V31) on this VCPU. set_vector_reg(&self, reg_num: u8, data: u128) -> Result<()>224 fn set_vector_reg(&self, reg_num: u8, data: u128) -> Result<()>; 225 226 /// Gets the value of a Neon vector register (V0-V31) on this VCPU. get_vector_reg(&self, reg_num: u8) -> Result<u128>227 fn get_vector_reg(&self, reg_num: u8) -> Result<u128>; 228 229 /// Gets the set of system registers accessible by the hypervisor get_system_regs(&self) -> Result<BTreeMap<AArch64SysRegId, u64>>230 fn get_system_regs(&self) -> Result<BTreeMap<AArch64SysRegId, u64>>; 231 232 /// Gets the hypervisor specific data for snapshot hypervisor_specific_snapshot(&self) -> anyhow::Result<AnySnapshot>233 fn hypervisor_specific_snapshot(&self) -> anyhow::Result<AnySnapshot>; 234 235 /// Restores the hypervisor specific data hypervisor_specific_restore(&self, _data: AnySnapshot) -> anyhow::Result<()>236 fn hypervisor_specific_restore(&self, _data: AnySnapshot) -> anyhow::Result<()>; 237 238 /// Gets the value of MPIDR_EL1 on this VCPU. get_mpidr(&self) -> Result<u64>239 fn get_mpidr(&self) -> Result<u64> { 240 const RES1: u64 = 1 << 31; 241 242 // Assume that MPIDR_EL1.{U,MT} = {0,0}. 243 244 let aff = u64::try_from(self.id()).unwrap(); 245 246 Ok(RES1 | aff) 247 } 248 249 /// Gets the current PSCI version. get_psci_version(&self) -> Result<PsciVersion>250 fn get_psci_version(&self) -> Result<PsciVersion>; 251 252 /// Sets up debug registers and configure vcpu for handling guest debug events. set_guest_debug(&self, addrs: &[GuestAddress], enable_singlestep: bool) -> Result<()>253 fn set_guest_debug(&self, addrs: &[GuestAddress], enable_singlestep: bool) -> Result<()>; 254 255 /// Gets the max number of hardware breakpoints. get_max_hw_bps(&self) -> Result<usize>256 fn get_max_hw_bps(&self) -> Result<usize>; 257 258 /// Gets the cache architecture information for all cache levels. 259 /// The keys of the map are the lower 4 lower significant bits of CSSELR_EL1, which represents 260 /// the cache level. cache level is actually located in bits [3:1], but the value saves also 261 /// if the cache is an instruction or data. 262 /// The values of the map are CCSIDR_EL1, which is the configuration of the cache. get_cache_info(&self) -> Result<BTreeMap<u8, u64>>263 fn get_cache_info(&self) -> Result<BTreeMap<u8, u64>>; 264 265 /// Sets the cache architecture information for all cache levels. set_cache_info(&self, cache_info: BTreeMap<u8, u64>) -> Result<()>266 fn set_cache_info(&self, cache_info: BTreeMap<u8, u64>) -> Result<()>; 267 snapshot(&self) -> anyhow::Result<VcpuSnapshot>268 fn snapshot(&self) -> anyhow::Result<VcpuSnapshot> { 269 let mut snap = VcpuSnapshot { 270 vcpu_id: self.id(), 271 sp: self 272 .get_one_reg(VcpuRegAArch64::Sp) 273 .context("Failed to get SP")?, 274 pc: self 275 .get_one_reg(VcpuRegAArch64::Pc) 276 .context("Failed to get PC")?, 277 pstate: self 278 .get_one_reg(VcpuRegAArch64::Pstate) 279 .context("Failed to get PState")?, 280 x: Default::default(), 281 v: Default::default(), 282 hypervisor_data: self 283 .hypervisor_specific_snapshot() 284 .context("Failed to get hyprevisor specific data")?, 285 sys: self 286 .get_system_regs() 287 .context("Failed to read system registers")?, 288 cache_arch_info: self.get_cache_info().context("Failed to get cache info")?, 289 }; 290 291 for (n, xn) in snap.x.iter_mut().enumerate() { 292 *xn = self 293 .get_one_reg(VcpuRegAArch64::X(n as u8)) 294 .with_context(|| format!("Failed to get X{}", n))?; 295 } 296 297 for (n, vn) in snap.v.iter_mut().enumerate() { 298 *vn = self 299 .get_vector_reg(n as u8) 300 .with_context(|| format!("Failed to get V{}", n))?; 301 } 302 303 Ok(snap) 304 } 305 306 /// Restore VCPU restore(&self, snapshot: &VcpuSnapshot) -> anyhow::Result<()>307 fn restore(&self, snapshot: &VcpuSnapshot) -> anyhow::Result<()> { 308 self.set_one_reg(VcpuRegAArch64::Sp, snapshot.sp) 309 .context("Failed to restore SP register")?; 310 self.set_one_reg(VcpuRegAArch64::Pc, snapshot.pc) 311 .context("Failed to restore PC register")?; 312 self.set_one_reg(VcpuRegAArch64::Pstate, snapshot.pstate) 313 .context("Failed to restore PState register")?; 314 315 for (n, xn) in snapshot.x.iter().enumerate() { 316 self.set_one_reg(VcpuRegAArch64::X(n as u8), *xn) 317 .with_context(|| format!("Failed to restore X{}", n))?; 318 } 319 for (n, vn) in snapshot.v.iter().enumerate() { 320 self.set_vector_reg(n as u8, *vn) 321 .with_context(|| format!("Failed to restore V{}", n))?; 322 } 323 for (id, val) in &snapshot.sys { 324 self.set_one_reg(VcpuRegAArch64::System(*id), *val) 325 .with_context(|| format!("Failed to restore system register {:?}", id))?; 326 } 327 self.set_cache_info(snapshot.cache_arch_info.clone()) 328 .context("Failed to restore cache info")?; 329 self.hypervisor_specific_restore(snapshot.hypervisor_data.clone()) 330 .context("Failed to restore hypervisor specific data")?; 331 Ok(()) 332 } 333 } 334 335 /// Aarch64 specific vCPU snapshot. 336 #[derive(Clone, Debug, Serialize, Deserialize)] 337 pub struct VcpuSnapshot { 338 pub vcpu_id: usize, 339 pub sp: u64, 340 pub pc: u64, 341 pub pstate: u64, 342 pub x: [u64; 31], 343 pub v: [u128; 32], 344 pub sys: BTreeMap<AArch64SysRegId, u64>, 345 pub cache_arch_info: BTreeMap<u8, u64>, 346 pub hypervisor_data: AnySnapshot, 347 } 348 349 impl_downcast!(VcpuAArch64); 350 351 /// Initial register state for AArch64 VCPUs. 352 #[derive(Clone, Default)] 353 pub struct VcpuInitAArch64 { 354 /// Initial register state as a map of register name to value pairs. Registers that do not have 355 /// a value specified in this map will retain the original value provided by the hypervisor. 356 pub regs: BTreeMap<VcpuRegAArch64, u64>, 357 } 358 359 #[derive(Clone, Debug, PartialEq, Eq)] 360 pub struct CpuConfigAArch64 {} 361 362 // Convenience constructors for IrqRoutes 363 impl IrqRoute { gic_irq_route(irq_num: u32) -> IrqRoute364 pub fn gic_irq_route(irq_num: u32) -> IrqRoute { 365 IrqRoute { 366 gsi: irq_num, 367 source: IrqSource::Irqchip { 368 chip: IrqSourceChip::Gic, 369 pin: irq_num, 370 }, 371 } 372 } 373 } 374 375 /// A feature that can be enabled on a VCPU with `VcpuAArch64::init`. 376 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 377 pub enum VcpuFeature { 378 /// Emulate PSCI v0.2 (or a future revision backward compatible with v0.2) for the VCPU. 379 PsciV0_2, 380 /// Emulate Performance Monitor Unit v3 for the VCPU. 381 PmuV3, 382 /// Starts the VCPU in a power-off state. 383 PowerOff, 384 /// Scalable Vector Extension support 385 Sve, 386 } 387 388 #[cfg(test)] 389 mod tests { 390 use super::*; 391 392 #[test] sysreg_new()393 fn sysreg_new() { 394 let sysreg = AArch64SysRegId::new(1, 2, 3, 4, 5).unwrap(); 395 assert_eq!(sysreg.op0(), 1); 396 assert_eq!(sysreg.op1(), 2); 397 assert_eq!(sysreg.crn(), 3); 398 assert_eq!(sysreg.crm(), 4); 399 assert_eq!(sysreg.op2(), 5); 400 assert_eq!(sysreg.encoded(), 0x51A5); 401 } 402 403 #[test] sysreg_new_max()404 fn sysreg_new_max() { 405 let sysreg = AArch64SysRegId::new(0b11, 0b111, 0b1111, 0b1111, 0b111).unwrap(); 406 assert_eq!(sysreg.op0(), 3); 407 assert_eq!(sysreg.op1(), 7); 408 assert_eq!(sysreg.crn(), 15); 409 assert_eq!(sysreg.crm(), 15); 410 assert_eq!(sysreg.op2(), 7); 411 assert_eq!(sysreg.encoded(), 0xFFFF); 412 } 413 414 #[test] sysreg_new_out_of_range()415 fn sysreg_new_out_of_range() { 416 AArch64SysRegId::new(4, 0, 0, 0, 0).expect_err("invalid Op0"); 417 AArch64SysRegId::new(0, 8, 0, 0, 0).expect_err("invalid Op1"); 418 AArch64SysRegId::new(0, 0, 16, 0, 0).expect_err("invalid CRn"); 419 AArch64SysRegId::new(0, 0, 0, 16, 0).expect_err("invalid CRm"); 420 AArch64SysRegId::new(0, 0, 0, 0, 8).expect_err("invalid Op2"); 421 } 422 423 #[test] sysreg_encoding_mpidr_el1()424 fn sysreg_encoding_mpidr_el1() { 425 assert_eq!(AArch64SysRegId::MPIDR_EL1.op0(), 3); 426 assert_eq!(AArch64SysRegId::MPIDR_EL1.op1(), 0); 427 assert_eq!(AArch64SysRegId::MPIDR_EL1.crn(), 0); 428 assert_eq!(AArch64SysRegId::MPIDR_EL1.crm(), 0); 429 assert_eq!(AArch64SysRegId::MPIDR_EL1.op2(), 5); 430 assert_eq!(AArch64SysRegId::MPIDR_EL1.encoded(), 0xC005); 431 assert_eq!( 432 AArch64SysRegId::MPIDR_EL1, 433 AArch64SysRegId::new(3, 0, 0, 0, 5).unwrap() 434 ); 435 } 436 437 #[test] sysreg_encoding_cntvct_el0()438 fn sysreg_encoding_cntvct_el0() { 439 assert_eq!(AArch64SysRegId::CNTVCT_EL0.op0(), 3); 440 assert_eq!(AArch64SysRegId::CNTVCT_EL0.op1(), 3); 441 assert_eq!(AArch64SysRegId::CNTVCT_EL0.crn(), 14); 442 assert_eq!(AArch64SysRegId::CNTVCT_EL0.crm(), 0); 443 assert_eq!(AArch64SysRegId::CNTVCT_EL0.op2(), 2); 444 assert_eq!(AArch64SysRegId::CNTVCT_EL0.encoded(), 0xDF02); 445 assert_eq!( 446 AArch64SysRegId::CNTVCT_EL0, 447 AArch64SysRegId::new(3, 3, 14, 0, 2).unwrap() 448 ); 449 } 450 451 #[test] sysreg_encoding_cntv_cval_el0()452 fn sysreg_encoding_cntv_cval_el0() { 453 assert_eq!(AArch64SysRegId::CNTV_CVAL_EL0.op0(), 3); 454 assert_eq!(AArch64SysRegId::CNTV_CVAL_EL0.op1(), 3); 455 assert_eq!(AArch64SysRegId::CNTV_CVAL_EL0.crn(), 14); 456 assert_eq!(AArch64SysRegId::CNTV_CVAL_EL0.crm(), 3); 457 assert_eq!(AArch64SysRegId::CNTV_CVAL_EL0.op2(), 2); 458 assert_eq!(AArch64SysRegId::CNTV_CVAL_EL0.encoded(), 0xDF1A); 459 assert_eq!( 460 AArch64SysRegId::CNTV_CVAL_EL0, 461 AArch64SysRegId::new(3, 3, 14, 3, 2).unwrap() 462 ); 463 } 464 465 #[test] sysreg_debug()466 fn sysreg_debug() { 467 assert_eq!( 468 format!("{:?}", AArch64SysRegId::MPIDR_EL1), 469 "AArch64SysRegId { Op0: 3, Op1: 0, CRn: 0, CRm: 0, Op2: 5 }" 470 ); 471 } 472 } 473