1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Fault injection for both 32 and 64bit guests.
4 *
5 * Copyright (C) 2012,2013 - ARM Ltd
6 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 *
8 * Based on arch/arm/kvm/emulate.c
9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
10 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
11 */
12
13 #include <hyp/adjust_pc.h>
14 #include <linux/kvm_host.h>
15 #include <asm/kvm_emulate.h>
16 #include <asm/kvm_mmu.h>
17 #include <asm/kvm_nested.h>
18
19 #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__)
20 #error Hypervisor code only!
21 #endif
22
__vcpu_read_sys_reg(const struct kvm_vcpu * vcpu,int reg)23 static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
24 {
25 u64 val;
26
27 if (unlikely(vcpu_has_nv(vcpu)))
28 return vcpu_read_sys_reg(vcpu, reg);
29 else if (__vcpu_read_sys_reg_from_cpu(reg, &val))
30 return val;
31
32 return __vcpu_sys_reg(vcpu, reg);
33 }
34
__vcpu_write_sys_reg(struct kvm_vcpu * vcpu,u64 val,int reg)35 static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
36 {
37 if (unlikely(vcpu_has_nv(vcpu)))
38 vcpu_write_sys_reg(vcpu, val, reg);
39 else if (!__vcpu_write_sys_reg_to_cpu(val, reg))
40 __vcpu_sys_reg(vcpu, reg) = val;
41 }
42
__vcpu_write_spsr(struct kvm_vcpu * vcpu,unsigned long target_mode,u64 val)43 static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long target_mode,
44 u64 val)
45 {
46 if (unlikely(vcpu_has_nv(vcpu))) {
47 if (target_mode == PSR_MODE_EL1h)
48 vcpu_write_sys_reg(vcpu, val, SPSR_EL1);
49 else
50 vcpu_write_sys_reg(vcpu, val, SPSR_EL2);
51 } else if (has_vhe()) {
52 write_sysreg_el1(val, SYS_SPSR);
53 } else {
54 __vcpu_sys_reg(vcpu, SPSR_EL1) = val;
55 }
56 }
57
__vcpu_write_spsr_abt(struct kvm_vcpu * vcpu,u64 val)58 static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val)
59 {
60 if (has_vhe())
61 write_sysreg(val, spsr_abt);
62 else
63 vcpu->arch.ctxt.spsr_abt = val;
64 }
65
__vcpu_write_spsr_und(struct kvm_vcpu * vcpu,u64 val)66 static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val)
67 {
68 if (has_vhe())
69 write_sysreg(val, spsr_und);
70 else
71 vcpu->arch.ctxt.spsr_und = val;
72 }
73
get_except64_offset(unsigned long psr,unsigned long target_mode,enum exception_type type)74 unsigned long get_except64_offset(unsigned long psr, unsigned long target_mode,
75 enum exception_type type)
76 {
77 u64 mode = psr & (PSR_MODE_MASK | PSR_MODE32_BIT);
78 u64 exc_offset;
79
80 if (mode == target_mode)
81 exc_offset = CURRENT_EL_SP_ELx_VECTOR;
82 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode)
83 exc_offset = CURRENT_EL_SP_EL0_VECTOR;
84 else if (!(mode & PSR_MODE32_BIT))
85 exc_offset = LOWER_EL_AArch64_VECTOR;
86 else
87 exc_offset = LOWER_EL_AArch32_VECTOR;
88
89 return exc_offset + type;
90 }
91
92 /*
93 * When an exception is taken, most PSTATE fields are left unchanged in the
94 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all
95 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx
96 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0.
97 *
98 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429.
99 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426.
100 *
101 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from
102 * MSB to LSB.
103 */
get_except64_cpsr(unsigned long old,bool has_mte,unsigned long sctlr,unsigned long target_mode)104 unsigned long get_except64_cpsr(unsigned long old, bool has_mte,
105 unsigned long sctlr, unsigned long target_mode)
106 {
107 u64 new = 0;
108
109 new |= (old & PSR_N_BIT);
110 new |= (old & PSR_Z_BIT);
111 new |= (old & PSR_C_BIT);
112 new |= (old & PSR_V_BIT);
113
114 if (has_mte)
115 new |= PSR_TCO_BIT;
116
117 new |= (old & PSR_DIT_BIT);
118
119 // PSTATE.UAO is set to zero upon any exception to AArch64
120 // See ARM DDI 0487E.a, page D5-2579.
121
122 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0
123 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented
124 // See ARM DDI 0487E.a, page D5-2578.
125 new |= (old & PSR_PAN_BIT);
126 if (!(sctlr & SCTLR_EL1_SPAN))
127 new |= PSR_PAN_BIT;
128
129 // PSTATE.SS is set to zero upon any exception to AArch64
130 // See ARM DDI 0487E.a, page D2-2452.
131
132 // PSTATE.IL is set to zero upon any exception to AArch64
133 // See ARM DDI 0487E.a, page D1-2306.
134
135 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64
136 // See ARM DDI 0487E.a, page D13-3258
137 if (sctlr & SCTLR_ELx_DSSBS)
138 new |= PSR_SSBS_BIT;
139
140 // PSTATE.BTYPE is set to zero upon any exception to AArch64
141 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294.
142
143 new |= PSR_D_BIT;
144 new |= PSR_A_BIT;
145 new |= PSR_I_BIT;
146 new |= PSR_F_BIT;
147
148 new |= target_mode;
149
150 return new;
151 }
152
153 /*
154 * This performs the exception entry at a given EL (@target_mode), stashing PC
155 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE.
156 * The EL passed to this function *must* be a non-secure, privileged mode with
157 * bit 0 being set (PSTATE.SP == 1).
158 */
enter_exception64(struct kvm_vcpu * vcpu,unsigned long target_mode,enum exception_type type)159 static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
160 enum exception_type type)
161 {
162 u64 offset = get_except64_offset(*vcpu_cpsr(vcpu), target_mode, type);
163 unsigned long sctlr, vbar, old, new;
164
165 switch (target_mode) {
166 case PSR_MODE_EL1h:
167 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1);
168 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
169 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
170 break;
171 case PSR_MODE_EL2h:
172 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL2);
173 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL2);
174 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2);
175 break;
176 default:
177 /* Don't do that */
178 BUG();
179 }
180
181 *vcpu_pc(vcpu) = vbar + offset;
182
183 old = *vcpu_cpsr(vcpu);
184 new = get_except64_cpsr(old, kvm_has_mte(kern_hyp_va(vcpu->kvm)), sctlr,
185 target_mode);
186 *vcpu_cpsr(vcpu) = new;
187 __vcpu_write_spsr(vcpu, target_mode, old);
188 }
189
190 /*
191 * When an exception is taken, most CPSR fields are left unchanged in the
192 * handler. However, some are explicitly overridden (e.g. M[4:0]).
193 *
194 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with
195 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was
196 * obsoleted by the ARMv7 virtualization extensions and is RES0.
197 *
198 * For the SPSR layout seen from AArch32, see:
199 * - ARM DDI 0406C.d, page B1-1148
200 * - ARM DDI 0487E.a, page G8-6264
201 *
202 * For the SPSR_ELx layout for AArch32 seen from AArch64, see:
203 * - ARM DDI 0487E.a, page C5-426
204 *
205 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from
206 * MSB to LSB.
207 */
get_except32_cpsr(struct kvm_vcpu * vcpu,u32 mode)208 static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode)
209 {
210 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
211 unsigned long old, new;
212
213 old = *vcpu_cpsr(vcpu);
214 new = 0;
215
216 new |= (old & PSR_AA32_N_BIT);
217 new |= (old & PSR_AA32_Z_BIT);
218 new |= (old & PSR_AA32_C_BIT);
219 new |= (old & PSR_AA32_V_BIT);
220 new |= (old & PSR_AA32_Q_BIT);
221
222 // CPSR.IT[7:0] are set to zero upon any exception
223 // See ARM DDI 0487E.a, section G1.12.3
224 // See ARM DDI 0406C.d, section B1.8.3
225
226 new |= (old & PSR_AA32_DIT_BIT);
227
228 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception
229 // See ARM DDI 0487E.a, page G8-6244
230 if (sctlr & BIT(31))
231 new |= PSR_AA32_SSBS_BIT;
232
233 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0
234 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented
235 // See ARM DDI 0487E.a, page G8-6246
236 new |= (old & PSR_AA32_PAN_BIT);
237 if (!(sctlr & BIT(23)))
238 new |= PSR_AA32_PAN_BIT;
239
240 // SS does not exist in AArch32, so ignore
241
242 // CPSR.IL is set to zero upon any exception
243 // See ARM DDI 0487E.a, page G1-5527
244
245 new |= (old & PSR_AA32_GE_MASK);
246
247 // CPSR.IT[7:0] are set to zero upon any exception
248 // See prior comment above
249
250 // CPSR.E is set to SCTLR.EE upon any exception
251 // See ARM DDI 0487E.a, page G8-6245
252 // See ARM DDI 0406C.d, page B4-1701
253 if (sctlr & BIT(25))
254 new |= PSR_AA32_E_BIT;
255
256 // CPSR.A is unchanged upon an exception to Undefined, Supervisor
257 // CPSR.A is set upon an exception to other modes
258 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
259 // See ARM DDI 0406C.d, page B1-1182
260 new |= (old & PSR_AA32_A_BIT);
261 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC)
262 new |= PSR_AA32_A_BIT;
263
264 // CPSR.I is set upon any exception
265 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
266 // See ARM DDI 0406C.d, page B1-1182
267 new |= PSR_AA32_I_BIT;
268
269 // CPSR.F is set upon an exception to FIQ
270 // CPSR.F is unchanged upon an exception to other modes
271 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
272 // See ARM DDI 0406C.d, page B1-1182
273 new |= (old & PSR_AA32_F_BIT);
274 if (mode == PSR_AA32_MODE_FIQ)
275 new |= PSR_AA32_F_BIT;
276
277 // CPSR.T is set to SCTLR.TE upon any exception
278 // See ARM DDI 0487E.a, page G8-5514
279 // See ARM DDI 0406C.d, page B1-1181
280 if (sctlr & BIT(30))
281 new |= PSR_AA32_T_BIT;
282
283 new |= mode;
284
285 return new;
286 }
287
288 /*
289 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10.
290 */
291 static const u8 return_offsets[8][2] = {
292 [0] = { 0, 0 }, /* Reset, unused */
293 [1] = { 4, 2 }, /* Undefined */
294 [2] = { 0, 0 }, /* SVC, unused */
295 [3] = { 4, 4 }, /* Prefetch abort */
296 [4] = { 8, 8 }, /* Data abort */
297 [5] = { 0, 0 }, /* HVC, unused */
298 [6] = { 4, 4 }, /* IRQ, unused */
299 [7] = { 4, 4 }, /* FIQ, unused */
300 };
301
enter_exception32(struct kvm_vcpu * vcpu,u32 mode,u32 vect_offset)302 static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
303 {
304 unsigned long spsr = *vcpu_cpsr(vcpu);
305 bool is_thumb = (spsr & PSR_AA32_T_BIT);
306 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
307 u32 return_address;
308
309 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
310 return_address = *vcpu_pc(vcpu);
311 return_address += return_offsets[vect_offset >> 2][is_thumb];
312
313 /* KVM only enters the ABT and UND modes, so only deal with those */
314 switch(mode) {
315 case PSR_AA32_MODE_ABT:
316 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr));
317 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address;
318 break;
319
320 case PSR_AA32_MODE_UND:
321 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr));
322 vcpu_gp_regs(vcpu)->compat_lr_und = return_address;
323 break;
324 }
325
326 /* Branch to exception vector */
327 if (sctlr & (1 << 13))
328 vect_offset += 0xffff0000;
329 else /* always have security exceptions */
330 vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1);
331
332 *vcpu_pc(vcpu) = vect_offset;
333 }
334
kvm_inject_exception(struct kvm_vcpu * vcpu)335 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
336 {
337 if (vcpu_el1_is_32bit(vcpu)) {
338 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
339 case unpack_vcpu_flag(EXCEPT_AA32_UND):
340 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4);
341 break;
342 case unpack_vcpu_flag(EXCEPT_AA32_IABT):
343 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12);
344 break;
345 case unpack_vcpu_flag(EXCEPT_AA32_DABT):
346 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16);
347 break;
348 default:
349 /* Err... */
350 break;
351 }
352 } else {
353 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
354 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
355 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
356 break;
357
358 case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
359 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync);
360 break;
361
362 case unpack_vcpu_flag(EXCEPT_AA64_EL2_IRQ):
363 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq);
364 break;
365
366 default:
367 /*
368 * Only EL1_SYNC and EL2_{SYNC,IRQ} makes
369 * sense so far. Everything else gets silently
370 * ignored.
371 */
372 break;
373 }
374 }
375 }
376
377 /*
378 * Adjust the guest PC (and potentially exception state) depending on
379 * flags provided by the emulation code.
380 */
__kvm_adjust_pc(struct kvm_vcpu * vcpu)381 void __kvm_adjust_pc(struct kvm_vcpu *vcpu)
382 {
383 if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) {
384 kvm_inject_exception(vcpu);
385 vcpu_clear_flag(vcpu, PENDING_EXCEPTION);
386 vcpu_clear_flag(vcpu, EXCEPT_MASK);
387 } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) {
388 kvm_skip_instr(vcpu);
389 vcpu_clear_flag(vcpu, INCREMENT_PC);
390 }
391 }
392