1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Fault injection for both 32 and 64bit guests.
4 *
5 * Copyright (C) 2012,2013 - ARM Ltd
6 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 *
8 * Based on arch/arm/kvm/emulate.c
9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
10 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
11 */
12
13 #include <hyp/adjust_pc.h>
14 #include <linux/kvm_host.h>
15 #include <asm/kvm_emulate.h>
16 #include <asm/kvm_mmu.h>
17
18 #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__)
19 #error Hypervisor code only!
20 #endif
21
__vcpu_read_sys_reg(const struct kvm_vcpu * vcpu,int reg)22 static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
23 {
24 u64 val;
25
26 if (__vcpu_read_sys_reg_from_cpu(reg, &val))
27 return val;
28
29 return __vcpu_sys_reg(vcpu, reg);
30 }
31
__vcpu_write_sys_reg(struct kvm_vcpu * vcpu,u64 val,int reg)32 static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
33 {
34 if (__vcpu_write_sys_reg_to_cpu(val, reg))
35 return;
36
37 __vcpu_sys_reg(vcpu, reg) = val;
38 }
39
__vcpu_write_spsr(struct kvm_vcpu * vcpu,u64 val)40 static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val)
41 {
42 if (has_vhe())
43 write_sysreg_el1(val, SYS_SPSR);
44 else
45 __vcpu_sys_reg(vcpu, SPSR_EL1) = val;
46 }
47
__vcpu_write_spsr_abt(struct kvm_vcpu * vcpu,u64 val)48 static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val)
49 {
50 if (has_vhe())
51 write_sysreg(val, spsr_abt);
52 else
53 vcpu->arch.ctxt.spsr_abt = val;
54 }
55
__vcpu_write_spsr_und(struct kvm_vcpu * vcpu,u64 val)56 static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val)
57 {
58 if (has_vhe())
59 write_sysreg(val, spsr_und);
60 else
61 vcpu->arch.ctxt.spsr_und = val;
62 }
63
get_except64_offset(unsigned long psr,unsigned long target_mode,enum exception_type type)64 unsigned long get_except64_offset(unsigned long psr, unsigned long target_mode,
65 enum exception_type type)
66 {
67 u64 mode = psr & (PSR_MODE_MASK | PSR_MODE32_BIT);
68 u64 exc_offset;
69
70 if (mode == target_mode)
71 exc_offset = CURRENT_EL_SP_ELx_VECTOR;
72 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode)
73 exc_offset = CURRENT_EL_SP_EL0_VECTOR;
74 else if (!(mode & PSR_MODE32_BIT))
75 exc_offset = LOWER_EL_AArch64_VECTOR;
76 else
77 exc_offset = LOWER_EL_AArch32_VECTOR;
78
79 return exc_offset + type;
80 }
81
82 /*
83 * When an exception is taken, most PSTATE fields are left unchanged in the
84 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all
85 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx
86 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0.
87 *
88 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429.
89 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426.
90 *
91 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from
92 * MSB to LSB.
93 */
get_except64_cpsr(unsigned long old,bool has_mte,unsigned long sctlr,unsigned long target_mode)94 unsigned long get_except64_cpsr(unsigned long old, bool has_mte,
95 unsigned long sctlr, unsigned long target_mode)
96 {
97 u64 new = 0;
98
99 new |= (old & PSR_N_BIT);
100 new |= (old & PSR_Z_BIT);
101 new |= (old & PSR_C_BIT);
102 new |= (old & PSR_V_BIT);
103
104 if (has_mte)
105 new |= PSR_TCO_BIT;
106
107 new |= (old & PSR_DIT_BIT);
108
109 // PSTATE.UAO is set to zero upon any exception to AArch64
110 // See ARM DDI 0487E.a, page D5-2579.
111
112 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0
113 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented
114 // See ARM DDI 0487E.a, page D5-2578.
115 new |= (old & PSR_PAN_BIT);
116 if (!(sctlr & SCTLR_EL1_SPAN))
117 new |= PSR_PAN_BIT;
118
119 // PSTATE.SS is set to zero upon any exception to AArch64
120 // See ARM DDI 0487E.a, page D2-2452.
121
122 // PSTATE.IL is set to zero upon any exception to AArch64
123 // See ARM DDI 0487E.a, page D1-2306.
124
125 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64
126 // See ARM DDI 0487E.a, page D13-3258
127 if (sctlr & SCTLR_ELx_DSSBS)
128 new |= PSR_SSBS_BIT;
129
130 // PSTATE.BTYPE is set to zero upon any exception to AArch64
131 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294.
132
133 new |= PSR_D_BIT;
134 new |= PSR_A_BIT;
135 new |= PSR_I_BIT;
136 new |= PSR_F_BIT;
137
138 new |= target_mode;
139
140 return new;
141 }
142
143 /*
144 * This performs the exception entry at a given EL (@target_mode), stashing PC
145 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE.
146 * The EL passed to this function *must* be a non-secure, privileged mode with
147 * bit 0 being set (PSTATE.SP == 1).
148 */
enter_exception64(struct kvm_vcpu * vcpu,unsigned long target_mode,enum exception_type type)149 static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
150 enum exception_type type)
151 {
152 u64 offset = get_except64_offset(*vcpu_cpsr(vcpu), target_mode, type);
153 unsigned long sctlr, vbar, old, new;
154
155 switch (target_mode) {
156 case PSR_MODE_EL1h:
157 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1);
158 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
159 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
160 break;
161 default:
162 /* Don't do that */
163 BUG();
164 }
165
166 *vcpu_pc(vcpu) = vbar + offset;
167
168 old = *vcpu_cpsr(vcpu);
169 new = get_except64_cpsr(old, kvm_has_mte(kern_hyp_va(vcpu->kvm)), sctlr,
170 target_mode);
171 *vcpu_cpsr(vcpu) = new;
172 __vcpu_write_spsr(vcpu, old);
173 }
174
175 /*
176 * When an exception is taken, most CPSR fields are left unchanged in the
177 * handler. However, some are explicitly overridden (e.g. M[4:0]).
178 *
179 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with
180 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was
181 * obsoleted by the ARMv7 virtualization extensions and is RES0.
182 *
183 * For the SPSR layout seen from AArch32, see:
184 * - ARM DDI 0406C.d, page B1-1148
185 * - ARM DDI 0487E.a, page G8-6264
186 *
187 * For the SPSR_ELx layout for AArch32 seen from AArch64, see:
188 * - ARM DDI 0487E.a, page C5-426
189 *
190 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from
191 * MSB to LSB.
192 */
get_except32_cpsr(struct kvm_vcpu * vcpu,u32 mode)193 static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode)
194 {
195 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
196 unsigned long old, new;
197
198 old = *vcpu_cpsr(vcpu);
199 new = 0;
200
201 new |= (old & PSR_AA32_N_BIT);
202 new |= (old & PSR_AA32_Z_BIT);
203 new |= (old & PSR_AA32_C_BIT);
204 new |= (old & PSR_AA32_V_BIT);
205 new |= (old & PSR_AA32_Q_BIT);
206
207 // CPSR.IT[7:0] are set to zero upon any exception
208 // See ARM DDI 0487E.a, section G1.12.3
209 // See ARM DDI 0406C.d, section B1.8.3
210
211 new |= (old & PSR_AA32_DIT_BIT);
212
213 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception
214 // See ARM DDI 0487E.a, page G8-6244
215 if (sctlr & BIT(31))
216 new |= PSR_AA32_SSBS_BIT;
217
218 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0
219 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented
220 // See ARM DDI 0487E.a, page G8-6246
221 new |= (old & PSR_AA32_PAN_BIT);
222 if (!(sctlr & BIT(23)))
223 new |= PSR_AA32_PAN_BIT;
224
225 // SS does not exist in AArch32, so ignore
226
227 // CPSR.IL is set to zero upon any exception
228 // See ARM DDI 0487E.a, page G1-5527
229
230 new |= (old & PSR_AA32_GE_MASK);
231
232 // CPSR.IT[7:0] are set to zero upon any exception
233 // See prior comment above
234
235 // CPSR.E is set to SCTLR.EE upon any exception
236 // See ARM DDI 0487E.a, page G8-6245
237 // See ARM DDI 0406C.d, page B4-1701
238 if (sctlr & BIT(25))
239 new |= PSR_AA32_E_BIT;
240
241 // CPSR.A is unchanged upon an exception to Undefined, Supervisor
242 // CPSR.A is set upon an exception to other modes
243 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
244 // See ARM DDI 0406C.d, page B1-1182
245 new |= (old & PSR_AA32_A_BIT);
246 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC)
247 new |= PSR_AA32_A_BIT;
248
249 // CPSR.I is set upon any exception
250 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
251 // See ARM DDI 0406C.d, page B1-1182
252 new |= PSR_AA32_I_BIT;
253
254 // CPSR.F is set upon an exception to FIQ
255 // CPSR.F is unchanged upon an exception to other modes
256 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
257 // See ARM DDI 0406C.d, page B1-1182
258 new |= (old & PSR_AA32_F_BIT);
259 if (mode == PSR_AA32_MODE_FIQ)
260 new |= PSR_AA32_F_BIT;
261
262 // CPSR.T is set to SCTLR.TE upon any exception
263 // See ARM DDI 0487E.a, page G8-5514
264 // See ARM DDI 0406C.d, page B1-1181
265 if (sctlr & BIT(30))
266 new |= PSR_AA32_T_BIT;
267
268 new |= mode;
269
270 return new;
271 }
272
273 /*
274 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10.
275 */
276 static const u8 return_offsets[8][2] = {
277 [0] = { 0, 0 }, /* Reset, unused */
278 [1] = { 4, 2 }, /* Undefined */
279 [2] = { 0, 0 }, /* SVC, unused */
280 [3] = { 4, 4 }, /* Prefetch abort */
281 [4] = { 8, 8 }, /* Data abort */
282 [5] = { 0, 0 }, /* HVC, unused */
283 [6] = { 4, 4 }, /* IRQ, unused */
284 [7] = { 4, 4 }, /* FIQ, unused */
285 };
286
enter_exception32(struct kvm_vcpu * vcpu,u32 mode,u32 vect_offset)287 static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
288 {
289 unsigned long spsr = *vcpu_cpsr(vcpu);
290 bool is_thumb = (spsr & PSR_AA32_T_BIT);
291 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
292 u32 return_address;
293
294 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
295 return_address = *vcpu_pc(vcpu);
296 return_address += return_offsets[vect_offset >> 2][is_thumb];
297
298 /* KVM only enters the ABT and UND modes, so only deal with those */
299 switch(mode) {
300 case PSR_AA32_MODE_ABT:
301 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr));
302 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address;
303 break;
304
305 case PSR_AA32_MODE_UND:
306 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr));
307 vcpu_gp_regs(vcpu)->compat_lr_und = return_address;
308 break;
309 }
310
311 /* Branch to exception vector */
312 if (sctlr & (1 << 13))
313 vect_offset += 0xffff0000;
314 else /* always have security exceptions */
315 vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1);
316
317 *vcpu_pc(vcpu) = vect_offset;
318 }
319
kvm_inject_exception(struct kvm_vcpu * vcpu)320 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
321 {
322 if (vcpu_el1_is_32bit(vcpu)) {
323 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
324 case unpack_vcpu_flag(EXCEPT_AA32_UND):
325 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4);
326 break;
327 case unpack_vcpu_flag(EXCEPT_AA32_IABT):
328 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12);
329 break;
330 case unpack_vcpu_flag(EXCEPT_AA32_DABT):
331 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16);
332 break;
333 default:
334 /* Err... */
335 break;
336 }
337 } else {
338 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
339 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
340 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
341 break;
342 default:
343 /*
344 * Only EL1_SYNC makes sense so far, EL2_{SYNC,IRQ}
345 * will be implemented at some point. Everything
346 * else gets silently ignored.
347 */
348 break;
349 }
350 }
351 }
352
353 /*
354 * Adjust the guest PC (and potentially exception state) depending on
355 * flags provided by the emulation code.
356 */
__kvm_adjust_pc(struct kvm_vcpu * vcpu)357 void __kvm_adjust_pc(struct kvm_vcpu *vcpu)
358 {
359 if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) {
360 kvm_inject_exception(vcpu);
361 vcpu_clear_flag(vcpu, PENDING_EXCEPTION);
362 vcpu_clear_flag(vcpu, EXCEPT_MASK);
363 } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) {
364 kvm_skip_instr(vcpu);
365 vcpu_clear_flag(vcpu, INCREMENT_PC);
366 }
367 }
368