1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/kvm_host.h>
4 #include <asm/asm-prototypes.h>
5 #include <asm/dbell.h>
6 #include <asm/kvm_ppc.h>
7 #include <asm/ppc-opcode.h>
8
9 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
__accumulate_time(struct kvm_vcpu * vcpu,struct kvmhv_tb_accumulator * next)10 static void __accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next)
11 {
12 struct kvmppc_vcore *vc = vcpu->arch.vcore;
13 struct kvmhv_tb_accumulator *curr;
14 u64 tb = mftb() - vc->tb_offset_applied;
15 u64 prev_tb;
16 u64 delta;
17 u64 seq;
18
19 curr = vcpu->arch.cur_activity;
20 vcpu->arch.cur_activity = next;
21 prev_tb = vcpu->arch.cur_tb_start;
22 vcpu->arch.cur_tb_start = tb;
23
24 if (!curr)
25 return;
26
27 delta = tb - prev_tb;
28
29 seq = curr->seqcount;
30 curr->seqcount = seq + 1;
31 smp_wmb();
32 curr->tb_total += delta;
33 if (seq == 0 || delta < curr->tb_min)
34 curr->tb_min = delta;
35 if (delta > curr->tb_max)
36 curr->tb_max = delta;
37 smp_wmb();
38 curr->seqcount = seq + 2;
39 }
40
41 #define start_timing(vcpu, next) __accumulate_time(vcpu, next)
42 #define end_timing(vcpu) __accumulate_time(vcpu, NULL)
43 #define accumulate_time(vcpu, next) __accumulate_time(vcpu, next)
44 #else
45 #define start_timing(vcpu, next) do {} while (0)
46 #define end_timing(vcpu) do {} while (0)
47 #define accumulate_time(vcpu, next) do {} while (0)
48 #endif
49
mfslb(unsigned int idx,u64 * slbee,u64 * slbev)50 static inline void mfslb(unsigned int idx, u64 *slbee, u64 *slbev)
51 {
52 asm volatile("slbmfev %0,%1" : "=r" (*slbev) : "r" (idx));
53 asm volatile("slbmfee %0,%1" : "=r" (*slbee) : "r" (idx));
54 }
55
mtslb(u64 slbee,u64 slbev)56 static inline void mtslb(u64 slbee, u64 slbev)
57 {
58 asm volatile("slbmte %0,%1" :: "r" (slbev), "r" (slbee));
59 }
60
clear_slb_entry(unsigned int idx)61 static inline void clear_slb_entry(unsigned int idx)
62 {
63 mtslb(idx, 0);
64 }
65
slb_clear_invalidate_partition(void)66 static inline void slb_clear_invalidate_partition(void)
67 {
68 clear_slb_entry(0);
69 asm volatile(PPC_SLBIA(6));
70 }
71
72 /*
73 * Malicious or buggy radix guests may have inserted SLB entries
74 * (only 0..3 because radix always runs with UPRT=1), so these must
75 * be cleared here to avoid side-channels. slbmte is used rather
76 * than slbia, as it won't clear cached translations.
77 */
radix_clear_slb(void)78 static void radix_clear_slb(void)
79 {
80 int i;
81
82 for (i = 0; i < 4; i++)
83 clear_slb_entry(i);
84 }
85
switch_mmu_to_guest_radix(struct kvm * kvm,struct kvm_vcpu * vcpu,u64 lpcr)86 static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
87 {
88 struct kvm_nested_guest *nested = vcpu->arch.nested;
89 u32 lpid;
90
91 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
92
93 /*
94 * All the isync()s are overkill but trivially follow the ISA
95 * requirements. Some can likely be replaced with justification
96 * comment for why they are not needed.
97 */
98 isync();
99 mtspr(SPRN_LPID, lpid);
100 isync();
101 mtspr(SPRN_LPCR, lpcr);
102 isync();
103 mtspr(SPRN_PID, vcpu->arch.pid);
104 isync();
105 }
106
switch_mmu_to_guest_hpt(struct kvm * kvm,struct kvm_vcpu * vcpu,u64 lpcr)107 static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
108 {
109 u32 lpid;
110 int i;
111
112 lpid = kvm->arch.lpid;
113
114 mtspr(SPRN_LPID, lpid);
115 mtspr(SPRN_LPCR, lpcr);
116 mtspr(SPRN_PID, vcpu->arch.pid);
117
118 for (i = 0; i < vcpu->arch.slb_max; i++)
119 mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv);
120
121 isync();
122 }
123
switch_mmu_to_host(struct kvm * kvm,u32 pid)124 static void switch_mmu_to_host(struct kvm *kvm, u32 pid)
125 {
126 isync();
127 mtspr(SPRN_PID, pid);
128 isync();
129 mtspr(SPRN_LPID, kvm->arch.host_lpid);
130 isync();
131 mtspr(SPRN_LPCR, kvm->arch.host_lpcr);
132 isync();
133
134 if (!radix_enabled())
135 slb_restore_bolted_realmode();
136 }
137
save_clear_host_mmu(struct kvm * kvm)138 static void save_clear_host_mmu(struct kvm *kvm)
139 {
140 if (!radix_enabled()) {
141 /*
142 * Hash host could save and restore host SLB entries to
143 * reduce SLB fault overheads of VM exits, but for now the
144 * existing code clears all entries and restores just the
145 * bolted ones when switching back to host.
146 */
147 slb_clear_invalidate_partition();
148 }
149 }
150
save_clear_guest_mmu(struct kvm * kvm,struct kvm_vcpu * vcpu)151 static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu)
152 {
153 if (kvm_is_radix(kvm)) {
154 radix_clear_slb();
155 } else {
156 int i;
157 int nr = 0;
158
159 /*
160 * This must run before switching to host (radix host can't
161 * access all SLBs).
162 */
163 for (i = 0; i < vcpu->arch.slb_nr; i++) {
164 u64 slbee, slbev;
165 mfslb(i, &slbee, &slbev);
166 if (slbee & SLB_ESID_V) {
167 vcpu->arch.slb[nr].orige = slbee | i;
168 vcpu->arch.slb[nr].origv = slbev;
169 nr++;
170 }
171 }
172 vcpu->arch.slb_max = nr;
173 slb_clear_invalidate_partition();
174 }
175 }
176
kvmhv_vcpu_entry_p9(struct kvm_vcpu * vcpu,u64 time_limit,unsigned long lpcr)177 int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr)
178 {
179 struct kvm *kvm = vcpu->kvm;
180 struct kvm_nested_guest *nested = vcpu->arch.nested;
181 struct kvmppc_vcore *vc = vcpu->arch.vcore;
182 s64 hdec;
183 u64 tb, purr, spurr;
184 u64 *exsave;
185 bool ri_set;
186 int trap;
187 unsigned long msr;
188 unsigned long host_hfscr;
189 unsigned long host_ciabr;
190 unsigned long host_dawr0;
191 unsigned long host_dawrx0;
192 unsigned long host_psscr;
193 unsigned long host_pidr;
194 unsigned long host_dawr1;
195 unsigned long host_dawrx1;
196
197 hdec = time_limit - mftb();
198 if (hdec < 0)
199 return BOOK3S_INTERRUPT_HV_DECREMENTER;
200
201 WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_HV);
202 WARN_ON_ONCE(!(vcpu->arch.shregs.msr & MSR_ME));
203
204 start_timing(vcpu, &vcpu->arch.rm_entry);
205
206 vcpu->arch.ceded = 0;
207
208 if (vc->tb_offset) {
209 u64 new_tb = mftb() + vc->tb_offset;
210 mtspr(SPRN_TBU40, new_tb);
211 tb = mftb();
212 if ((tb & 0xffffff) < (new_tb & 0xffffff))
213 mtspr(SPRN_TBU40, new_tb + 0x1000000);
214 vc->tb_offset_applied = vc->tb_offset;
215 }
216
217 msr = mfmsr();
218
219 host_hfscr = mfspr(SPRN_HFSCR);
220 host_ciabr = mfspr(SPRN_CIABR);
221 host_dawr0 = mfspr(SPRN_DAWR0);
222 host_dawrx0 = mfspr(SPRN_DAWRX0);
223 host_psscr = mfspr(SPRN_PSSCR);
224 host_pidr = mfspr(SPRN_PID);
225 if (cpu_has_feature(CPU_FTR_DAWR1)) {
226 host_dawr1 = mfspr(SPRN_DAWR1);
227 host_dawrx1 = mfspr(SPRN_DAWRX1);
228 }
229
230 if (vc->pcr)
231 mtspr(SPRN_PCR, vc->pcr | PCR_MASK);
232 mtspr(SPRN_DPDES, vc->dpdes);
233 mtspr(SPRN_VTB, vc->vtb);
234
235 local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR);
236 local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR);
237 mtspr(SPRN_PURR, vcpu->arch.purr);
238 mtspr(SPRN_SPURR, vcpu->arch.spurr);
239
240 if (dawr_enabled()) {
241 mtspr(SPRN_DAWR0, vcpu->arch.dawr0);
242 mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0);
243 if (cpu_has_feature(CPU_FTR_DAWR1)) {
244 mtspr(SPRN_DAWR1, vcpu->arch.dawr1);
245 mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1);
246 }
247 }
248 mtspr(SPRN_CIABR, vcpu->arch.ciabr);
249 mtspr(SPRN_IC, vcpu->arch.ic);
250
251 mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC |
252 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
253
254 mtspr(SPRN_HFSCR, vcpu->arch.hfscr);
255
256 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip);
257 mtspr(SPRN_HSRR1, (vcpu->arch.shregs.msr & ~MSR_HV) | MSR_ME);
258
259 /*
260 * On POWER9 DD2.1 and below, sometimes on a Hypervisor Data Storage
261 * Interrupt (HDSI) the HDSISR is not be updated at all.
262 *
263 * To work around this we put a canary value into the HDSISR before
264 * returning to a guest and then check for this canary when we take a
265 * HDSI. If we find the canary on a HDSI, we know the hardware didn't
266 * update the HDSISR. In this case we return to the guest to retake the
267 * HDSI which should correctly update the HDSISR the second time HDSI
268 * entry.
269 *
270 * Just do this on all p9 processors for now.
271 */
272 mtspr(SPRN_HDSISR, HDSISR_CANARY);
273
274 mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0);
275 mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1);
276 mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2);
277 mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3);
278
279 mtspr(SPRN_AMOR, ~0UL);
280
281 local_paca->kvm_hstate.in_guest = KVM_GUEST_MODE_HV_P9;
282
283 /*
284 * Hash host, hash guest, or radix guest with prefetch bug, all have
285 * to disable the MMU before switching to guest MMU state.
286 */
287 if (!radix_enabled() || !kvm_is_radix(kvm) ||
288 cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
289 __mtmsrd(msr & ~(MSR_IR|MSR_DR|MSR_RI), 0);
290
291 save_clear_host_mmu(kvm);
292
293 if (kvm_is_radix(kvm)) {
294 switch_mmu_to_guest_radix(kvm, vcpu, lpcr);
295 if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
296 __mtmsrd(0, 1); /* clear RI */
297
298 } else {
299 switch_mmu_to_guest_hpt(kvm, vcpu, lpcr);
300 }
301
302 /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */
303 kvmppc_check_need_tlb_flush(kvm, vc->pcpu, nested);
304
305 /*
306 * P9 suppresses the HDEC exception when LPCR[HDICE] = 0,
307 * so set guest LPCR (with HDICE) before writing HDEC.
308 */
309 mtspr(SPRN_HDEC, hdec);
310
311 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
312 tm_return_to_guest:
313 #endif
314 mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
315 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
316 mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0);
317 mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1);
318
319 accumulate_time(vcpu, &vcpu->arch.guest_time);
320
321 kvmppc_p9_enter_guest(vcpu);
322
323 accumulate_time(vcpu, &vcpu->arch.rm_intr);
324
325 /* XXX: Could get these from r11/12 and paca exsave instead */
326 vcpu->arch.shregs.srr0 = mfspr(SPRN_SRR0);
327 vcpu->arch.shregs.srr1 = mfspr(SPRN_SRR1);
328 vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
329 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
330
331 /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */
332 trap = local_paca->kvm_hstate.scratch0 & ~0x2;
333
334 /* HSRR interrupts leave MSR[RI] unchanged, SRR interrupts clear it. */
335 ri_set = false;
336 if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK)) {
337 if (trap != BOOK3S_INTERRUPT_SYSCALL &&
338 (vcpu->arch.shregs.msr & MSR_RI))
339 ri_set = true;
340 exsave = local_paca->exgen;
341 } else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET) {
342 exsave = local_paca->exnmi;
343 } else { /* trap == 0x200 */
344 exsave = local_paca->exmc;
345 }
346
347 vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1;
348 vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2;
349
350 /*
351 * Only set RI after reading machine check regs (DAR, DSISR, SRR0/1)
352 * and hstate scratch (which we need to move into exsave to make
353 * re-entrant vs SRESET/MCE)
354 */
355 if (ri_set) {
356 if (unlikely(!(mfmsr() & MSR_RI))) {
357 __mtmsrd(MSR_RI, 1);
358 WARN_ON_ONCE(1);
359 }
360 } else {
361 WARN_ON_ONCE(mfmsr() & MSR_RI);
362 __mtmsrd(MSR_RI, 1);
363 }
364
365 vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)];
366 vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)];
367 vcpu->arch.regs.gpr[11] = exsave[EX_R11/sizeof(u64)];
368 vcpu->arch.regs.gpr[12] = exsave[EX_R12/sizeof(u64)];
369 vcpu->arch.regs.gpr[13] = exsave[EX_R13/sizeof(u64)];
370 vcpu->arch.ppr = exsave[EX_PPR/sizeof(u64)];
371 vcpu->arch.cfar = exsave[EX_CFAR/sizeof(u64)];
372 vcpu->arch.regs.ctr = exsave[EX_CTR/sizeof(u64)];
373
374 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
375
376 if (unlikely(trap == BOOK3S_INTERRUPT_MACHINE_CHECK)) {
377 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)];
378 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)];
379 kvmppc_realmode_machine_check(vcpu);
380
381 } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) {
382 kvmppc_realmode_hmi_handler();
383
384 } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) {
385 vcpu->arch.emul_inst = mfspr(SPRN_HEIR);
386
387 } else if (trap == BOOK3S_INTERRUPT_H_DATA_STORAGE) {
388 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)];
389 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)];
390 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR);
391
392 } else if (trap == BOOK3S_INTERRUPT_H_INST_STORAGE) {
393 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR);
394
395 } else if (trap == BOOK3S_INTERRUPT_H_FAC_UNAVAIL) {
396 vcpu->arch.hfscr = mfspr(SPRN_HFSCR);
397
398 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
399 /*
400 * Softpatch interrupt for transactional memory emulation cases
401 * on POWER9 DD2.2. This is early in the guest exit path - we
402 * haven't saved registers or done a treclaim yet.
403 */
404 } else if (trap == BOOK3S_INTERRUPT_HV_SOFTPATCH) {
405 vcpu->arch.emul_inst = mfspr(SPRN_HEIR);
406
407 /*
408 * The cases we want to handle here are those where the guest
409 * is in real suspend mode and is trying to transition to
410 * transactional mode.
411 */
412 if (!local_paca->kvm_hstate.fake_suspend &&
413 (vcpu->arch.shregs.msr & MSR_TS_S)) {
414 if (kvmhv_p9_tm_emulation_early(vcpu)) {
415 /*
416 * Go straight back into the guest with the
417 * new NIP/MSR as set by TM emulation.
418 */
419 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip);
420 mtspr(SPRN_HSRR1, vcpu->arch.shregs.msr);
421
422 /*
423 * tm_return_to_guest re-loads SRR0/1, DAR,
424 * DSISR after RI is cleared, in case they had
425 * been clobbered by a MCE.
426 */
427 __mtmsrd(0, 1); /* clear RI */
428 goto tm_return_to_guest;
429 }
430 }
431 #endif
432 }
433
434 accumulate_time(vcpu, &vcpu->arch.rm_exit);
435
436 /* Advance host PURR/SPURR by the amount used by guest */
437 purr = mfspr(SPRN_PURR);
438 spurr = mfspr(SPRN_SPURR);
439 mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr +
440 purr - vcpu->arch.purr);
441 mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr +
442 spurr - vcpu->arch.spurr);
443 vcpu->arch.purr = purr;
444 vcpu->arch.spurr = spurr;
445
446 vcpu->arch.ic = mfspr(SPRN_IC);
447 vcpu->arch.pid = mfspr(SPRN_PID);
448 vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS;
449
450 vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0);
451 vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1);
452 vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2);
453 vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3);
454
455 /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */
456 mtspr(SPRN_PSSCR, host_psscr |
457 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
458 mtspr(SPRN_HFSCR, host_hfscr);
459 mtspr(SPRN_CIABR, host_ciabr);
460 mtspr(SPRN_DAWR0, host_dawr0);
461 mtspr(SPRN_DAWRX0, host_dawrx0);
462 if (cpu_has_feature(CPU_FTR_DAWR1)) {
463 mtspr(SPRN_DAWR1, host_dawr1);
464 mtspr(SPRN_DAWRX1, host_dawrx1);
465 }
466
467 if (kvm_is_radix(kvm)) {
468 /*
469 * Since this is radix, do a eieio; tlbsync; ptesync sequence
470 * in case we interrupted the guest between a tlbie and a
471 * ptesync.
472 */
473 asm volatile("eieio; tlbsync; ptesync");
474 }
475
476 /*
477 * cp_abort is required if the processor supports local copy-paste
478 * to clear the copy buffer that was under control of the guest.
479 */
480 if (cpu_has_feature(CPU_FTR_ARCH_31))
481 asm volatile(PPC_CP_ABORT);
482
483 vc->dpdes = mfspr(SPRN_DPDES);
484 vc->vtb = mfspr(SPRN_VTB);
485 mtspr(SPRN_DPDES, 0);
486 if (vc->pcr)
487 mtspr(SPRN_PCR, PCR_MASK);
488
489 if (vc->tb_offset_applied) {
490 u64 new_tb = mftb() - vc->tb_offset_applied;
491 mtspr(SPRN_TBU40, new_tb);
492 tb = mftb();
493 if ((tb & 0xffffff) < (new_tb & 0xffffff))
494 mtspr(SPRN_TBU40, new_tb + 0x1000000);
495 vc->tb_offset_applied = 0;
496 }
497
498 mtspr(SPRN_HDEC, 0x7fffffff);
499
500 save_clear_guest_mmu(kvm, vcpu);
501 switch_mmu_to_host(kvm, host_pidr);
502 local_paca->kvm_hstate.in_guest = KVM_GUEST_MODE_NONE;
503
504 /*
505 * If we are in real mode, only switch MMU on after the MMU is
506 * switched to host, to avoid the P9_RADIX_PREFETCH_BUG.
507 */
508 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
509 vcpu->arch.shregs.msr & MSR_TS_MASK)
510 msr |= MSR_TS_S;
511
512 __mtmsrd(msr, 0);
513
514 end_timing(vcpu);
515
516 return trap;
517 }
518 EXPORT_SYMBOL_GPL(kvmhv_vcpu_entry_p9);
519