1 /*
2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
4 *
5 * Authors:
6 * Paul Mackerras <paulus@au1.ibm.com>
7 * Alexander Graf <agraf@suse.de>
8 * Kevin Wolf <mail@kevin-wolf.de>
9 *
10 * Description: KVM functions specific to running on Book 3S
11 * processors in hypervisor mode (specifically POWER7 and later).
12 *
13 * This file is derived from arch/powerpc/kvm/book3s.c,
14 * by Alexander Graf <agraf@suse.de>.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License, version 2, as
18 * published by the Free Software Foundation.
19 */
20
21 #include <linux/kvm_host.h>
22 #include <linux/err.h>
23 #include <linux/slab.h>
24 #include <linux/preempt.h>
25 #include <linux/sched.h>
26 #include <linux/delay.h>
27 #include <linux/export.h>
28 #include <linux/fs.h>
29 #include <linux/anon_inodes.h>
30 #include <linux/cpumask.h>
31 #include <linux/spinlock.h>
32 #include <linux/page-flags.h>
33 #include <linux/srcu.h>
34 #include <linux/miscdevice.h>
35
36 #include <asm/reg.h>
37 #include <asm/cputable.h>
38 #include <asm/cache.h>
39 #include <asm/cacheflush.h>
40 #include <asm/tlbflush.h>
41 #include <asm/uaccess.h>
42 #include <asm/io.h>
43 #include <asm/kvm_ppc.h>
44 #include <asm/kvm_book3s.h>
45 #include <asm/mmu_context.h>
46 #include <asm/lppaca.h>
47 #include <asm/processor.h>
48 #include <asm/cputhreads.h>
49 #include <asm/page.h>
50 #include <asm/hvcall.h>
51 #include <asm/switch_to.h>
52 #include <asm/smp.h>
53 #include <linux/gfp.h>
54 #include <linux/vmalloc.h>
55 #include <linux/highmem.h>
56 #include <linux/hugetlb.h>
57 #include <linux/module.h>
58
59 #include "book3s.h"
60
61 /* #define EXIT_DEBUG */
62 /* #define EXIT_DEBUG_SIMPLE */
63 /* #define EXIT_DEBUG_INT */
64
65 /* Used to indicate that a guest page fault needs to be handled */
66 #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1)
67
68 /* Used as a "null" value for timebase values */
69 #define TB_NIL (~(u64)0)
70
71 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
72
73 #if defined(CONFIG_PPC_64K_PAGES)
74 #define MPP_BUFFER_ORDER 0
75 #elif defined(CONFIG_PPC_4K_PAGES)
76 #define MPP_BUFFER_ORDER 3
77 #endif
78
79
80 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
81 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
82
kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu * vcpu)83 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
84 {
85 int me;
86 int cpu = vcpu->cpu;
87 wait_queue_head_t *wqp;
88
89 wqp = kvm_arch_vcpu_wq(vcpu);
90 if (waitqueue_active(wqp)) {
91 wake_up_interruptible(wqp);
92 ++vcpu->stat.halt_wakeup;
93 }
94
95 me = get_cpu();
96
97 /* CPU points to the first thread of the core */
98 if (cpu != me && cpu >= 0 && cpu < nr_cpu_ids) {
99 #ifdef CONFIG_PPC_ICP_NATIVE
100 int real_cpu = cpu + vcpu->arch.ptid;
101 if (paca[real_cpu].kvm_hstate.xics_phys)
102 xics_wake_cpu(real_cpu);
103 else
104 #endif
105 if (cpu_online(cpu))
106 smp_send_reschedule(cpu);
107 }
108 put_cpu();
109 }
110
111 /*
112 * We use the vcpu_load/put functions to measure stolen time.
113 * Stolen time is counted as time when either the vcpu is able to
114 * run as part of a virtual core, but the task running the vcore
115 * is preempted or sleeping, or when the vcpu needs something done
116 * in the kernel by the task running the vcpu, but that task is
117 * preempted or sleeping. Those two things have to be counted
118 * separately, since one of the vcpu tasks will take on the job
119 * of running the core, and the other vcpu tasks in the vcore will
120 * sleep waiting for it to do that, but that sleep shouldn't count
121 * as stolen time.
122 *
123 * Hence we accumulate stolen time when the vcpu can run as part of
124 * a vcore using vc->stolen_tb, and the stolen time when the vcpu
125 * needs its task to do other things in the kernel (for example,
126 * service a page fault) in busy_stolen. We don't accumulate
127 * stolen time for a vcore when it is inactive, or for a vcpu
128 * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of
129 * a misnomer; it means that the vcpu task is not executing in
130 * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
131 * the kernel. We don't have any way of dividing up that time
132 * between time that the vcpu is genuinely stopped, time that
133 * the task is actively working on behalf of the vcpu, and time
134 * that the task is preempted, so we don't count any of it as
135 * stolen.
136 *
137 * Updates to busy_stolen are protected by arch.tbacct_lock;
138 * updates to vc->stolen_tb are protected by the arch.tbacct_lock
139 * of the vcpu that has taken responsibility for running the vcore
140 * (i.e. vc->runner). The stolen times are measured in units of
141 * timebase ticks. (Note that the != TB_NIL checks below are
142 * purely defensive; they should never fail.)
143 */
144
kvmppc_core_vcpu_load_hv(struct kvm_vcpu * vcpu,int cpu)145 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
146 {
147 struct kvmppc_vcore *vc = vcpu->arch.vcore;
148 unsigned long flags;
149
150 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
151 if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE &&
152 vc->preempt_tb != TB_NIL) {
153 vc->stolen_tb += mftb() - vc->preempt_tb;
154 vc->preempt_tb = TB_NIL;
155 }
156 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
157 vcpu->arch.busy_preempt != TB_NIL) {
158 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
159 vcpu->arch.busy_preempt = TB_NIL;
160 }
161 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
162 }
163
kvmppc_core_vcpu_put_hv(struct kvm_vcpu * vcpu)164 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
165 {
166 struct kvmppc_vcore *vc = vcpu->arch.vcore;
167 unsigned long flags;
168
169 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
170 if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE)
171 vc->preempt_tb = mftb();
172 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
173 vcpu->arch.busy_preempt = mftb();
174 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
175 }
176
kvmppc_set_msr_hv(struct kvm_vcpu * vcpu,u64 msr)177 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
178 {
179 vcpu->arch.shregs.msr = msr;
180 kvmppc_end_cede(vcpu);
181 }
182
kvmppc_set_pvr_hv(struct kvm_vcpu * vcpu,u32 pvr)183 void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
184 {
185 vcpu->arch.pvr = pvr;
186 }
187
kvmppc_set_arch_compat(struct kvm_vcpu * vcpu,u32 arch_compat)188 int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
189 {
190 unsigned long pcr = 0;
191 struct kvmppc_vcore *vc = vcpu->arch.vcore;
192
193 if (arch_compat) {
194 if (!cpu_has_feature(CPU_FTR_ARCH_206))
195 return -EINVAL; /* 970 has no compat mode support */
196
197 switch (arch_compat) {
198 case PVR_ARCH_205:
199 /*
200 * If an arch bit is set in PCR, all the defined
201 * higher-order arch bits also have to be set.
202 */
203 pcr = PCR_ARCH_206 | PCR_ARCH_205;
204 break;
205 case PVR_ARCH_206:
206 case PVR_ARCH_206p:
207 pcr = PCR_ARCH_206;
208 break;
209 case PVR_ARCH_207:
210 break;
211 default:
212 return -EINVAL;
213 }
214
215 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) {
216 /* POWER7 can't emulate POWER8 */
217 if (!(pcr & PCR_ARCH_206))
218 return -EINVAL;
219 pcr &= ~PCR_ARCH_206;
220 }
221 }
222
223 spin_lock(&vc->lock);
224 vc->arch_compat = arch_compat;
225 vc->pcr = pcr;
226 spin_unlock(&vc->lock);
227
228 return 0;
229 }
230
kvmppc_dump_regs(struct kvm_vcpu * vcpu)231 void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
232 {
233 int r;
234
235 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
236 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
237 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
238 for (r = 0; r < 16; ++r)
239 pr_err("r%2d = %.16lx r%d = %.16lx\n",
240 r, kvmppc_get_gpr(vcpu, r),
241 r+16, kvmppc_get_gpr(vcpu, r+16));
242 pr_err("ctr = %.16lx lr = %.16lx\n",
243 vcpu->arch.ctr, vcpu->arch.lr);
244 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
245 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
246 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
247 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
248 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
249 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
250 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n",
251 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
252 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
253 pr_err("fault dar = %.16lx dsisr = %.8x\n",
254 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
255 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
256 for (r = 0; r < vcpu->arch.slb_max; ++r)
257 pr_err(" ESID = %.16llx VSID = %.16llx\n",
258 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
259 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
260 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
261 vcpu->arch.last_inst);
262 }
263
kvmppc_find_vcpu(struct kvm * kvm,int id)264 struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
265 {
266 int r;
267 struct kvm_vcpu *v, *ret = NULL;
268
269 mutex_lock(&kvm->lock);
270 kvm_for_each_vcpu(r, v, kvm) {
271 if (v->vcpu_id == id) {
272 ret = v;
273 break;
274 }
275 }
276 mutex_unlock(&kvm->lock);
277 return ret;
278 }
279
init_vpa(struct kvm_vcpu * vcpu,struct lppaca * vpa)280 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
281 {
282 vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
283 vpa->yield_count = cpu_to_be32(1);
284 }
285
set_vpa(struct kvm_vcpu * vcpu,struct kvmppc_vpa * v,unsigned long addr,unsigned long len)286 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
287 unsigned long addr, unsigned long len)
288 {
289 /* check address is cacheline aligned */
290 if (addr & (L1_CACHE_BYTES - 1))
291 return -EINVAL;
292 spin_lock(&vcpu->arch.vpa_update_lock);
293 if (v->next_gpa != addr || v->len != len) {
294 v->next_gpa = addr;
295 v->len = addr ? len : 0;
296 v->update_pending = 1;
297 }
298 spin_unlock(&vcpu->arch.vpa_update_lock);
299 return 0;
300 }
301
302 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
303 struct reg_vpa {
304 u32 dummy;
305 union {
306 __be16 hword;
307 __be32 word;
308 } length;
309 };
310
vpa_is_registered(struct kvmppc_vpa * vpap)311 static int vpa_is_registered(struct kvmppc_vpa *vpap)
312 {
313 if (vpap->update_pending)
314 return vpap->next_gpa != 0;
315 return vpap->pinned_addr != NULL;
316 }
317
do_h_register_vpa(struct kvm_vcpu * vcpu,unsigned long flags,unsigned long vcpuid,unsigned long vpa)318 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
319 unsigned long flags,
320 unsigned long vcpuid, unsigned long vpa)
321 {
322 struct kvm *kvm = vcpu->kvm;
323 unsigned long len, nb;
324 void *va;
325 struct kvm_vcpu *tvcpu;
326 int err;
327 int subfunc;
328 struct kvmppc_vpa *vpap;
329
330 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
331 if (!tvcpu)
332 return H_PARAMETER;
333
334 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
335 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
336 subfunc == H_VPA_REG_SLB) {
337 /* Registering new area - address must be cache-line aligned */
338 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
339 return H_PARAMETER;
340
341 /* convert logical addr to kernel addr and read length */
342 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
343 if (va == NULL)
344 return H_PARAMETER;
345 if (subfunc == H_VPA_REG_VPA)
346 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
347 else
348 len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
349 kvmppc_unpin_guest_page(kvm, va, vpa, false);
350
351 /* Check length */
352 if (len > nb || len < sizeof(struct reg_vpa))
353 return H_PARAMETER;
354 } else {
355 vpa = 0;
356 len = 0;
357 }
358
359 err = H_PARAMETER;
360 vpap = NULL;
361 spin_lock(&tvcpu->arch.vpa_update_lock);
362
363 switch (subfunc) {
364 case H_VPA_REG_VPA: /* register VPA */
365 if (len < sizeof(struct lppaca))
366 break;
367 vpap = &tvcpu->arch.vpa;
368 err = 0;
369 break;
370
371 case H_VPA_REG_DTL: /* register DTL */
372 if (len < sizeof(struct dtl_entry))
373 break;
374 len -= len % sizeof(struct dtl_entry);
375
376 /* Check that they have previously registered a VPA */
377 err = H_RESOURCE;
378 if (!vpa_is_registered(&tvcpu->arch.vpa))
379 break;
380
381 vpap = &tvcpu->arch.dtl;
382 err = 0;
383 break;
384
385 case H_VPA_REG_SLB: /* register SLB shadow buffer */
386 /* Check that they have previously registered a VPA */
387 err = H_RESOURCE;
388 if (!vpa_is_registered(&tvcpu->arch.vpa))
389 break;
390
391 vpap = &tvcpu->arch.slb_shadow;
392 err = 0;
393 break;
394
395 case H_VPA_DEREG_VPA: /* deregister VPA */
396 /* Check they don't still have a DTL or SLB buf registered */
397 err = H_RESOURCE;
398 if (vpa_is_registered(&tvcpu->arch.dtl) ||
399 vpa_is_registered(&tvcpu->arch.slb_shadow))
400 break;
401
402 vpap = &tvcpu->arch.vpa;
403 err = 0;
404 break;
405
406 case H_VPA_DEREG_DTL: /* deregister DTL */
407 vpap = &tvcpu->arch.dtl;
408 err = 0;
409 break;
410
411 case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */
412 vpap = &tvcpu->arch.slb_shadow;
413 err = 0;
414 break;
415 }
416
417 if (vpap) {
418 vpap->next_gpa = vpa;
419 vpap->len = len;
420 vpap->update_pending = 1;
421 }
422
423 spin_unlock(&tvcpu->arch.vpa_update_lock);
424
425 return err;
426 }
427
kvmppc_update_vpa(struct kvm_vcpu * vcpu,struct kvmppc_vpa * vpap)428 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
429 {
430 struct kvm *kvm = vcpu->kvm;
431 void *va;
432 unsigned long nb;
433 unsigned long gpa;
434
435 /*
436 * We need to pin the page pointed to by vpap->next_gpa,
437 * but we can't call kvmppc_pin_guest_page under the lock
438 * as it does get_user_pages() and down_read(). So we
439 * have to drop the lock, pin the page, then get the lock
440 * again and check that a new area didn't get registered
441 * in the meantime.
442 */
443 for (;;) {
444 gpa = vpap->next_gpa;
445 spin_unlock(&vcpu->arch.vpa_update_lock);
446 va = NULL;
447 nb = 0;
448 if (gpa)
449 va = kvmppc_pin_guest_page(kvm, gpa, &nb);
450 spin_lock(&vcpu->arch.vpa_update_lock);
451 if (gpa == vpap->next_gpa)
452 break;
453 /* sigh... unpin that one and try again */
454 if (va)
455 kvmppc_unpin_guest_page(kvm, va, gpa, false);
456 }
457
458 vpap->update_pending = 0;
459 if (va && nb < vpap->len) {
460 /*
461 * If it's now too short, it must be that userspace
462 * has changed the mappings underlying guest memory,
463 * so unregister the region.
464 */
465 kvmppc_unpin_guest_page(kvm, va, gpa, false);
466 va = NULL;
467 }
468 if (vpap->pinned_addr)
469 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
470 vpap->dirty);
471 vpap->gpa = gpa;
472 vpap->pinned_addr = va;
473 vpap->dirty = false;
474 if (va)
475 vpap->pinned_end = va + vpap->len;
476 }
477
kvmppc_update_vpas(struct kvm_vcpu * vcpu)478 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
479 {
480 if (!(vcpu->arch.vpa.update_pending ||
481 vcpu->arch.slb_shadow.update_pending ||
482 vcpu->arch.dtl.update_pending))
483 return;
484
485 spin_lock(&vcpu->arch.vpa_update_lock);
486 if (vcpu->arch.vpa.update_pending) {
487 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
488 if (vcpu->arch.vpa.pinned_addr)
489 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
490 }
491 if (vcpu->arch.dtl.update_pending) {
492 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
493 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
494 vcpu->arch.dtl_index = 0;
495 }
496 if (vcpu->arch.slb_shadow.update_pending)
497 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
498 spin_unlock(&vcpu->arch.vpa_update_lock);
499 }
500
501 /*
502 * Return the accumulated stolen time for the vcore up until `now'.
503 * The caller should hold the vcore lock.
504 */
vcore_stolen_time(struct kvmppc_vcore * vc,u64 now)505 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
506 {
507 u64 p;
508
509 /*
510 * If we are the task running the vcore, then since we hold
511 * the vcore lock, we can't be preempted, so stolen_tb/preempt_tb
512 * can't be updated, so we don't need the tbacct_lock.
513 * If the vcore is inactive, it can't become active (since we
514 * hold the vcore lock), so the vcpu load/put functions won't
515 * update stolen_tb/preempt_tb, and we don't need tbacct_lock.
516 */
517 if (vc->vcore_state != VCORE_INACTIVE &&
518 vc->runner->arch.run_task != current) {
519 spin_lock_irq(&vc->runner->arch.tbacct_lock);
520 p = vc->stolen_tb;
521 if (vc->preempt_tb != TB_NIL)
522 p += now - vc->preempt_tb;
523 spin_unlock_irq(&vc->runner->arch.tbacct_lock);
524 } else {
525 p = vc->stolen_tb;
526 }
527 return p;
528 }
529
kvmppc_create_dtl_entry(struct kvm_vcpu * vcpu,struct kvmppc_vcore * vc)530 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
531 struct kvmppc_vcore *vc)
532 {
533 struct dtl_entry *dt;
534 struct lppaca *vpa;
535 unsigned long stolen;
536 unsigned long core_stolen;
537 u64 now;
538
539 dt = vcpu->arch.dtl_ptr;
540 vpa = vcpu->arch.vpa.pinned_addr;
541 now = mftb();
542 core_stolen = vcore_stolen_time(vc, now);
543 stolen = core_stolen - vcpu->arch.stolen_logged;
544 vcpu->arch.stolen_logged = core_stolen;
545 spin_lock_irq(&vcpu->arch.tbacct_lock);
546 stolen += vcpu->arch.busy_stolen;
547 vcpu->arch.busy_stolen = 0;
548 spin_unlock_irq(&vcpu->arch.tbacct_lock);
549 if (!dt || !vpa)
550 return;
551 memset(dt, 0, sizeof(struct dtl_entry));
552 dt->dispatch_reason = 7;
553 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
554 dt->timebase = cpu_to_be64(now + vc->tb_offset);
555 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
556 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
557 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
558 ++dt;
559 if (dt == vcpu->arch.dtl.pinned_end)
560 dt = vcpu->arch.dtl.pinned_addr;
561 vcpu->arch.dtl_ptr = dt;
562 /* order writing *dt vs. writing vpa->dtl_idx */
563 smp_wmb();
564 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
565 vcpu->arch.dtl.dirty = true;
566 }
567
kvmppc_power8_compatible(struct kvm_vcpu * vcpu)568 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
569 {
570 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
571 return true;
572 if ((!vcpu->arch.vcore->arch_compat) &&
573 cpu_has_feature(CPU_FTR_ARCH_207S))
574 return true;
575 return false;
576 }
577
kvmppc_h_set_mode(struct kvm_vcpu * vcpu,unsigned long mflags,unsigned long resource,unsigned long value1,unsigned long value2)578 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
579 unsigned long resource, unsigned long value1,
580 unsigned long value2)
581 {
582 switch (resource) {
583 case H_SET_MODE_RESOURCE_SET_CIABR:
584 if (!kvmppc_power8_compatible(vcpu))
585 return H_P2;
586 if (value2)
587 return H_P4;
588 if (mflags)
589 return H_UNSUPPORTED_FLAG_START;
590 /* Guests can't breakpoint the hypervisor */
591 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
592 return H_P3;
593 vcpu->arch.ciabr = value1;
594 return H_SUCCESS;
595 case H_SET_MODE_RESOURCE_SET_DAWR:
596 if (!kvmppc_power8_compatible(vcpu))
597 return H_P2;
598 if (mflags)
599 return H_UNSUPPORTED_FLAG_START;
600 if (value2 & DABRX_HYP)
601 return H_P4;
602 vcpu->arch.dawr = value1;
603 vcpu->arch.dawrx = value2;
604 return H_SUCCESS;
605 default:
606 return H_TOO_HARD;
607 }
608 }
609
kvmppc_pseries_do_hcall(struct kvm_vcpu * vcpu)610 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
611 {
612 unsigned long req = kvmppc_get_gpr(vcpu, 3);
613 unsigned long target, ret = H_SUCCESS;
614 struct kvm_vcpu *tvcpu;
615 int idx, rc;
616
617 if (req <= MAX_HCALL_OPCODE &&
618 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
619 return RESUME_HOST;
620
621 switch (req) {
622 case H_ENTER:
623 idx = srcu_read_lock(&vcpu->kvm->srcu);
624 ret = kvmppc_virtmode_h_enter(vcpu, kvmppc_get_gpr(vcpu, 4),
625 kvmppc_get_gpr(vcpu, 5),
626 kvmppc_get_gpr(vcpu, 6),
627 kvmppc_get_gpr(vcpu, 7));
628 srcu_read_unlock(&vcpu->kvm->srcu, idx);
629 break;
630 case H_CEDE:
631 break;
632 case H_PROD:
633 target = kvmppc_get_gpr(vcpu, 4);
634 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
635 if (!tvcpu) {
636 ret = H_PARAMETER;
637 break;
638 }
639 tvcpu->arch.prodded = 1;
640 smp_mb();
641 if (vcpu->arch.ceded) {
642 if (waitqueue_active(&vcpu->wq)) {
643 wake_up_interruptible(&vcpu->wq);
644 vcpu->stat.halt_wakeup++;
645 }
646 }
647 break;
648 case H_CONFER:
649 target = kvmppc_get_gpr(vcpu, 4);
650 if (target == -1)
651 break;
652 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
653 if (!tvcpu) {
654 ret = H_PARAMETER;
655 break;
656 }
657 kvm_vcpu_yield_to(tvcpu);
658 break;
659 case H_REGISTER_VPA:
660 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
661 kvmppc_get_gpr(vcpu, 5),
662 kvmppc_get_gpr(vcpu, 6));
663 break;
664 case H_RTAS:
665 if (list_empty(&vcpu->kvm->arch.rtas_tokens))
666 return RESUME_HOST;
667
668 idx = srcu_read_lock(&vcpu->kvm->srcu);
669 rc = kvmppc_rtas_hcall(vcpu);
670 srcu_read_unlock(&vcpu->kvm->srcu, idx);
671
672 if (rc == -ENOENT)
673 return RESUME_HOST;
674 else if (rc == 0)
675 break;
676
677 /* Send the error out to userspace via KVM_RUN */
678 return rc;
679 case H_SET_MODE:
680 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
681 kvmppc_get_gpr(vcpu, 5),
682 kvmppc_get_gpr(vcpu, 6),
683 kvmppc_get_gpr(vcpu, 7));
684 if (ret == H_TOO_HARD)
685 return RESUME_HOST;
686 break;
687 case H_XIRR:
688 case H_CPPR:
689 case H_EOI:
690 case H_IPI:
691 case H_IPOLL:
692 case H_XIRR_X:
693 if (kvmppc_xics_enabled(vcpu)) {
694 ret = kvmppc_xics_hcall(vcpu, req);
695 break;
696 } /* fallthrough */
697 default:
698 return RESUME_HOST;
699 }
700 kvmppc_set_gpr(vcpu, 3, ret);
701 vcpu->arch.hcall_needed = 0;
702 return RESUME_GUEST;
703 }
704
kvmppc_hcall_impl_hv(unsigned long cmd)705 static int kvmppc_hcall_impl_hv(unsigned long cmd)
706 {
707 switch (cmd) {
708 case H_CEDE:
709 case H_PROD:
710 case H_CONFER:
711 case H_REGISTER_VPA:
712 case H_SET_MODE:
713 #ifdef CONFIG_KVM_XICS
714 case H_XIRR:
715 case H_CPPR:
716 case H_EOI:
717 case H_IPI:
718 case H_IPOLL:
719 case H_XIRR_X:
720 #endif
721 return 1;
722 }
723
724 /* See if it's in the real-mode table */
725 return kvmppc_hcall_impl_hv_realmode(cmd);
726 }
727
kvmppc_emulate_debug_inst(struct kvm_run * run,struct kvm_vcpu * vcpu)728 static int kvmppc_emulate_debug_inst(struct kvm_run *run,
729 struct kvm_vcpu *vcpu)
730 {
731 u32 last_inst;
732
733 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
734 EMULATE_DONE) {
735 /*
736 * Fetch failed, so return to guest and
737 * try executing it again.
738 */
739 return RESUME_GUEST;
740 }
741
742 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
743 run->exit_reason = KVM_EXIT_DEBUG;
744 run->debug.arch.address = kvmppc_get_pc(vcpu);
745 return RESUME_HOST;
746 } else {
747 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
748 return RESUME_GUEST;
749 }
750 }
751
kvmppc_handle_exit_hv(struct kvm_run * run,struct kvm_vcpu * vcpu,struct task_struct * tsk)752 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
753 struct task_struct *tsk)
754 {
755 int r = RESUME_HOST;
756
757 vcpu->stat.sum_exits++;
758
759 run->exit_reason = KVM_EXIT_UNKNOWN;
760 run->ready_for_interrupt_injection = 1;
761 switch (vcpu->arch.trap) {
762 /* We're good on these - the host merely wanted to get our attention */
763 case BOOK3S_INTERRUPT_HV_DECREMENTER:
764 vcpu->stat.dec_exits++;
765 r = RESUME_GUEST;
766 break;
767 case BOOK3S_INTERRUPT_EXTERNAL:
768 case BOOK3S_INTERRUPT_H_DOORBELL:
769 vcpu->stat.ext_intr_exits++;
770 r = RESUME_GUEST;
771 break;
772 case BOOK3S_INTERRUPT_PERFMON:
773 r = RESUME_GUEST;
774 break;
775 case BOOK3S_INTERRUPT_MACHINE_CHECK:
776 /*
777 * Deliver a machine check interrupt to the guest.
778 * We have to do this, even if the host has handled the
779 * machine check, because machine checks use SRR0/1 and
780 * the interrupt might have trashed guest state in them.
781 */
782 kvmppc_book3s_queue_irqprio(vcpu,
783 BOOK3S_INTERRUPT_MACHINE_CHECK);
784 r = RESUME_GUEST;
785 break;
786 case BOOK3S_INTERRUPT_PROGRAM:
787 {
788 ulong flags;
789 /*
790 * Normally program interrupts are delivered directly
791 * to the guest by the hardware, but we can get here
792 * as a result of a hypervisor emulation interrupt
793 * (e40) getting turned into a 700 by BML RTAS.
794 */
795 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
796 kvmppc_core_queue_program(vcpu, flags);
797 r = RESUME_GUEST;
798 break;
799 }
800 case BOOK3S_INTERRUPT_SYSCALL:
801 {
802 /* hcall - punt to userspace */
803 int i;
804
805 /* hypercall with MSR_PR has already been handled in rmode,
806 * and never reaches here.
807 */
808
809 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
810 for (i = 0; i < 9; ++i)
811 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
812 run->exit_reason = KVM_EXIT_PAPR_HCALL;
813 vcpu->arch.hcall_needed = 1;
814 r = RESUME_HOST;
815 break;
816 }
817 /*
818 * We get these next two if the guest accesses a page which it thinks
819 * it has mapped but which is not actually present, either because
820 * it is for an emulated I/O device or because the corresonding
821 * host page has been paged out. Any other HDSI/HISI interrupts
822 * have been handled already.
823 */
824 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
825 r = RESUME_PAGE_FAULT;
826 break;
827 case BOOK3S_INTERRUPT_H_INST_STORAGE:
828 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
829 vcpu->arch.fault_dsisr = 0;
830 r = RESUME_PAGE_FAULT;
831 break;
832 /*
833 * This occurs if the guest executes an illegal instruction.
834 * If the guest debug is disabled, generate a program interrupt
835 * to the guest. If guest debug is enabled, we need to check
836 * whether the instruction is a software breakpoint instruction.
837 * Accordingly return to Guest or Host.
838 */
839 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
840 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
841 r = kvmppc_emulate_debug_inst(run, vcpu);
842 } else {
843 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
844 r = RESUME_GUEST;
845 }
846 break;
847 /*
848 * This occurs if the guest (kernel or userspace), does something that
849 * is prohibited by HFSCR. We just generate a program interrupt to
850 * the guest.
851 */
852 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
853 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
854 r = RESUME_GUEST;
855 break;
856 default:
857 kvmppc_dump_regs(vcpu);
858 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
859 vcpu->arch.trap, kvmppc_get_pc(vcpu),
860 vcpu->arch.shregs.msr);
861 run->hw.hardware_exit_reason = vcpu->arch.trap;
862 r = RESUME_HOST;
863 break;
864 }
865
866 return r;
867 }
868
kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)869 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
870 struct kvm_sregs *sregs)
871 {
872 int i;
873
874 memset(sregs, 0, sizeof(struct kvm_sregs));
875 sregs->pvr = vcpu->arch.pvr;
876 for (i = 0; i < vcpu->arch.slb_max; i++) {
877 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
878 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
879 }
880
881 return 0;
882 }
883
kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)884 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
885 struct kvm_sregs *sregs)
886 {
887 int i, j;
888
889 /* Only accept the same PVR as the host's, since we can't spoof it */
890 if (sregs->pvr != vcpu->arch.pvr)
891 return -EINVAL;
892
893 j = 0;
894 for (i = 0; i < vcpu->arch.slb_nr; i++) {
895 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
896 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
897 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
898 ++j;
899 }
900 }
901 vcpu->arch.slb_max = j;
902
903 return 0;
904 }
905
kvmppc_set_lpcr(struct kvm_vcpu * vcpu,u64 new_lpcr,bool preserve_top32)906 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
907 bool preserve_top32)
908 {
909 struct kvmppc_vcore *vc = vcpu->arch.vcore;
910 u64 mask;
911
912 spin_lock(&vc->lock);
913 /*
914 * If ILE (interrupt little-endian) has changed, update the
915 * MSR_LE bit in the intr_msr for each vcpu in this vcore.
916 */
917 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
918 struct kvm *kvm = vcpu->kvm;
919 struct kvm_vcpu *vcpu;
920 int i;
921
922 mutex_lock(&kvm->lock);
923 kvm_for_each_vcpu(i, vcpu, kvm) {
924 if (vcpu->arch.vcore != vc)
925 continue;
926 if (new_lpcr & LPCR_ILE)
927 vcpu->arch.intr_msr |= MSR_LE;
928 else
929 vcpu->arch.intr_msr &= ~MSR_LE;
930 }
931 mutex_unlock(&kvm->lock);
932 }
933
934 /*
935 * Userspace can only modify DPFD (default prefetch depth),
936 * ILE (interrupt little-endian) and TC (translation control).
937 * On POWER8 userspace can also modify AIL (alt. interrupt loc.)
938 */
939 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
940 if (cpu_has_feature(CPU_FTR_ARCH_207S))
941 mask |= LPCR_AIL;
942
943 /* Broken 32-bit version of LPCR must not clear top bits */
944 if (preserve_top32)
945 mask &= 0xFFFFFFFF;
946 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
947 spin_unlock(&vc->lock);
948 }
949
kvmppc_get_one_reg_hv(struct kvm_vcpu * vcpu,u64 id,union kvmppc_one_reg * val)950 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
951 union kvmppc_one_reg *val)
952 {
953 int r = 0;
954 long int i;
955
956 switch (id) {
957 case KVM_REG_PPC_DEBUG_INST:
958 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
959 break;
960 case KVM_REG_PPC_HIOR:
961 *val = get_reg_val(id, 0);
962 break;
963 case KVM_REG_PPC_DABR:
964 *val = get_reg_val(id, vcpu->arch.dabr);
965 break;
966 case KVM_REG_PPC_DABRX:
967 *val = get_reg_val(id, vcpu->arch.dabrx);
968 break;
969 case KVM_REG_PPC_DSCR:
970 *val = get_reg_val(id, vcpu->arch.dscr);
971 break;
972 case KVM_REG_PPC_PURR:
973 *val = get_reg_val(id, vcpu->arch.purr);
974 break;
975 case KVM_REG_PPC_SPURR:
976 *val = get_reg_val(id, vcpu->arch.spurr);
977 break;
978 case KVM_REG_PPC_AMR:
979 *val = get_reg_val(id, vcpu->arch.amr);
980 break;
981 case KVM_REG_PPC_UAMOR:
982 *val = get_reg_val(id, vcpu->arch.uamor);
983 break;
984 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
985 i = id - KVM_REG_PPC_MMCR0;
986 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
987 break;
988 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
989 i = id - KVM_REG_PPC_PMC1;
990 *val = get_reg_val(id, vcpu->arch.pmc[i]);
991 break;
992 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
993 i = id - KVM_REG_PPC_SPMC1;
994 *val = get_reg_val(id, vcpu->arch.spmc[i]);
995 break;
996 case KVM_REG_PPC_SIAR:
997 *val = get_reg_val(id, vcpu->arch.siar);
998 break;
999 case KVM_REG_PPC_SDAR:
1000 *val = get_reg_val(id, vcpu->arch.sdar);
1001 break;
1002 case KVM_REG_PPC_SIER:
1003 *val = get_reg_val(id, vcpu->arch.sier);
1004 break;
1005 case KVM_REG_PPC_IAMR:
1006 *val = get_reg_val(id, vcpu->arch.iamr);
1007 break;
1008 case KVM_REG_PPC_PSPB:
1009 *val = get_reg_val(id, vcpu->arch.pspb);
1010 break;
1011 case KVM_REG_PPC_DPDES:
1012 *val = get_reg_val(id, vcpu->arch.vcore->dpdes);
1013 break;
1014 case KVM_REG_PPC_DAWR:
1015 *val = get_reg_val(id, vcpu->arch.dawr);
1016 break;
1017 case KVM_REG_PPC_DAWRX:
1018 *val = get_reg_val(id, vcpu->arch.dawrx);
1019 break;
1020 case KVM_REG_PPC_CIABR:
1021 *val = get_reg_val(id, vcpu->arch.ciabr);
1022 break;
1023 case KVM_REG_PPC_CSIGR:
1024 *val = get_reg_val(id, vcpu->arch.csigr);
1025 break;
1026 case KVM_REG_PPC_TACR:
1027 *val = get_reg_val(id, vcpu->arch.tacr);
1028 break;
1029 case KVM_REG_PPC_TCSCR:
1030 *val = get_reg_val(id, vcpu->arch.tcscr);
1031 break;
1032 case KVM_REG_PPC_PID:
1033 *val = get_reg_val(id, vcpu->arch.pid);
1034 break;
1035 case KVM_REG_PPC_ACOP:
1036 *val = get_reg_val(id, vcpu->arch.acop);
1037 break;
1038 case KVM_REG_PPC_WORT:
1039 *val = get_reg_val(id, vcpu->arch.wort);
1040 break;
1041 case KVM_REG_PPC_VPA_ADDR:
1042 spin_lock(&vcpu->arch.vpa_update_lock);
1043 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1044 spin_unlock(&vcpu->arch.vpa_update_lock);
1045 break;
1046 case KVM_REG_PPC_VPA_SLB:
1047 spin_lock(&vcpu->arch.vpa_update_lock);
1048 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1049 val->vpaval.length = vcpu->arch.slb_shadow.len;
1050 spin_unlock(&vcpu->arch.vpa_update_lock);
1051 break;
1052 case KVM_REG_PPC_VPA_DTL:
1053 spin_lock(&vcpu->arch.vpa_update_lock);
1054 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1055 val->vpaval.length = vcpu->arch.dtl.len;
1056 spin_unlock(&vcpu->arch.vpa_update_lock);
1057 break;
1058 case KVM_REG_PPC_TB_OFFSET:
1059 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1060 break;
1061 case KVM_REG_PPC_LPCR:
1062 case KVM_REG_PPC_LPCR_64:
1063 *val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1064 break;
1065 case KVM_REG_PPC_PPR:
1066 *val = get_reg_val(id, vcpu->arch.ppr);
1067 break;
1068 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1069 case KVM_REG_PPC_TFHAR:
1070 *val = get_reg_val(id, vcpu->arch.tfhar);
1071 break;
1072 case KVM_REG_PPC_TFIAR:
1073 *val = get_reg_val(id, vcpu->arch.tfiar);
1074 break;
1075 case KVM_REG_PPC_TEXASR:
1076 *val = get_reg_val(id, vcpu->arch.texasr);
1077 break;
1078 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1079 i = id - KVM_REG_PPC_TM_GPR0;
1080 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1081 break;
1082 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1083 {
1084 int j;
1085 i = id - KVM_REG_PPC_TM_VSR0;
1086 if (i < 32)
1087 for (j = 0; j < TS_FPRWIDTH; j++)
1088 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1089 else {
1090 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1091 val->vval = vcpu->arch.vr_tm.vr[i-32];
1092 else
1093 r = -ENXIO;
1094 }
1095 break;
1096 }
1097 case KVM_REG_PPC_TM_CR:
1098 *val = get_reg_val(id, vcpu->arch.cr_tm);
1099 break;
1100 case KVM_REG_PPC_TM_XER:
1101 *val = get_reg_val(id, vcpu->arch.xer_tm);
1102 break;
1103 case KVM_REG_PPC_TM_LR:
1104 *val = get_reg_val(id, vcpu->arch.lr_tm);
1105 break;
1106 case KVM_REG_PPC_TM_CTR:
1107 *val = get_reg_val(id, vcpu->arch.ctr_tm);
1108 break;
1109 case KVM_REG_PPC_TM_FPSCR:
1110 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1111 break;
1112 case KVM_REG_PPC_TM_AMR:
1113 *val = get_reg_val(id, vcpu->arch.amr_tm);
1114 break;
1115 case KVM_REG_PPC_TM_PPR:
1116 *val = get_reg_val(id, vcpu->arch.ppr_tm);
1117 break;
1118 case KVM_REG_PPC_TM_VRSAVE:
1119 *val = get_reg_val(id, vcpu->arch.vrsave_tm);
1120 break;
1121 case KVM_REG_PPC_TM_VSCR:
1122 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1123 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1124 else
1125 r = -ENXIO;
1126 break;
1127 case KVM_REG_PPC_TM_DSCR:
1128 *val = get_reg_val(id, vcpu->arch.dscr_tm);
1129 break;
1130 case KVM_REG_PPC_TM_TAR:
1131 *val = get_reg_val(id, vcpu->arch.tar_tm);
1132 break;
1133 #endif
1134 case KVM_REG_PPC_ARCH_COMPAT:
1135 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1136 break;
1137 default:
1138 r = -EINVAL;
1139 break;
1140 }
1141
1142 return r;
1143 }
1144
kvmppc_set_one_reg_hv(struct kvm_vcpu * vcpu,u64 id,union kvmppc_one_reg * val)1145 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1146 union kvmppc_one_reg *val)
1147 {
1148 int r = 0;
1149 long int i;
1150 unsigned long addr, len;
1151
1152 switch (id) {
1153 case KVM_REG_PPC_HIOR:
1154 /* Only allow this to be set to zero */
1155 if (set_reg_val(id, *val))
1156 r = -EINVAL;
1157 break;
1158 case KVM_REG_PPC_DABR:
1159 vcpu->arch.dabr = set_reg_val(id, *val);
1160 break;
1161 case KVM_REG_PPC_DABRX:
1162 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1163 break;
1164 case KVM_REG_PPC_DSCR:
1165 vcpu->arch.dscr = set_reg_val(id, *val);
1166 break;
1167 case KVM_REG_PPC_PURR:
1168 vcpu->arch.purr = set_reg_val(id, *val);
1169 break;
1170 case KVM_REG_PPC_SPURR:
1171 vcpu->arch.spurr = set_reg_val(id, *val);
1172 break;
1173 case KVM_REG_PPC_AMR:
1174 vcpu->arch.amr = set_reg_val(id, *val);
1175 break;
1176 case KVM_REG_PPC_UAMOR:
1177 vcpu->arch.uamor = set_reg_val(id, *val);
1178 break;
1179 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1180 i = id - KVM_REG_PPC_MMCR0;
1181 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
1182 break;
1183 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1184 i = id - KVM_REG_PPC_PMC1;
1185 vcpu->arch.pmc[i] = set_reg_val(id, *val);
1186 break;
1187 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1188 i = id - KVM_REG_PPC_SPMC1;
1189 vcpu->arch.spmc[i] = set_reg_val(id, *val);
1190 break;
1191 case KVM_REG_PPC_SIAR:
1192 vcpu->arch.siar = set_reg_val(id, *val);
1193 break;
1194 case KVM_REG_PPC_SDAR:
1195 vcpu->arch.sdar = set_reg_val(id, *val);
1196 break;
1197 case KVM_REG_PPC_SIER:
1198 vcpu->arch.sier = set_reg_val(id, *val);
1199 break;
1200 case KVM_REG_PPC_IAMR:
1201 vcpu->arch.iamr = set_reg_val(id, *val);
1202 break;
1203 case KVM_REG_PPC_PSPB:
1204 vcpu->arch.pspb = set_reg_val(id, *val);
1205 break;
1206 case KVM_REG_PPC_DPDES:
1207 vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
1208 break;
1209 case KVM_REG_PPC_DAWR:
1210 vcpu->arch.dawr = set_reg_val(id, *val);
1211 break;
1212 case KVM_REG_PPC_DAWRX:
1213 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
1214 break;
1215 case KVM_REG_PPC_CIABR:
1216 vcpu->arch.ciabr = set_reg_val(id, *val);
1217 /* Don't allow setting breakpoints in hypervisor code */
1218 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
1219 vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */
1220 break;
1221 case KVM_REG_PPC_CSIGR:
1222 vcpu->arch.csigr = set_reg_val(id, *val);
1223 break;
1224 case KVM_REG_PPC_TACR:
1225 vcpu->arch.tacr = set_reg_val(id, *val);
1226 break;
1227 case KVM_REG_PPC_TCSCR:
1228 vcpu->arch.tcscr = set_reg_val(id, *val);
1229 break;
1230 case KVM_REG_PPC_PID:
1231 vcpu->arch.pid = set_reg_val(id, *val);
1232 break;
1233 case KVM_REG_PPC_ACOP:
1234 vcpu->arch.acop = set_reg_val(id, *val);
1235 break;
1236 case KVM_REG_PPC_WORT:
1237 vcpu->arch.wort = set_reg_val(id, *val);
1238 break;
1239 case KVM_REG_PPC_VPA_ADDR:
1240 addr = set_reg_val(id, *val);
1241 r = -EINVAL;
1242 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
1243 vcpu->arch.dtl.next_gpa))
1244 break;
1245 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
1246 break;
1247 case KVM_REG_PPC_VPA_SLB:
1248 addr = val->vpaval.addr;
1249 len = val->vpaval.length;
1250 r = -EINVAL;
1251 if (addr && !vcpu->arch.vpa.next_gpa)
1252 break;
1253 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
1254 break;
1255 case KVM_REG_PPC_VPA_DTL:
1256 addr = val->vpaval.addr;
1257 len = val->vpaval.length;
1258 r = -EINVAL;
1259 if (addr && (len < sizeof(struct dtl_entry) ||
1260 !vcpu->arch.vpa.next_gpa))
1261 break;
1262 len -= len % sizeof(struct dtl_entry);
1263 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1264 break;
1265 case KVM_REG_PPC_TB_OFFSET:
1266 /* round up to multiple of 2^24 */
1267 vcpu->arch.vcore->tb_offset =
1268 ALIGN(set_reg_val(id, *val), 1UL << 24);
1269 break;
1270 case KVM_REG_PPC_LPCR:
1271 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
1272 break;
1273 case KVM_REG_PPC_LPCR_64:
1274 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1275 break;
1276 case KVM_REG_PPC_PPR:
1277 vcpu->arch.ppr = set_reg_val(id, *val);
1278 break;
1279 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1280 case KVM_REG_PPC_TFHAR:
1281 vcpu->arch.tfhar = set_reg_val(id, *val);
1282 break;
1283 case KVM_REG_PPC_TFIAR:
1284 vcpu->arch.tfiar = set_reg_val(id, *val);
1285 break;
1286 case KVM_REG_PPC_TEXASR:
1287 vcpu->arch.texasr = set_reg_val(id, *val);
1288 break;
1289 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1290 i = id - KVM_REG_PPC_TM_GPR0;
1291 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
1292 break;
1293 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1294 {
1295 int j;
1296 i = id - KVM_REG_PPC_TM_VSR0;
1297 if (i < 32)
1298 for (j = 0; j < TS_FPRWIDTH; j++)
1299 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
1300 else
1301 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1302 vcpu->arch.vr_tm.vr[i-32] = val->vval;
1303 else
1304 r = -ENXIO;
1305 break;
1306 }
1307 case KVM_REG_PPC_TM_CR:
1308 vcpu->arch.cr_tm = set_reg_val(id, *val);
1309 break;
1310 case KVM_REG_PPC_TM_XER:
1311 vcpu->arch.xer_tm = set_reg_val(id, *val);
1312 break;
1313 case KVM_REG_PPC_TM_LR:
1314 vcpu->arch.lr_tm = set_reg_val(id, *val);
1315 break;
1316 case KVM_REG_PPC_TM_CTR:
1317 vcpu->arch.ctr_tm = set_reg_val(id, *val);
1318 break;
1319 case KVM_REG_PPC_TM_FPSCR:
1320 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
1321 break;
1322 case KVM_REG_PPC_TM_AMR:
1323 vcpu->arch.amr_tm = set_reg_val(id, *val);
1324 break;
1325 case KVM_REG_PPC_TM_PPR:
1326 vcpu->arch.ppr_tm = set_reg_val(id, *val);
1327 break;
1328 case KVM_REG_PPC_TM_VRSAVE:
1329 vcpu->arch.vrsave_tm = set_reg_val(id, *val);
1330 break;
1331 case KVM_REG_PPC_TM_VSCR:
1332 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1333 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
1334 else
1335 r = - ENXIO;
1336 break;
1337 case KVM_REG_PPC_TM_DSCR:
1338 vcpu->arch.dscr_tm = set_reg_val(id, *val);
1339 break;
1340 case KVM_REG_PPC_TM_TAR:
1341 vcpu->arch.tar_tm = set_reg_val(id, *val);
1342 break;
1343 #endif
1344 case KVM_REG_PPC_ARCH_COMPAT:
1345 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
1346 break;
1347 default:
1348 r = -EINVAL;
1349 break;
1350 }
1351
1352 return r;
1353 }
1354
kvmppc_vcore_create(struct kvm * kvm,int core)1355 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)
1356 {
1357 struct kvmppc_vcore *vcore;
1358
1359 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
1360
1361 if (vcore == NULL)
1362 return NULL;
1363
1364 INIT_LIST_HEAD(&vcore->runnable_threads);
1365 spin_lock_init(&vcore->lock);
1366 init_waitqueue_head(&vcore->wq);
1367 vcore->preempt_tb = TB_NIL;
1368 vcore->lpcr = kvm->arch.lpcr;
1369 vcore->first_vcpuid = core * threads_per_subcore;
1370 vcore->kvm = kvm;
1371
1372 vcore->mpp_buffer_is_valid = false;
1373
1374 if (cpu_has_feature(CPU_FTR_ARCH_207S))
1375 vcore->mpp_buffer = (void *)__get_free_pages(
1376 GFP_KERNEL|__GFP_ZERO,
1377 MPP_BUFFER_ORDER);
1378
1379 return vcore;
1380 }
1381
kvmppc_core_vcpu_create_hv(struct kvm * kvm,unsigned int id)1382 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
1383 unsigned int id)
1384 {
1385 struct kvm_vcpu *vcpu;
1386 int err = -EINVAL;
1387 int core;
1388 struct kvmppc_vcore *vcore;
1389
1390 core = id / threads_per_subcore;
1391 if (core >= KVM_MAX_VCORES)
1392 goto out;
1393
1394 err = -ENOMEM;
1395 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1396 if (!vcpu)
1397 goto out;
1398
1399 err = kvm_vcpu_init(vcpu, kvm, id);
1400 if (err)
1401 goto free_vcpu;
1402
1403 vcpu->arch.shared = &vcpu->arch.shregs;
1404 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1405 /*
1406 * The shared struct is never shared on HV,
1407 * so we can always use host endianness
1408 */
1409 #ifdef __BIG_ENDIAN__
1410 vcpu->arch.shared_big_endian = true;
1411 #else
1412 vcpu->arch.shared_big_endian = false;
1413 #endif
1414 #endif
1415 vcpu->arch.mmcr[0] = MMCR0_FC;
1416 vcpu->arch.ctrl = CTRL_RUNLATCH;
1417 /* default to host PVR, since we can't spoof it */
1418 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
1419 spin_lock_init(&vcpu->arch.vpa_update_lock);
1420 spin_lock_init(&vcpu->arch.tbacct_lock);
1421 vcpu->arch.busy_preempt = TB_NIL;
1422 vcpu->arch.intr_msr = MSR_SF | MSR_ME;
1423
1424 kvmppc_mmu_book3s_hv_init(vcpu);
1425
1426 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
1427
1428 init_waitqueue_head(&vcpu->arch.cpu_run);
1429
1430 mutex_lock(&kvm->lock);
1431 vcore = kvm->arch.vcores[core];
1432 if (!vcore) {
1433 vcore = kvmppc_vcore_create(kvm, core);
1434 kvm->arch.vcores[core] = vcore;
1435 kvm->arch.online_vcores++;
1436 }
1437 mutex_unlock(&kvm->lock);
1438
1439 if (!vcore)
1440 goto free_vcpu;
1441
1442 spin_lock(&vcore->lock);
1443 ++vcore->num_threads;
1444 spin_unlock(&vcore->lock);
1445 vcpu->arch.vcore = vcore;
1446 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
1447
1448 vcpu->arch.cpu_type = KVM_CPU_3S_64;
1449 kvmppc_sanity_check(vcpu);
1450
1451 return vcpu;
1452
1453 free_vcpu:
1454 kmem_cache_free(kvm_vcpu_cache, vcpu);
1455 out:
1456 return ERR_PTR(err);
1457 }
1458
unpin_vpa(struct kvm * kvm,struct kvmppc_vpa * vpa)1459 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
1460 {
1461 if (vpa->pinned_addr)
1462 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
1463 vpa->dirty);
1464 }
1465
kvmppc_core_vcpu_free_hv(struct kvm_vcpu * vcpu)1466 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
1467 {
1468 spin_lock(&vcpu->arch.vpa_update_lock);
1469 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
1470 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
1471 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
1472 spin_unlock(&vcpu->arch.vpa_update_lock);
1473 kvm_vcpu_uninit(vcpu);
1474 kmem_cache_free(kvm_vcpu_cache, vcpu);
1475 }
1476
kvmppc_core_check_requests_hv(struct kvm_vcpu * vcpu)1477 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
1478 {
1479 /* Indicate we want to get back into the guest */
1480 return 1;
1481 }
1482
kvmppc_set_timer(struct kvm_vcpu * vcpu)1483 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
1484 {
1485 unsigned long dec_nsec, now;
1486
1487 now = get_tb();
1488 if (now > vcpu->arch.dec_expires) {
1489 /* decrementer has already gone negative */
1490 kvmppc_core_queue_dec(vcpu);
1491 kvmppc_core_prepare_to_enter(vcpu);
1492 return;
1493 }
1494 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC
1495 / tb_ticks_per_sec;
1496 hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec),
1497 HRTIMER_MODE_REL);
1498 vcpu->arch.timer_running = 1;
1499 }
1500
kvmppc_end_cede(struct kvm_vcpu * vcpu)1501 static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
1502 {
1503 vcpu->arch.ceded = 0;
1504 if (vcpu->arch.timer_running) {
1505 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
1506 vcpu->arch.timer_running = 0;
1507 }
1508 }
1509
1510 extern void __kvmppc_vcore_entry(void);
1511
kvmppc_remove_runnable(struct kvmppc_vcore * vc,struct kvm_vcpu * vcpu)1512 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
1513 struct kvm_vcpu *vcpu)
1514 {
1515 u64 now;
1516
1517 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
1518 return;
1519 spin_lock_irq(&vcpu->arch.tbacct_lock);
1520 now = mftb();
1521 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
1522 vcpu->arch.stolen_logged;
1523 vcpu->arch.busy_preempt = now;
1524 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
1525 spin_unlock_irq(&vcpu->arch.tbacct_lock);
1526 --vc->n_runnable;
1527 list_del(&vcpu->arch.run_list);
1528 }
1529
kvmppc_grab_hwthread(int cpu)1530 static int kvmppc_grab_hwthread(int cpu)
1531 {
1532 struct paca_struct *tpaca;
1533 long timeout = 10000;
1534
1535 tpaca = &paca[cpu];
1536
1537 /* Ensure the thread won't go into the kernel if it wakes */
1538 tpaca->kvm_hstate.hwthread_req = 1;
1539 tpaca->kvm_hstate.kvm_vcpu = NULL;
1540
1541 /*
1542 * If the thread is already executing in the kernel (e.g. handling
1543 * a stray interrupt), wait for it to get back to nap mode.
1544 * The smp_mb() is to ensure that our setting of hwthread_req
1545 * is visible before we look at hwthread_state, so if this
1546 * races with the code at system_reset_pSeries and the thread
1547 * misses our setting of hwthread_req, we are sure to see its
1548 * setting of hwthread_state, and vice versa.
1549 */
1550 smp_mb();
1551 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
1552 if (--timeout <= 0) {
1553 pr_err("KVM: couldn't grab cpu %d\n", cpu);
1554 return -EBUSY;
1555 }
1556 udelay(1);
1557 }
1558 return 0;
1559 }
1560
kvmppc_release_hwthread(int cpu)1561 static void kvmppc_release_hwthread(int cpu)
1562 {
1563 struct paca_struct *tpaca;
1564
1565 tpaca = &paca[cpu];
1566 tpaca->kvm_hstate.hwthread_req = 0;
1567 tpaca->kvm_hstate.kvm_vcpu = NULL;
1568 }
1569
kvmppc_start_thread(struct kvm_vcpu * vcpu)1570 static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
1571 {
1572 int cpu;
1573 struct paca_struct *tpaca;
1574 struct kvmppc_vcore *vc = vcpu->arch.vcore;
1575
1576 if (vcpu->arch.timer_running) {
1577 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
1578 vcpu->arch.timer_running = 0;
1579 }
1580 cpu = vc->pcpu + vcpu->arch.ptid;
1581 tpaca = &paca[cpu];
1582 tpaca->kvm_hstate.kvm_vcpu = vcpu;
1583 tpaca->kvm_hstate.kvm_vcore = vc;
1584 tpaca->kvm_hstate.ptid = vcpu->arch.ptid;
1585 vcpu->cpu = vc->pcpu;
1586 smp_wmb();
1587 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
1588 if (cpu != smp_processor_id()) {
1589 xics_wake_cpu(cpu);
1590 if (vcpu->arch.ptid)
1591 ++vc->n_woken;
1592 }
1593 #endif
1594 }
1595
kvmppc_wait_for_nap(struct kvmppc_vcore * vc)1596 static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc)
1597 {
1598 int i;
1599
1600 HMT_low();
1601 i = 0;
1602 while (vc->nap_count < vc->n_woken) {
1603 if (++i >= 1000000) {
1604 pr_err("kvmppc_wait_for_nap timeout %d %d\n",
1605 vc->nap_count, vc->n_woken);
1606 break;
1607 }
1608 cpu_relax();
1609 }
1610 HMT_medium();
1611 }
1612
1613 /*
1614 * Check that we are on thread 0 and that any other threads in
1615 * this core are off-line. Then grab the threads so they can't
1616 * enter the kernel.
1617 */
on_primary_thread(void)1618 static int on_primary_thread(void)
1619 {
1620 int cpu = smp_processor_id();
1621 int thr;
1622
1623 /* Are we on a primary subcore? */
1624 if (cpu_thread_in_subcore(cpu))
1625 return 0;
1626
1627 thr = 0;
1628 while (++thr < threads_per_subcore)
1629 if (cpu_online(cpu + thr))
1630 return 0;
1631
1632 /* Grab all hw threads so they can't go into the kernel */
1633 for (thr = 1; thr < threads_per_subcore; ++thr) {
1634 if (kvmppc_grab_hwthread(cpu + thr)) {
1635 /* Couldn't grab one; let the others go */
1636 do {
1637 kvmppc_release_hwthread(cpu + thr);
1638 } while (--thr > 0);
1639 return 0;
1640 }
1641 }
1642 return 1;
1643 }
1644
kvmppc_start_saving_l2_cache(struct kvmppc_vcore * vc)1645 static void kvmppc_start_saving_l2_cache(struct kvmppc_vcore *vc)
1646 {
1647 phys_addr_t phy_addr, mpp_addr;
1648
1649 phy_addr = (phys_addr_t)virt_to_phys(vc->mpp_buffer);
1650 mpp_addr = phy_addr & PPC_MPPE_ADDRESS_MASK;
1651
1652 mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_ABORT);
1653 logmpp(mpp_addr | PPC_LOGMPP_LOG_L2);
1654
1655 vc->mpp_buffer_is_valid = true;
1656 }
1657
kvmppc_start_restoring_l2_cache(const struct kvmppc_vcore * vc)1658 static void kvmppc_start_restoring_l2_cache(const struct kvmppc_vcore *vc)
1659 {
1660 phys_addr_t phy_addr, mpp_addr;
1661
1662 phy_addr = virt_to_phys(vc->mpp_buffer);
1663 mpp_addr = phy_addr & PPC_MPPE_ADDRESS_MASK;
1664
1665 /* We must abort any in-progress save operations to ensure
1666 * the table is valid so that prefetch engine knows when to
1667 * stop prefetching. */
1668 logmpp(mpp_addr | PPC_LOGMPP_LOG_ABORT);
1669 mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_WHOLE_TABLE);
1670 }
1671
1672 /*
1673 * Run a set of guest threads on a physical core.
1674 * Called with vc->lock held.
1675 */
kvmppc_run_core(struct kvmppc_vcore * vc)1676 static void kvmppc_run_core(struct kvmppc_vcore *vc)
1677 {
1678 struct kvm_vcpu *vcpu, *vnext;
1679 long ret;
1680 u64 now;
1681 int i, need_vpa_update;
1682 int srcu_idx;
1683 struct kvm_vcpu *vcpus_to_update[threads_per_core];
1684
1685 /* don't start if any threads have a signal pending */
1686 need_vpa_update = 0;
1687 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1688 if (signal_pending(vcpu->arch.run_task))
1689 return;
1690 if (vcpu->arch.vpa.update_pending ||
1691 vcpu->arch.slb_shadow.update_pending ||
1692 vcpu->arch.dtl.update_pending)
1693 vcpus_to_update[need_vpa_update++] = vcpu;
1694 }
1695
1696 /*
1697 * Initialize *vc, in particular vc->vcore_state, so we can
1698 * drop the vcore lock if necessary.
1699 */
1700 vc->n_woken = 0;
1701 vc->nap_count = 0;
1702 vc->entry_exit_count = 0;
1703 vc->vcore_state = VCORE_STARTING;
1704 vc->in_guest = 0;
1705 vc->napping_threads = 0;
1706
1707 /*
1708 * Updating any of the vpas requires calling kvmppc_pin_guest_page,
1709 * which can't be called with any spinlocks held.
1710 */
1711 if (need_vpa_update) {
1712 spin_unlock(&vc->lock);
1713 for (i = 0; i < need_vpa_update; ++i)
1714 kvmppc_update_vpas(vcpus_to_update[i]);
1715 spin_lock(&vc->lock);
1716 }
1717
1718 /*
1719 * Make sure we are running on primary threads, and that secondary
1720 * threads are offline. Also check if the number of threads in this
1721 * guest are greater than the current system threads per guest.
1722 */
1723 if ((threads_per_core > 1) &&
1724 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
1725 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1726 vcpu->arch.ret = -EBUSY;
1727 goto out;
1728 }
1729
1730
1731 vc->pcpu = smp_processor_id();
1732 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1733 kvmppc_start_thread(vcpu);
1734 kvmppc_create_dtl_entry(vcpu, vc);
1735 }
1736
1737 /* Set this explicitly in case thread 0 doesn't have a vcpu */
1738 get_paca()->kvm_hstate.kvm_vcore = vc;
1739 get_paca()->kvm_hstate.ptid = 0;
1740
1741 vc->vcore_state = VCORE_RUNNING;
1742 preempt_disable();
1743 spin_unlock(&vc->lock);
1744
1745 kvm_guest_enter();
1746
1747 srcu_idx = srcu_read_lock(&vc->kvm->srcu);
1748
1749 if (vc->mpp_buffer_is_valid)
1750 kvmppc_start_restoring_l2_cache(vc);
1751
1752 __kvmppc_vcore_entry();
1753
1754 spin_lock(&vc->lock);
1755
1756 if (vc->mpp_buffer)
1757 kvmppc_start_saving_l2_cache(vc);
1758
1759 /* disable sending of IPIs on virtual external irqs */
1760 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1761 vcpu->cpu = -1;
1762 /* wait for secondary threads to finish writing their state to memory */
1763 if (vc->nap_count < vc->n_woken)
1764 kvmppc_wait_for_nap(vc);
1765 for (i = 0; i < threads_per_subcore; ++i)
1766 kvmppc_release_hwthread(vc->pcpu + i);
1767 /* prevent other vcpu threads from doing kvmppc_start_thread() now */
1768 vc->vcore_state = VCORE_EXITING;
1769 spin_unlock(&vc->lock);
1770
1771 srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
1772
1773 /* make sure updates to secondary vcpu structs are visible now */
1774 smp_mb();
1775 kvm_guest_exit();
1776
1777 preempt_enable();
1778 cond_resched();
1779
1780 spin_lock(&vc->lock);
1781 now = get_tb();
1782 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1783 /* cancel pending dec exception if dec is positive */
1784 if (now < vcpu->arch.dec_expires &&
1785 kvmppc_core_pending_dec(vcpu))
1786 kvmppc_core_dequeue_dec(vcpu);
1787
1788 ret = RESUME_GUEST;
1789 if (vcpu->arch.trap)
1790 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
1791 vcpu->arch.run_task);
1792
1793 vcpu->arch.ret = ret;
1794 vcpu->arch.trap = 0;
1795
1796 if (vcpu->arch.ceded) {
1797 if (!is_kvmppc_resume_guest(ret))
1798 kvmppc_end_cede(vcpu);
1799 else
1800 kvmppc_set_timer(vcpu);
1801 }
1802 }
1803
1804 out:
1805 vc->vcore_state = VCORE_INACTIVE;
1806 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
1807 arch.run_list) {
1808 if (!is_kvmppc_resume_guest(vcpu->arch.ret)) {
1809 kvmppc_remove_runnable(vc, vcpu);
1810 wake_up(&vcpu->arch.cpu_run);
1811 }
1812 }
1813 }
1814
1815 /*
1816 * Wait for some other vcpu thread to execute us, and
1817 * wake us up when we need to handle something in the host.
1818 */
kvmppc_wait_for_exec(struct kvm_vcpu * vcpu,int wait_state)1819 static void kvmppc_wait_for_exec(struct kvm_vcpu *vcpu, int wait_state)
1820 {
1821 DEFINE_WAIT(wait);
1822
1823 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
1824 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE)
1825 schedule();
1826 finish_wait(&vcpu->arch.cpu_run, &wait);
1827 }
1828
1829 /*
1830 * All the vcpus in this vcore are idle, so wait for a decrementer
1831 * or external interrupt to one of the vcpus. vc->lock is held.
1832 */
kvmppc_vcore_blocked(struct kvmppc_vcore * vc)1833 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
1834 {
1835 DEFINE_WAIT(wait);
1836
1837 prepare_to_wait(&vc->wq, &wait, TASK_INTERRUPTIBLE);
1838 vc->vcore_state = VCORE_SLEEPING;
1839 spin_unlock(&vc->lock);
1840 schedule();
1841 finish_wait(&vc->wq, &wait);
1842 spin_lock(&vc->lock);
1843 vc->vcore_state = VCORE_INACTIVE;
1844 }
1845
kvmppc_run_vcpu(struct kvm_run * kvm_run,struct kvm_vcpu * vcpu)1846 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1847 {
1848 int n_ceded;
1849 struct kvmppc_vcore *vc;
1850 struct kvm_vcpu *v, *vn;
1851
1852 kvm_run->exit_reason = 0;
1853 vcpu->arch.ret = RESUME_GUEST;
1854 vcpu->arch.trap = 0;
1855 kvmppc_update_vpas(vcpu);
1856
1857 /*
1858 * Synchronize with other threads in this virtual core
1859 */
1860 vc = vcpu->arch.vcore;
1861 spin_lock(&vc->lock);
1862 vcpu->arch.ceded = 0;
1863 vcpu->arch.run_task = current;
1864 vcpu->arch.kvm_run = kvm_run;
1865 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
1866 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
1867 vcpu->arch.busy_preempt = TB_NIL;
1868 list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads);
1869 ++vc->n_runnable;
1870
1871 /*
1872 * This happens the first time this is called for a vcpu.
1873 * If the vcore is already running, we may be able to start
1874 * this thread straight away and have it join in.
1875 */
1876 if (!signal_pending(current)) {
1877 if (vc->vcore_state == VCORE_RUNNING &&
1878 VCORE_EXIT_COUNT(vc) == 0) {
1879 kvmppc_create_dtl_entry(vcpu, vc);
1880 kvmppc_start_thread(vcpu);
1881 } else if (vc->vcore_state == VCORE_SLEEPING) {
1882 wake_up(&vc->wq);
1883 }
1884
1885 }
1886
1887 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
1888 !signal_pending(current)) {
1889 if (vc->vcore_state != VCORE_INACTIVE) {
1890 spin_unlock(&vc->lock);
1891 kvmppc_wait_for_exec(vcpu, TASK_INTERRUPTIBLE);
1892 spin_lock(&vc->lock);
1893 continue;
1894 }
1895 list_for_each_entry_safe(v, vn, &vc->runnable_threads,
1896 arch.run_list) {
1897 kvmppc_core_prepare_to_enter(v);
1898 if (signal_pending(v->arch.run_task)) {
1899 kvmppc_remove_runnable(vc, v);
1900 v->stat.signal_exits++;
1901 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
1902 v->arch.ret = -EINTR;
1903 wake_up(&v->arch.cpu_run);
1904 }
1905 }
1906 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
1907 break;
1908 vc->runner = vcpu;
1909 n_ceded = 0;
1910 list_for_each_entry(v, &vc->runnable_threads, arch.run_list) {
1911 if (!v->arch.pending_exceptions)
1912 n_ceded += v->arch.ceded;
1913 else
1914 v->arch.ceded = 0;
1915 }
1916 if (n_ceded == vc->n_runnable)
1917 kvmppc_vcore_blocked(vc);
1918 else
1919 kvmppc_run_core(vc);
1920 vc->runner = NULL;
1921 }
1922
1923 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
1924 (vc->vcore_state == VCORE_RUNNING ||
1925 vc->vcore_state == VCORE_EXITING)) {
1926 spin_unlock(&vc->lock);
1927 kvmppc_wait_for_exec(vcpu, TASK_UNINTERRUPTIBLE);
1928 spin_lock(&vc->lock);
1929 }
1930
1931 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
1932 kvmppc_remove_runnable(vc, vcpu);
1933 vcpu->stat.signal_exits++;
1934 kvm_run->exit_reason = KVM_EXIT_INTR;
1935 vcpu->arch.ret = -EINTR;
1936 }
1937
1938 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
1939 /* Wake up some vcpu to run the core */
1940 v = list_first_entry(&vc->runnable_threads,
1941 struct kvm_vcpu, arch.run_list);
1942 wake_up(&v->arch.cpu_run);
1943 }
1944
1945 spin_unlock(&vc->lock);
1946 return vcpu->arch.ret;
1947 }
1948
kvmppc_vcpu_run_hv(struct kvm_run * run,struct kvm_vcpu * vcpu)1949 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
1950 {
1951 int r;
1952 int srcu_idx;
1953 unsigned long ebb_regs[3] = {}; /* shut up GCC */
1954 unsigned long user_tar = 0;
1955 unsigned long proc_fscr = 0;
1956 unsigned int user_vrsave;
1957
1958 if (!vcpu->arch.sane) {
1959 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1960 return -EINVAL;
1961 }
1962
1963 /*
1964 * Don't allow entry with a suspended transaction, because
1965 * the guest entry/exit code will lose it.
1966 * If the guest has TM enabled, save away their TM-related SPRs
1967 * (they will get restored by the TM unavailable interrupt).
1968 */
1969 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1970 if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
1971 (current->thread.regs->msr & MSR_TM)) {
1972 if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
1973 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1974 run->fail_entry.hardware_entry_failure_reason = 0;
1975 return -EINVAL;
1976 }
1977 /* Enable TM so we can read the TM SPRs */
1978 mtmsr(mfmsr() | MSR_TM);
1979 current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
1980 current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
1981 current->thread.tm_texasr = mfspr(SPRN_TEXASR);
1982 }
1983 #endif
1984
1985 kvmppc_core_prepare_to_enter(vcpu);
1986
1987 /* No need to go into the guest when all we'll do is come back out */
1988 if (signal_pending(current)) {
1989 run->exit_reason = KVM_EXIT_INTR;
1990 return -EINTR;
1991 }
1992
1993 atomic_inc(&vcpu->kvm->arch.vcpus_running);
1994 /* Order vcpus_running vs. rma_setup_done, see kvmppc_alloc_reset_hpt */
1995 smp_mb();
1996
1997 /* On the first time here, set up HTAB and VRMA or RMA */
1998 if (!vcpu->kvm->arch.rma_setup_done) {
1999 r = kvmppc_hv_setup_htab_rma(vcpu);
2000 if (r)
2001 goto out;
2002 }
2003
2004 flush_fp_to_thread(current);
2005 flush_altivec_to_thread(current);
2006 flush_vsx_to_thread(current);
2007
2008 /* Save userspace EBB and other register values */
2009 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
2010 ebb_regs[0] = mfspr(SPRN_EBBHR);
2011 ebb_regs[1] = mfspr(SPRN_EBBRR);
2012 ebb_regs[2] = mfspr(SPRN_BESCR);
2013 user_tar = mfspr(SPRN_TAR);
2014 proc_fscr = mfspr(SPRN_FSCR);
2015 }
2016 user_vrsave = mfspr(SPRN_VRSAVE);
2017
2018 vcpu->arch.wqp = &vcpu->arch.vcore->wq;
2019 vcpu->arch.pgdir = current->mm->pgd;
2020 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2021
2022 do {
2023 r = kvmppc_run_vcpu(run, vcpu);
2024
2025 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
2026 !(vcpu->arch.shregs.msr & MSR_PR)) {
2027 r = kvmppc_pseries_do_hcall(vcpu);
2028 kvmppc_core_prepare_to_enter(vcpu);
2029 } else if (r == RESUME_PAGE_FAULT) {
2030 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2031 r = kvmppc_book3s_hv_page_fault(run, vcpu,
2032 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
2033 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2034 }
2035 } while (is_kvmppc_resume_guest(r));
2036
2037 /* Restore userspace EBB and other register values */
2038 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
2039 mtspr(SPRN_EBBHR, ebb_regs[0]);
2040 mtspr(SPRN_EBBRR, ebb_regs[1]);
2041 mtspr(SPRN_BESCR, ebb_regs[2]);
2042 mtspr(SPRN_TAR, user_tar);
2043 mtspr(SPRN_FSCR, proc_fscr);
2044 }
2045 mtspr(SPRN_VRSAVE, user_vrsave);
2046
2047 /*
2048 * Since we don't do lazy TM reload, we need to reload
2049 * the TM registers here.
2050 */
2051 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2052 if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
2053 (current->thread.regs->msr & MSR_TM)) {
2054 mtspr(SPRN_TFHAR, current->thread.tm_tfhar);
2055 mtspr(SPRN_TFIAR, current->thread.tm_tfiar);
2056 mtspr(SPRN_TEXASR, current->thread.tm_texasr);
2057 }
2058 #endif
2059
2060 out:
2061 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2062 atomic_dec(&vcpu->kvm->arch.vcpus_running);
2063 return r;
2064 }
2065
2066
2067 /* Work out RMLS (real mode limit selector) field value for a given RMA size.
2068 Assumes POWER7 or PPC970. */
lpcr_rmls(unsigned long rma_size)2069 static inline int lpcr_rmls(unsigned long rma_size)
2070 {
2071 switch (rma_size) {
2072 case 32ul << 20: /* 32 MB */
2073 if (cpu_has_feature(CPU_FTR_ARCH_206))
2074 return 8; /* only supported on POWER7 */
2075 return -1;
2076 case 64ul << 20: /* 64 MB */
2077 return 3;
2078 case 128ul << 20: /* 128 MB */
2079 return 7;
2080 case 256ul << 20: /* 256 MB */
2081 return 4;
2082 case 1ul << 30: /* 1 GB */
2083 return 2;
2084 case 16ul << 30: /* 16 GB */
2085 return 1;
2086 case 256ul << 30: /* 256 GB */
2087 return 0;
2088 default:
2089 return -1;
2090 }
2091 }
2092
kvm_rma_fault(struct vm_area_struct * vma,struct vm_fault * vmf)2093 static int kvm_rma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2094 {
2095 struct page *page;
2096 struct kvm_rma_info *ri = vma->vm_file->private_data;
2097
2098 if (vmf->pgoff >= kvm_rma_pages)
2099 return VM_FAULT_SIGBUS;
2100
2101 page = pfn_to_page(ri->base_pfn + vmf->pgoff);
2102 get_page(page);
2103 vmf->page = page;
2104 return 0;
2105 }
2106
2107 static const struct vm_operations_struct kvm_rma_vm_ops = {
2108 .fault = kvm_rma_fault,
2109 };
2110
kvm_rma_mmap(struct file * file,struct vm_area_struct * vma)2111 static int kvm_rma_mmap(struct file *file, struct vm_area_struct *vma)
2112 {
2113 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
2114 vma->vm_ops = &kvm_rma_vm_ops;
2115 return 0;
2116 }
2117
kvm_rma_release(struct inode * inode,struct file * filp)2118 static int kvm_rma_release(struct inode *inode, struct file *filp)
2119 {
2120 struct kvm_rma_info *ri = filp->private_data;
2121
2122 kvm_release_rma(ri);
2123 return 0;
2124 }
2125
2126 static const struct file_operations kvm_rma_fops = {
2127 .mmap = kvm_rma_mmap,
2128 .release = kvm_rma_release,
2129 };
2130
kvm_vm_ioctl_allocate_rma(struct kvm * kvm,struct kvm_allocate_rma * ret)2131 static long kvm_vm_ioctl_allocate_rma(struct kvm *kvm,
2132 struct kvm_allocate_rma *ret)
2133 {
2134 long fd;
2135 struct kvm_rma_info *ri;
2136 /*
2137 * Only do this on PPC970 in HV mode
2138 */
2139 if (!cpu_has_feature(CPU_FTR_HVMODE) ||
2140 !cpu_has_feature(CPU_FTR_ARCH_201))
2141 return -EINVAL;
2142
2143 if (!kvm_rma_pages)
2144 return -EINVAL;
2145
2146 ri = kvm_alloc_rma();
2147 if (!ri)
2148 return -ENOMEM;
2149
2150 fd = anon_inode_getfd("kvm-rma", &kvm_rma_fops, ri, O_RDWR | O_CLOEXEC);
2151 if (fd < 0)
2152 kvm_release_rma(ri);
2153
2154 ret->rma_size = kvm_rma_pages << PAGE_SHIFT;
2155 return fd;
2156 }
2157
kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size ** sps,int linux_psize)2158 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
2159 int linux_psize)
2160 {
2161 struct mmu_psize_def *def = &mmu_psize_defs[linux_psize];
2162
2163 if (!def->shift)
2164 return;
2165 (*sps)->page_shift = def->shift;
2166 (*sps)->slb_enc = def->sllp;
2167 (*sps)->enc[0].page_shift = def->shift;
2168 (*sps)->enc[0].pte_enc = def->penc[linux_psize];
2169 /*
2170 * Add 16MB MPSS support if host supports it
2171 */
2172 if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) {
2173 (*sps)->enc[1].page_shift = 24;
2174 (*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M];
2175 }
2176 (*sps)++;
2177 }
2178
kvm_vm_ioctl_get_smmu_info_hv(struct kvm * kvm,struct kvm_ppc_smmu_info * info)2179 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
2180 struct kvm_ppc_smmu_info *info)
2181 {
2182 struct kvm_ppc_one_seg_page_size *sps;
2183
2184 info->flags = KVM_PPC_PAGE_SIZES_REAL;
2185 if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
2186 info->flags |= KVM_PPC_1T_SEGMENTS;
2187 info->slb_size = mmu_slb_size;
2188
2189 /* We only support these sizes for now, and no muti-size segments */
2190 sps = &info->sps[0];
2191 kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K);
2192 kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K);
2193 kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M);
2194
2195 return 0;
2196 }
2197
2198 /*
2199 * Get (and clear) the dirty memory log for a memory slot.
2200 */
kvm_vm_ioctl_get_dirty_log_hv(struct kvm * kvm,struct kvm_dirty_log * log)2201 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
2202 struct kvm_dirty_log *log)
2203 {
2204 struct kvm_memory_slot *memslot;
2205 int r;
2206 unsigned long n;
2207
2208 mutex_lock(&kvm->slots_lock);
2209
2210 r = -EINVAL;
2211 if (log->slot >= KVM_USER_MEM_SLOTS)
2212 goto out;
2213
2214 memslot = id_to_memslot(kvm->memslots, log->slot);
2215 r = -ENOENT;
2216 if (!memslot->dirty_bitmap)
2217 goto out;
2218
2219 n = kvm_dirty_bitmap_bytes(memslot);
2220 memset(memslot->dirty_bitmap, 0, n);
2221
2222 r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);
2223 if (r)
2224 goto out;
2225
2226 r = -EFAULT;
2227 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
2228 goto out;
2229
2230 r = 0;
2231 out:
2232 mutex_unlock(&kvm->slots_lock);
2233 return r;
2234 }
2235
unpin_slot(struct kvm_memory_slot * memslot)2236 static void unpin_slot(struct kvm_memory_slot *memslot)
2237 {
2238 unsigned long *physp;
2239 unsigned long j, npages, pfn;
2240 struct page *page;
2241
2242 physp = memslot->arch.slot_phys;
2243 npages = memslot->npages;
2244 if (!physp)
2245 return;
2246 for (j = 0; j < npages; j++) {
2247 if (!(physp[j] & KVMPPC_GOT_PAGE))
2248 continue;
2249 pfn = physp[j] >> PAGE_SHIFT;
2250 page = pfn_to_page(pfn);
2251 SetPageDirty(page);
2252 put_page(page);
2253 }
2254 }
2255
kvmppc_core_free_memslot_hv(struct kvm_memory_slot * free,struct kvm_memory_slot * dont)2256 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free,
2257 struct kvm_memory_slot *dont)
2258 {
2259 if (!dont || free->arch.rmap != dont->arch.rmap) {
2260 vfree(free->arch.rmap);
2261 free->arch.rmap = NULL;
2262 }
2263 if (!dont || free->arch.slot_phys != dont->arch.slot_phys) {
2264 unpin_slot(free);
2265 vfree(free->arch.slot_phys);
2266 free->arch.slot_phys = NULL;
2267 }
2268 }
2269
kvmppc_core_create_memslot_hv(struct kvm_memory_slot * slot,unsigned long npages)2270 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot,
2271 unsigned long npages)
2272 {
2273 slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap));
2274 if (!slot->arch.rmap)
2275 return -ENOMEM;
2276 slot->arch.slot_phys = NULL;
2277
2278 return 0;
2279 }
2280
kvmppc_core_prepare_memory_region_hv(struct kvm * kvm,struct kvm_memory_slot * memslot,struct kvm_userspace_memory_region * mem)2281 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
2282 struct kvm_memory_slot *memslot,
2283 struct kvm_userspace_memory_region *mem)
2284 {
2285 unsigned long *phys;
2286
2287 /* Allocate a slot_phys array if needed */
2288 phys = memslot->arch.slot_phys;
2289 if (!kvm->arch.using_mmu_notifiers && !phys && memslot->npages) {
2290 phys = vzalloc(memslot->npages * sizeof(unsigned long));
2291 if (!phys)
2292 return -ENOMEM;
2293 memslot->arch.slot_phys = phys;
2294 }
2295
2296 return 0;
2297 }
2298
kvmppc_core_commit_memory_region_hv(struct kvm * kvm,struct kvm_userspace_memory_region * mem,const struct kvm_memory_slot * old)2299 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
2300 struct kvm_userspace_memory_region *mem,
2301 const struct kvm_memory_slot *old)
2302 {
2303 unsigned long npages = mem->memory_size >> PAGE_SHIFT;
2304 struct kvm_memory_slot *memslot;
2305
2306 if (npages && old->npages) {
2307 /*
2308 * If modifying a memslot, reset all the rmap dirty bits.
2309 * If this is a new memslot, we don't need to do anything
2310 * since the rmap array starts out as all zeroes,
2311 * i.e. no pages are dirty.
2312 */
2313 memslot = id_to_memslot(kvm->memslots, mem->slot);
2314 kvmppc_hv_get_dirty_log(kvm, memslot, NULL);
2315 }
2316 }
2317
2318 /*
2319 * Update LPCR values in kvm->arch and in vcores.
2320 * Caller must hold kvm->lock.
2321 */
kvmppc_update_lpcr(struct kvm * kvm,unsigned long lpcr,unsigned long mask)2322 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
2323 {
2324 long int i;
2325 u32 cores_done = 0;
2326
2327 if ((kvm->arch.lpcr & mask) == lpcr)
2328 return;
2329
2330 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
2331
2332 for (i = 0; i < KVM_MAX_VCORES; ++i) {
2333 struct kvmppc_vcore *vc = kvm->arch.vcores[i];
2334 if (!vc)
2335 continue;
2336 spin_lock(&vc->lock);
2337 vc->lpcr = (vc->lpcr & ~mask) | lpcr;
2338 spin_unlock(&vc->lock);
2339 if (++cores_done >= kvm->arch.online_vcores)
2340 break;
2341 }
2342 }
2343
kvmppc_mmu_destroy_hv(struct kvm_vcpu * vcpu)2344 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
2345 {
2346 return;
2347 }
2348
kvmppc_hv_setup_htab_rma(struct kvm_vcpu * vcpu)2349 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
2350 {
2351 int err = 0;
2352 struct kvm *kvm = vcpu->kvm;
2353 struct kvm_rma_info *ri = NULL;
2354 unsigned long hva;
2355 struct kvm_memory_slot *memslot;
2356 struct vm_area_struct *vma;
2357 unsigned long lpcr = 0, senc;
2358 unsigned long lpcr_mask = 0;
2359 unsigned long psize, porder;
2360 unsigned long rma_size;
2361 unsigned long rmls;
2362 unsigned long *physp;
2363 unsigned long i, npages;
2364 int srcu_idx;
2365
2366 mutex_lock(&kvm->lock);
2367 if (kvm->arch.rma_setup_done)
2368 goto out; /* another vcpu beat us to it */
2369
2370 /* Allocate hashed page table (if not done already) and reset it */
2371 if (!kvm->arch.hpt_virt) {
2372 err = kvmppc_alloc_hpt(kvm, NULL);
2373 if (err) {
2374 pr_err("KVM: Couldn't alloc HPT\n");
2375 goto out;
2376 }
2377 }
2378
2379 /* Look up the memslot for guest physical address 0 */
2380 srcu_idx = srcu_read_lock(&kvm->srcu);
2381 memslot = gfn_to_memslot(kvm, 0);
2382
2383 /* We must have some memory at 0 by now */
2384 err = -EINVAL;
2385 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
2386 goto out_srcu;
2387
2388 /* Look up the VMA for the start of this memory slot */
2389 hva = memslot->userspace_addr;
2390 down_read(¤t->mm->mmap_sem);
2391 vma = find_vma(current->mm, hva);
2392 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
2393 goto up_out;
2394
2395 psize = vma_kernel_pagesize(vma);
2396 porder = __ilog2(psize);
2397
2398 /* Is this one of our preallocated RMAs? */
2399 if (vma->vm_file && vma->vm_file->f_op == &kvm_rma_fops &&
2400 hva == vma->vm_start)
2401 ri = vma->vm_file->private_data;
2402
2403 up_read(¤t->mm->mmap_sem);
2404
2405 if (!ri) {
2406 /* On POWER7, use VRMA; on PPC970, give up */
2407 err = -EPERM;
2408 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
2409 pr_err("KVM: CPU requires an RMO\n");
2410 goto out_srcu;
2411 }
2412
2413 /* We can handle 4k, 64k or 16M pages in the VRMA */
2414 err = -EINVAL;
2415 if (!(psize == 0x1000 || psize == 0x10000 ||
2416 psize == 0x1000000))
2417 goto out_srcu;
2418
2419 /* Update VRMASD field in the LPCR */
2420 senc = slb_pgsize_encoding(psize);
2421 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
2422 (VRMA_VSID << SLB_VSID_SHIFT_1T);
2423 lpcr_mask = LPCR_VRMASD;
2424 /* the -4 is to account for senc values starting at 0x10 */
2425 lpcr = senc << (LPCR_VRMASD_SH - 4);
2426
2427 /* Create HPTEs in the hash page table for the VRMA */
2428 kvmppc_map_vrma(vcpu, memslot, porder);
2429
2430 } else {
2431 /* Set up to use an RMO region */
2432 rma_size = kvm_rma_pages;
2433 if (rma_size > memslot->npages)
2434 rma_size = memslot->npages;
2435 rma_size <<= PAGE_SHIFT;
2436 rmls = lpcr_rmls(rma_size);
2437 err = -EINVAL;
2438 if ((long)rmls < 0) {
2439 pr_err("KVM: Can't use RMA of 0x%lx bytes\n", rma_size);
2440 goto out_srcu;
2441 }
2442 atomic_inc(&ri->use_count);
2443 kvm->arch.rma = ri;
2444
2445 /* Update LPCR and RMOR */
2446 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
2447 /* PPC970; insert RMLS value (split field) in HID4 */
2448 lpcr_mask = (1ul << HID4_RMLS0_SH) |
2449 (3ul << HID4_RMLS2_SH) | HID4_RMOR;
2450 lpcr = ((rmls >> 2) << HID4_RMLS0_SH) |
2451 ((rmls & 3) << HID4_RMLS2_SH);
2452 /* RMOR is also in HID4 */
2453 lpcr |= ((ri->base_pfn >> (26 - PAGE_SHIFT)) & 0xffff)
2454 << HID4_RMOR_SH;
2455 } else {
2456 /* POWER7 */
2457 lpcr_mask = LPCR_VPM0 | LPCR_VRMA_L | LPCR_RMLS;
2458 lpcr = rmls << LPCR_RMLS_SH;
2459 kvm->arch.rmor = ri->base_pfn << PAGE_SHIFT;
2460 }
2461 pr_info("KVM: Using RMO at %lx size %lx (LPCR = %lx)\n",
2462 ri->base_pfn << PAGE_SHIFT, rma_size, lpcr);
2463
2464 /* Initialize phys addrs of pages in RMO */
2465 npages = kvm_rma_pages;
2466 porder = __ilog2(npages);
2467 physp = memslot->arch.slot_phys;
2468 if (physp) {
2469 if (npages > memslot->npages)
2470 npages = memslot->npages;
2471 spin_lock(&kvm->arch.slot_phys_lock);
2472 for (i = 0; i < npages; ++i)
2473 physp[i] = ((ri->base_pfn + i) << PAGE_SHIFT) +
2474 porder;
2475 spin_unlock(&kvm->arch.slot_phys_lock);
2476 }
2477 }
2478
2479 kvmppc_update_lpcr(kvm, lpcr, lpcr_mask);
2480
2481 /* Order updates to kvm->arch.lpcr etc. vs. rma_setup_done */
2482 smp_wmb();
2483 kvm->arch.rma_setup_done = 1;
2484 err = 0;
2485 out_srcu:
2486 srcu_read_unlock(&kvm->srcu, srcu_idx);
2487 out:
2488 mutex_unlock(&kvm->lock);
2489 return err;
2490
2491 up_out:
2492 up_read(¤t->mm->mmap_sem);
2493 goto out_srcu;
2494 }
2495
kvmppc_core_init_vm_hv(struct kvm * kvm)2496 static int kvmppc_core_init_vm_hv(struct kvm *kvm)
2497 {
2498 unsigned long lpcr, lpid;
2499
2500 /* Allocate the guest's logical partition ID */
2501
2502 lpid = kvmppc_alloc_lpid();
2503 if ((long)lpid < 0)
2504 return -ENOMEM;
2505 kvm->arch.lpid = lpid;
2506
2507 /*
2508 * Since we don't flush the TLB when tearing down a VM,
2509 * and this lpid might have previously been used,
2510 * make sure we flush on each core before running the new VM.
2511 */
2512 cpumask_setall(&kvm->arch.need_tlb_flush);
2513
2514 /* Start out with the default set of hcalls enabled */
2515 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
2516 sizeof(kvm->arch.enabled_hcalls));
2517
2518 kvm->arch.rma = NULL;
2519
2520 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
2521
2522 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
2523 /* PPC970; HID4 is effectively the LPCR */
2524 kvm->arch.host_lpid = 0;
2525 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_HID4);
2526 lpcr &= ~((3 << HID4_LPID1_SH) | (0xful << HID4_LPID5_SH));
2527 lpcr |= ((lpid >> 4) << HID4_LPID1_SH) |
2528 ((lpid & 0xf) << HID4_LPID5_SH);
2529 } else {
2530 /* POWER7; init LPCR for virtual RMA mode */
2531 kvm->arch.host_lpid = mfspr(SPRN_LPID);
2532 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
2533 lpcr &= LPCR_PECE | LPCR_LPES;
2534 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
2535 LPCR_VPM0 | LPCR_VPM1;
2536 kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
2537 (VRMA_VSID << SLB_VSID_SHIFT_1T);
2538 /* On POWER8 turn on online bit to enable PURR/SPURR */
2539 if (cpu_has_feature(CPU_FTR_ARCH_207S))
2540 lpcr |= LPCR_ONL;
2541 }
2542 kvm->arch.lpcr = lpcr;
2543
2544 kvm->arch.using_mmu_notifiers = !!cpu_has_feature(CPU_FTR_ARCH_206);
2545 spin_lock_init(&kvm->arch.slot_phys_lock);
2546
2547 /*
2548 * Track that we now have a HV mode VM active. This blocks secondary
2549 * CPU threads from coming online.
2550 */
2551 kvm_hv_vm_activated();
2552
2553 return 0;
2554 }
2555
kvmppc_free_vcores(struct kvm * kvm)2556 static void kvmppc_free_vcores(struct kvm *kvm)
2557 {
2558 long int i;
2559
2560 for (i = 0; i < KVM_MAX_VCORES; ++i) {
2561 if (kvm->arch.vcores[i] && kvm->arch.vcores[i]->mpp_buffer) {
2562 struct kvmppc_vcore *vc = kvm->arch.vcores[i];
2563 free_pages((unsigned long)vc->mpp_buffer,
2564 MPP_BUFFER_ORDER);
2565 }
2566 kfree(kvm->arch.vcores[i]);
2567 }
2568 kvm->arch.online_vcores = 0;
2569 }
2570
kvmppc_core_destroy_vm_hv(struct kvm * kvm)2571 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
2572 {
2573 kvm_hv_vm_deactivated();
2574
2575 kvmppc_free_vcores(kvm);
2576 if (kvm->arch.rma) {
2577 kvm_release_rma(kvm->arch.rma);
2578 kvm->arch.rma = NULL;
2579 }
2580
2581 kvmppc_free_hpt(kvm);
2582 }
2583
2584 /* We don't need to emulate any privileged instructions or dcbz */
kvmppc_core_emulate_op_hv(struct kvm_run * run,struct kvm_vcpu * vcpu,unsigned int inst,int * advance)2585 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
2586 unsigned int inst, int *advance)
2587 {
2588 return EMULATE_FAIL;
2589 }
2590
kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu * vcpu,int sprn,ulong spr_val)2591 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
2592 ulong spr_val)
2593 {
2594 return EMULATE_FAIL;
2595 }
2596
kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu * vcpu,int sprn,ulong * spr_val)2597 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
2598 ulong *spr_val)
2599 {
2600 return EMULATE_FAIL;
2601 }
2602
kvmppc_core_check_processor_compat_hv(void)2603 static int kvmppc_core_check_processor_compat_hv(void)
2604 {
2605 if (!cpu_has_feature(CPU_FTR_HVMODE))
2606 return -EIO;
2607 return 0;
2608 }
2609
kvm_arch_vm_ioctl_hv(struct file * filp,unsigned int ioctl,unsigned long arg)2610 static long kvm_arch_vm_ioctl_hv(struct file *filp,
2611 unsigned int ioctl, unsigned long arg)
2612 {
2613 struct kvm *kvm __maybe_unused = filp->private_data;
2614 void __user *argp = (void __user *)arg;
2615 long r;
2616
2617 switch (ioctl) {
2618
2619 case KVM_ALLOCATE_RMA: {
2620 struct kvm_allocate_rma rma;
2621 struct kvm *kvm = filp->private_data;
2622
2623 r = kvm_vm_ioctl_allocate_rma(kvm, &rma);
2624 if (r >= 0 && copy_to_user(argp, &rma, sizeof(rma)))
2625 r = -EFAULT;
2626 break;
2627 }
2628
2629 case KVM_PPC_ALLOCATE_HTAB: {
2630 u32 htab_order;
2631
2632 r = -EFAULT;
2633 if (get_user(htab_order, (u32 __user *)argp))
2634 break;
2635 r = kvmppc_alloc_reset_hpt(kvm, &htab_order);
2636 if (r)
2637 break;
2638 r = -EFAULT;
2639 if (put_user(htab_order, (u32 __user *)argp))
2640 break;
2641 r = 0;
2642 break;
2643 }
2644
2645 case KVM_PPC_GET_HTAB_FD: {
2646 struct kvm_get_htab_fd ghf;
2647
2648 r = -EFAULT;
2649 if (copy_from_user(&ghf, argp, sizeof(ghf)))
2650 break;
2651 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
2652 break;
2653 }
2654
2655 default:
2656 r = -ENOTTY;
2657 }
2658
2659 return r;
2660 }
2661
2662 /*
2663 * List of hcall numbers to enable by default.
2664 * For compatibility with old userspace, we enable by default
2665 * all hcalls that were implemented before the hcall-enabling
2666 * facility was added. Note this list should not include H_RTAS.
2667 */
2668 static unsigned int default_hcall_list[] = {
2669 H_REMOVE,
2670 H_ENTER,
2671 H_READ,
2672 H_PROTECT,
2673 H_BULK_REMOVE,
2674 H_GET_TCE,
2675 H_PUT_TCE,
2676 H_SET_DABR,
2677 H_SET_XDABR,
2678 H_CEDE,
2679 H_PROD,
2680 H_CONFER,
2681 H_REGISTER_VPA,
2682 #ifdef CONFIG_KVM_XICS
2683 H_EOI,
2684 H_CPPR,
2685 H_IPI,
2686 H_IPOLL,
2687 H_XIRR,
2688 H_XIRR_X,
2689 #endif
2690 0
2691 };
2692
init_default_hcalls(void)2693 static void init_default_hcalls(void)
2694 {
2695 int i;
2696 unsigned int hcall;
2697
2698 for (i = 0; default_hcall_list[i]; ++i) {
2699 hcall = default_hcall_list[i];
2700 WARN_ON(!kvmppc_hcall_impl_hv(hcall));
2701 __set_bit(hcall / 4, default_enabled_hcalls);
2702 }
2703 }
2704
2705 static struct kvmppc_ops kvm_ops_hv = {
2706 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
2707 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
2708 .get_one_reg = kvmppc_get_one_reg_hv,
2709 .set_one_reg = kvmppc_set_one_reg_hv,
2710 .vcpu_load = kvmppc_core_vcpu_load_hv,
2711 .vcpu_put = kvmppc_core_vcpu_put_hv,
2712 .set_msr = kvmppc_set_msr_hv,
2713 .vcpu_run = kvmppc_vcpu_run_hv,
2714 .vcpu_create = kvmppc_core_vcpu_create_hv,
2715 .vcpu_free = kvmppc_core_vcpu_free_hv,
2716 .check_requests = kvmppc_core_check_requests_hv,
2717 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv,
2718 .flush_memslot = kvmppc_core_flush_memslot_hv,
2719 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
2720 .commit_memory_region = kvmppc_core_commit_memory_region_hv,
2721 .unmap_hva = kvm_unmap_hva_hv,
2722 .unmap_hva_range = kvm_unmap_hva_range_hv,
2723 .age_hva = kvm_age_hva_hv,
2724 .test_age_hva = kvm_test_age_hva_hv,
2725 .set_spte_hva = kvm_set_spte_hva_hv,
2726 .mmu_destroy = kvmppc_mmu_destroy_hv,
2727 .free_memslot = kvmppc_core_free_memslot_hv,
2728 .create_memslot = kvmppc_core_create_memslot_hv,
2729 .init_vm = kvmppc_core_init_vm_hv,
2730 .destroy_vm = kvmppc_core_destroy_vm_hv,
2731 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
2732 .emulate_op = kvmppc_core_emulate_op_hv,
2733 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
2734 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
2735 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
2736 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv,
2737 .hcall_implemented = kvmppc_hcall_impl_hv,
2738 };
2739
kvmppc_book3s_init_hv(void)2740 static int kvmppc_book3s_init_hv(void)
2741 {
2742 int r;
2743 /*
2744 * FIXME!! Do we need to check on all cpus ?
2745 */
2746 r = kvmppc_core_check_processor_compat_hv();
2747 if (r < 0)
2748 return -ENODEV;
2749
2750 kvm_ops_hv.owner = THIS_MODULE;
2751 kvmppc_hv_ops = &kvm_ops_hv;
2752
2753 init_default_hcalls();
2754
2755 r = kvmppc_mmu_hv_init();
2756 return r;
2757 }
2758
kvmppc_book3s_exit_hv(void)2759 static void kvmppc_book3s_exit_hv(void)
2760 {
2761 kvmppc_hv_ops = NULL;
2762 }
2763
2764 module_init(kvmppc_book3s_init_hv);
2765 module_exit(kvmppc_book3s_exit_hv);
2766 MODULE_LICENSE("GPL");
2767 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2768 MODULE_ALIAS("devname:kvm");
2769