1 /*
2 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation.
7 */
8
9 #define pr_fmt(fmt) "xive-kvm: " fmt
10
11 #include <linux/kernel.h>
12 #include <linux/kvm_host.h>
13 #include <linux/err.h>
14 #include <linux/gfp.h>
15 #include <linux/spinlock.h>
16 #include <linux/delay.h>
17 #include <linux/percpu.h>
18 #include <linux/cpumask.h>
19 #include <asm/uaccess.h>
20 #include <asm/kvm_book3s.h>
21 #include <asm/kvm_ppc.h>
22 #include <asm/hvcall.h>
23 #include <asm/xics.h>
24 #include <asm/xive.h>
25 #include <asm/xive-regs.h>
26 #include <asm/debug.h>
27 #include <asm/debugfs.h>
28 #include <asm/time.h>
29 #include <asm/opal.h>
30
31 #include <linux/debugfs.h>
32 #include <linux/seq_file.h>
33
34 #include "book3s_xive.h"
35
36
37 /*
38 * Virtual mode variants of the hcalls for use on radix/radix
39 * with AIL. They require the VCPU's VP to be "pushed"
40 *
41 * We still instanciate them here because we use some of the
42 * generated utility functions as well in this file.
43 */
44 #define XIVE_RUNTIME_CHECKS
45 #define X_PFX xive_vm_
46 #define X_STATIC static
47 #define X_STAT_PFX stat_vm_
48 #define __x_tima xive_tima
49 #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio))
50 #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio))
51 #define __x_writeb __raw_writeb
52 #define __x_readw __raw_readw
53 #define __x_readq __raw_readq
54 #define __x_writeq __raw_writeq
55
56 #include "book3s_xive_template.c"
57
58 /*
59 * We leave a gap of a couple of interrupts in the queue to
60 * account for the IPI and additional safety guard.
61 */
62 #define XIVE_Q_GAP 2
63
64 /*
65 * This is a simple trigger for a generic XIVE IRQ. This must
66 * only be called for interrupts that support a trigger page
67 */
xive_irq_trigger(struct xive_irq_data * xd)68 static bool xive_irq_trigger(struct xive_irq_data *xd)
69 {
70 /* This should be only for MSIs */
71 if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
72 return false;
73
74 /* Those interrupts should always have a trigger page */
75 if (WARN_ON(!xd->trig_mmio))
76 return false;
77
78 out_be64(xd->trig_mmio, 0);
79
80 return true;
81 }
82
xive_esc_irq(int irq,void * data)83 static irqreturn_t xive_esc_irq(int irq, void *data)
84 {
85 struct kvm_vcpu *vcpu = data;
86
87 /* We use the existing H_PROD mechanism to wake up the target */
88 vcpu->arch.prodded = 1;
89 smp_mb();
90 if (vcpu->arch.ceded)
91 kvmppc_fast_vcpu_kick(vcpu);
92
93 return IRQ_HANDLED;
94 }
95
xive_attach_escalation(struct kvm_vcpu * vcpu,u8 prio)96 static int xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio)
97 {
98 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
99 struct xive_q *q = &xc->queues[prio];
100 char *name = NULL;
101 int rc;
102
103 /* Already there ? */
104 if (xc->esc_virq[prio])
105 return 0;
106
107 /* Hook up the escalation interrupt */
108 xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
109 if (!xc->esc_virq[prio]) {
110 pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
111 prio, xc->server_num);
112 return -EIO;
113 }
114
115 /*
116 * Future improvement: start with them disabled
117 * and handle DD2 and later scheme of merged escalation
118 * interrupts
119 */
120 name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
121 vcpu->kvm->arch.lpid, xc->server_num, prio);
122 if (!name) {
123 pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
124 prio, xc->server_num);
125 rc = -ENOMEM;
126 goto error;
127 }
128 rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
129 IRQF_NO_THREAD, name, vcpu);
130 if (rc) {
131 pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
132 prio, xc->server_num);
133 goto error;
134 }
135 xc->esc_virq_names[prio] = name;
136 return 0;
137 error:
138 irq_dispose_mapping(xc->esc_virq[prio]);
139 xc->esc_virq[prio] = 0;
140 kfree(name);
141 return rc;
142 }
143
xive_provision_queue(struct kvm_vcpu * vcpu,u8 prio)144 static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
145 {
146 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
147 struct kvmppc_xive *xive = xc->xive;
148 struct xive_q *q = &xc->queues[prio];
149 void *qpage;
150 int rc;
151
152 if (WARN_ON(q->qpage))
153 return 0;
154
155 /* Allocate the queue and retrieve infos on current node for now */
156 qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
157 if (!qpage) {
158 pr_err("Failed to allocate queue %d for VCPU %d\n",
159 prio, xc->server_num);
160 return -ENOMEM;;
161 }
162 memset(qpage, 0, 1 << xive->q_order);
163
164 /*
165 * Reconfigure the queue. This will set q->qpage only once the
166 * queue is fully configured. This is a requirement for prio 0
167 * as we will stop doing EOIs for every IPI as soon as we observe
168 * qpage being non-NULL, and instead will only EOI when we receive
169 * corresponding queue 0 entries
170 */
171 rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
172 xive->q_order, true);
173 if (rc)
174 pr_err("Failed to configure queue %d for VCPU %d\n",
175 prio, xc->server_num);
176 return rc;
177 }
178
179 /* Called with kvm_lock held */
xive_check_provisioning(struct kvm * kvm,u8 prio)180 static int xive_check_provisioning(struct kvm *kvm, u8 prio)
181 {
182 struct kvmppc_xive *xive = kvm->arch.xive;
183 struct kvm_vcpu *vcpu;
184 int i, rc;
185
186 lockdep_assert_held(&kvm->lock);
187
188 /* Already provisioned ? */
189 if (xive->qmap & (1 << prio))
190 return 0;
191
192 pr_devel("Provisioning prio... %d\n", prio);
193
194 /* Provision each VCPU and enable escalations */
195 kvm_for_each_vcpu(i, vcpu, kvm) {
196 if (!vcpu->arch.xive_vcpu)
197 continue;
198 rc = xive_provision_queue(vcpu, prio);
199 if (rc == 0)
200 xive_attach_escalation(vcpu, prio);
201 if (rc)
202 return rc;
203 }
204
205 /* Order previous stores and mark it as provisioned */
206 mb();
207 xive->qmap |= (1 << prio);
208 return 0;
209 }
210
xive_inc_q_pending(struct kvm * kvm,u32 server,u8 prio)211 static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
212 {
213 struct kvm_vcpu *vcpu;
214 struct kvmppc_xive_vcpu *xc;
215 struct xive_q *q;
216
217 /* Locate target server */
218 vcpu = kvmppc_xive_find_server(kvm, server);
219 if (!vcpu) {
220 pr_warn("%s: Can't find server %d\n", __func__, server);
221 return;
222 }
223 xc = vcpu->arch.xive_vcpu;
224 if (WARN_ON(!xc))
225 return;
226
227 q = &xc->queues[prio];
228 atomic_inc(&q->pending_count);
229 }
230
xive_try_pick_queue(struct kvm_vcpu * vcpu,u8 prio)231 static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
232 {
233 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
234 struct xive_q *q;
235 u32 max;
236
237 if (WARN_ON(!xc))
238 return -ENXIO;
239 if (!xc->valid)
240 return -ENXIO;
241
242 q = &xc->queues[prio];
243 if (WARN_ON(!q->qpage))
244 return -ENXIO;
245
246 /* Calculate max number of interrupts in that queue. */
247 max = (q->msk + 1) - XIVE_Q_GAP;
248 return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
249 }
250
xive_select_target(struct kvm * kvm,u32 * server,u8 prio)251 static int xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
252 {
253 struct kvm_vcpu *vcpu;
254 int i, rc;
255
256 /* Locate target server */
257 vcpu = kvmppc_xive_find_server(kvm, *server);
258 if (!vcpu) {
259 pr_devel("Can't find server %d\n", *server);
260 return -EINVAL;
261 }
262
263 pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
264
265 /* Try pick it */
266 rc = xive_try_pick_queue(vcpu, prio);
267 if (rc == 0)
268 return rc;
269
270 pr_devel(" .. failed, looking up candidate...\n");
271
272 /* Failed, pick another VCPU */
273 kvm_for_each_vcpu(i, vcpu, kvm) {
274 if (!vcpu->arch.xive_vcpu)
275 continue;
276 rc = xive_try_pick_queue(vcpu, prio);
277 if (rc == 0) {
278 *server = vcpu->arch.xive_vcpu->server_num;
279 pr_devel(" found on 0x%x/%d\n", *server, prio);
280 return rc;
281 }
282 }
283 pr_devel(" no available target !\n");
284
285 /* No available target ! */
286 return -EBUSY;
287 }
288
xive_lock_and_mask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)289 static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
290 struct kvmppc_xive_src_block *sb,
291 struct kvmppc_xive_irq_state *state)
292 {
293 struct xive_irq_data *xd;
294 u32 hw_num;
295 u8 old_prio;
296 u64 val;
297
298 /*
299 * Take the lock, set masked, try again if racing
300 * with H_EOI
301 */
302 for (;;) {
303 arch_spin_lock(&sb->lock);
304 old_prio = state->guest_priority;
305 state->guest_priority = MASKED;
306 mb();
307 if (!state->in_eoi)
308 break;
309 state->guest_priority = old_prio;
310 arch_spin_unlock(&sb->lock);
311 }
312
313 /* No change ? Bail */
314 if (old_prio == MASKED)
315 return old_prio;
316
317 /* Get the right irq */
318 kvmppc_xive_select_irq(state, &hw_num, &xd);
319
320 /*
321 * If the interrupt is marked as needing masking via
322 * firmware, we do it here. Firmware masking however
323 * is "lossy", it won't return the old p and q bits
324 * and won't set the interrupt to a state where it will
325 * record queued ones. If this is an issue we should do
326 * lazy masking instead.
327 *
328 * For now, we work around this in unmask by forcing
329 * an interrupt whenever we unmask a non-LSI via FW
330 * (if ever).
331 */
332 if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) {
333 xive_native_configure_irq(hw_num,
334 xive->vp_base + state->act_server,
335 MASKED, state->number);
336 /* set old_p so we can track if an H_EOI was done */
337 state->old_p = true;
338 state->old_q = false;
339 } else {
340 /* Set PQ to 10, return old P and old Q and remember them */
341 val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
342 state->old_p = !!(val & 2);
343 state->old_q = !!(val & 1);
344
345 /*
346 * Synchronize hardware to sensure the queues are updated
347 * when masking
348 */
349 xive_native_sync_source(hw_num);
350 }
351
352 return old_prio;
353 }
354
xive_lock_for_unmask(struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)355 static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
356 struct kvmppc_xive_irq_state *state)
357 {
358 /*
359 * Take the lock try again if racing with H_EOI
360 */
361 for (;;) {
362 arch_spin_lock(&sb->lock);
363 if (!state->in_eoi)
364 break;
365 arch_spin_unlock(&sb->lock);
366 }
367 }
368
xive_finish_unmask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state,u8 prio)369 static void xive_finish_unmask(struct kvmppc_xive *xive,
370 struct kvmppc_xive_src_block *sb,
371 struct kvmppc_xive_irq_state *state,
372 u8 prio)
373 {
374 struct xive_irq_data *xd;
375 u32 hw_num;
376
377 /* If we aren't changing a thing, move on */
378 if (state->guest_priority != MASKED)
379 goto bail;
380
381 /* Get the right irq */
382 kvmppc_xive_select_irq(state, &hw_num, &xd);
383
384 /*
385 * See command in xive_lock_and_mask() concerning masking
386 * via firmware.
387 */
388 if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) {
389 xive_native_configure_irq(hw_num,
390 xive->vp_base + state->act_server,
391 state->act_priority, state->number);
392 /* If an EOI is needed, do it here */
393 if (!state->old_p)
394 xive_vm_source_eoi(hw_num, xd);
395 /* If this is not an LSI, force a trigger */
396 if (!(xd->flags & OPAL_XIVE_IRQ_LSI))
397 xive_irq_trigger(xd);
398 goto bail;
399 }
400
401 /* Old Q set, set PQ to 11 */
402 if (state->old_q)
403 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
404
405 /*
406 * If not old P, then perform an "effective" EOI,
407 * on the source. This will handle the cases where
408 * FW EOI is needed.
409 */
410 if (!state->old_p)
411 xive_vm_source_eoi(hw_num, xd);
412
413 /* Synchronize ordering and mark unmasked */
414 mb();
415 bail:
416 state->guest_priority = prio;
417 }
418
419 /*
420 * Target an interrupt to a given server/prio, this will fallback
421 * to another server if necessary and perform the HW targetting
422 * updates as needed
423 *
424 * NOTE: Must be called with the state lock held
425 */
xive_target_interrupt(struct kvm * kvm,struct kvmppc_xive_irq_state * state,u32 server,u8 prio)426 static int xive_target_interrupt(struct kvm *kvm,
427 struct kvmppc_xive_irq_state *state,
428 u32 server, u8 prio)
429 {
430 struct kvmppc_xive *xive = kvm->arch.xive;
431 u32 hw_num;
432 int rc;
433
434 /*
435 * This will return a tentative server and actual
436 * priority. The count for that new target will have
437 * already been incremented.
438 */
439 rc = xive_select_target(kvm, &server, prio);
440
441 /*
442 * We failed to find a target ? Not much we can do
443 * at least until we support the GIQ.
444 */
445 if (rc)
446 return rc;
447
448 /*
449 * Increment the old queue pending count if there
450 * was one so that the old queue count gets adjusted later
451 * when observed to be empty.
452 */
453 if (state->act_priority != MASKED)
454 xive_inc_q_pending(kvm,
455 state->act_server,
456 state->act_priority);
457 /*
458 * Update state and HW
459 */
460 state->act_priority = prio;
461 state->act_server = server;
462
463 /* Get the right irq */
464 kvmppc_xive_select_irq(state, &hw_num, NULL);
465
466 return xive_native_configure_irq(hw_num,
467 xive->vp_base + server,
468 prio, state->number);
469 }
470
471 /*
472 * Targetting rules: In order to avoid losing track of
473 * pending interrupts accross mask and unmask, which would
474 * allow queue overflows, we implement the following rules:
475 *
476 * - Unless it was never enabled (or we run out of capacity)
477 * an interrupt is always targetted at a valid server/queue
478 * pair even when "masked" by the guest. This pair tends to
479 * be the last one used but it can be changed under some
480 * circumstances. That allows us to separate targetting
481 * from masking, we only handle accounting during (re)targetting,
482 * this also allows us to let an interrupt drain into its target
483 * queue after masking, avoiding complex schemes to remove
484 * interrupts out of remote processor queues.
485 *
486 * - When masking, we set PQ to 10 and save the previous value
487 * of P and Q.
488 *
489 * - When unmasking, if saved Q was set, we set PQ to 11
490 * otherwise we leave PQ to the HW state which will be either
491 * 10 if nothing happened or 11 if the interrupt fired while
492 * masked. Effectively we are OR'ing the previous Q into the
493 * HW Q.
494 *
495 * Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
496 * which will unmask the interrupt and shoot a new one if Q was
497 * set.
498 *
499 * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
500 * effectively meaning an H_EOI from the guest is still expected
501 * for that interrupt).
502 *
503 * - If H_EOI occurs while masked, we clear the saved P.
504 *
505 * - When changing target, we account on the new target and
506 * increment a separate "pending" counter on the old one.
507 * This pending counter will be used to decrement the old
508 * target's count when its queue has been observed empty.
509 */
510
kvmppc_xive_set_xive(struct kvm * kvm,u32 irq,u32 server,u32 priority)511 int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
512 u32 priority)
513 {
514 struct kvmppc_xive *xive = kvm->arch.xive;
515 struct kvmppc_xive_src_block *sb;
516 struct kvmppc_xive_irq_state *state;
517 u8 new_act_prio;
518 int rc = 0;
519 u16 idx;
520
521 if (!xive)
522 return -ENODEV;
523
524 pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
525 irq, server, priority);
526
527 /* First, check provisioning of queues */
528 if (priority != MASKED)
529 rc = xive_check_provisioning(xive->kvm,
530 xive_prio_from_guest(priority));
531 if (rc) {
532 pr_devel(" provisioning failure %d !\n", rc);
533 return rc;
534 }
535
536 sb = kvmppc_xive_find_source(xive, irq, &idx);
537 if (!sb)
538 return -EINVAL;
539 state = &sb->irq_state[idx];
540
541 /*
542 * We first handle masking/unmasking since the locking
543 * might need to be retried due to EOIs, we'll handle
544 * targetting changes later. These functions will return
545 * with the SB lock held.
546 *
547 * xive_lock_and_mask() will also set state->guest_priority
548 * but won't otherwise change other fields of the state.
549 *
550 * xive_lock_for_unmask will not actually unmask, this will
551 * be done later by xive_finish_unmask() once the targetting
552 * has been done, so we don't try to unmask an interrupt
553 * that hasn't yet been targetted.
554 */
555 if (priority == MASKED)
556 xive_lock_and_mask(xive, sb, state);
557 else
558 xive_lock_for_unmask(sb, state);
559
560
561 /*
562 * Then we handle targetting.
563 *
564 * First calculate a new "actual priority"
565 */
566 new_act_prio = state->act_priority;
567 if (priority != MASKED)
568 new_act_prio = xive_prio_from_guest(priority);
569
570 pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
571 new_act_prio, state->act_server, state->act_priority);
572
573 /*
574 * Then check if we actually need to change anything,
575 *
576 * The condition for re-targetting the interrupt is that
577 * we have a valid new priority (new_act_prio is not 0xff)
578 * and either the server or the priority changed.
579 *
580 * Note: If act_priority was ff and the new priority is
581 * also ff, we don't do anything and leave the interrupt
582 * untargetted. An attempt of doing an int_on on an
583 * untargetted interrupt will fail. If that is a problem
584 * we could initialize interrupts with valid default
585 */
586
587 if (new_act_prio != MASKED &&
588 (state->act_server != server ||
589 state->act_priority != new_act_prio))
590 rc = xive_target_interrupt(kvm, state, server, new_act_prio);
591
592 /*
593 * Perform the final unmasking of the interrupt source
594 * if necessary
595 */
596 if (priority != MASKED)
597 xive_finish_unmask(xive, sb, state, priority);
598
599 /*
600 * Finally Update saved_priority to match. Only int_on/off
601 * set this field to a different value.
602 */
603 state->saved_priority = priority;
604
605 arch_spin_unlock(&sb->lock);
606 return rc;
607 }
608
kvmppc_xive_get_xive(struct kvm * kvm,u32 irq,u32 * server,u32 * priority)609 int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
610 u32 *priority)
611 {
612 struct kvmppc_xive *xive = kvm->arch.xive;
613 struct kvmppc_xive_src_block *sb;
614 struct kvmppc_xive_irq_state *state;
615 u16 idx;
616
617 if (!xive)
618 return -ENODEV;
619
620 sb = kvmppc_xive_find_source(xive, irq, &idx);
621 if (!sb)
622 return -EINVAL;
623 state = &sb->irq_state[idx];
624 arch_spin_lock(&sb->lock);
625 *server = state->act_server;
626 *priority = state->guest_priority;
627 arch_spin_unlock(&sb->lock);
628
629 return 0;
630 }
631
kvmppc_xive_int_on(struct kvm * kvm,u32 irq)632 int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
633 {
634 struct kvmppc_xive *xive = kvm->arch.xive;
635 struct kvmppc_xive_src_block *sb;
636 struct kvmppc_xive_irq_state *state;
637 u16 idx;
638
639 if (!xive)
640 return -ENODEV;
641
642 sb = kvmppc_xive_find_source(xive, irq, &idx);
643 if (!sb)
644 return -EINVAL;
645 state = &sb->irq_state[idx];
646
647 pr_devel("int_on(irq=0x%x)\n", irq);
648
649 /*
650 * Check if interrupt was not targetted
651 */
652 if (state->act_priority == MASKED) {
653 pr_devel("int_on on untargetted interrupt\n");
654 return -EINVAL;
655 }
656
657 /* If saved_priority is 0xff, do nothing */
658 if (state->saved_priority == MASKED)
659 return 0;
660
661 /*
662 * Lock and unmask it.
663 */
664 xive_lock_for_unmask(sb, state);
665 xive_finish_unmask(xive, sb, state, state->saved_priority);
666 arch_spin_unlock(&sb->lock);
667
668 return 0;
669 }
670
kvmppc_xive_int_off(struct kvm * kvm,u32 irq)671 int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
672 {
673 struct kvmppc_xive *xive = kvm->arch.xive;
674 struct kvmppc_xive_src_block *sb;
675 struct kvmppc_xive_irq_state *state;
676 u16 idx;
677
678 if (!xive)
679 return -ENODEV;
680
681 sb = kvmppc_xive_find_source(xive, irq, &idx);
682 if (!sb)
683 return -EINVAL;
684 state = &sb->irq_state[idx];
685
686 pr_devel("int_off(irq=0x%x)\n", irq);
687
688 /*
689 * Lock and mask
690 */
691 state->saved_priority = xive_lock_and_mask(xive, sb, state);
692 arch_spin_unlock(&sb->lock);
693
694 return 0;
695 }
696
xive_restore_pending_irq(struct kvmppc_xive * xive,u32 irq)697 static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
698 {
699 struct kvmppc_xive_src_block *sb;
700 struct kvmppc_xive_irq_state *state;
701 u16 idx;
702
703 sb = kvmppc_xive_find_source(xive, irq, &idx);
704 if (!sb)
705 return false;
706 state = &sb->irq_state[idx];
707 if (!state->valid)
708 return false;
709
710 /*
711 * Trigger the IPI. This assumes we never restore a pass-through
712 * interrupt which should be safe enough
713 */
714 xive_irq_trigger(&state->ipi_data);
715
716 return true;
717 }
718
kvmppc_xive_get_icp(struct kvm_vcpu * vcpu)719 u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
720 {
721 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
722
723 if (!xc)
724 return 0;
725
726 /* Return the per-cpu state for state saving/migration */
727 return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
728 (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
729 (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
730 }
731
kvmppc_xive_set_icp(struct kvm_vcpu * vcpu,u64 icpval)732 int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
733 {
734 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
735 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
736 u8 cppr, mfrr;
737 u32 xisr;
738
739 if (!xc || !xive)
740 return -ENOENT;
741
742 /* Grab individual state fields. We don't use pending_pri */
743 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
744 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
745 KVM_REG_PPC_ICP_XISR_MASK;
746 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
747
748 pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
749 xc->server_num, cppr, mfrr, xisr);
750
751 /*
752 * We can't update the state of a "pushed" VCPU, but that
753 * shouldn't happen.
754 */
755 if (WARN_ON(vcpu->arch.xive_pushed))
756 return -EIO;
757
758 /* Update VCPU HW saved state */
759 vcpu->arch.xive_saved_state.cppr = cppr;
760 xc->hw_cppr = xc->cppr = cppr;
761
762 /*
763 * Update MFRR state. If it's not 0xff, we mark the VCPU as
764 * having a pending MFRR change, which will re-evaluate the
765 * target. The VCPU will thus potentially get a spurious
766 * interrupt but that's not a big deal.
767 */
768 xc->mfrr = mfrr;
769 if (mfrr < cppr)
770 xive_irq_trigger(&xc->vp_ipi_data);
771
772 /*
773 * Now saved XIRR is "interesting". It means there's something in
774 * the legacy "1 element" queue... for an IPI we simply ignore it,
775 * as the MFRR restore will handle that. For anything else we need
776 * to force a resend of the source.
777 * However the source may not have been setup yet. If that's the
778 * case, we keep that info and increment a counter in the xive to
779 * tell subsequent xive_set_source() to go look.
780 */
781 if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
782 xc->delayed_irq = xisr;
783 xive->delayed_irqs++;
784 pr_devel(" xisr restore delayed\n");
785 }
786
787 return 0;
788 }
789
kvmppc_xive_set_mapped(struct kvm * kvm,unsigned long guest_irq,struct irq_desc * host_desc)790 int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
791 struct irq_desc *host_desc)
792 {
793 struct kvmppc_xive *xive = kvm->arch.xive;
794 struct kvmppc_xive_src_block *sb;
795 struct kvmppc_xive_irq_state *state;
796 struct irq_data *host_data = irq_desc_get_irq_data(host_desc);
797 unsigned int host_irq = irq_desc_get_irq(host_desc);
798 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
799 u16 idx;
800 u8 prio;
801 int rc;
802
803 if (!xive)
804 return -ENODEV;
805
806 pr_devel("set_mapped girq 0x%lx host HW irq 0x%x...\n",guest_irq, hw_irq);
807
808 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
809 if (!sb)
810 return -EINVAL;
811 state = &sb->irq_state[idx];
812
813 /*
814 * Mark the passed-through interrupt as going to a VCPU,
815 * this will prevent further EOIs and similar operations
816 * from the XIVE code. It will also mask the interrupt
817 * to either PQ=10 or 11 state, the latter if the interrupt
818 * is pending. This will allow us to unmask or retrigger it
819 * after routing it to the guest with a simple EOI.
820 *
821 * The "state" argument is a "token", all it needs is to be
822 * non-NULL to switch to passed-through or NULL for the
823 * other way around. We may not yet have an actual VCPU
824 * target here and we don't really care.
825 */
826 rc = irq_set_vcpu_affinity(host_irq, state);
827 if (rc) {
828 pr_err("Failed to set VCPU affinity for irq %d\n", host_irq);
829 return rc;
830 }
831
832 /*
833 * Mask and read state of IPI. We need to know if its P bit
834 * is set as that means it's potentially already using a
835 * queue entry in the target
836 */
837 prio = xive_lock_and_mask(xive, sb, state);
838 pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
839 state->old_p, state->old_q);
840
841 /* Turn the IPI hard off */
842 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
843
844 /* Grab info about irq */
845 state->pt_number = hw_irq;
846 state->pt_data = irq_data_get_irq_handler_data(host_data);
847
848 /*
849 * Configure the IRQ to match the existing configuration of
850 * the IPI if it was already targetted. Otherwise this will
851 * mask the interrupt in a lossy way (act_priority is 0xff)
852 * which is fine for a never started interrupt.
853 */
854 xive_native_configure_irq(hw_irq,
855 xive->vp_base + state->act_server,
856 state->act_priority, state->number);
857
858 /*
859 * We do an EOI to enable the interrupt (and retrigger if needed)
860 * if the guest has the interrupt unmasked and the P bit was *not*
861 * set in the IPI. If it was set, we know a slot may still be in
862 * use in the target queue thus we have to wait for a guest
863 * originated EOI
864 */
865 if (prio != MASKED && !state->old_p)
866 xive_vm_source_eoi(hw_irq, state->pt_data);
867
868 /* Clear old_p/old_q as they are no longer relevant */
869 state->old_p = state->old_q = false;
870
871 /* Restore guest prio (unlocks EOI) */
872 mb();
873 state->guest_priority = prio;
874 arch_spin_unlock(&sb->lock);
875
876 return 0;
877 }
878 EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
879
kvmppc_xive_clr_mapped(struct kvm * kvm,unsigned long guest_irq,struct irq_desc * host_desc)880 int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
881 struct irq_desc *host_desc)
882 {
883 struct kvmppc_xive *xive = kvm->arch.xive;
884 struct kvmppc_xive_src_block *sb;
885 struct kvmppc_xive_irq_state *state;
886 unsigned int host_irq = irq_desc_get_irq(host_desc);
887 u16 idx;
888 u8 prio;
889 int rc;
890
891 if (!xive)
892 return -ENODEV;
893
894 pr_devel("clr_mapped girq 0x%lx...\n", guest_irq);
895
896 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
897 if (!sb)
898 return -EINVAL;
899 state = &sb->irq_state[idx];
900
901 /*
902 * Mask and read state of IRQ. We need to know if its P bit
903 * is set as that means it's potentially already using a
904 * queue entry in the target
905 */
906 prio = xive_lock_and_mask(xive, sb, state);
907 pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
908 state->old_p, state->old_q);
909
910 /*
911 * If old_p is set, the interrupt is pending, we switch it to
912 * PQ=11. This will force a resend in the host so the interrupt
913 * isn't lost to whatver host driver may pick it up
914 */
915 if (state->old_p)
916 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
917
918 /* Release the passed-through interrupt to the host */
919 rc = irq_set_vcpu_affinity(host_irq, NULL);
920 if (rc) {
921 pr_err("Failed to clr VCPU affinity for irq %d\n", host_irq);
922 return rc;
923 }
924
925 /* Forget about the IRQ */
926 state->pt_number = 0;
927 state->pt_data = NULL;
928
929 /* Reconfigure the IPI */
930 xive_native_configure_irq(state->ipi_number,
931 xive->vp_base + state->act_server,
932 state->act_priority, state->number);
933
934 /*
935 * If old_p is set (we have a queue entry potentially
936 * occupied) or the interrupt is masked, we set the IPI
937 * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
938 */
939 if (prio == MASKED || state->old_p)
940 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
941 else
942 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
943
944 /* Restore guest prio (unlocks EOI) */
945 mb();
946 state->guest_priority = prio;
947 arch_spin_unlock(&sb->lock);
948
949 return 0;
950 }
951 EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
952
kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu * vcpu)953 static void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
954 {
955 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
956 struct kvm *kvm = vcpu->kvm;
957 struct kvmppc_xive *xive = kvm->arch.xive;
958 int i, j;
959
960 for (i = 0; i <= xive->max_sbid; i++) {
961 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
962
963 if (!sb)
964 continue;
965 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
966 struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
967
968 if (!state->valid)
969 continue;
970 if (state->act_priority == MASKED)
971 continue;
972 if (state->act_server != xc->server_num)
973 continue;
974
975 /* Clean it up */
976 arch_spin_lock(&sb->lock);
977 state->act_priority = MASKED;
978 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
979 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
980 if (state->pt_number) {
981 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
982 xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
983 }
984 arch_spin_unlock(&sb->lock);
985 }
986 }
987 }
988
kvmppc_xive_cleanup_vcpu(struct kvm_vcpu * vcpu)989 void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
990 {
991 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
992 struct kvmppc_xive *xive = xc->xive;
993 int i;
994
995 pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
996
997 /* Ensure no interrupt is still routed to that VP */
998 xc->valid = false;
999 kvmppc_xive_disable_vcpu_interrupts(vcpu);
1000
1001 /* Mask the VP IPI */
1002 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1003
1004 /* Free escalations */
1005 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1006 if (xc->esc_virq[i]) {
1007 free_irq(xc->esc_virq[i], vcpu);
1008 irq_dispose_mapping(xc->esc_virq[i]);
1009 kfree(xc->esc_virq_names[i]);
1010 }
1011 }
1012
1013 /* Disable the VP */
1014 xive_native_disable_vp(xc->vp_id);
1015
1016 /* Free the queues */
1017 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1018 struct xive_q *q = &xc->queues[i];
1019
1020 xive_native_disable_queue(xc->vp_id, q, i);
1021 if (q->qpage) {
1022 free_pages((unsigned long)q->qpage,
1023 xive->q_page_order);
1024 q->qpage = NULL;
1025 }
1026 }
1027
1028 /* Free the IPI */
1029 if (xc->vp_ipi) {
1030 xive_cleanup_irq_data(&xc->vp_ipi_data);
1031 xive_native_free_irq(xc->vp_ipi);
1032 }
1033 /* Free the VP */
1034 kfree(xc);
1035 }
1036
kvmppc_xive_connect_vcpu(struct kvm_device * dev,struct kvm_vcpu * vcpu,u32 cpu)1037 int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
1038 struct kvm_vcpu *vcpu, u32 cpu)
1039 {
1040 struct kvmppc_xive *xive = dev->private;
1041 struct kvmppc_xive_vcpu *xc;
1042 int i, r = -EBUSY;
1043
1044 pr_devel("connect_vcpu(cpu=%d)\n", cpu);
1045
1046 if (dev->ops != &kvm_xive_ops) {
1047 pr_devel("Wrong ops !\n");
1048 return -EPERM;
1049 }
1050 if (xive->kvm != vcpu->kvm)
1051 return -EPERM;
1052 if (vcpu->arch.irq_type)
1053 return -EBUSY;
1054 if (kvmppc_xive_find_server(vcpu->kvm, cpu)) {
1055 pr_devel("Duplicate !\n");
1056 return -EEXIST;
1057 }
1058 if (cpu >= KVM_MAX_VCPUS) {
1059 pr_devel("Out of bounds !\n");
1060 return -EINVAL;
1061 }
1062 xc = kzalloc(sizeof(*xc), GFP_KERNEL);
1063 if (!xc)
1064 return -ENOMEM;
1065
1066 /* We need to synchronize with queue provisioning */
1067 mutex_lock(&vcpu->kvm->lock);
1068 vcpu->arch.xive_vcpu = xc;
1069 xc->xive = xive;
1070 xc->vcpu = vcpu;
1071 xc->server_num = cpu;
1072 xc->vp_id = xive->vp_base + cpu;
1073 xc->mfrr = 0xff;
1074 xc->valid = true;
1075
1076 r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
1077 if (r)
1078 goto bail;
1079
1080 /* Configure VCPU fields for use by assembly push/pull */
1081 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
1082 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
1083
1084 /* Allocate IPI */
1085 xc->vp_ipi = xive_native_alloc_irq();
1086 if (!xc->vp_ipi) {
1087 r = -EIO;
1088 goto bail;
1089 }
1090 pr_devel(" IPI=0x%x\n", xc->vp_ipi);
1091
1092 r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
1093 if (r)
1094 goto bail;
1095
1096 /*
1097 * Initialize queues. Initially we set them all for no queueing
1098 * and we enable escalation for queue 0 only which we'll use for
1099 * our mfrr change notifications. If the VCPU is hot-plugged, we
1100 * do handle provisioning however.
1101 */
1102 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1103 struct xive_q *q = &xc->queues[i];
1104
1105 /* Is queue already enabled ? Provision it */
1106 if (xive->qmap & (1 << i)) {
1107 r = xive_provision_queue(vcpu, i);
1108 if (r == 0)
1109 xive_attach_escalation(vcpu, i);
1110 if (r)
1111 goto bail;
1112 } else {
1113 r = xive_native_configure_queue(xc->vp_id,
1114 q, i, NULL, 0, true);
1115 if (r) {
1116 pr_err("Failed to configure queue %d for VCPU %d\n",
1117 i, cpu);
1118 goto bail;
1119 }
1120 }
1121 }
1122
1123 /* If not done above, attach priority 0 escalation */
1124 r = xive_attach_escalation(vcpu, 0);
1125 if (r)
1126 goto bail;
1127
1128 /* Enable the VP */
1129 r = xive_native_enable_vp(xc->vp_id);
1130 if (r)
1131 goto bail;
1132
1133 /* Route the IPI */
1134 r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
1135 if (!r)
1136 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
1137
1138 bail:
1139 mutex_unlock(&vcpu->kvm->lock);
1140 if (r) {
1141 kvmppc_xive_cleanup_vcpu(vcpu);
1142 return r;
1143 }
1144
1145 vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
1146 return 0;
1147 }
1148
1149 /*
1150 * Scanning of queues before/after migration save
1151 */
xive_pre_save_set_queued(struct kvmppc_xive * xive,u32 irq)1152 static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
1153 {
1154 struct kvmppc_xive_src_block *sb;
1155 struct kvmppc_xive_irq_state *state;
1156 u16 idx;
1157
1158 sb = kvmppc_xive_find_source(xive, irq, &idx);
1159 if (!sb)
1160 return;
1161
1162 state = &sb->irq_state[idx];
1163
1164 /* Some sanity checking */
1165 if (!state->valid) {
1166 pr_err("invalid irq 0x%x in cpu queue!\n", irq);
1167 return;
1168 }
1169
1170 /*
1171 * If the interrupt is in a queue it should have P set.
1172 * We warn so that gets reported. A backtrace isn't useful
1173 * so no need to use a WARN_ON.
1174 */
1175 if (!state->saved_p)
1176 pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
1177
1178 /* Set flag */
1179 state->in_queue = true;
1180 }
1181
xive_pre_save_mask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)1182 static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
1183 struct kvmppc_xive_src_block *sb,
1184 u32 irq)
1185 {
1186 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
1187
1188 if (!state->valid)
1189 return;
1190
1191 /* Mask and save state, this will also sync HW queues */
1192 state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
1193
1194 /* Transfer P and Q */
1195 state->saved_p = state->old_p;
1196 state->saved_q = state->old_q;
1197
1198 /* Unlock */
1199 arch_spin_unlock(&sb->lock);
1200 }
1201
xive_pre_save_unmask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)1202 static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
1203 struct kvmppc_xive_src_block *sb,
1204 u32 irq)
1205 {
1206 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
1207
1208 if (!state->valid)
1209 return;
1210
1211 /*
1212 * Lock / exclude EOI (not technically necessary if the
1213 * guest isn't running concurrently. If this becomes a
1214 * performance issue we can probably remove the lock.
1215 */
1216 xive_lock_for_unmask(sb, state);
1217
1218 /* Restore mask/prio if it wasn't masked */
1219 if (state->saved_scan_prio != MASKED)
1220 xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
1221
1222 /* Unlock */
1223 arch_spin_unlock(&sb->lock);
1224 }
1225
xive_pre_save_queue(struct kvmppc_xive * xive,struct xive_q * q)1226 static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
1227 {
1228 u32 idx = q->idx;
1229 u32 toggle = q->toggle;
1230 u32 irq;
1231
1232 do {
1233 irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
1234 if (irq > XICS_IPI)
1235 xive_pre_save_set_queued(xive, irq);
1236 } while(irq);
1237 }
1238
xive_pre_save_scan(struct kvmppc_xive * xive)1239 static void xive_pre_save_scan(struct kvmppc_xive *xive)
1240 {
1241 struct kvm_vcpu *vcpu = NULL;
1242 int i, j;
1243
1244 /*
1245 * See comment in xive_get_source() about how this
1246 * work. Collect a stable state for all interrupts
1247 */
1248 for (i = 0; i <= xive->max_sbid; i++) {
1249 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1250 if (!sb)
1251 continue;
1252 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1253 xive_pre_save_mask_irq(xive, sb, j);
1254 }
1255
1256 /* Then scan the queues and update the "in_queue" flag */
1257 kvm_for_each_vcpu(i, vcpu, xive->kvm) {
1258 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1259 if (!xc)
1260 continue;
1261 for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
1262 if (xc->queues[j].qpage)
1263 xive_pre_save_queue(xive, &xc->queues[j]);
1264 }
1265 }
1266
1267 /* Finally restore interrupt states */
1268 for (i = 0; i <= xive->max_sbid; i++) {
1269 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1270 if (!sb)
1271 continue;
1272 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1273 xive_pre_save_unmask_irq(xive, sb, j);
1274 }
1275 }
1276
xive_post_save_scan(struct kvmppc_xive * xive)1277 static void xive_post_save_scan(struct kvmppc_xive *xive)
1278 {
1279 u32 i, j;
1280
1281 /* Clear all the in_queue flags */
1282 for (i = 0; i <= xive->max_sbid; i++) {
1283 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1284 if (!sb)
1285 continue;
1286 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1287 sb->irq_state[j].in_queue = false;
1288 }
1289
1290 /* Next get_source() will do a new scan */
1291 xive->saved_src_count = 0;
1292 }
1293
1294 /*
1295 * This returns the source configuration and state to user space.
1296 */
xive_get_source(struct kvmppc_xive * xive,long irq,u64 addr)1297 static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
1298 {
1299 struct kvmppc_xive_src_block *sb;
1300 struct kvmppc_xive_irq_state *state;
1301 u64 __user *ubufp = (u64 __user *) addr;
1302 u64 val, prio;
1303 u16 idx;
1304
1305 sb = kvmppc_xive_find_source(xive, irq, &idx);
1306 if (!sb)
1307 return -ENOENT;
1308
1309 state = &sb->irq_state[idx];
1310
1311 if (!state->valid)
1312 return -ENOENT;
1313
1314 pr_devel("get_source(%ld)...\n", irq);
1315
1316 /*
1317 * So to properly save the state into something that looks like a
1318 * XICS migration stream we cannot treat interrupts individually.
1319 *
1320 * We need, instead, mask them all (& save their previous PQ state)
1321 * to get a stable state in the HW, then sync them to ensure that
1322 * any interrupt that had already fired hits its queue, and finally
1323 * scan all the queues to collect which interrupts are still present
1324 * in the queues, so we can set the "pending" flag on them and
1325 * they can be resent on restore.
1326 *
1327 * So we do it all when the "first" interrupt gets saved, all the
1328 * state is collected at that point, the rest of xive_get_source()
1329 * will merely collect and convert that state to the expected
1330 * userspace bit mask.
1331 */
1332 if (xive->saved_src_count == 0)
1333 xive_pre_save_scan(xive);
1334 xive->saved_src_count++;
1335
1336 /* Convert saved state into something compatible with xics */
1337 val = state->act_server;
1338 prio = state->saved_scan_prio;
1339
1340 if (prio == MASKED) {
1341 val |= KVM_XICS_MASKED;
1342 prio = state->saved_priority;
1343 }
1344 val |= prio << KVM_XICS_PRIORITY_SHIFT;
1345 if (state->lsi) {
1346 val |= KVM_XICS_LEVEL_SENSITIVE;
1347 if (state->saved_p)
1348 val |= KVM_XICS_PENDING;
1349 } else {
1350 if (state->saved_p)
1351 val |= KVM_XICS_PRESENTED;
1352
1353 if (state->saved_q)
1354 val |= KVM_XICS_QUEUED;
1355
1356 /*
1357 * We mark it pending (which will attempt a re-delivery)
1358 * if we are in a queue *or* we were masked and had
1359 * Q set which is equivalent to the XICS "masked pending"
1360 * state
1361 */
1362 if (state->in_queue || (prio == MASKED && state->saved_q))
1363 val |= KVM_XICS_PENDING;
1364 }
1365
1366 /*
1367 * If that was the last interrupt saved, reset the
1368 * in_queue flags
1369 */
1370 if (xive->saved_src_count == xive->src_count)
1371 xive_post_save_scan(xive);
1372
1373 /* Copy the result to userspace */
1374 if (put_user(val, ubufp))
1375 return -EFAULT;
1376
1377 return 0;
1378 }
1379
xive_create_src_block(struct kvmppc_xive * xive,int irq)1380 static struct kvmppc_xive_src_block *xive_create_src_block(struct kvmppc_xive *xive,
1381 int irq)
1382 {
1383 struct kvm *kvm = xive->kvm;
1384 struct kvmppc_xive_src_block *sb;
1385 int i, bid;
1386
1387 bid = irq >> KVMPPC_XICS_ICS_SHIFT;
1388
1389 mutex_lock(&kvm->lock);
1390
1391 /* block already exists - somebody else got here first */
1392 if (xive->src_blocks[bid])
1393 goto out;
1394
1395 /* Create the ICS */
1396 sb = kzalloc(sizeof(*sb), GFP_KERNEL);
1397 if (!sb)
1398 goto out;
1399
1400 sb->id = bid;
1401
1402 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
1403 sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
1404 sb->irq_state[i].guest_priority = MASKED;
1405 sb->irq_state[i].saved_priority = MASKED;
1406 sb->irq_state[i].act_priority = MASKED;
1407 }
1408 smp_wmb();
1409 xive->src_blocks[bid] = sb;
1410
1411 if (bid > xive->max_sbid)
1412 xive->max_sbid = bid;
1413
1414 out:
1415 mutex_unlock(&kvm->lock);
1416 return xive->src_blocks[bid];
1417 }
1418
xive_check_delayed_irq(struct kvmppc_xive * xive,u32 irq)1419 static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
1420 {
1421 struct kvm *kvm = xive->kvm;
1422 struct kvm_vcpu *vcpu = NULL;
1423 int i;
1424
1425 kvm_for_each_vcpu(i, vcpu, kvm) {
1426 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1427
1428 if (!xc)
1429 continue;
1430
1431 if (xc->delayed_irq == irq) {
1432 xc->delayed_irq = 0;
1433 xive->delayed_irqs--;
1434 return true;
1435 }
1436 }
1437 return false;
1438 }
1439
xive_set_source(struct kvmppc_xive * xive,long irq,u64 addr)1440 static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
1441 {
1442 struct kvmppc_xive_src_block *sb;
1443 struct kvmppc_xive_irq_state *state;
1444 u64 __user *ubufp = (u64 __user *) addr;
1445 u16 idx;
1446 u64 val;
1447 u8 act_prio, guest_prio;
1448 u32 server;
1449 int rc = 0;
1450
1451 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
1452 return -ENOENT;
1453
1454 pr_devel("set_source(irq=0x%lx)\n", irq);
1455
1456 /* Find the source */
1457 sb = kvmppc_xive_find_source(xive, irq, &idx);
1458 if (!sb) {
1459 pr_devel("No source, creating source block...\n");
1460 sb = xive_create_src_block(xive, irq);
1461 if (!sb) {
1462 pr_devel("Failed to create block...\n");
1463 return -ENOMEM;
1464 }
1465 }
1466 state = &sb->irq_state[idx];
1467
1468 /* Read user passed data */
1469 if (get_user(val, ubufp)) {
1470 pr_devel("fault getting user info !\n");
1471 return -EFAULT;
1472 }
1473
1474 server = val & KVM_XICS_DESTINATION_MASK;
1475 guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
1476
1477 pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n",
1478 val, server, guest_prio);
1479 /*
1480 * If the source doesn't already have an IPI, allocate
1481 * one and get the corresponding data
1482 */
1483 if (!state->ipi_number) {
1484 state->ipi_number = xive_native_alloc_irq();
1485 if (state->ipi_number == 0) {
1486 pr_devel("Failed to allocate IPI !\n");
1487 return -ENOMEM;
1488 }
1489 xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
1490 pr_devel(" src_ipi=0x%x\n", state->ipi_number);
1491 }
1492
1493 /*
1494 * We use lock_and_mask() to set us in the right masked
1495 * state. We will override that state from the saved state
1496 * further down, but this will handle the cases of interrupts
1497 * that need FW masking. We set the initial guest_priority to
1498 * 0 before calling it to ensure it actually performs the masking.
1499 */
1500 state->guest_priority = 0;
1501 xive_lock_and_mask(xive, sb, state);
1502
1503 /*
1504 * Now, we select a target if we have one. If we don't we
1505 * leave the interrupt untargetted. It means that an interrupt
1506 * can become "untargetted" accross migration if it was masked
1507 * by set_xive() but there is little we can do about it.
1508 */
1509
1510 /* First convert prio and mark interrupt as untargetted */
1511 act_prio = xive_prio_from_guest(guest_prio);
1512 state->act_priority = MASKED;
1513
1514 /*
1515 * We need to drop the lock due to the mutex below. Hopefully
1516 * nothing is touching that interrupt yet since it hasn't been
1517 * advertized to a running guest yet
1518 */
1519 arch_spin_unlock(&sb->lock);
1520
1521 /* If we have a priority target the interrupt */
1522 if (act_prio != MASKED) {
1523 /* First, check provisioning of queues */
1524 mutex_lock(&xive->kvm->lock);
1525 rc = xive_check_provisioning(xive->kvm, act_prio);
1526 mutex_unlock(&xive->kvm->lock);
1527
1528 /* Target interrupt */
1529 if (rc == 0)
1530 rc = xive_target_interrupt(xive->kvm, state,
1531 server, act_prio);
1532 /*
1533 * If provisioning or targetting failed, leave it
1534 * alone and masked. It will remain disabled until
1535 * the guest re-targets it.
1536 */
1537 }
1538
1539 /*
1540 * Find out if this was a delayed irq stashed in an ICP,
1541 * in which case, treat it as pending
1542 */
1543 if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
1544 val |= KVM_XICS_PENDING;
1545 pr_devel(" Found delayed ! forcing PENDING !\n");
1546 }
1547
1548 /* Cleanup the SW state */
1549 state->old_p = false;
1550 state->old_q = false;
1551 state->lsi = false;
1552 state->asserted = false;
1553
1554 /* Restore LSI state */
1555 if (val & KVM_XICS_LEVEL_SENSITIVE) {
1556 state->lsi = true;
1557 if (val & KVM_XICS_PENDING)
1558 state->asserted = true;
1559 pr_devel(" LSI ! Asserted=%d\n", state->asserted);
1560 }
1561
1562 /*
1563 * Restore P and Q. If the interrupt was pending, we
1564 * force Q and !P, which will trigger a resend.
1565 *
1566 * That means that a guest that had both an interrupt
1567 * pending (queued) and Q set will restore with only
1568 * one instance of that interrupt instead of 2, but that
1569 * is perfectly fine as coalescing interrupts that haven't
1570 * been presented yet is always allowed.
1571 */
1572 if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
1573 state->old_p = true;
1574 if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
1575 state->old_q = true;
1576
1577 pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q);
1578
1579 /*
1580 * If the interrupt was unmasked, update guest priority and
1581 * perform the appropriate state transition and do a
1582 * re-trigger if necessary.
1583 */
1584 if (val & KVM_XICS_MASKED) {
1585 pr_devel(" masked, saving prio\n");
1586 state->guest_priority = MASKED;
1587 state->saved_priority = guest_prio;
1588 } else {
1589 pr_devel(" unmasked, restoring to prio %d\n", guest_prio);
1590 xive_finish_unmask(xive, sb, state, guest_prio);
1591 state->saved_priority = guest_prio;
1592 }
1593
1594 /* Increment the number of valid sources and mark this one valid */
1595 if (!state->valid)
1596 xive->src_count++;
1597 state->valid = true;
1598
1599 return 0;
1600 }
1601
kvmppc_xive_set_irq(struct kvm * kvm,int irq_source_id,u32 irq,int level,bool line_status)1602 int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
1603 bool line_status)
1604 {
1605 struct kvmppc_xive *xive = kvm->arch.xive;
1606 struct kvmppc_xive_src_block *sb;
1607 struct kvmppc_xive_irq_state *state;
1608 u16 idx;
1609
1610 if (!xive)
1611 return -ENODEV;
1612
1613 sb = kvmppc_xive_find_source(xive, irq, &idx);
1614 if (!sb)
1615 return -EINVAL;
1616
1617 /* Perform locklessly .... (we need to do some RCUisms here...) */
1618 state = &sb->irq_state[idx];
1619 if (!state->valid)
1620 return -EINVAL;
1621
1622 /* We don't allow a trigger on a passed-through interrupt */
1623 if (state->pt_number)
1624 return -EINVAL;
1625
1626 if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
1627 state->asserted = 1;
1628 else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
1629 state->asserted = 0;
1630 return 0;
1631 }
1632
1633 /* Trigger the IPI */
1634 xive_irq_trigger(&state->ipi_data);
1635
1636 return 0;
1637 }
1638
xive_set_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1639 static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1640 {
1641 struct kvmppc_xive *xive = dev->private;
1642
1643 /* We honor the existing XICS ioctl */
1644 switch (attr->group) {
1645 case KVM_DEV_XICS_GRP_SOURCES:
1646 return xive_set_source(xive, attr->attr, attr->addr);
1647 }
1648 return -ENXIO;
1649 }
1650
xive_get_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1651 static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1652 {
1653 struct kvmppc_xive *xive = dev->private;
1654
1655 /* We honor the existing XICS ioctl */
1656 switch (attr->group) {
1657 case KVM_DEV_XICS_GRP_SOURCES:
1658 return xive_get_source(xive, attr->attr, attr->addr);
1659 }
1660 return -ENXIO;
1661 }
1662
xive_has_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1663 static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1664 {
1665 /* We honor the same limits as XICS, at least for now */
1666 switch (attr->group) {
1667 case KVM_DEV_XICS_GRP_SOURCES:
1668 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
1669 attr->attr < KVMPPC_XICS_NR_IRQS)
1670 return 0;
1671 break;
1672 }
1673 return -ENXIO;
1674 }
1675
kvmppc_xive_cleanup_irq(u32 hw_num,struct xive_irq_data * xd)1676 static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
1677 {
1678 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
1679 xive_native_configure_irq(hw_num, 0, MASKED, 0);
1680 }
1681
kvmppc_xive_free_sources(struct kvmppc_xive_src_block * sb)1682 static void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
1683 {
1684 int i;
1685
1686 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
1687 struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
1688
1689 if (!state->valid)
1690 continue;
1691
1692 kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
1693 xive_cleanup_irq_data(&state->ipi_data);
1694 xive_native_free_irq(state->ipi_number);
1695
1696 /* Pass-through, cleanup too but keep IRQ hw data */
1697 if (state->pt_number)
1698 kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
1699
1700 state->valid = false;
1701 }
1702 }
1703
kvmppc_xive_free(struct kvm_device * dev)1704 static void kvmppc_xive_free(struct kvm_device *dev)
1705 {
1706 struct kvmppc_xive *xive = dev->private;
1707 struct kvm *kvm = xive->kvm;
1708 int i;
1709
1710 debugfs_remove(xive->dentry);
1711
1712 if (kvm)
1713 kvm->arch.xive = NULL;
1714
1715 /* Mask and free interrupts */
1716 for (i = 0; i <= xive->max_sbid; i++) {
1717 if (xive->src_blocks[i])
1718 kvmppc_xive_free_sources(xive->src_blocks[i]);
1719 kfree(xive->src_blocks[i]);
1720 xive->src_blocks[i] = NULL;
1721 }
1722
1723 if (xive->vp_base != XIVE_INVALID_VP)
1724 xive_native_free_vp_block(xive->vp_base);
1725
1726
1727 kfree(xive);
1728 kfree(dev);
1729 }
1730
kvmppc_xive_create(struct kvm_device * dev,u32 type)1731 static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
1732 {
1733 struct kvmppc_xive *xive;
1734 struct kvm *kvm = dev->kvm;
1735 int ret = 0;
1736
1737 pr_devel("Creating xive for partition\n");
1738
1739 xive = kzalloc(sizeof(*xive), GFP_KERNEL);
1740 if (!xive)
1741 return -ENOMEM;
1742
1743 dev->private = xive;
1744 xive->dev = dev;
1745 xive->kvm = kvm;
1746
1747 /* Already there ? */
1748 if (kvm->arch.xive)
1749 ret = -EEXIST;
1750 else
1751 kvm->arch.xive = xive;
1752
1753 /* We use the default queue size set by the host */
1754 xive->q_order = xive_native_default_eq_shift();
1755 if (xive->q_order < PAGE_SHIFT)
1756 xive->q_page_order = 0;
1757 else
1758 xive->q_page_order = xive->q_order - PAGE_SHIFT;
1759
1760 /* Allocate a bunch of VPs */
1761 xive->vp_base = xive_native_alloc_vp_block(KVM_MAX_VCPUS);
1762 pr_devel("VP_Base=%x\n", xive->vp_base);
1763
1764 if (xive->vp_base == XIVE_INVALID_VP)
1765 ret = -ENOMEM;
1766
1767 if (ret) {
1768 kfree(xive);
1769 return ret;
1770 }
1771
1772 return 0;
1773 }
1774
1775
xive_debug_show(struct seq_file * m,void * private)1776 static int xive_debug_show(struct seq_file *m, void *private)
1777 {
1778 struct kvmppc_xive *xive = m->private;
1779 struct kvm *kvm = xive->kvm;
1780 struct kvm_vcpu *vcpu;
1781 u64 t_rm_h_xirr = 0;
1782 u64 t_rm_h_ipoll = 0;
1783 u64 t_rm_h_cppr = 0;
1784 u64 t_rm_h_eoi = 0;
1785 u64 t_rm_h_ipi = 0;
1786 u64 t_vm_h_xirr = 0;
1787 u64 t_vm_h_ipoll = 0;
1788 u64 t_vm_h_cppr = 0;
1789 u64 t_vm_h_eoi = 0;
1790 u64 t_vm_h_ipi = 0;
1791 unsigned int i;
1792
1793 if (!kvm)
1794 return 0;
1795
1796 seq_printf(m, "=========\nVCPU state\n=========\n");
1797
1798 kvm_for_each_vcpu(i, vcpu, kvm) {
1799 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1800
1801 if (!xc)
1802 continue;
1803
1804 seq_printf(m, "cpu server %#x CPPR:%#x HWCPPR:%#x"
1805 " MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
1806 xc->server_num, xc->cppr, xc->hw_cppr,
1807 xc->mfrr, xc->pending,
1808 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
1809
1810 t_rm_h_xirr += xc->stat_rm_h_xirr;
1811 t_rm_h_ipoll += xc->stat_rm_h_ipoll;
1812 t_rm_h_cppr += xc->stat_rm_h_cppr;
1813 t_rm_h_eoi += xc->stat_rm_h_eoi;
1814 t_rm_h_ipi += xc->stat_rm_h_ipi;
1815 t_vm_h_xirr += xc->stat_vm_h_xirr;
1816 t_vm_h_ipoll += xc->stat_vm_h_ipoll;
1817 t_vm_h_cppr += xc->stat_vm_h_cppr;
1818 t_vm_h_eoi += xc->stat_vm_h_eoi;
1819 t_vm_h_ipi += xc->stat_vm_h_ipi;
1820 }
1821
1822 seq_printf(m, "Hcalls totals\n");
1823 seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
1824 seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
1825 seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
1826 seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
1827 seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
1828
1829 return 0;
1830 }
1831
xive_debug_open(struct inode * inode,struct file * file)1832 static int xive_debug_open(struct inode *inode, struct file *file)
1833 {
1834 return single_open(file, xive_debug_show, inode->i_private);
1835 }
1836
1837 static const struct file_operations xive_debug_fops = {
1838 .open = xive_debug_open,
1839 .read = seq_read,
1840 .llseek = seq_lseek,
1841 .release = single_release,
1842 };
1843
xive_debugfs_init(struct kvmppc_xive * xive)1844 static void xive_debugfs_init(struct kvmppc_xive *xive)
1845 {
1846 char *name;
1847
1848 name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive);
1849 if (!name) {
1850 pr_err("%s: no memory for name\n", __func__);
1851 return;
1852 }
1853
1854 xive->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root,
1855 xive, &xive_debug_fops);
1856
1857 pr_debug("%s: created %s\n", __func__, name);
1858 kfree(name);
1859 }
1860
kvmppc_xive_init(struct kvm_device * dev)1861 static void kvmppc_xive_init(struct kvm_device *dev)
1862 {
1863 struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private;
1864
1865 /* Register some debug interfaces */
1866 xive_debugfs_init(xive);
1867 }
1868
1869 struct kvm_device_ops kvm_xive_ops = {
1870 .name = "kvm-xive",
1871 .create = kvmppc_xive_create,
1872 .init = kvmppc_xive_init,
1873 .destroy = kvmppc_xive_free,
1874 .set_attr = xive_set_attr,
1875 .get_attr = xive_get_attr,
1876 .has_attr = xive_has_attr,
1877 };
1878
kvmppc_xive_init_module(void)1879 void kvmppc_xive_init_module(void)
1880 {
1881 __xive_vm_h_xirr = xive_vm_h_xirr;
1882 __xive_vm_h_ipoll = xive_vm_h_ipoll;
1883 __xive_vm_h_ipi = xive_vm_h_ipi;
1884 __xive_vm_h_cppr = xive_vm_h_cppr;
1885 __xive_vm_h_eoi = xive_vm_h_eoi;
1886 }
1887
kvmppc_xive_exit_module(void)1888 void kvmppc_xive_exit_module(void)
1889 {
1890 __xive_vm_h_xirr = NULL;
1891 __xive_vm_h_ipoll = NULL;
1892 __xive_vm_h_ipi = NULL;
1893 __xive_vm_h_cppr = NULL;
1894 __xive_vm_h_eoi = NULL;
1895 }
1896