1 /*
2 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation.
7 */
8
9 /* File to be included by other .c files */
10
11 #define XGLUE(a,b) a##b
12 #define GLUE(a,b) XGLUE(a,b)
13
GLUE(X_PFX,ack_pending)14 static void GLUE(X_PFX,ack_pending)(struct kvmppc_xive_vcpu *xc)
15 {
16 u8 cppr;
17 u16 ack;
18
19 /*
20 * Ensure any previous store to CPPR is ordered vs.
21 * the subsequent loads from PIPR or ACK.
22 */
23 eieio();
24
25 /*
26 * DD1 bug workaround: If PIPR is less favored than CPPR
27 * ignore the interrupt or we might incorrectly lose an IPB
28 * bit.
29 */
30 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
31 __be64 qw1 = __x_readq(__x_tima + TM_QW1_OS);
32 u8 pipr = be64_to_cpu(qw1) & 0xff;
33 if (pipr >= xc->hw_cppr)
34 return;
35 }
36
37 /* Perform the acknowledge OS to register cycle. */
38 ack = be16_to_cpu(__x_readw(__x_tima + TM_SPC_ACK_OS_REG));
39
40 /* Synchronize subsequent queue accesses */
41 mb();
42
43 /* XXX Check grouping level */
44
45 /* Anything ? */
46 if (!((ack >> 8) & TM_QW1_NSR_EO))
47 return;
48
49 /* Grab CPPR of the most favored pending interrupt */
50 cppr = ack & 0xff;
51 if (cppr < 8)
52 xc->pending |= 1 << cppr;
53
54 #ifdef XIVE_RUNTIME_CHECKS
55 /* Check consistency */
56 if (cppr >= xc->hw_cppr)
57 pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n",
58 smp_processor_id(), cppr, xc->hw_cppr);
59 #endif
60
61 /*
62 * Update our image of the HW CPPR. We don't yet modify
63 * xc->cppr, this will be done as we scan for interrupts
64 * in the queues.
65 */
66 xc->hw_cppr = cppr;
67 }
68
GLUE(X_PFX,esb_load)69 static u8 GLUE(X_PFX,esb_load)(struct xive_irq_data *xd, u32 offset)
70 {
71 u64 val;
72
73 if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)
74 offset |= offset << 4;
75
76 val =__x_readq(__x_eoi_page(xd) + offset);
77 #ifdef __LITTLE_ENDIAN__
78 val >>= 64-8;
79 #endif
80 return (u8)val;
81 }
82
83
GLUE(X_PFX,source_eoi)84 static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd)
85 {
86 /* If the XIVE supports the new "store EOI facility, use it */
87 if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
88 __x_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
89 else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) {
90 opal_int_eoi(hw_irq);
91 } else {
92 uint64_t eoi_val;
93
94 /*
95 * Otherwise for EOI, we use the special MMIO that does
96 * a clear of both P and Q and returns the old Q,
97 * except for LSIs where we use the "EOI cycle" special
98 * load.
99 *
100 * This allows us to then do a re-trigger if Q was set
101 * rather than synthetizing an interrupt in software
102 *
103 * For LSIs, using the HW EOI cycle works around a problem
104 * on P9 DD1 PHBs where the other ESB accesses don't work
105 * properly.
106 */
107 if (xd->flags & XIVE_IRQ_FLAG_LSI)
108 __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
109 else {
110 eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00);
111
112 /* Re-trigger if needed */
113 if ((eoi_val & 1) && __x_trig_page(xd))
114 __x_writeq(0, __x_trig_page(xd));
115 }
116 }
117 }
118
119 enum {
120 scan_fetch,
121 scan_poll,
122 scan_eoi,
123 };
124
GLUE(X_PFX,scan_interrupts)125 static u32 GLUE(X_PFX,scan_interrupts)(struct kvmppc_xive_vcpu *xc,
126 u8 pending, int scan_type)
127 {
128 u32 hirq = 0;
129 u8 prio = 0xff;
130
131 /* Find highest pending priority */
132 while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) {
133 struct xive_q *q;
134 u32 idx, toggle;
135 __be32 *qpage;
136
137 /*
138 * If pending is 0 this will return 0xff which is what
139 * we want
140 */
141 prio = ffs(pending) - 1;
142
143 /*
144 * If the most favoured prio we found pending is less
145 * favored (or equal) than a pending IPI, we return
146 * the IPI instead.
147 *
148 * Note: If pending was 0 and mfrr is 0xff, we will
149 * not spurriously take an IPI because mfrr cannot
150 * then be smaller than cppr.
151 */
152 if (prio >= xc->mfrr && xc->mfrr < xc->cppr) {
153 prio = xc->mfrr;
154 hirq = XICS_IPI;
155 break;
156 }
157
158 /* Don't scan past the guest cppr */
159 if (prio >= xc->cppr || prio > 7)
160 break;
161
162 /* Grab queue and pointers */
163 q = &xc->queues[prio];
164 idx = q->idx;
165 toggle = q->toggle;
166
167 /*
168 * Snapshot the queue page. The test further down for EOI
169 * must use the same "copy" that was used by __xive_read_eq
170 * since qpage can be set concurrently and we don't want
171 * to miss an EOI.
172 */
173 qpage = READ_ONCE(q->qpage);
174
175 skip_ipi:
176 /*
177 * Try to fetch from the queue. Will return 0 for a
178 * non-queueing priority (ie, qpage = 0).
179 */
180 hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle);
181
182 /*
183 * If this was a signal for an MFFR change done by
184 * H_IPI we skip it. Additionally, if we were fetching
185 * we EOI it now, thus re-enabling reception of a new
186 * such signal.
187 *
188 * We also need to do that if prio is 0 and we had no
189 * page for the queue. In this case, we have non-queued
190 * IPI that needs to be EOId.
191 *
192 * This is safe because if we have another pending MFRR
193 * change that wasn't observed above, the Q bit will have
194 * been set and another occurrence of the IPI will trigger.
195 */
196 if (hirq == XICS_IPI || (prio == 0 && !qpage)) {
197 if (scan_type == scan_fetch)
198 GLUE(X_PFX,source_eoi)(xc->vp_ipi,
199 &xc->vp_ipi_data);
200 /* Loop back on same queue with updated idx/toggle */
201 #ifdef XIVE_RUNTIME_CHECKS
202 WARN_ON(hirq && hirq != XICS_IPI);
203 #endif
204 if (hirq)
205 goto skip_ipi;
206 }
207
208 /* If fetching, update queue pointers */
209 if (scan_type == scan_fetch) {
210 q->idx = idx;
211 q->toggle = toggle;
212 }
213
214 /* Something found, stop searching */
215 if (hirq)
216 break;
217
218 /* Clear the pending bit on the now empty queue */
219 pending &= ~(1 << prio);
220
221 /*
222 * Check if the queue count needs adjusting due to
223 * interrupts being moved away.
224 */
225 if (atomic_read(&q->pending_count)) {
226 int p = atomic_xchg(&q->pending_count, 0);
227 if (p) {
228 #ifdef XIVE_RUNTIME_CHECKS
229 WARN_ON(p > atomic_read(&q->count));
230 #endif
231 atomic_sub(p, &q->count);
232 }
233 }
234 }
235
236 /* If we are just taking a "peek", do nothing else */
237 if (scan_type == scan_poll)
238 return hirq;
239
240 /* Update the pending bits */
241 xc->pending = pending;
242
243 /*
244 * If this is an EOI that's it, no CPPR adjustment done here,
245 * all we needed was cleanup the stale pending bits and check
246 * if there's anything left.
247 */
248 if (scan_type == scan_eoi)
249 return hirq;
250
251 /*
252 * If we found an interrupt, adjust what the guest CPPR should
253 * be as if we had just fetched that interrupt from HW.
254 *
255 * Note: This can only make xc->cppr smaller as the previous
256 * loop will only exit with hirq != 0 if prio is lower than
257 * the current xc->cppr. Thus we don't need to re-check xc->mfrr
258 * for pending IPIs.
259 */
260 if (hirq)
261 xc->cppr = prio;
262 /*
263 * If it was an IPI the HW CPPR might have been lowered too much
264 * as the HW interrupt we use for IPIs is routed to priority 0.
265 *
266 * We re-sync it here.
267 */
268 if (xc->cppr != xc->hw_cppr) {
269 xc->hw_cppr = xc->cppr;
270 __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR);
271 }
272
273 return hirq;
274 }
275
GLUE(X_PFX,h_xirr)276 X_STATIC unsigned long GLUE(X_PFX,h_xirr)(struct kvm_vcpu *vcpu)
277 {
278 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
279 u8 old_cppr;
280 u32 hirq;
281
282 pr_devel("H_XIRR\n");
283
284 xc->GLUE(X_STAT_PFX,h_xirr)++;
285
286 /* First collect pending bits from HW */
287 GLUE(X_PFX,ack_pending)(xc);
288
289 /*
290 * Cleanup the old-style bits if needed (they may have been
291 * set by pull or an escalation interrupts).
292 */
293 if (test_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions))
294 clear_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL,
295 &vcpu->arch.pending_exceptions);
296
297 pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n",
298 xc->pending, xc->hw_cppr, xc->cppr);
299
300 /* Grab previous CPPR and reverse map it */
301 old_cppr = xive_prio_to_guest(xc->cppr);
302
303 /* Scan for actual interrupts */
304 hirq = GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_fetch);
305
306 pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n",
307 hirq, xc->hw_cppr, xc->cppr);
308
309 #ifdef XIVE_RUNTIME_CHECKS
310 /* That should never hit */
311 if (hirq & 0xff000000)
312 pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq);
313 #endif
314
315 /*
316 * XXX We could check if the interrupt is masked here and
317 * filter it. If we chose to do so, we would need to do:
318 *
319 * if (masked) {
320 * lock();
321 * if (masked) {
322 * old_Q = true;
323 * hirq = 0;
324 * }
325 * unlock();
326 * }
327 */
328
329 /* Return interrupt and old CPPR in GPR4 */
330 vcpu->arch.gpr[4] = hirq | (old_cppr << 24);
331
332 return H_SUCCESS;
333 }
334
GLUE(X_PFX,h_ipoll)335 X_STATIC unsigned long GLUE(X_PFX,h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server)
336 {
337 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
338 u8 pending = xc->pending;
339 u32 hirq;
340
341 pr_devel("H_IPOLL(server=%ld)\n", server);
342
343 xc->GLUE(X_STAT_PFX,h_ipoll)++;
344
345 /* Grab the target VCPU if not the current one */
346 if (xc->server_num != server) {
347 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
348 if (!vcpu)
349 return H_PARAMETER;
350 xc = vcpu->arch.xive_vcpu;
351
352 /* Scan all priorities */
353 pending = 0xff;
354 } else {
355 /* Grab pending interrupt if any */
356 __be64 qw1 = __x_readq(__x_tima + TM_QW1_OS);
357 u8 pipr = be64_to_cpu(qw1) & 0xff;
358 if (pipr < 8)
359 pending |= 1 << pipr;
360 }
361
362 hirq = GLUE(X_PFX,scan_interrupts)(xc, pending, scan_poll);
363
364 /* Return interrupt and old CPPR in GPR4 */
365 vcpu->arch.gpr[4] = hirq | (xc->cppr << 24);
366
367 return H_SUCCESS;
368 }
369
GLUE(X_PFX,push_pending_to_hw)370 static void GLUE(X_PFX,push_pending_to_hw)(struct kvmppc_xive_vcpu *xc)
371 {
372 u8 pending, prio;
373
374 pending = xc->pending;
375 if (xc->mfrr != 0xff) {
376 if (xc->mfrr < 8)
377 pending |= 1 << xc->mfrr;
378 else
379 pending |= 0x80;
380 }
381 if (!pending)
382 return;
383 prio = ffs(pending) - 1;
384
385 __x_writeb(prio, __x_tima + TM_SPC_SET_OS_PENDING);
386 }
387
GLUE(X_PFX,h_cppr)388 X_STATIC int GLUE(X_PFX,h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr)
389 {
390 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
391 u8 old_cppr;
392
393 pr_devel("H_CPPR(cppr=%ld)\n", cppr);
394
395 xc->GLUE(X_STAT_PFX,h_cppr)++;
396
397 /* Map CPPR */
398 cppr = xive_prio_from_guest(cppr);
399
400 /* Remember old and update SW state */
401 old_cppr = xc->cppr;
402 xc->cppr = cppr;
403
404 /*
405 * Order the above update of xc->cppr with the subsequent
406 * read of xc->mfrr inside push_pending_to_hw()
407 */
408 smp_mb();
409
410 /*
411 * We are masking less, we need to look for pending things
412 * to deliver and set VP pending bits accordingly to trigger
413 * a new interrupt otherwise we might miss MFRR changes for
414 * which we have optimized out sending an IPI signal.
415 */
416 if (cppr > old_cppr)
417 GLUE(X_PFX,push_pending_to_hw)(xc);
418
419 /* Apply new CPPR */
420 xc->hw_cppr = cppr;
421 __x_writeb(cppr, __x_tima + TM_QW1_OS + TM_CPPR);
422
423 return H_SUCCESS;
424 }
425
GLUE(X_PFX,h_eoi)426 X_STATIC int GLUE(X_PFX,h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr)
427 {
428 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
429 struct kvmppc_xive_src_block *sb;
430 struct kvmppc_xive_irq_state *state;
431 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
432 struct xive_irq_data *xd;
433 u8 new_cppr = xirr >> 24;
434 u32 irq = xirr & 0x00ffffff, hw_num;
435 u16 src;
436 int rc = 0;
437
438 pr_devel("H_EOI(xirr=%08lx)\n", xirr);
439
440 xc->GLUE(X_STAT_PFX,h_eoi)++;
441
442 xc->cppr = xive_prio_from_guest(new_cppr);
443
444 /*
445 * IPIs are synthetized from MFRR and thus don't need
446 * any special EOI handling. The underlying interrupt
447 * used to signal MFRR changes is EOId when fetched from
448 * the queue.
449 */
450 if (irq == XICS_IPI || irq == 0) {
451 /*
452 * This barrier orders the setting of xc->cppr vs.
453 * subsquent test of xc->mfrr done inside
454 * scan_interrupts and push_pending_to_hw
455 */
456 smp_mb();
457 goto bail;
458 }
459
460 /* Find interrupt source */
461 sb = kvmppc_xive_find_source(xive, irq, &src);
462 if (!sb) {
463 pr_devel(" source not found !\n");
464 rc = H_PARAMETER;
465 /* Same as above */
466 smp_mb();
467 goto bail;
468 }
469 state = &sb->irq_state[src];
470 kvmppc_xive_select_irq(state, &hw_num, &xd);
471
472 state->in_eoi = true;
473
474 /*
475 * This barrier orders both setting of in_eoi above vs,
476 * subsequent test of guest_priority, and the setting
477 * of xc->cppr vs. subsquent test of xc->mfrr done inside
478 * scan_interrupts and push_pending_to_hw
479 */
480 smp_mb();
481
482 again:
483 if (state->guest_priority == MASKED) {
484 arch_spin_lock(&sb->lock);
485 if (state->guest_priority != MASKED) {
486 arch_spin_unlock(&sb->lock);
487 goto again;
488 }
489 pr_devel(" EOI on saved P...\n");
490
491 /* Clear old_p, that will cause unmask to perform an EOI */
492 state->old_p = false;
493
494 arch_spin_unlock(&sb->lock);
495 } else {
496 pr_devel(" EOI on source...\n");
497
498 /* Perform EOI on the source */
499 GLUE(X_PFX,source_eoi)(hw_num, xd);
500
501 /* If it's an emulated LSI, check level and resend */
502 if (state->lsi && state->asserted)
503 __x_writeq(0, __x_trig_page(xd));
504
505 }
506
507 /*
508 * This barrier orders the above guest_priority check
509 * and spin_lock/unlock with clearing in_eoi below.
510 *
511 * It also has to be a full mb() as it must ensure
512 * the MMIOs done in source_eoi() are completed before
513 * state->in_eoi is visible.
514 */
515 mb();
516 state->in_eoi = false;
517 bail:
518
519 /* Re-evaluate pending IRQs and update HW */
520 GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_eoi);
521 GLUE(X_PFX,push_pending_to_hw)(xc);
522 pr_devel(" after scan pending=%02x\n", xc->pending);
523
524 /* Apply new CPPR */
525 xc->hw_cppr = xc->cppr;
526 __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR);
527
528 return rc;
529 }
530
GLUE(X_PFX,h_ipi)531 X_STATIC int GLUE(X_PFX,h_ipi)(struct kvm_vcpu *vcpu, unsigned long server,
532 unsigned long mfrr)
533 {
534 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
535
536 pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr);
537
538 xc->GLUE(X_STAT_PFX,h_ipi)++;
539
540 /* Find target */
541 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
542 if (!vcpu)
543 return H_PARAMETER;
544 xc = vcpu->arch.xive_vcpu;
545
546 /* Locklessly write over MFRR */
547 xc->mfrr = mfrr;
548
549 /*
550 * The load of xc->cppr below and the subsequent MMIO store
551 * to the IPI must happen after the above mfrr update is
552 * globally visible so that:
553 *
554 * - Synchronize with another CPU doing an H_EOI or a H_CPPR
555 * updating xc->cppr then reading xc->mfrr.
556 *
557 * - The target of the IPI sees the xc->mfrr update
558 */
559 mb();
560
561 /* Shoot the IPI if most favored than target cppr */
562 if (mfrr < xc->cppr)
563 __x_writeq(0, __x_trig_page(&xc->vp_ipi_data));
564
565 return H_SUCCESS;
566 }
567