• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * OpenPIC emulation
3  *
4  * Copyright (c) 2004 Jocelyn Mayer
5  *               2011 Alexander Graf
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a copy
8  * of this software and associated documentation files (the "Software"), to deal
9  * in the Software without restriction, including without limitation the rights
10  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11  * copies of the Software, and to permit persons to whom the Software is
12  * furnished to do so, subject to the following conditions:
13  *
14  * The above copyright notice and this permission notice shall be included in
15  * all copies or substantial portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23  * THE SOFTWARE.
24  */
25 
26 #include <linux/slab.h>
27 #include <linux/mutex.h>
28 #include <linux/kvm_host.h>
29 #include <linux/errno.h>
30 #include <linux/fs.h>
31 #include <linux/anon_inodes.h>
32 #include <asm/uaccess.h>
33 #include <asm/mpic.h>
34 #include <asm/kvm_para.h>
35 #include <asm/kvm_host.h>
36 #include <asm/kvm_ppc.h>
37 #include <kvm/iodev.h>
38 
39 #define MAX_CPU     32
40 #define MAX_SRC     256
41 #define MAX_TMR     4
42 #define MAX_IPI     4
43 #define MAX_MSI     8
44 #define MAX_IRQ     (MAX_SRC + MAX_IPI + MAX_TMR)
45 #define VID         0x03	/* MPIC version ID */
46 
47 /* OpenPIC capability flags */
48 #define OPENPIC_FLAG_IDR_CRIT     (1 << 0)
49 #define OPENPIC_FLAG_ILR          (2 << 0)
50 
51 /* OpenPIC address map */
52 #define OPENPIC_REG_SIZE             0x40000
53 #define OPENPIC_GLB_REG_START        0x0
54 #define OPENPIC_GLB_REG_SIZE         0x10F0
55 #define OPENPIC_TMR_REG_START        0x10F0
56 #define OPENPIC_TMR_REG_SIZE         0x220
57 #define OPENPIC_MSI_REG_START        0x1600
58 #define OPENPIC_MSI_REG_SIZE         0x200
59 #define OPENPIC_SUMMARY_REG_START    0x3800
60 #define OPENPIC_SUMMARY_REG_SIZE     0x800
61 #define OPENPIC_SRC_REG_START        0x10000
62 #define OPENPIC_SRC_REG_SIZE         (MAX_SRC * 0x20)
63 #define OPENPIC_CPU_REG_START        0x20000
64 #define OPENPIC_CPU_REG_SIZE         (0x100 + ((MAX_CPU - 1) * 0x1000))
65 
66 struct fsl_mpic_info {
67 	int max_ext;
68 };
69 
70 static struct fsl_mpic_info fsl_mpic_20 = {
71 	.max_ext = 12,
72 };
73 
74 static struct fsl_mpic_info fsl_mpic_42 = {
75 	.max_ext = 12,
76 };
77 
78 #define FRR_NIRQ_SHIFT    16
79 #define FRR_NCPU_SHIFT     8
80 #define FRR_VID_SHIFT      0
81 
82 #define VID_REVISION_1_2   2
83 #define VID_REVISION_1_3   3
84 
85 #define VIR_GENERIC      0x00000000	/* Generic Vendor ID */
86 
87 #define GCR_RESET        0x80000000
88 #define GCR_MODE_PASS    0x00000000
89 #define GCR_MODE_MIXED   0x20000000
90 #define GCR_MODE_PROXY   0x60000000
91 
92 #define TBCR_CI           0x80000000	/* count inhibit */
93 #define TCCR_TOG          0x80000000	/* toggles when decrement to zero */
94 
95 #define IDR_EP_SHIFT      31
96 #define IDR_EP_MASK       (1 << IDR_EP_SHIFT)
97 #define IDR_CI0_SHIFT     30
98 #define IDR_CI1_SHIFT     29
99 #define IDR_P1_SHIFT      1
100 #define IDR_P0_SHIFT      0
101 
102 #define ILR_INTTGT_MASK   0x000000ff
103 #define ILR_INTTGT_INT    0x00
104 #define ILR_INTTGT_CINT   0x01	/* critical */
105 #define ILR_INTTGT_MCP    0x02	/* machine check */
106 #define NUM_OUTPUTS       3
107 
108 #define MSIIR_OFFSET       0x140
109 #define MSIIR_SRS_SHIFT    29
110 #define MSIIR_SRS_MASK     (0x7 << MSIIR_SRS_SHIFT)
111 #define MSIIR_IBS_SHIFT    24
112 #define MSIIR_IBS_MASK     (0x1f << MSIIR_IBS_SHIFT)
113 
get_current_cpu(void)114 static int get_current_cpu(void)
115 {
116 #if defined(CONFIG_KVM) && defined(CONFIG_BOOKE)
117 	struct kvm_vcpu *vcpu = current->thread.kvm_vcpu;
118 	return vcpu ? vcpu->arch.irq_cpu_id : -1;
119 #else
120 	/* XXX */
121 	return -1;
122 #endif
123 }
124 
125 static int openpic_cpu_write_internal(void *opaque, gpa_t addr,
126 				      u32 val, int idx);
127 static int openpic_cpu_read_internal(void *opaque, gpa_t addr,
128 				     u32 *ptr, int idx);
129 static inline void write_IRQreg_idr(struct openpic *opp, int n_IRQ,
130 				    uint32_t val);
131 
132 enum irq_type {
133 	IRQ_TYPE_NORMAL = 0,
134 	IRQ_TYPE_FSLINT,	/* FSL internal interrupt -- level only */
135 	IRQ_TYPE_FSLSPECIAL,	/* FSL timer/IPI interrupt, edge, no polarity */
136 };
137 
138 struct irq_queue {
139 	/* Round up to the nearest 64 IRQs so that the queue length
140 	 * won't change when moving between 32 and 64 bit hosts.
141 	 */
142 	unsigned long queue[BITS_TO_LONGS((MAX_IRQ + 63) & ~63)];
143 	int next;
144 	int priority;
145 };
146 
147 struct irq_source {
148 	uint32_t ivpr;		/* IRQ vector/priority register */
149 	uint32_t idr;		/* IRQ destination register */
150 	uint32_t destmask;	/* bitmap of CPU destinations */
151 	int last_cpu;
152 	int output;		/* IRQ level, e.g. ILR_INTTGT_INT */
153 	int pending;		/* TRUE if IRQ is pending */
154 	enum irq_type type;
155 	bool level:1;		/* level-triggered */
156 	bool nomask:1;	/* critical interrupts ignore mask on some FSL MPICs */
157 };
158 
159 #define IVPR_MASK_SHIFT       31
160 #define IVPR_MASK_MASK        (1 << IVPR_MASK_SHIFT)
161 #define IVPR_ACTIVITY_SHIFT   30
162 #define IVPR_ACTIVITY_MASK    (1 << IVPR_ACTIVITY_SHIFT)
163 #define IVPR_MODE_SHIFT       29
164 #define IVPR_MODE_MASK        (1 << IVPR_MODE_SHIFT)
165 #define IVPR_POLARITY_SHIFT   23
166 #define IVPR_POLARITY_MASK    (1 << IVPR_POLARITY_SHIFT)
167 #define IVPR_SENSE_SHIFT      22
168 #define IVPR_SENSE_MASK       (1 << IVPR_SENSE_SHIFT)
169 
170 #define IVPR_PRIORITY_MASK     (0xF << 16)
171 #define IVPR_PRIORITY(_ivprr_) ((int)(((_ivprr_) & IVPR_PRIORITY_MASK) >> 16))
172 #define IVPR_VECTOR(opp, _ivprr_) ((_ivprr_) & (opp)->vector_mask)
173 
174 /* IDR[EP/CI] are only for FSL MPIC prior to v4.0 */
175 #define IDR_EP      0x80000000	/* external pin */
176 #define IDR_CI      0x40000000	/* critical interrupt */
177 
178 struct irq_dest {
179 	struct kvm_vcpu *vcpu;
180 
181 	int32_t ctpr;		/* CPU current task priority */
182 	struct irq_queue raised;
183 	struct irq_queue servicing;
184 
185 	/* Count of IRQ sources asserting on non-INT outputs */
186 	uint32_t outputs_active[NUM_OUTPUTS];
187 };
188 
189 #define MAX_MMIO_REGIONS 10
190 
191 struct openpic {
192 	struct kvm *kvm;
193 	struct kvm_device *dev;
194 	struct kvm_io_device mmio;
195 	const struct mem_reg *mmio_regions[MAX_MMIO_REGIONS];
196 	int num_mmio_regions;
197 
198 	gpa_t reg_base;
199 	spinlock_t lock;
200 
201 	/* Behavior control */
202 	struct fsl_mpic_info *fsl;
203 	uint32_t model;
204 	uint32_t flags;
205 	uint32_t nb_irqs;
206 	uint32_t vid;
207 	uint32_t vir;		/* Vendor identification register */
208 	uint32_t vector_mask;
209 	uint32_t tfrr_reset;
210 	uint32_t ivpr_reset;
211 	uint32_t idr_reset;
212 	uint32_t brr1;
213 	uint32_t mpic_mode_mask;
214 
215 	/* Global registers */
216 	uint32_t frr;		/* Feature reporting register */
217 	uint32_t gcr;		/* Global configuration register  */
218 	uint32_t pir;		/* Processor initialization register */
219 	uint32_t spve;		/* Spurious vector register */
220 	uint32_t tfrr;		/* Timer frequency reporting register */
221 	/* Source registers */
222 	struct irq_source src[MAX_IRQ];
223 	/* Local registers per output pin */
224 	struct irq_dest dst[MAX_CPU];
225 	uint32_t nb_cpus;
226 	/* Timer registers */
227 	struct {
228 		uint32_t tccr;	/* Global timer current count register */
229 		uint32_t tbcr;	/* Global timer base count register */
230 	} timers[MAX_TMR];
231 	/* Shared MSI registers */
232 	struct {
233 		uint32_t msir;	/* Shared Message Signaled Interrupt Register */
234 	} msi[MAX_MSI];
235 	uint32_t max_irq;
236 	uint32_t irq_ipi0;
237 	uint32_t irq_tim0;
238 	uint32_t irq_msi;
239 };
240 
241 
mpic_irq_raise(struct openpic * opp,struct irq_dest * dst,int output)242 static void mpic_irq_raise(struct openpic *opp, struct irq_dest *dst,
243 			   int output)
244 {
245 	struct kvm_interrupt irq = {
246 		.irq = KVM_INTERRUPT_SET_LEVEL,
247 	};
248 
249 	if (!dst->vcpu) {
250 		pr_debug("%s: destination cpu %d does not exist\n",
251 			 __func__, (int)(dst - &opp->dst[0]));
252 		return;
253 	}
254 
255 	pr_debug("%s: cpu %d output %d\n", __func__, dst->vcpu->arch.irq_cpu_id,
256 		output);
257 
258 	if (output != ILR_INTTGT_INT)	/* TODO */
259 		return;
260 
261 	kvm_vcpu_ioctl_interrupt(dst->vcpu, &irq);
262 }
263 
mpic_irq_lower(struct openpic * opp,struct irq_dest * dst,int output)264 static void mpic_irq_lower(struct openpic *opp, struct irq_dest *dst,
265 			   int output)
266 {
267 	if (!dst->vcpu) {
268 		pr_debug("%s: destination cpu %d does not exist\n",
269 			 __func__, (int)(dst - &opp->dst[0]));
270 		return;
271 	}
272 
273 	pr_debug("%s: cpu %d output %d\n", __func__, dst->vcpu->arch.irq_cpu_id,
274 		output);
275 
276 	if (output != ILR_INTTGT_INT)	/* TODO */
277 		return;
278 
279 	kvmppc_core_dequeue_external(dst->vcpu);
280 }
281 
IRQ_setbit(struct irq_queue * q,int n_IRQ)282 static inline void IRQ_setbit(struct irq_queue *q, int n_IRQ)
283 {
284 	set_bit(n_IRQ, q->queue);
285 }
286 
IRQ_resetbit(struct irq_queue * q,int n_IRQ)287 static inline void IRQ_resetbit(struct irq_queue *q, int n_IRQ)
288 {
289 	clear_bit(n_IRQ, q->queue);
290 }
291 
IRQ_check(struct openpic * opp,struct irq_queue * q)292 static void IRQ_check(struct openpic *opp, struct irq_queue *q)
293 {
294 	int irq = -1;
295 	int next = -1;
296 	int priority = -1;
297 
298 	for (;;) {
299 		irq = find_next_bit(q->queue, opp->max_irq, irq + 1);
300 		if (irq == opp->max_irq)
301 			break;
302 
303 		pr_debug("IRQ_check: irq %d set ivpr_pr=%d pr=%d\n",
304 			irq, IVPR_PRIORITY(opp->src[irq].ivpr), priority);
305 
306 		if (IVPR_PRIORITY(opp->src[irq].ivpr) > priority) {
307 			next = irq;
308 			priority = IVPR_PRIORITY(opp->src[irq].ivpr);
309 		}
310 	}
311 
312 	q->next = next;
313 	q->priority = priority;
314 }
315 
IRQ_get_next(struct openpic * opp,struct irq_queue * q)316 static int IRQ_get_next(struct openpic *opp, struct irq_queue *q)
317 {
318 	/* XXX: optimize */
319 	IRQ_check(opp, q);
320 
321 	return q->next;
322 }
323 
IRQ_local_pipe(struct openpic * opp,int n_CPU,int n_IRQ,bool active,bool was_active)324 static void IRQ_local_pipe(struct openpic *opp, int n_CPU, int n_IRQ,
325 			   bool active, bool was_active)
326 {
327 	struct irq_dest *dst;
328 	struct irq_source *src;
329 	int priority;
330 
331 	dst = &opp->dst[n_CPU];
332 	src = &opp->src[n_IRQ];
333 
334 	pr_debug("%s: IRQ %d active %d was %d\n",
335 		__func__, n_IRQ, active, was_active);
336 
337 	if (src->output != ILR_INTTGT_INT) {
338 		pr_debug("%s: output %d irq %d active %d was %d count %d\n",
339 			__func__, src->output, n_IRQ, active, was_active,
340 			dst->outputs_active[src->output]);
341 
342 		/* On Freescale MPIC, critical interrupts ignore priority,
343 		 * IACK, EOI, etc.  Before MPIC v4.1 they also ignore
344 		 * masking.
345 		 */
346 		if (active) {
347 			if (!was_active &&
348 			    dst->outputs_active[src->output]++ == 0) {
349 				pr_debug("%s: Raise OpenPIC output %d cpu %d irq %d\n",
350 					__func__, src->output, n_CPU, n_IRQ);
351 				mpic_irq_raise(opp, dst, src->output);
352 			}
353 		} else {
354 			if (was_active &&
355 			    --dst->outputs_active[src->output] == 0) {
356 				pr_debug("%s: Lower OpenPIC output %d cpu %d irq %d\n",
357 					__func__, src->output, n_CPU, n_IRQ);
358 				mpic_irq_lower(opp, dst, src->output);
359 			}
360 		}
361 
362 		return;
363 	}
364 
365 	priority = IVPR_PRIORITY(src->ivpr);
366 
367 	/* Even if the interrupt doesn't have enough priority,
368 	 * it is still raised, in case ctpr is lowered later.
369 	 */
370 	if (active)
371 		IRQ_setbit(&dst->raised, n_IRQ);
372 	else
373 		IRQ_resetbit(&dst->raised, n_IRQ);
374 
375 	IRQ_check(opp, &dst->raised);
376 
377 	if (active && priority <= dst->ctpr) {
378 		pr_debug("%s: IRQ %d priority %d too low for ctpr %d on CPU %d\n",
379 			__func__, n_IRQ, priority, dst->ctpr, n_CPU);
380 		active = 0;
381 	}
382 
383 	if (active) {
384 		if (IRQ_get_next(opp, &dst->servicing) >= 0 &&
385 		    priority <= dst->servicing.priority) {
386 			pr_debug("%s: IRQ %d is hidden by servicing IRQ %d on CPU %d\n",
387 				__func__, n_IRQ, dst->servicing.next, n_CPU);
388 		} else {
389 			pr_debug("%s: Raise OpenPIC INT output cpu %d irq %d/%d\n",
390 				__func__, n_CPU, n_IRQ, dst->raised.next);
391 			mpic_irq_raise(opp, dst, ILR_INTTGT_INT);
392 		}
393 	} else {
394 		IRQ_get_next(opp, &dst->servicing);
395 		if (dst->raised.priority > dst->ctpr &&
396 		    dst->raised.priority > dst->servicing.priority) {
397 			pr_debug("%s: IRQ %d inactive, IRQ %d prio %d above %d/%d, CPU %d\n",
398 				__func__, n_IRQ, dst->raised.next,
399 				dst->raised.priority, dst->ctpr,
400 				dst->servicing.priority, n_CPU);
401 			/* IRQ line stays asserted */
402 		} else {
403 			pr_debug("%s: IRQ %d inactive, current prio %d/%d, CPU %d\n",
404 				__func__, n_IRQ, dst->ctpr,
405 				dst->servicing.priority, n_CPU);
406 			mpic_irq_lower(opp, dst, ILR_INTTGT_INT);
407 		}
408 	}
409 }
410 
411 /* update pic state because registers for n_IRQ have changed value */
openpic_update_irq(struct openpic * opp,int n_IRQ)412 static void openpic_update_irq(struct openpic *opp, int n_IRQ)
413 {
414 	struct irq_source *src;
415 	bool active, was_active;
416 	int i;
417 
418 	src = &opp->src[n_IRQ];
419 	active = src->pending;
420 
421 	if ((src->ivpr & IVPR_MASK_MASK) && !src->nomask) {
422 		/* Interrupt source is disabled */
423 		pr_debug("%s: IRQ %d is disabled\n", __func__, n_IRQ);
424 		active = false;
425 	}
426 
427 	was_active = !!(src->ivpr & IVPR_ACTIVITY_MASK);
428 
429 	/*
430 	 * We don't have a similar check for already-active because
431 	 * ctpr may have changed and we need to withdraw the interrupt.
432 	 */
433 	if (!active && !was_active) {
434 		pr_debug("%s: IRQ %d is already inactive\n", __func__, n_IRQ);
435 		return;
436 	}
437 
438 	if (active)
439 		src->ivpr |= IVPR_ACTIVITY_MASK;
440 	else
441 		src->ivpr &= ~IVPR_ACTIVITY_MASK;
442 
443 	if (src->destmask == 0) {
444 		/* No target */
445 		pr_debug("%s: IRQ %d has no target\n", __func__, n_IRQ);
446 		return;
447 	}
448 
449 	if (src->destmask == (1 << src->last_cpu)) {
450 		/* Only one CPU is allowed to receive this IRQ */
451 		IRQ_local_pipe(opp, src->last_cpu, n_IRQ, active, was_active);
452 	} else if (!(src->ivpr & IVPR_MODE_MASK)) {
453 		/* Directed delivery mode */
454 		for (i = 0; i < opp->nb_cpus; i++) {
455 			if (src->destmask & (1 << i)) {
456 				IRQ_local_pipe(opp, i, n_IRQ, active,
457 					       was_active);
458 			}
459 		}
460 	} else {
461 		/* Distributed delivery mode */
462 		for (i = src->last_cpu + 1; i != src->last_cpu; i++) {
463 			if (i == opp->nb_cpus)
464 				i = 0;
465 
466 			if (src->destmask & (1 << i)) {
467 				IRQ_local_pipe(opp, i, n_IRQ, active,
468 					       was_active);
469 				src->last_cpu = i;
470 				break;
471 			}
472 		}
473 	}
474 }
475 
openpic_set_irq(void * opaque,int n_IRQ,int level)476 static void openpic_set_irq(void *opaque, int n_IRQ, int level)
477 {
478 	struct openpic *opp = opaque;
479 	struct irq_source *src;
480 
481 	if (n_IRQ >= MAX_IRQ) {
482 		WARN_ONCE(1, "%s: IRQ %d out of range\n", __func__, n_IRQ);
483 		return;
484 	}
485 
486 	src = &opp->src[n_IRQ];
487 	pr_debug("openpic: set irq %d = %d ivpr=0x%08x\n",
488 		n_IRQ, level, src->ivpr);
489 	if (src->level) {
490 		/* level-sensitive irq */
491 		src->pending = level;
492 		openpic_update_irq(opp, n_IRQ);
493 	} else {
494 		/* edge-sensitive irq */
495 		if (level) {
496 			src->pending = 1;
497 			openpic_update_irq(opp, n_IRQ);
498 		}
499 
500 		if (src->output != ILR_INTTGT_INT) {
501 			/* Edge-triggered interrupts shouldn't be used
502 			 * with non-INT delivery, but just in case,
503 			 * try to make it do something sane rather than
504 			 * cause an interrupt storm.  This is close to
505 			 * what you'd probably see happen in real hardware.
506 			 */
507 			src->pending = 0;
508 			openpic_update_irq(opp, n_IRQ);
509 		}
510 	}
511 }
512 
openpic_reset(struct openpic * opp)513 static void openpic_reset(struct openpic *opp)
514 {
515 	int i;
516 
517 	opp->gcr = GCR_RESET;
518 	/* Initialise controller registers */
519 	opp->frr = ((opp->nb_irqs - 1) << FRR_NIRQ_SHIFT) |
520 	    (opp->vid << FRR_VID_SHIFT);
521 
522 	opp->pir = 0;
523 	opp->spve = -1 & opp->vector_mask;
524 	opp->tfrr = opp->tfrr_reset;
525 	/* Initialise IRQ sources */
526 	for (i = 0; i < opp->max_irq; i++) {
527 		opp->src[i].ivpr = opp->ivpr_reset;
528 
529 		switch (opp->src[i].type) {
530 		case IRQ_TYPE_NORMAL:
531 			opp->src[i].level =
532 			    !!(opp->ivpr_reset & IVPR_SENSE_MASK);
533 			break;
534 
535 		case IRQ_TYPE_FSLINT:
536 			opp->src[i].ivpr |= IVPR_POLARITY_MASK;
537 			break;
538 
539 		case IRQ_TYPE_FSLSPECIAL:
540 			break;
541 		}
542 
543 		write_IRQreg_idr(opp, i, opp->idr_reset);
544 	}
545 	/* Initialise IRQ destinations */
546 	for (i = 0; i < MAX_CPU; i++) {
547 		opp->dst[i].ctpr = 15;
548 		memset(&opp->dst[i].raised, 0, sizeof(struct irq_queue));
549 		opp->dst[i].raised.next = -1;
550 		memset(&opp->dst[i].servicing, 0, sizeof(struct irq_queue));
551 		opp->dst[i].servicing.next = -1;
552 	}
553 	/* Initialise timers */
554 	for (i = 0; i < MAX_TMR; i++) {
555 		opp->timers[i].tccr = 0;
556 		opp->timers[i].tbcr = TBCR_CI;
557 	}
558 	/* Go out of RESET state */
559 	opp->gcr = 0;
560 }
561 
read_IRQreg_idr(struct openpic * opp,int n_IRQ)562 static inline uint32_t read_IRQreg_idr(struct openpic *opp, int n_IRQ)
563 {
564 	return opp->src[n_IRQ].idr;
565 }
566 
read_IRQreg_ilr(struct openpic * opp,int n_IRQ)567 static inline uint32_t read_IRQreg_ilr(struct openpic *opp, int n_IRQ)
568 {
569 	if (opp->flags & OPENPIC_FLAG_ILR)
570 		return opp->src[n_IRQ].output;
571 
572 	return 0xffffffff;
573 }
574 
read_IRQreg_ivpr(struct openpic * opp,int n_IRQ)575 static inline uint32_t read_IRQreg_ivpr(struct openpic *opp, int n_IRQ)
576 {
577 	return opp->src[n_IRQ].ivpr;
578 }
579 
write_IRQreg_idr(struct openpic * opp,int n_IRQ,uint32_t val)580 static inline void write_IRQreg_idr(struct openpic *opp, int n_IRQ,
581 				    uint32_t val)
582 {
583 	struct irq_source *src = &opp->src[n_IRQ];
584 	uint32_t normal_mask = (1UL << opp->nb_cpus) - 1;
585 	uint32_t crit_mask = 0;
586 	uint32_t mask = normal_mask;
587 	int crit_shift = IDR_EP_SHIFT - opp->nb_cpus;
588 	int i;
589 
590 	if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
591 		crit_mask = mask << crit_shift;
592 		mask |= crit_mask | IDR_EP;
593 	}
594 
595 	src->idr = val & mask;
596 	pr_debug("Set IDR %d to 0x%08x\n", n_IRQ, src->idr);
597 
598 	if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
599 		if (src->idr & crit_mask) {
600 			if (src->idr & normal_mask) {
601 				pr_debug("%s: IRQ configured for multiple output types, using critical\n",
602 					__func__);
603 			}
604 
605 			src->output = ILR_INTTGT_CINT;
606 			src->nomask = true;
607 			src->destmask = 0;
608 
609 			for (i = 0; i < opp->nb_cpus; i++) {
610 				int n_ci = IDR_CI0_SHIFT - i;
611 
612 				if (src->idr & (1UL << n_ci))
613 					src->destmask |= 1UL << i;
614 			}
615 		} else {
616 			src->output = ILR_INTTGT_INT;
617 			src->nomask = false;
618 			src->destmask = src->idr & normal_mask;
619 		}
620 	} else {
621 		src->destmask = src->idr;
622 	}
623 }
624 
write_IRQreg_ilr(struct openpic * opp,int n_IRQ,uint32_t val)625 static inline void write_IRQreg_ilr(struct openpic *opp, int n_IRQ,
626 				    uint32_t val)
627 {
628 	if (opp->flags & OPENPIC_FLAG_ILR) {
629 		struct irq_source *src = &opp->src[n_IRQ];
630 
631 		src->output = val & ILR_INTTGT_MASK;
632 		pr_debug("Set ILR %d to 0x%08x, output %d\n", n_IRQ, src->idr,
633 			src->output);
634 
635 		/* TODO: on MPIC v4.0 only, set nomask for non-INT */
636 	}
637 }
638 
write_IRQreg_ivpr(struct openpic * opp,int n_IRQ,uint32_t val)639 static inline void write_IRQreg_ivpr(struct openpic *opp, int n_IRQ,
640 				     uint32_t val)
641 {
642 	uint32_t mask;
643 
644 	/* NOTE when implementing newer FSL MPIC models: starting with v4.0,
645 	 * the polarity bit is read-only on internal interrupts.
646 	 */
647 	mask = IVPR_MASK_MASK | IVPR_PRIORITY_MASK | IVPR_SENSE_MASK |
648 	    IVPR_POLARITY_MASK | opp->vector_mask;
649 
650 	/* ACTIVITY bit is read-only */
651 	opp->src[n_IRQ].ivpr =
652 	    (opp->src[n_IRQ].ivpr & IVPR_ACTIVITY_MASK) | (val & mask);
653 
654 	/* For FSL internal interrupts, The sense bit is reserved and zero,
655 	 * and the interrupt is always level-triggered.  Timers and IPIs
656 	 * have no sense or polarity bits, and are edge-triggered.
657 	 */
658 	switch (opp->src[n_IRQ].type) {
659 	case IRQ_TYPE_NORMAL:
660 		opp->src[n_IRQ].level =
661 		    !!(opp->src[n_IRQ].ivpr & IVPR_SENSE_MASK);
662 		break;
663 
664 	case IRQ_TYPE_FSLINT:
665 		opp->src[n_IRQ].ivpr &= ~IVPR_SENSE_MASK;
666 		break;
667 
668 	case IRQ_TYPE_FSLSPECIAL:
669 		opp->src[n_IRQ].ivpr &= ~(IVPR_POLARITY_MASK | IVPR_SENSE_MASK);
670 		break;
671 	}
672 
673 	openpic_update_irq(opp, n_IRQ);
674 	pr_debug("Set IVPR %d to 0x%08x -> 0x%08x\n", n_IRQ, val,
675 		opp->src[n_IRQ].ivpr);
676 }
677 
openpic_gcr_write(struct openpic * opp,uint64_t val)678 static void openpic_gcr_write(struct openpic *opp, uint64_t val)
679 {
680 	if (val & GCR_RESET) {
681 		openpic_reset(opp);
682 		return;
683 	}
684 
685 	opp->gcr &= ~opp->mpic_mode_mask;
686 	opp->gcr |= val & opp->mpic_mode_mask;
687 }
688 
openpic_gbl_write(void * opaque,gpa_t addr,u32 val)689 static int openpic_gbl_write(void *opaque, gpa_t addr, u32 val)
690 {
691 	struct openpic *opp = opaque;
692 	int err = 0;
693 
694 	pr_debug("%s: addr %#llx <= %08x\n", __func__, addr, val);
695 	if (addr & 0xF)
696 		return 0;
697 
698 	switch (addr) {
699 	case 0x00:	/* Block Revision Register1 (BRR1) is Readonly */
700 		break;
701 	case 0x40:
702 	case 0x50:
703 	case 0x60:
704 	case 0x70:
705 	case 0x80:
706 	case 0x90:
707 	case 0xA0:
708 	case 0xB0:
709 		err = openpic_cpu_write_internal(opp, addr, val,
710 						 get_current_cpu());
711 		break;
712 	case 0x1000:		/* FRR */
713 		break;
714 	case 0x1020:		/* GCR */
715 		openpic_gcr_write(opp, val);
716 		break;
717 	case 0x1080:		/* VIR */
718 		break;
719 	case 0x1090:		/* PIR */
720 		/*
721 		 * This register is used to reset a CPU core --
722 		 * let userspace handle it.
723 		 */
724 		err = -ENXIO;
725 		break;
726 	case 0x10A0:		/* IPI_IVPR */
727 	case 0x10B0:
728 	case 0x10C0:
729 	case 0x10D0: {
730 		int idx;
731 		idx = (addr - 0x10A0) >> 4;
732 		write_IRQreg_ivpr(opp, opp->irq_ipi0 + idx, val);
733 		break;
734 	}
735 	case 0x10E0:		/* SPVE */
736 		opp->spve = val & opp->vector_mask;
737 		break;
738 	default:
739 		break;
740 	}
741 
742 	return err;
743 }
744 
openpic_gbl_read(void * opaque,gpa_t addr,u32 * ptr)745 static int openpic_gbl_read(void *opaque, gpa_t addr, u32 *ptr)
746 {
747 	struct openpic *opp = opaque;
748 	u32 retval;
749 	int err = 0;
750 
751 	pr_debug("%s: addr %#llx\n", __func__, addr);
752 	retval = 0xFFFFFFFF;
753 	if (addr & 0xF)
754 		goto out;
755 
756 	switch (addr) {
757 	case 0x1000:		/* FRR */
758 		retval = opp->frr;
759 		retval |= (opp->nb_cpus - 1) << FRR_NCPU_SHIFT;
760 		break;
761 	case 0x1020:		/* GCR */
762 		retval = opp->gcr;
763 		break;
764 	case 0x1080:		/* VIR */
765 		retval = opp->vir;
766 		break;
767 	case 0x1090:		/* PIR */
768 		retval = 0x00000000;
769 		break;
770 	case 0x00:		/* Block Revision Register1 (BRR1) */
771 		retval = opp->brr1;
772 		break;
773 	case 0x40:
774 	case 0x50:
775 	case 0x60:
776 	case 0x70:
777 	case 0x80:
778 	case 0x90:
779 	case 0xA0:
780 	case 0xB0:
781 		err = openpic_cpu_read_internal(opp, addr,
782 			&retval, get_current_cpu());
783 		break;
784 	case 0x10A0:		/* IPI_IVPR */
785 	case 0x10B0:
786 	case 0x10C0:
787 	case 0x10D0:
788 		{
789 			int idx;
790 			idx = (addr - 0x10A0) >> 4;
791 			retval = read_IRQreg_ivpr(opp, opp->irq_ipi0 + idx);
792 		}
793 		break;
794 	case 0x10E0:		/* SPVE */
795 		retval = opp->spve;
796 		break;
797 	default:
798 		break;
799 	}
800 
801 out:
802 	pr_debug("%s: => 0x%08x\n", __func__, retval);
803 	*ptr = retval;
804 	return err;
805 }
806 
openpic_tmr_write(void * opaque,gpa_t addr,u32 val)807 static int openpic_tmr_write(void *opaque, gpa_t addr, u32 val)
808 {
809 	struct openpic *opp = opaque;
810 	int idx;
811 
812 	addr += 0x10f0;
813 
814 	pr_debug("%s: addr %#llx <= %08x\n", __func__, addr, val);
815 	if (addr & 0xF)
816 		return 0;
817 
818 	if (addr == 0x10f0) {
819 		/* TFRR */
820 		opp->tfrr = val;
821 		return 0;
822 	}
823 
824 	idx = (addr >> 6) & 0x3;
825 	addr = addr & 0x30;
826 
827 	switch (addr & 0x30) {
828 	case 0x00:		/* TCCR */
829 		break;
830 	case 0x10:		/* TBCR */
831 		if ((opp->timers[idx].tccr & TCCR_TOG) != 0 &&
832 		    (val & TBCR_CI) == 0 &&
833 		    (opp->timers[idx].tbcr & TBCR_CI) != 0)
834 			opp->timers[idx].tccr &= ~TCCR_TOG;
835 
836 		opp->timers[idx].tbcr = val;
837 		break;
838 	case 0x20:		/* TVPR */
839 		write_IRQreg_ivpr(opp, opp->irq_tim0 + idx, val);
840 		break;
841 	case 0x30:		/* TDR */
842 		write_IRQreg_idr(opp, opp->irq_tim0 + idx, val);
843 		break;
844 	}
845 
846 	return 0;
847 }
848 
openpic_tmr_read(void * opaque,gpa_t addr,u32 * ptr)849 static int openpic_tmr_read(void *opaque, gpa_t addr, u32 *ptr)
850 {
851 	struct openpic *opp = opaque;
852 	uint32_t retval = -1;
853 	int idx;
854 
855 	pr_debug("%s: addr %#llx\n", __func__, addr);
856 	if (addr & 0xF)
857 		goto out;
858 
859 	idx = (addr >> 6) & 0x3;
860 	if (addr == 0x0) {
861 		/* TFRR */
862 		retval = opp->tfrr;
863 		goto out;
864 	}
865 
866 	switch (addr & 0x30) {
867 	case 0x00:		/* TCCR */
868 		retval = opp->timers[idx].tccr;
869 		break;
870 	case 0x10:		/* TBCR */
871 		retval = opp->timers[idx].tbcr;
872 		break;
873 	case 0x20:		/* TIPV */
874 		retval = read_IRQreg_ivpr(opp, opp->irq_tim0 + idx);
875 		break;
876 	case 0x30:		/* TIDE (TIDR) */
877 		retval = read_IRQreg_idr(opp, opp->irq_tim0 + idx);
878 		break;
879 	}
880 
881 out:
882 	pr_debug("%s: => 0x%08x\n", __func__, retval);
883 	*ptr = retval;
884 	return 0;
885 }
886 
openpic_src_write(void * opaque,gpa_t addr,u32 val)887 static int openpic_src_write(void *opaque, gpa_t addr, u32 val)
888 {
889 	struct openpic *opp = opaque;
890 	int idx;
891 
892 	pr_debug("%s: addr %#llx <= %08x\n", __func__, addr, val);
893 
894 	addr = addr & 0xffff;
895 	idx = addr >> 5;
896 
897 	switch (addr & 0x1f) {
898 	case 0x00:
899 		write_IRQreg_ivpr(opp, idx, val);
900 		break;
901 	case 0x10:
902 		write_IRQreg_idr(opp, idx, val);
903 		break;
904 	case 0x18:
905 		write_IRQreg_ilr(opp, idx, val);
906 		break;
907 	}
908 
909 	return 0;
910 }
911 
openpic_src_read(void * opaque,gpa_t addr,u32 * ptr)912 static int openpic_src_read(void *opaque, gpa_t addr, u32 *ptr)
913 {
914 	struct openpic *opp = opaque;
915 	uint32_t retval;
916 	int idx;
917 
918 	pr_debug("%s: addr %#llx\n", __func__, addr);
919 	retval = 0xFFFFFFFF;
920 
921 	addr = addr & 0xffff;
922 	idx = addr >> 5;
923 
924 	switch (addr & 0x1f) {
925 	case 0x00:
926 		retval = read_IRQreg_ivpr(opp, idx);
927 		break;
928 	case 0x10:
929 		retval = read_IRQreg_idr(opp, idx);
930 		break;
931 	case 0x18:
932 		retval = read_IRQreg_ilr(opp, idx);
933 		break;
934 	}
935 
936 	pr_debug("%s: => 0x%08x\n", __func__, retval);
937 	*ptr = retval;
938 	return 0;
939 }
940 
openpic_msi_write(void * opaque,gpa_t addr,u32 val)941 static int openpic_msi_write(void *opaque, gpa_t addr, u32 val)
942 {
943 	struct openpic *opp = opaque;
944 	int idx = opp->irq_msi;
945 	int srs, ibs;
946 
947 	pr_debug("%s: addr %#llx <= 0x%08x\n", __func__, addr, val);
948 	if (addr & 0xF)
949 		return 0;
950 
951 	switch (addr) {
952 	case MSIIR_OFFSET:
953 		srs = val >> MSIIR_SRS_SHIFT;
954 		idx += srs;
955 		ibs = (val & MSIIR_IBS_MASK) >> MSIIR_IBS_SHIFT;
956 		opp->msi[srs].msir |= 1 << ibs;
957 		openpic_set_irq(opp, idx, 1);
958 		break;
959 	default:
960 		/* most registers are read-only, thus ignored */
961 		break;
962 	}
963 
964 	return 0;
965 }
966 
openpic_msi_read(void * opaque,gpa_t addr,u32 * ptr)967 static int openpic_msi_read(void *opaque, gpa_t addr, u32 *ptr)
968 {
969 	struct openpic *opp = opaque;
970 	uint32_t r = 0;
971 	int i, srs;
972 
973 	pr_debug("%s: addr %#llx\n", __func__, addr);
974 	if (addr & 0xF)
975 		return -ENXIO;
976 
977 	srs = addr >> 4;
978 
979 	switch (addr) {
980 	case 0x00:
981 	case 0x10:
982 	case 0x20:
983 	case 0x30:
984 	case 0x40:
985 	case 0x50:
986 	case 0x60:
987 	case 0x70:		/* MSIRs */
988 		r = opp->msi[srs].msir;
989 		/* Clear on read */
990 		opp->msi[srs].msir = 0;
991 		openpic_set_irq(opp, opp->irq_msi + srs, 0);
992 		break;
993 	case 0x120:		/* MSISR */
994 		for (i = 0; i < MAX_MSI; i++)
995 			r |= (opp->msi[i].msir ? 1 : 0) << i;
996 		break;
997 	}
998 
999 	pr_debug("%s: => 0x%08x\n", __func__, r);
1000 	*ptr = r;
1001 	return 0;
1002 }
1003 
openpic_summary_read(void * opaque,gpa_t addr,u32 * ptr)1004 static int openpic_summary_read(void *opaque, gpa_t addr, u32 *ptr)
1005 {
1006 	uint32_t r = 0;
1007 
1008 	pr_debug("%s: addr %#llx\n", __func__, addr);
1009 
1010 	/* TODO: EISR/EIMR */
1011 
1012 	*ptr = r;
1013 	return 0;
1014 }
1015 
openpic_summary_write(void * opaque,gpa_t addr,u32 val)1016 static int openpic_summary_write(void *opaque, gpa_t addr, u32 val)
1017 {
1018 	pr_debug("%s: addr %#llx <= 0x%08x\n", __func__, addr, val);
1019 
1020 	/* TODO: EISR/EIMR */
1021 	return 0;
1022 }
1023 
openpic_cpu_write_internal(void * opaque,gpa_t addr,u32 val,int idx)1024 static int openpic_cpu_write_internal(void *opaque, gpa_t addr,
1025 				      u32 val, int idx)
1026 {
1027 	struct openpic *opp = opaque;
1028 	struct irq_source *src;
1029 	struct irq_dest *dst;
1030 	int s_IRQ, n_IRQ;
1031 
1032 	pr_debug("%s: cpu %d addr %#llx <= 0x%08x\n", __func__, idx,
1033 		addr, val);
1034 
1035 	if (idx < 0)
1036 		return 0;
1037 
1038 	if (addr & 0xF)
1039 		return 0;
1040 
1041 	dst = &opp->dst[idx];
1042 	addr &= 0xFF0;
1043 	switch (addr) {
1044 	case 0x40:		/* IPIDR */
1045 	case 0x50:
1046 	case 0x60:
1047 	case 0x70:
1048 		idx = (addr - 0x40) >> 4;
1049 		/* we use IDE as mask which CPUs to deliver the IPI to still. */
1050 		opp->src[opp->irq_ipi0 + idx].destmask |= val;
1051 		openpic_set_irq(opp, opp->irq_ipi0 + idx, 1);
1052 		openpic_set_irq(opp, opp->irq_ipi0 + idx, 0);
1053 		break;
1054 	case 0x80:		/* CTPR */
1055 		dst->ctpr = val & 0x0000000F;
1056 
1057 		pr_debug("%s: set CPU %d ctpr to %d, raised %d servicing %d\n",
1058 			__func__, idx, dst->ctpr, dst->raised.priority,
1059 			dst->servicing.priority);
1060 
1061 		if (dst->raised.priority <= dst->ctpr) {
1062 			pr_debug("%s: Lower OpenPIC INT output cpu %d due to ctpr\n",
1063 				__func__, idx);
1064 			mpic_irq_lower(opp, dst, ILR_INTTGT_INT);
1065 		} else if (dst->raised.priority > dst->servicing.priority) {
1066 			pr_debug("%s: Raise OpenPIC INT output cpu %d irq %d\n",
1067 				__func__, idx, dst->raised.next);
1068 			mpic_irq_raise(opp, dst, ILR_INTTGT_INT);
1069 		}
1070 
1071 		break;
1072 	case 0x90:		/* WHOAMI */
1073 		/* Read-only register */
1074 		break;
1075 	case 0xA0:		/* IACK */
1076 		/* Read-only register */
1077 		break;
1078 	case 0xB0: {		/* EOI */
1079 		int notify_eoi;
1080 
1081 		pr_debug("EOI\n");
1082 		s_IRQ = IRQ_get_next(opp, &dst->servicing);
1083 
1084 		if (s_IRQ < 0) {
1085 			pr_debug("%s: EOI with no interrupt in service\n",
1086 				__func__);
1087 			break;
1088 		}
1089 
1090 		IRQ_resetbit(&dst->servicing, s_IRQ);
1091 		/* Notify listeners that the IRQ is over */
1092 		notify_eoi = s_IRQ;
1093 		/* Set up next servicing IRQ */
1094 		s_IRQ = IRQ_get_next(opp, &dst->servicing);
1095 		/* Check queued interrupts. */
1096 		n_IRQ = IRQ_get_next(opp, &dst->raised);
1097 		src = &opp->src[n_IRQ];
1098 		if (n_IRQ != -1 &&
1099 		    (s_IRQ == -1 ||
1100 		     IVPR_PRIORITY(src->ivpr) > dst->servicing.priority)) {
1101 			pr_debug("Raise OpenPIC INT output cpu %d irq %d\n",
1102 				idx, n_IRQ);
1103 			mpic_irq_raise(opp, dst, ILR_INTTGT_INT);
1104 		}
1105 
1106 		spin_unlock(&opp->lock);
1107 		kvm_notify_acked_irq(opp->kvm, 0, notify_eoi);
1108 		spin_lock(&opp->lock);
1109 
1110 		break;
1111 	}
1112 	default:
1113 		break;
1114 	}
1115 
1116 	return 0;
1117 }
1118 
openpic_cpu_write(void * opaque,gpa_t addr,u32 val)1119 static int openpic_cpu_write(void *opaque, gpa_t addr, u32 val)
1120 {
1121 	struct openpic *opp = opaque;
1122 
1123 	return openpic_cpu_write_internal(opp, addr, val,
1124 					 (addr & 0x1f000) >> 12);
1125 }
1126 
openpic_iack(struct openpic * opp,struct irq_dest * dst,int cpu)1127 static uint32_t openpic_iack(struct openpic *opp, struct irq_dest *dst,
1128 			     int cpu)
1129 {
1130 	struct irq_source *src;
1131 	int retval, irq;
1132 
1133 	pr_debug("Lower OpenPIC INT output\n");
1134 	mpic_irq_lower(opp, dst, ILR_INTTGT_INT);
1135 
1136 	irq = IRQ_get_next(opp, &dst->raised);
1137 	pr_debug("IACK: irq=%d\n", irq);
1138 
1139 	if (irq == -1)
1140 		/* No more interrupt pending */
1141 		return opp->spve;
1142 
1143 	src = &opp->src[irq];
1144 	if (!(src->ivpr & IVPR_ACTIVITY_MASK) ||
1145 	    !(IVPR_PRIORITY(src->ivpr) > dst->ctpr)) {
1146 		pr_err("%s: bad raised IRQ %d ctpr %d ivpr 0x%08x\n",
1147 			__func__, irq, dst->ctpr, src->ivpr);
1148 		openpic_update_irq(opp, irq);
1149 		retval = opp->spve;
1150 	} else {
1151 		/* IRQ enter servicing state */
1152 		IRQ_setbit(&dst->servicing, irq);
1153 		retval = IVPR_VECTOR(opp, src->ivpr);
1154 	}
1155 
1156 	if (!src->level) {
1157 		/* edge-sensitive IRQ */
1158 		src->ivpr &= ~IVPR_ACTIVITY_MASK;
1159 		src->pending = 0;
1160 		IRQ_resetbit(&dst->raised, irq);
1161 	}
1162 
1163 	if ((irq >= opp->irq_ipi0) && (irq < (opp->irq_ipi0 + MAX_IPI))) {
1164 		src->destmask &= ~(1 << cpu);
1165 		if (src->destmask && !src->level) {
1166 			/* trigger on CPUs that didn't know about it yet */
1167 			openpic_set_irq(opp, irq, 1);
1168 			openpic_set_irq(opp, irq, 0);
1169 			/* if all CPUs knew about it, set active bit again */
1170 			src->ivpr |= IVPR_ACTIVITY_MASK;
1171 		}
1172 	}
1173 
1174 	return retval;
1175 }
1176 
kvmppc_mpic_set_epr(struct kvm_vcpu * vcpu)1177 void kvmppc_mpic_set_epr(struct kvm_vcpu *vcpu)
1178 {
1179 	struct openpic *opp = vcpu->arch.mpic;
1180 	int cpu = vcpu->arch.irq_cpu_id;
1181 	unsigned long flags;
1182 
1183 	spin_lock_irqsave(&opp->lock, flags);
1184 
1185 	if ((opp->gcr & opp->mpic_mode_mask) == GCR_MODE_PROXY)
1186 		kvmppc_set_epr(vcpu, openpic_iack(opp, &opp->dst[cpu], cpu));
1187 
1188 	spin_unlock_irqrestore(&opp->lock, flags);
1189 }
1190 
openpic_cpu_read_internal(void * opaque,gpa_t addr,u32 * ptr,int idx)1191 static int openpic_cpu_read_internal(void *opaque, gpa_t addr,
1192 				     u32 *ptr, int idx)
1193 {
1194 	struct openpic *opp = opaque;
1195 	struct irq_dest *dst;
1196 	uint32_t retval;
1197 
1198 	pr_debug("%s: cpu %d addr %#llx\n", __func__, idx, addr);
1199 	retval = 0xFFFFFFFF;
1200 
1201 	if (idx < 0)
1202 		goto out;
1203 
1204 	if (addr & 0xF)
1205 		goto out;
1206 
1207 	dst = &opp->dst[idx];
1208 	addr &= 0xFF0;
1209 	switch (addr) {
1210 	case 0x80:		/* CTPR */
1211 		retval = dst->ctpr;
1212 		break;
1213 	case 0x90:		/* WHOAMI */
1214 		retval = idx;
1215 		break;
1216 	case 0xA0:		/* IACK */
1217 		retval = openpic_iack(opp, dst, idx);
1218 		break;
1219 	case 0xB0:		/* EOI */
1220 		retval = 0;
1221 		break;
1222 	default:
1223 		break;
1224 	}
1225 	pr_debug("%s: => 0x%08x\n", __func__, retval);
1226 
1227 out:
1228 	*ptr = retval;
1229 	return 0;
1230 }
1231 
openpic_cpu_read(void * opaque,gpa_t addr,u32 * ptr)1232 static int openpic_cpu_read(void *opaque, gpa_t addr, u32 *ptr)
1233 {
1234 	struct openpic *opp = opaque;
1235 
1236 	return openpic_cpu_read_internal(opp, addr, ptr,
1237 					 (addr & 0x1f000) >> 12);
1238 }
1239 
1240 struct mem_reg {
1241 	int (*read)(void *opaque, gpa_t addr, u32 *ptr);
1242 	int (*write)(void *opaque, gpa_t addr, u32 val);
1243 	gpa_t start_addr;
1244 	int size;
1245 };
1246 
1247 static const struct mem_reg openpic_gbl_mmio = {
1248 	.write = openpic_gbl_write,
1249 	.read = openpic_gbl_read,
1250 	.start_addr = OPENPIC_GLB_REG_START,
1251 	.size = OPENPIC_GLB_REG_SIZE,
1252 };
1253 
1254 static const struct mem_reg openpic_tmr_mmio = {
1255 	.write = openpic_tmr_write,
1256 	.read = openpic_tmr_read,
1257 	.start_addr = OPENPIC_TMR_REG_START,
1258 	.size = OPENPIC_TMR_REG_SIZE,
1259 };
1260 
1261 static const struct mem_reg openpic_cpu_mmio = {
1262 	.write = openpic_cpu_write,
1263 	.read = openpic_cpu_read,
1264 	.start_addr = OPENPIC_CPU_REG_START,
1265 	.size = OPENPIC_CPU_REG_SIZE,
1266 };
1267 
1268 static const struct mem_reg openpic_src_mmio = {
1269 	.write = openpic_src_write,
1270 	.read = openpic_src_read,
1271 	.start_addr = OPENPIC_SRC_REG_START,
1272 	.size = OPENPIC_SRC_REG_SIZE,
1273 };
1274 
1275 static const struct mem_reg openpic_msi_mmio = {
1276 	.read = openpic_msi_read,
1277 	.write = openpic_msi_write,
1278 	.start_addr = OPENPIC_MSI_REG_START,
1279 	.size = OPENPIC_MSI_REG_SIZE,
1280 };
1281 
1282 static const struct mem_reg openpic_summary_mmio = {
1283 	.read = openpic_summary_read,
1284 	.write = openpic_summary_write,
1285 	.start_addr = OPENPIC_SUMMARY_REG_START,
1286 	.size = OPENPIC_SUMMARY_REG_SIZE,
1287 };
1288 
add_mmio_region(struct openpic * opp,const struct mem_reg * mr)1289 static void add_mmio_region(struct openpic *opp, const struct mem_reg *mr)
1290 {
1291 	if (opp->num_mmio_regions >= MAX_MMIO_REGIONS) {
1292 		WARN(1, "kvm mpic: too many mmio regions\n");
1293 		return;
1294 	}
1295 
1296 	opp->mmio_regions[opp->num_mmio_regions++] = mr;
1297 }
1298 
fsl_common_init(struct openpic * opp)1299 static void fsl_common_init(struct openpic *opp)
1300 {
1301 	int i;
1302 	int virq = MAX_SRC;
1303 
1304 	add_mmio_region(opp, &openpic_msi_mmio);
1305 	add_mmio_region(opp, &openpic_summary_mmio);
1306 
1307 	opp->vid = VID_REVISION_1_2;
1308 	opp->vir = VIR_GENERIC;
1309 	opp->vector_mask = 0xFFFF;
1310 	opp->tfrr_reset = 0;
1311 	opp->ivpr_reset = IVPR_MASK_MASK;
1312 	opp->idr_reset = 1 << 0;
1313 	opp->max_irq = MAX_IRQ;
1314 
1315 	opp->irq_ipi0 = virq;
1316 	virq += MAX_IPI;
1317 	opp->irq_tim0 = virq;
1318 	virq += MAX_TMR;
1319 
1320 	BUG_ON(virq > MAX_IRQ);
1321 
1322 	opp->irq_msi = 224;
1323 
1324 	for (i = 0; i < opp->fsl->max_ext; i++)
1325 		opp->src[i].level = false;
1326 
1327 	/* Internal interrupts, including message and MSI */
1328 	for (i = 16; i < MAX_SRC; i++) {
1329 		opp->src[i].type = IRQ_TYPE_FSLINT;
1330 		opp->src[i].level = true;
1331 	}
1332 
1333 	/* timers and IPIs */
1334 	for (i = MAX_SRC; i < virq; i++) {
1335 		opp->src[i].type = IRQ_TYPE_FSLSPECIAL;
1336 		opp->src[i].level = false;
1337 	}
1338 }
1339 
kvm_mpic_read_internal(struct openpic * opp,gpa_t addr,u32 * ptr)1340 static int kvm_mpic_read_internal(struct openpic *opp, gpa_t addr, u32 *ptr)
1341 {
1342 	int i;
1343 
1344 	for (i = 0; i < opp->num_mmio_regions; i++) {
1345 		const struct mem_reg *mr = opp->mmio_regions[i];
1346 
1347 		if (mr->start_addr > addr || addr >= mr->start_addr + mr->size)
1348 			continue;
1349 
1350 		return mr->read(opp, addr - mr->start_addr, ptr);
1351 	}
1352 
1353 	return -ENXIO;
1354 }
1355 
kvm_mpic_write_internal(struct openpic * opp,gpa_t addr,u32 val)1356 static int kvm_mpic_write_internal(struct openpic *opp, gpa_t addr, u32 val)
1357 {
1358 	int i;
1359 
1360 	for (i = 0; i < opp->num_mmio_regions; i++) {
1361 		const struct mem_reg *mr = opp->mmio_regions[i];
1362 
1363 		if (mr->start_addr > addr || addr >= mr->start_addr + mr->size)
1364 			continue;
1365 
1366 		return mr->write(opp, addr - mr->start_addr, val);
1367 	}
1368 
1369 	return -ENXIO;
1370 }
1371 
kvm_mpic_read(struct kvm_vcpu * vcpu,struct kvm_io_device * this,gpa_t addr,int len,void * ptr)1372 static int kvm_mpic_read(struct kvm_vcpu *vcpu,
1373 			 struct kvm_io_device *this,
1374 			 gpa_t addr, int len, void *ptr)
1375 {
1376 	struct openpic *opp = container_of(this, struct openpic, mmio);
1377 	int ret;
1378 	union {
1379 		u32 val;
1380 		u8 bytes[4];
1381 	} u;
1382 
1383 	if (addr & (len - 1)) {
1384 		pr_debug("%s: bad alignment %llx/%d\n",
1385 			 __func__, addr, len);
1386 		return -EINVAL;
1387 	}
1388 
1389 	spin_lock_irq(&opp->lock);
1390 	ret = kvm_mpic_read_internal(opp, addr - opp->reg_base, &u.val);
1391 	spin_unlock_irq(&opp->lock);
1392 
1393 	/*
1394 	 * Technically only 32-bit accesses are allowed, but be nice to
1395 	 * people dumping registers a byte at a time -- it works in real
1396 	 * hardware (reads only, not writes).
1397 	 */
1398 	if (len == 4) {
1399 		*(u32 *)ptr = u.val;
1400 		pr_debug("%s: addr %llx ret %d len 4 val %x\n",
1401 			 __func__, addr, ret, u.val);
1402 	} else if (len == 1) {
1403 		*(u8 *)ptr = u.bytes[addr & 3];
1404 		pr_debug("%s: addr %llx ret %d len 1 val %x\n",
1405 			 __func__, addr, ret, u.bytes[addr & 3]);
1406 	} else {
1407 		pr_debug("%s: bad length %d\n", __func__, len);
1408 		return -EINVAL;
1409 	}
1410 
1411 	return ret;
1412 }
1413 
kvm_mpic_write(struct kvm_vcpu * vcpu,struct kvm_io_device * this,gpa_t addr,int len,const void * ptr)1414 static int kvm_mpic_write(struct kvm_vcpu *vcpu,
1415 			  struct kvm_io_device *this,
1416 			  gpa_t addr, int len, const void *ptr)
1417 {
1418 	struct openpic *opp = container_of(this, struct openpic, mmio);
1419 	int ret;
1420 
1421 	if (len != 4) {
1422 		pr_debug("%s: bad length %d\n", __func__, len);
1423 		return -EOPNOTSUPP;
1424 	}
1425 	if (addr & 3) {
1426 		pr_debug("%s: bad alignment %llx/%d\n", __func__, addr, len);
1427 		return -EOPNOTSUPP;
1428 	}
1429 
1430 	spin_lock_irq(&opp->lock);
1431 	ret = kvm_mpic_write_internal(opp, addr - opp->reg_base,
1432 				      *(const u32 *)ptr);
1433 	spin_unlock_irq(&opp->lock);
1434 
1435 	pr_debug("%s: addr %llx ret %d val %x\n",
1436 		 __func__, addr, ret, *(const u32 *)ptr);
1437 
1438 	return ret;
1439 }
1440 
1441 static const struct kvm_io_device_ops mpic_mmio_ops = {
1442 	.read = kvm_mpic_read,
1443 	.write = kvm_mpic_write,
1444 };
1445 
map_mmio(struct openpic * opp)1446 static void map_mmio(struct openpic *opp)
1447 {
1448 	kvm_iodevice_init(&opp->mmio, &mpic_mmio_ops);
1449 
1450 	kvm_io_bus_register_dev(opp->kvm, KVM_MMIO_BUS,
1451 				opp->reg_base, OPENPIC_REG_SIZE,
1452 				&opp->mmio);
1453 }
1454 
unmap_mmio(struct openpic * opp)1455 static void unmap_mmio(struct openpic *opp)
1456 {
1457 	kvm_io_bus_unregister_dev(opp->kvm, KVM_MMIO_BUS, &opp->mmio);
1458 }
1459 
set_base_addr(struct openpic * opp,struct kvm_device_attr * attr)1460 static int set_base_addr(struct openpic *opp, struct kvm_device_attr *attr)
1461 {
1462 	u64 base;
1463 
1464 	if (copy_from_user(&base, (u64 __user *)(long)attr->addr, sizeof(u64)))
1465 		return -EFAULT;
1466 
1467 	if (base & 0x3ffff) {
1468 		pr_debug("kvm mpic %s: KVM_DEV_MPIC_BASE_ADDR %08llx not aligned\n",
1469 			 __func__, base);
1470 		return -EINVAL;
1471 	}
1472 
1473 	if (base == opp->reg_base)
1474 		return 0;
1475 
1476 	mutex_lock(&opp->kvm->slots_lock);
1477 
1478 	unmap_mmio(opp);
1479 	opp->reg_base = base;
1480 
1481 	pr_debug("kvm mpic %s: KVM_DEV_MPIC_BASE_ADDR %08llx\n",
1482 		 __func__, base);
1483 
1484 	if (base == 0)
1485 		goto out;
1486 
1487 	map_mmio(opp);
1488 
1489 out:
1490 	mutex_unlock(&opp->kvm->slots_lock);
1491 	return 0;
1492 }
1493 
1494 #define ATTR_SET		0
1495 #define ATTR_GET		1
1496 
access_reg(struct openpic * opp,gpa_t addr,u32 * val,int type)1497 static int access_reg(struct openpic *opp, gpa_t addr, u32 *val, int type)
1498 {
1499 	int ret;
1500 
1501 	if (addr & 3)
1502 		return -ENXIO;
1503 
1504 	spin_lock_irq(&opp->lock);
1505 
1506 	if (type == ATTR_SET)
1507 		ret = kvm_mpic_write_internal(opp, addr, *val);
1508 	else
1509 		ret = kvm_mpic_read_internal(opp, addr, val);
1510 
1511 	spin_unlock_irq(&opp->lock);
1512 
1513 	pr_debug("%s: type %d addr %llx val %x\n", __func__, type, addr, *val);
1514 
1515 	return ret;
1516 }
1517 
mpic_set_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1518 static int mpic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1519 {
1520 	struct openpic *opp = dev->private;
1521 	u32 attr32;
1522 
1523 	switch (attr->group) {
1524 	case KVM_DEV_MPIC_GRP_MISC:
1525 		switch (attr->attr) {
1526 		case KVM_DEV_MPIC_BASE_ADDR:
1527 			return set_base_addr(opp, attr);
1528 		}
1529 
1530 		break;
1531 
1532 	case KVM_DEV_MPIC_GRP_REGISTER:
1533 		if (get_user(attr32, (u32 __user *)(long)attr->addr))
1534 			return -EFAULT;
1535 
1536 		return access_reg(opp, attr->attr, &attr32, ATTR_SET);
1537 
1538 	case KVM_DEV_MPIC_GRP_IRQ_ACTIVE:
1539 		if (attr->attr > MAX_SRC)
1540 			return -EINVAL;
1541 
1542 		if (get_user(attr32, (u32 __user *)(long)attr->addr))
1543 			return -EFAULT;
1544 
1545 		if (attr32 != 0 && attr32 != 1)
1546 			return -EINVAL;
1547 
1548 		spin_lock_irq(&opp->lock);
1549 		openpic_set_irq(opp, attr->attr, attr32);
1550 		spin_unlock_irq(&opp->lock);
1551 		return 0;
1552 	}
1553 
1554 	return -ENXIO;
1555 }
1556 
mpic_get_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1557 static int mpic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1558 {
1559 	struct openpic *opp = dev->private;
1560 	u64 attr64;
1561 	u32 attr32;
1562 	int ret;
1563 
1564 	switch (attr->group) {
1565 	case KVM_DEV_MPIC_GRP_MISC:
1566 		switch (attr->attr) {
1567 		case KVM_DEV_MPIC_BASE_ADDR:
1568 			mutex_lock(&opp->kvm->slots_lock);
1569 			attr64 = opp->reg_base;
1570 			mutex_unlock(&opp->kvm->slots_lock);
1571 
1572 			if (copy_to_user((u64 __user *)(long)attr->addr,
1573 					 &attr64, sizeof(u64)))
1574 				return -EFAULT;
1575 
1576 			return 0;
1577 		}
1578 
1579 		break;
1580 
1581 	case KVM_DEV_MPIC_GRP_REGISTER:
1582 		ret = access_reg(opp, attr->attr, &attr32, ATTR_GET);
1583 		if (ret)
1584 			return ret;
1585 
1586 		if (put_user(attr32, (u32 __user *)(long)attr->addr))
1587 			return -EFAULT;
1588 
1589 		return 0;
1590 
1591 	case KVM_DEV_MPIC_GRP_IRQ_ACTIVE:
1592 		if (attr->attr > MAX_SRC)
1593 			return -EINVAL;
1594 
1595 		spin_lock_irq(&opp->lock);
1596 		attr32 = opp->src[attr->attr].pending;
1597 		spin_unlock_irq(&opp->lock);
1598 
1599 		if (put_user(attr32, (u32 __user *)(long)attr->addr))
1600 			return -EFAULT;
1601 
1602 		return 0;
1603 	}
1604 
1605 	return -ENXIO;
1606 }
1607 
mpic_has_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1608 static int mpic_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1609 {
1610 	switch (attr->group) {
1611 	case KVM_DEV_MPIC_GRP_MISC:
1612 		switch (attr->attr) {
1613 		case KVM_DEV_MPIC_BASE_ADDR:
1614 			return 0;
1615 		}
1616 
1617 		break;
1618 
1619 	case KVM_DEV_MPIC_GRP_REGISTER:
1620 		return 0;
1621 
1622 	case KVM_DEV_MPIC_GRP_IRQ_ACTIVE:
1623 		if (attr->attr > MAX_SRC)
1624 			break;
1625 
1626 		return 0;
1627 	}
1628 
1629 	return -ENXIO;
1630 }
1631 
mpic_destroy(struct kvm_device * dev)1632 static void mpic_destroy(struct kvm_device *dev)
1633 {
1634 	struct openpic *opp = dev->private;
1635 
1636 	dev->kvm->arch.mpic = NULL;
1637 	kfree(opp);
1638 	kfree(dev);
1639 }
1640 
mpic_set_default_irq_routing(struct openpic * opp)1641 static int mpic_set_default_irq_routing(struct openpic *opp)
1642 {
1643 	struct kvm_irq_routing_entry *routing;
1644 
1645 	/* Create a nop default map, so that dereferencing it still works */
1646 	routing = kzalloc((sizeof(*routing)), GFP_KERNEL);
1647 	if (!routing)
1648 		return -ENOMEM;
1649 
1650 	kvm_set_irq_routing(opp->kvm, routing, 0, 0);
1651 
1652 	kfree(routing);
1653 	return 0;
1654 }
1655 
mpic_create(struct kvm_device * dev,u32 type)1656 static int mpic_create(struct kvm_device *dev, u32 type)
1657 {
1658 	struct openpic *opp;
1659 	int ret;
1660 
1661 	/* We only support one MPIC at a time for now */
1662 	if (dev->kvm->arch.mpic)
1663 		return -EINVAL;
1664 
1665 	opp = kzalloc(sizeof(struct openpic), GFP_KERNEL);
1666 	if (!opp)
1667 		return -ENOMEM;
1668 
1669 	dev->private = opp;
1670 	opp->kvm = dev->kvm;
1671 	opp->dev = dev;
1672 	opp->model = type;
1673 	spin_lock_init(&opp->lock);
1674 
1675 	add_mmio_region(opp, &openpic_gbl_mmio);
1676 	add_mmio_region(opp, &openpic_tmr_mmio);
1677 	add_mmio_region(opp, &openpic_src_mmio);
1678 	add_mmio_region(opp, &openpic_cpu_mmio);
1679 
1680 	switch (opp->model) {
1681 	case KVM_DEV_TYPE_FSL_MPIC_20:
1682 		opp->fsl = &fsl_mpic_20;
1683 		opp->brr1 = 0x00400200;
1684 		opp->flags |= OPENPIC_FLAG_IDR_CRIT;
1685 		opp->nb_irqs = 80;
1686 		opp->mpic_mode_mask = GCR_MODE_MIXED;
1687 
1688 		fsl_common_init(opp);
1689 
1690 		break;
1691 
1692 	case KVM_DEV_TYPE_FSL_MPIC_42:
1693 		opp->fsl = &fsl_mpic_42;
1694 		opp->brr1 = 0x00400402;
1695 		opp->flags |= OPENPIC_FLAG_ILR;
1696 		opp->nb_irqs = 196;
1697 		opp->mpic_mode_mask = GCR_MODE_PROXY;
1698 
1699 		fsl_common_init(opp);
1700 
1701 		break;
1702 
1703 	default:
1704 		ret = -ENODEV;
1705 		goto err;
1706 	}
1707 
1708 	ret = mpic_set_default_irq_routing(opp);
1709 	if (ret)
1710 		goto err;
1711 
1712 	openpic_reset(opp);
1713 
1714 	smp_wmb();
1715 	dev->kvm->arch.mpic = opp;
1716 
1717 	return 0;
1718 
1719 err:
1720 	kfree(opp);
1721 	return ret;
1722 }
1723 
1724 struct kvm_device_ops kvm_mpic_ops = {
1725 	.name = "kvm-mpic",
1726 	.create = mpic_create,
1727 	.destroy = mpic_destroy,
1728 	.set_attr = mpic_set_attr,
1729 	.get_attr = mpic_get_attr,
1730 	.has_attr = mpic_has_attr,
1731 };
1732 
kvmppc_mpic_connect_vcpu(struct kvm_device * dev,struct kvm_vcpu * vcpu,u32 cpu)1733 int kvmppc_mpic_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu,
1734 			     u32 cpu)
1735 {
1736 	struct openpic *opp = dev->private;
1737 	int ret = 0;
1738 
1739 	if (dev->ops != &kvm_mpic_ops)
1740 		return -EPERM;
1741 	if (opp->kvm != vcpu->kvm)
1742 		return -EPERM;
1743 	if (cpu < 0 || cpu >= MAX_CPU)
1744 		return -EPERM;
1745 
1746 	spin_lock_irq(&opp->lock);
1747 
1748 	if (opp->dst[cpu].vcpu) {
1749 		ret = -EEXIST;
1750 		goto out;
1751 	}
1752 	if (vcpu->arch.irq_type) {
1753 		ret = -EBUSY;
1754 		goto out;
1755 	}
1756 
1757 	opp->dst[cpu].vcpu = vcpu;
1758 	opp->nb_cpus = max(opp->nb_cpus, cpu + 1);
1759 
1760 	vcpu->arch.mpic = opp;
1761 	vcpu->arch.irq_cpu_id = cpu;
1762 	vcpu->arch.irq_type = KVMPPC_IRQ_MPIC;
1763 
1764 	/* This might need to be changed if GCR gets extended */
1765 	if (opp->mpic_mode_mask == GCR_MODE_PROXY)
1766 		vcpu->arch.epr_flags |= KVMPPC_EPR_KERNEL;
1767 
1768 out:
1769 	spin_unlock_irq(&opp->lock);
1770 	return ret;
1771 }
1772 
1773 /*
1774  * This should only happen immediately before the mpic is destroyed,
1775  * so we shouldn't need to worry about anything still trying to
1776  * access the vcpu pointer.
1777  */
kvmppc_mpic_disconnect_vcpu(struct openpic * opp,struct kvm_vcpu * vcpu)1778 void kvmppc_mpic_disconnect_vcpu(struct openpic *opp, struct kvm_vcpu *vcpu)
1779 {
1780 	BUG_ON(!opp->dst[vcpu->arch.irq_cpu_id].vcpu);
1781 
1782 	opp->dst[vcpu->arch.irq_cpu_id].vcpu = NULL;
1783 }
1784 
1785 /*
1786  * Return value:
1787  *  < 0   Interrupt was ignored (masked or not delivered for other reasons)
1788  *  = 0   Interrupt was coalesced (previous irq is still pending)
1789  *  > 0   Number of CPUs interrupt was delivered to
1790  */
mpic_set_irq(struct kvm_kernel_irq_routing_entry * e,struct kvm * kvm,int irq_source_id,int level,bool line_status)1791 static int mpic_set_irq(struct kvm_kernel_irq_routing_entry *e,
1792 			struct kvm *kvm, int irq_source_id, int level,
1793 			bool line_status)
1794 {
1795 	u32 irq = e->irqchip.pin;
1796 	struct openpic *opp = kvm->arch.mpic;
1797 	unsigned long flags;
1798 
1799 	spin_lock_irqsave(&opp->lock, flags);
1800 	openpic_set_irq(opp, irq, level);
1801 	spin_unlock_irqrestore(&opp->lock, flags);
1802 
1803 	/* All code paths we care about don't check for the return value */
1804 	return 0;
1805 }
1806 
kvm_set_msi(struct kvm_kernel_irq_routing_entry * e,struct kvm * kvm,int irq_source_id,int level,bool line_status)1807 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
1808 		struct kvm *kvm, int irq_source_id, int level, bool line_status)
1809 {
1810 	struct openpic *opp = kvm->arch.mpic;
1811 	unsigned long flags;
1812 
1813 	spin_lock_irqsave(&opp->lock, flags);
1814 
1815 	/*
1816 	 * XXX We ignore the target address for now, as we only support
1817 	 *     a single MSI bank.
1818 	 */
1819 	openpic_msi_write(kvm->arch.mpic, MSIIR_OFFSET, e->msi.data);
1820 	spin_unlock_irqrestore(&opp->lock, flags);
1821 
1822 	/* All code paths we care about don't check for the return value */
1823 	return 0;
1824 }
1825 
kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry * e,const struct kvm_irq_routing_entry * ue)1826 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e,
1827 			  const struct kvm_irq_routing_entry *ue)
1828 {
1829 	int r = -EINVAL;
1830 
1831 	switch (ue->type) {
1832 	case KVM_IRQ_ROUTING_IRQCHIP:
1833 		e->set = mpic_set_irq;
1834 		e->irqchip.irqchip = ue->u.irqchip.irqchip;
1835 		e->irqchip.pin = ue->u.irqchip.pin;
1836 		if (e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS)
1837 			goto out;
1838 		break;
1839 	case KVM_IRQ_ROUTING_MSI:
1840 		e->set = kvm_set_msi;
1841 		e->msi.address_lo = ue->u.msi.address_lo;
1842 		e->msi.address_hi = ue->u.msi.address_hi;
1843 		e->msi.data = ue->u.msi.data;
1844 		break;
1845 	default:
1846 		goto out;
1847 	}
1848 
1849 	r = 0;
1850 out:
1851 	return r;
1852 }
1853