• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012-2015 - ARM Ltd
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #include <hyp/adjust_pc.h>
8 
9 #include <linux/compiler.h>
10 #include <linux/irqchip/arm-gic-v3.h>
11 #include <linux/kvm_host.h>
12 
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_hyp.h>
15 #include <asm/kvm_mmu.h>
16 
17 #define vtr_to_max_lr_idx(v)		((v) & 0xf)
18 #define vtr_to_nr_pre_bits(v)		((((u32)(v) >> 26) & 7) + 1)
19 #define vtr_to_nr_apr_regs(v)		(1 << (vtr_to_nr_pre_bits(v) - 5))
20 
__gic_v3_get_lr(unsigned int lr)21 static u64 __gic_v3_get_lr(unsigned int lr)
22 {
23 	switch (lr & 0xf) {
24 	case 0:
25 		return read_gicreg(ICH_LR0_EL2);
26 	case 1:
27 		return read_gicreg(ICH_LR1_EL2);
28 	case 2:
29 		return read_gicreg(ICH_LR2_EL2);
30 	case 3:
31 		return read_gicreg(ICH_LR3_EL2);
32 	case 4:
33 		return read_gicreg(ICH_LR4_EL2);
34 	case 5:
35 		return read_gicreg(ICH_LR5_EL2);
36 	case 6:
37 		return read_gicreg(ICH_LR6_EL2);
38 	case 7:
39 		return read_gicreg(ICH_LR7_EL2);
40 	case 8:
41 		return read_gicreg(ICH_LR8_EL2);
42 	case 9:
43 		return read_gicreg(ICH_LR9_EL2);
44 	case 10:
45 		return read_gicreg(ICH_LR10_EL2);
46 	case 11:
47 		return read_gicreg(ICH_LR11_EL2);
48 	case 12:
49 		return read_gicreg(ICH_LR12_EL2);
50 	case 13:
51 		return read_gicreg(ICH_LR13_EL2);
52 	case 14:
53 		return read_gicreg(ICH_LR14_EL2);
54 	case 15:
55 		return read_gicreg(ICH_LR15_EL2);
56 	}
57 
58 	unreachable();
59 }
60 
__gic_v3_set_lr(u64 val,int lr)61 static void __gic_v3_set_lr(u64 val, int lr)
62 {
63 	switch (lr & 0xf) {
64 	case 0:
65 		write_gicreg(val, ICH_LR0_EL2);
66 		break;
67 	case 1:
68 		write_gicreg(val, ICH_LR1_EL2);
69 		break;
70 	case 2:
71 		write_gicreg(val, ICH_LR2_EL2);
72 		break;
73 	case 3:
74 		write_gicreg(val, ICH_LR3_EL2);
75 		break;
76 	case 4:
77 		write_gicreg(val, ICH_LR4_EL2);
78 		break;
79 	case 5:
80 		write_gicreg(val, ICH_LR5_EL2);
81 		break;
82 	case 6:
83 		write_gicreg(val, ICH_LR6_EL2);
84 		break;
85 	case 7:
86 		write_gicreg(val, ICH_LR7_EL2);
87 		break;
88 	case 8:
89 		write_gicreg(val, ICH_LR8_EL2);
90 		break;
91 	case 9:
92 		write_gicreg(val, ICH_LR9_EL2);
93 		break;
94 	case 10:
95 		write_gicreg(val, ICH_LR10_EL2);
96 		break;
97 	case 11:
98 		write_gicreg(val, ICH_LR11_EL2);
99 		break;
100 	case 12:
101 		write_gicreg(val, ICH_LR12_EL2);
102 		break;
103 	case 13:
104 		write_gicreg(val, ICH_LR13_EL2);
105 		break;
106 	case 14:
107 		write_gicreg(val, ICH_LR14_EL2);
108 		break;
109 	case 15:
110 		write_gicreg(val, ICH_LR15_EL2);
111 		break;
112 	}
113 }
114 
__vgic_v3_write_ap0rn(u32 val,int n)115 static void __vgic_v3_write_ap0rn(u32 val, int n)
116 {
117 	switch (n) {
118 	case 0:
119 		write_gicreg(val, ICH_AP0R0_EL2);
120 		break;
121 	case 1:
122 		write_gicreg(val, ICH_AP0R1_EL2);
123 		break;
124 	case 2:
125 		write_gicreg(val, ICH_AP0R2_EL2);
126 		break;
127 	case 3:
128 		write_gicreg(val, ICH_AP0R3_EL2);
129 		break;
130 	}
131 }
132 
__vgic_v3_write_ap1rn(u32 val,int n)133 static void __vgic_v3_write_ap1rn(u32 val, int n)
134 {
135 	switch (n) {
136 	case 0:
137 		write_gicreg(val, ICH_AP1R0_EL2);
138 		break;
139 	case 1:
140 		write_gicreg(val, ICH_AP1R1_EL2);
141 		break;
142 	case 2:
143 		write_gicreg(val, ICH_AP1R2_EL2);
144 		break;
145 	case 3:
146 		write_gicreg(val, ICH_AP1R3_EL2);
147 		break;
148 	}
149 }
150 
__vgic_v3_read_ap0rn(int n)151 static u32 __vgic_v3_read_ap0rn(int n)
152 {
153 	u32 val;
154 
155 	switch (n) {
156 	case 0:
157 		val = read_gicreg(ICH_AP0R0_EL2);
158 		break;
159 	case 1:
160 		val = read_gicreg(ICH_AP0R1_EL2);
161 		break;
162 	case 2:
163 		val = read_gicreg(ICH_AP0R2_EL2);
164 		break;
165 	case 3:
166 		val = read_gicreg(ICH_AP0R3_EL2);
167 		break;
168 	default:
169 		unreachable();
170 	}
171 
172 	return val;
173 }
174 
__vgic_v3_read_ap1rn(int n)175 static u32 __vgic_v3_read_ap1rn(int n)
176 {
177 	u32 val;
178 
179 	switch (n) {
180 	case 0:
181 		val = read_gicreg(ICH_AP1R0_EL2);
182 		break;
183 	case 1:
184 		val = read_gicreg(ICH_AP1R1_EL2);
185 		break;
186 	case 2:
187 		val = read_gicreg(ICH_AP1R2_EL2);
188 		break;
189 	case 3:
190 		val = read_gicreg(ICH_AP1R3_EL2);
191 		break;
192 	default:
193 		unreachable();
194 	}
195 
196 	return val;
197 }
198 
__vgic_v3_save_state(struct vgic_v3_cpu_if * cpu_if)199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
200 {
201 	u64 used_lrs = cpu_if->used_lrs;
202 
203 	/*
204 	 * Make sure stores to the GIC via the memory mapped interface
205 	 * are now visible to the system register interface when reading the
206 	 * LRs, and when reading back the VMCR on non-VHE systems.
207 	 */
208 	if (used_lrs || !has_vhe()) {
209 		if (!cpu_if->vgic_sre) {
210 			dsb(sy);
211 			isb();
212 		}
213 	}
214 
215 	if (used_lrs || cpu_if->its_vpe.its_vm) {
216 		int i;
217 		u32 elrsr;
218 
219 		elrsr = read_gicreg(ICH_ELRSR_EL2);
220 
221 		write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2);
222 
223 		for (i = 0; i < used_lrs; i++) {
224 			if (elrsr & (1 << i))
225 				cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
226 			else
227 				cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
228 
229 			__gic_v3_set_lr(0, i);
230 		}
231 	}
232 }
233 
__vgic_v3_restore_state(struct vgic_v3_cpu_if * cpu_if)234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
235 {
236 	u64 used_lrs = cpu_if->used_lrs;
237 	int i;
238 
239 	if (used_lrs || cpu_if->its_vpe.its_vm) {
240 		write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
241 
242 		for (i = 0; i < used_lrs; i++)
243 			__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
244 	}
245 
246 	/*
247 	 * Ensure that writes to the LRs, and on non-VHE systems ensure that
248 	 * the write to the VMCR in __vgic_v3_activate_traps(), will have
249 	 * reached the (re)distributors. This ensure the guest will read the
250 	 * correct values from the memory-mapped interface.
251 	 */
252 	if (used_lrs || !has_vhe()) {
253 		if (!cpu_if->vgic_sre) {
254 			isb();
255 			dsb(sy);
256 		}
257 	}
258 }
259 
__vgic_v3_activate_traps(struct vgic_v3_cpu_if * cpu_if)260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
261 {
262 	/*
263 	 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
264 	 * Group0 interrupt (as generated in GICv2 mode) to be
265 	 * delivered as a FIQ to the guest, with potentially fatal
266 	 * consequences. So we must make sure that ICC_SRE_EL1 has
267 	 * been actually programmed with the value we want before
268 	 * starting to mess with the rest of the GIC, and VMCR_EL2 in
269 	 * particular.  This logic must be called before
270 	 * __vgic_v3_restore_state().
271 	 */
272 	if (!cpu_if->vgic_sre) {
273 		write_gicreg(0, ICC_SRE_EL1);
274 		isb();
275 		write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
276 
277 
278 		if (has_vhe()) {
279 			/*
280 			 * Ensure that the write to the VMCR will have reached
281 			 * the (re)distributors. This ensure the guest will
282 			 * read the correct values from the memory-mapped
283 			 * interface.
284 			 */
285 			isb();
286 			dsb(sy);
287 		}
288 	}
289 
290 	/*
291 	 * Prevent the guest from touching the GIC system registers if
292 	 * SRE isn't enabled for GICv3 emulation.
293 	 */
294 	write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
295 		     ICC_SRE_EL2);
296 
297 	/*
298 	 * If we need to trap system registers, we must write
299 	 * ICH_HCR_EL2 anyway, even if no interrupts are being
300 	 * injected,
301 	 */
302 	if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
303 	    cpu_if->its_vpe.its_vm)
304 		write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
305 }
306 
__vgic_v3_deactivate_traps(struct vgic_v3_cpu_if * cpu_if)307 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
308 {
309 	u64 val;
310 
311 	if (!cpu_if->vgic_sre) {
312 		cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
313 	}
314 
315 	val = read_gicreg(ICC_SRE_EL2);
316 	write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
317 
318 	if (!cpu_if->vgic_sre) {
319 		/* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
320 		isb();
321 		write_gicreg(1, ICC_SRE_EL1);
322 	}
323 
324 	/*
325 	 * If we were trapping system registers, we enabled the VGIC even if
326 	 * no interrupts were being injected, and we disable it again here.
327 	 */
328 	if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
329 	    cpu_if->its_vpe.its_vm)
330 		write_gicreg(0, ICH_HCR_EL2);
331 }
332 
__vgic_v3_save_aprs(struct vgic_v3_cpu_if * cpu_if)333 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
334 {
335 	u64 val;
336 	u32 nr_pre_bits;
337 
338 	val = read_gicreg(ICH_VTR_EL2);
339 	nr_pre_bits = vtr_to_nr_pre_bits(val);
340 
341 	switch (nr_pre_bits) {
342 	case 7:
343 		cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
344 		cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
345 		fallthrough;
346 	case 6:
347 		cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
348 		fallthrough;
349 	default:
350 		cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
351 	}
352 
353 	switch (nr_pre_bits) {
354 	case 7:
355 		cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
356 		cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
357 		fallthrough;
358 	case 6:
359 		cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
360 		fallthrough;
361 	default:
362 		cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
363 	}
364 }
365 
__vgic_v3_restore_aprs(struct vgic_v3_cpu_if * cpu_if)366 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
367 {
368 	u64 val;
369 	u32 nr_pre_bits;
370 
371 	val = read_gicreg(ICH_VTR_EL2);
372 	nr_pre_bits = vtr_to_nr_pre_bits(val);
373 
374 	switch (nr_pre_bits) {
375 	case 7:
376 		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
377 		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
378 		fallthrough;
379 	case 6:
380 		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
381 		fallthrough;
382 	default:
383 		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
384 	}
385 
386 	switch (nr_pre_bits) {
387 	case 7:
388 		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
389 		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
390 		fallthrough;
391 	case 6:
392 		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
393 		fallthrough;
394 	default:
395 		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
396 	}
397 }
398 
__vgic_v3_init_lrs(void)399 void __vgic_v3_init_lrs(void)
400 {
401 	int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
402 	int i;
403 
404 	for (i = 0; i <= max_lr_idx; i++)
405 		__gic_v3_set_lr(0, i);
406 }
407 
408 /*
409  * Return the GIC CPU configuration:
410  * - [31:0]  ICH_VTR_EL2
411  * - [62:32] RES0
412  * - [63]    MMIO (GICv2) capable
413  */
__vgic_v3_get_gic_config(void)414 u64 __vgic_v3_get_gic_config(void)
415 {
416 	u64 val, sre = read_gicreg(ICC_SRE_EL1);
417 	unsigned long flags = 0;
418 
419 	/*
420 	 * To check whether we have a MMIO-based (GICv2 compatible)
421 	 * CPU interface, we need to disable the system register
422 	 * view. To do that safely, we have to prevent any interrupt
423 	 * from firing (which would be deadly).
424 	 *
425 	 * Note that this only makes sense on VHE, as interrupts are
426 	 * already masked for nVHE as part of the exception entry to
427 	 * EL2.
428 	 */
429 	if (has_vhe())
430 		flags = local_daif_save();
431 
432 	/*
433 	 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
434 	 * that to be able to set ICC_SRE_EL1.SRE to 0, all the
435 	 * interrupt overrides must be set. You've got to love this.
436 	 */
437 	sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
438 	isb();
439 	write_gicreg(0, ICC_SRE_EL1);
440 	isb();
441 
442 	val = read_gicreg(ICC_SRE_EL1);
443 
444 	write_gicreg(sre, ICC_SRE_EL1);
445 	isb();
446 	sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
447 	isb();
448 
449 	if (has_vhe())
450 		local_daif_restore(flags);
451 
452 	val  = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
453 	val |= read_gicreg(ICH_VTR_EL2);
454 
455 	return val;
456 }
457 
__vgic_v3_read_vmcr(void)458 static u64 __vgic_v3_read_vmcr(void)
459 {
460 	return read_gicreg(ICH_VMCR_EL2);
461 }
462 
__vgic_v3_write_vmcr(u32 vmcr)463 static void __vgic_v3_write_vmcr(u32 vmcr)
464 {
465 	write_gicreg(vmcr, ICH_VMCR_EL2);
466 }
467 
__vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if * cpu_if)468 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
469 {
470 	__vgic_v3_save_aprs(cpu_if);
471 	if (cpu_if->vgic_sre)
472 		cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
473 }
474 
__vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if * cpu_if)475 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
476 {
477 	/*
478 	 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
479 	 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
480 	 * VMCR_EL2 save/restore in the world switch.
481 	 */
482 	if (cpu_if->vgic_sre)
483 		__vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
484 	__vgic_v3_restore_aprs(cpu_if);
485 }
486 
__vgic_v3_bpr_min(void)487 static int __vgic_v3_bpr_min(void)
488 {
489 	/* See Pseudocode for VPriorityGroup */
490 	return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
491 }
492 
__vgic_v3_get_group(struct kvm_vcpu * vcpu)493 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
494 {
495 	u32 esr = kvm_vcpu_get_esr(vcpu);
496 	u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
497 
498 	return crm != 8;
499 }
500 
501 #define GICv3_IDLE_PRIORITY	0xff
502 
__vgic_v3_highest_priority_lr(struct kvm_vcpu * vcpu,u32 vmcr,u64 * lr_val)503 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
504 					 u64 *lr_val)
505 {
506 	unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
507 	u8 priority = GICv3_IDLE_PRIORITY;
508 	int i, lr = -1;
509 
510 	for (i = 0; i < used_lrs; i++) {
511 		u64 val = __gic_v3_get_lr(i);
512 		u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
513 
514 		/* Not pending in the state? */
515 		if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
516 			continue;
517 
518 		/* Group-0 interrupt, but Group-0 disabled? */
519 		if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
520 			continue;
521 
522 		/* Group-1 interrupt, but Group-1 disabled? */
523 		if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
524 			continue;
525 
526 		/* Not the highest priority? */
527 		if (lr_prio >= priority)
528 			continue;
529 
530 		/* This is a candidate */
531 		priority = lr_prio;
532 		*lr_val = val;
533 		lr = i;
534 	}
535 
536 	if (lr == -1)
537 		*lr_val = ICC_IAR1_EL1_SPURIOUS;
538 
539 	return lr;
540 }
541 
__vgic_v3_find_active_lr(struct kvm_vcpu * vcpu,int intid,u64 * lr_val)542 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
543 				    u64 *lr_val)
544 {
545 	unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
546 	int i;
547 
548 	for (i = 0; i < used_lrs; i++) {
549 		u64 val = __gic_v3_get_lr(i);
550 
551 		if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
552 		    (val & ICH_LR_ACTIVE_BIT)) {
553 			*lr_val = val;
554 			return i;
555 		}
556 	}
557 
558 	*lr_val = ICC_IAR1_EL1_SPURIOUS;
559 	return -1;
560 }
561 
__vgic_v3_get_highest_active_priority(void)562 static int __vgic_v3_get_highest_active_priority(void)
563 {
564 	u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
565 	u32 hap = 0;
566 	int i;
567 
568 	for (i = 0; i < nr_apr_regs; i++) {
569 		u32 val;
570 
571 		/*
572 		 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
573 		 * contain the active priority levels for this VCPU
574 		 * for the maximum number of supported priority
575 		 * levels, and we return the full priority level only
576 		 * if the BPR is programmed to its minimum, otherwise
577 		 * we return a combination of the priority level and
578 		 * subpriority, as determined by the setting of the
579 		 * BPR, but without the full subpriority.
580 		 */
581 		val  = __vgic_v3_read_ap0rn(i);
582 		val |= __vgic_v3_read_ap1rn(i);
583 		if (!val) {
584 			hap += 32;
585 			continue;
586 		}
587 
588 		return (hap + __ffs(val)) << __vgic_v3_bpr_min();
589 	}
590 
591 	return GICv3_IDLE_PRIORITY;
592 }
593 
__vgic_v3_get_bpr0(u32 vmcr)594 static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
595 {
596 	return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
597 }
598 
__vgic_v3_get_bpr1(u32 vmcr)599 static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
600 {
601 	unsigned int bpr;
602 
603 	if (vmcr & ICH_VMCR_CBPR_MASK) {
604 		bpr = __vgic_v3_get_bpr0(vmcr);
605 		if (bpr < 7)
606 			bpr++;
607 	} else {
608 		bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
609 	}
610 
611 	return bpr;
612 }
613 
614 /*
615  * Convert a priority to a preemption level, taking the relevant BPR
616  * into account by zeroing the sub-priority bits.
617  */
__vgic_v3_pri_to_pre(u8 pri,u32 vmcr,int grp)618 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
619 {
620 	unsigned int bpr;
621 
622 	if (!grp)
623 		bpr = __vgic_v3_get_bpr0(vmcr) + 1;
624 	else
625 		bpr = __vgic_v3_get_bpr1(vmcr);
626 
627 	return pri & (GENMASK(7, 0) << bpr);
628 }
629 
630 /*
631  * The priority value is independent of any of the BPR values, so we
632  * normalize it using the minimal BPR value. This guarantees that no
633  * matter what the guest does with its BPR, we can always set/get the
634  * same value of a priority.
635  */
__vgic_v3_set_active_priority(u8 pri,u32 vmcr,int grp)636 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
637 {
638 	u8 pre, ap;
639 	u32 val;
640 	int apr;
641 
642 	pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
643 	ap = pre >> __vgic_v3_bpr_min();
644 	apr = ap / 32;
645 
646 	if (!grp) {
647 		val = __vgic_v3_read_ap0rn(apr);
648 		__vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
649 	} else {
650 		val = __vgic_v3_read_ap1rn(apr);
651 		__vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
652 	}
653 }
654 
__vgic_v3_clear_highest_active_priority(void)655 static int __vgic_v3_clear_highest_active_priority(void)
656 {
657 	u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
658 	u32 hap = 0;
659 	int i;
660 
661 	for (i = 0; i < nr_apr_regs; i++) {
662 		u32 ap0, ap1;
663 		int c0, c1;
664 
665 		ap0 = __vgic_v3_read_ap0rn(i);
666 		ap1 = __vgic_v3_read_ap1rn(i);
667 		if (!ap0 && !ap1) {
668 			hap += 32;
669 			continue;
670 		}
671 
672 		c0 = ap0 ? __ffs(ap0) : 32;
673 		c1 = ap1 ? __ffs(ap1) : 32;
674 
675 		/* Always clear the LSB, which is the highest priority */
676 		if (c0 < c1) {
677 			ap0 &= ~BIT(c0);
678 			__vgic_v3_write_ap0rn(ap0, i);
679 			hap += c0;
680 		} else {
681 			ap1 &= ~BIT(c1);
682 			__vgic_v3_write_ap1rn(ap1, i);
683 			hap += c1;
684 		}
685 
686 		/* Rescale to 8 bits of priority */
687 		return hap << __vgic_v3_bpr_min();
688 	}
689 
690 	return GICv3_IDLE_PRIORITY;
691 }
692 
__vgic_v3_read_iar(struct kvm_vcpu * vcpu,u32 vmcr,int rt)693 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
694 {
695 	u64 lr_val;
696 	u8 lr_prio, pmr;
697 	int lr, grp;
698 
699 	grp = __vgic_v3_get_group(vcpu);
700 
701 	lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
702 	if (lr < 0)
703 		goto spurious;
704 
705 	if (grp != !!(lr_val & ICH_LR_GROUP))
706 		goto spurious;
707 
708 	pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
709 	lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
710 	if (pmr <= lr_prio)
711 		goto spurious;
712 
713 	if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
714 		goto spurious;
715 
716 	lr_val &= ~ICH_LR_STATE;
717 	lr_val |= ICH_LR_ACTIVE_BIT;
718 	__gic_v3_set_lr(lr_val, lr);
719 	__vgic_v3_set_active_priority(lr_prio, vmcr, grp);
720 	vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
721 	return;
722 
723 spurious:
724 	vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
725 }
726 
__vgic_v3_clear_active_lr(int lr,u64 lr_val)727 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
728 {
729 	lr_val &= ~ICH_LR_ACTIVE_BIT;
730 	if (lr_val & ICH_LR_HW) {
731 		u32 pid;
732 
733 		pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
734 		gic_write_dir(pid);
735 	}
736 
737 	__gic_v3_set_lr(lr_val, lr);
738 }
739 
__vgic_v3_bump_eoicount(void)740 static void __vgic_v3_bump_eoicount(void)
741 {
742 	u32 hcr;
743 
744 	hcr = read_gicreg(ICH_HCR_EL2);
745 	hcr += 1 << ICH_HCR_EOIcount_SHIFT;
746 	write_gicreg(hcr, ICH_HCR_EL2);
747 }
748 
__vgic_v3_write_dir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)749 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
750 {
751 	u32 vid = vcpu_get_reg(vcpu, rt);
752 	u64 lr_val;
753 	int lr;
754 
755 	/* EOImode == 0, nothing to be done here */
756 	if (!(vmcr & ICH_VMCR_EOIM_MASK))
757 		return;
758 
759 	/* No deactivate to be performed on an LPI */
760 	if (vid >= VGIC_MIN_LPI)
761 		return;
762 
763 	lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
764 	if (lr == -1) {
765 		__vgic_v3_bump_eoicount();
766 		return;
767 	}
768 
769 	__vgic_v3_clear_active_lr(lr, lr_val);
770 }
771 
__vgic_v3_write_eoir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)772 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
773 {
774 	u32 vid = vcpu_get_reg(vcpu, rt);
775 	u64 lr_val;
776 	u8 lr_prio, act_prio;
777 	int lr, grp;
778 
779 	grp = __vgic_v3_get_group(vcpu);
780 
781 	/* Drop priority in any case */
782 	act_prio = __vgic_v3_clear_highest_active_priority();
783 
784 	lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
785 	if (lr == -1) {
786 		/* Do not bump EOIcount for LPIs that aren't in the LRs */
787 		if (!(vid >= VGIC_MIN_LPI))
788 			__vgic_v3_bump_eoicount();
789 		return;
790 	}
791 
792 	/* EOImode == 1 and not an LPI, nothing to be done here */
793 	if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
794 		return;
795 
796 	lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
797 
798 	/* If priorities or group do not match, the guest has fscked-up. */
799 	if (grp != !!(lr_val & ICH_LR_GROUP) ||
800 	    __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
801 		return;
802 
803 	/* Let's now perform the deactivation */
804 	__vgic_v3_clear_active_lr(lr, lr_val);
805 }
806 
__vgic_v3_read_igrpen0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)807 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
808 {
809 	vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
810 }
811 
__vgic_v3_read_igrpen1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)812 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
813 {
814 	vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
815 }
816 
__vgic_v3_write_igrpen0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)817 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
818 {
819 	u64 val = vcpu_get_reg(vcpu, rt);
820 
821 	if (val & 1)
822 		vmcr |= ICH_VMCR_ENG0_MASK;
823 	else
824 		vmcr &= ~ICH_VMCR_ENG0_MASK;
825 
826 	__vgic_v3_write_vmcr(vmcr);
827 }
828 
__vgic_v3_write_igrpen1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)829 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
830 {
831 	u64 val = vcpu_get_reg(vcpu, rt);
832 
833 	if (val & 1)
834 		vmcr |= ICH_VMCR_ENG1_MASK;
835 	else
836 		vmcr &= ~ICH_VMCR_ENG1_MASK;
837 
838 	__vgic_v3_write_vmcr(vmcr);
839 }
840 
__vgic_v3_read_bpr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)841 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
842 {
843 	vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
844 }
845 
__vgic_v3_read_bpr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)846 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
847 {
848 	vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
849 }
850 
__vgic_v3_write_bpr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)851 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
852 {
853 	u64 val = vcpu_get_reg(vcpu, rt);
854 	u8 bpr_min = __vgic_v3_bpr_min() - 1;
855 
856 	/* Enforce BPR limiting */
857 	if (val < bpr_min)
858 		val = bpr_min;
859 
860 	val <<= ICH_VMCR_BPR0_SHIFT;
861 	val &= ICH_VMCR_BPR0_MASK;
862 	vmcr &= ~ICH_VMCR_BPR0_MASK;
863 	vmcr |= val;
864 
865 	__vgic_v3_write_vmcr(vmcr);
866 }
867 
__vgic_v3_write_bpr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)868 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
869 {
870 	u64 val = vcpu_get_reg(vcpu, rt);
871 	u8 bpr_min = __vgic_v3_bpr_min();
872 
873 	if (vmcr & ICH_VMCR_CBPR_MASK)
874 		return;
875 
876 	/* Enforce BPR limiting */
877 	if (val < bpr_min)
878 		val = bpr_min;
879 
880 	val <<= ICH_VMCR_BPR1_SHIFT;
881 	val &= ICH_VMCR_BPR1_MASK;
882 	vmcr &= ~ICH_VMCR_BPR1_MASK;
883 	vmcr |= val;
884 
885 	__vgic_v3_write_vmcr(vmcr);
886 }
887 
__vgic_v3_read_apxrn(struct kvm_vcpu * vcpu,int rt,int n)888 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
889 {
890 	u32 val;
891 
892 	if (!__vgic_v3_get_group(vcpu))
893 		val = __vgic_v3_read_ap0rn(n);
894 	else
895 		val = __vgic_v3_read_ap1rn(n);
896 
897 	vcpu_set_reg(vcpu, rt, val);
898 }
899 
__vgic_v3_write_apxrn(struct kvm_vcpu * vcpu,int rt,int n)900 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
901 {
902 	u32 val = vcpu_get_reg(vcpu, rt);
903 
904 	if (!__vgic_v3_get_group(vcpu))
905 		__vgic_v3_write_ap0rn(val, n);
906 	else
907 		__vgic_v3_write_ap1rn(val, n);
908 }
909 
__vgic_v3_read_apxr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)910 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
911 					    u32 vmcr, int rt)
912 {
913 	__vgic_v3_read_apxrn(vcpu, rt, 0);
914 }
915 
__vgic_v3_read_apxr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)916 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
917 					    u32 vmcr, int rt)
918 {
919 	__vgic_v3_read_apxrn(vcpu, rt, 1);
920 }
921 
__vgic_v3_read_apxr2(struct kvm_vcpu * vcpu,u32 vmcr,int rt)922 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
923 {
924 	__vgic_v3_read_apxrn(vcpu, rt, 2);
925 }
926 
__vgic_v3_read_apxr3(struct kvm_vcpu * vcpu,u32 vmcr,int rt)927 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
928 {
929 	__vgic_v3_read_apxrn(vcpu, rt, 3);
930 }
931 
__vgic_v3_write_apxr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)932 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
933 {
934 	__vgic_v3_write_apxrn(vcpu, rt, 0);
935 }
936 
__vgic_v3_write_apxr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)937 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
938 {
939 	__vgic_v3_write_apxrn(vcpu, rt, 1);
940 }
941 
__vgic_v3_write_apxr2(struct kvm_vcpu * vcpu,u32 vmcr,int rt)942 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
943 {
944 	__vgic_v3_write_apxrn(vcpu, rt, 2);
945 }
946 
__vgic_v3_write_apxr3(struct kvm_vcpu * vcpu,u32 vmcr,int rt)947 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
948 {
949 	__vgic_v3_write_apxrn(vcpu, rt, 3);
950 }
951 
__vgic_v3_read_hppir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)952 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
953 {
954 	u64 lr_val;
955 	int lr, lr_grp, grp;
956 
957 	grp = __vgic_v3_get_group(vcpu);
958 
959 	lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
960 	if (lr == -1)
961 		goto spurious;
962 
963 	lr_grp = !!(lr_val & ICH_LR_GROUP);
964 	if (lr_grp != grp)
965 		lr_val = ICC_IAR1_EL1_SPURIOUS;
966 
967 spurious:
968 	vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
969 }
970 
__vgic_v3_read_pmr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)971 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
972 {
973 	vmcr &= ICH_VMCR_PMR_MASK;
974 	vmcr >>= ICH_VMCR_PMR_SHIFT;
975 	vcpu_set_reg(vcpu, rt, vmcr);
976 }
977 
__vgic_v3_write_pmr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)978 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
979 {
980 	u32 val = vcpu_get_reg(vcpu, rt);
981 
982 	val <<= ICH_VMCR_PMR_SHIFT;
983 	val &= ICH_VMCR_PMR_MASK;
984 	vmcr &= ~ICH_VMCR_PMR_MASK;
985 	vmcr |= val;
986 
987 	write_gicreg(vmcr, ICH_VMCR_EL2);
988 }
989 
__vgic_v3_read_rpr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)990 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
991 {
992 	u32 val = __vgic_v3_get_highest_active_priority();
993 	vcpu_set_reg(vcpu, rt, val);
994 }
995 
__vgic_v3_read_ctlr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)996 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
997 {
998 	u32 vtr, val;
999 
1000 	vtr = read_gicreg(ICH_VTR_EL2);
1001 	/* PRIbits */
1002 	val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
1003 	/* IDbits */
1004 	val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
1005 	/* SEIS */
1006 	if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK)
1007 		val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT);
1008 	/* A3V */
1009 	val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
1010 	/* EOImode */
1011 	val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
1012 	/* CBPR */
1013 	val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
1014 
1015 	vcpu_set_reg(vcpu, rt, val);
1016 }
1017 
__vgic_v3_write_ctlr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1018 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1019 {
1020 	u32 val = vcpu_get_reg(vcpu, rt);
1021 
1022 	if (val & ICC_CTLR_EL1_CBPR_MASK)
1023 		vmcr |= ICH_VMCR_CBPR_MASK;
1024 	else
1025 		vmcr &= ~ICH_VMCR_CBPR_MASK;
1026 
1027 	if (val & ICC_CTLR_EL1_EOImode_MASK)
1028 		vmcr |= ICH_VMCR_EOIM_MASK;
1029 	else
1030 		vmcr &= ~ICH_VMCR_EOIM_MASK;
1031 
1032 	write_gicreg(vmcr, ICH_VMCR_EL2);
1033 }
1034 
__vgic_v3_perform_cpuif_access(struct kvm_vcpu * vcpu)1035 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1036 {
1037 	int rt;
1038 	u32 esr;
1039 	u32 vmcr;
1040 	void (*fn)(struct kvm_vcpu *, u32, int);
1041 	bool is_read;
1042 	u32 sysreg;
1043 
1044 	esr = kvm_vcpu_get_esr(vcpu);
1045 	if (vcpu_mode_is_32bit(vcpu)) {
1046 		if (!kvm_condition_valid(vcpu)) {
1047 			__kvm_skip_instr(vcpu);
1048 			return 1;
1049 		}
1050 
1051 		sysreg = esr_cp15_to_sysreg(esr);
1052 	} else {
1053 		sysreg = esr_sys64_to_sysreg(esr);
1054 	}
1055 
1056 	is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1057 
1058 	switch (sysreg) {
1059 	case SYS_ICC_IAR0_EL1:
1060 	case SYS_ICC_IAR1_EL1:
1061 		if (unlikely(!is_read))
1062 			return 0;
1063 		fn = __vgic_v3_read_iar;
1064 		break;
1065 	case SYS_ICC_EOIR0_EL1:
1066 	case SYS_ICC_EOIR1_EL1:
1067 		if (unlikely(is_read))
1068 			return 0;
1069 		fn = __vgic_v3_write_eoir;
1070 		break;
1071 	case SYS_ICC_IGRPEN1_EL1:
1072 		if (is_read)
1073 			fn = __vgic_v3_read_igrpen1;
1074 		else
1075 			fn = __vgic_v3_write_igrpen1;
1076 		break;
1077 	case SYS_ICC_BPR1_EL1:
1078 		if (is_read)
1079 			fn = __vgic_v3_read_bpr1;
1080 		else
1081 			fn = __vgic_v3_write_bpr1;
1082 		break;
1083 	case SYS_ICC_AP0Rn_EL1(0):
1084 	case SYS_ICC_AP1Rn_EL1(0):
1085 		if (is_read)
1086 			fn = __vgic_v3_read_apxr0;
1087 		else
1088 			fn = __vgic_v3_write_apxr0;
1089 		break;
1090 	case SYS_ICC_AP0Rn_EL1(1):
1091 	case SYS_ICC_AP1Rn_EL1(1):
1092 		if (is_read)
1093 			fn = __vgic_v3_read_apxr1;
1094 		else
1095 			fn = __vgic_v3_write_apxr1;
1096 		break;
1097 	case SYS_ICC_AP0Rn_EL1(2):
1098 	case SYS_ICC_AP1Rn_EL1(2):
1099 		if (is_read)
1100 			fn = __vgic_v3_read_apxr2;
1101 		else
1102 			fn = __vgic_v3_write_apxr2;
1103 		break;
1104 	case SYS_ICC_AP0Rn_EL1(3):
1105 	case SYS_ICC_AP1Rn_EL1(3):
1106 		if (is_read)
1107 			fn = __vgic_v3_read_apxr3;
1108 		else
1109 			fn = __vgic_v3_write_apxr3;
1110 		break;
1111 	case SYS_ICC_HPPIR0_EL1:
1112 	case SYS_ICC_HPPIR1_EL1:
1113 		if (unlikely(!is_read))
1114 			return 0;
1115 		fn = __vgic_v3_read_hppir;
1116 		break;
1117 	case SYS_ICC_IGRPEN0_EL1:
1118 		if (is_read)
1119 			fn = __vgic_v3_read_igrpen0;
1120 		else
1121 			fn = __vgic_v3_write_igrpen0;
1122 		break;
1123 	case SYS_ICC_BPR0_EL1:
1124 		if (is_read)
1125 			fn = __vgic_v3_read_bpr0;
1126 		else
1127 			fn = __vgic_v3_write_bpr0;
1128 		break;
1129 	case SYS_ICC_DIR_EL1:
1130 		if (unlikely(is_read))
1131 			return 0;
1132 		fn = __vgic_v3_write_dir;
1133 		break;
1134 	case SYS_ICC_RPR_EL1:
1135 		if (unlikely(!is_read))
1136 			return 0;
1137 		fn = __vgic_v3_read_rpr;
1138 		break;
1139 	case SYS_ICC_CTLR_EL1:
1140 		if (is_read)
1141 			fn = __vgic_v3_read_ctlr;
1142 		else
1143 			fn = __vgic_v3_write_ctlr;
1144 		break;
1145 	case SYS_ICC_PMR_EL1:
1146 		if (is_read)
1147 			fn = __vgic_v3_read_pmr;
1148 		else
1149 			fn = __vgic_v3_write_pmr;
1150 		break;
1151 	default:
1152 		return 0;
1153 	}
1154 
1155 	vmcr = __vgic_v3_read_vmcr();
1156 	rt = kvm_vcpu_sys_get_rt(vcpu);
1157 	fn(vcpu, vmcr, rt);
1158 
1159 	__kvm_skip_instr(vcpu);
1160 
1161 	return 1;
1162 }
1163