1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/guest.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
9 */
10
11 #include <linux/bits.h>
12 #include <linux/errno.h>
13 #include <linux/err.h>
14 #include <linux/nospec.h>
15 #include <linux/kvm_host.h>
16 #include <linux/module.h>
17 #include <linux/stddef.h>
18 #include <linux/string.h>
19 #include <linux/vmalloc.h>
20 #include <linux/fs.h>
21 #include <kvm/arm_hypercalls.h>
22 #include <asm/cputype.h>
23 #include <linux/uaccess.h>
24 #include <asm/fpsimd.h>
25 #include <asm/kvm.h>
26 #include <asm/kvm_emulate.h>
27 #include <asm/kvm_nested.h>
28 #include <asm/sigcontext.h>
29
30 #include "trace.h"
31
32 const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
33 KVM_GENERIC_VM_STATS(),
34 STATS_DESC_ICOUNTER(VM, protected_hyp_mem),
35 STATS_DESC_ICOUNTER(VM, protected_shared_mem),
36 };
37
38 const struct kvm_stats_header kvm_vm_stats_header = {
39 .name_size = KVM_STATS_NAME_SIZE,
40 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
41 .id_offset = sizeof(struct kvm_stats_header),
42 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
43 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
44 sizeof(kvm_vm_stats_desc),
45 };
46
47 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
48 KVM_GENERIC_VCPU_STATS(),
49 STATS_DESC_COUNTER(VCPU, hvc_exit_stat),
50 STATS_DESC_COUNTER(VCPU, wfe_exit_stat),
51 STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
52 STATS_DESC_COUNTER(VCPU, mmio_exit_user),
53 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
54 STATS_DESC_COUNTER(VCPU, signal_exits),
55 STATS_DESC_COUNTER(VCPU, exits)
56 };
57
58 const struct kvm_stats_header kvm_vcpu_stats_header = {
59 .name_size = KVM_STATS_NAME_SIZE,
60 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
61 .id_offset = sizeof(struct kvm_stats_header),
62 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
63 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
64 sizeof(kvm_vcpu_stats_desc),
65 };
66
core_reg_offset_is_vreg(u64 off)67 static bool core_reg_offset_is_vreg(u64 off)
68 {
69 return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) &&
70 off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr);
71 }
72
core_reg_offset_from_id(u64 id)73 static u64 core_reg_offset_from_id(u64 id)
74 {
75 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
76 }
77
core_reg_size_from_offset(const struct kvm_vcpu * vcpu,u64 off)78 static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off)
79 {
80 int size;
81
82 switch (off) {
83 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
84 KVM_REG_ARM_CORE_REG(regs.regs[30]):
85 case KVM_REG_ARM_CORE_REG(regs.sp):
86 case KVM_REG_ARM_CORE_REG(regs.pc):
87 case KVM_REG_ARM_CORE_REG(regs.pstate):
88 case KVM_REG_ARM_CORE_REG(sp_el1):
89 case KVM_REG_ARM_CORE_REG(elr_el1):
90 case KVM_REG_ARM_CORE_REG(spsr[0]) ...
91 KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
92 size = sizeof(__u64);
93 break;
94
95 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
96 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
97 size = sizeof(__uint128_t);
98 break;
99
100 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
101 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
102 size = sizeof(__u32);
103 break;
104
105 default:
106 return -EINVAL;
107 }
108
109 if (!IS_ALIGNED(off, size / sizeof(__u32)))
110 return -EINVAL;
111
112 /*
113 * The KVM_REG_ARM64_SVE regs must be used instead of
114 * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on
115 * SVE-enabled vcpus:
116 */
117 if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off))
118 return -EINVAL;
119
120 return size;
121 }
122
core_reg_addr(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)123 static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
124 {
125 u64 off = core_reg_offset_from_id(reg->id);
126 int size = core_reg_size_from_offset(vcpu, off);
127
128 if (size < 0)
129 return NULL;
130
131 if (KVM_REG_SIZE(reg->id) != size)
132 return NULL;
133
134 switch (off) {
135 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
136 KVM_REG_ARM_CORE_REG(regs.regs[30]):
137 off -= KVM_REG_ARM_CORE_REG(regs.regs[0]);
138 off /= 2;
139 return &vcpu->arch.ctxt.regs.regs[off];
140
141 case KVM_REG_ARM_CORE_REG(regs.sp):
142 return &vcpu->arch.ctxt.regs.sp;
143
144 case KVM_REG_ARM_CORE_REG(regs.pc):
145 return &vcpu->arch.ctxt.regs.pc;
146
147 case KVM_REG_ARM_CORE_REG(regs.pstate):
148 return &vcpu->arch.ctxt.regs.pstate;
149
150 case KVM_REG_ARM_CORE_REG(sp_el1):
151 return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1);
152
153 case KVM_REG_ARM_CORE_REG(elr_el1):
154 return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1);
155
156 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]):
157 return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1);
158
159 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]):
160 return &vcpu->arch.ctxt.spsr_abt;
161
162 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]):
163 return &vcpu->arch.ctxt.spsr_und;
164
165 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]):
166 return &vcpu->arch.ctxt.spsr_irq;
167
168 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]):
169 return &vcpu->arch.ctxt.spsr_fiq;
170
171 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
172 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
173 off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]);
174 off /= 4;
175 return &vcpu->arch.ctxt.fp_regs.vregs[off];
176
177 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
178 return &vcpu->arch.ctxt.fp_regs.fpsr;
179
180 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
181 return &vcpu->arch.ctxt.fp_regs.fpcr;
182
183 default:
184 return NULL;
185 }
186 }
187
get_core_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)188 static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
189 {
190 /*
191 * Because the kvm_regs structure is a mix of 32, 64 and
192 * 128bit fields, we index it as if it was a 32bit
193 * array. Hence below, nr_regs is the number of entries, and
194 * off the index in the "array".
195 */
196 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
197 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
198 void *addr;
199 u32 off;
200
201 /* Our ID is an index into the kvm_regs struct. */
202 off = core_reg_offset_from_id(reg->id);
203 if (off >= nr_regs ||
204 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
205 return -ENOENT;
206
207 addr = core_reg_addr(vcpu, reg);
208 if (!addr)
209 return -EINVAL;
210
211 if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id)))
212 return -EFAULT;
213
214 return 0;
215 }
216
set_core_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)217 static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
218 {
219 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
220 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
221 __uint128_t tmp;
222 void *valp = &tmp, *addr;
223 u64 off;
224 int err = 0;
225
226 /* Our ID is an index into the kvm_regs struct. */
227 off = core_reg_offset_from_id(reg->id);
228 if (off >= nr_regs ||
229 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
230 return -ENOENT;
231
232 addr = core_reg_addr(vcpu, reg);
233 if (!addr)
234 return -EINVAL;
235
236 if (KVM_REG_SIZE(reg->id) > sizeof(tmp))
237 return -EINVAL;
238
239 if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) {
240 err = -EFAULT;
241 goto out;
242 }
243
244 if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) {
245 u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK;
246 switch (mode) {
247 case PSR_AA32_MODE_USR:
248 if (!kvm_supports_32bit_el0())
249 return -EINVAL;
250 break;
251 case PSR_AA32_MODE_FIQ:
252 case PSR_AA32_MODE_IRQ:
253 case PSR_AA32_MODE_SVC:
254 case PSR_AA32_MODE_ABT:
255 case PSR_AA32_MODE_UND:
256 case PSR_AA32_MODE_SYS:
257 if (!vcpu_el1_is_32bit(vcpu))
258 return -EINVAL;
259 break;
260 case PSR_MODE_EL2h:
261 case PSR_MODE_EL2t:
262 if (!vcpu_has_nv(vcpu))
263 return -EINVAL;
264 fallthrough;
265 case PSR_MODE_EL0t:
266 case PSR_MODE_EL1t:
267 case PSR_MODE_EL1h:
268 if (vcpu_el1_is_32bit(vcpu))
269 return -EINVAL;
270 break;
271 default:
272 err = -EINVAL;
273 goto out;
274 }
275 }
276
277 memcpy(addr, valp, KVM_REG_SIZE(reg->id));
278
279 if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) {
280 int i, nr_reg;
281
282 switch (*vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK) {
283 /*
284 * Either we are dealing with user mode, and only the
285 * first 15 registers (+ PC) must be narrowed to 32bit.
286 * AArch32 r0-r14 conveniently map to AArch64 x0-x14.
287 */
288 case PSR_AA32_MODE_USR:
289 case PSR_AA32_MODE_SYS:
290 nr_reg = 15;
291 break;
292
293 /*
294 * Otherwise, this is a privileged mode, and *all* the
295 * registers must be narrowed to 32bit.
296 */
297 default:
298 nr_reg = 31;
299 break;
300 }
301
302 for (i = 0; i < nr_reg; i++)
303 vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i));
304
305 *vcpu_pc(vcpu) = (u32)*vcpu_pc(vcpu);
306 }
307 out:
308 return err;
309 }
310
311 #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64)
312 #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64)
313 #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq)))
314
get_sve_vls(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)315 static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
316 {
317 unsigned int max_vq, vq;
318 u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
319
320 if (!vcpu_has_sve(vcpu))
321 return -ENOENT;
322
323 if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl)))
324 return -EINVAL;
325
326 memset(vqs, 0, sizeof(vqs));
327
328 max_vq = vcpu_sve_max_vq(vcpu);
329 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
330 if (sve_vq_available(vq))
331 vqs[vq_word(vq)] |= vq_mask(vq);
332
333 if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs)))
334 return -EFAULT;
335
336 return 0;
337 }
338
set_sve_vls(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)339 static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
340 {
341 unsigned int max_vq, vq;
342 u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
343
344 if (!vcpu_has_sve(vcpu))
345 return -ENOENT;
346
347 if (kvm_arm_vcpu_sve_finalized(vcpu))
348 return -EPERM; /* too late! */
349
350 if (WARN_ON(vcpu->arch.sve_state))
351 return -EINVAL;
352
353 if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs)))
354 return -EFAULT;
355
356 max_vq = 0;
357 for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq)
358 if (vq_present(vqs, vq))
359 max_vq = vq;
360
361 if (max_vq > sve_vq_from_vl(kvm_sve_max_vl))
362 return -EINVAL;
363
364 /*
365 * Vector lengths supported by the host can't currently be
366 * hidden from the guest individually: instead we can only set a
367 * maximum via ZCR_EL2.LEN. So, make sure the available vector
368 * lengths match the set requested exactly up to the requested
369 * maximum:
370 */
371 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
372 if (vq_present(vqs, vq) != sve_vq_available(vq))
373 return -EINVAL;
374
375 /* Can't run with no vector lengths at all: */
376 if (max_vq < SVE_VQ_MIN)
377 return -EINVAL;
378
379 /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */
380 vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq);
381
382 return 0;
383 }
384
385 #define SVE_REG_SLICE_SHIFT 0
386 #define SVE_REG_SLICE_BITS 5
387 #define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS)
388 #define SVE_REG_ID_BITS 5
389
390 #define SVE_REG_SLICE_MASK \
391 GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \
392 SVE_REG_SLICE_SHIFT)
393 #define SVE_REG_ID_MASK \
394 GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT)
395
396 #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS)
397
398 #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0))
399 #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0))
400
401 /*
402 * Number of register slices required to cover each whole SVE register.
403 * NOTE: Only the first slice every exists, for now.
404 * If you are tempted to modify this, you must also rework sve_reg_to_region()
405 * to match:
406 */
407 #define vcpu_sve_slices(vcpu) 1
408
409 /* Bounds of a single SVE register slice within vcpu->arch.sve_state */
410 struct sve_state_reg_region {
411 unsigned int koffset; /* offset into sve_state in kernel memory */
412 unsigned int klen; /* length in kernel memory */
413 unsigned int upad; /* extra trailing padding in user memory */
414 };
415
416 /*
417 * Validate SVE register ID and get sanitised bounds for user/kernel SVE
418 * register copy
419 */
sve_reg_to_region(struct sve_state_reg_region * region,struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)420 static int sve_reg_to_region(struct sve_state_reg_region *region,
421 struct kvm_vcpu *vcpu,
422 const struct kvm_one_reg *reg)
423 {
424 /* reg ID ranges for Z- registers */
425 const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0);
426 const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1,
427 SVE_NUM_SLICES - 1);
428
429 /* reg ID ranges for P- registers and FFR (which are contiguous) */
430 const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0);
431 const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1);
432
433 unsigned int vq;
434 unsigned int reg_num;
435
436 unsigned int reqoffset, reqlen; /* User-requested offset and length */
437 unsigned int maxlen; /* Maximum permitted length */
438
439 size_t sve_state_size;
440
441 const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1,
442 SVE_NUM_SLICES - 1);
443
444 /* Verify that the P-regs and FFR really do have contiguous IDs: */
445 BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1);
446
447 /* Verify that we match the UAPI header: */
448 BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES);
449
450 reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
451
452 if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) {
453 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
454 return -ENOENT;
455
456 vq = vcpu_sve_max_vq(vcpu);
457
458 reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) -
459 SVE_SIG_REGS_OFFSET;
460 reqlen = KVM_SVE_ZREG_SIZE;
461 maxlen = SVE_SIG_ZREG_SIZE(vq);
462 } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) {
463 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
464 return -ENOENT;
465
466 vq = vcpu_sve_max_vq(vcpu);
467
468 reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) -
469 SVE_SIG_REGS_OFFSET;
470 reqlen = KVM_SVE_PREG_SIZE;
471 maxlen = SVE_SIG_PREG_SIZE(vq);
472 } else {
473 return -EINVAL;
474 }
475
476 sve_state_size = vcpu_sve_state_size(vcpu);
477 if (WARN_ON(!sve_state_size))
478 return -EINVAL;
479
480 region->koffset = array_index_nospec(reqoffset, sve_state_size);
481 region->klen = min(maxlen, reqlen);
482 region->upad = reqlen - region->klen;
483
484 return 0;
485 }
486
get_sve_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)487 static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
488 {
489 int ret;
490 struct sve_state_reg_region region;
491 char __user *uptr = (char __user *)reg->addr;
492
493 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
494 if (reg->id == KVM_REG_ARM64_SVE_VLS)
495 return get_sve_vls(vcpu, reg);
496
497 /* Try to interpret reg ID as an architectural SVE register... */
498 ret = sve_reg_to_region(®ion, vcpu, reg);
499 if (ret)
500 return ret;
501
502 if (!kvm_arm_vcpu_sve_finalized(vcpu))
503 return -EPERM;
504
505 if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset,
506 region.klen) ||
507 clear_user(uptr + region.klen, region.upad))
508 return -EFAULT;
509
510 return 0;
511 }
512
set_sve_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)513 static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
514 {
515 int ret;
516 struct sve_state_reg_region region;
517 const char __user *uptr = (const char __user *)reg->addr;
518
519 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
520 if (reg->id == KVM_REG_ARM64_SVE_VLS)
521 return set_sve_vls(vcpu, reg);
522
523 /* Try to interpret reg ID as an architectural SVE register... */
524 ret = sve_reg_to_region(®ion, vcpu, reg);
525 if (ret)
526 return ret;
527
528 if (!kvm_arm_vcpu_sve_finalized(vcpu))
529 return -EPERM;
530
531 if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr,
532 region.klen))
533 return -EFAULT;
534
535 return 0;
536 }
537
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)538 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
539 {
540 return -EINVAL;
541 }
542
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)543 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
544 {
545 return -EINVAL;
546 }
547
copy_core_reg_indices(const struct kvm_vcpu * vcpu,u64 __user * uindices)548 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu,
549 u64 __user *uindices)
550 {
551 unsigned int i;
552 int n = 0;
553
554 for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {
555 u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i;
556 int size = core_reg_size_from_offset(vcpu, i);
557
558 if (size < 0)
559 continue;
560
561 switch (size) {
562 case sizeof(__u32):
563 reg |= KVM_REG_SIZE_U32;
564 break;
565
566 case sizeof(__u64):
567 reg |= KVM_REG_SIZE_U64;
568 break;
569
570 case sizeof(__uint128_t):
571 reg |= KVM_REG_SIZE_U128;
572 break;
573
574 default:
575 WARN_ON(1);
576 continue;
577 }
578
579 if (uindices) {
580 if (put_user(reg, uindices))
581 return -EFAULT;
582 uindices++;
583 }
584
585 n++;
586 }
587
588 return n;
589 }
590
num_core_regs(const struct kvm_vcpu * vcpu)591 static unsigned long num_core_regs(const struct kvm_vcpu *vcpu)
592 {
593 return copy_core_reg_indices(vcpu, NULL);
594 }
595
596 static const u64 timer_reg_list[] = {
597 KVM_REG_ARM_TIMER_CTL,
598 KVM_REG_ARM_TIMER_CNT,
599 KVM_REG_ARM_TIMER_CVAL,
600 KVM_REG_ARM_PTIMER_CTL,
601 KVM_REG_ARM_PTIMER_CNT,
602 KVM_REG_ARM_PTIMER_CVAL,
603 };
604
605 #define NUM_TIMER_REGS ARRAY_SIZE(timer_reg_list)
606
is_timer_reg(u64 index)607 static bool is_timer_reg(u64 index)
608 {
609 switch (index) {
610 case KVM_REG_ARM_TIMER_CTL:
611 case KVM_REG_ARM_TIMER_CNT:
612 case KVM_REG_ARM_TIMER_CVAL:
613 case KVM_REG_ARM_PTIMER_CTL:
614 case KVM_REG_ARM_PTIMER_CNT:
615 case KVM_REG_ARM_PTIMER_CVAL:
616 return true;
617 }
618 return false;
619 }
620
copy_timer_indices(struct kvm_vcpu * vcpu,u64 __user * uindices)621 static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
622 {
623 for (int i = 0; i < NUM_TIMER_REGS; i++) {
624 if (put_user(timer_reg_list[i], uindices))
625 return -EFAULT;
626 uindices++;
627 }
628
629 return 0;
630 }
631
set_timer_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)632 static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
633 {
634 void __user *uaddr = (void __user *)(long)reg->addr;
635 u64 val;
636 int ret;
637
638 ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id));
639 if (ret != 0)
640 return -EFAULT;
641
642 return kvm_arm_timer_set_reg(vcpu, reg->id, val);
643 }
644
get_timer_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)645 static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
646 {
647 void __user *uaddr = (void __user *)(long)reg->addr;
648 u64 val;
649
650 val = kvm_arm_timer_get_reg(vcpu, reg->id);
651 return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0;
652 }
653
num_sve_regs(const struct kvm_vcpu * vcpu)654 static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
655 {
656 const unsigned int slices = vcpu_sve_slices(vcpu);
657
658 if (!vcpu_has_sve(vcpu))
659 return 0;
660
661 /* Policed by KVM_GET_REG_LIST: */
662 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
663
664 return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */)
665 + 1; /* KVM_REG_ARM64_SVE_VLS */
666 }
667
copy_sve_reg_indices(const struct kvm_vcpu * vcpu,u64 __user * uindices)668 static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu,
669 u64 __user *uindices)
670 {
671 const unsigned int slices = vcpu_sve_slices(vcpu);
672 u64 reg;
673 unsigned int i, n;
674 int num_regs = 0;
675
676 if (!vcpu_has_sve(vcpu))
677 return 0;
678
679 /* Policed by KVM_GET_REG_LIST: */
680 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
681
682 /*
683 * Enumerate this first, so that userspace can save/restore in
684 * the order reported by KVM_GET_REG_LIST:
685 */
686 reg = KVM_REG_ARM64_SVE_VLS;
687 if (put_user(reg, uindices++))
688 return -EFAULT;
689 ++num_regs;
690
691 for (i = 0; i < slices; i++) {
692 for (n = 0; n < SVE_NUM_ZREGS; n++) {
693 reg = KVM_REG_ARM64_SVE_ZREG(n, i);
694 if (put_user(reg, uindices++))
695 return -EFAULT;
696 num_regs++;
697 }
698
699 for (n = 0; n < SVE_NUM_PREGS; n++) {
700 reg = KVM_REG_ARM64_SVE_PREG(n, i);
701 if (put_user(reg, uindices++))
702 return -EFAULT;
703 num_regs++;
704 }
705
706 reg = KVM_REG_ARM64_SVE_FFR(i);
707 if (put_user(reg, uindices++))
708 return -EFAULT;
709 num_regs++;
710 }
711
712 return num_regs;
713 }
714
715 /**
716 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG
717 * @vcpu: the vCPU pointer
718 *
719 * This is for all registers.
720 */
kvm_arm_num_regs(struct kvm_vcpu * vcpu)721 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
722 {
723 unsigned long res = 0;
724
725 res += num_core_regs(vcpu);
726 res += num_sve_regs(vcpu);
727 res += kvm_arm_num_sys_reg_descs(vcpu);
728 res += kvm_arm_get_fw_num_regs(vcpu);
729 res += NUM_TIMER_REGS;
730
731 return res;
732 }
733
734 /**
735 * kvm_arm_copy_reg_indices - get indices of all registers.
736 * @vcpu: the vCPU pointer
737 * @uindices: register list to copy
738 *
739 * We do core registers right here, then we append system regs.
740 */
kvm_arm_copy_reg_indices(struct kvm_vcpu * vcpu,u64 __user * uindices)741 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
742 {
743 int ret;
744
745 ret = copy_core_reg_indices(vcpu, uindices);
746 if (ret < 0)
747 return ret;
748 uindices += ret;
749
750 ret = copy_sve_reg_indices(vcpu, uindices);
751 if (ret < 0)
752 return ret;
753 uindices += ret;
754
755 ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices);
756 if (ret < 0)
757 return ret;
758 uindices += kvm_arm_get_fw_num_regs(vcpu);
759
760 ret = copy_timer_indices(vcpu, uindices);
761 if (ret < 0)
762 return ret;
763 uindices += NUM_TIMER_REGS;
764
765 return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
766 }
767
kvm_arm_get_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)768 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
769 {
770 /* We currently use nothing arch-specific in upper 32 bits */
771 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
772 return -EINVAL;
773
774 switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
775 case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg);
776 case KVM_REG_ARM_FW:
777 case KVM_REG_ARM_FW_FEAT_BMAP:
778 return kvm_arm_get_fw_reg(vcpu, reg);
779 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg);
780 }
781
782 if (is_timer_reg(reg->id))
783 return get_timer_reg(vcpu, reg);
784
785 return kvm_arm_sys_reg_get_reg(vcpu, reg);
786 }
787
kvm_arm_set_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)788 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
789 {
790 /* We currently use nothing arch-specific in upper 32 bits */
791 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
792 return -EINVAL;
793
794 switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
795 case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg);
796 case KVM_REG_ARM_FW:
797 case KVM_REG_ARM_FW_FEAT_BMAP:
798 return kvm_arm_set_fw_reg(vcpu, reg);
799 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg);
800 }
801
802 if (is_timer_reg(reg->id))
803 return set_timer_reg(vcpu, reg);
804
805 return kvm_arm_sys_reg_set_reg(vcpu, reg);
806 }
807
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)808 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
809 struct kvm_sregs *sregs)
810 {
811 return -EINVAL;
812 }
813
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)814 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
815 struct kvm_sregs *sregs)
816 {
817 return -EINVAL;
818 }
819
__kvm_arm_vcpu_get_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)820 int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
821 struct kvm_vcpu_events *events)
822 {
823 events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE);
824 events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
825
826 if (events->exception.serror_pending && events->exception.serror_has_esr)
827 events->exception.serror_esr = vcpu_get_vsesr(vcpu);
828
829 /*
830 * We never return a pending ext_dabt here because we deliver it to
831 * the virtual CPU directly when setting the event and it's no longer
832 * 'pending' at this point.
833 */
834
835 return 0;
836 }
837
__kvm_arm_vcpu_set_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)838 int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
839 struct kvm_vcpu_events *events)
840 {
841 bool serror_pending = events->exception.serror_pending;
842 bool has_esr = events->exception.serror_has_esr;
843 bool ext_dabt_pending = events->exception.ext_dabt_pending;
844
845 if (serror_pending && has_esr) {
846 if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
847 return -EINVAL;
848
849 if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK))
850 kvm_set_sei_esr(vcpu, events->exception.serror_esr);
851 else
852 return -EINVAL;
853 } else if (serror_pending) {
854 kvm_inject_vabt(vcpu);
855 }
856
857 if (ext_dabt_pending)
858 kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
859
860 return 0;
861 }
862
kvm_target_cpu(void)863 u32 __attribute_const__ kvm_target_cpu(void)
864 {
865 unsigned long implementor = read_cpuid_implementor();
866 unsigned long part_number = read_cpuid_part_number();
867
868 switch (implementor) {
869 case ARM_CPU_IMP_ARM:
870 switch (part_number) {
871 case ARM_CPU_PART_AEM_V8:
872 return KVM_ARM_TARGET_AEM_V8;
873 case ARM_CPU_PART_FOUNDATION:
874 return KVM_ARM_TARGET_FOUNDATION_V8;
875 case ARM_CPU_PART_CORTEX_A53:
876 return KVM_ARM_TARGET_CORTEX_A53;
877 case ARM_CPU_PART_CORTEX_A57:
878 return KVM_ARM_TARGET_CORTEX_A57;
879 }
880 break;
881 case ARM_CPU_IMP_APM:
882 switch (part_number) {
883 case APM_CPU_PART_XGENE:
884 return KVM_ARM_TARGET_XGENE_POTENZA;
885 }
886 break;
887 }
888
889 /* Return a default generic target */
890 return KVM_ARM_TARGET_GENERIC_V8;
891 }
892
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)893 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
894 {
895 return -EINVAL;
896 }
897
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)898 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
899 {
900 return -EINVAL;
901 }
902
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)903 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
904 struct kvm_translation *tr)
905 {
906 return -EINVAL;
907 }
908
909 /**
910 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging
911 * @vcpu: the vCPU pointer
912 * @dbg: the ioctl data buffer
913 *
914 * This sets up and enables the VM for guest debugging. Userspace
915 * passes in a control flag to enable different debug types and
916 * potentially other architecture specific information in the rest of
917 * the structure.
918 */
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)919 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
920 struct kvm_guest_debug *dbg)
921 {
922 int ret = 0;
923
924 trace_kvm_set_guest_debug(vcpu, dbg->control);
925
926 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) {
927 ret = -EINVAL;
928 goto out;
929 }
930
931 if (dbg->control & KVM_GUESTDBG_ENABLE) {
932 vcpu->guest_debug = dbg->control;
933
934 /* Hardware assisted Break and Watch points */
935 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) {
936 vcpu->arch.external_debug_state = dbg->arch;
937 }
938
939 } else {
940 /* If not enabled clear all flags */
941 vcpu->guest_debug = 0;
942 vcpu_clear_flag(vcpu, DBG_SS_ACTIVE_PENDING);
943 }
944
945 out:
946 return ret;
947 }
948
kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)949 int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
950 struct kvm_device_attr *attr)
951 {
952 int ret;
953
954 switch (attr->group) {
955 case KVM_ARM_VCPU_PMU_V3_CTRL:
956 mutex_lock(&vcpu->kvm->arch.config_lock);
957 ret = kvm_arm_pmu_v3_set_attr(vcpu, attr);
958 mutex_unlock(&vcpu->kvm->arch.config_lock);
959 break;
960 case KVM_ARM_VCPU_TIMER_CTRL:
961 ret = kvm_arm_timer_set_attr(vcpu, attr);
962 break;
963 case KVM_ARM_VCPU_PVTIME_CTRL:
964 ret = kvm_arm_pvtime_set_attr(vcpu, attr);
965 break;
966 default:
967 ret = -ENXIO;
968 break;
969 }
970
971 return ret;
972 }
973
kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)974 int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
975 struct kvm_device_attr *attr)
976 {
977 int ret;
978
979 switch (attr->group) {
980 case KVM_ARM_VCPU_PMU_V3_CTRL:
981 ret = kvm_arm_pmu_v3_get_attr(vcpu, attr);
982 break;
983 case KVM_ARM_VCPU_TIMER_CTRL:
984 ret = kvm_arm_timer_get_attr(vcpu, attr);
985 break;
986 case KVM_ARM_VCPU_PVTIME_CTRL:
987 ret = kvm_arm_pvtime_get_attr(vcpu, attr);
988 break;
989 default:
990 ret = -ENXIO;
991 break;
992 }
993
994 return ret;
995 }
996
kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)997 int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
998 struct kvm_device_attr *attr)
999 {
1000 int ret;
1001
1002 switch (attr->group) {
1003 case KVM_ARM_VCPU_PMU_V3_CTRL:
1004 ret = kvm_arm_pmu_v3_has_attr(vcpu, attr);
1005 break;
1006 case KVM_ARM_VCPU_TIMER_CTRL:
1007 ret = kvm_arm_timer_has_attr(vcpu, attr);
1008 break;
1009 case KVM_ARM_VCPU_PVTIME_CTRL:
1010 ret = kvm_arm_pvtime_has_attr(vcpu, attr);
1011 break;
1012 default:
1013 ret = -ENXIO;
1014 break;
1015 }
1016
1017 return ret;
1018 }
1019
kvm_vm_ioctl_mte_copy_tags(struct kvm * kvm,struct kvm_arm_copy_mte_tags * copy_tags)1020 int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,
1021 struct kvm_arm_copy_mte_tags *copy_tags)
1022 {
1023 gpa_t guest_ipa = copy_tags->guest_ipa;
1024 size_t length = copy_tags->length;
1025 void __user *tags = copy_tags->addr;
1026 gpa_t gfn;
1027 bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST);
1028 int ret = 0;
1029
1030 if (!kvm_has_mte(kvm))
1031 return -EINVAL;
1032
1033 if (copy_tags->reserved[0] || copy_tags->reserved[1])
1034 return -EINVAL;
1035
1036 if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST)
1037 return -EINVAL;
1038
1039 if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK)
1040 return -EINVAL;
1041
1042 /* Lengths above INT_MAX cannot be represented in the return value */
1043 if (length > INT_MAX)
1044 return -EINVAL;
1045
1046 gfn = gpa_to_gfn(guest_ipa);
1047
1048 mutex_lock(&kvm->slots_lock);
1049
1050 if (write && atomic_read(&kvm->nr_memslots_dirty_logging)) {
1051 ret = -EBUSY;
1052 goto out;
1053 }
1054
1055 while (length > 0) {
1056 kvm_pfn_t pfn = gfn_to_pfn_prot(kvm, gfn, write, NULL);
1057 void *maddr;
1058 unsigned long num_tags;
1059 struct page *page;
1060
1061 if (is_error_noslot_pfn(pfn)) {
1062 ret = -EFAULT;
1063 goto out;
1064 }
1065
1066 page = pfn_to_online_page(pfn);
1067 if (!page) {
1068 /* Reject ZONE_DEVICE memory */
1069 kvm_release_pfn_clean(pfn);
1070 ret = -EFAULT;
1071 goto out;
1072 }
1073 maddr = page_address(page);
1074
1075 if (!write) {
1076 if (page_mte_tagged(page))
1077 num_tags = mte_copy_tags_to_user(tags, maddr,
1078 MTE_GRANULES_PER_PAGE);
1079 else
1080 /* No tags in memory, so write zeros */
1081 num_tags = MTE_GRANULES_PER_PAGE -
1082 clear_user(tags, MTE_GRANULES_PER_PAGE);
1083 kvm_release_pfn_clean(pfn);
1084 } else {
1085 /*
1086 * Only locking to serialise with a concurrent
1087 * __set_ptes() in the VMM but still overriding the
1088 * tags, hence ignoring the return value.
1089 */
1090 try_page_mte_tagging(page);
1091 num_tags = mte_copy_tags_from_user(maddr, tags,
1092 MTE_GRANULES_PER_PAGE);
1093
1094 /* uaccess failed, don't leave stale tags */
1095 if (num_tags != MTE_GRANULES_PER_PAGE)
1096 mte_clear_page_tags(maddr);
1097 set_page_mte_tagged(page);
1098
1099 kvm_release_pfn_dirty(pfn);
1100 }
1101
1102 if (num_tags != MTE_GRANULES_PER_PAGE) {
1103 ret = -EFAULT;
1104 goto out;
1105 }
1106
1107 gfn++;
1108 tags += num_tags;
1109 length -= PAGE_SIZE;
1110 }
1111
1112 out:
1113 mutex_unlock(&kvm->slots_lock);
1114 /* If some data has been copied report the number of bytes copied */
1115 if (length != copy_tags->length)
1116 return copy_tags->length - length;
1117 return ret;
1118 }
1119