1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5 */
6
7 #include <linux/bug.h>
8 #include <linux/cpu_pm.h>
9 #include <linux/entry-kvm.h>
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kvm_host.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/vmalloc.h>
16 #include <linux/fs.h>
17 #include <linux/mman.h>
18 #include <linux/sched.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_irqfd.h>
21 #include <linux/irqbypass.h>
22 #include <linux/sched/stat.h>
23 #include <linux/psci.h>
24 #include <trace/events/kvm.h>
25
26 #define CREATE_TRACE_POINTS
27 #include "trace_arm.h"
28
29 #include "hyp_trace.h"
30
31 #include <linux/uaccess.h>
32 #include <asm/ptrace.h>
33 #include <asm/mman.h>
34 #include <asm/tlbflush.h>
35 #include <asm/cacheflush.h>
36 #include <asm/cpufeature.h>
37 #include <asm/virt.h>
38 #include <asm/kvm_arm.h>
39 #include <asm/kvm_asm.h>
40 #include <asm/kvm_mmu.h>
41 #include <asm/kvm_pkvm.h>
42 #include <asm/kvm_emulate.h>
43 #include <asm/sections.h>
44
45 #include <kvm/arm_hypercalls.h>
46 #include <kvm/arm_pmu.h>
47 #include <kvm/arm_psci.h>
48
49 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
50
51 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
52
53 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
54 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
55 DECLARE_KVM_NVHE_PER_CPU(int, hyp_cpu_number);
56
57 static bool vgic_present;
58
59 static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
60 DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
61
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)62 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
63 {
64 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
65 }
66
kvm_arch_hardware_setup(void * opaque)67 int kvm_arch_hardware_setup(void *opaque)
68 {
69 return 0;
70 }
71
kvm_arch_check_processor_compat(void * opaque)72 int kvm_arch_check_processor_compat(void *opaque)
73 {
74 return 0;
75 }
76
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)77 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
78 struct kvm_enable_cap *cap)
79 {
80 int r;
81
82 /* Capabilities with flags */
83 switch (cap->cap) {
84 case KVM_CAP_ARM_PROTECTED_VM:
85 return pkvm_vm_ioctl_enable_cap(kvm, cap);
86 default:
87 if (cap->flags)
88 return -EINVAL;
89 }
90
91 /* Capabilities without flags */
92 switch (cap->cap) {
93 case KVM_CAP_ARM_NISV_TO_USER:
94 if (kvm_vm_is_protected(kvm)) {
95 r = -EINVAL;
96 } else {
97 r = 0;
98 set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
99 &kvm->arch.flags);
100 }
101 break;
102 case KVM_CAP_ARM_MTE:
103 mutex_lock(&kvm->lock);
104 if (!system_supports_mte() ||
105 kvm_vm_is_protected(kvm) ||
106 kvm->created_vcpus) {
107 r = -EINVAL;
108 } else {
109 r = 0;
110 set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
111 }
112 mutex_unlock(&kvm->lock);
113 break;
114 case KVM_CAP_ARM_SYSTEM_SUSPEND:
115 r = 0;
116 set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
117 break;
118 default:
119 r = -EINVAL;
120 break;
121 }
122
123 return r;
124 }
125
kvm_arm_default_max_vcpus(void)126 static int kvm_arm_default_max_vcpus(void)
127 {
128 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
129 }
130
set_default_spectre(struct kvm * kvm)131 static void set_default_spectre(struct kvm *kvm)
132 {
133 /*
134 * The default is to expose CSV2 == 1 if the HW isn't affected.
135 * Although this is a per-CPU feature, we make it global because
136 * asymmetric systems are just a nuisance.
137 *
138 * Userspace can override this as long as it doesn't promise
139 * the impossible.
140 */
141 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED)
142 kvm->arch.pfr0_csv2 = 1;
143 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED)
144 kvm->arch.pfr0_csv3 = 1;
145 }
146
147 /**
148 * kvm_arch_init_vm - initializes a VM data structure
149 * @kvm: pointer to the KVM struct
150 */
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)151 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
152 {
153 int ret;
154
155 if (type & ~KVM_VM_TYPE_MASK)
156 return -EINVAL;
157
158 mutex_init(&kvm->arch.config_lock);
159
160 #ifdef CONFIG_LOCKDEP
161 /* Clue in lockdep that the config_lock must be taken inside kvm->lock */
162 mutex_lock(&kvm->lock);
163 mutex_lock(&kvm->arch.config_lock);
164 mutex_unlock(&kvm->arch.config_lock);
165 mutex_unlock(&kvm->lock);
166 #endif
167
168 ret = kvm_share_hyp(kvm, kvm + 1);
169 if (ret)
170 return ret;
171
172 ret = pkvm_init_host_vm(kvm, type);
173 if (ret)
174 goto err_unshare_kvm;
175
176 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL)) {
177 ret = -ENOMEM;
178 goto err_unshare_kvm;
179 }
180 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
181
182 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
183 if (ret)
184 goto err_free_cpumask;
185
186 kvm_vgic_early_init(kvm);
187
188 /* The maximum number of VCPUs is limited by the host's GIC model */
189 kvm->arch.max_vcpus = kvm_arm_default_max_vcpus();
190
191 set_default_spectre(kvm);
192 kvm_arm_init_hypercalls(kvm);
193
194 return 0;
195
196 err_free_cpumask:
197 free_cpumask_var(kvm->arch.supported_cpus);
198 err_unshare_kvm:
199 kvm_unshare_hyp(kvm, kvm + 1);
200 return ret;
201 }
202
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)203 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
204 {
205 return VM_FAULT_SIGBUS;
206 }
207
208
209 /**
210 * kvm_arch_destroy_vm - destroy the VM data structure
211 * @kvm: pointer to the KVM struct
212 */
kvm_arch_destroy_vm(struct kvm * kvm)213 void kvm_arch_destroy_vm(struct kvm *kvm)
214 {
215 bitmap_free(kvm->arch.pmu_filter);
216 free_cpumask_var(kvm->arch.supported_cpus);
217
218 kvm_vgic_destroy(kvm);
219
220 if (is_protected_kvm_enabled())
221 pkvm_destroy_hyp_vm(kvm);
222
223 kvm_destroy_vcpus(kvm);
224
225 if (atomic64_read(&kvm->stat.protected_hyp_mem))
226 pr_warn("%lluB of donations to the nVHE hyp are missing\n",
227 atomic64_read(&kvm->stat.protected_hyp_mem));
228
229 kvm_unshare_hyp(kvm, kvm + 1);
230 }
231
kvm_check_extension(struct kvm * kvm,long ext)232 static int kvm_check_extension(struct kvm *kvm, long ext)
233 {
234 int r;
235
236 switch (ext) {
237 case KVM_CAP_IRQCHIP:
238 r = vgic_present;
239 break;
240 case KVM_CAP_IOEVENTFD:
241 case KVM_CAP_DEVICE_CTRL:
242 case KVM_CAP_USER_MEMORY:
243 case KVM_CAP_SYNC_MMU:
244 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
245 case KVM_CAP_ONE_REG:
246 case KVM_CAP_ARM_PSCI:
247 case KVM_CAP_ARM_PSCI_0_2:
248 case KVM_CAP_READONLY_MEM:
249 case KVM_CAP_MP_STATE:
250 case KVM_CAP_IMMEDIATE_EXIT:
251 case KVM_CAP_VCPU_EVENTS:
252 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
253 case KVM_CAP_ARM_INJECT_EXT_DABT:
254 case KVM_CAP_SET_GUEST_DEBUG:
255 case KVM_CAP_VCPU_ATTRIBUTES:
256 case KVM_CAP_PTP_KVM:
257 case KVM_CAP_ARM_SYSTEM_SUSPEND:
258 r = 1;
259 break;
260 case KVM_CAP_ARM_NISV_TO_USER:
261 r = !kvm || !kvm_vm_is_protected(kvm);
262 break;
263 case KVM_CAP_SET_GUEST_DEBUG2:
264 return KVM_GUESTDBG_VALID_MASK;
265 case KVM_CAP_ARM_SET_DEVICE_ADDR:
266 r = 1;
267 break;
268 case KVM_CAP_NR_VCPUS:
269 /*
270 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
271 * architectures, as it does not always bound it to
272 * KVM_CAP_MAX_VCPUS. It should not matter much because
273 * this is just an advisory value.
274 */
275 r = min_t(unsigned int, num_online_cpus(),
276 kvm_arm_default_max_vcpus());
277 break;
278 case KVM_CAP_MAX_VCPUS:
279 case KVM_CAP_MAX_VCPU_ID:
280 if (kvm)
281 r = kvm->arch.max_vcpus;
282 else
283 r = kvm_arm_default_max_vcpus();
284 break;
285 case KVM_CAP_MSI_DEVID:
286 if (!kvm)
287 r = -EINVAL;
288 else
289 r = kvm->arch.vgic.msis_require_devid;
290 break;
291 case KVM_CAP_ARM_USER_IRQ:
292 /*
293 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
294 * (bump this number if adding more devices)
295 */
296 r = 1;
297 break;
298 case KVM_CAP_ARM_MTE:
299 r = system_supports_mte();
300 break;
301 case KVM_CAP_STEAL_TIME:
302 r = kvm_arm_pvtime_supported();
303 break;
304 case KVM_CAP_ARM_EL1_32BIT:
305 r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1);
306 break;
307 case KVM_CAP_GUEST_DEBUG_HW_BPS:
308 r = get_num_brps();
309 break;
310 case KVM_CAP_GUEST_DEBUG_HW_WPS:
311 r = get_num_wrps();
312 break;
313 case KVM_CAP_ARM_PMU_V3:
314 r = kvm_arm_support_pmu_v3();
315 break;
316 case KVM_CAP_ARM_INJECT_SERROR_ESR:
317 r = cpus_have_const_cap(ARM64_HAS_RAS_EXTN);
318 break;
319 case KVM_CAP_ARM_VM_IPA_SIZE:
320 r = get_kvm_ipa_limit();
321 break;
322 case KVM_CAP_ARM_SVE:
323 r = system_supports_sve();
324 break;
325 case KVM_CAP_ARM_PTRAUTH_ADDRESS:
326 case KVM_CAP_ARM_PTRAUTH_GENERIC:
327 r = system_has_full_ptr_auth();
328 break;
329 default:
330 r = 0;
331 }
332
333 return r;
334 }
335
336 /*
337 * Checks whether the extension specified in ext is supported in protected
338 * mode for the specified vm.
339 * The capabilities supported by kvm in general are passed in kvm_cap.
340 */
pkvm_check_extension(struct kvm * kvm,long ext,int kvm_cap)341 static int pkvm_check_extension(struct kvm *kvm, long ext, int kvm_cap)
342 {
343 int r;
344
345 switch (ext) {
346 case KVM_CAP_IRQCHIP:
347 case KVM_CAP_ARM_PSCI:
348 case KVM_CAP_ARM_PSCI_0_2:
349 case KVM_CAP_NR_VCPUS:
350 case KVM_CAP_MAX_VCPUS:
351 case KVM_CAP_MAX_VCPU_ID:
352 case KVM_CAP_MSI_DEVID:
353 case KVM_CAP_ARM_VM_IPA_SIZE:
354 r = kvm_cap;
355 break;
356 case KVM_CAP_GUEST_DEBUG_HW_BPS:
357 r = min(kvm_cap, pkvm_get_max_brps());
358 break;
359 case KVM_CAP_GUEST_DEBUG_HW_WPS:
360 r = min(kvm_cap, pkvm_get_max_wrps());
361 break;
362 case KVM_CAP_ARM_PMU_V3:
363 r = kvm_cap && FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer),
364 PVM_ID_AA64DFR0_ALLOW);
365 break;
366 case KVM_CAP_ARM_SVE:
367 r = kvm_cap && FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE),
368 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED);
369 break;
370 case KVM_CAP_ARM_PTRAUTH_ADDRESS:
371 r = kvm_cap &&
372 FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API),
373 PVM_ID_AA64ISAR1_ALLOW) &&
374 FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA),
375 PVM_ID_AA64ISAR1_ALLOW);
376 break;
377 case KVM_CAP_ARM_PTRAUTH_GENERIC:
378 r = kvm_cap &&
379 FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI),
380 PVM_ID_AA64ISAR1_ALLOW) &&
381 FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA),
382 PVM_ID_AA64ISAR1_ALLOW);
383 break;
384 case KVM_CAP_ARM_PROTECTED_VM:
385 r = 1;
386 break;
387 default:
388 r = 0;
389 break;
390 }
391
392 return r;
393 }
394
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)395 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
396 {
397 int r = kvm_check_extension(kvm, ext);
398
399 if (kvm && kvm_vm_is_protected(kvm))
400 r = pkvm_check_extension(kvm, ext, r);
401
402 return r;
403 }
404
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)405 long kvm_arch_dev_ioctl(struct file *filp,
406 unsigned int ioctl, unsigned long arg)
407 {
408 return -EINVAL;
409 }
410
kvm_arch_alloc_vm(void)411 struct kvm *kvm_arch_alloc_vm(void)
412 {
413 size_t sz = sizeof(struct kvm);
414
415 if (!has_vhe())
416 return kzalloc(sz, GFP_KERNEL_ACCOUNT);
417
418 return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
419 }
420
kvm_arch_free_vm(struct kvm * kvm)421 void kvm_arch_free_vm(struct kvm *kvm)
422 {
423 if (!has_vhe())
424 kfree(kvm);
425 else
426 vfree(kvm);
427 }
428
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)429 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
430 {
431 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
432 return -EBUSY;
433
434 if (id >= kvm->arch.max_vcpus)
435 return -EINVAL;
436
437 return 0;
438 }
439
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)440 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
441 {
442 int err;
443
444 spin_lock_init(&vcpu->arch.mp_state_lock);
445
446 #ifdef CONFIG_LOCKDEP
447 /* Inform lockdep that the config_lock is acquired after vcpu->mutex */
448 mutex_lock(&vcpu->mutex);
449 mutex_lock(&vcpu->kvm->arch.config_lock);
450 mutex_unlock(&vcpu->kvm->arch.config_lock);
451 mutex_unlock(&vcpu->mutex);
452 #endif
453
454 /* Force users to call KVM_ARM_VCPU_INIT */
455 vcpu->arch.target = -1;
456 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
457
458 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
459
460 /*
461 * Default value for the FP state, will be overloaded at load
462 * time if we support FP (pretty likely)
463 */
464 vcpu->arch.fp_state = FP_STATE_FREE;
465
466 /* Set up the timer */
467 kvm_timer_vcpu_init(vcpu);
468
469 kvm_pmu_vcpu_init(vcpu);
470
471 kvm_arm_reset_debug_ptr(vcpu);
472
473 kvm_arm_pvtime_vcpu_init(&vcpu->arch);
474
475 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
476
477 err = kvm_vgic_vcpu_init(vcpu);
478 if (err)
479 return err;
480
481 return kvm_share_hyp(vcpu, vcpu + 1);
482 }
483
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)484 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
485 {
486 }
487
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)488 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
489 {
490 if (vcpu_has_run_once(vcpu) && unlikely(!irqchip_in_kernel(vcpu->kvm)))
491 static_branch_dec(&userspace_irqchip_in_use);
492
493 if (is_protected_kvm_enabled())
494 free_hyp_stage2_memcache(&vcpu->arch.pkvm_memcache, vcpu->kvm);
495 else
496 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
497
498 kvm_timer_vcpu_terminate(vcpu);
499 kvm_pmu_vcpu_destroy(vcpu);
500 kvm_vgic_vcpu_destroy(vcpu);
501 kvm_arm_vcpu_destroy(vcpu);
502 }
503
kvm_arch_vcpu_blocking(struct kvm_vcpu * vcpu)504 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
505 {
506
507 }
508
kvm_arch_vcpu_unblocking(struct kvm_vcpu * vcpu)509 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
510 {
511
512 }
513
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)514 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
515 {
516 struct kvm_s2_mmu *mmu;
517 int *last_ran;
518
519 if (is_protected_kvm_enabled())
520 goto nommu;
521
522 mmu = vcpu->arch.hw_mmu;
523 last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
524
525 /*
526 * We guarantee that both TLBs and I-cache are private to each
527 * vcpu. If detecting that a vcpu from the same VM has
528 * previously run on the same physical CPU, call into the
529 * hypervisor code to nuke the relevant contexts.
530 *
531 * We might get preempted before the vCPU actually runs, but
532 * over-invalidation doesn't affect correctness.
533 */
534 if (*last_ran != vcpu->vcpu_id) {
535 kvm_call_hyp(__kvm_flush_cpu_context, mmu);
536 *last_ran = vcpu->vcpu_id;
537 }
538
539 nommu:
540 vcpu->cpu = cpu;
541
542 kvm_vgic_load(vcpu);
543 kvm_timer_vcpu_load(vcpu);
544 if (has_vhe())
545 kvm_vcpu_load_sysregs_vhe(vcpu);
546 kvm_arch_vcpu_load_fp(vcpu);
547 kvm_vcpu_pmu_restore_guest(vcpu);
548 if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
549 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
550
551 if (single_task_running())
552 vcpu_clear_wfx_traps(vcpu);
553 else
554 vcpu_set_wfx_traps(vcpu);
555
556 if (vcpu_has_ptrauth(vcpu))
557 vcpu_ptrauth_disable(vcpu);
558 kvm_arch_vcpu_load_debug_state_flags(vcpu);
559
560 if (is_protected_kvm_enabled()) {
561 kvm_call_hyp_nvhe(__pkvm_vcpu_load,
562 vcpu->kvm->arch.pkvm.handle,
563 vcpu->vcpu_idx, vcpu->arch.hcr_el2);
564 kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
565 &vcpu->arch.vgic_cpu.vgic_v3);
566 }
567
568 if (!cpumask_test_cpu(smp_processor_id(), vcpu->kvm->arch.supported_cpus))
569 vcpu_set_on_unsupported_cpu(vcpu);
570 }
571
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)572 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
573 {
574 if (is_protected_kvm_enabled()) {
575 kvm_call_hyp(__vgic_v3_save_vmcr_aprs,
576 &vcpu->arch.vgic_cpu.vgic_v3);
577 kvm_call_hyp_nvhe(__pkvm_vcpu_put);
578
579 /* __pkvm_vcpu_put implies a sync of the state */
580 if (!kvm_vm_is_protected(vcpu->kvm))
581 vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
582 }
583
584 kvm_arch_vcpu_put_debug_state_flags(vcpu);
585 kvm_arch_vcpu_put_fp(vcpu);
586 if (has_vhe())
587 kvm_vcpu_put_sysregs_vhe(vcpu);
588 kvm_timer_vcpu_put(vcpu);
589 kvm_vgic_put(vcpu, false);
590 kvm_vcpu_pmu_restore_host(vcpu);
591 kvm_arm_vmid_clear_active();
592
593 vcpu_clear_on_unsupported_cpu(vcpu);
594 vcpu->cpu = -1;
595 }
596
__kvm_arm_vcpu_power_off(struct kvm_vcpu * vcpu)597 static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
598 {
599 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
600 kvm_make_request(KVM_REQ_SLEEP, vcpu);
601 kvm_vcpu_kick(vcpu);
602 }
603
kvm_arm_vcpu_power_off(struct kvm_vcpu * vcpu)604 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
605 {
606 spin_lock(&vcpu->arch.mp_state_lock);
607 __kvm_arm_vcpu_power_off(vcpu);
608 spin_unlock(&vcpu->arch.mp_state_lock);
609 }
610
kvm_arm_vcpu_stopped(struct kvm_vcpu * vcpu)611 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
612 {
613 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
614 }
615
kvm_arm_vcpu_suspend(struct kvm_vcpu * vcpu)616 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
617 {
618 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);
619 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
620 kvm_vcpu_kick(vcpu);
621 }
622
kvm_arm_vcpu_suspended(struct kvm_vcpu * vcpu)623 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
624 {
625 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
626 }
627
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)628 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
629 struct kvm_mp_state *mp_state)
630 {
631 *mp_state = READ_ONCE(vcpu->arch.mp_state);
632
633 return 0;
634 }
635
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)636 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
637 struct kvm_mp_state *mp_state)
638 {
639 int ret = 0;
640
641 spin_lock(&vcpu->arch.mp_state_lock);
642
643 switch (mp_state->mp_state) {
644 case KVM_MP_STATE_RUNNABLE:
645 WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
646 break;
647 case KVM_MP_STATE_STOPPED:
648 __kvm_arm_vcpu_power_off(vcpu);
649 break;
650 case KVM_MP_STATE_SUSPENDED:
651 kvm_arm_vcpu_suspend(vcpu);
652 break;
653 default:
654 ret = -EINVAL;
655 }
656
657 spin_unlock(&vcpu->arch.mp_state_lock);
658
659 return ret;
660 }
661
662 /**
663 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
664 * @v: The VCPU pointer
665 *
666 * If the guest CPU is not waiting for interrupts or an interrupt line is
667 * asserted, the CPU is by definition runnable.
668 */
kvm_arch_vcpu_runnable(struct kvm_vcpu * v)669 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
670 {
671 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
672 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
673 && !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
674 }
675
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)676 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
677 {
678 return vcpu_mode_priv(vcpu);
679 }
680
kvm_arch_vcpu_get_ip(struct kvm_vcpu * vcpu)681 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
682 {
683 return *vcpu_pc(vcpu);
684 }
685
kvm_vcpu_initialized(struct kvm_vcpu * vcpu)686 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
687 {
688 return vcpu->arch.target >= 0;
689 }
690
691 /*
692 * Handle both the initialisation that is being done when the vcpu is
693 * run for the first time, as well as the updates that must be
694 * performed each time we get a new thread dealing with this vcpu.
695 */
kvm_arch_vcpu_run_pid_change(struct kvm_vcpu * vcpu)696 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
697 {
698 struct kvm *kvm = vcpu->kvm;
699 int ret;
700
701 if (!kvm_vcpu_initialized(vcpu))
702 return -ENOEXEC;
703
704 if (!kvm_arm_vcpu_is_finalized(vcpu))
705 return -EPERM;
706
707 ret = kvm_arch_vcpu_run_map_fp(vcpu);
708 if (ret)
709 return ret;
710
711 if (likely(vcpu_has_run_once(vcpu)))
712 return 0;
713
714 kvm_arm_vcpu_init_debug(vcpu);
715
716 if (likely(irqchip_in_kernel(kvm))) {
717 /*
718 * Map the VGIC hardware resources before running a vcpu the
719 * first time on this VM.
720 */
721 ret = kvm_vgic_map_resources(kvm);
722 if (ret)
723 return ret;
724 }
725
726 ret = kvm_timer_enable(vcpu);
727 if (ret)
728 return ret;
729
730 ret = kvm_arm_pmu_v3_enable(vcpu);
731 if (ret)
732 return ret;
733
734 if (is_protected_kvm_enabled()) {
735 /* Start with the vcpu in a dirty state */
736 if (!kvm_vm_is_protected(vcpu->kvm))
737 vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
738 ret = pkvm_create_hyp_vm(kvm);
739 if (ret)
740 return ret;
741 }
742
743 if (!irqchip_in_kernel(kvm)) {
744 /*
745 * Tell the rest of the code that there are userspace irqchip
746 * VMs in the wild.
747 */
748 static_branch_inc(&userspace_irqchip_in_use);
749 }
750
751 mutex_lock(&kvm->arch.config_lock);
752 set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
753 mutex_unlock(&kvm->arch.config_lock);
754
755 return ret;
756 }
757
kvm_arch_intc_initialized(struct kvm * kvm)758 bool kvm_arch_intc_initialized(struct kvm *kvm)
759 {
760 return vgic_initialized(kvm);
761 }
762
kvm_arm_halt_guest(struct kvm * kvm)763 void kvm_arm_halt_guest(struct kvm *kvm)
764 {
765 unsigned long i;
766 struct kvm_vcpu *vcpu;
767
768 kvm_for_each_vcpu(i, vcpu, kvm)
769 vcpu->arch.pause = true;
770 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
771 }
772
kvm_arm_resume_guest(struct kvm * kvm)773 void kvm_arm_resume_guest(struct kvm *kvm)
774 {
775 unsigned long i;
776 struct kvm_vcpu *vcpu;
777
778 kvm_for_each_vcpu(i, vcpu, kvm) {
779 vcpu->arch.pause = false;
780 rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu));
781 }
782 }
783
kvm_vcpu_sleep(struct kvm_vcpu * vcpu)784 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
785 {
786 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
787
788 rcuwait_wait_event(wait,
789 (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
790 TASK_INTERRUPTIBLE);
791
792 if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
793 /* Awaken to handle a signal, request we sleep again later. */
794 kvm_make_request(KVM_REQ_SLEEP, vcpu);
795 }
796
797 /*
798 * Make sure we will observe a potential reset request if we've
799 * observed a change to the power state. Pairs with the smp_wmb() in
800 * kvm_psci_vcpu_on().
801 */
802 smp_rmb();
803 }
804
805 /**
806 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
807 * @vcpu: The VCPU pointer
808 *
809 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
810 * the vCPU is runnable. The vCPU may or may not be scheduled out, depending
811 * on when a wake event arrives, e.g. there may already be a pending wake event.
812 */
kvm_vcpu_wfi(struct kvm_vcpu * vcpu)813 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
814 {
815 /*
816 * Sync back the state of the GIC CPU interface so that we have
817 * the latest PMR and group enables. This ensures that
818 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
819 * we have pending interrupts, e.g. when determining if the
820 * vCPU should block.
821 *
822 * For the same reason, we want to tell GICv4 that we need
823 * doorbells to be signalled, should an interrupt become pending.
824 */
825 preempt_disable();
826 kvm_vgic_put(vcpu, true);
827 preempt_enable();
828
829 kvm_vcpu_block(vcpu);
830 vcpu_clear_flag(vcpu, IN_WFIT);
831 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
832
833 preempt_disable();
834 kvm_vgic_load(vcpu);
835 preempt_enable();
836 }
837
kvm_vcpu_suspend(struct kvm_vcpu * vcpu)838 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
839 {
840 if (!kvm_arm_vcpu_suspended(vcpu))
841 return 1;
842
843 kvm_vcpu_wfi(vcpu);
844
845 /*
846 * The suspend state is sticky; we do not leave it until userspace
847 * explicitly marks the vCPU as runnable. Request that we suspend again
848 * later.
849 */
850 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
851
852 /*
853 * Check to make sure the vCPU is actually runnable. If so, exit to
854 * userspace informing it of the wakeup condition.
855 */
856 if (kvm_arch_vcpu_runnable(vcpu)) {
857 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
858 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
859 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
860 return 0;
861 }
862
863 /*
864 * Otherwise, we were unblocked to process a different event, such as a
865 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
866 * process the event.
867 */
868 return 1;
869 }
870
871 /**
872 * check_vcpu_requests - check and handle pending vCPU requests
873 * @vcpu: the VCPU pointer
874 *
875 * Return: 1 if we should enter the guest
876 * 0 if we should exit to userspace
877 * < 0 if we should exit to userspace, where the return value indicates
878 * an error
879 */
check_vcpu_requests(struct kvm_vcpu * vcpu)880 static int check_vcpu_requests(struct kvm_vcpu *vcpu)
881 {
882 if (kvm_request_pending(vcpu)) {
883 if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
884 kvm_vcpu_sleep(vcpu);
885
886 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
887 kvm_reset_vcpu(vcpu);
888
889 /*
890 * Clear IRQ_PENDING requests that were made to guarantee
891 * that a VCPU sees new virtual interrupts.
892 */
893 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
894
895 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
896 kvm_update_stolen_time(vcpu);
897
898 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
899 /* The distributor enable bits were changed */
900 preempt_disable();
901 vgic_v4_put(vcpu, false);
902 vgic_v4_load(vcpu);
903 preempt_enable();
904 }
905
906 if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
907 kvm_pmu_handle_pmcr(vcpu,
908 __vcpu_sys_reg(vcpu, PMCR_EL0));
909
910 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
911 return kvm_vcpu_suspend(vcpu);
912 }
913
914 return 1;
915 }
916
vcpu_mode_is_bad_32bit(struct kvm_vcpu * vcpu)917 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
918 {
919 if (likely(!vcpu_mode_is_32bit(vcpu)))
920 return false;
921
922 return !kvm_supports_32bit_el0();
923 }
924
925 /**
926 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
927 * @vcpu: The VCPU pointer
928 * @ret: Pointer to write optional return code
929 *
930 * Returns: true if the VCPU needs to return to a preemptible + interruptible
931 * and skip guest entry.
932 *
933 * This function disambiguates between two different types of exits: exits to a
934 * preemptible + interruptible kernel context and exits to userspace. For an
935 * exit to userspace, this function will write the return code to ret and return
936 * true. For an exit to preemptible + interruptible kernel context (i.e. check
937 * for pending work and re-enter), return true without writing to ret.
938 */
kvm_vcpu_exit_request(struct kvm_vcpu * vcpu,int * ret)939 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
940 {
941 struct kvm_run *run = vcpu->run;
942
943 /*
944 * If we're using a userspace irqchip, then check if we need
945 * to tell a userspace irqchip about timer or PMU level
946 * changes and if so, exit to userspace (the actual level
947 * state gets updated in kvm_timer_update_run and
948 * kvm_pmu_update_run below).
949 */
950 if (static_branch_unlikely(&userspace_irqchip_in_use)) {
951 if (kvm_timer_should_notify_user(vcpu) ||
952 kvm_pmu_should_notify_user(vcpu)) {
953 *ret = -EINTR;
954 run->exit_reason = KVM_EXIT_INTR;
955 return true;
956 }
957 }
958
959 if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
960 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
961 run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
962 run->fail_entry.cpu = smp_processor_id();
963 *ret = 0;
964 return true;
965 }
966
967 return kvm_request_pending(vcpu) ||
968 xfer_to_guest_mode_work_pending();
969 }
970
971 /*
972 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
973 * the vCPU is running.
974 *
975 * This must be noinstr as instrumentation may make use of RCU, and this is not
976 * safe during the EQS.
977 */
kvm_arm_vcpu_enter_exit(struct kvm_vcpu * vcpu)978 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
979 {
980 int ret;
981
982 guest_state_enter_irqoff();
983 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
984 guest_state_exit_irqoff();
985
986 return ret;
987 }
988
989 /**
990 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
991 * @vcpu: The VCPU pointer
992 *
993 * This function is called through the VCPU_RUN ioctl called from user space. It
994 * will execute VM code in a loop until the time slice for the process is used
995 * or some emulation is needed from user space in which case the function will
996 * return with return value 0 and with the kvm_run structure filled in with the
997 * required data for the requested emulation.
998 */
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)999 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1000 {
1001 struct kvm_run *run = vcpu->run;
1002 int ret;
1003
1004 if (run->exit_reason == KVM_EXIT_MMIO) {
1005 ret = kvm_handle_mmio_return(vcpu);
1006 if (ret)
1007 return ret;
1008 }
1009
1010 vcpu_load(vcpu);
1011
1012 if (run->immediate_exit) {
1013 ret = -EINTR;
1014 goto out;
1015 }
1016
1017 kvm_sigset_activate(vcpu);
1018
1019 ret = 1;
1020 run->exit_reason = KVM_EXIT_UNKNOWN;
1021 while (ret > 0) {
1022 /*
1023 * Check conditions before entering the guest
1024 */
1025 ret = xfer_to_guest_mode_handle_work(vcpu);
1026 if (!ret)
1027 ret = 1;
1028
1029 if (ret > 0)
1030 ret = check_vcpu_requests(vcpu);
1031
1032 /*
1033 * Preparing the interrupts to be injected also
1034 * involves poking the GIC, which must be done in a
1035 * non-preemptible context.
1036 */
1037 preempt_disable();
1038
1039 /*
1040 * The VMID allocator only tracks active VMIDs per
1041 * physical CPU, and therefore the VMID allocated may not be
1042 * preserved on VMID roll-over if the task was preempted,
1043 * making a thread's VMID inactive. So we need to call
1044 * kvm_arm_vmid_update() in non-premptible context.
1045 */
1046 kvm_arm_vmid_update(&vcpu->arch.hw_mmu->vmid);
1047
1048 kvm_pmu_flush_hwstate(vcpu);
1049
1050 local_irq_disable();
1051
1052 kvm_vgic_flush_hwstate(vcpu);
1053
1054 kvm_pmu_update_vcpu_events(vcpu);
1055
1056 /*
1057 * Ensure we set mode to IN_GUEST_MODE after we disable
1058 * interrupts and before the final VCPU requests check.
1059 * See the comment in kvm_vcpu_exiting_guest_mode() and
1060 * Documentation/virt/kvm/vcpu-requests.rst
1061 */
1062 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
1063
1064 if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
1065 vcpu->mode = OUTSIDE_GUEST_MODE;
1066 isb(); /* Ensure work in x_flush_hwstate is committed */
1067 kvm_pmu_sync_hwstate(vcpu);
1068 if (static_branch_unlikely(&userspace_irqchip_in_use))
1069 kvm_timer_sync_user(vcpu);
1070 kvm_vgic_sync_hwstate(vcpu);
1071 local_irq_enable();
1072 preempt_enable();
1073 continue;
1074 }
1075
1076 kvm_arm_setup_debug(vcpu);
1077 kvm_arch_vcpu_ctxflush_fp(vcpu);
1078
1079 /**************************************************************
1080 * Enter the guest
1081 */
1082 trace_kvm_entry(*vcpu_pc(vcpu));
1083 guest_timing_enter_irqoff();
1084
1085 ret = kvm_arm_vcpu_enter_exit(vcpu);
1086
1087 vcpu->mode = OUTSIDE_GUEST_MODE;
1088 vcpu->stat.exits++;
1089 /*
1090 * Back from guest
1091 *************************************************************/
1092
1093 kvm_arm_clear_debug(vcpu);
1094
1095 /*
1096 * We must sync the PMU state before the vgic state so
1097 * that the vgic can properly sample the updated state of the
1098 * interrupt line.
1099 */
1100 kvm_pmu_sync_hwstate(vcpu);
1101
1102 /*
1103 * Sync the vgic state before syncing the timer state because
1104 * the timer code needs to know if the virtual timer
1105 * interrupts are active.
1106 */
1107 kvm_vgic_sync_hwstate(vcpu);
1108
1109 /*
1110 * Sync the timer hardware state before enabling interrupts as
1111 * we don't want vtimer interrupts to race with syncing the
1112 * timer virtual interrupt state.
1113 */
1114 if (static_branch_unlikely(&userspace_irqchip_in_use))
1115 kvm_timer_sync_user(vcpu);
1116
1117 kvm_arch_vcpu_ctxsync_fp(vcpu);
1118
1119 /*
1120 * We must ensure that any pending interrupts are taken before
1121 * we exit guest timing so that timer ticks are accounted as
1122 * guest time. Transiently unmask interrupts so that any
1123 * pending interrupts are taken.
1124 *
1125 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
1126 * context synchronization event) is necessary to ensure that
1127 * pending interrupts are taken.
1128 */
1129 if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
1130 local_irq_enable();
1131 isb();
1132 local_irq_disable();
1133 }
1134
1135 guest_timing_exit_irqoff();
1136
1137 local_irq_enable();
1138
1139 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
1140
1141 /* Exit types that need handling before we can be preempted */
1142 handle_exit_early(vcpu, ret);
1143
1144 preempt_enable();
1145
1146 /*
1147 * The ARMv8 architecture doesn't give the hypervisor
1148 * a mechanism to prevent a guest from dropping to AArch32 EL0
1149 * if implemented by the CPU. If we spot the guest in such
1150 * state and that we decided it wasn't supposed to do so (like
1151 * with the asymmetric AArch32 case), return to userspace with
1152 * a fatal error.
1153 */
1154 if (vcpu_mode_is_bad_32bit(vcpu)) {
1155 /*
1156 * As we have caught the guest red-handed, decide that
1157 * it isn't fit for purpose anymore by making the vcpu
1158 * invalid. The VMM can try and fix it by issuing a
1159 * KVM_ARM_VCPU_INIT if it really wants to.
1160 */
1161 vcpu->arch.target = -1;
1162 ret = ARM_EXCEPTION_IL;
1163 }
1164
1165 ret = handle_exit(vcpu, ret);
1166 }
1167
1168 /* Tell userspace about in-kernel device output levels */
1169 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1170 kvm_timer_update_run(vcpu);
1171 kvm_pmu_update_run(vcpu);
1172 }
1173
1174 kvm_sigset_deactivate(vcpu);
1175
1176 out:
1177 /*
1178 * In the unlikely event that we are returning to userspace
1179 * with pending exceptions or PC adjustment, commit these
1180 * adjustments in order to give userspace a consistent view of
1181 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1182 * being preempt-safe on VHE.
1183 */
1184 if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1185 vcpu_get_flag(vcpu, INCREMENT_PC)))
1186 kvm_call_hyp(__kvm_adjust_pc, vcpu);
1187
1188 vcpu_put(vcpu);
1189 return ret;
1190 }
1191
vcpu_interrupt_line(struct kvm_vcpu * vcpu,int number,bool level)1192 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1193 {
1194 int bit_index;
1195 bool set;
1196 unsigned long *hcr;
1197
1198 if (number == KVM_ARM_IRQ_CPU_IRQ)
1199 bit_index = __ffs(HCR_VI);
1200 else /* KVM_ARM_IRQ_CPU_FIQ */
1201 bit_index = __ffs(HCR_VF);
1202
1203 hcr = vcpu_hcr(vcpu);
1204 if (level)
1205 set = test_and_set_bit(bit_index, hcr);
1206 else
1207 set = test_and_clear_bit(bit_index, hcr);
1208
1209 /*
1210 * If we didn't change anything, no need to wake up or kick other CPUs
1211 */
1212 if (set == level)
1213 return 0;
1214
1215 /*
1216 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1217 * trigger a world-switch round on the running physical CPU to set the
1218 * virtual IRQ/FIQ fields in the HCR appropriately.
1219 */
1220 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1221 kvm_vcpu_kick(vcpu);
1222
1223 return 0;
1224 }
1225
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_level,bool line_status)1226 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1227 bool line_status)
1228 {
1229 u32 irq = irq_level->irq;
1230 unsigned int irq_type, vcpu_idx, irq_num;
1231 int nrcpus = atomic_read(&kvm->online_vcpus);
1232 struct kvm_vcpu *vcpu = NULL;
1233 bool level = irq_level->level;
1234
1235 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1236 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1237 vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1238 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1239
1240 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
1241
1242 switch (irq_type) {
1243 case KVM_ARM_IRQ_TYPE_CPU:
1244 if (irqchip_in_kernel(kvm))
1245 return -ENXIO;
1246
1247 if (vcpu_idx >= nrcpus)
1248 return -EINVAL;
1249
1250 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1251 if (!vcpu)
1252 return -EINVAL;
1253
1254 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1255 return -EINVAL;
1256
1257 return vcpu_interrupt_line(vcpu, irq_num, level);
1258 case KVM_ARM_IRQ_TYPE_PPI:
1259 if (!irqchip_in_kernel(kvm))
1260 return -ENXIO;
1261
1262 if (vcpu_idx >= nrcpus)
1263 return -EINVAL;
1264
1265 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1266 if (!vcpu)
1267 return -EINVAL;
1268
1269 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1270 return -EINVAL;
1271
1272 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);
1273 case KVM_ARM_IRQ_TYPE_SPI:
1274 if (!irqchip_in_kernel(kvm))
1275 return -ENXIO;
1276
1277 if (irq_num < VGIC_NR_PRIVATE_IRQS)
1278 return -EINVAL;
1279
1280 return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);
1281 }
1282
1283 return -EINVAL;
1284 }
1285
kvm_vcpu_set_target(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1286 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1287 const struct kvm_vcpu_init *init)
1288 {
1289 unsigned int i, ret;
1290 u32 phys_target = kvm_target_cpu();
1291
1292 if (init->target != phys_target)
1293 return -EINVAL;
1294
1295 /*
1296 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1297 * use the same target.
1298 */
1299 if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
1300 return -EINVAL;
1301
1302 /* -ENOENT for unknown features, -EINVAL for invalid combinations. */
1303 for (i = 0; i < sizeof(init->features) * 8; i++) {
1304 bool set = (init->features[i / 32] & (1 << (i % 32)));
1305
1306 if (set && i >= KVM_VCPU_MAX_FEATURES)
1307 return -ENOENT;
1308
1309 /*
1310 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1311 * use the same feature set.
1312 */
1313 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
1314 test_bit(i, vcpu->arch.features) != set)
1315 return -EINVAL;
1316
1317 if (set)
1318 set_bit(i, vcpu->arch.features);
1319 }
1320
1321 vcpu->arch.target = phys_target;
1322
1323 /* Now we know what it is, we can reset it. */
1324 ret = kvm_reset_vcpu(vcpu);
1325 if (ret) {
1326 vcpu->arch.target = -1;
1327 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
1328 }
1329
1330 return ret;
1331 }
1332
kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu * vcpu,struct kvm_vcpu_init * init)1333 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1334 struct kvm_vcpu_init *init)
1335 {
1336 int ret;
1337
1338 ret = kvm_vcpu_set_target(vcpu, init);
1339 if (ret)
1340 return ret;
1341
1342 /*
1343 * Ensure a rebooted VM will fault in RAM pages and detect if the
1344 * guest MMU is turned off and flush the caches as needed.
1345 *
1346 * S2FWB enforces all memory accesses to RAM being cacheable,
1347 * ensuring that the data side is always coherent. We still
1348 * need to invalidate the I-cache though, as FWB does *not*
1349 * imply CTR_EL0.DIC.
1350 */
1351 if (vcpu_has_run_once(vcpu)) {
1352 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1353 stage2_unmap_vm(vcpu->kvm);
1354 else
1355 icache_inval_all_pou();
1356 }
1357
1358 vcpu_reset_hcr(vcpu);
1359 vcpu->arch.cptr_el2 = CPTR_EL2_DEFAULT;
1360
1361 /*
1362 * Handle the "start in power-off" case.
1363 */
1364 if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
1365 kvm_arm_vcpu_power_off(vcpu);
1366 else
1367 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1368
1369 return 0;
1370 }
1371
kvm_arm_vcpu_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1372 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1373 struct kvm_device_attr *attr)
1374 {
1375 int ret = -ENXIO;
1376
1377 switch (attr->group) {
1378 default:
1379 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1380 break;
1381 }
1382
1383 return ret;
1384 }
1385
kvm_arm_vcpu_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1386 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1387 struct kvm_device_attr *attr)
1388 {
1389 int ret = -ENXIO;
1390
1391 switch (attr->group) {
1392 default:
1393 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1394 break;
1395 }
1396
1397 return ret;
1398 }
1399
kvm_arm_vcpu_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1400 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1401 struct kvm_device_attr *attr)
1402 {
1403 int ret = -ENXIO;
1404
1405 switch (attr->group) {
1406 default:
1407 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1408 break;
1409 }
1410
1411 return ret;
1412 }
1413
kvm_arm_vcpu_get_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1414 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1415 struct kvm_vcpu_events *events)
1416 {
1417 memset(events, 0, sizeof(*events));
1418
1419 return __kvm_arm_vcpu_get_events(vcpu, events);
1420 }
1421
kvm_arm_vcpu_set_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1422 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1423 struct kvm_vcpu_events *events)
1424 {
1425 int i;
1426
1427 /* check whether the reserved field is zero */
1428 for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1429 if (events->reserved[i])
1430 return -EINVAL;
1431
1432 /* check whether the pad field is zero */
1433 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1434 if (events->exception.pad[i])
1435 return -EINVAL;
1436
1437 return __kvm_arm_vcpu_set_events(vcpu, events);
1438 }
1439
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1440 long kvm_arch_vcpu_ioctl(struct file *filp,
1441 unsigned int ioctl, unsigned long arg)
1442 {
1443 struct kvm_vcpu *vcpu = filp->private_data;
1444 void __user *argp = (void __user *)arg;
1445 struct kvm_device_attr attr;
1446 long r;
1447
1448 switch (ioctl) {
1449 case KVM_ARM_VCPU_INIT: {
1450 struct kvm_vcpu_init init;
1451
1452 r = -EFAULT;
1453 if (copy_from_user(&init, argp, sizeof(init)))
1454 break;
1455
1456 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1457 break;
1458 }
1459 case KVM_SET_ONE_REG:
1460 case KVM_GET_ONE_REG: {
1461 struct kvm_one_reg reg;
1462
1463 r = -ENOEXEC;
1464 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1465 break;
1466
1467 r = -EFAULT;
1468 if (copy_from_user(®, argp, sizeof(reg)))
1469 break;
1470
1471 /*
1472 * We could owe a reset due to PSCI. Handle the pending reset
1473 * here to ensure userspace register accesses are ordered after
1474 * the reset.
1475 */
1476 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1477 kvm_reset_vcpu(vcpu);
1478
1479 if (ioctl == KVM_SET_ONE_REG)
1480 r = kvm_arm_set_reg(vcpu, ®);
1481 else
1482 r = kvm_arm_get_reg(vcpu, ®);
1483 break;
1484 }
1485 case KVM_GET_REG_LIST: {
1486 struct kvm_reg_list __user *user_list = argp;
1487 struct kvm_reg_list reg_list;
1488 unsigned n;
1489
1490 r = -ENOEXEC;
1491 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1492 break;
1493
1494 r = -EPERM;
1495 if (!kvm_arm_vcpu_is_finalized(vcpu))
1496 break;
1497
1498 r = -EFAULT;
1499 if (copy_from_user(®_list, user_list, sizeof(reg_list)))
1500 break;
1501 n = reg_list.n;
1502 reg_list.n = kvm_arm_num_regs(vcpu);
1503 if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
1504 break;
1505 r = -E2BIG;
1506 if (n < reg_list.n)
1507 break;
1508 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1509 break;
1510 }
1511 case KVM_SET_DEVICE_ATTR: {
1512 r = -EFAULT;
1513 if (copy_from_user(&attr, argp, sizeof(attr)))
1514 break;
1515 r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1516 break;
1517 }
1518 case KVM_GET_DEVICE_ATTR: {
1519 r = -EFAULT;
1520 if (copy_from_user(&attr, argp, sizeof(attr)))
1521 break;
1522 r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1523 break;
1524 }
1525 case KVM_HAS_DEVICE_ATTR: {
1526 r = -EFAULT;
1527 if (copy_from_user(&attr, argp, sizeof(attr)))
1528 break;
1529 r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1530 break;
1531 }
1532 case KVM_GET_VCPU_EVENTS: {
1533 struct kvm_vcpu_events events;
1534
1535 if (kvm_arm_vcpu_get_events(vcpu, &events))
1536 return -EINVAL;
1537
1538 if (copy_to_user(argp, &events, sizeof(events)))
1539 return -EFAULT;
1540
1541 return 0;
1542 }
1543 case KVM_SET_VCPU_EVENTS: {
1544 struct kvm_vcpu_events events;
1545
1546 if (copy_from_user(&events, argp, sizeof(events)))
1547 return -EFAULT;
1548
1549 return kvm_arm_vcpu_set_events(vcpu, &events);
1550 }
1551 case KVM_ARM_VCPU_FINALIZE: {
1552 int what;
1553
1554 if (!kvm_vcpu_initialized(vcpu))
1555 return -ENOEXEC;
1556
1557 if (get_user(what, (const int __user *)argp))
1558 return -EFAULT;
1559
1560 return kvm_arm_vcpu_finalize(vcpu, what);
1561 }
1562 default:
1563 r = -EINVAL;
1564 }
1565
1566 return r;
1567 }
1568
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)1569 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1570 {
1571
1572 }
1573
kvm_arch_flush_remote_tlbs_memslot(struct kvm * kvm,const struct kvm_memory_slot * memslot)1574 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
1575 const struct kvm_memory_slot *memslot)
1576 {
1577 kvm_flush_remote_tlbs(kvm);
1578 }
1579
kvm_vm_ioctl_set_device_addr(struct kvm * kvm,struct kvm_arm_device_addr * dev_addr)1580 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1581 struct kvm_arm_device_addr *dev_addr)
1582 {
1583 switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1584 case KVM_ARM_DEVICE_VGIC_V2:
1585 if (!vgic_present)
1586 return -ENXIO;
1587 return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1588 default:
1589 return -ENODEV;
1590 }
1591 }
1592
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1593 long kvm_arch_vm_ioctl(struct file *filp,
1594 unsigned int ioctl, unsigned long arg)
1595 {
1596 struct kvm *kvm = filp->private_data;
1597 void __user *argp = (void __user *)arg;
1598
1599 switch (ioctl) {
1600 case KVM_CREATE_IRQCHIP: {
1601 int ret;
1602 if (!vgic_present)
1603 return -ENXIO;
1604 mutex_lock(&kvm->lock);
1605 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1606 mutex_unlock(&kvm->lock);
1607 return ret;
1608 }
1609 case KVM_ARM_SET_DEVICE_ADDR: {
1610 struct kvm_arm_device_addr dev_addr;
1611
1612 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1613 return -EFAULT;
1614 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1615 }
1616 case KVM_ARM_PREFERRED_TARGET: {
1617 struct kvm_vcpu_init init;
1618
1619 kvm_vcpu_preferred_target(&init);
1620
1621 if (copy_to_user(argp, &init, sizeof(init)))
1622 return -EFAULT;
1623
1624 return 0;
1625 }
1626 case KVM_ARM_MTE_COPY_TAGS: {
1627 struct kvm_arm_copy_mte_tags copy_tags;
1628
1629 if (copy_from_user(©_tags, argp, sizeof(copy_tags)))
1630 return -EFAULT;
1631 return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags);
1632 }
1633 default:
1634 return -EINVAL;
1635 }
1636 }
1637
nvhe_percpu_size(void)1638 static unsigned long nvhe_percpu_size(void)
1639 {
1640 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1641 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1642 }
1643
nvhe_percpu_order(void)1644 static unsigned long nvhe_percpu_order(void)
1645 {
1646 unsigned long size = nvhe_percpu_size();
1647
1648 return size ? get_order(size) : 0;
1649 }
1650
pkvm_host_fp_state_order(void)1651 static inline size_t pkvm_host_fp_state_order(void)
1652 {
1653 return get_order(pkvm_host_fp_state_size());
1654 }
1655
1656 /* A lookup table holding the hypervisor VA for each vector slot */
1657 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1658
kvm_init_vector_slot(void * base,enum arm64_hyp_spectre_vector slot)1659 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1660 {
1661 hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1662 }
1663
kvm_init_vector_slots(void)1664 static int kvm_init_vector_slots(void)
1665 {
1666 int err;
1667 void *base;
1668
1669 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1670 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1671
1672 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1673 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1674
1675 if (kvm_system_needs_idmapped_vectors() &&
1676 !is_protected_kvm_enabled()) {
1677 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1678 __BP_HARDEN_HYP_VECS_SZ, &base);
1679 if (err)
1680 return err;
1681 }
1682
1683 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
1684 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
1685 return 0;
1686 }
1687
cpu_prepare_hyp_mode(int cpu)1688 static void cpu_prepare_hyp_mode(int cpu)
1689 {
1690 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
1691 unsigned long tcr;
1692 int *hyp_cpu_number_ptr = per_cpu_ptr_nvhe_sym(hyp_cpu_number, cpu);
1693
1694 *hyp_cpu_number_ptr = cpu;
1695
1696 /*
1697 * Calculate the raw per-cpu offset without a translation from the
1698 * kernel's mapping to the linear mapping, and store it in tpidr_el2
1699 * so that we can use adr_l to access per-cpu variables in EL2.
1700 * Also drop the KASAN tag which gets in the way...
1701 */
1702 params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
1703 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
1704
1705 params->mair_el2 = read_sysreg(mair_el1);
1706
1707 /*
1708 * The ID map may be configured to use an extended virtual address
1709 * range. This is only the case if system RAM is out of range for the
1710 * currently configured page size and VA_BITS, in which case we will
1711 * also need the extended virtual range for the HYP ID map, or we won't
1712 * be able to enable the EL2 MMU.
1713 *
1714 * However, at EL2, there is only one TTBR register, and we can't switch
1715 * between translation tables *and* update TCR_EL2.T0SZ at the same
1716 * time. Bottom line: we need to use the extended range with *both* our
1717 * translation tables.
1718 *
1719 * So use the same T0SZ value we use for the ID map.
1720 */
1721 tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;
1722 tcr &= ~TCR_T0SZ_MASK;
1723 tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
1724 params->tcr_el2 = tcr;
1725
1726 params->pgd_pa = kvm_mmu_get_httbr();
1727 if (is_protected_kvm_enabled())
1728 params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
1729 else
1730 params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
1731 params->vttbr = params->vtcr = 0;
1732 params->hfgwtr_el2 = HFGxTR_EL2_nSMPRI_EL1_MASK | HFGxTR_EL2_nTPIDR2_EL0_MASK;
1733
1734 /*
1735 * Flush the init params from the data cache because the struct will
1736 * be read while the MMU is off.
1737 */
1738 kvm_flush_dcache_to_poc(params, sizeof(*params));
1739 }
1740
hyp_install_host_vector(void)1741 static void hyp_install_host_vector(void)
1742 {
1743 struct kvm_nvhe_init_params *params;
1744 struct arm_smccc_res res;
1745
1746 /* Switch from the HYP stub to our own HYP init vector */
1747 __hyp_set_vectors(kvm_get_idmap_vector());
1748
1749 /*
1750 * Call initialization code, and switch to the full blown HYP code.
1751 * If the cpucaps haven't been finalized yet, something has gone very
1752 * wrong, and hyp will crash and burn when it uses any
1753 * cpus_have_const_cap() wrapper.
1754 */
1755 BUG_ON(!system_capabilities_finalized());
1756 params = this_cpu_ptr_nvhe_sym(kvm_init_params);
1757 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
1758 WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
1759 }
1760
cpu_init_hyp_mode(void)1761 static void cpu_init_hyp_mode(void)
1762 {
1763 hyp_install_host_vector();
1764
1765 /*
1766 * Disabling SSBD on a non-VHE system requires us to enable SSBS
1767 * at EL2.
1768 */
1769 if (this_cpu_has_cap(ARM64_SSBS) &&
1770 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
1771 kvm_call_hyp_nvhe(__kvm_enable_ssbs);
1772 }
1773 }
1774
cpu_hyp_reset(void)1775 static void cpu_hyp_reset(void)
1776 {
1777 if (!is_kernel_in_hyp_mode())
1778 __hyp_reset_vectors();
1779 }
1780
1781 /*
1782 * EL2 vectors can be mapped and rerouted in a number of ways,
1783 * depending on the kernel configuration and CPU present:
1784 *
1785 * - If the CPU is affected by Spectre-v2, the hardening sequence is
1786 * placed in one of the vector slots, which is executed before jumping
1787 * to the real vectors.
1788 *
1789 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
1790 * containing the hardening sequence is mapped next to the idmap page,
1791 * and executed before jumping to the real vectors.
1792 *
1793 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
1794 * empty slot is selected, mapped next to the idmap page, and
1795 * executed before jumping to the real vectors.
1796 *
1797 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
1798 * VHE, as we don't have hypervisor-specific mappings. If the system
1799 * is VHE and yet selects this capability, it will be ignored.
1800 */
cpu_set_hyp_vector(void)1801 static void cpu_set_hyp_vector(void)
1802 {
1803 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
1804 void *vector = hyp_spectre_vector_selector[data->slot];
1805
1806 if (!is_protected_kvm_enabled())
1807 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
1808 else
1809 kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
1810 }
1811
cpu_hyp_init_context(void)1812 static void cpu_hyp_init_context(void)
1813 {
1814 kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt);
1815
1816 if (!is_kernel_in_hyp_mode())
1817 cpu_init_hyp_mode();
1818 }
1819
cpu_hyp_init_features(void)1820 static void cpu_hyp_init_features(void)
1821 {
1822 cpu_set_hyp_vector();
1823 kvm_arm_init_debug();
1824
1825 if (is_kernel_in_hyp_mode())
1826 kvm_timer_init_vhe();
1827
1828 if (vgic_present)
1829 kvm_vgic_init_cpu_hardware();
1830 }
1831
cpu_hyp_reinit(void)1832 static void cpu_hyp_reinit(void)
1833 {
1834 cpu_hyp_reset();
1835 cpu_hyp_init_context();
1836 cpu_hyp_init_features();
1837 }
1838
_kvm_arch_hardware_enable(void * discard)1839 static void _kvm_arch_hardware_enable(void *discard)
1840 {
1841 if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
1842 cpu_hyp_reinit();
1843 __this_cpu_write(kvm_arm_hardware_enabled, 1);
1844 }
1845 }
1846
kvm_arch_hardware_enable(void)1847 int kvm_arch_hardware_enable(void)
1848 {
1849 _kvm_arch_hardware_enable(NULL);
1850 return 0;
1851 }
1852
_kvm_arch_hardware_disable(void * discard)1853 static void _kvm_arch_hardware_disable(void *discard)
1854 {
1855 if (__this_cpu_read(kvm_arm_hardware_enabled)) {
1856 cpu_hyp_reset();
1857 __this_cpu_write(kvm_arm_hardware_enabled, 0);
1858 }
1859 }
1860
kvm_arch_hardware_disable(void)1861 void kvm_arch_hardware_disable(void)
1862 {
1863 if (!is_protected_kvm_enabled())
1864 _kvm_arch_hardware_disable(NULL);
1865 }
1866
1867 #ifdef CONFIG_CPU_PM
hyp_init_cpu_pm_notifier(struct notifier_block * self,unsigned long cmd,void * v)1868 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
1869 unsigned long cmd,
1870 void *v)
1871 {
1872 /*
1873 * kvm_arm_hardware_enabled is left with its old value over
1874 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
1875 * re-enable hyp.
1876 */
1877 switch (cmd) {
1878 case CPU_PM_ENTER:
1879 if (__this_cpu_read(kvm_arm_hardware_enabled))
1880 /*
1881 * don't update kvm_arm_hardware_enabled here
1882 * so that the hardware will be re-enabled
1883 * when we resume. See below.
1884 */
1885 cpu_hyp_reset();
1886
1887 return NOTIFY_OK;
1888 case CPU_PM_ENTER_FAILED:
1889 case CPU_PM_EXIT:
1890 if (__this_cpu_read(kvm_arm_hardware_enabled))
1891 /* The hardware was enabled before suspend. */
1892 cpu_hyp_reinit();
1893
1894 return NOTIFY_OK;
1895
1896 default:
1897 return NOTIFY_DONE;
1898 }
1899 }
1900
1901 static struct notifier_block hyp_init_cpu_pm_nb = {
1902 .notifier_call = hyp_init_cpu_pm_notifier,
1903 };
1904
hyp_cpu_pm_init(void)1905 static void hyp_cpu_pm_init(void)
1906 {
1907 if (!is_protected_kvm_enabled())
1908 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
1909 }
hyp_cpu_pm_exit(void)1910 static void hyp_cpu_pm_exit(void)
1911 {
1912 if (!is_protected_kvm_enabled())
1913 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
1914 }
1915 #else
hyp_cpu_pm_init(void)1916 static inline void hyp_cpu_pm_init(void)
1917 {
1918 }
hyp_cpu_pm_exit(void)1919 static inline void hyp_cpu_pm_exit(void)
1920 {
1921 }
1922 #endif
1923
init_cpu_logical_map(void)1924 static void init_cpu_logical_map(void)
1925 {
1926 unsigned int cpu;
1927
1928 /*
1929 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
1930 * Only copy the set of online CPUs whose features have been checked
1931 * against the finalized system capabilities. The hypervisor will not
1932 * allow any other CPUs from the `possible` set to boot.
1933 */
1934 for_each_online_cpu(cpu)
1935 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
1936 }
1937
1938 #define init_psci_0_1_impl_state(config, what) \
1939 config.psci_0_1_ ## what ## _implemented = psci_ops.what
1940
init_psci_relay(void)1941 static bool init_psci_relay(void)
1942 {
1943 /*
1944 * If PSCI has not been initialized, protected KVM cannot install
1945 * itself on newly booted CPUs.
1946 */
1947 if (!psci_ops.get_version) {
1948 kvm_err("Cannot initialize protected mode without PSCI\n");
1949 return false;
1950 }
1951
1952 kvm_host_psci_config.version = psci_ops.get_version();
1953 kvm_host_psci_config.smccc_version = arm_smccc_get_version();
1954
1955 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
1956 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
1957 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
1958 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
1959 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
1960 init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
1961 }
1962 return true;
1963 }
1964
init_subsystems(void)1965 static int init_subsystems(void)
1966 {
1967 int err = 0;
1968
1969 /*
1970 * Enable hardware so that subsystem initialisation can access EL2.
1971 */
1972 on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
1973
1974 /*
1975 * Register CPU lower-power notifier
1976 */
1977 hyp_cpu_pm_init();
1978
1979 /*
1980 * Init HYP view of VGIC
1981 */
1982 err = kvm_vgic_hyp_init();
1983 switch (err) {
1984 case 0:
1985 vgic_present = true;
1986 break;
1987 case -ENODEV:
1988 case -ENXIO:
1989 vgic_present = false;
1990 err = 0;
1991 break;
1992 default:
1993 goto out;
1994 }
1995
1996 /*
1997 * Init HYP architected timer support
1998 */
1999 err = kvm_timer_hyp_init(vgic_present);
2000 if (err)
2001 goto out;
2002
2003 kvm_register_perf_callbacks(NULL);
2004
2005 out:
2006 if (err || !is_protected_kvm_enabled())
2007 on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
2008
2009 return err;
2010 }
2011
teardown_hyp_mode(void)2012 static void teardown_hyp_mode(void)
2013 {
2014 int cpu;
2015
2016 free_hyp_pgds();
2017 for_each_possible_cpu(cpu) {
2018 free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT);
2019 free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2020 free_pages(kvm_nvhe_sym(kvm_arm_hyp_host_fp_state)[cpu],
2021 pkvm_host_fp_state_order());
2022 }
2023 }
2024
do_pkvm_init(u32 hyp_va_bits)2025 static int do_pkvm_init(u32 hyp_va_bits)
2026 {
2027 void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
2028 int ret;
2029
2030 preempt_disable();
2031 cpu_hyp_init_context();
2032 ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
2033 num_possible_cpus(), kern_hyp_va(per_cpu_base),
2034 hyp_va_bits);
2035 cpu_hyp_init_features();
2036
2037 /*
2038 * The stub hypercalls are now disabled, so set our local flag to
2039 * prevent a later re-init attempt in kvm_arch_hardware_enable().
2040 */
2041 __this_cpu_write(kvm_arm_hardware_enabled, 1);
2042 preempt_enable();
2043
2044 return ret;
2045 }
2046
get_hyp_id_aa64pfr0_el1(void)2047 static u64 get_hyp_id_aa64pfr0_el1(void)
2048 {
2049 /*
2050 * Track whether the system isn't affected by spectre/meltdown in the
2051 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs.
2052 * Although this is per-CPU, we make it global for simplicity, e.g., not
2053 * to have to worry about vcpu migration.
2054 *
2055 * Unlike for non-protected VMs, userspace cannot override this for
2056 * protected VMs.
2057 */
2058 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
2059
2060 val &= ~(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2) |
2061 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3));
2062
2063 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2),
2064 arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED);
2065 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3),
2066 arm64_get_meltdown_state() == SPECTRE_UNAFFECTED);
2067
2068 return val;
2069 }
2070
kvm_hyp_init_symbols(void)2071 static void kvm_hyp_init_symbols(void)
2072 {
2073 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
2074 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
2075 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
2076 kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2077 kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
2078 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2079 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
2080 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
2081 kvm_nvhe_sym(__icache_flags) = __icache_flags;
2082 kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
2083 kvm_nvhe_sym(smccc_trng_available) = smccc_trng_available;
2084 kvm_nvhe_sym(kvm_host_sve_max_vl) = kvm_host_sve_max_vl;
2085 }
2086
2087 int kvm_hyp_init_events(void);
2088
kvm_hyp_init_protection(u32 hyp_va_bits)2089 static int kvm_hyp_init_protection(u32 hyp_va_bits)
2090 {
2091 void *addr = phys_to_virt(hyp_mem_base);
2092 int ret;
2093
2094 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
2095 if (ret)
2096 return ret;
2097
2098 ret = do_pkvm_init(hyp_va_bits);
2099 if (ret)
2100 return ret;
2101
2102 free_hyp_pgds();
2103
2104 return 0;
2105 }
2106
init_pkvm_host_fp_state(void)2107 static int init_pkvm_host_fp_state(void)
2108 {
2109 int cpu;
2110
2111 if (!is_protected_kvm_enabled())
2112 return 0;
2113
2114 /* Allocate pages for protected-mode host-fp state. */
2115 for_each_possible_cpu(cpu) {
2116 struct page *page;
2117 unsigned long addr;
2118
2119 page = alloc_pages(GFP_KERNEL, pkvm_host_fp_state_order());
2120 if (!page)
2121 return -ENOMEM;
2122
2123 addr = (unsigned long)page_address(page);
2124 kvm_nvhe_sym(kvm_arm_hyp_host_fp_state)[cpu] = addr;
2125 }
2126
2127 /*
2128 * Don't map the pages in hyp since these are only used in protected
2129 * mode, which will (re)create its own mapping when initialized.
2130 */
2131
2132 return 0;
2133 }
2134
2135 /*
2136 * Finalizes the initialization of hyp mode, once everything else is initialized
2137 * and the initialziation process cannot fail.
2138 */
finalize_init_hyp_mode(void)2139 static void finalize_init_hyp_mode(void)
2140 {
2141 int cpu;
2142
2143 for_each_possible_cpu(cpu) {
2144 kvm_nvhe_sym(kvm_arm_hyp_host_fp_state)[cpu] =
2145 kern_hyp_va(kvm_nvhe_sym(kvm_arm_hyp_host_fp_state)[cpu]);
2146 }
2147 }
2148
2149 /**
2150 * Inits Hyp-mode on all online CPUs
2151 */
init_hyp_mode(void)2152 static int init_hyp_mode(void)
2153 {
2154 u32 hyp_va_bits;
2155 int cpu;
2156 int err = -ENOMEM;
2157
2158 /*
2159 * The protected Hyp-mode cannot be initialized if the memory pool
2160 * allocation has failed.
2161 */
2162 if (is_protected_kvm_enabled() && !hyp_mem_base)
2163 goto out_err;
2164
2165 /*
2166 * Allocate Hyp PGD and setup Hyp identity mapping
2167 */
2168 err = kvm_mmu_init(&hyp_va_bits);
2169 if (err)
2170 goto out_err;
2171
2172 /*
2173 * Allocate stack pages for Hypervisor-mode
2174 */
2175 for_each_possible_cpu(cpu) {
2176 unsigned long stack_base;
2177
2178 stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT);
2179 if (!stack_base) {
2180 err = -ENOMEM;
2181 goto out_err;
2182 }
2183
2184 per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base;
2185 }
2186
2187 /*
2188 * Allocate and initialize pages for Hypervisor-mode percpu regions.
2189 */
2190 for_each_possible_cpu(cpu) {
2191 struct page *page;
2192 void *page_addr;
2193
2194 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
2195 if (!page) {
2196 err = -ENOMEM;
2197 goto out_err;
2198 }
2199
2200 page_addr = page_address(page);
2201 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
2202 kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
2203 }
2204
2205 /*
2206 * Map the Hyp-code called directly from the host
2207 */
2208 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
2209 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
2210 if (err) {
2211 kvm_err("Cannot map world-switch code\n");
2212 goto out_err;
2213 }
2214
2215 err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start),
2216 kvm_ksym_ref(__hyp_data_end), PAGE_HYP);
2217 if (err) {
2218 kvm_err("Cannot map .hyp.data section\n");
2219 goto out_err;
2220 }
2221
2222 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
2223 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
2224 if (err) {
2225 kvm_err("Cannot map .hyp.rodata section\n");
2226 goto out_err;
2227 }
2228
2229 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
2230 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
2231 if (err) {
2232 kvm_err("Cannot map rodata section\n");
2233 goto out_err;
2234 }
2235
2236 /*
2237 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
2238 * section thanks to an assertion in the linker script. Map it RW and
2239 * the rest of .bss RO.
2240 */
2241 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
2242 kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
2243 if (err) {
2244 kvm_err("Cannot map hyp bss section: %d\n", err);
2245 goto out_err;
2246 }
2247
2248 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
2249 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
2250 if (err) {
2251 kvm_err("Cannot map bss section\n");
2252 goto out_err;
2253 }
2254
2255 /*
2256 * Map the Hyp stack pages
2257 */
2258 for_each_possible_cpu(cpu) {
2259 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2260 char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu);
2261 unsigned long hyp_addr;
2262
2263 /*
2264 * Allocate a contiguous HYP private VA range for the stack
2265 * and guard page. The allocation is also aligned based on
2266 * the order of its size.
2267 */
2268 err = hyp_alloc_private_va_range(NVHE_STACK_SIZE * 2, &hyp_addr);
2269 if (err) {
2270 kvm_err("Cannot allocate hyp stack guard page\n");
2271 goto out_err;
2272 }
2273
2274 /*
2275 * Since the stack grows downwards, map the stack to the page
2276 * at the higher address and leave the lower guard page
2277 * unbacked.
2278 *
2279 * Any valid stack address now has the NVHE_STACK_SHIFT bit as 1
2280 * and addresses corresponding to the guard page have the
2281 * NVHE_STACK_SHIFT bit as 0 - this is used for overflow detection.
2282 */
2283 err = __create_hyp_mappings(hyp_addr + NVHE_STACK_SIZE, NVHE_STACK_SIZE,
2284 __pa(stack_base), PAGE_HYP);
2285 if (err) {
2286 kvm_err("Cannot map hyp stack\n");
2287 goto out_err;
2288 }
2289
2290 /*
2291 * Save the stack PA in nvhe_init_params. This will be needed
2292 * to recreate the stack mapping in protected nVHE mode.
2293 * __hyp_pa() won't do the right thing there, since the stack
2294 * has been mapped in the flexible private VA space.
2295 */
2296 params->stack_pa = __pa(stack_base);
2297
2298 params->stack_hyp_va = hyp_addr + (2 * NVHE_STACK_SIZE);
2299 }
2300
2301 for_each_possible_cpu(cpu) {
2302 char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2303 char *percpu_end = percpu_begin + nvhe_percpu_size();
2304
2305 /* Map Hyp percpu pages */
2306 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2307 if (err) {
2308 kvm_err("Cannot map hyp percpu region\n");
2309 goto out_err;
2310 }
2311
2312 /* Prepare the CPU initialization parameters */
2313 cpu_prepare_hyp_mode(cpu);
2314 }
2315
2316 err = init_pkvm_host_fp_state();
2317 if (err)
2318 goto out_err;
2319
2320 kvm_hyp_init_symbols();
2321
2322 /* TODO: Real .h interface */
2323 #ifdef CONFIG_TRACING
2324 kvm_hyp_init_events();
2325 #endif
2326
2327 if (is_protected_kvm_enabled()) {
2328 init_cpu_logical_map();
2329
2330 if (!init_psci_relay()) {
2331 err = -ENODEV;
2332 goto out_err;
2333 }
2334
2335 err = kvm_hyp_init_protection(hyp_va_bits);
2336 if (err) {
2337 kvm_err("Failed to init hyp memory protection\n");
2338 goto out_err;
2339 }
2340 }
2341
2342 return 0;
2343
2344 out_err:
2345 teardown_hyp_mode();
2346 kvm_err("error initializing Hyp mode: %d\n", err);
2347 return err;
2348 }
2349
kvm_mpidr_to_vcpu(struct kvm * kvm,unsigned long mpidr)2350 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2351 {
2352 struct kvm_vcpu *vcpu;
2353 unsigned long i;
2354
2355 mpidr &= MPIDR_HWID_BITMASK;
2356 kvm_for_each_vcpu(i, vcpu, kvm) {
2357 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2358 return vcpu;
2359 }
2360 return NULL;
2361 }
2362
kvm_arch_has_irq_bypass(void)2363 bool kvm_arch_has_irq_bypass(void)
2364 {
2365 return true;
2366 }
2367
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2368 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2369 struct irq_bypass_producer *prod)
2370 {
2371 struct kvm_kernel_irqfd *irqfd =
2372 container_of(cons, struct kvm_kernel_irqfd, consumer);
2373
2374 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2375 &irqfd->irq_entry);
2376 }
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2377 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2378 struct irq_bypass_producer *prod)
2379 {
2380 struct kvm_kernel_irqfd *irqfd =
2381 container_of(cons, struct kvm_kernel_irqfd, consumer);
2382
2383 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
2384 &irqfd->irq_entry);
2385 }
2386
kvm_arch_irq_bypass_stop(struct irq_bypass_consumer * cons)2387 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2388 {
2389 struct kvm_kernel_irqfd *irqfd =
2390 container_of(cons, struct kvm_kernel_irqfd, consumer);
2391
2392 kvm_arm_halt_guest(irqfd->kvm);
2393 }
2394
kvm_arch_irq_bypass_start(struct irq_bypass_consumer * cons)2395 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2396 {
2397 struct kvm_kernel_irqfd *irqfd =
2398 container_of(cons, struct kvm_kernel_irqfd, consumer);
2399
2400 kvm_arm_resume_guest(irqfd->kvm);
2401 }
2402
2403 /**
2404 * Initialize Hyp-mode and memory mappings on all CPUs.
2405 */
kvm_arch_init(void * opaque)2406 int kvm_arch_init(void *opaque)
2407 {
2408 int err;
2409 bool in_hyp_mode;
2410
2411 if (!is_hyp_mode_available()) {
2412 kvm_info("HYP mode not available\n");
2413 return -ENODEV;
2414 }
2415
2416 if (kvm_get_mode() == KVM_MODE_NONE) {
2417 kvm_info("KVM disabled from command line\n");
2418 return -ENODEV;
2419 }
2420
2421 err = kvm_sys_reg_table_init();
2422 if (err) {
2423 kvm_info("Error initializing system register tables");
2424 return err;
2425 }
2426
2427 in_hyp_mode = is_kernel_in_hyp_mode();
2428
2429 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2430 cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2431 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2432 "Only trusted guests should be used on this system.\n");
2433
2434 err = kvm_set_ipa_limit();
2435 if (err)
2436 return err;
2437
2438 err = kvm_arm_init_sve();
2439 if (err)
2440 return err;
2441
2442 err = kvm_arm_vmid_alloc_init();
2443 if (err) {
2444 kvm_err("Failed to initialize VMID allocator.\n");
2445 return err;
2446 }
2447
2448 if (!in_hyp_mode) {
2449 err = init_hyp_mode();
2450 if (err)
2451 goto out_err;
2452 }
2453
2454 err = kvm_init_vector_slots();
2455 if (err) {
2456 kvm_err("Cannot initialise vector slots\n");
2457 goto out_err;
2458 }
2459
2460 err = init_subsystems();
2461 if (err)
2462 goto out_hyp;
2463
2464 if (!in_hyp_mode) {
2465 err = init_hyp_tracefs();
2466 if (err)
2467 kvm_err("Failed to initialize Hyp tracing\n");
2468 }
2469
2470 if (is_protected_kvm_enabled()) {
2471 kvm_info("Protected nVHE mode initialized successfully\n");
2472 } else if (in_hyp_mode) {
2473 kvm_info("VHE mode initialized successfully\n");
2474 } else {
2475 kvm_info("Hyp mode initialized successfully\n");
2476 }
2477
2478 /*
2479 * This should be called after initialization is done and failure isn't
2480 * possible anymore.
2481 */
2482 if (!in_hyp_mode)
2483 finalize_init_hyp_mode();
2484
2485 return 0;
2486
2487 out_hyp:
2488 hyp_cpu_pm_exit();
2489 if (!in_hyp_mode)
2490 teardown_hyp_mode();
2491 out_err:
2492 kvm_arm_vmid_alloc_free();
2493 return err;
2494 }
2495
2496 /* NOP: Compiling as a module not supported */
kvm_arch_exit(void)2497 void kvm_arch_exit(void)
2498 {
2499 kvm_unregister_perf_callbacks();
2500 }
2501
early_kvm_mode_cfg(char * arg)2502 static int __init early_kvm_mode_cfg(char *arg)
2503 {
2504 if (!arg)
2505 return -EINVAL;
2506
2507 if (strcmp(arg, "none") == 0) {
2508 kvm_mode = KVM_MODE_NONE;
2509 return 0;
2510 }
2511
2512 if (!is_hyp_mode_available()) {
2513 pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2514 return 0;
2515 }
2516
2517 if (strcmp(arg, "protected") == 0) {
2518 if (!is_kernel_in_hyp_mode())
2519 kvm_mode = KVM_MODE_PROTECTED;
2520 else
2521 pr_warn_once("Protected KVM not available with VHE\n");
2522
2523 return 0;
2524 }
2525
2526 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2527 kvm_mode = KVM_MODE_DEFAULT;
2528 return 0;
2529 }
2530
2531 return -EINVAL;
2532 }
2533 early_param("kvm-arm.mode", early_kvm_mode_cfg);
2534
kvm_get_mode(void)2535 enum kvm_mode kvm_get_mode(void)
2536 {
2537 return kvm_mode;
2538 }
2539
arm_init(void)2540 static int arm_init(void)
2541 {
2542 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2543 return rc;
2544 }
2545
2546 module_init(arm_init);
2547