• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5  */
6 
7 #include <linux/bug.h>
8 #include <linux/cpu_pm.h>
9 #include <linux/errno.h>
10 #include <linux/err.h>
11 #include <linux/kvm_host.h>
12 #include <linux/list.h>
13 #include <linux/module.h>
14 #include <linux/vmalloc.h>
15 #include <linux/fs.h>
16 #include <linux/mman.h>
17 #include <linux/sched.h>
18 #include <linux/kmemleak.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_irqfd.h>
21 #include <linux/irqbypass.h>
22 #include <linux/sched/stat.h>
23 #include <linux/psci.h>
24 #include <trace/events/kvm.h>
25 
26 #define CREATE_TRACE_POINTS
27 #include "trace_arm.h"
28 
29 #include <linux/uaccess.h>
30 #include <asm/ptrace.h>
31 #include <asm/mman.h>
32 #include <asm/tlbflush.h>
33 #include <asm/cacheflush.h>
34 #include <asm/cpufeature.h>
35 #include <asm/virt.h>
36 #include <asm/kvm_arm.h>
37 #include <asm/kvm_asm.h>
38 #include <asm/kvm_mmu.h>
39 #include <asm/kvm_emulate.h>
40 #include <asm/sections.h>
41 
42 #include <kvm/arm_hypercalls.h>
43 #include <kvm/arm_pmu.h>
44 #include <kvm/arm_psci.h>
45 
46 #ifdef REQUIRES_VIRT
47 __asm__(".arch_extension	virt");
48 #endif
49 
50 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
51 DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
52 
53 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
54 
55 static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
56 unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
57 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
58 
59 /* The VMID used in the VTTBR */
60 static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
61 static u32 kvm_next_vmid;
62 static DEFINE_SPINLOCK(kvm_vmid_lock);
63 
64 static bool vgic_present;
65 
66 static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
67 DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
68 
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)69 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
70 {
71 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
72 }
73 
kvm_arch_hardware_setup(void * opaque)74 int kvm_arch_hardware_setup(void *opaque)
75 {
76 	return 0;
77 }
78 
kvm_arch_check_processor_compat(void * opaque)79 int kvm_arch_check_processor_compat(void *opaque)
80 {
81 	return 0;
82 }
83 
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)84 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
85 			    struct kvm_enable_cap *cap)
86 {
87 	int r;
88 
89 	if (cap->flags)
90 		return -EINVAL;
91 
92 	switch (cap->cap) {
93 	case KVM_CAP_ARM_NISV_TO_USER:
94 		r = 0;
95 		kvm->arch.return_nisv_io_abort_to_user = true;
96 		break;
97 	default:
98 		r = -EINVAL;
99 		break;
100 	}
101 
102 	return r;
103 }
104 
kvm_arm_default_max_vcpus(void)105 static int kvm_arm_default_max_vcpus(void)
106 {
107 	return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
108 }
109 
set_default_spectre(struct kvm * kvm)110 static void set_default_spectre(struct kvm *kvm)
111 {
112 	/*
113 	 * The default is to expose CSV2 == 1 if the HW isn't affected.
114 	 * Although this is a per-CPU feature, we make it global because
115 	 * asymmetric systems are just a nuisance.
116 	 *
117 	 * Userspace can override this as long as it doesn't promise
118 	 * the impossible.
119 	 */
120 	if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED)
121 		kvm->arch.pfr0_csv2 = 1;
122 	if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED)
123 		kvm->arch.pfr0_csv3 = 1;
124 }
125 
126 /**
127  * kvm_arch_init_vm - initializes a VM data structure
128  * @kvm:	pointer to the KVM struct
129  */
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)130 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
131 {
132 	int ret;
133 
134 	ret = kvm_arm_setup_stage2(kvm, type);
135 	if (ret)
136 		return ret;
137 
138 	ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu);
139 	if (ret)
140 		return ret;
141 
142 	ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
143 	if (ret)
144 		goto out_free_stage2_pgd;
145 
146 	kvm_vgic_early_init(kvm);
147 
148 	/* The maximum number of VCPUs is limited by the host's GIC model */
149 	kvm->arch.max_vcpus = kvm_arm_default_max_vcpus();
150 
151 	set_default_spectre(kvm);
152 
153 	return ret;
154 out_free_stage2_pgd:
155 	kvm_free_stage2_pgd(&kvm->arch.mmu);
156 	return ret;
157 }
158 
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)159 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
160 {
161 	return VM_FAULT_SIGBUS;
162 }
163 
164 
165 /**
166  * kvm_arch_destroy_vm - destroy the VM data structure
167  * @kvm:	pointer to the KVM struct
168  */
kvm_arch_destroy_vm(struct kvm * kvm)169 void kvm_arch_destroy_vm(struct kvm *kvm)
170 {
171 	int i;
172 
173 	bitmap_free(kvm->arch.pmu_filter);
174 
175 	kvm_vgic_destroy(kvm);
176 
177 	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
178 		if (kvm->vcpus[i]) {
179 			kvm_vcpu_destroy(kvm->vcpus[i]);
180 			kvm->vcpus[i] = NULL;
181 		}
182 	}
183 	atomic_set(&kvm->online_vcpus, 0);
184 }
185 
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)186 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
187 {
188 	int r;
189 	switch (ext) {
190 	case KVM_CAP_IRQCHIP:
191 		r = vgic_present;
192 		break;
193 	case KVM_CAP_IOEVENTFD:
194 	case KVM_CAP_DEVICE_CTRL:
195 	case KVM_CAP_USER_MEMORY:
196 	case KVM_CAP_SYNC_MMU:
197 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
198 	case KVM_CAP_ONE_REG:
199 	case KVM_CAP_ARM_PSCI:
200 	case KVM_CAP_ARM_PSCI_0_2:
201 	case KVM_CAP_READONLY_MEM:
202 	case KVM_CAP_MP_STATE:
203 	case KVM_CAP_IMMEDIATE_EXIT:
204 	case KVM_CAP_VCPU_EVENTS:
205 	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
206 	case KVM_CAP_ARM_NISV_TO_USER:
207 	case KVM_CAP_ARM_INJECT_EXT_DABT:
208 	case KVM_CAP_SET_GUEST_DEBUG:
209 	case KVM_CAP_VCPU_ATTRIBUTES:
210 		r = 1;
211 		break;
212 	case KVM_CAP_ARM_SET_DEVICE_ADDR:
213 		r = 1;
214 		break;
215 	case KVM_CAP_NR_VCPUS:
216 		r = num_online_cpus();
217 		break;
218 	case KVM_CAP_MAX_VCPUS:
219 	case KVM_CAP_MAX_VCPU_ID:
220 		if (kvm)
221 			r = kvm->arch.max_vcpus;
222 		else
223 			r = kvm_arm_default_max_vcpus();
224 		break;
225 	case KVM_CAP_MSI_DEVID:
226 		if (!kvm)
227 			r = -EINVAL;
228 		else
229 			r = kvm->arch.vgic.msis_require_devid;
230 		break;
231 	case KVM_CAP_ARM_USER_IRQ:
232 		/*
233 		 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
234 		 * (bump this number if adding more devices)
235 		 */
236 		r = 1;
237 		break;
238 	case KVM_CAP_STEAL_TIME:
239 		r = kvm_arm_pvtime_supported();
240 		break;
241 	case KVM_CAP_ARM_EL1_32BIT:
242 		r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1);
243 		break;
244 	case KVM_CAP_GUEST_DEBUG_HW_BPS:
245 		r = get_num_brps();
246 		break;
247 	case KVM_CAP_GUEST_DEBUG_HW_WPS:
248 		r = get_num_wrps();
249 		break;
250 	case KVM_CAP_ARM_PMU_V3:
251 		r = kvm_arm_support_pmu_v3();
252 		break;
253 	case KVM_CAP_ARM_INJECT_SERROR_ESR:
254 		r = cpus_have_const_cap(ARM64_HAS_RAS_EXTN);
255 		break;
256 	case KVM_CAP_ARM_VM_IPA_SIZE:
257 		r = get_kvm_ipa_limit();
258 		break;
259 	case KVM_CAP_ARM_SVE:
260 		r = system_supports_sve();
261 		break;
262 	case KVM_CAP_ARM_PTRAUTH_ADDRESS:
263 	case KVM_CAP_ARM_PTRAUTH_GENERIC:
264 		r = system_has_full_ptr_auth();
265 		break;
266 	default:
267 		r = 0;
268 	}
269 
270 	return r;
271 }
272 
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)273 long kvm_arch_dev_ioctl(struct file *filp,
274 			unsigned int ioctl, unsigned long arg)
275 {
276 	return -EINVAL;
277 }
278 
kvm_arch_alloc_vm(void)279 struct kvm *kvm_arch_alloc_vm(void)
280 {
281 	if (!has_vhe())
282 		return kzalloc(sizeof(struct kvm), GFP_KERNEL);
283 
284 	return vzalloc(sizeof(struct kvm));
285 }
286 
kvm_arch_free_vm(struct kvm * kvm)287 void kvm_arch_free_vm(struct kvm *kvm)
288 {
289 	if (!has_vhe())
290 		kfree(kvm);
291 	else
292 		vfree(kvm);
293 }
294 
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)295 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
296 {
297 	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
298 		return -EBUSY;
299 
300 	if (id >= kvm->arch.max_vcpus)
301 		return -EINVAL;
302 
303 	return 0;
304 }
305 
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)306 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
307 {
308 	int err;
309 
310 	/* Force users to call KVM_ARM_VCPU_INIT */
311 	vcpu->arch.target = -1;
312 	bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
313 
314 	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
315 
316 	/* Set up the timer */
317 	kvm_timer_vcpu_init(vcpu);
318 
319 	kvm_pmu_vcpu_init(vcpu);
320 
321 	kvm_arm_reset_debug_ptr(vcpu);
322 
323 	kvm_arm_pvtime_vcpu_init(&vcpu->arch);
324 
325 	vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
326 
327 	err = kvm_vgic_vcpu_init(vcpu);
328 	if (err)
329 		return err;
330 
331 	return create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
332 }
333 
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)334 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
335 {
336 }
337 
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)338 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
339 {
340 	if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm)))
341 		static_branch_dec(&userspace_irqchip_in_use);
342 
343 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
344 	kvm_timer_vcpu_terminate(vcpu);
345 	kvm_pmu_vcpu_destroy(vcpu);
346 
347 	kvm_arm_vcpu_destroy(vcpu);
348 }
349 
kvm_cpu_has_pending_timer(struct kvm_vcpu * vcpu)350 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
351 {
352 	return kvm_timer_is_pending(vcpu);
353 }
354 
kvm_arch_vcpu_blocking(struct kvm_vcpu * vcpu)355 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
356 {
357 	/*
358 	 * If we're about to block (most likely because we've just hit a
359 	 * WFI), we need to sync back the state of the GIC CPU interface
360 	 * so that we have the latest PMR and group enables. This ensures
361 	 * that kvm_arch_vcpu_runnable has up-to-date data to decide
362 	 * whether we have pending interrupts.
363 	 *
364 	 * For the same reason, we want to tell GICv4 that we need
365 	 * doorbells to be signalled, should an interrupt become pending.
366 	 */
367 	preempt_disable();
368 	kvm_vgic_vmcr_sync(vcpu);
369 	vgic_v4_put(vcpu, true);
370 	preempt_enable();
371 }
372 
kvm_arch_vcpu_unblocking(struct kvm_vcpu * vcpu)373 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
374 {
375 	preempt_disable();
376 	vgic_v4_load(vcpu);
377 	preempt_enable();
378 }
379 
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)380 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
381 {
382 	struct kvm_s2_mmu *mmu;
383 	int *last_ran;
384 
385 	mmu = vcpu->arch.hw_mmu;
386 	last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
387 
388 	/*
389 	 * We guarantee that both TLBs and I-cache are private to each
390 	 * vcpu. If detecting that a vcpu from the same VM has
391 	 * previously run on the same physical CPU, call into the
392 	 * hypervisor code to nuke the relevant contexts.
393 	 *
394 	 * We might get preempted before the vCPU actually runs, but
395 	 * over-invalidation doesn't affect correctness.
396 	 */
397 	if (*last_ran != vcpu->vcpu_id) {
398 		kvm_call_hyp(__kvm_flush_cpu_context, mmu);
399 		*last_ran = vcpu->vcpu_id;
400 	}
401 
402 	vcpu->cpu = cpu;
403 
404 	kvm_vgic_load(vcpu);
405 	kvm_timer_vcpu_load(vcpu);
406 	if (has_vhe())
407 		kvm_vcpu_load_sysregs_vhe(vcpu);
408 	kvm_arch_vcpu_load_fp(vcpu);
409 	kvm_vcpu_pmu_restore_guest(vcpu);
410 	if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
411 		kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
412 
413 	if (single_task_running())
414 		vcpu_clear_wfx_traps(vcpu);
415 	else
416 		vcpu_set_wfx_traps(vcpu);
417 
418 	if (vcpu_has_ptrauth(vcpu))
419 		vcpu_ptrauth_disable(vcpu);
420 	kvm_arch_vcpu_load_debug_state_flags(vcpu);
421 }
422 
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)423 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
424 {
425 	kvm_arch_vcpu_put_debug_state_flags(vcpu);
426 	kvm_arch_vcpu_put_fp(vcpu);
427 	if (has_vhe())
428 		kvm_vcpu_put_sysregs_vhe(vcpu);
429 	kvm_timer_vcpu_put(vcpu);
430 	kvm_vgic_put(vcpu);
431 	kvm_vcpu_pmu_restore_host(vcpu);
432 
433 	vcpu->cpu = -1;
434 }
435 
vcpu_power_off(struct kvm_vcpu * vcpu)436 static void vcpu_power_off(struct kvm_vcpu *vcpu)
437 {
438 	vcpu->arch.power_off = true;
439 	kvm_make_request(KVM_REQ_SLEEP, vcpu);
440 	kvm_vcpu_kick(vcpu);
441 }
442 
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)443 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
444 				    struct kvm_mp_state *mp_state)
445 {
446 	if (vcpu->arch.power_off)
447 		mp_state->mp_state = KVM_MP_STATE_STOPPED;
448 	else
449 		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
450 
451 	return 0;
452 }
453 
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)454 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
455 				    struct kvm_mp_state *mp_state)
456 {
457 	int ret = 0;
458 
459 	switch (mp_state->mp_state) {
460 	case KVM_MP_STATE_RUNNABLE:
461 		vcpu->arch.power_off = false;
462 		break;
463 	case KVM_MP_STATE_STOPPED:
464 		vcpu_power_off(vcpu);
465 		break;
466 	default:
467 		ret = -EINVAL;
468 	}
469 
470 	return ret;
471 }
472 
473 /**
474  * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
475  * @v:		The VCPU pointer
476  *
477  * If the guest CPU is not waiting for interrupts or an interrupt line is
478  * asserted, the CPU is by definition runnable.
479  */
kvm_arch_vcpu_runnable(struct kvm_vcpu * v)480 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
481 {
482 	bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
483 	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
484 		&& !v->arch.power_off && !v->arch.pause);
485 }
486 
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)487 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
488 {
489 	return vcpu_mode_priv(vcpu);
490 }
491 
492 /* Just ensure a guest exit from a particular CPU */
exit_vm_noop(void * info)493 static void exit_vm_noop(void *info)
494 {
495 }
496 
force_vm_exit(const cpumask_t * mask)497 void force_vm_exit(const cpumask_t *mask)
498 {
499 	preempt_disable();
500 	smp_call_function_many(mask, exit_vm_noop, NULL, true);
501 	preempt_enable();
502 }
503 
504 /**
505  * need_new_vmid_gen - check that the VMID is still valid
506  * @vmid: The VMID to check
507  *
508  * return true if there is a new generation of VMIDs being used
509  *
510  * The hardware supports a limited set of values with the value zero reserved
511  * for the host, so we check if an assigned value belongs to a previous
512  * generation, which requires us to assign a new value. If we're the first to
513  * use a VMID for the new generation, we must flush necessary caches and TLBs
514  * on all CPUs.
515  */
need_new_vmid_gen(struct kvm_vmid * vmid)516 static bool need_new_vmid_gen(struct kvm_vmid *vmid)
517 {
518 	u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen);
519 	smp_rmb(); /* Orders read of kvm_vmid_gen and kvm->arch.vmid */
520 	return unlikely(READ_ONCE(vmid->vmid_gen) != current_vmid_gen);
521 }
522 
523 /**
524  * update_vmid - Update the vmid with a valid VMID for the current generation
525  * @vmid: The stage-2 VMID information struct
526  */
update_vmid(struct kvm_vmid * vmid)527 static void update_vmid(struct kvm_vmid *vmid)
528 {
529 	if (!need_new_vmid_gen(vmid))
530 		return;
531 
532 	spin_lock(&kvm_vmid_lock);
533 
534 	/*
535 	 * We need to re-check the vmid_gen here to ensure that if another vcpu
536 	 * already allocated a valid vmid for this vm, then this vcpu should
537 	 * use the same vmid.
538 	 */
539 	if (!need_new_vmid_gen(vmid)) {
540 		spin_unlock(&kvm_vmid_lock);
541 		return;
542 	}
543 
544 	/* First user of a new VMID generation? */
545 	if (unlikely(kvm_next_vmid == 0)) {
546 		atomic64_inc(&kvm_vmid_gen);
547 		kvm_next_vmid = 1;
548 
549 		/*
550 		 * On SMP we know no other CPUs can use this CPU's or each
551 		 * other's VMID after force_vm_exit returns since the
552 		 * kvm_vmid_lock blocks them from reentry to the guest.
553 		 */
554 		force_vm_exit(cpu_all_mask);
555 		/*
556 		 * Now broadcast TLB + ICACHE invalidation over the inner
557 		 * shareable domain to make sure all data structures are
558 		 * clean.
559 		 */
560 		kvm_call_hyp(__kvm_flush_vm_context);
561 	}
562 
563 	vmid->vmid = kvm_next_vmid;
564 	kvm_next_vmid++;
565 	kvm_next_vmid &= (1 << kvm_get_vmid_bits()) - 1;
566 
567 	smp_wmb();
568 	WRITE_ONCE(vmid->vmid_gen, atomic64_read(&kvm_vmid_gen));
569 
570 	spin_unlock(&kvm_vmid_lock);
571 }
572 
kvm_vcpu_first_run_init(struct kvm_vcpu * vcpu)573 static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
574 {
575 	struct kvm *kvm = vcpu->kvm;
576 	int ret = 0;
577 
578 	if (likely(vcpu->arch.has_run_once))
579 		return 0;
580 
581 	if (!kvm_arm_vcpu_is_finalized(vcpu))
582 		return -EPERM;
583 
584 	vcpu->arch.has_run_once = true;
585 
586 	kvm_arm_vcpu_init_debug(vcpu);
587 
588 	if (likely(irqchip_in_kernel(kvm))) {
589 		/*
590 		 * Map the VGIC hardware resources before running a vcpu the
591 		 * first time on this VM.
592 		 */
593 		ret = kvm_vgic_map_resources(kvm);
594 		if (ret)
595 			return ret;
596 	} else {
597 		/*
598 		 * Tell the rest of the code that there are userspace irqchip
599 		 * VMs in the wild.
600 		 */
601 		static_branch_inc(&userspace_irqchip_in_use);
602 	}
603 
604 	ret = kvm_timer_enable(vcpu);
605 	if (ret)
606 		return ret;
607 
608 	ret = kvm_arm_pmu_v3_enable(vcpu);
609 
610 	return ret;
611 }
612 
kvm_arch_intc_initialized(struct kvm * kvm)613 bool kvm_arch_intc_initialized(struct kvm *kvm)
614 {
615 	return vgic_initialized(kvm);
616 }
617 
kvm_arm_halt_guest(struct kvm * kvm)618 void kvm_arm_halt_guest(struct kvm *kvm)
619 {
620 	int i;
621 	struct kvm_vcpu *vcpu;
622 
623 	kvm_for_each_vcpu(i, vcpu, kvm)
624 		vcpu->arch.pause = true;
625 	kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
626 }
627 
kvm_arm_resume_guest(struct kvm * kvm)628 void kvm_arm_resume_guest(struct kvm *kvm)
629 {
630 	int i;
631 	struct kvm_vcpu *vcpu;
632 
633 	kvm_for_each_vcpu(i, vcpu, kvm) {
634 		vcpu->arch.pause = false;
635 		rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu));
636 	}
637 }
638 
vcpu_req_sleep(struct kvm_vcpu * vcpu)639 static void vcpu_req_sleep(struct kvm_vcpu *vcpu)
640 {
641 	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
642 
643 	rcuwait_wait_event(wait,
644 			   (!vcpu->arch.power_off) &&(!vcpu->arch.pause),
645 			   TASK_INTERRUPTIBLE);
646 
647 	if (vcpu->arch.power_off || vcpu->arch.pause) {
648 		/* Awaken to handle a signal, request we sleep again later. */
649 		kvm_make_request(KVM_REQ_SLEEP, vcpu);
650 	}
651 
652 	/*
653 	 * Make sure we will observe a potential reset request if we've
654 	 * observed a change to the power state. Pairs with the smp_wmb() in
655 	 * kvm_psci_vcpu_on().
656 	 */
657 	smp_rmb();
658 }
659 
kvm_vcpu_initialized(struct kvm_vcpu * vcpu)660 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
661 {
662 	return vcpu->arch.target >= 0;
663 }
664 
check_vcpu_requests(struct kvm_vcpu * vcpu)665 static void check_vcpu_requests(struct kvm_vcpu *vcpu)
666 {
667 	if (kvm_request_pending(vcpu)) {
668 		if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
669 			vcpu_req_sleep(vcpu);
670 
671 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
672 			kvm_reset_vcpu(vcpu);
673 
674 		/*
675 		 * Clear IRQ_PENDING requests that were made to guarantee
676 		 * that a VCPU sees new virtual interrupts.
677 		 */
678 		kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
679 
680 		if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
681 			kvm_update_stolen_time(vcpu);
682 
683 		if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
684 			/* The distributor enable bits were changed */
685 			preempt_disable();
686 			vgic_v4_put(vcpu, false);
687 			vgic_v4_load(vcpu);
688 			preempt_enable();
689 		}
690 	}
691 }
692 
vcpu_mode_is_bad_32bit(struct kvm_vcpu * vcpu)693 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
694 {
695 	if (likely(!vcpu_mode_is_32bit(vcpu)))
696 		return false;
697 
698 	return !kvm_supports_32bit_el0();
699 }
700 
701 /**
702  * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
703  * @vcpu:	The VCPU pointer
704  *
705  * This function is called through the VCPU_RUN ioctl called from user space. It
706  * will execute VM code in a loop until the time slice for the process is used
707  * or some emulation is needed from user space in which case the function will
708  * return with return value 0 and with the kvm_run structure filled in with the
709  * required data for the requested emulation.
710  */
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)711 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
712 {
713 	struct kvm_run *run = vcpu->run;
714 	int ret;
715 
716 	if (unlikely(!kvm_vcpu_initialized(vcpu)))
717 		return -ENOEXEC;
718 
719 	ret = kvm_vcpu_first_run_init(vcpu);
720 	if (ret)
721 		return ret;
722 
723 	if (run->exit_reason == KVM_EXIT_MMIO) {
724 		ret = kvm_handle_mmio_return(vcpu);
725 		if (ret)
726 			return ret;
727 	}
728 
729 	if (run->immediate_exit)
730 		return -EINTR;
731 
732 	vcpu_load(vcpu);
733 
734 	kvm_sigset_activate(vcpu);
735 
736 	ret = 1;
737 	run->exit_reason = KVM_EXIT_UNKNOWN;
738 	while (ret > 0) {
739 		/*
740 		 * Check conditions before entering the guest
741 		 */
742 		cond_resched();
743 
744 		update_vmid(&vcpu->arch.hw_mmu->vmid);
745 
746 		check_vcpu_requests(vcpu);
747 
748 		/*
749 		 * Preparing the interrupts to be injected also
750 		 * involves poking the GIC, which must be done in a
751 		 * non-preemptible context.
752 		 */
753 		preempt_disable();
754 
755 		kvm_pmu_flush_hwstate(vcpu);
756 
757 		local_irq_disable();
758 
759 		kvm_vgic_flush_hwstate(vcpu);
760 
761 		/*
762 		 * Exit if we have a signal pending so that we can deliver the
763 		 * signal to user space.
764 		 */
765 		if (signal_pending(current)) {
766 			ret = -EINTR;
767 			run->exit_reason = KVM_EXIT_INTR;
768 		}
769 
770 		/*
771 		 * If we're using a userspace irqchip, then check if we need
772 		 * to tell a userspace irqchip about timer or PMU level
773 		 * changes and if so, exit to userspace (the actual level
774 		 * state gets updated in kvm_timer_update_run and
775 		 * kvm_pmu_update_run below).
776 		 */
777 		if (static_branch_unlikely(&userspace_irqchip_in_use)) {
778 			if (kvm_timer_should_notify_user(vcpu) ||
779 			    kvm_pmu_should_notify_user(vcpu)) {
780 				ret = -EINTR;
781 				run->exit_reason = KVM_EXIT_INTR;
782 			}
783 		}
784 
785 		/*
786 		 * Ensure we set mode to IN_GUEST_MODE after we disable
787 		 * interrupts and before the final VCPU requests check.
788 		 * See the comment in kvm_vcpu_exiting_guest_mode() and
789 		 * Documentation/virt/kvm/vcpu-requests.rst
790 		 */
791 		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
792 
793 		if (ret <= 0 || need_new_vmid_gen(&vcpu->arch.hw_mmu->vmid) ||
794 		    kvm_request_pending(vcpu)) {
795 			vcpu->mode = OUTSIDE_GUEST_MODE;
796 			isb(); /* Ensure work in x_flush_hwstate is committed */
797 			kvm_pmu_sync_hwstate(vcpu);
798 			if (static_branch_unlikely(&userspace_irqchip_in_use))
799 				kvm_timer_sync_user(vcpu);
800 			kvm_vgic_sync_hwstate(vcpu);
801 			local_irq_enable();
802 			preempt_enable();
803 			continue;
804 		}
805 
806 		kvm_arm_setup_debug(vcpu);
807 
808 		/**************************************************************
809 		 * Enter the guest
810 		 */
811 		trace_kvm_entry(*vcpu_pc(vcpu));
812 		guest_enter_irqoff();
813 
814 		ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
815 
816 		vcpu->mode = OUTSIDE_GUEST_MODE;
817 		vcpu->stat.exits++;
818 		/*
819 		 * Back from guest
820 		 *************************************************************/
821 
822 		kvm_arm_clear_debug(vcpu);
823 
824 		/*
825 		 * We must sync the PMU state before the vgic state so
826 		 * that the vgic can properly sample the updated state of the
827 		 * interrupt line.
828 		 */
829 		kvm_pmu_sync_hwstate(vcpu);
830 
831 		/*
832 		 * Sync the vgic state before syncing the timer state because
833 		 * the timer code needs to know if the virtual timer
834 		 * interrupts are active.
835 		 */
836 		kvm_vgic_sync_hwstate(vcpu);
837 
838 		/*
839 		 * Sync the timer hardware state before enabling interrupts as
840 		 * we don't want vtimer interrupts to race with syncing the
841 		 * timer virtual interrupt state.
842 		 */
843 		if (static_branch_unlikely(&userspace_irqchip_in_use))
844 			kvm_timer_sync_user(vcpu);
845 
846 		kvm_arch_vcpu_ctxsync_fp(vcpu);
847 
848 		/*
849 		 * We may have taken a host interrupt in HYP mode (ie
850 		 * while executing the guest). This interrupt is still
851 		 * pending, as we haven't serviced it yet!
852 		 *
853 		 * We're now back in SVC mode, with interrupts
854 		 * disabled.  Enabling the interrupts now will have
855 		 * the effect of taking the interrupt again, in SVC
856 		 * mode this time.
857 		 */
858 		local_irq_enable();
859 
860 		/*
861 		 * We do local_irq_enable() before calling guest_exit() so
862 		 * that if a timer interrupt hits while running the guest we
863 		 * account that tick as being spent in the guest.  We enable
864 		 * preemption after calling guest_exit() so that if we get
865 		 * preempted we make sure ticks after that is not counted as
866 		 * guest time.
867 		 */
868 		guest_exit();
869 		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
870 
871 		/* Exit types that need handling before we can be preempted */
872 		handle_exit_early(vcpu, ret);
873 
874 		preempt_enable();
875 
876 		/*
877 		 * The ARMv8 architecture doesn't give the hypervisor
878 		 * a mechanism to prevent a guest from dropping to AArch32 EL0
879 		 * if implemented by the CPU. If we spot the guest in such
880 		 * state and that we decided it wasn't supposed to do so (like
881 		 * with the asymmetric AArch32 case), return to userspace with
882 		 * a fatal error.
883 		 */
884 		if (vcpu_mode_is_bad_32bit(vcpu)) {
885 			/*
886 			 * As we have caught the guest red-handed, decide that
887 			 * it isn't fit for purpose anymore by making the vcpu
888 			 * invalid. The VMM can try and fix it by issuing  a
889 			 * KVM_ARM_VCPU_INIT if it really wants to.
890 			 */
891 			vcpu->arch.target = -1;
892 			ret = ARM_EXCEPTION_IL;
893 		}
894 
895 		ret = handle_exit(vcpu, ret);
896 	}
897 
898 	/* Tell userspace about in-kernel device output levels */
899 	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
900 		kvm_timer_update_run(vcpu);
901 		kvm_pmu_update_run(vcpu);
902 	}
903 
904 	kvm_sigset_deactivate(vcpu);
905 
906 	vcpu_put(vcpu);
907 	return ret;
908 }
909 
vcpu_interrupt_line(struct kvm_vcpu * vcpu,int number,bool level)910 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
911 {
912 	int bit_index;
913 	bool set;
914 	unsigned long *hcr;
915 
916 	if (number == KVM_ARM_IRQ_CPU_IRQ)
917 		bit_index = __ffs(HCR_VI);
918 	else /* KVM_ARM_IRQ_CPU_FIQ */
919 		bit_index = __ffs(HCR_VF);
920 
921 	hcr = vcpu_hcr(vcpu);
922 	if (level)
923 		set = test_and_set_bit(bit_index, hcr);
924 	else
925 		set = test_and_clear_bit(bit_index, hcr);
926 
927 	/*
928 	 * If we didn't change anything, no need to wake up or kick other CPUs
929 	 */
930 	if (set == level)
931 		return 0;
932 
933 	/*
934 	 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
935 	 * trigger a world-switch round on the running physical CPU to set the
936 	 * virtual IRQ/FIQ fields in the HCR appropriately.
937 	 */
938 	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
939 	kvm_vcpu_kick(vcpu);
940 
941 	return 0;
942 }
943 
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_level,bool line_status)944 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
945 			  bool line_status)
946 {
947 	u32 irq = irq_level->irq;
948 	unsigned int irq_type, vcpu_idx, irq_num;
949 	int nrcpus = atomic_read(&kvm->online_vcpus);
950 	struct kvm_vcpu *vcpu = NULL;
951 	bool level = irq_level->level;
952 
953 	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
954 	vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
955 	vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
956 	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
957 
958 	trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
959 
960 	switch (irq_type) {
961 	case KVM_ARM_IRQ_TYPE_CPU:
962 		if (irqchip_in_kernel(kvm))
963 			return -ENXIO;
964 
965 		if (vcpu_idx >= nrcpus)
966 			return -EINVAL;
967 
968 		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
969 		if (!vcpu)
970 			return -EINVAL;
971 
972 		if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
973 			return -EINVAL;
974 
975 		return vcpu_interrupt_line(vcpu, irq_num, level);
976 	case KVM_ARM_IRQ_TYPE_PPI:
977 		if (!irqchip_in_kernel(kvm))
978 			return -ENXIO;
979 
980 		if (vcpu_idx >= nrcpus)
981 			return -EINVAL;
982 
983 		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
984 		if (!vcpu)
985 			return -EINVAL;
986 
987 		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
988 			return -EINVAL;
989 
990 		return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);
991 	case KVM_ARM_IRQ_TYPE_SPI:
992 		if (!irqchip_in_kernel(kvm))
993 			return -ENXIO;
994 
995 		if (irq_num < VGIC_NR_PRIVATE_IRQS)
996 			return -EINVAL;
997 
998 		return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);
999 	}
1000 
1001 	return -EINVAL;
1002 }
1003 
kvm_vcpu_set_target(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1004 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1005 			       const struct kvm_vcpu_init *init)
1006 {
1007 	unsigned int i, ret;
1008 	int phys_target = kvm_target_cpu();
1009 
1010 	if (init->target != phys_target)
1011 		return -EINVAL;
1012 
1013 	/*
1014 	 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1015 	 * use the same target.
1016 	 */
1017 	if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
1018 		return -EINVAL;
1019 
1020 	/* -ENOENT for unknown features, -EINVAL for invalid combinations. */
1021 	for (i = 0; i < sizeof(init->features) * 8; i++) {
1022 		bool set = (init->features[i / 32] & (1 << (i % 32)));
1023 
1024 		if (set && i >= KVM_VCPU_MAX_FEATURES)
1025 			return -ENOENT;
1026 
1027 		/*
1028 		 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1029 		 * use the same feature set.
1030 		 */
1031 		if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
1032 		    test_bit(i, vcpu->arch.features) != set)
1033 			return -EINVAL;
1034 
1035 		if (set)
1036 			set_bit(i, vcpu->arch.features);
1037 	}
1038 
1039 	vcpu->arch.target = phys_target;
1040 
1041 	/* Now we know what it is, we can reset it. */
1042 	ret = kvm_reset_vcpu(vcpu);
1043 	if (ret) {
1044 		vcpu->arch.target = -1;
1045 		bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
1046 	}
1047 
1048 	return ret;
1049 }
1050 
kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu * vcpu,struct kvm_vcpu_init * init)1051 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1052 					 struct kvm_vcpu_init *init)
1053 {
1054 	int ret;
1055 
1056 	ret = kvm_vcpu_set_target(vcpu, init);
1057 	if (ret)
1058 		return ret;
1059 
1060 	/*
1061 	 * Ensure a rebooted VM will fault in RAM pages and detect if the
1062 	 * guest MMU is turned off and flush the caches as needed.
1063 	 *
1064 	 * S2FWB enforces all memory accesses to RAM being cacheable,
1065 	 * ensuring that the data side is always coherent. We still
1066 	 * need to invalidate the I-cache though, as FWB does *not*
1067 	 * imply CTR_EL0.DIC.
1068 	 */
1069 	if (vcpu->arch.has_run_once) {
1070 		if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1071 			stage2_unmap_vm(vcpu->kvm);
1072 		else
1073 			__flush_icache_all();
1074 	}
1075 
1076 	vcpu_reset_hcr(vcpu);
1077 
1078 	/*
1079 	 * Handle the "start in power-off" case.
1080 	 */
1081 	if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
1082 		vcpu_power_off(vcpu);
1083 	else
1084 		vcpu->arch.power_off = false;
1085 
1086 	return 0;
1087 }
1088 
kvm_arm_vcpu_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1089 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1090 				 struct kvm_device_attr *attr)
1091 {
1092 	int ret = -ENXIO;
1093 
1094 	switch (attr->group) {
1095 	default:
1096 		ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1097 		break;
1098 	}
1099 
1100 	return ret;
1101 }
1102 
kvm_arm_vcpu_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1103 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1104 				 struct kvm_device_attr *attr)
1105 {
1106 	int ret = -ENXIO;
1107 
1108 	switch (attr->group) {
1109 	default:
1110 		ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1111 		break;
1112 	}
1113 
1114 	return ret;
1115 }
1116 
kvm_arm_vcpu_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1117 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1118 				 struct kvm_device_attr *attr)
1119 {
1120 	int ret = -ENXIO;
1121 
1122 	switch (attr->group) {
1123 	default:
1124 		ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1125 		break;
1126 	}
1127 
1128 	return ret;
1129 }
1130 
kvm_arm_vcpu_get_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1131 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1132 				   struct kvm_vcpu_events *events)
1133 {
1134 	memset(events, 0, sizeof(*events));
1135 
1136 	return __kvm_arm_vcpu_get_events(vcpu, events);
1137 }
1138 
kvm_arm_vcpu_set_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1139 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1140 				   struct kvm_vcpu_events *events)
1141 {
1142 	int i;
1143 
1144 	/* check whether the reserved field is zero */
1145 	for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1146 		if (events->reserved[i])
1147 			return -EINVAL;
1148 
1149 	/* check whether the pad field is zero */
1150 	for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1151 		if (events->exception.pad[i])
1152 			return -EINVAL;
1153 
1154 	return __kvm_arm_vcpu_set_events(vcpu, events);
1155 }
1156 
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1157 long kvm_arch_vcpu_ioctl(struct file *filp,
1158 			 unsigned int ioctl, unsigned long arg)
1159 {
1160 	struct kvm_vcpu *vcpu = filp->private_data;
1161 	void __user *argp = (void __user *)arg;
1162 	struct kvm_device_attr attr;
1163 	long r;
1164 
1165 	switch (ioctl) {
1166 	case KVM_ARM_VCPU_INIT: {
1167 		struct kvm_vcpu_init init;
1168 
1169 		r = -EFAULT;
1170 		if (copy_from_user(&init, argp, sizeof(init)))
1171 			break;
1172 
1173 		r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1174 		break;
1175 	}
1176 	case KVM_SET_ONE_REG:
1177 	case KVM_GET_ONE_REG: {
1178 		struct kvm_one_reg reg;
1179 
1180 		r = -ENOEXEC;
1181 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1182 			break;
1183 
1184 		r = -EFAULT;
1185 		if (copy_from_user(&reg, argp, sizeof(reg)))
1186 			break;
1187 
1188 		/*
1189 		 * We could owe a reset due to PSCI. Handle the pending reset
1190 		 * here to ensure userspace register accesses are ordered after
1191 		 * the reset.
1192 		 */
1193 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1194 			kvm_reset_vcpu(vcpu);
1195 
1196 		if (ioctl == KVM_SET_ONE_REG)
1197 			r = kvm_arm_set_reg(vcpu, &reg);
1198 		else
1199 			r = kvm_arm_get_reg(vcpu, &reg);
1200 		break;
1201 	}
1202 	case KVM_GET_REG_LIST: {
1203 		struct kvm_reg_list __user *user_list = argp;
1204 		struct kvm_reg_list reg_list;
1205 		unsigned n;
1206 
1207 		r = -ENOEXEC;
1208 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1209 			break;
1210 
1211 		r = -EPERM;
1212 		if (!kvm_arm_vcpu_is_finalized(vcpu))
1213 			break;
1214 
1215 		r = -EFAULT;
1216 		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
1217 			break;
1218 		n = reg_list.n;
1219 		reg_list.n = kvm_arm_num_regs(vcpu);
1220 		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
1221 			break;
1222 		r = -E2BIG;
1223 		if (n < reg_list.n)
1224 			break;
1225 		r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1226 		break;
1227 	}
1228 	case KVM_SET_DEVICE_ATTR: {
1229 		r = -EFAULT;
1230 		if (copy_from_user(&attr, argp, sizeof(attr)))
1231 			break;
1232 		r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1233 		break;
1234 	}
1235 	case KVM_GET_DEVICE_ATTR: {
1236 		r = -EFAULT;
1237 		if (copy_from_user(&attr, argp, sizeof(attr)))
1238 			break;
1239 		r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1240 		break;
1241 	}
1242 	case KVM_HAS_DEVICE_ATTR: {
1243 		r = -EFAULT;
1244 		if (copy_from_user(&attr, argp, sizeof(attr)))
1245 			break;
1246 		r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1247 		break;
1248 	}
1249 	case KVM_GET_VCPU_EVENTS: {
1250 		struct kvm_vcpu_events events;
1251 
1252 		if (kvm_arm_vcpu_get_events(vcpu, &events))
1253 			return -EINVAL;
1254 
1255 		if (copy_to_user(argp, &events, sizeof(events)))
1256 			return -EFAULT;
1257 
1258 		return 0;
1259 	}
1260 	case KVM_SET_VCPU_EVENTS: {
1261 		struct kvm_vcpu_events events;
1262 
1263 		if (copy_from_user(&events, argp, sizeof(events)))
1264 			return -EFAULT;
1265 
1266 		return kvm_arm_vcpu_set_events(vcpu, &events);
1267 	}
1268 	case KVM_ARM_VCPU_FINALIZE: {
1269 		int what;
1270 
1271 		if (!kvm_vcpu_initialized(vcpu))
1272 			return -ENOEXEC;
1273 
1274 		if (get_user(what, (const int __user *)argp))
1275 			return -EFAULT;
1276 
1277 		return kvm_arm_vcpu_finalize(vcpu, what);
1278 	}
1279 	default:
1280 		r = -EINVAL;
1281 	}
1282 
1283 	return r;
1284 }
1285 
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)1286 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1287 {
1288 
1289 }
1290 
kvm_arch_flush_remote_tlbs_memslot(struct kvm * kvm,struct kvm_memory_slot * memslot)1291 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
1292 					struct kvm_memory_slot *memslot)
1293 {
1294 	kvm_flush_remote_tlbs(kvm);
1295 }
1296 
kvm_vm_ioctl_set_device_addr(struct kvm * kvm,struct kvm_arm_device_addr * dev_addr)1297 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1298 					struct kvm_arm_device_addr *dev_addr)
1299 {
1300 	unsigned long dev_id, type;
1301 
1302 	dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
1303 		KVM_ARM_DEVICE_ID_SHIFT;
1304 	type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
1305 		KVM_ARM_DEVICE_TYPE_SHIFT;
1306 
1307 	switch (dev_id) {
1308 	case KVM_ARM_DEVICE_VGIC_V2:
1309 		if (!vgic_present)
1310 			return -ENXIO;
1311 		return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
1312 	default:
1313 		return -ENODEV;
1314 	}
1315 }
1316 
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1317 long kvm_arch_vm_ioctl(struct file *filp,
1318 		       unsigned int ioctl, unsigned long arg)
1319 {
1320 	struct kvm *kvm = filp->private_data;
1321 	void __user *argp = (void __user *)arg;
1322 
1323 	switch (ioctl) {
1324 	case KVM_CREATE_IRQCHIP: {
1325 		int ret;
1326 		if (!vgic_present)
1327 			return -ENXIO;
1328 		mutex_lock(&kvm->lock);
1329 		ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1330 		mutex_unlock(&kvm->lock);
1331 		return ret;
1332 	}
1333 	case KVM_ARM_SET_DEVICE_ADDR: {
1334 		struct kvm_arm_device_addr dev_addr;
1335 
1336 		if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1337 			return -EFAULT;
1338 		return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1339 	}
1340 	case KVM_ARM_PREFERRED_TARGET: {
1341 		int err;
1342 		struct kvm_vcpu_init init;
1343 
1344 		err = kvm_vcpu_preferred_target(&init);
1345 		if (err)
1346 			return err;
1347 
1348 		if (copy_to_user(argp, &init, sizeof(init)))
1349 			return -EFAULT;
1350 
1351 		return 0;
1352 	}
1353 	default:
1354 		return -EINVAL;
1355 	}
1356 }
1357 
nvhe_percpu_size(void)1358 static unsigned long nvhe_percpu_size(void)
1359 {
1360 	return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1361 		(unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1362 }
1363 
nvhe_percpu_order(void)1364 static unsigned long nvhe_percpu_order(void)
1365 {
1366 	unsigned long size = nvhe_percpu_size();
1367 
1368 	return size ? get_order(size) : 0;
1369 }
1370 
1371 /* A lookup table holding the hypervisor VA for each vector slot */
1372 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1373 
kvm_init_vector_slot(void * base,enum arm64_hyp_spectre_vector slot)1374 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1375 {
1376 	hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1377 }
1378 
kvm_init_vector_slots(void)1379 static int kvm_init_vector_slots(void)
1380 {
1381 	int err;
1382 	void *base;
1383 
1384 	base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1385 	kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1386 
1387 	base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1388 	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1389 
1390 	if (kvm_system_needs_idmapped_vectors() && !has_vhe()) {
1391 		err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1392 					       __BP_HARDEN_HYP_VECS_SZ, &base);
1393 		if (err)
1394 			return err;
1395 	}
1396 
1397 	kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
1398 	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
1399 	return 0;
1400 }
1401 
cpu_prepare_hyp_mode(int cpu)1402 static void cpu_prepare_hyp_mode(int cpu)
1403 {
1404 	struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
1405 	unsigned long tcr;
1406 
1407 	/*
1408 	 * Calculate the raw per-cpu offset without a translation from the
1409 	 * kernel's mapping to the linear mapping, and store it in tpidr_el2
1410 	 * so that we can use adr_l to access per-cpu variables in EL2.
1411 	 * Also drop the KASAN tag which gets in the way...
1412 	 */
1413 	params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
1414 			    (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
1415 
1416 	params->mair_el2 = read_sysreg(mair_el1);
1417 
1418 	/*
1419 	 * The ID map may be configured to use an extended virtual address
1420 	 * range. This is only the case if system RAM is out of range for the
1421 	 * currently configured page size and VA_BITS, in which case we will
1422 	 * also need the extended virtual range for the HYP ID map, or we won't
1423 	 * be able to enable the EL2 MMU.
1424 	 *
1425 	 * However, at EL2, there is only one TTBR register, and we can't switch
1426 	 * between translation tables *and* update TCR_EL2.T0SZ at the same
1427 	 * time. Bottom line: we need to use the extended range with *both* our
1428 	 * translation tables.
1429 	 *
1430 	 * So use the same T0SZ value we use for the ID map.
1431 	 */
1432 	tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;
1433 	tcr &= ~TCR_T0SZ_MASK;
1434 	tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
1435 	params->tcr_el2 = tcr;
1436 
1437 	params->stack_hyp_va = kern_hyp_va(per_cpu(kvm_arm_hyp_stack_page, cpu) + PAGE_SIZE);
1438 	params->pgd_pa = kvm_mmu_get_httbr();
1439 	if (is_protected_kvm_enabled())
1440 		params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
1441 	else
1442 		params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
1443 	params->vttbr = params->vtcr = 0;
1444 
1445 	/*
1446 	 * Flush the init params from the data cache because the struct will
1447 	 * be read while the MMU is off.
1448 	 */
1449 	kvm_flush_dcache_to_poc(params, sizeof(*params));
1450 }
1451 
hyp_install_host_vector(void)1452 static void hyp_install_host_vector(void)
1453 {
1454 	struct kvm_nvhe_init_params *params;
1455 	struct arm_smccc_res res;
1456 
1457 	/* Switch from the HYP stub to our own HYP init vector */
1458 	__hyp_set_vectors(kvm_get_idmap_vector());
1459 
1460 	/*
1461 	 * Call initialization code, and switch to the full blown HYP code.
1462 	 * If the cpucaps haven't been finalized yet, something has gone very
1463 	 * wrong, and hyp will crash and burn when it uses any
1464 	 * cpus_have_const_cap() wrapper.
1465 	 */
1466 	BUG_ON(!system_capabilities_finalized());
1467 	params = this_cpu_ptr_nvhe_sym(kvm_init_params);
1468 	arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
1469 	WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
1470 }
1471 
cpu_init_hyp_mode(void)1472 static void cpu_init_hyp_mode(void)
1473 {
1474 	hyp_install_host_vector();
1475 
1476 	/*
1477 	 * Disabling SSBD on a non-VHE system requires us to enable SSBS
1478 	 * at EL2.
1479 	 */
1480 	if (this_cpu_has_cap(ARM64_SSBS) &&
1481 	    arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
1482 		kvm_call_hyp_nvhe(__kvm_enable_ssbs);
1483 	}
1484 }
1485 
cpu_hyp_reset(void)1486 static void cpu_hyp_reset(void)
1487 {
1488 	if (!is_kernel_in_hyp_mode())
1489 		__hyp_reset_vectors();
1490 }
1491 
1492 /*
1493  * EL2 vectors can be mapped and rerouted in a number of ways,
1494  * depending on the kernel configuration and CPU present:
1495  *
1496  * - If the CPU is affected by Spectre-v2, the hardening sequence is
1497  *   placed in one of the vector slots, which is executed before jumping
1498  *   to the real vectors.
1499  *
1500  * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
1501  *   containing the hardening sequence is mapped next to the idmap page,
1502  *   and executed before jumping to the real vectors.
1503  *
1504  * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
1505  *   empty slot is selected, mapped next to the idmap page, and
1506  *   executed before jumping to the real vectors.
1507  *
1508  * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
1509  * VHE, as we don't have hypervisor-specific mappings. If the system
1510  * is VHE and yet selects this capability, it will be ignored.
1511  */
cpu_set_hyp_vector(void)1512 static void cpu_set_hyp_vector(void)
1513 {
1514 	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
1515 	void *vector = hyp_spectre_vector_selector[data->slot];
1516 
1517 	if (!is_protected_kvm_enabled())
1518 		*this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
1519 	else
1520 		kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
1521 }
1522 
cpu_hyp_reinit(void)1523 static void cpu_hyp_reinit(void)
1524 {
1525 	kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt);
1526 
1527 	cpu_hyp_reset();
1528 
1529 	if (is_kernel_in_hyp_mode())
1530 		kvm_timer_init_vhe();
1531 	else
1532 		cpu_init_hyp_mode();
1533 
1534 	cpu_set_hyp_vector();
1535 
1536 	kvm_arm_init_debug();
1537 
1538 	if (vgic_present)
1539 		kvm_vgic_init_cpu_hardware();
1540 }
1541 
_kvm_arch_hardware_enable(void * discard)1542 static void _kvm_arch_hardware_enable(void *discard)
1543 {
1544 	if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
1545 		cpu_hyp_reinit();
1546 		__this_cpu_write(kvm_arm_hardware_enabled, 1);
1547 	}
1548 }
1549 
kvm_arch_hardware_enable(void)1550 int kvm_arch_hardware_enable(void)
1551 {
1552 	_kvm_arch_hardware_enable(NULL);
1553 	return 0;
1554 }
1555 
_kvm_arch_hardware_disable(void * discard)1556 static void _kvm_arch_hardware_disable(void *discard)
1557 {
1558 	if (__this_cpu_read(kvm_arm_hardware_enabled)) {
1559 		cpu_hyp_reset();
1560 		__this_cpu_write(kvm_arm_hardware_enabled, 0);
1561 	}
1562 }
1563 
kvm_arch_hardware_disable(void)1564 void kvm_arch_hardware_disable(void)
1565 {
1566 	if (!is_protected_kvm_enabled())
1567 		_kvm_arch_hardware_disable(NULL);
1568 }
1569 
1570 #ifdef CONFIG_CPU_PM
hyp_init_cpu_pm_notifier(struct notifier_block * self,unsigned long cmd,void * v)1571 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
1572 				    unsigned long cmd,
1573 				    void *v)
1574 {
1575 	/*
1576 	 * kvm_arm_hardware_enabled is left with its old value over
1577 	 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
1578 	 * re-enable hyp.
1579 	 */
1580 	switch (cmd) {
1581 	case CPU_PM_ENTER:
1582 		if (__this_cpu_read(kvm_arm_hardware_enabled))
1583 			/*
1584 			 * don't update kvm_arm_hardware_enabled here
1585 			 * so that the hardware will be re-enabled
1586 			 * when we resume. See below.
1587 			 */
1588 			cpu_hyp_reset();
1589 
1590 		return NOTIFY_OK;
1591 	case CPU_PM_ENTER_FAILED:
1592 	case CPU_PM_EXIT:
1593 		if (__this_cpu_read(kvm_arm_hardware_enabled))
1594 			/* The hardware was enabled before suspend. */
1595 			cpu_hyp_reinit();
1596 
1597 		return NOTIFY_OK;
1598 
1599 	default:
1600 		return NOTIFY_DONE;
1601 	}
1602 }
1603 
1604 static struct notifier_block hyp_init_cpu_pm_nb = {
1605 	.notifier_call = hyp_init_cpu_pm_notifier,
1606 };
1607 
hyp_cpu_pm_init(void)1608 static void hyp_cpu_pm_init(void)
1609 {
1610 	if (!is_protected_kvm_enabled())
1611 		cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
1612 }
hyp_cpu_pm_exit(void)1613 static void hyp_cpu_pm_exit(void)
1614 {
1615 	if (!is_protected_kvm_enabled())
1616 		cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
1617 }
1618 #else
hyp_cpu_pm_init(void)1619 static inline void hyp_cpu_pm_init(void)
1620 {
1621 }
hyp_cpu_pm_exit(void)1622 static inline void hyp_cpu_pm_exit(void)
1623 {
1624 }
1625 #endif
1626 
init_cpu_logical_map(void)1627 static void init_cpu_logical_map(void)
1628 {
1629 	unsigned int cpu;
1630 
1631 	/*
1632 	 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
1633 	 * Only copy the set of online CPUs whose features have been chacked
1634 	 * against the finalized system capabilities. The hypervisor will not
1635 	 * allow any other CPUs from the `possible` set to boot.
1636 	 */
1637 	for_each_online_cpu(cpu)
1638 		hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
1639 }
1640 
1641 #define init_psci_0_1_impl_state(config, what)	\
1642 	config.psci_0_1_ ## what ## _implemented = psci_ops.what
1643 
init_psci_relay(void)1644 static bool init_psci_relay(void)
1645 {
1646 	/*
1647 	 * If PSCI has not been initialized, protected KVM cannot install
1648 	 * itself on newly booted CPUs.
1649 	 */
1650 	if (!psci_ops.get_version) {
1651 		kvm_err("Cannot initialize protected mode without PSCI\n");
1652 		return false;
1653 	}
1654 
1655 	kvm_host_psci_config.version = psci_ops.get_version();
1656 
1657 	if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
1658 		kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
1659 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
1660 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
1661 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
1662 		init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
1663 	}
1664 	return true;
1665 }
1666 
init_common_resources(void)1667 static int init_common_resources(void)
1668 {
1669 	return kvm_set_ipa_limit();
1670 }
1671 
init_subsystems(void)1672 static int init_subsystems(void)
1673 {
1674 	int err = 0;
1675 
1676 	/*
1677 	 * Enable hardware so that subsystem initialisation can access EL2.
1678 	 */
1679 	on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
1680 
1681 	/*
1682 	 * Register CPU lower-power notifier
1683 	 */
1684 	hyp_cpu_pm_init();
1685 
1686 	/*
1687 	 * Init HYP view of VGIC
1688 	 */
1689 	err = kvm_vgic_hyp_init();
1690 	switch (err) {
1691 	case 0:
1692 		vgic_present = true;
1693 		break;
1694 	case -ENODEV:
1695 	case -ENXIO:
1696 		vgic_present = false;
1697 		err = 0;
1698 		break;
1699 	default:
1700 		goto out;
1701 	}
1702 
1703 	/*
1704 	 * Init HYP architected timer support
1705 	 */
1706 	err = kvm_timer_hyp_init(vgic_present);
1707 	if (err)
1708 		goto out;
1709 
1710 	kvm_perf_init();
1711 	kvm_sys_reg_table_init();
1712 
1713 out:
1714 	if (err || !is_protected_kvm_enabled())
1715 		on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
1716 
1717 	return err;
1718 }
1719 
teardown_hyp_mode(void)1720 static void teardown_hyp_mode(void)
1721 {
1722 	int cpu;
1723 
1724 	free_hyp_pgds();
1725 	for_each_possible_cpu(cpu) {
1726 		free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1727 		free_pages(kvm_arm_hyp_percpu_base[cpu], nvhe_percpu_order());
1728 	}
1729 }
1730 
do_pkvm_init(u32 hyp_va_bits)1731 static int do_pkvm_init(u32 hyp_va_bits)
1732 {
1733 	void *per_cpu_base = kvm_ksym_ref(kvm_arm_hyp_percpu_base);
1734 	int ret;
1735 
1736 	preempt_disable();
1737 	hyp_install_host_vector();
1738 	ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
1739 				num_possible_cpus(), kern_hyp_va(per_cpu_base),
1740 				hyp_va_bits);
1741 	preempt_enable();
1742 
1743 	return ret;
1744 }
1745 
kvm_hyp_init_protection(u32 hyp_va_bits)1746 static int kvm_hyp_init_protection(u32 hyp_va_bits)
1747 {
1748 	void *addr = phys_to_virt(hyp_mem_base);
1749 	int ret;
1750 
1751 	kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
1752 	kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
1753 
1754 	ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
1755 	if (ret)
1756 		return ret;
1757 
1758 	ret = do_pkvm_init(hyp_va_bits);
1759 	if (ret)
1760 		return ret;
1761 
1762 	free_hyp_pgds();
1763 
1764 	return 0;
1765 }
1766 
1767 /**
1768  * Inits Hyp-mode on all online CPUs
1769  */
init_hyp_mode(void)1770 static int init_hyp_mode(void)
1771 {
1772 	u32 hyp_va_bits;
1773 	int cpu;
1774 	int err = -ENOMEM;
1775 
1776 	/*
1777 	 * The protected Hyp-mode cannot be initialized if the memory pool
1778 	 * allocation has failed.
1779 	 */
1780 	if (is_protected_kvm_enabled() && !hyp_mem_base)
1781 		goto out_err;
1782 
1783 	/*
1784 	 * Allocate Hyp PGD and setup Hyp identity mapping
1785 	 */
1786 	err = kvm_mmu_init(&hyp_va_bits);
1787 	if (err)
1788 		goto out_err;
1789 
1790 	/*
1791 	 * Allocate stack pages for Hypervisor-mode
1792 	 */
1793 	for_each_possible_cpu(cpu) {
1794 		unsigned long stack_page;
1795 
1796 		stack_page = __get_free_page(GFP_KERNEL);
1797 		if (!stack_page) {
1798 			err = -ENOMEM;
1799 			goto out_err;
1800 		}
1801 
1802 		per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
1803 	}
1804 
1805 	/*
1806 	 * Allocate and initialize pages for Hypervisor-mode percpu regions.
1807 	 */
1808 	for_each_possible_cpu(cpu) {
1809 		struct page *page;
1810 		void *page_addr;
1811 
1812 		page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
1813 		if (!page) {
1814 			err = -ENOMEM;
1815 			goto out_err;
1816 		}
1817 
1818 		page_addr = page_address(page);
1819 		memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
1820 		kvm_arm_hyp_percpu_base[cpu] = (unsigned long)page_addr;
1821 	}
1822 
1823 	/*
1824 	 * Map the Hyp-code called directly from the host
1825 	 */
1826 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
1827 				  kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
1828 	if (err) {
1829 		kvm_err("Cannot map world-switch code\n");
1830 		goto out_err;
1831 	}
1832 
1833 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
1834 				  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
1835 	if (err) {
1836 		kvm_err("Cannot map .hyp.rodata section\n");
1837 		goto out_err;
1838 	}
1839 
1840 	err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
1841 				  kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
1842 	if (err) {
1843 		kvm_err("Cannot map rodata section\n");
1844 		goto out_err;
1845 	}
1846 
1847 	/*
1848 	 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
1849 	 * section thanks to an assertion in the linker script. Map it RW and
1850 	 * the rest of .bss RO.
1851 	 */
1852 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
1853 				  kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
1854 	if (err) {
1855 		kvm_err("Cannot map hyp bss section: %d\n", err);
1856 		goto out_err;
1857 	}
1858 
1859 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
1860 				  kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
1861 	if (err) {
1862 		kvm_err("Cannot map bss section\n");
1863 		goto out_err;
1864 	}
1865 
1866 	/*
1867 	 * Map the Hyp stack pages
1868 	 */
1869 	for_each_possible_cpu(cpu) {
1870 		char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
1871 		err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,
1872 					  PAGE_HYP);
1873 
1874 		if (err) {
1875 			kvm_err("Cannot map hyp stack\n");
1876 			goto out_err;
1877 		}
1878 	}
1879 
1880 	for_each_possible_cpu(cpu) {
1881 		char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu];
1882 		char *percpu_end = percpu_begin + nvhe_percpu_size();
1883 
1884 		/* Map Hyp percpu pages */
1885 		err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
1886 		if (err) {
1887 			kvm_err("Cannot map hyp percpu region\n");
1888 			goto out_err;
1889 		}
1890 
1891 		/* Prepare the CPU initialization parameters */
1892 		cpu_prepare_hyp_mode(cpu);
1893 	}
1894 
1895 	if (is_protected_kvm_enabled()) {
1896 		init_cpu_logical_map();
1897 
1898 		if (!init_psci_relay()) {
1899 			err = -ENODEV;
1900 			goto out_err;
1901 		}
1902 	}
1903 
1904 	if (is_protected_kvm_enabled()) {
1905 		err = kvm_hyp_init_protection(hyp_va_bits);
1906 		if (err) {
1907 			kvm_err("Failed to init hyp memory protection\n");
1908 			goto out_err;
1909 		}
1910 	}
1911 
1912 	return 0;
1913 
1914 out_err:
1915 	teardown_hyp_mode();
1916 	kvm_err("error initializing Hyp mode: %d\n", err);
1917 	return err;
1918 }
1919 
_kvm_host_prot_finalize(void * discard)1920 void _kvm_host_prot_finalize(void *discard)
1921 {
1922 	WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize));
1923 }
1924 
pkvm_mark_hyp(phys_addr_t start,phys_addr_t end)1925 static inline int pkvm_mark_hyp(phys_addr_t start, phys_addr_t end)
1926 {
1927 	return kvm_call_hyp_nvhe(__pkvm_mark_hyp, start, end);
1928 }
1929 
1930 #define pkvm_mark_hyp_section(__section)		\
1931 	pkvm_mark_hyp(__pa_symbol(__section##_start),	\
1932 			__pa_symbol(__section##_end))
1933 
finalize_hyp_mode(void)1934 static int finalize_hyp_mode(void)
1935 {
1936 	int cpu, ret;
1937 
1938 	if (!is_protected_kvm_enabled())
1939 		return 0;
1940 
1941 	ret = pkvm_mark_hyp_section(__hyp_idmap_text);
1942 	if (ret)
1943 		return ret;
1944 
1945 	ret = pkvm_mark_hyp_section(__hyp_text);
1946 	if (ret)
1947 		return ret;
1948 
1949 	ret = pkvm_mark_hyp_section(__hyp_rodata);
1950 	if (ret)
1951 		return ret;
1952 
1953 	/*
1954 	 * Exclude HYP BSS from kmemleak so that it doesn't get peeked
1955 	 * at, which would end badly once the section is inaccessible.
1956 	 * None of other sections should ever be introspected.
1957 	 */
1958 	kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start);
1959 	ret = pkvm_mark_hyp_section(__hyp_bss);
1960 	if (ret)
1961 		return ret;
1962 
1963 	ret = pkvm_mark_hyp(hyp_mem_base, hyp_mem_base + hyp_mem_size);
1964 	if (ret)
1965 		return ret;
1966 
1967 	for_each_possible_cpu(cpu) {
1968 		phys_addr_t start = virt_to_phys((void *)kvm_arm_hyp_percpu_base[cpu]);
1969 		phys_addr_t end = start + (PAGE_SIZE << nvhe_percpu_order());
1970 
1971 		ret = pkvm_mark_hyp(start, end);
1972 		if (ret)
1973 			return ret;
1974 
1975 		start = virt_to_phys((void *)per_cpu(kvm_arm_hyp_stack_page, cpu));
1976 		end = start + PAGE_SIZE;
1977 		ret = pkvm_mark_hyp(start, end);
1978 		if (ret)
1979 			return ret;
1980 	}
1981 
1982 	/*
1983 	 * Flip the static key upfront as that may no longer be possible
1984 	 * once the host stage 2 is installed.
1985 	 */
1986 	static_branch_enable(&kvm_protected_mode_initialized);
1987 	on_each_cpu(_kvm_host_prot_finalize, NULL, 1);
1988 
1989 	return 0;
1990 }
1991 
check_kvm_target_cpu(void * ret)1992 static void check_kvm_target_cpu(void *ret)
1993 {
1994 	*(int *)ret = kvm_target_cpu();
1995 }
1996 
kvm_mpidr_to_vcpu(struct kvm * kvm,unsigned long mpidr)1997 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
1998 {
1999 	struct kvm_vcpu *vcpu;
2000 	int i;
2001 
2002 	mpidr &= MPIDR_HWID_BITMASK;
2003 	kvm_for_each_vcpu(i, vcpu, kvm) {
2004 		if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2005 			return vcpu;
2006 	}
2007 	return NULL;
2008 }
2009 
kvm_arch_has_irq_bypass(void)2010 bool kvm_arch_has_irq_bypass(void)
2011 {
2012 	return true;
2013 }
2014 
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2015 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2016 				      struct irq_bypass_producer *prod)
2017 {
2018 	struct kvm_kernel_irqfd *irqfd =
2019 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2020 
2021 	return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2022 					  &irqfd->irq_entry);
2023 }
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2024 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2025 				      struct irq_bypass_producer *prod)
2026 {
2027 	struct kvm_kernel_irqfd *irqfd =
2028 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2029 
2030 	kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
2031 				     &irqfd->irq_entry);
2032 }
2033 
kvm_arch_irq_bypass_stop(struct irq_bypass_consumer * cons)2034 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2035 {
2036 	struct kvm_kernel_irqfd *irqfd =
2037 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2038 
2039 	kvm_arm_halt_guest(irqfd->kvm);
2040 }
2041 
kvm_arch_irq_bypass_start(struct irq_bypass_consumer * cons)2042 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2043 {
2044 	struct kvm_kernel_irqfd *irqfd =
2045 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2046 
2047 	kvm_arm_resume_guest(irqfd->kvm);
2048 }
2049 
2050 /**
2051  * Initialize Hyp-mode and memory mappings on all CPUs.
2052  */
kvm_arch_init(void * opaque)2053 int kvm_arch_init(void *opaque)
2054 {
2055 	int err;
2056 	int ret, cpu;
2057 	bool in_hyp_mode;
2058 
2059 	if (!is_hyp_mode_available()) {
2060 		kvm_info("HYP mode not available\n");
2061 		return -ENODEV;
2062 	}
2063 
2064 	if (kvm_get_mode() == KVM_MODE_NONE) {
2065 		kvm_info("KVM disabled from command line\n");
2066 		return -ENODEV;
2067 	}
2068 
2069 	in_hyp_mode = is_kernel_in_hyp_mode();
2070 
2071 	if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2072 	    cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2073 		kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2074 			 "Only trusted guests should be used on this system.\n");
2075 
2076 	for_each_online_cpu(cpu) {
2077 		smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
2078 		if (ret < 0) {
2079 			kvm_err("Error, CPU %d not supported!\n", cpu);
2080 			return -ENODEV;
2081 		}
2082 	}
2083 
2084 	err = init_common_resources();
2085 	if (err)
2086 		return err;
2087 
2088 	err = kvm_arm_init_sve();
2089 	if (err)
2090 		return err;
2091 
2092 	if (!in_hyp_mode) {
2093 		err = init_hyp_mode();
2094 		if (err)
2095 			goto out_err;
2096 	}
2097 
2098 	err = kvm_init_vector_slots();
2099 	if (err) {
2100 		kvm_err("Cannot initialise vector slots\n");
2101 		goto out_hyp;
2102 	}
2103 
2104 	err = init_subsystems();
2105 	if (err)
2106 		goto out_subs;
2107 
2108 	if (!in_hyp_mode) {
2109 		err = finalize_hyp_mode();
2110 		if (err) {
2111 			kvm_err("Failed to finalize Hyp protection\n");
2112 			goto out_subs;
2113 		}
2114 	}
2115 
2116 	if (is_protected_kvm_enabled()) {
2117 		kvm_info("Protected nVHE mode initialized successfully\n");
2118 	} else if (in_hyp_mode) {
2119 		kvm_info("VHE mode initialized successfully\n");
2120 	} else {
2121 		kvm_info("Hyp mode initialized successfully\n");
2122 	}
2123 
2124 	return 0;
2125 
2126 out_subs:
2127 	hyp_cpu_pm_exit();
2128 out_hyp:
2129 	if (!in_hyp_mode)
2130 		teardown_hyp_mode();
2131 out_err:
2132 	return err;
2133 }
2134 
2135 /* NOP: Compiling as a module not supported */
kvm_arch_exit(void)2136 void kvm_arch_exit(void)
2137 {
2138 	kvm_perf_teardown();
2139 }
2140 
early_kvm_mode_cfg(char * arg)2141 static int __init early_kvm_mode_cfg(char *arg)
2142 {
2143 	if (!arg)
2144 		return -EINVAL;
2145 
2146 	if (strcmp(arg, "none") == 0) {
2147 		kvm_mode = KVM_MODE_NONE;
2148 		return 0;
2149 	}
2150 
2151 	if (!is_hyp_mode_available()) {
2152 		pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2153 		return 0;
2154 	}
2155 
2156 	if (strcmp(arg, "protected") == 0) {
2157 		kvm_mode = KVM_MODE_PROTECTED;
2158 		return 0;
2159 	}
2160 
2161 	if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2162 		kvm_mode = KVM_MODE_DEFAULT;
2163 		return 0;
2164 	}
2165 
2166 	return -EINVAL;
2167 }
2168 early_param("kvm-arm.mode", early_kvm_mode_cfg);
2169 
kvm_get_mode(void)2170 enum kvm_mode kvm_get_mode(void)
2171 {
2172 	return kvm_mode;
2173 }
2174 
arm_init(void)2175 static int arm_init(void)
2176 {
2177 	int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2178 	return rc;
2179 }
2180 
2181 module_init(arm_init);
2182