• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2021 Google LLC
4  * Author: Fuad Tabba <tabba@google.com>
5  */
6 
7 #include <linux/kvm_host.h>
8 #include <linux/mm.h>
9 
10 #include <kvm/arm_hypercalls.h>
11 #include <kvm/arm_psci.h>
12 
13 #include <asm/kvm_emulate.h>
14 
15 #include <nvhe/arm-smccc.h>
16 #include <nvhe/mem_protect.h>
17 #include <nvhe/memory.h>
18 #include <nvhe/mm.h>
19 #include <nvhe/pkvm.h>
20 #include <nvhe/trap_handler.h>
21 
22 /* Used by icache_is_vpipt(). */
23 unsigned long __icache_flags;
24 
25 /* Used by kvm_get_vttbr(). */
26 unsigned int kvm_arm_vmid_bits;
27 
28 unsigned int kvm_host_sve_max_vl;
29 
30 /*
31  * The currently loaded hyp vCPU for each physical CPU. Used only when
32  * protected KVM is enabled, but for both protected and non-protected VMs.
33  */
34 static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu);
35 
36 /*
37  * Host fp state for all cpus. This could include the host simd state, as well
38  * as the sve and sme states if supported. Written to when the guest accesses
39  * its own FPSIMD state, and read when the guest state is live and we need to
40  * switch back to the host.
41  *
42  * Only valid when (fp_state == FP_STATE_GUEST_OWNED) in the hyp vCPU structure.
43  */
44 unsigned long __ro_after_init kvm_arm_hyp_host_fp_state[NR_CPUS];
45 
__get_host_fpsimd_bytes(void)46 static void *__get_host_fpsimd_bytes(void)
47 {
48 	/*
49 	 * The addresses in this array have been converted to hyp addresses
50 	 * in finalize_init_hyp_mode().
51 	 */
52 	return (void *)kvm_arm_hyp_host_fp_state[hyp_smp_processor_id()];
53 }
54 
get_host_fpsimd_state(struct kvm_vcpu * vcpu)55 struct user_fpsimd_state *get_host_fpsimd_state(struct kvm_vcpu *vcpu)
56 {
57 	if (likely(!is_protected_kvm_enabled()))
58 		return vcpu->arch.host_fpsimd_state;
59 
60 	WARN_ON(system_supports_sve());
61 	return __get_host_fpsimd_bytes();
62 }
63 
get_host_sve_state(struct kvm_vcpu * vcpu)64 struct kvm_host_sve_state *get_host_sve_state(struct kvm_vcpu *vcpu)
65 {
66 	WARN_ON(!system_supports_sve());
67 	WARN_ON(!is_protected_kvm_enabled());
68 	return __get_host_fpsimd_bytes();
69 }
70 
71 /*
72  * Set trap register values based on features in ID_AA64PFR0.
73  */
pvm_init_traps_aa64pfr0(struct kvm_vcpu * vcpu)74 static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu)
75 {
76 	const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR0_EL1);
77 	u64 hcr_set = HCR_RW;
78 	u64 hcr_clear = 0;
79 	u64 cptr_set = 0;
80 
81 	/* Protected KVM does not support AArch32 guests. */
82 	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0),
83 		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
84 	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1),
85 		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
86 
87 	/*
88 	 * Linux guests assume support for floating-point and Advanced SIMD. Do
89 	 * not change the trapping behavior for these from the KVM default.
90 	 */
91 	BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP),
92 				PVM_ID_AA64PFR0_ALLOW));
93 	BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD),
94 				PVM_ID_AA64PFR0_ALLOW));
95 
96 	/* Trap RAS unless all current versions are supported */
97 	if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), feature_ids) <
98 	    ID_AA64PFR0_EL1_RAS_V1P1) {
99 		hcr_set |= HCR_TERR | HCR_TEA;
100 		hcr_clear |= HCR_FIEN;
101 	}
102 
103 	/* Trap AMU */
104 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), feature_ids)) {
105 		hcr_clear |= HCR_AMVOFFEN;
106 		cptr_set |= CPTR_EL2_TAM;
107 	}
108 
109 	/* Trap SVE */
110 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids))
111 		cptr_set |= CPTR_EL2_TZ;
112 
113 	vcpu->arch.hcr_el2 |= hcr_set;
114 	vcpu->arch.hcr_el2 &= ~hcr_clear;
115 	vcpu->arch.cptr_el2 |= cptr_set;
116 }
117 
118 /*
119  * Set trap register values based on features in ID_AA64PFR1.
120  */
pvm_init_traps_aa64pfr1(struct kvm_vcpu * vcpu)121 static void pvm_init_traps_aa64pfr1(struct kvm_vcpu *vcpu)
122 {
123 	const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR1_EL1);
124 	u64 hcr_set = 0;
125 	u64 hcr_clear = 0;
126 
127 	/* Memory Tagging: Trap and Treat as Untagged if not supported. */
128 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE), feature_ids)) {
129 		hcr_set |= HCR_TID5;
130 		hcr_clear |= HCR_DCT | HCR_ATA;
131 	}
132 
133 	vcpu->arch.hcr_el2 |= hcr_set;
134 	vcpu->arch.hcr_el2 &= ~hcr_clear;
135 }
136 
137 /*
138  * Set trap register values based on features in ID_AA64DFR0.
139  */
pvm_init_traps_aa64dfr0(struct kvm_vcpu * vcpu)140 static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu)
141 {
142 	const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64DFR0_EL1);
143 	u64 mdcr_set = 0;
144 	u64 mdcr_clear = 0;
145 	u64 cptr_set = 0;
146 
147 	/* Trap/constrain PMU */
148 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), feature_ids)) {
149 		mdcr_set |= MDCR_EL2_TPM | MDCR_EL2_TPMCR;
150 		mdcr_clear |= MDCR_EL2_HPME | MDCR_EL2_MTPME |
151 			      MDCR_EL2_HPMN_MASK;
152 	}
153 
154 	/* Trap Debug */
155 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), feature_ids))
156 		mdcr_set |= MDCR_EL2_TDRA | MDCR_EL2_TDA;
157 
158 	/* Trap OS Double Lock */
159 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DoubleLock), feature_ids))
160 		mdcr_set |= MDCR_EL2_TDOSA;
161 
162 	/* Trap SPE */
163 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer), feature_ids)) {
164 		mdcr_set |= MDCR_EL2_TPMS;
165 		mdcr_clear |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
166 	}
167 
168 	/* Trap Trace Filter */
169 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceFilt), feature_ids))
170 		mdcr_set |= MDCR_EL2_TTRF;
171 
172 	/* Trap Trace */
173 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids))
174 		cptr_set |= CPTR_EL2_TTA;
175 
176 	vcpu->arch.mdcr_el2 |= mdcr_set;
177 	vcpu->arch.mdcr_el2 &= ~mdcr_clear;
178 	vcpu->arch.cptr_el2 |= cptr_set;
179 }
180 
181 /*
182  * Set trap register values based on features in ID_AA64MMFR0.
183  */
pvm_init_traps_aa64mmfr0(struct kvm_vcpu * vcpu)184 static void pvm_init_traps_aa64mmfr0(struct kvm_vcpu *vcpu)
185 {
186 	const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR0_EL1);
187 	u64 mdcr_set = 0;
188 
189 	/* Trap Debug Communications Channel registers */
190 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_FGT), feature_ids))
191 		mdcr_set |= MDCR_EL2_TDCC;
192 
193 	vcpu->arch.mdcr_el2 |= mdcr_set;
194 }
195 
196 /*
197  * Set trap register values based on features in ID_AA64MMFR1.
198  */
pvm_init_traps_aa64mmfr1(struct kvm_vcpu * vcpu)199 static void pvm_init_traps_aa64mmfr1(struct kvm_vcpu *vcpu)
200 {
201 	const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR1_EL1);
202 	u64 hcr_set = 0;
203 
204 	/* Trap LOR */
205 	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_LO), feature_ids))
206 		hcr_set |= HCR_TLOR;
207 
208 	vcpu->arch.hcr_el2 |= hcr_set;
209 }
210 
211 /*
212  * Set baseline trap register values.
213  */
pvm_init_trap_regs(struct kvm_vcpu * vcpu)214 static void pvm_init_trap_regs(struct kvm_vcpu *vcpu)
215 {
216 	/*
217 	 * Always trap:
218 	 * - Feature id registers: to control features exposed to guests
219 	 * - Implementation-defined features
220 	 */
221 	vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS |
222 			     HCR_TID3 | HCR_TACR | HCR_TIDCP | HCR_TID1;
223 
224 	if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) {
225 		/* route synchronous external abort exceptions to EL2 */
226 		vcpu->arch.hcr_el2 |= HCR_TEA;
227 		/* trap error record accesses */
228 		vcpu->arch.hcr_el2 |= HCR_TERR;
229 	}
230 
231 	if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
232 		vcpu->arch.hcr_el2 |= HCR_FWB;
233 
234 	if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE))
235 		vcpu->arch.hcr_el2 |= HCR_TID2;
236 }
237 
238 /*
239  * Initialize trap register values for protected VMs.
240  */
pkvm_vcpu_init_traps(struct pkvm_hyp_vcpu * hyp_vcpu)241 static void pkvm_vcpu_init_traps(struct pkvm_hyp_vcpu *hyp_vcpu)
242 {
243 	hyp_vcpu->vcpu.arch.cptr_el2 = CPTR_EL2_DEFAULT;
244 	hyp_vcpu->vcpu.arch.mdcr_el2 = 0;
245 
246 	if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) {
247 		u64 hcr = READ_ONCE(hyp_vcpu->host_vcpu->arch.hcr_el2);
248 
249 		hyp_vcpu->vcpu.arch.hcr_el2 = HCR_GUEST_FLAGS | hcr;
250 		return;
251 	}
252 
253 	pvm_init_trap_regs(&hyp_vcpu->vcpu);
254 	pvm_init_traps_aa64pfr0(&hyp_vcpu->vcpu);
255 	pvm_init_traps_aa64pfr1(&hyp_vcpu->vcpu);
256 	pvm_init_traps_aa64dfr0(&hyp_vcpu->vcpu);
257 	pvm_init_traps_aa64mmfr0(&hyp_vcpu->vcpu);
258 	pvm_init_traps_aa64mmfr1(&hyp_vcpu->vcpu);
259 }
260 
261 /*
262  * Start the VM table handle at the offset defined instead of at 0.
263  * Mainly for sanity checking and debugging.
264  */
265 #define HANDLE_OFFSET 0x1000
266 
vm_handle_to_idx(pkvm_handle_t handle)267 static unsigned int vm_handle_to_idx(pkvm_handle_t handle)
268 {
269 	return handle - HANDLE_OFFSET;
270 }
271 
idx_to_vm_handle(unsigned int idx)272 static pkvm_handle_t idx_to_vm_handle(unsigned int idx)
273 {
274 	return idx + HANDLE_OFFSET;
275 }
276 
277 /*
278  * Spinlock for protecting state related to the VM table. Protects writes
279  * to 'vm_table' and 'nr_table_entries' as well as reads and writes to
280  * 'last_hyp_vcpu_lookup'.
281  */
282 static DEFINE_HYP_SPINLOCK(vm_table_lock);
283 
284 /*
285  * The table of VM entries for protected VMs in hyp.
286  * Allocated at hyp initialization and setup.
287  */
288 static struct pkvm_hyp_vm **vm_table;
289 
pkvm_hyp_vm_table_init(void * tbl)290 void pkvm_hyp_vm_table_init(void *tbl)
291 {
292 	WARN_ON(vm_table);
293 	vm_table = tbl;
294 }
295 
296 /*
297  * Return the hyp vm structure corresponding to the handle.
298  */
get_vm_by_handle(pkvm_handle_t handle)299 static struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle)
300 {
301 	unsigned int idx = vm_handle_to_idx(handle);
302 
303 	if (unlikely(idx >= KVM_MAX_PVMS))
304 		return NULL;
305 
306 	return vm_table[idx];
307 }
308 
__pkvm_reclaim_dying_guest_page(pkvm_handle_t handle,u64 pfn,u64 ipa)309 int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 pfn, u64 ipa)
310 {
311 	struct pkvm_hyp_vm *hyp_vm;
312 	int ret = -EINVAL;
313 
314 	hyp_spin_lock(&vm_table_lock);
315 	hyp_vm = get_vm_by_handle(handle);
316 	if (!hyp_vm || !hyp_vm->is_dying)
317 		goto unlock;
318 
319 	ret = __pkvm_host_reclaim_page(hyp_vm, pfn, ipa);
320 	if (ret)
321 		goto unlock;
322 
323 	drain_hyp_pool(hyp_vm, &hyp_vm->host_kvm->arch.pkvm.teardown_stage2_mc);
324 unlock:
325 	hyp_spin_unlock(&vm_table_lock);
326 
327 	return ret;
328 }
329 
pkvm_load_hyp_vcpu(pkvm_handle_t handle,unsigned int vcpu_idx)330 struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
331 					 unsigned int vcpu_idx)
332 {
333 	struct pkvm_hyp_vcpu *hyp_vcpu = NULL;
334 	struct pkvm_hyp_vm *hyp_vm;
335 
336 	/* Cannot load a new vcpu without putting the old one first. */
337 	if (__this_cpu_read(loaded_hyp_vcpu))
338 		return NULL;
339 
340 	hyp_spin_lock(&vm_table_lock);
341 	hyp_vm = get_vm_by_handle(handle);
342 	if (!hyp_vm || hyp_vm->is_dying || hyp_vm->nr_vcpus <= vcpu_idx)
343 		goto unlock;
344 
345 	hyp_vcpu = hyp_vm->vcpus[vcpu_idx];
346 
347 	/* Ensure vcpu isn't loaded on more than one cpu simultaneously. */
348 	if (unlikely(hyp_vcpu->loaded_hyp_vcpu)) {
349 		hyp_vcpu = NULL;
350 		goto unlock;
351 	}
352 
353 	hyp_vcpu->loaded_hyp_vcpu = this_cpu_ptr(&loaded_hyp_vcpu);
354 	hyp_page_ref_inc(hyp_virt_to_page(hyp_vm));
355 unlock:
356 	hyp_spin_unlock(&vm_table_lock);
357 
358 	if (hyp_vcpu)
359 		__this_cpu_write(loaded_hyp_vcpu, hyp_vcpu);
360 	return hyp_vcpu;
361 }
362 
pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu * hyp_vcpu)363 void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
364 {
365 	struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
366 
367 	hyp_spin_lock(&vm_table_lock);
368 	hyp_vcpu->loaded_hyp_vcpu = NULL;
369 	__this_cpu_write(loaded_hyp_vcpu, NULL);
370 	hyp_page_ref_dec(hyp_virt_to_page(hyp_vm));
371 	hyp_spin_unlock(&vm_table_lock);
372 }
373 
pkvm_get_loaded_hyp_vcpu(void)374 struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void)
375 {
376 	return __this_cpu_read(loaded_hyp_vcpu);
377 }
378 
pkvm_vcpu_init_features_from_host(struct pkvm_hyp_vcpu * hyp_vcpu)379 static void pkvm_vcpu_init_features_from_host(struct pkvm_hyp_vcpu *hyp_vcpu)
380 {
381 	struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
382 	DECLARE_BITMAP(allowed_features, KVM_VCPU_MAX_FEATURES);
383 
384 	/* No restrictions for non-protected VMs. */
385 	if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) {
386 		bitmap_copy(hyp_vcpu->vcpu.arch.features,
387 			    host_vcpu->arch.features,
388 			    KVM_VCPU_MAX_FEATURES);
389 		return;
390 	}
391 
392 	bitmap_zero(allowed_features, KVM_VCPU_MAX_FEATURES);
393 
394 	/*
395 	 * For protected vms, always allow:
396 	 * - CPU starting in poweroff state
397 	 * - PSCI v0.2
398 	 */
399 	set_bit(KVM_ARM_VCPU_POWER_OFF, allowed_features);
400 	set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed_features);
401 
402 	/*
403 	 * Check if remaining features are allowed:
404 	 * - Performance Monitoring
405 	 * - Scalable Vectors
406 	 * - Pointer Authentication
407 	 */
408 	if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), PVM_ID_AA64DFR0_ALLOW))
409 		set_bit(KVM_ARM_VCPU_PMU_V3, allowed_features);
410 
411 	if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), PVM_ID_AA64PFR0_ALLOW))
412 		set_bit(KVM_ARM_VCPU_SVE, allowed_features);
413 
414 	if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API), PVM_ID_AA64ISAR1_ALLOW) &&
415 	    FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA), PVM_ID_AA64ISAR1_ALLOW))
416 		set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed_features);
417 
418 	if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI), PVM_ID_AA64ISAR1_ALLOW) &&
419 	    FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA), PVM_ID_AA64ISAR1_ALLOW))
420 		set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features);
421 
422 	bitmap_and(hyp_vcpu->vcpu.arch.features, host_vcpu->arch.features,
423 		   allowed_features, KVM_VCPU_MAX_FEATURES);
424 
425 	/*
426 	 * Now sanitise the configuration flags that we have inherited
427 	 * from the host, as they may refer to features that protected
428 	 * mode doesn't support.
429 	 */
430 	if (!vcpu_has_feature(&hyp_vcpu->vcpu,(KVM_ARM_VCPU_SVE))) {
431 		vcpu_clear_flag(&hyp_vcpu->vcpu, GUEST_HAS_SVE);
432 		vcpu_clear_flag(&hyp_vcpu->vcpu, VCPU_SVE_FINALIZED);
433 	}
434 
435 	if (!vcpu_has_feature(&hyp_vcpu->vcpu, KVM_ARM_VCPU_PTRAUTH_ADDRESS) ||
436 	    !vcpu_has_feature(&hyp_vcpu->vcpu, KVM_ARM_VCPU_PTRAUTH_GENERIC))
437 		vcpu_clear_flag(&hyp_vcpu->vcpu, GUEST_HAS_PTRAUTH);
438 }
439 
pkvm_vcpu_init_ptrauth(struct pkvm_hyp_vcpu * hyp_vcpu)440 static int pkvm_vcpu_init_ptrauth(struct pkvm_hyp_vcpu *hyp_vcpu)
441 {
442 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
443 	int ret = 0;
444 
445 	if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, vcpu->arch.features) ||
446 	    test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, vcpu->arch.features))
447 		ret = kvm_vcpu_enable_ptrauth(vcpu);
448 
449 	return ret;
450 }
451 
pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu * hyp_vcpu)452 static int pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu *hyp_vcpu)
453 {
454 	struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state;
455 	struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
456 
457 	if (test_bit(KVM_ARM_VCPU_POWER_OFF, hyp_vcpu->vcpu.arch.features)) {
458 		reset_state->reset = false;
459 		hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_OFF;
460 	} else if (pkvm_hyp_vm_has_pvmfw(hyp_vm)) {
461 		if (hyp_vm->pvmfw_entry_vcpu)
462 			return -EINVAL;
463 
464 		hyp_vm->pvmfw_entry_vcpu = hyp_vcpu;
465 		reset_state->reset = true;
466 		hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING;
467 	} else {
468 		struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
469 
470 		reset_state->pc = READ_ONCE(host_vcpu->arch.ctxt.regs.pc);
471 		reset_state->r0 = READ_ONCE(host_vcpu->arch.ctxt.regs.regs[0]);
472 		reset_state->reset = true;
473 		hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING;
474 	}
475 
476 	return 0;
477 }
478 
unpin_host_vcpu(struct kvm_vcpu * host_vcpu)479 static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu)
480 {
481 	if (host_vcpu)
482 		hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1);
483 }
484 
unpin_host_sve_state(struct pkvm_hyp_vcpu * hyp_vcpu)485 static void unpin_host_sve_state(struct pkvm_hyp_vcpu *hyp_vcpu)
486 {
487 	void *sve_state;
488 
489 	if (!test_bit(KVM_ARM_VCPU_SVE, hyp_vcpu->vcpu.arch.features))
490 		return;
491 
492 	sve_state = kern_hyp_va(hyp_vcpu->vcpu.arch.sve_state);
493 	hyp_unpin_shared_mem(sve_state,
494 			     sve_state + vcpu_sve_state_size(&hyp_vcpu->vcpu));
495 }
496 
unpin_host_vcpus(struct pkvm_hyp_vcpu * hyp_vcpus[],unsigned int nr_vcpus)497 static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[],
498 			     unsigned int nr_vcpus)
499 {
500 	int i;
501 
502 	for (i = 0; i < nr_vcpus; i++) {
503 		struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vcpus[i];
504 
505 		unpin_host_vcpu(hyp_vcpu->host_vcpu);
506 		unpin_host_sve_state(hyp_vcpu);
507 	}
508 }
509 
pkvm_get_last_ran_size(void)510 static size_t pkvm_get_last_ran_size(void)
511 {
512 	return array_size(hyp_nr_cpus, sizeof(int));
513 }
514 
init_pkvm_hyp_vm(struct kvm * host_kvm,struct pkvm_hyp_vm * hyp_vm,int * last_ran,unsigned int nr_vcpus)515 static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
516 			     int *last_ran, unsigned int nr_vcpus)
517 {
518 	u64 pvmfw_load_addr = PVMFW_INVALID_LOAD_ADDR;
519 
520 	hyp_vm->host_kvm = host_kvm;
521 	hyp_vm->kvm.created_vcpus = nr_vcpus;
522 	hyp_vm->kvm.arch.vtcr = host_mmu.arch.vtcr;
523 	hyp_vm->kvm.arch.pkvm.enabled = READ_ONCE(host_kvm->arch.pkvm.enabled);
524 
525 	if (hyp_vm->kvm.arch.pkvm.enabled)
526 		pvmfw_load_addr = READ_ONCE(host_kvm->arch.pkvm.pvmfw_load_addr);
527 	hyp_vm->kvm.arch.pkvm.pvmfw_load_addr = pvmfw_load_addr;
528 
529 	hyp_vm->kvm.arch.mmu.last_vcpu_ran = (int __percpu *)last_ran;
530 	memset(last_ran, -1, pkvm_get_last_ran_size());
531 }
532 
init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu * hyp_vcpu,struct pkvm_hyp_vm * hyp_vm,struct kvm_vcpu * host_vcpu,unsigned int vcpu_idx)533 static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
534 			      struct pkvm_hyp_vm *hyp_vm,
535 			      struct kvm_vcpu *host_vcpu,
536 			      unsigned int vcpu_idx)
537 {
538 	int ret = 0;
539 
540 	if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1))
541 		return -EBUSY;
542 
543 	if (host_vcpu->vcpu_idx != vcpu_idx) {
544 		ret = -EINVAL;
545 		goto done;
546 	}
547 
548 	hyp_vcpu->host_vcpu = host_vcpu;
549 
550 	hyp_vcpu->vcpu.kvm = &hyp_vm->kvm;
551 	hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id);
552 	hyp_vcpu->vcpu.vcpu_idx = vcpu_idx;
553 
554 	hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu;
555 	hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags);
556 	hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED;
557 	hyp_vcpu->vcpu.arch.debug_ptr = &host_vcpu->arch.vcpu_debug_state;
558 
559 	pkvm_vcpu_init_features_from_host(hyp_vcpu);
560 
561 	ret = pkvm_vcpu_init_ptrauth(hyp_vcpu);
562 	if (ret)
563 		goto done;
564 
565 	ret = pkvm_vcpu_init_psci(hyp_vcpu);
566 	if (ret)
567 		goto done;
568 
569 	if (test_bit(KVM_ARM_VCPU_SVE, hyp_vcpu->vcpu.arch.features)) {
570 		size_t sve_state_size;
571 		void *sve_state;
572 
573 		hyp_vcpu->vcpu.arch.sve_state = READ_ONCE(host_vcpu->arch.sve_state);
574 		hyp_vcpu->vcpu.arch.sve_max_vl = READ_ONCE(host_vcpu->arch.sve_max_vl);
575 
576 		sve_state = kern_hyp_va(hyp_vcpu->vcpu.arch.sve_state);
577 		sve_state_size = vcpu_sve_state_size(&hyp_vcpu->vcpu);
578 
579 		if (!hyp_vcpu->vcpu.arch.sve_state || !sve_state_size ||
580 		    hyp_pin_shared_mem(sve_state, sve_state + sve_state_size)) {
581 			clear_bit(KVM_ARM_VCPU_SVE, hyp_vcpu->vcpu.arch.features);
582 			hyp_vcpu->vcpu.arch.sve_state = NULL;
583 			hyp_vcpu->vcpu.arch.sve_max_vl = 0;
584 			ret = -EINVAL;
585 			goto done;
586 		}
587 	}
588 
589 	pkvm_vcpu_init_traps(hyp_vcpu);
590 	kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu);
591 done:
592 	if (ret)
593 		unpin_host_vcpu(host_vcpu);
594 	return ret;
595 }
596 
find_free_vm_table_entry(struct kvm * host_kvm)597 static int find_free_vm_table_entry(struct kvm *host_kvm)
598 {
599 	int i;
600 
601 	for (i = 0; i < KVM_MAX_PVMS; ++i) {
602 		if (!vm_table[i])
603 			return i;
604 	}
605 
606 	return -ENOMEM;
607 }
608 
609 /*
610  * Allocate a VM table entry and insert a pointer to the new vm.
611  *
612  * Return a unique handle to the protected VM on success,
613  * negative error code on failure.
614  */
insert_vm_table_entry(struct kvm * host_kvm,struct pkvm_hyp_vm * hyp_vm)615 static pkvm_handle_t insert_vm_table_entry(struct kvm *host_kvm,
616 					   struct pkvm_hyp_vm *hyp_vm)
617 {
618 	struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu;
619 	int idx;
620 
621 	hyp_assert_lock_held(&vm_table_lock);
622 
623 	/*
624 	 * Initializing protected state might have failed, yet a malicious
625 	 * host could trigger this function. Thus, ensure that 'vm_table'
626 	 * exists.
627 	 */
628 	if (unlikely(!vm_table))
629 		return -EINVAL;
630 
631 	idx = find_free_vm_table_entry(host_kvm);
632 	if (idx < 0)
633 		return idx;
634 
635 	hyp_vm->kvm.arch.pkvm.handle = idx_to_vm_handle(idx);
636 
637 	/* VMID 0 is reserved for the host */
638 	atomic64_set(&mmu->vmid.id, idx + 1);
639 
640 	mmu->arch = &hyp_vm->kvm.arch;
641 	mmu->pgt = &hyp_vm->pgt;
642 
643 	vm_table[idx] = hyp_vm;
644 	return hyp_vm->kvm.arch.pkvm.handle;
645 }
646 
647 /*
648  * Deallocate and remove the VM table entry corresponding to the handle.
649  */
remove_vm_table_entry(pkvm_handle_t handle)650 static void remove_vm_table_entry(pkvm_handle_t handle)
651 {
652 	hyp_assert_lock_held(&vm_table_lock);
653 	vm_table[vm_handle_to_idx(handle)] = NULL;
654 }
655 
pkvm_get_hyp_vm_size(unsigned int nr_vcpus)656 static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus)
657 {
658 	return size_add(sizeof(struct pkvm_hyp_vm),
659 		size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus));
660 }
661 
map_donated_memory_noclear(unsigned long host_va,size_t size)662 static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
663 {
664 	void *va = (void *)kern_hyp_va(host_va);
665 
666 	if (!PAGE_ALIGNED(va))
667 		return NULL;
668 
669 	if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va),
670 				   PAGE_ALIGN(size) >> PAGE_SHIFT))
671 		return NULL;
672 
673 	return va;
674 }
675 
map_donated_memory(unsigned long host_va,size_t size)676 static void *map_donated_memory(unsigned long host_va, size_t size)
677 {
678 	void *va = map_donated_memory_noclear(host_va, size);
679 
680 	if (va)
681 		memset(va, 0, size);
682 
683 	return va;
684 }
685 
__unmap_donated_memory(void * va,size_t size)686 static void __unmap_donated_memory(void *va, size_t size)
687 {
688 	kvm_flush_dcache_to_poc(va, size);
689 	WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va),
690 				       PAGE_ALIGN(size) >> PAGE_SHIFT));
691 }
692 
unmap_donated_memory(void * va,size_t size)693 static void unmap_donated_memory(void *va, size_t size)
694 {
695 	if (!va)
696 		return;
697 
698 	memset(va, 0, size);
699 	__unmap_donated_memory(va, size);
700 }
701 
unmap_donated_memory_noclear(void * va,size_t size)702 static void unmap_donated_memory_noclear(void *va, size_t size)
703 {
704 	if (!va)
705 		return;
706 
707 	__unmap_donated_memory(va, size);
708 }
709 
710 /*
711  * Initialize the hypervisor copy of the protected VM state using the
712  * memory donated by the host.
713  *
714  * Unmaps the donated memory from the host at stage 2.
715  *
716  * host_kvm: A pointer to the host's struct kvm.
717  * vm_hva: The host va of the area being donated for the VM state.
718  *	   Must be page aligned.
719  * pgd_hva: The host va of the area being donated for the stage-2 PGD for
720  *	    the VM. Must be page aligned. Its size is implied by the VM's
721  *	    VTCR.
722  * last_ran_hva: The host va of the area being donated for hyp to use to track
723  *		 the most recent physical cpu on which each vcpu has run.
724  * Return a unique handle to the protected VM on success,
725  * negative error code on failure.
726  */
__pkvm_init_vm(struct kvm * host_kvm,unsigned long vm_hva,unsigned long pgd_hva,unsigned long last_ran_hva)727 int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
728 		   unsigned long pgd_hva, unsigned long last_ran_hva)
729 {
730 	struct pkvm_hyp_vm *hyp_vm = NULL;
731 	int *last_ran = NULL;
732 	size_t vm_size, pgd_size, last_ran_size;
733 	unsigned int nr_vcpus;
734 	void *pgd = NULL;
735 	int ret;
736 
737 	ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1);
738 	if (ret)
739 		return ret;
740 
741 	nr_vcpus = READ_ONCE(host_kvm->created_vcpus);
742 	if (nr_vcpus < 1) {
743 		ret = -EINVAL;
744 		goto err_unpin_kvm;
745 	}
746 
747 	vm_size = pkvm_get_hyp_vm_size(nr_vcpus);
748 	last_ran_size = pkvm_get_last_ran_size();
749 	pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.vtcr);
750 
751 	ret = -ENOMEM;
752 
753 	hyp_vm = map_donated_memory(vm_hva, vm_size);
754 	if (!hyp_vm)
755 		goto err_remove_mappings;
756 
757 	last_ran = map_donated_memory(last_ran_hva, last_ran_size);
758 	if (!last_ran)
759 		goto err_remove_mappings;
760 
761 	pgd = map_donated_memory_noclear(pgd_hva, pgd_size);
762 	if (!pgd)
763 		goto err_remove_mappings;
764 
765 	init_pkvm_hyp_vm(host_kvm, hyp_vm, last_ran, nr_vcpus);
766 
767 	hyp_spin_lock(&vm_table_lock);
768 	ret = insert_vm_table_entry(host_kvm, hyp_vm);
769 	if (ret < 0)
770 		goto err_unlock;
771 
772 	ret = kvm_guest_prepare_stage2(hyp_vm, pgd);
773 	if (ret)
774 		goto err_remove_vm_table_entry;
775 	hyp_spin_unlock(&vm_table_lock);
776 
777 	return hyp_vm->kvm.arch.pkvm.handle;
778 
779 err_remove_vm_table_entry:
780 	remove_vm_table_entry(hyp_vm->kvm.arch.pkvm.handle);
781 err_unlock:
782 	hyp_spin_unlock(&vm_table_lock);
783 err_remove_mappings:
784 	unmap_donated_memory(hyp_vm, vm_size);
785 	unmap_donated_memory(last_ran, last_ran_size);
786 	unmap_donated_memory(pgd, pgd_size);
787 err_unpin_kvm:
788 	hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
789 	return ret;
790 }
791 
792 /*
793  * Initialize the hypervisor copy of the protected vCPU state using the
794  * memory donated by the host.
795  *
796  * handle: The handle for the protected vm.
797  * host_vcpu: A pointer to the corresponding host vcpu.
798  * vcpu_hva: The host va of the area being donated for the vcpu state.
799  *	     Must be page aligned. The size of the area must be equal to
800  *	     the page-aligned size of 'struct pkvm_hyp_vcpu'.
801  * Return 0 on success, negative error code on failure.
802  */
__pkvm_init_vcpu(pkvm_handle_t handle,struct kvm_vcpu * host_vcpu,unsigned long vcpu_hva)803 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
804 		     unsigned long vcpu_hva)
805 {
806 	struct pkvm_hyp_vcpu *hyp_vcpu;
807 	struct pkvm_hyp_vm *hyp_vm;
808 	unsigned int idx;
809 	int ret;
810 
811 	hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu));
812 	if (!hyp_vcpu)
813 		return -ENOMEM;
814 
815 	hyp_spin_lock(&vm_table_lock);
816 
817 	hyp_vm = get_vm_by_handle(handle);
818 	if (!hyp_vm) {
819 		ret = -ENOENT;
820 		goto unlock;
821 	}
822 
823 	idx = hyp_vm->nr_vcpus;
824 	if (idx >= hyp_vm->kvm.created_vcpus) {
825 		ret = -EINVAL;
826 		goto unlock;
827 	}
828 
829 	ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx);
830 	if (ret)
831 		goto unlock;
832 
833 	hyp_vm->vcpus[idx] = hyp_vcpu;
834 	hyp_vm->nr_vcpus++;
835 unlock:
836 	hyp_spin_unlock(&vm_table_lock);
837 
838 	if (ret)
839 		unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu));
840 
841 	return ret;
842 }
843 
844 static void
teardown_donated_memory(struct kvm_hyp_memcache * mc,void * addr,size_t size)845 teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size)
846 {
847 	void *start;
848 
849 	size = PAGE_ALIGN(size);
850 	memset(addr, 0, size);
851 
852 	for (start = addr; start < addr + size; start += PAGE_SIZE)
853 		push_hyp_memcache(mc, start, hyp_virt_to_phys);
854 
855 	unmap_donated_memory_noclear(addr, size);
856 }
857 
__pkvm_start_teardown_vm(pkvm_handle_t handle)858 int __pkvm_start_teardown_vm(pkvm_handle_t handle)
859 {
860 	struct pkvm_hyp_vm *hyp_vm;
861 	int ret = 0;
862 
863 	hyp_spin_lock(&vm_table_lock);
864 	hyp_vm = get_vm_by_handle(handle);
865 	if (!hyp_vm) {
866 		ret = -ENOENT;
867 		goto unlock;
868 	} else if (WARN_ON(hyp_page_count(hyp_vm))) {
869 		ret = -EBUSY;
870 		goto unlock;
871 	} else if (hyp_vm->is_dying) {
872 		ret = -EINVAL;
873 		goto unlock;
874 	}
875 
876 	hyp_vm->is_dying = true;
877 
878 unlock:
879 	hyp_spin_unlock(&vm_table_lock);
880 
881 	return ret;
882 }
883 
__pkvm_finalize_teardown_vm(pkvm_handle_t handle)884 int __pkvm_finalize_teardown_vm(pkvm_handle_t handle)
885 {
886 	struct kvm_hyp_memcache *mc, *stage2_mc;
887 	size_t vm_size, last_ran_size;
888 	int __percpu *last_vcpu_ran;
889 	struct pkvm_hyp_vm *hyp_vm;
890 	struct kvm *host_kvm;
891 	unsigned int idx;
892 	int err;
893 
894 	hyp_spin_lock(&vm_table_lock);
895 	hyp_vm = get_vm_by_handle(handle);
896 	if (!hyp_vm) {
897 		err = -ENOENT;
898 		goto err_unlock;
899 	} else if (!hyp_vm->is_dying) {
900 		err = -EBUSY;
901 		goto err_unlock;
902 	}
903 
904 	host_kvm = hyp_vm->host_kvm;
905 
906 	/* Ensure the VMID is clean before it can be reallocated */
907 	__kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu);
908 	remove_vm_table_entry(handle);
909 	hyp_spin_unlock(&vm_table_lock);
910 
911 	mc = &host_kvm->arch.pkvm.teardown_mc;
912 	stage2_mc = &host_kvm->arch.pkvm.teardown_stage2_mc;
913 
914 	destroy_hyp_vm_pgt(hyp_vm);
915 	drain_hyp_pool(hyp_vm, stage2_mc);
916 	unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus);
917 
918 	/* Push the metadata pages to the teardown memcache */
919 	for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) {
920 		struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx];
921 		struct kvm_hyp_memcache *vcpu_mc;
922 		void *addr;
923 
924 		vcpu_mc = &hyp_vcpu->vcpu.arch.pkvm_memcache;
925 		while (vcpu_mc->nr_pages) {
926 			addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt);
927 			push_hyp_memcache(stage2_mc, addr, hyp_virt_to_phys);
928 			unmap_donated_memory_noclear(addr, PAGE_SIZE);
929 		}
930 
931 		teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu));
932 	}
933 
934 	last_vcpu_ran = hyp_vm->kvm.arch.mmu.last_vcpu_ran;
935 	last_ran_size = pkvm_get_last_ran_size();
936 	teardown_donated_memory(mc, (__force void *)last_vcpu_ran,
937 				last_ran_size);
938 
939 	vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus);
940 	teardown_donated_memory(mc, hyp_vm, vm_size);
941 	hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
942 	return 0;
943 
944 err_unlock:
945 	hyp_spin_unlock(&vm_table_lock);
946 	return err;
947 }
948 
pkvm_load_pvmfw_pages(struct pkvm_hyp_vm * vm,u64 ipa,phys_addr_t phys,u64 size)949 int pkvm_load_pvmfw_pages(struct pkvm_hyp_vm *vm, u64 ipa, phys_addr_t phys,
950 			  u64 size)
951 {
952 	struct kvm_protected_vm *pkvm = &vm->kvm.arch.pkvm;
953 	u64 npages, offset = ipa - pkvm->pvmfw_load_addr;
954 	void *src = hyp_phys_to_virt(pvmfw_base) + offset;
955 
956 	if (offset >= pvmfw_size)
957 		return -EINVAL;
958 
959 	size = min(size, pvmfw_size - offset);
960 	if (!PAGE_ALIGNED(size) || !PAGE_ALIGNED(src))
961 		return -EINVAL;
962 
963 	npages = size >> PAGE_SHIFT;
964 	while (npages--) {
965 		/*
966 		 * No need for cache maintenance here, as the pgtable code will
967 		 * take care of this when installing the pte in the guest's
968 		 * stage-2 page table.
969 		 */
970 		memcpy(hyp_fixmap_map(phys), src, PAGE_SIZE);
971 		hyp_fixmap_unmap();
972 
973 		src += PAGE_SIZE;
974 		phys += PAGE_SIZE;
975 	}
976 
977 	return 0;
978 }
979 
pkvm_poison_pvmfw_pages(void)980 void pkvm_poison_pvmfw_pages(void)
981 {
982 	u64 npages = pvmfw_size >> PAGE_SHIFT;
983 	phys_addr_t addr = pvmfw_base;
984 
985 	while (npages--) {
986 		hyp_poison_page(addr);
987 		addr += PAGE_SIZE;
988 	}
989 }
990 
991 /*
992  * This function sets the registers on the vcpu to their architecturally defined
993  * reset values.
994  *
995  * Note: Can only be called by the vcpu on itself, after it has been turned on.
996  */
pkvm_reset_vcpu(struct pkvm_hyp_vcpu * hyp_vcpu)997 void pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
998 {
999 	struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state;
1000 	struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
1001 
1002 	WARN_ON(!reset_state->reset);
1003 
1004 	pkvm_vcpu_init_ptrauth(hyp_vcpu);
1005 	kvm_reset_vcpu_core(&hyp_vcpu->vcpu);
1006 	kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu);
1007 
1008 	/* Must be done after reseting sys registers. */
1009 	kvm_reset_vcpu_psci(&hyp_vcpu->vcpu, reset_state);
1010 	if (hyp_vm->pvmfw_entry_vcpu == hyp_vcpu) {
1011 		struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
1012 		u64 entry = hyp_vm->kvm.arch.pkvm.pvmfw_load_addr;
1013 		int i;
1014 
1015 		/* X0 - X14 provided by the VMM (preserved) */
1016 		for (i = 0; i <= 14; ++i) {
1017 			u64 val = vcpu_get_reg(host_vcpu, i);
1018 
1019 			vcpu_set_reg(&hyp_vcpu->vcpu, i, val);
1020 		}
1021 
1022 		/* X15: Boot protocol version */
1023 		vcpu_set_reg(&hyp_vcpu->vcpu, 15, 0);
1024 
1025 		/* PC: IPA of pvmfw base */
1026 		*vcpu_pc(&hyp_vcpu->vcpu) = entry;
1027 		hyp_vm->pvmfw_entry_vcpu = NULL;
1028 
1029 		/* Auto enroll MMIO guard */
1030 		set_bit(KVM_ARCH_FLAG_MMIO_GUARD, &hyp_vm->kvm.arch.flags);
1031 	}
1032 
1033 	reset_state->reset = false;
1034 
1035 	hyp_vcpu->exit_code = 0;
1036 
1037 	WARN_ON(hyp_vcpu->power_state != PSCI_0_2_AFFINITY_LEVEL_ON_PENDING);
1038 	WRITE_ONCE(hyp_vcpu->vcpu.arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1039 	WRITE_ONCE(hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_ON);
1040 }
1041 
pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm * hyp_vm,u64 mpidr)1042 struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm,
1043 					     u64 mpidr)
1044 {
1045 	int i;
1046 
1047 	mpidr &= MPIDR_HWID_BITMASK;
1048 
1049 	for (i = 0; i < hyp_vm->nr_vcpus; i++) {
1050 		struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[i];
1051 
1052 		if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu))
1053 			return hyp_vcpu;
1054 	}
1055 
1056 	return NULL;
1057 }
1058 
1059 /*
1060  * Returns true if the hypervisor has handled the PSCI call, and control should
1061  * go back to the guest, or false if the host needs to do some additional work
1062  * (i.e., wake up the vcpu).
1063  */
pvm_psci_vcpu_on(struct pkvm_hyp_vcpu * hyp_vcpu)1064 static bool pvm_psci_vcpu_on(struct pkvm_hyp_vcpu *hyp_vcpu)
1065 {
1066 	struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
1067 	struct vcpu_reset_state *reset_state;
1068 	struct pkvm_hyp_vcpu *target;
1069 	unsigned long cpu_id, ret;
1070 	int power_state;
1071 
1072 	cpu_id = smccc_get_arg1(&hyp_vcpu->vcpu);
1073 	if (!kvm_psci_valid_affinity(&hyp_vcpu->vcpu, cpu_id)) {
1074 		ret = PSCI_RET_INVALID_PARAMS;
1075 		goto error;
1076 	}
1077 
1078 	target = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id);
1079 	if (!target) {
1080 		ret = PSCI_RET_INVALID_PARAMS;
1081 		goto error;
1082 	}
1083 
1084 	/*
1085 	 * Make sure the requested vcpu is not on to begin with.
1086 	 * Atomic to avoid race between vcpus trying to power on the same vcpu.
1087 	 */
1088 	power_state = cmpxchg(&target->power_state,
1089 			      PSCI_0_2_AFFINITY_LEVEL_OFF,
1090 			      PSCI_0_2_AFFINITY_LEVEL_ON_PENDING);
1091 	switch (power_state) {
1092 	case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING:
1093 		ret = PSCI_RET_ON_PENDING;
1094 		goto error;
1095 	case PSCI_0_2_AFFINITY_LEVEL_ON:
1096 		ret = PSCI_RET_ALREADY_ON;
1097 		goto error;
1098 	case PSCI_0_2_AFFINITY_LEVEL_OFF:
1099 		break;
1100 	default:
1101 		ret = PSCI_RET_INTERNAL_FAILURE;
1102 		goto error;
1103 	}
1104 
1105 	reset_state = &target->vcpu.arch.reset_state;
1106 	reset_state->pc = smccc_get_arg2(&hyp_vcpu->vcpu);
1107 	reset_state->r0 = smccc_get_arg3(&hyp_vcpu->vcpu);
1108 	/* Propagate caller endianness */
1109 	reset_state->be = kvm_vcpu_is_be(&hyp_vcpu->vcpu);
1110 	reset_state->reset = true;
1111 
1112 	/*
1113 	 * Return to the host, which should make the KVM_REQ_VCPU_RESET request
1114 	 * as well as kvm_vcpu_wake_up() to schedule the vcpu.
1115 	 */
1116 	return false;
1117 
1118 error:
1119 	/* If there's an error go back straight to the guest. */
1120 	smccc_set_retval(&hyp_vcpu->vcpu, ret, 0, 0, 0);
1121 	return true;
1122 }
1123 
pvm_psci_vcpu_affinity_info(struct pkvm_hyp_vcpu * hyp_vcpu)1124 static bool pvm_psci_vcpu_affinity_info(struct pkvm_hyp_vcpu *hyp_vcpu)
1125 {
1126 	unsigned long target_affinity_mask, target_affinity, lowest_affinity_level;
1127 	struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
1128 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1129 	unsigned long mpidr, ret;
1130 	int i, matching_cpus = 0;
1131 
1132 	target_affinity = smccc_get_arg1(vcpu);
1133 	lowest_affinity_level = smccc_get_arg2(vcpu);
1134 	if (!kvm_psci_valid_affinity(vcpu, target_affinity)) {
1135 		ret = PSCI_RET_INVALID_PARAMS;
1136 		goto done;
1137 	}
1138 
1139 	/* Determine target affinity mask */
1140 	target_affinity_mask = psci_affinity_mask(lowest_affinity_level);
1141 	if (!target_affinity_mask) {
1142 		ret = PSCI_RET_INVALID_PARAMS;
1143 		goto done;
1144 	}
1145 
1146 	/* Ignore other bits of target affinity */
1147 	target_affinity &= target_affinity_mask;
1148 	ret = PSCI_0_2_AFFINITY_LEVEL_OFF;
1149 
1150 	/*
1151 	 * If at least one vcpu matching target affinity is ON then return ON,
1152 	 * then if at least one is PENDING_ON then return PENDING_ON.
1153 	 * Otherwise, return OFF.
1154 	 */
1155 	for (i = 0; i < hyp_vm->nr_vcpus; i++) {
1156 		struct pkvm_hyp_vcpu *target = hyp_vm->vcpus[i];
1157 
1158 		mpidr = kvm_vcpu_get_mpidr_aff(&target->vcpu);
1159 
1160 		if ((mpidr & target_affinity_mask) == target_affinity) {
1161 			int power_state;
1162 
1163 			matching_cpus++;
1164 			power_state = READ_ONCE(target->power_state);
1165 			switch (power_state) {
1166 			case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING:
1167 				ret = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING;
1168 				break;
1169 			case PSCI_0_2_AFFINITY_LEVEL_ON:
1170 				ret = PSCI_0_2_AFFINITY_LEVEL_ON;
1171 				goto done;
1172 			case PSCI_0_2_AFFINITY_LEVEL_OFF:
1173 				break;
1174 			default:
1175 				ret = PSCI_RET_INTERNAL_FAILURE;
1176 				goto done;
1177 			}
1178 		}
1179 	}
1180 
1181 	if (!matching_cpus)
1182 		ret = PSCI_RET_INVALID_PARAMS;
1183 
1184 done:
1185 	/* Nothing to be handled by the host. Go back to the guest. */
1186 	smccc_set_retval(vcpu, ret, 0, 0, 0);
1187 	return true;
1188 }
1189 
1190 /*
1191  * Returns true if the hypervisor has handled the PSCI call, and control should
1192  * go back to the guest, or false if the host needs to do some additional work
1193  * (e.g., turn off and update vcpu scheduling status).
1194  */
pvm_psci_vcpu_off(struct pkvm_hyp_vcpu * hyp_vcpu)1195 static bool pvm_psci_vcpu_off(struct pkvm_hyp_vcpu *hyp_vcpu)
1196 {
1197 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1198 
1199 	WARN_ON(vcpu->arch.mp_state.mp_state == KVM_MP_STATE_STOPPED);
1200 	WARN_ON(hyp_vcpu->power_state != PSCI_0_2_AFFINITY_LEVEL_ON);
1201 
1202 	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
1203 	WRITE_ONCE(hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_OFF);
1204 
1205 	/* Return to the host so that it can finish powering off the vcpu. */
1206 	return false;
1207 }
1208 
pvm_psci_version(struct pkvm_hyp_vcpu * hyp_vcpu)1209 static bool pvm_psci_version(struct pkvm_hyp_vcpu *hyp_vcpu)
1210 {
1211 	/* Nothing to be handled by the host. Go back to the guest. */
1212 	smccc_set_retval(&hyp_vcpu->vcpu, KVM_ARM_PSCI_1_1, 0, 0, 0);
1213 	return true;
1214 }
1215 
pvm_psci_not_supported(struct pkvm_hyp_vcpu * hyp_vcpu)1216 static bool pvm_psci_not_supported(struct pkvm_hyp_vcpu *hyp_vcpu)
1217 {
1218 	/* Nothing to be handled by the host. Go back to the guest. */
1219 	smccc_set_retval(&hyp_vcpu->vcpu, PSCI_RET_NOT_SUPPORTED, 0, 0, 0);
1220 	return true;
1221 }
1222 
pvm_psci_features(struct pkvm_hyp_vcpu * hyp_vcpu)1223 static bool pvm_psci_features(struct pkvm_hyp_vcpu *hyp_vcpu)
1224 {
1225 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1226 	u32 feature = smccc_get_arg1(vcpu);
1227 	unsigned long val;
1228 
1229 	switch (feature) {
1230 	case PSCI_0_2_FN_PSCI_VERSION:
1231 	case PSCI_0_2_FN_CPU_SUSPEND:
1232 	case PSCI_0_2_FN64_CPU_SUSPEND:
1233 	case PSCI_0_2_FN_CPU_OFF:
1234 	case PSCI_0_2_FN_CPU_ON:
1235 	case PSCI_0_2_FN64_CPU_ON:
1236 	case PSCI_0_2_FN_AFFINITY_INFO:
1237 	case PSCI_0_2_FN64_AFFINITY_INFO:
1238 	case PSCI_0_2_FN_SYSTEM_OFF:
1239 	case PSCI_0_2_FN_SYSTEM_RESET:
1240 	case PSCI_1_0_FN_PSCI_FEATURES:
1241 	case PSCI_1_1_FN_SYSTEM_RESET2:
1242 	case PSCI_1_1_FN64_SYSTEM_RESET2:
1243 	case ARM_SMCCC_VERSION_FUNC_ID:
1244 		val = PSCI_RET_SUCCESS;
1245 		break;
1246 	default:
1247 		val = PSCI_RET_NOT_SUPPORTED;
1248 		break;
1249 	}
1250 
1251 	/* Nothing to be handled by the host. Go back to the guest. */
1252 	smccc_set_retval(vcpu, val, 0, 0, 0);
1253 	return true;
1254 }
1255 
pkvm_handle_psci(struct pkvm_hyp_vcpu * hyp_vcpu)1256 static bool pkvm_handle_psci(struct pkvm_hyp_vcpu *hyp_vcpu)
1257 {
1258 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1259 	u32 psci_fn = smccc_get_function(vcpu);
1260 
1261 	switch (psci_fn) {
1262 	case PSCI_0_2_FN_CPU_ON:
1263 		kvm_psci_narrow_to_32bit(vcpu);
1264 		fallthrough;
1265 	case PSCI_0_2_FN64_CPU_ON:
1266 		return pvm_psci_vcpu_on(hyp_vcpu);
1267 	case PSCI_0_2_FN_CPU_OFF:
1268 		return pvm_psci_vcpu_off(hyp_vcpu);
1269 	case PSCI_0_2_FN_AFFINITY_INFO:
1270 		kvm_psci_narrow_to_32bit(vcpu);
1271 		fallthrough;
1272 	case PSCI_0_2_FN64_AFFINITY_INFO:
1273 		return pvm_psci_vcpu_affinity_info(hyp_vcpu);
1274 	case PSCI_0_2_FN_PSCI_VERSION:
1275 		return pvm_psci_version(hyp_vcpu);
1276 	case PSCI_1_0_FN_PSCI_FEATURES:
1277 		return pvm_psci_features(hyp_vcpu);
1278 	case PSCI_0_2_FN_SYSTEM_RESET:
1279 	case PSCI_0_2_FN_CPU_SUSPEND:
1280 	case PSCI_0_2_FN64_CPU_SUSPEND:
1281 	case PSCI_0_2_FN_SYSTEM_OFF:
1282 	case PSCI_1_1_FN_SYSTEM_RESET2:
1283 	case PSCI_1_1_FN64_SYSTEM_RESET2:
1284 		return false; /* Handled by the host. */
1285 	default:
1286 		break;
1287 	}
1288 
1289 	return pvm_psci_not_supported(hyp_vcpu);
1290 }
1291 
__pkvm_memshare_page_req(struct pkvm_hyp_vcpu * hyp_vcpu,u64 ipa)1292 static u64 __pkvm_memshare_page_req(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa)
1293 {
1294 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1295 	u64 elr;
1296 
1297 	/* Fake up a data abort (Level 3 translation fault on write) */
1298 	vcpu->arch.fault.esr_el2 = (u32)ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT |
1299 				   ESR_ELx_WNR | ESR_ELx_FSC_FAULT |
1300 				   FIELD_PREP(ESR_ELx_FSC_LEVEL, 3);
1301 
1302 	/* Shuffle the IPA around into the HPFAR */
1303 	vcpu->arch.fault.hpfar_el2 = (ipa >> 8) & HPFAR_MASK;
1304 
1305 	/* This is a virtual address. 0's good. Let's go with 0. */
1306 	vcpu->arch.fault.far_el2 = 0;
1307 
1308 	/* Rewind the ELR so we return to the HVC once the IPA is mapped */
1309 	elr = read_sysreg(elr_el2);
1310 	elr -= 4;
1311 	write_sysreg(elr, elr_el2);
1312 
1313 	return ARM_EXCEPTION_TRAP;
1314 }
1315 
pkvm_memshare_call(struct pkvm_hyp_vcpu * hyp_vcpu,u64 * exit_code)1316 static bool pkvm_memshare_call(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code)
1317 {
1318 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1319 	u64 ipa = smccc_get_arg1(vcpu);
1320 	u64 arg2 = smccc_get_arg2(vcpu);
1321 	u64 arg3 = smccc_get_arg3(vcpu);
1322 	int err;
1323 
1324 	if (arg2 || arg3)
1325 		goto out_guest_err;
1326 
1327 	err = __pkvm_guest_share_host(hyp_vcpu, ipa);
1328 	switch (err) {
1329 	case 0:
1330 		/* Success! Now tell the host. */
1331 		goto out_host;
1332 	case -EFAULT:
1333 		/*
1334 		 * Convert the exception into a data abort so that the page
1335 		 * being shared is mapped into the guest next time.
1336 		 */
1337 		*exit_code = __pkvm_memshare_page_req(hyp_vcpu, ipa);
1338 		goto out_host;
1339 	}
1340 
1341 out_guest_err:
1342 	smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0);
1343 	return true;
1344 
1345 out_host:
1346 	return false;
1347 }
1348 
pkvm_memunshare_call(struct pkvm_hyp_vcpu * hyp_vcpu)1349 static bool pkvm_memunshare_call(struct pkvm_hyp_vcpu *hyp_vcpu)
1350 {
1351 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1352 	u64 ipa = smccc_get_arg1(vcpu);
1353 	u64 arg2 = smccc_get_arg2(vcpu);
1354 	u64 arg3 = smccc_get_arg3(vcpu);
1355 	int err;
1356 
1357 	if (arg2 || arg3)
1358 		goto out_guest_err;
1359 
1360 	err = __pkvm_guest_unshare_host(hyp_vcpu, ipa);
1361 	if (err)
1362 		goto out_guest_err;
1363 
1364 	return false;
1365 
1366 out_guest_err:
1367 	smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0);
1368 	return true;
1369 }
1370 
pkvm_meminfo_call(struct pkvm_hyp_vcpu * hyp_vcpu)1371 static bool pkvm_meminfo_call(struct pkvm_hyp_vcpu *hyp_vcpu)
1372 {
1373 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1374 	u64 arg1 = smccc_get_arg1(vcpu);
1375 	u64 arg2 = smccc_get_arg2(vcpu);
1376 	u64 arg3 = smccc_get_arg3(vcpu);
1377 
1378 	if (arg1 || arg2 || arg3)
1379 		goto out_guest_err;
1380 
1381 	smccc_set_retval(vcpu, PAGE_SIZE, 0, 0, 0);
1382 	return true;
1383 
1384 out_guest_err:
1385 	smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0);
1386 	return true;
1387 }
1388 
pkvm_memrelinquish_call(struct pkvm_hyp_vcpu * hyp_vcpu)1389 static bool pkvm_memrelinquish_call(struct pkvm_hyp_vcpu *hyp_vcpu)
1390 {
1391 	struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
1392 	u64 ipa = smccc_get_arg1(vcpu);
1393 	u64 arg2 = smccc_get_arg2(vcpu);
1394 	u64 arg3 = smccc_get_arg3(vcpu);
1395 	u64 pa = 0;
1396 	int ret;
1397 
1398 	if (arg2 || arg3)
1399 		goto out_guest_err;
1400 
1401 	ret = __pkvm_guest_relinquish_to_host(hyp_vcpu, ipa, &pa);
1402 	if (ret)
1403 		goto out_guest_err;
1404 
1405 	if (pa != 0) {
1406 		/* Now pass to host. */
1407 		return false;
1408 	}
1409 
1410 	/* This was a NOP as no page was actually mapped at the IPA. */
1411 	smccc_set_retval(vcpu, 0, 0, 0, 0);
1412 	return true;
1413 
1414 out_guest_err:
1415 	smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0);
1416 	return true;
1417 }
1418 
pkvm_install_ioguard_page(struct pkvm_hyp_vcpu * hyp_vcpu,u64 * exit_code)1419 static bool pkvm_install_ioguard_page(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code)
1420 {
1421 	u64 retval = SMCCC_RET_SUCCESS;
1422 	u64 ipa = smccc_get_arg1(&hyp_vcpu->vcpu);
1423 	int ret;
1424 
1425 	ret = __pkvm_install_ioguard_page(hyp_vcpu, ipa);
1426 	if (ret == -ENOMEM) {
1427 		/*
1428 		 * We ran out of memcache, let's ask for more. Cancel
1429 		 * the effects of the HVC that took us here, and
1430 		 * forward the hypercall to the host for page donation
1431 		 * purposes.
1432 		 */
1433 		write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR);
1434 		return false;
1435 	}
1436 
1437 	if (ret)
1438 		retval = SMCCC_RET_INVALID_PARAMETER;
1439 
1440 	smccc_set_retval(&hyp_vcpu->vcpu, retval, 0, 0, 0);
1441 	return true;
1442 }
1443 
1444 bool smccc_trng_available;
1445 
pkvm_forward_trng(struct kvm_vcpu * vcpu)1446 static bool pkvm_forward_trng(struct kvm_vcpu *vcpu)
1447 {
1448 	u32 fn = smccc_get_function(vcpu);
1449 	struct arm_smccc_res res;
1450 	unsigned long arg1 = 0;
1451 
1452 	/*
1453 	 * Forward TRNG calls to EL3, as we can't trust the host to handle
1454 	 * these for us.
1455 	 */
1456 	switch (fn) {
1457 	case ARM_SMCCC_TRNG_FEATURES:
1458 	case ARM_SMCCC_TRNG_RND32:
1459 	case ARM_SMCCC_TRNG_RND64:
1460 		arg1 = smccc_get_arg1(vcpu);
1461 		fallthrough;
1462 	case ARM_SMCCC_TRNG_VERSION:
1463 	case ARM_SMCCC_TRNG_GET_UUID:
1464 		arm_smccc_1_1_smc(fn, arg1, &res);
1465 		smccc_set_retval(vcpu, res.a0, res.a1, res.a2, res.a3);
1466 		memzero_explicit(&res, sizeof(res));
1467 		break;
1468 	}
1469 
1470 	return true;
1471 }
1472 
1473 /*
1474  * Handler for protected VM HVC calls.
1475  *
1476  * Returns true if the hypervisor has handled the exit, and control should go
1477  * back to the guest, or false if it hasn't.
1478  */
kvm_handle_pvm_hvc64(struct kvm_vcpu * vcpu,u64 * exit_code)1479 bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code)
1480 {
1481 	u64 val[4] = { SMCCC_RET_NOT_SUPPORTED };
1482 	u32 fn = smccc_get_function(vcpu);
1483 	struct pkvm_hyp_vcpu *hyp_vcpu;
1484 
1485 	hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu);
1486 
1487 	switch (fn) {
1488 	case ARM_SMCCC_VERSION_FUNC_ID:
1489 		/* Nothing to be handled by the host. Go back to the guest. */
1490 		val[0] = ARM_SMCCC_VERSION_1_1;
1491 		break;
1492 	case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID:
1493 		val[0] = ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_0;
1494 		val[1] = ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_1;
1495 		val[2] = ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_2;
1496 		val[3] = ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_3;
1497 		break;
1498 	case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID:
1499 		val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES);
1500 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO);
1501 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_SHARE);
1502 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_UNSHARE);
1503 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_RELINQUISH);
1504 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MMIO_GUARD_INFO);
1505 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MMIO_GUARD_ENROLL);
1506 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MMIO_GUARD_MAP);
1507 		val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MMIO_GUARD_UNMAP);
1508 		break;
1509 	case ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_ENROLL_FUNC_ID:
1510 		set_bit(KVM_ARCH_FLAG_MMIO_GUARD, &vcpu->kvm->arch.flags);
1511 		val[0] = SMCCC_RET_SUCCESS;
1512 		break;
1513 	case ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_MAP_FUNC_ID:
1514 		return pkvm_install_ioguard_page(hyp_vcpu, exit_code);
1515 	case ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_UNMAP_FUNC_ID:
1516 		if (__pkvm_remove_ioguard_page(hyp_vcpu, vcpu_get_reg(vcpu, 1)))
1517 			val[0] = SMCCC_RET_INVALID_PARAMETER;
1518 		else
1519 			val[0] = SMCCC_RET_SUCCESS;
1520 		break;
1521 	case ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_INFO_FUNC_ID:
1522 	case ARM_SMCCC_VENDOR_HYP_KVM_HYP_MEMINFO_FUNC_ID:
1523 		return pkvm_meminfo_call(hyp_vcpu);
1524 	case ARM_SMCCC_VENDOR_HYP_KVM_MEM_SHARE_FUNC_ID:
1525 		return pkvm_memshare_call(hyp_vcpu, exit_code);
1526 	case ARM_SMCCC_VENDOR_HYP_KVM_MEM_UNSHARE_FUNC_ID:
1527 		return pkvm_memunshare_call(hyp_vcpu);
1528 	case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID:
1529 		return pkvm_memrelinquish_call(hyp_vcpu);
1530 	case ARM_SMCCC_TRNG_VERSION ... ARM_SMCCC_TRNG_RND32:
1531 	case ARM_SMCCC_TRNG_RND64:
1532 		if (smccc_trng_available)
1533 			return pkvm_forward_trng(vcpu);
1534 		break;
1535 	default:
1536 		return pkvm_handle_psci(hyp_vcpu);
1537 	}
1538 
1539 	smccc_set_retval(vcpu, val[0], val[1], val[2], val[3]);
1540 	return true;
1541 }
1542 
1543 /*
1544  * Handler for non-protected VM HVC calls.
1545  *
1546  * Returns true if the hypervisor has handled the exit, and control should go
1547  * back to the guest, or false if it hasn't.
1548  */
kvm_hyp_handle_hvc64(struct kvm_vcpu * vcpu,u64 * exit_code)1549 bool kvm_hyp_handle_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code)
1550 {
1551 	u32 fn = smccc_get_function(vcpu);
1552 	struct pkvm_hyp_vcpu *hyp_vcpu;
1553 
1554 	hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu);
1555 
1556 	switch (fn) {
1557 	case ARM_SMCCC_VENDOR_HYP_KVM_HYP_MEMINFO_FUNC_ID:
1558 		return pkvm_meminfo_call(hyp_vcpu);
1559 	case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID:
1560 		return pkvm_memrelinquish_call(hyp_vcpu);
1561 	}
1562 
1563 	return false;
1564 }
1565