1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/reset.c
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
9 */
10
11 #include <linux/errno.h>
12 #include <linux/kernel.h>
13 #include <linux/kvm_host.h>
14 #include <linux/kvm.h>
15 #include <linux/hw_breakpoint.h>
16 #include <linux/slab.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19
20 #include <kvm/arm_arch_timer.h>
21
22 #include <asm/cpufeature.h>
23 #include <asm/cputype.h>
24 #include <asm/fpsimd.h>
25 #include <asm/ptrace.h>
26 #include <asm/kvm_arm.h>
27 #include <asm/kvm_asm.h>
28 #include <asm/kvm_emulate.h>
29 #include <asm/kvm_mmu.h>
30 #include <asm/virt.h>
31
32 /* Maximum phys_shift supported for any VM on this host */
33 static u32 kvm_ipa_limit;
34
35 unsigned int kvm_sve_max_vl;
36 unsigned int kvm_host_sve_max_vl;
37
kvm_arm_init_sve(void)38 int kvm_arm_init_sve(void)
39 {
40 if (system_supports_sve()) {
41 kvm_sve_max_vl = sve_max_virtualisable_vl();
42 kvm_host_sve_max_vl = sve_max_vl();
43
44 /*
45 * The get_sve_reg()/set_sve_reg() ioctl interface will need
46 * to be extended with multiple register slice support in
47 * order to support vector lengths greater than
48 * VL_ARCH_MAX:
49 */
50 if (WARN_ON(kvm_sve_max_vl > VL_ARCH_MAX))
51 kvm_sve_max_vl = VL_ARCH_MAX;
52
53 /*
54 * Don't even try to make use of vector lengths that
55 * aren't available on all CPUs, for now:
56 */
57 if (kvm_sve_max_vl < sve_max_vl())
58 pr_warn("KVM: SVE vector length for guests limited to %u bytes\n",
59 kvm_sve_max_vl);
60 }
61
62 return 0;
63 }
64
kvm_vcpu_enable_sve(struct kvm_vcpu * vcpu)65 static int kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu)
66 {
67 if (!system_supports_sve())
68 return -EINVAL;
69
70 vcpu->arch.sve_max_vl = kvm_sve_max_vl;
71
72 /*
73 * Userspace can still customize the vector lengths by writing
74 * KVM_REG_ARM64_SVE_VLS. Allocation is deferred until
75 * kvm_arm_vcpu_finalize(), which freezes the configuration.
76 */
77 vcpu_set_flag(vcpu, GUEST_HAS_SVE);
78
79 return 0;
80 }
81
82 /*
83 * Finalize vcpu's maximum SVE vector length, allocating
84 * vcpu->arch.sve_state as necessary.
85 */
kvm_vcpu_finalize_sve(struct kvm_vcpu * vcpu)86 static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu)
87 {
88 void *buf;
89 unsigned int vl;
90 size_t reg_sz;
91 int ret;
92
93 vl = vcpu->arch.sve_max_vl;
94
95 /*
96 * Responsibility for these properties is shared between
97 * kvm_arm_init_sve(), kvm_vcpu_enable_sve() and
98 * set_sve_vls(). Double-check here just to be sure:
99 */
100 if (WARN_ON(!sve_vl_valid(vl) || vl > sve_max_virtualisable_vl() ||
101 vl > VL_ARCH_MAX))
102 return -EIO;
103
104 reg_sz = vcpu_sve_state_size(vcpu);
105 buf = kzalloc(reg_sz, GFP_KERNEL_ACCOUNT);
106 if (!buf)
107 return -ENOMEM;
108
109 ret = kvm_share_hyp(buf, buf + reg_sz);
110 if (ret) {
111 kfree(buf);
112 return ret;
113 }
114
115 vcpu->arch.sve_state = buf;
116 vcpu_set_flag(vcpu, VCPU_SVE_FINALIZED);
117 return 0;
118 }
119
kvm_arm_vcpu_finalize(struct kvm_vcpu * vcpu,int feature)120 int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature)
121 {
122 switch (feature) {
123 case KVM_ARM_VCPU_SVE:
124 if (!vcpu_has_sve(vcpu))
125 return -EINVAL;
126
127 if (kvm_arm_vcpu_sve_finalized(vcpu))
128 return -EPERM;
129
130 return kvm_vcpu_finalize_sve(vcpu);
131 }
132
133 return -EINVAL;
134 }
135
kvm_arm_vcpu_is_finalized(struct kvm_vcpu * vcpu)136 bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu)
137 {
138 if (vcpu_has_sve(vcpu) && !kvm_arm_vcpu_sve_finalized(vcpu))
139 return false;
140
141 return true;
142 }
143
kvm_arm_vcpu_destroy(struct kvm_vcpu * vcpu)144 void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu)
145 {
146 void *sve_state = vcpu->arch.sve_state;
147
148 kvm_unshare_hyp(vcpu, vcpu + 1);
149 if (sve_state)
150 kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));
151 kfree(sve_state);
152 }
153
kvm_vcpu_reset_sve(struct kvm_vcpu * vcpu)154 static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu)
155 {
156 if (vcpu_has_sve(vcpu))
157 memset(vcpu->arch.sve_state, 0, vcpu_sve_state_size(vcpu));
158 }
159
160 /**
161 * kvm_set_vm_width() - set the register width for the guest
162 * @vcpu: Pointer to the vcpu being configured
163 *
164 * Set both KVM_ARCH_FLAG_EL1_32BIT and KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED
165 * in the VM flags based on the vcpu's requested register width, the HW
166 * capabilities and other options (such as MTE).
167 * When REG_WIDTH_CONFIGURED is already set, the vcpu settings must be
168 * consistent with the value of the FLAG_EL1_32BIT bit in the flags.
169 *
170 * Return: 0 on success, negative error code on failure.
171 */
kvm_set_vm_width(struct kvm_vcpu * vcpu)172 static int kvm_set_vm_width(struct kvm_vcpu *vcpu)
173 {
174 struct kvm *kvm = vcpu->kvm;
175 bool is32bit;
176
177 is32bit = vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT);
178
179 lockdep_assert_held(&kvm->arch.config_lock);
180
181 if (test_bit(KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED, &kvm->arch.flags)) {
182 /*
183 * The guest's register width is already configured.
184 * Make sure that the vcpu is consistent with it.
185 */
186 if (is32bit == test_bit(KVM_ARCH_FLAG_EL1_32BIT, &kvm->arch.flags))
187 return 0;
188
189 return -EINVAL;
190 }
191
192 if (!cpus_have_const_cap(ARM64_HAS_32BIT_EL1) && is32bit)
193 return -EINVAL;
194
195 /* MTE is incompatible with AArch32 */
196 if (kvm_has_mte(kvm) && is32bit)
197 return -EINVAL;
198
199 if (is32bit)
200 set_bit(KVM_ARCH_FLAG_EL1_32BIT, &kvm->arch.flags);
201
202 set_bit(KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED, &kvm->arch.flags);
203
204 return 0;
205 }
206
207 /**
208 * kvm_reset_vcpu - sets core registers and sys_regs to reset value
209 * @vcpu: The VCPU pointer
210 *
211 * This function sets the registers on the virtual CPU struct to their
212 * architecturally defined reset values, except for registers whose reset is
213 * deferred until kvm_arm_vcpu_finalize().
214 *
215 * Note: This function can be called from two paths: The KVM_ARM_VCPU_INIT
216 * ioctl or as part of handling a request issued by another VCPU in the PSCI
217 * handling code. In the first case, the VCPU will not be loaded, and in the
218 * second case the VCPU will be loaded. Because this function operates purely
219 * on the memory-backed values of system registers, we want to do a full put if
220 * we were loaded (handling a request) and load the values back at the end of
221 * the function. Otherwise we leave the state alone. In both cases, we
222 * disable preemption around the vcpu reset as we would otherwise race with
223 * preempt notifiers which also call put/load.
224 */
kvm_reset_vcpu(struct kvm_vcpu * vcpu)225 int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
226 {
227 struct vcpu_reset_state reset_state;
228 int ret;
229 bool loaded;
230
231 mutex_lock(&vcpu->kvm->arch.config_lock);
232 ret = kvm_set_vm_width(vcpu);
233 mutex_unlock(&vcpu->kvm->arch.config_lock);
234
235 if (ret)
236 return ret;
237
238 spin_lock(&vcpu->arch.mp_state_lock);
239 reset_state = vcpu->arch.reset_state;
240 vcpu->arch.reset_state.reset = false;
241 spin_unlock(&vcpu->arch.mp_state_lock);
242
243 /* Reset PMU outside of the non-preemptible section */
244 kvm_pmu_vcpu_reset(vcpu);
245
246 preempt_disable();
247 loaded = (vcpu->cpu != -1);
248 if (loaded)
249 kvm_arch_vcpu_put(vcpu);
250
251 if (!kvm_arm_vcpu_sve_finalized(vcpu)) {
252 if (test_bit(KVM_ARM_VCPU_SVE, vcpu->arch.features)) {
253 ret = kvm_vcpu_enable_sve(vcpu);
254 if (ret)
255 goto out;
256 }
257 } else {
258 kvm_vcpu_reset_sve(vcpu);
259 }
260
261 if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, vcpu->arch.features) ||
262 test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, vcpu->arch.features)) {
263 if (kvm_vcpu_enable_ptrauth(vcpu)) {
264 ret = -EINVAL;
265 goto out;
266 }
267 }
268
269 if (kvm_vcpu_has_pmu(vcpu) && !kvm_arm_support_pmu_v3()) {
270 ret = -EINVAL;
271 goto out;
272 }
273
274 /* Reset core registers */
275 kvm_reset_vcpu_core(vcpu);
276
277 /* Reset system registers */
278 kvm_reset_sys_regs(vcpu);
279
280 /*
281 * Additional reset state handling that PSCI may have imposed on us.
282 * Must be done after all the sys_reg reset.
283 */
284 if (reset_state.reset)
285 kvm_reset_vcpu_psci(vcpu, &reset_state);
286
287 /* Reset timer */
288 ret = kvm_timer_vcpu_reset(vcpu);
289 out:
290 if (loaded)
291 kvm_arch_vcpu_load(vcpu, smp_processor_id());
292 preempt_enable();
293 return ret;
294 }
295
get_kvm_ipa_limit(void)296 u32 get_kvm_ipa_limit(void)
297 {
298 return kvm_ipa_limit;
299 }
300
kvm_set_ipa_limit(void)301 int kvm_set_ipa_limit(void)
302 {
303 unsigned int parange;
304 u64 mmfr0;
305
306 mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
307 parange = cpuid_feature_extract_unsigned_field(mmfr0,
308 ID_AA64MMFR0_EL1_PARANGE_SHIFT);
309 /*
310 * IPA size beyond 48 bits could not be supported
311 * on either 4K or 16K page size. Hence let's cap
312 * it to 48 bits, in case it's reported as larger
313 * on the system.
314 */
315 if (PAGE_SIZE != SZ_64K)
316 parange = min(parange, (unsigned int)ID_AA64MMFR0_EL1_PARANGE_48);
317
318 /*
319 * Check with ARMv8.5-GTG that our PAGE_SIZE is supported at
320 * Stage-2. If not, things will stop very quickly.
321 */
322 switch (cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_TGRAN_2_SHIFT)) {
323 case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_NONE:
324 kvm_err("PAGE_SIZE not supported at Stage-2, giving up\n");
325 return -EINVAL;
326 case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_DEFAULT:
327 kvm_debug("PAGE_SIZE supported at Stage-2 (default)\n");
328 break;
329 case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MIN ... ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MAX:
330 kvm_debug("PAGE_SIZE supported at Stage-2 (advertised)\n");
331 break;
332 default:
333 kvm_err("Unsupported value for TGRAN_2, giving up\n");
334 return -EINVAL;
335 }
336
337 kvm_ipa_limit = id_aa64mmfr0_parange_to_phys_shift(parange);
338 kvm_info("IPA Size Limit: %d bits%s\n", kvm_ipa_limit,
339 ((kvm_ipa_limit < KVM_PHYS_SHIFT) ?
340 " (Reduced IPA size, limited VM/VMM compatibility)" : ""));
341
342 return 0;
343 }
344