1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Kernel-based Virtual Machine driver for Linux
4 * cpuid support routines
5 *
6 * derived from arch/x86/kvm/x86.c
7 *
8 * Copyright 2011 Red Hat, Inc. and/or its affiliates.
9 * Copyright IBM Corporation, 2008
10 */
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13 #include <linux/kvm_host.h>
14 #include "linux/lockdep.h"
15 #include <linux/export.h>
16 #include <linux/vmalloc.h>
17 #include <linux/uaccess.h>
18 #include <linux/sched/stat.h>
19
20 #include <asm/processor.h>
21 #include <asm/user.h>
22 #include <asm/fpu/xstate.h>
23 #include <asm/sgx.h>
24 #include <asm/cpuid.h>
25 #include "cpuid.h"
26 #include "lapic.h"
27 #include "mmu.h"
28 #include "trace.h"
29 #include "pmu.h"
30 #include "xen.h"
31
32 /*
33 * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
34 * aligned to sizeof(unsigned long) because it's not accessed via bitops.
35 */
36 u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly;
37 EXPORT_SYMBOL_GPL(kvm_cpu_caps);
38
39 struct cpuid_xstate_sizes {
40 u32 eax;
41 u32 ebx;
42 u32 ecx;
43 };
44
45 static struct cpuid_xstate_sizes xstate_sizes[XFEATURE_MAX] __ro_after_init;
46
kvm_init_xstate_sizes(void)47 void __init kvm_init_xstate_sizes(void)
48 {
49 u32 ign;
50 int i;
51
52 for (i = XFEATURE_YMM; i < ARRAY_SIZE(xstate_sizes); i++) {
53 struct cpuid_xstate_sizes *xs = &xstate_sizes[i];
54
55 cpuid_count(0xD, i, &xs->eax, &xs->ebx, &xs->ecx, &ign);
56 }
57 }
58
xstate_required_size(u64 xstate_bv,bool compacted)59 u32 xstate_required_size(u64 xstate_bv, bool compacted)
60 {
61 int feature_bit = 0;
62 u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
63
64 xstate_bv &= XFEATURE_MASK_EXTEND;
65 while (xstate_bv) {
66 if (xstate_bv & 0x1) {
67 struct cpuid_xstate_sizes *xs = &xstate_sizes[feature_bit];
68 u32 offset;
69
70 /* ECX[1]: 64B alignment in compacted form */
71 if (compacted)
72 offset = (xs->ecx & 0x2) ? ALIGN(ret, 64) : ret;
73 else
74 offset = xs->ebx;
75 ret = max(ret, offset + xs->eax);
76 }
77
78 xstate_bv >>= 1;
79 feature_bit++;
80 }
81
82 return ret;
83 }
84
85 #define F feature_bit
86
87 /* Scattered Flag - For features that are scattered by cpufeatures.h. */
88 #define SF(name) \
89 ({ \
90 BUILD_BUG_ON(X86_FEATURE_##name >= MAX_CPU_FEATURES); \
91 (boot_cpu_has(X86_FEATURE_##name) ? F(name) : 0); \
92 })
93
94 /*
95 * Magic value used by KVM when querying userspace-provided CPUID entries and
96 * doesn't care about the CPIUD index because the index of the function in
97 * question is not significant. Note, this magic value must have at least one
98 * bit set in bits[63:32] and must be consumed as a u64 by cpuid_entry2_find()
99 * to avoid false positives when processing guest CPUID input.
100 */
101 #define KVM_CPUID_INDEX_NOT_SIGNIFICANT -1ull
102
cpuid_entry2_find(struct kvm_cpuid_entry2 * entries,int nent,u32 function,u64 index)103 static inline struct kvm_cpuid_entry2 *cpuid_entry2_find(
104 struct kvm_cpuid_entry2 *entries, int nent, u32 function, u64 index)
105 {
106 struct kvm_cpuid_entry2 *e;
107 int i;
108
109 /*
110 * KVM has a semi-arbitrary rule that querying the guest's CPUID model
111 * with IRQs disabled is disallowed. The CPUID model can legitimately
112 * have over one hundred entries, i.e. the lookup is slow, and IRQs are
113 * typically disabled in KVM only when KVM is in a performance critical
114 * path, e.g. the core VM-Enter/VM-Exit run loop. Nothing will break
115 * if this rule is violated, this assertion is purely to flag potential
116 * performance issues. If this fires, consider moving the lookup out
117 * of the hotpath, e.g. by caching information during CPUID updates.
118 */
119 lockdep_assert_irqs_enabled();
120
121 for (i = 0; i < nent; i++) {
122 e = &entries[i];
123
124 if (e->function != function)
125 continue;
126
127 /*
128 * If the index isn't significant, use the first entry with a
129 * matching function. It's userspace's responsibility to not
130 * provide "duplicate" entries in all cases.
131 */
132 if (!(e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) || e->index == index)
133 return e;
134
135
136 /*
137 * Similarly, use the first matching entry if KVM is doing a
138 * lookup (as opposed to emulating CPUID) for a function that's
139 * architecturally defined as not having a significant index.
140 */
141 if (index == KVM_CPUID_INDEX_NOT_SIGNIFICANT) {
142 /*
143 * Direct lookups from KVM should not diverge from what
144 * KVM defines internally (the architectural behavior).
145 */
146 WARN_ON_ONCE(cpuid_function_is_indexed(function));
147 return e;
148 }
149 }
150
151 return NULL;
152 }
153
kvm_check_cpuid(struct kvm_vcpu * vcpu,struct kvm_cpuid_entry2 * entries,int nent)154 static int kvm_check_cpuid(struct kvm_vcpu *vcpu,
155 struct kvm_cpuid_entry2 *entries,
156 int nent)
157 {
158 struct kvm_cpuid_entry2 *best;
159 u64 xfeatures;
160
161 /*
162 * The existing code assumes virtual address is 48-bit or 57-bit in the
163 * canonical address checks; exit if it is ever changed.
164 */
165 best = cpuid_entry2_find(entries, nent, 0x80000008,
166 KVM_CPUID_INDEX_NOT_SIGNIFICANT);
167 if (best) {
168 int vaddr_bits = (best->eax & 0xff00) >> 8;
169
170 if (vaddr_bits != 48 && vaddr_bits != 57 && vaddr_bits != 0)
171 return -EINVAL;
172 }
173
174 /*
175 * Exposing dynamic xfeatures to the guest requires additional
176 * enabling in the FPU, e.g. to expand the guest XSAVE state size.
177 */
178 best = cpuid_entry2_find(entries, nent, 0xd, 0);
179 if (!best)
180 return 0;
181
182 xfeatures = best->eax | ((u64)best->edx << 32);
183 xfeatures &= XFEATURE_MASK_USER_DYNAMIC;
184 if (!xfeatures)
185 return 0;
186
187 return fpu_enable_guest_xfd_features(&vcpu->arch.guest_fpu, xfeatures);
188 }
189
190 /* Check whether the supplied CPUID data is equal to what is already set for the vCPU. */
kvm_cpuid_check_equal(struct kvm_vcpu * vcpu,struct kvm_cpuid_entry2 * e2,int nent)191 static int kvm_cpuid_check_equal(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
192 int nent)
193 {
194 struct kvm_cpuid_entry2 *orig;
195 int i;
196
197 if (nent != vcpu->arch.cpuid_nent)
198 return -EINVAL;
199
200 for (i = 0; i < nent; i++) {
201 orig = &vcpu->arch.cpuid_entries[i];
202 if (e2[i].function != orig->function ||
203 e2[i].index != orig->index ||
204 e2[i].flags != orig->flags ||
205 e2[i].eax != orig->eax || e2[i].ebx != orig->ebx ||
206 e2[i].ecx != orig->ecx || e2[i].edx != orig->edx)
207 return -EINVAL;
208 }
209
210 return 0;
211 }
212
__kvm_get_hypervisor_cpuid(struct kvm_cpuid_entry2 * entries,int nent,const char * sig)213 static struct kvm_hypervisor_cpuid __kvm_get_hypervisor_cpuid(struct kvm_cpuid_entry2 *entries,
214 int nent, const char *sig)
215 {
216 struct kvm_hypervisor_cpuid cpuid = {};
217 struct kvm_cpuid_entry2 *entry;
218 u32 base;
219
220 for_each_possible_hypervisor_cpuid_base(base) {
221 entry = cpuid_entry2_find(entries, nent, base, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
222
223 if (entry) {
224 u32 signature[3];
225
226 signature[0] = entry->ebx;
227 signature[1] = entry->ecx;
228 signature[2] = entry->edx;
229
230 if (!memcmp(signature, sig, sizeof(signature))) {
231 cpuid.base = base;
232 cpuid.limit = entry->eax;
233 break;
234 }
235 }
236 }
237
238 return cpuid;
239 }
240
kvm_get_hypervisor_cpuid(struct kvm_vcpu * vcpu,const char * sig)241 static struct kvm_hypervisor_cpuid kvm_get_hypervisor_cpuid(struct kvm_vcpu *vcpu,
242 const char *sig)
243 {
244 return __kvm_get_hypervisor_cpuid(vcpu->arch.cpuid_entries,
245 vcpu->arch.cpuid_nent, sig);
246 }
247
__kvm_find_kvm_cpuid_features(struct kvm_cpuid_entry2 * entries,int nent,u32 kvm_cpuid_base)248 static struct kvm_cpuid_entry2 *__kvm_find_kvm_cpuid_features(struct kvm_cpuid_entry2 *entries,
249 int nent, u32 kvm_cpuid_base)
250 {
251 return cpuid_entry2_find(entries, nent, kvm_cpuid_base | KVM_CPUID_FEATURES,
252 KVM_CPUID_INDEX_NOT_SIGNIFICANT);
253 }
254
kvm_find_kvm_cpuid_features(struct kvm_vcpu * vcpu)255 static struct kvm_cpuid_entry2 *kvm_find_kvm_cpuid_features(struct kvm_vcpu *vcpu)
256 {
257 u32 base = vcpu->arch.kvm_cpuid.base;
258
259 if (!base)
260 return NULL;
261
262 return __kvm_find_kvm_cpuid_features(vcpu->arch.cpuid_entries,
263 vcpu->arch.cpuid_nent, base);
264 }
265
kvm_update_pv_runtime(struct kvm_vcpu * vcpu)266 void kvm_update_pv_runtime(struct kvm_vcpu *vcpu)
267 {
268 struct kvm_cpuid_entry2 *best = kvm_find_kvm_cpuid_features(vcpu);
269
270 /*
271 * save the feature bitmap to avoid cpuid lookup for every PV
272 * operation
273 */
274 if (best)
275 vcpu->arch.pv_cpuid.features = best->eax;
276 }
277
278 /*
279 * Calculate guest's supported XCR0 taking into account guest CPUID data and
280 * KVM's supported XCR0 (comprised of host's XCR0 and KVM_SUPPORTED_XCR0).
281 */
cpuid_get_supported_xcr0(struct kvm_cpuid_entry2 * entries,int nent)282 static u64 cpuid_get_supported_xcr0(struct kvm_cpuid_entry2 *entries, int nent)
283 {
284 struct kvm_cpuid_entry2 *best;
285
286 best = cpuid_entry2_find(entries, nent, 0xd, 0);
287 if (!best)
288 return 0;
289
290 return (best->eax | ((u64)best->edx << 32)) & kvm_caps.supported_xcr0;
291 }
292
__kvm_update_cpuid_runtime(struct kvm_vcpu * vcpu,struct kvm_cpuid_entry2 * entries,int nent)293 static void __kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *entries,
294 int nent)
295 {
296 struct kvm_cpuid_entry2 *best;
297 struct kvm_hypervisor_cpuid kvm_cpuid;
298
299 best = cpuid_entry2_find(entries, nent, 1, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
300 if (best) {
301 /* Update OSXSAVE bit */
302 if (boot_cpu_has(X86_FEATURE_XSAVE))
303 cpuid_entry_change(best, X86_FEATURE_OSXSAVE,
304 kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE));
305
306 cpuid_entry_change(best, X86_FEATURE_APIC,
307 vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE);
308 }
309
310 best = cpuid_entry2_find(entries, nent, 7, 0);
311 if (best && boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7)
312 cpuid_entry_change(best, X86_FEATURE_OSPKE,
313 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE));
314
315 best = cpuid_entry2_find(entries, nent, 0xD, 0);
316 if (best)
317 best->ebx = xstate_required_size(vcpu->arch.xcr0, false);
318
319 best = cpuid_entry2_find(entries, nent, 0xD, 1);
320 if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) ||
321 cpuid_entry_has(best, X86_FEATURE_XSAVEC)))
322 best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
323
324 kvm_cpuid = __kvm_get_hypervisor_cpuid(entries, nent, KVM_SIGNATURE);
325 if (kvm_cpuid.base) {
326 best = __kvm_find_kvm_cpuid_features(entries, nent, kvm_cpuid.base);
327 if (kvm_hlt_in_guest(vcpu->kvm) && best)
328 best->eax &= ~(1 << KVM_FEATURE_PV_UNHALT);
329 }
330
331 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) {
332 best = cpuid_entry2_find(entries, nent, 0x1, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
333 if (best)
334 cpuid_entry_change(best, X86_FEATURE_MWAIT,
335 vcpu->arch.ia32_misc_enable_msr &
336 MSR_IA32_MISC_ENABLE_MWAIT);
337 }
338 }
339
kvm_update_cpuid_runtime(struct kvm_vcpu * vcpu)340 void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
341 {
342 __kvm_update_cpuid_runtime(vcpu, vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent);
343 }
344 EXPORT_SYMBOL_GPL(kvm_update_cpuid_runtime);
345
kvm_cpuid_has_hyperv(struct kvm_cpuid_entry2 * entries,int nent)346 static bool kvm_cpuid_has_hyperv(struct kvm_cpuid_entry2 *entries, int nent)
347 {
348 #ifdef CONFIG_KVM_HYPERV
349 struct kvm_cpuid_entry2 *entry;
350
351 entry = cpuid_entry2_find(entries, nent, HYPERV_CPUID_INTERFACE,
352 KVM_CPUID_INDEX_NOT_SIGNIFICANT);
353 return entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX;
354 #else
355 return false;
356 #endif
357 }
358
guest_cpuid_is_amd_or_hygon(struct kvm_vcpu * vcpu)359 static bool guest_cpuid_is_amd_or_hygon(struct kvm_vcpu *vcpu)
360 {
361 struct kvm_cpuid_entry2 *entry;
362
363 entry = kvm_find_cpuid_entry(vcpu, 0);
364 if (!entry)
365 return false;
366
367 return is_guest_vendor_amd(entry->ebx, entry->ecx, entry->edx) ||
368 is_guest_vendor_hygon(entry->ebx, entry->ecx, entry->edx);
369 }
370
kvm_vcpu_after_set_cpuid(struct kvm_vcpu * vcpu)371 static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
372 {
373 struct kvm_lapic *apic = vcpu->arch.apic;
374 struct kvm_cpuid_entry2 *best;
375 bool allow_gbpages;
376
377 BUILD_BUG_ON(KVM_NR_GOVERNED_FEATURES > KVM_MAX_NR_GOVERNED_FEATURES);
378 bitmap_zero(vcpu->arch.governed_features.enabled,
379 KVM_MAX_NR_GOVERNED_FEATURES);
380
381 /*
382 * If TDP is enabled, let the guest use GBPAGES if they're supported in
383 * hardware. The hardware page walker doesn't let KVM disable GBPAGES,
384 * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA
385 * walk for performance and complexity reasons. Not to mention KVM
386 * _can't_ solve the problem because GVA->GPA walks aren't visible to
387 * KVM once a TDP translation is installed. Mimic hardware behavior so
388 * that KVM's is at least consistent, i.e. doesn't randomly inject #PF.
389 * If TDP is disabled, honor *only* guest CPUID as KVM has full control
390 * and can install smaller shadow pages if the host lacks 1GiB support.
391 */
392 allow_gbpages = tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) :
393 guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES);
394 if (allow_gbpages)
395 kvm_governed_feature_set(vcpu, X86_FEATURE_GBPAGES);
396
397 best = kvm_find_cpuid_entry(vcpu, 1);
398 if (best && apic) {
399 if (cpuid_entry_has(best, X86_FEATURE_TSC_DEADLINE_TIMER))
400 apic->lapic_timer.timer_mode_mask = 3 << 17;
401 else
402 apic->lapic_timer.timer_mode_mask = 1 << 17;
403
404 kvm_apic_set_version(vcpu);
405 }
406
407 vcpu->arch.guest_supported_xcr0 =
408 cpuid_get_supported_xcr0(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent);
409
410 kvm_update_pv_runtime(vcpu);
411
412 vcpu->arch.is_amd_compatible = guest_cpuid_is_amd_or_hygon(vcpu);
413 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
414 vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
415
416 kvm_pmu_refresh(vcpu);
417 vcpu->arch.cr4_guest_rsvd_bits =
418 __cr4_reserved_bits(guest_cpuid_has, vcpu);
419
420 kvm_hv_set_cpuid(vcpu, kvm_cpuid_has_hyperv(vcpu->arch.cpuid_entries,
421 vcpu->arch.cpuid_nent));
422
423 /* Invoke the vendor callback only after the above state is updated. */
424 kvm_x86_call(vcpu_after_set_cpuid)(vcpu);
425
426 /*
427 * Except for the MMU, which needs to do its thing any vendor specific
428 * adjustments to the reserved GPA bits.
429 */
430 kvm_mmu_after_set_cpuid(vcpu);
431 }
432
cpuid_query_maxphyaddr(struct kvm_vcpu * vcpu)433 int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu)
434 {
435 struct kvm_cpuid_entry2 *best;
436
437 best = kvm_find_cpuid_entry(vcpu, 0x80000000);
438 if (!best || best->eax < 0x80000008)
439 goto not_found;
440 best = kvm_find_cpuid_entry(vcpu, 0x80000008);
441 if (best)
442 return best->eax & 0xff;
443 not_found:
444 return 36;
445 }
446
447 /*
448 * This "raw" version returns the reserved GPA bits without any adjustments for
449 * encryption technologies that usurp bits. The raw mask should be used if and
450 * only if hardware does _not_ strip the usurped bits, e.g. in virtual MTRRs.
451 */
kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu * vcpu)452 u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu)
453 {
454 return rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
455 }
456
kvm_set_cpuid(struct kvm_vcpu * vcpu,struct kvm_cpuid_entry2 * e2,int nent)457 static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
458 int nent)
459 {
460 int r;
461
462 __kvm_update_cpuid_runtime(vcpu, e2, nent);
463
464 /*
465 * KVM does not correctly handle changing guest CPUID after KVM_RUN, as
466 * MAXPHYADDR, GBPAGES support, AMD reserved bit behavior, etc.. aren't
467 * tracked in kvm_mmu_page_role. As a result, KVM may miss guest page
468 * faults due to reusing SPs/SPTEs. In practice no sane VMM mucks with
469 * the core vCPU model on the fly. It would've been better to forbid any
470 * KVM_SET_CPUID{,2} calls after KVM_RUN altogether but unfortunately
471 * some VMMs (e.g. QEMU) reuse vCPU fds for CPU hotplug/unplug and do
472 * KVM_SET_CPUID{,2} again. To support this legacy behavior, check
473 * whether the supplied CPUID data is equal to what's already set.
474 */
475 if (kvm_vcpu_has_run(vcpu)) {
476 r = kvm_cpuid_check_equal(vcpu, e2, nent);
477 if (r)
478 return r;
479
480 kvfree(e2);
481 return 0;
482 }
483
484 #ifdef CONFIG_KVM_HYPERV
485 if (kvm_cpuid_has_hyperv(e2, nent)) {
486 r = kvm_hv_vcpu_init(vcpu);
487 if (r)
488 return r;
489 }
490 #endif
491
492 r = kvm_check_cpuid(vcpu, e2, nent);
493 if (r)
494 return r;
495
496 kvfree(vcpu->arch.cpuid_entries);
497 vcpu->arch.cpuid_entries = e2;
498 vcpu->arch.cpuid_nent = nent;
499
500 vcpu->arch.kvm_cpuid = kvm_get_hypervisor_cpuid(vcpu, KVM_SIGNATURE);
501 #ifdef CONFIG_KVM_XEN
502 vcpu->arch.xen.cpuid = kvm_get_hypervisor_cpuid(vcpu, XEN_SIGNATURE);
503 #endif
504 kvm_vcpu_after_set_cpuid(vcpu);
505
506 return 0;
507 }
508
509 /* when an old userspace process fills a new kernel module */
kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu * vcpu,struct kvm_cpuid * cpuid,struct kvm_cpuid_entry __user * entries)510 int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
511 struct kvm_cpuid *cpuid,
512 struct kvm_cpuid_entry __user *entries)
513 {
514 int r, i;
515 struct kvm_cpuid_entry *e = NULL;
516 struct kvm_cpuid_entry2 *e2 = NULL;
517
518 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
519 return -E2BIG;
520
521 if (cpuid->nent) {
522 e = vmemdup_array_user(entries, cpuid->nent, sizeof(*e));
523 if (IS_ERR(e))
524 return PTR_ERR(e);
525
526 e2 = kvmalloc_array(cpuid->nent, sizeof(*e2), GFP_KERNEL_ACCOUNT);
527 if (!e2) {
528 r = -ENOMEM;
529 goto out_free_cpuid;
530 }
531 }
532 for (i = 0; i < cpuid->nent; i++) {
533 e2[i].function = e[i].function;
534 e2[i].eax = e[i].eax;
535 e2[i].ebx = e[i].ebx;
536 e2[i].ecx = e[i].ecx;
537 e2[i].edx = e[i].edx;
538 e2[i].index = 0;
539 e2[i].flags = 0;
540 e2[i].padding[0] = 0;
541 e2[i].padding[1] = 0;
542 e2[i].padding[2] = 0;
543 }
544
545 r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
546 if (r)
547 kvfree(e2);
548
549 out_free_cpuid:
550 kvfree(e);
551
552 return r;
553 }
554
kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu * vcpu,struct kvm_cpuid2 * cpuid,struct kvm_cpuid_entry2 __user * entries)555 int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
556 struct kvm_cpuid2 *cpuid,
557 struct kvm_cpuid_entry2 __user *entries)
558 {
559 struct kvm_cpuid_entry2 *e2 = NULL;
560 int r;
561
562 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
563 return -E2BIG;
564
565 if (cpuid->nent) {
566 e2 = vmemdup_array_user(entries, cpuid->nent, sizeof(*e2));
567 if (IS_ERR(e2))
568 return PTR_ERR(e2);
569 }
570
571 r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
572 if (r)
573 kvfree(e2);
574
575 return r;
576 }
577
kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu * vcpu,struct kvm_cpuid2 * cpuid,struct kvm_cpuid_entry2 __user * entries)578 int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
579 struct kvm_cpuid2 *cpuid,
580 struct kvm_cpuid_entry2 __user *entries)
581 {
582 if (cpuid->nent < vcpu->arch.cpuid_nent)
583 return -E2BIG;
584
585 if (copy_to_user(entries, vcpu->arch.cpuid_entries,
586 vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
587 return -EFAULT;
588
589 cpuid->nent = vcpu->arch.cpuid_nent;
590 return 0;
591 }
592
593 /* Mask kvm_cpu_caps for @leaf with the raw CPUID capabilities of this CPU. */
__kvm_cpu_cap_mask(unsigned int leaf)594 static __always_inline void __kvm_cpu_cap_mask(unsigned int leaf)
595 {
596 const struct cpuid_reg cpuid = x86_feature_cpuid(leaf * 32);
597 struct kvm_cpuid_entry2 entry;
598
599 reverse_cpuid_check(leaf);
600
601 cpuid_count(cpuid.function, cpuid.index,
602 &entry.eax, &entry.ebx, &entry.ecx, &entry.edx);
603
604 kvm_cpu_caps[leaf] &= *__cpuid_entry_get_reg(&entry, cpuid.reg);
605 }
606
607 static __always_inline
kvm_cpu_cap_init_kvm_defined(enum kvm_only_cpuid_leafs leaf,u32 mask)608 void kvm_cpu_cap_init_kvm_defined(enum kvm_only_cpuid_leafs leaf, u32 mask)
609 {
610 /* Use kvm_cpu_cap_mask for leafs that aren't KVM-only. */
611 BUILD_BUG_ON(leaf < NCAPINTS);
612
613 kvm_cpu_caps[leaf] = mask;
614
615 __kvm_cpu_cap_mask(leaf);
616 }
617
kvm_cpu_cap_mask(enum cpuid_leafs leaf,u32 mask)618 static __always_inline void kvm_cpu_cap_mask(enum cpuid_leafs leaf, u32 mask)
619 {
620 /* Use kvm_cpu_cap_init_kvm_defined for KVM-only leafs. */
621 BUILD_BUG_ON(leaf >= NCAPINTS);
622
623 kvm_cpu_caps[leaf] &= mask;
624
625 __kvm_cpu_cap_mask(leaf);
626 }
627
kvm_set_cpu_caps(void)628 void kvm_set_cpu_caps(void)
629 {
630 #ifdef CONFIG_X86_64
631 unsigned int f_gbpages = F(GBPAGES);
632 unsigned int f_lm = F(LM);
633 unsigned int f_xfd = F(XFD);
634 #else
635 unsigned int f_gbpages = 0;
636 unsigned int f_lm = 0;
637 unsigned int f_xfd = 0;
638 #endif
639 memset(kvm_cpu_caps, 0, sizeof(kvm_cpu_caps));
640
641 BUILD_BUG_ON(sizeof(kvm_cpu_caps) - (NKVMCAPINTS * sizeof(*kvm_cpu_caps)) >
642 sizeof(boot_cpu_data.x86_capability));
643
644 memcpy(&kvm_cpu_caps, &boot_cpu_data.x86_capability,
645 sizeof(kvm_cpu_caps) - (NKVMCAPINTS * sizeof(*kvm_cpu_caps)));
646
647 kvm_cpu_cap_mask(CPUID_1_ECX,
648 /*
649 * NOTE: MONITOR (and MWAIT) are emulated as NOP, but *not*
650 * advertised to guests via CPUID!
651 */
652 F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ |
653 0 /* DS-CPL, VMX, SMX, EST */ |
654 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
655 F(FMA) | F(CX16) | 0 /* xTPR Update */ | F(PDCM) |
656 F(PCID) | 0 /* Reserved, DCA */ | F(XMM4_1) |
657 F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
658 0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) |
659 F(F16C) | F(RDRAND)
660 );
661 /* KVM emulates x2apic in software irrespective of host support. */
662 kvm_cpu_cap_set(X86_FEATURE_X2APIC);
663
664 kvm_cpu_cap_mask(CPUID_1_EDX,
665 F(FPU) | F(VME) | F(DE) | F(PSE) |
666 F(TSC) | F(MSR) | F(PAE) | F(MCE) |
667 F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
668 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
669 F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLUSH) |
670 0 /* Reserved, DS, ACPI */ | F(MMX) |
671 F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
672 0 /* HTT, TM, Reserved, PBE */
673 );
674
675 kvm_cpu_cap_mask(CPUID_7_0_EBX,
676 F(FSGSBASE) | F(SGX) | F(BMI1) | F(HLE) | F(AVX2) |
677 F(FDP_EXCPTN_ONLY) | F(SMEP) | F(BMI2) | F(ERMS) | F(INVPCID) |
678 F(RTM) | F(ZERO_FCS_FDS) | 0 /*MPX*/ | F(AVX512F) |
679 F(AVX512DQ) | F(RDSEED) | F(ADX) | F(SMAP) | F(AVX512IFMA) |
680 F(CLFLUSHOPT) | F(CLWB) | 0 /*INTEL_PT*/ | F(AVX512PF) |
681 F(AVX512ER) | F(AVX512CD) | F(SHA_NI) | F(AVX512BW) |
682 F(AVX512VL));
683
684 kvm_cpu_cap_mask(CPUID_7_ECX,
685 F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ | F(RDPID) |
686 F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
687 F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) |
688 F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B) | 0 /*WAITPKG*/ |
689 F(SGX_LC) | F(BUS_LOCK_DETECT)
690 );
691 /* Set LA57 based on hardware capability. */
692 if (cpuid_ecx(7) & F(LA57))
693 kvm_cpu_cap_set(X86_FEATURE_LA57);
694
695 /*
696 * PKU not yet implemented for shadow paging and requires OSPKE
697 * to be set on the host. Clear it if that is not the case
698 */
699 if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE))
700 kvm_cpu_cap_clear(X86_FEATURE_PKU);
701
702 kvm_cpu_cap_mask(CPUID_7_EDX,
703 F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
704 F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) |
705 F(MD_CLEAR) | F(AVX512_VP2INTERSECT) | F(FSRM) |
706 F(SERIALIZE) | F(TSXLDTRK) | F(AVX512_FP16) |
707 F(AMX_TILE) | F(AMX_INT8) | F(AMX_BF16) | F(FLUSH_L1D)
708 );
709
710 /* TSC_ADJUST and ARCH_CAPABILITIES are emulated in software. */
711 kvm_cpu_cap_set(X86_FEATURE_TSC_ADJUST);
712 kvm_cpu_cap_set(X86_FEATURE_ARCH_CAPABILITIES);
713
714 if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS))
715 kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL);
716 if (boot_cpu_has(X86_FEATURE_STIBP))
717 kvm_cpu_cap_set(X86_FEATURE_INTEL_STIBP);
718 if (boot_cpu_has(X86_FEATURE_AMD_SSBD))
719 kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
720
721 kvm_cpu_cap_mask(CPUID_7_1_EAX,
722 F(AVX_VNNI) | F(AVX512_BF16) | F(CMPCCXADD) |
723 F(FZRM) | F(FSRS) | F(FSRC) |
724 F(AMX_FP16) | F(AVX_IFMA) | F(LAM)
725 );
726
727 kvm_cpu_cap_init_kvm_defined(CPUID_7_1_EDX,
728 F(AVX_VNNI_INT8) | F(AVX_NE_CONVERT) | F(PREFETCHITI) |
729 F(AMX_COMPLEX) | F(AVX10)
730 );
731
732 kvm_cpu_cap_init_kvm_defined(CPUID_7_2_EDX,
733 F(INTEL_PSFD) | F(IPRED_CTRL) | F(RRSBA_CTRL) | F(DDPD_U) |
734 F(BHI_CTRL) | F(MCDT_NO)
735 );
736
737 kvm_cpu_cap_mask(CPUID_D_1_EAX,
738 F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES) | f_xfd
739 );
740
741 kvm_cpu_cap_init_kvm_defined(CPUID_12_EAX,
742 SF(SGX1) | SF(SGX2) | SF(SGX_EDECCSSA)
743 );
744
745 kvm_cpu_cap_init_kvm_defined(CPUID_24_0_EBX,
746 F(AVX10_128) | F(AVX10_256) | F(AVX10_512)
747 );
748
749 kvm_cpu_cap_mask(CPUID_8000_0001_ECX,
750 F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ |
751 F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
752 F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
753 0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
754 F(TOPOEXT) | 0 /* PERFCTR_CORE */
755 );
756
757 kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
758 F(FPU) | F(VME) | F(DE) | F(PSE) |
759 F(TSC) | F(MSR) | F(PAE) | F(MCE) |
760 F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
761 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
762 F(PAT) | F(PSE36) | 0 /* Reserved */ |
763 F(NX) | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
764 F(FXSR) | F(FXSR_OPT) | f_gbpages | F(RDTSCP) |
765 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW)
766 );
767
768 if (!tdp_enabled && IS_ENABLED(CONFIG_X86_64))
769 kvm_cpu_cap_set(X86_FEATURE_GBPAGES);
770
771 kvm_cpu_cap_init_kvm_defined(CPUID_8000_0007_EDX,
772 SF(CONSTANT_TSC)
773 );
774
775 kvm_cpu_cap_mask(CPUID_8000_0008_EBX,
776 F(CLZERO) | F(XSAVEERPTR) |
777 F(WBNOINVD) | F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) |
778 F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON) |
779 F(AMD_PSFD)
780 );
781
782 /*
783 * AMD has separate bits for each SPEC_CTRL bit.
784 * arch/x86/kernel/cpu/bugs.c is kind enough to
785 * record that in cpufeatures so use them.
786 */
787 if (boot_cpu_has(X86_FEATURE_IBPB))
788 kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB);
789 if (boot_cpu_has(X86_FEATURE_IBRS))
790 kvm_cpu_cap_set(X86_FEATURE_AMD_IBRS);
791 if (boot_cpu_has(X86_FEATURE_STIBP))
792 kvm_cpu_cap_set(X86_FEATURE_AMD_STIBP);
793 if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
794 kvm_cpu_cap_set(X86_FEATURE_AMD_SSBD);
795 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
796 kvm_cpu_cap_set(X86_FEATURE_AMD_SSB_NO);
797 /*
798 * The preference is to use SPEC CTRL MSR instead of the
799 * VIRT_SPEC MSR.
800 */
801 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
802 !boot_cpu_has(X86_FEATURE_AMD_SSBD))
803 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
804
805 /*
806 * Hide all SVM features by default, SVM will set the cap bits for
807 * features it emulates and/or exposes for L1.
808 */
809 kvm_cpu_cap_mask(CPUID_8000_000A_EDX, 0);
810
811 kvm_cpu_cap_mask(CPUID_8000_001F_EAX,
812 0 /* SME */ | 0 /* SEV */ | 0 /* VM_PAGE_FLUSH */ | 0 /* SEV_ES */ |
813 F(SME_COHERENT));
814
815 kvm_cpu_cap_mask(CPUID_8000_0021_EAX,
816 F(NO_NESTED_DATA_BP) | F(LFENCE_RDTSC) | 0 /* SmmPgCfgLock */ |
817 F(VERW_CLEAR) |
818 F(NULL_SEL_CLR_BASE) | F(AUTOIBRS) | 0 /* PrefetchCtlMsr */ |
819 F(WRMSR_XX_BASE_NS)
820 );
821
822 kvm_cpu_cap_check_and_set(X86_FEATURE_SBPB);
823 kvm_cpu_cap_check_and_set(X86_FEATURE_IBPB_BRTYPE);
824 kvm_cpu_cap_check_and_set(X86_FEATURE_SRSO_NO);
825 kvm_cpu_cap_check_and_set(X86_FEATURE_VERW_CLEAR);
826
827 kvm_cpu_cap_init_kvm_defined(CPUID_8000_0022_EAX,
828 F(PERFMON_V2)
829 );
830
831 kvm_cpu_cap_init_kvm_defined(CPUID_8000_0021_ECX,
832 F(TSA_SQ_NO) | F(TSA_L1_NO)
833 );
834
835 kvm_cpu_cap_check_and_set(X86_FEATURE_TSA_SQ_NO);
836 kvm_cpu_cap_check_and_set(X86_FEATURE_TSA_L1_NO);
837
838 /*
839 * Synthesize "LFENCE is serializing" into the AMD-defined entry in
840 * KVM's supported CPUID if the feature is reported as supported by the
841 * kernel. LFENCE_RDTSC was a Linux-defined synthetic feature long
842 * before AMD joined the bandwagon, e.g. LFENCE is serializing on most
843 * CPUs that support SSE2. On CPUs that don't support AMD's leaf,
844 * kvm_cpu_cap_mask() will unfortunately drop the flag due to ANDing
845 * the mask with the raw host CPUID, and reporting support in AMD's
846 * leaf can make it easier for userspace to detect the feature.
847 */
848 if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
849 kvm_cpu_cap_set(X86_FEATURE_LFENCE_RDTSC);
850 if (!static_cpu_has_bug(X86_BUG_NULL_SEG))
851 kvm_cpu_cap_set(X86_FEATURE_NULL_SEL_CLR_BASE);
852 kvm_cpu_cap_set(X86_FEATURE_NO_SMM_CTL_MSR);
853
854 kvm_cpu_cap_mask(CPUID_C000_0001_EDX,
855 F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) |
856 F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) |
857 F(PMM) | F(PMM_EN)
858 );
859
860 /*
861 * Hide RDTSCP and RDPID if either feature is reported as supported but
862 * probing MSR_TSC_AUX failed. This is purely a sanity check and
863 * should never happen, but the guest will likely crash if RDTSCP or
864 * RDPID is misreported, and KVM has botched MSR_TSC_AUX emulation in
865 * the past. For example, the sanity check may fire if this instance of
866 * KVM is running as L1 on top of an older, broken KVM.
867 */
868 if (WARN_ON((kvm_cpu_cap_has(X86_FEATURE_RDTSCP) ||
869 kvm_cpu_cap_has(X86_FEATURE_RDPID)) &&
870 !kvm_is_supported_user_return_msr(MSR_TSC_AUX))) {
871 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP);
872 kvm_cpu_cap_clear(X86_FEATURE_RDPID);
873 }
874 }
875 EXPORT_SYMBOL_GPL(kvm_set_cpu_caps);
876
877 struct kvm_cpuid_array {
878 struct kvm_cpuid_entry2 *entries;
879 int maxnent;
880 int nent;
881 };
882
get_next_cpuid(struct kvm_cpuid_array * array)883 static struct kvm_cpuid_entry2 *get_next_cpuid(struct kvm_cpuid_array *array)
884 {
885 if (array->nent >= array->maxnent)
886 return NULL;
887
888 return &array->entries[array->nent++];
889 }
890
do_host_cpuid(struct kvm_cpuid_array * array,u32 function,u32 index)891 static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array,
892 u32 function, u32 index)
893 {
894 struct kvm_cpuid_entry2 *entry = get_next_cpuid(array);
895
896 if (!entry)
897 return NULL;
898
899 memset(entry, 0, sizeof(*entry));
900 entry->function = function;
901 entry->index = index;
902 switch (function & 0xC0000000) {
903 case 0x40000000:
904 /* Hypervisor leaves are always synthesized by __do_cpuid_func. */
905 return entry;
906
907 case 0x80000000:
908 /*
909 * 0x80000021 is sometimes synthesized by __do_cpuid_func, which
910 * would result in out-of-bounds calls to do_host_cpuid.
911 */
912 {
913 static int max_cpuid_80000000;
914 if (!READ_ONCE(max_cpuid_80000000))
915 WRITE_ONCE(max_cpuid_80000000, cpuid_eax(0x80000000));
916 if (function > READ_ONCE(max_cpuid_80000000))
917 return entry;
918 }
919 break;
920
921 default:
922 break;
923 }
924
925 cpuid_count(entry->function, entry->index,
926 &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
927
928 if (cpuid_function_is_indexed(function))
929 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
930
931 return entry;
932 }
933
__do_cpuid_func_emulated(struct kvm_cpuid_array * array,u32 func)934 static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func)
935 {
936 struct kvm_cpuid_entry2 *entry;
937
938 if (array->nent >= array->maxnent)
939 return -E2BIG;
940
941 entry = &array->entries[array->nent];
942 entry->function = func;
943 entry->index = 0;
944 entry->flags = 0;
945
946 switch (func) {
947 case 0:
948 entry->eax = 7;
949 ++array->nent;
950 break;
951 case 1:
952 entry->ecx = F(MOVBE);
953 ++array->nent;
954 break;
955 case 7:
956 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
957 entry->eax = 0;
958 if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP))
959 entry->ecx = F(RDPID);
960 ++array->nent;
961 break;
962 default:
963 break;
964 }
965
966 return 0;
967 }
968
__do_cpuid_func(struct kvm_cpuid_array * array,u32 function)969 static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
970 {
971 struct kvm_cpuid_entry2 *entry;
972 int r, i, max_idx;
973
974 /* all calls to cpuid_count() should be made on the same cpu */
975 get_cpu();
976
977 r = -E2BIG;
978
979 entry = do_host_cpuid(array, function, 0);
980 if (!entry)
981 goto out;
982
983 switch (function) {
984 case 0:
985 /* Limited to the highest leaf implemented in KVM. */
986 entry->eax = min(entry->eax, 0x24U);
987 break;
988 case 1:
989 cpuid_entry_override(entry, CPUID_1_EDX);
990 cpuid_entry_override(entry, CPUID_1_ECX);
991 break;
992 case 2:
993 /*
994 * On ancient CPUs, function 2 entries are STATEFUL. That is,
995 * CPUID(function=2, index=0) may return different results each
996 * time, with the least-significant byte in EAX enumerating the
997 * number of times software should do CPUID(2, 0).
998 *
999 * Modern CPUs, i.e. every CPU KVM has *ever* run on are less
1000 * idiotic. Intel's SDM states that EAX & 0xff "will always
1001 * return 01H. Software should ignore this value and not
1002 * interpret it as an informational descriptor", while AMD's
1003 * APM states that CPUID(2) is reserved.
1004 *
1005 * WARN if a frankenstein CPU that supports virtualization and
1006 * a stateful CPUID.0x2 is encountered.
1007 */
1008 WARN_ON_ONCE((entry->eax & 0xff) > 1);
1009 break;
1010 /* functions 4 and 0x8000001d have additional index. */
1011 case 4:
1012 case 0x8000001d:
1013 /*
1014 * Read entries until the cache type in the previous entry is
1015 * zero, i.e. indicates an invalid entry.
1016 */
1017 for (i = 1; entry->eax & 0x1f; ++i) {
1018 entry = do_host_cpuid(array, function, i);
1019 if (!entry)
1020 goto out;
1021 }
1022 break;
1023 case 6: /* Thermal management */
1024 entry->eax = 0x4; /* allow ARAT */
1025 entry->ebx = 0;
1026 entry->ecx = 0;
1027 entry->edx = 0;
1028 break;
1029 /* function 7 has additional index. */
1030 case 7:
1031 max_idx = entry->eax = min(entry->eax, 2u);
1032 cpuid_entry_override(entry, CPUID_7_0_EBX);
1033 cpuid_entry_override(entry, CPUID_7_ECX);
1034 cpuid_entry_override(entry, CPUID_7_EDX);
1035
1036 /* KVM only supports up to 0x7.2, capped above via min(). */
1037 if (max_idx >= 1) {
1038 entry = do_host_cpuid(array, function, 1);
1039 if (!entry)
1040 goto out;
1041
1042 cpuid_entry_override(entry, CPUID_7_1_EAX);
1043 cpuid_entry_override(entry, CPUID_7_1_EDX);
1044 entry->ebx = 0;
1045 entry->ecx = 0;
1046 }
1047 if (max_idx >= 2) {
1048 entry = do_host_cpuid(array, function, 2);
1049 if (!entry)
1050 goto out;
1051
1052 cpuid_entry_override(entry, CPUID_7_2_EDX);
1053 entry->ecx = 0;
1054 entry->ebx = 0;
1055 entry->eax = 0;
1056 }
1057 break;
1058 case 0xa: { /* Architectural Performance Monitoring */
1059 union cpuid10_eax eax = { };
1060 union cpuid10_edx edx = { };
1061
1062 if (!enable_pmu || !static_cpu_has(X86_FEATURE_ARCH_PERFMON)) {
1063 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1064 break;
1065 }
1066
1067 eax.split.version_id = kvm_pmu_cap.version;
1068 eax.split.num_counters = kvm_pmu_cap.num_counters_gp;
1069 eax.split.bit_width = kvm_pmu_cap.bit_width_gp;
1070 eax.split.mask_length = kvm_pmu_cap.events_mask_len;
1071 edx.split.num_counters_fixed = kvm_pmu_cap.num_counters_fixed;
1072 edx.split.bit_width_fixed = kvm_pmu_cap.bit_width_fixed;
1073
1074 if (kvm_pmu_cap.version)
1075 edx.split.anythread_deprecated = 1;
1076
1077 entry->eax = eax.full;
1078 entry->ebx = kvm_pmu_cap.events_mask;
1079 entry->ecx = 0;
1080 entry->edx = edx.full;
1081 break;
1082 }
1083 case 0x1f:
1084 case 0xb:
1085 /*
1086 * No topology; a valid topology is indicated by the presence
1087 * of subleaf 1.
1088 */
1089 entry->eax = entry->ebx = entry->ecx = 0;
1090 break;
1091 case 0xd: {
1092 u64 permitted_xcr0 = kvm_get_filtered_xcr0();
1093 u64 permitted_xss = kvm_caps.supported_xss;
1094
1095 entry->eax &= permitted_xcr0;
1096 entry->ebx = xstate_required_size(permitted_xcr0, false);
1097 entry->ecx = entry->ebx;
1098 entry->edx &= permitted_xcr0 >> 32;
1099 if (!permitted_xcr0)
1100 break;
1101
1102 entry = do_host_cpuid(array, function, 1);
1103 if (!entry)
1104 goto out;
1105
1106 cpuid_entry_override(entry, CPUID_D_1_EAX);
1107 if (entry->eax & (F(XSAVES)|F(XSAVEC)))
1108 entry->ebx = xstate_required_size(permitted_xcr0 | permitted_xss,
1109 true);
1110 else {
1111 WARN_ON_ONCE(permitted_xss != 0);
1112 entry->ebx = 0;
1113 }
1114 entry->ecx &= permitted_xss;
1115 entry->edx &= permitted_xss >> 32;
1116
1117 for (i = 2; i < 64; ++i) {
1118 bool s_state;
1119 if (permitted_xcr0 & BIT_ULL(i))
1120 s_state = false;
1121 else if (permitted_xss & BIT_ULL(i))
1122 s_state = true;
1123 else
1124 continue;
1125
1126 entry = do_host_cpuid(array, function, i);
1127 if (!entry)
1128 goto out;
1129
1130 /*
1131 * The supported check above should have filtered out
1132 * invalid sub-leafs. Only valid sub-leafs should
1133 * reach this point, and they should have a non-zero
1134 * save state size. Furthermore, check whether the
1135 * processor agrees with permitted_xcr0/permitted_xss
1136 * on whether this is an XCR0- or IA32_XSS-managed area.
1137 */
1138 if (WARN_ON_ONCE(!entry->eax || (entry->ecx & 0x1) != s_state)) {
1139 --array->nent;
1140 continue;
1141 }
1142
1143 if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
1144 entry->ecx &= ~BIT_ULL(2);
1145 entry->edx = 0;
1146 }
1147 break;
1148 }
1149 case 0x12:
1150 /* Intel SGX */
1151 if (!kvm_cpu_cap_has(X86_FEATURE_SGX)) {
1152 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1153 break;
1154 }
1155
1156 /*
1157 * Index 0: Sub-features, MISCSELECT (a.k.a extended features)
1158 * and max enclave sizes. The SGX sub-features and MISCSELECT
1159 * are restricted by kernel and KVM capabilities (like most
1160 * feature flags), while enclave size is unrestricted.
1161 */
1162 cpuid_entry_override(entry, CPUID_12_EAX);
1163 entry->ebx &= SGX_MISC_EXINFO;
1164
1165 entry = do_host_cpuid(array, function, 1);
1166 if (!entry)
1167 goto out;
1168
1169 /*
1170 * Index 1: SECS.ATTRIBUTES. ATTRIBUTES are restricted a la
1171 * feature flags. Advertise all supported flags, including
1172 * privileged attributes that require explicit opt-in from
1173 * userspace. ATTRIBUTES.XFRM is not adjusted as userspace is
1174 * expected to derive it from supported XCR0.
1175 */
1176 entry->eax &= SGX_ATTR_PRIV_MASK | SGX_ATTR_UNPRIV_MASK;
1177 entry->ebx &= 0;
1178 break;
1179 /* Intel PT */
1180 case 0x14:
1181 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) {
1182 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1183 break;
1184 }
1185
1186 for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
1187 if (!do_host_cpuid(array, function, i))
1188 goto out;
1189 }
1190 break;
1191 /* Intel AMX TILE */
1192 case 0x1d:
1193 if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) {
1194 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1195 break;
1196 }
1197
1198 for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
1199 if (!do_host_cpuid(array, function, i))
1200 goto out;
1201 }
1202 break;
1203 case 0x1e: /* TMUL information */
1204 if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) {
1205 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1206 break;
1207 }
1208 break;
1209 case 0x24: {
1210 u8 avx10_version;
1211
1212 if (!kvm_cpu_cap_has(X86_FEATURE_AVX10)) {
1213 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1214 break;
1215 }
1216
1217 /*
1218 * The AVX10 version is encoded in EBX[7:0]. Note, the version
1219 * is guaranteed to be >=1 if AVX10 is supported. Note #2, the
1220 * version needs to be captured before overriding EBX features!
1221 */
1222 avx10_version = min_t(u8, entry->ebx & 0xff, 1);
1223 cpuid_entry_override(entry, CPUID_24_0_EBX);
1224 entry->ebx |= avx10_version;
1225
1226 entry->eax = 0;
1227 entry->ecx = 0;
1228 entry->edx = 0;
1229 break;
1230 }
1231 case KVM_CPUID_SIGNATURE: {
1232 const u32 *sigptr = (const u32 *)KVM_SIGNATURE;
1233 entry->eax = KVM_CPUID_FEATURES;
1234 entry->ebx = sigptr[0];
1235 entry->ecx = sigptr[1];
1236 entry->edx = sigptr[2];
1237 break;
1238 }
1239 case KVM_CPUID_FEATURES:
1240 entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) |
1241 (1 << KVM_FEATURE_NOP_IO_DELAY) |
1242 (1 << KVM_FEATURE_CLOCKSOURCE2) |
1243 (1 << KVM_FEATURE_ASYNC_PF) |
1244 (1 << KVM_FEATURE_PV_EOI) |
1245 (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
1246 (1 << KVM_FEATURE_PV_UNHALT) |
1247 (1 << KVM_FEATURE_PV_TLB_FLUSH) |
1248 (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
1249 (1 << KVM_FEATURE_PV_SEND_IPI) |
1250 (1 << KVM_FEATURE_POLL_CONTROL) |
1251 (1 << KVM_FEATURE_PV_SCHED_YIELD) |
1252 (1 << KVM_FEATURE_ASYNC_PF_INT);
1253
1254 if (sched_info_on())
1255 entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
1256
1257 entry->ebx = 0;
1258 entry->ecx = 0;
1259 entry->edx = 0;
1260 break;
1261 case 0x80000000:
1262 entry->eax = min(entry->eax, 0x80000022);
1263 /*
1264 * Serializing LFENCE is reported in a multitude of ways, and
1265 * NullSegClearsBase is not reported in CPUID on Zen2; help
1266 * userspace by providing the CPUID leaf ourselves.
1267 *
1268 * However, only do it if the host has CPUID leaf 0x8000001d.
1269 * QEMU thinks that it can query the host blindly for that
1270 * CPUID leaf if KVM reports that it supports 0x8000001d or
1271 * above. The processor merrily returns values from the
1272 * highest Intel leaf which QEMU tries to use as the guest's
1273 * 0x8000001d. Even worse, this can result in an infinite
1274 * loop if said highest leaf has no subleaves indexed by ECX.
1275 */
1276 if (entry->eax >= 0x8000001d &&
1277 (static_cpu_has(X86_FEATURE_LFENCE_RDTSC)
1278 || !static_cpu_has_bug(X86_BUG_NULL_SEG)))
1279 entry->eax = max(entry->eax, 0x80000021);
1280 break;
1281 case 0x80000001:
1282 entry->ebx &= ~GENMASK(27, 16);
1283 cpuid_entry_override(entry, CPUID_8000_0001_EDX);
1284 cpuid_entry_override(entry, CPUID_8000_0001_ECX);
1285 break;
1286 case 0x80000005:
1287 /* Pass host L1 cache and TLB info. */
1288 break;
1289 case 0x80000006:
1290 /* Drop reserved bits, pass host L2 cache and TLB info. */
1291 entry->edx &= ~GENMASK(17, 16);
1292 break;
1293 case 0x80000007: /* Advanced power management */
1294 cpuid_entry_override(entry, CPUID_8000_0007_EDX);
1295
1296 /* mask against host */
1297 entry->edx &= boot_cpu_data.x86_power;
1298 entry->eax = entry->ebx = entry->ecx = 0;
1299 break;
1300 case 0x80000008: {
1301 /*
1302 * GuestPhysAddrSize (EAX[23:16]) is intended for software
1303 * use.
1304 *
1305 * KVM's ABI is to report the effective MAXPHYADDR for the
1306 * guest in PhysAddrSize (phys_as), and the maximum
1307 * *addressable* GPA in GuestPhysAddrSize (g_phys_as).
1308 *
1309 * GuestPhysAddrSize is valid if and only if TDP is enabled,
1310 * in which case the max GPA that can be addressed by KVM may
1311 * be less than the max GPA that can be legally generated by
1312 * the guest, e.g. if MAXPHYADDR>48 but the CPU doesn't
1313 * support 5-level TDP.
1314 */
1315 unsigned int virt_as = max((entry->eax >> 8) & 0xff, 48U);
1316 unsigned int phys_as, g_phys_as;
1317
1318 /*
1319 * If TDP (NPT) is disabled use the adjusted host MAXPHYADDR as
1320 * the guest operates in the same PA space as the host, i.e.
1321 * reductions in MAXPHYADDR for memory encryption affect shadow
1322 * paging, too.
1323 *
1324 * If TDP is enabled, use the raw bare metal MAXPHYADDR as
1325 * reductions to the HPAs do not affect GPAs. The max
1326 * addressable GPA is the same as the max effective GPA, except
1327 * that it's capped at 48 bits if 5-level TDP isn't supported
1328 * (hardware processes bits 51:48 only when walking the fifth
1329 * level page table).
1330 */
1331 if (!tdp_enabled) {
1332 phys_as = boot_cpu_data.x86_phys_bits;
1333 g_phys_as = 0;
1334 } else {
1335 phys_as = entry->eax & 0xff;
1336 g_phys_as = phys_as;
1337 if (kvm_mmu_get_max_tdp_level() < 5)
1338 g_phys_as = min(g_phys_as, 48);
1339 }
1340
1341 entry->eax = phys_as | (virt_as << 8) | (g_phys_as << 16);
1342 entry->ecx &= ~(GENMASK(31, 16) | GENMASK(11, 8));
1343 entry->edx = 0;
1344 cpuid_entry_override(entry, CPUID_8000_0008_EBX);
1345 break;
1346 }
1347 case 0x8000000A:
1348 if (!kvm_cpu_cap_has(X86_FEATURE_SVM)) {
1349 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1350 break;
1351 }
1352 entry->eax = 1; /* SVM revision 1 */
1353 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
1354 ASID emulation to nested SVM */
1355 entry->ecx = 0; /* Reserved */
1356 cpuid_entry_override(entry, CPUID_8000_000A_EDX);
1357 break;
1358 case 0x80000019:
1359 entry->ecx = entry->edx = 0;
1360 break;
1361 case 0x8000001a:
1362 entry->eax &= GENMASK(2, 0);
1363 entry->ebx = entry->ecx = entry->edx = 0;
1364 break;
1365 case 0x8000001e:
1366 /* Do not return host topology information. */
1367 entry->eax = entry->ebx = entry->ecx = 0;
1368 entry->edx = 0; /* reserved */
1369 break;
1370 case 0x8000001F:
1371 if (!kvm_cpu_cap_has(X86_FEATURE_SEV)) {
1372 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1373 } else {
1374 cpuid_entry_override(entry, CPUID_8000_001F_EAX);
1375 /* Clear NumVMPL since KVM does not support VMPL. */
1376 entry->ebx &= ~GENMASK(31, 12);
1377 /*
1378 * Enumerate '0' for "PA bits reduction", the adjusted
1379 * MAXPHYADDR is enumerated directly (see 0x80000008).
1380 */
1381 entry->ebx &= ~GENMASK(11, 6);
1382 }
1383 break;
1384 case 0x80000020:
1385 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1386 break;
1387 case 0x80000021:
1388 entry->ebx = entry->edx = 0;
1389 cpuid_entry_override(entry, CPUID_8000_0021_EAX);
1390 cpuid_entry_override(entry, CPUID_8000_0021_ECX);
1391 break;
1392 /* AMD Extended Performance Monitoring and Debug */
1393 case 0x80000022: {
1394 union cpuid_0x80000022_ebx ebx = { };
1395
1396 entry->ecx = entry->edx = 0;
1397 if (!enable_pmu || !kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) {
1398 entry->eax = entry->ebx = 0;
1399 break;
1400 }
1401
1402 cpuid_entry_override(entry, CPUID_8000_0022_EAX);
1403
1404 if (kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2))
1405 ebx.split.num_core_pmc = kvm_pmu_cap.num_counters_gp;
1406 else if (kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
1407 ebx.split.num_core_pmc = AMD64_NUM_COUNTERS_CORE;
1408 else
1409 ebx.split.num_core_pmc = AMD64_NUM_COUNTERS;
1410
1411 entry->ebx = ebx.full;
1412 break;
1413 }
1414 /*Add support for Centaur's CPUID instruction*/
1415 case 0xC0000000:
1416 /*Just support up to 0xC0000004 now*/
1417 entry->eax = min(entry->eax, 0xC0000004);
1418 break;
1419 case 0xC0000001:
1420 cpuid_entry_override(entry, CPUID_C000_0001_EDX);
1421 break;
1422 case 3: /* Processor serial number */
1423 case 5: /* MONITOR/MWAIT */
1424 case 0xC0000002:
1425 case 0xC0000003:
1426 case 0xC0000004:
1427 default:
1428 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1429 break;
1430 }
1431
1432 r = 0;
1433
1434 out:
1435 put_cpu();
1436
1437 return r;
1438 }
1439
do_cpuid_func(struct kvm_cpuid_array * array,u32 func,unsigned int type)1440 static int do_cpuid_func(struct kvm_cpuid_array *array, u32 func,
1441 unsigned int type)
1442 {
1443 if (type == KVM_GET_EMULATED_CPUID)
1444 return __do_cpuid_func_emulated(array, func);
1445
1446 return __do_cpuid_func(array, func);
1447 }
1448
1449 #define CENTAUR_CPUID_SIGNATURE 0xC0000000
1450
get_cpuid_func(struct kvm_cpuid_array * array,u32 func,unsigned int type)1451 static int get_cpuid_func(struct kvm_cpuid_array *array, u32 func,
1452 unsigned int type)
1453 {
1454 u32 limit;
1455 int r;
1456
1457 if (func == CENTAUR_CPUID_SIGNATURE &&
1458 boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR)
1459 return 0;
1460
1461 r = do_cpuid_func(array, func, type);
1462 if (r)
1463 return r;
1464
1465 limit = array->entries[array->nent - 1].eax;
1466 for (func = func + 1; func <= limit; ++func) {
1467 r = do_cpuid_func(array, func, type);
1468 if (r)
1469 break;
1470 }
1471
1472 return r;
1473 }
1474
sanity_check_entries(struct kvm_cpuid_entry2 __user * entries,__u32 num_entries,unsigned int ioctl_type)1475 static bool sanity_check_entries(struct kvm_cpuid_entry2 __user *entries,
1476 __u32 num_entries, unsigned int ioctl_type)
1477 {
1478 int i;
1479 __u32 pad[3];
1480
1481 if (ioctl_type != KVM_GET_EMULATED_CPUID)
1482 return false;
1483
1484 /*
1485 * We want to make sure that ->padding is being passed clean from
1486 * userspace in case we want to use it for something in the future.
1487 *
1488 * Sadly, this wasn't enforced for KVM_GET_SUPPORTED_CPUID and so we
1489 * have to give ourselves satisfied only with the emulated side. /me
1490 * sheds a tear.
1491 */
1492 for (i = 0; i < num_entries; i++) {
1493 if (copy_from_user(pad, entries[i].padding, sizeof(pad)))
1494 return true;
1495
1496 if (pad[0] || pad[1] || pad[2])
1497 return true;
1498 }
1499 return false;
1500 }
1501
kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 * cpuid,struct kvm_cpuid_entry2 __user * entries,unsigned int type)1502 int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid,
1503 struct kvm_cpuid_entry2 __user *entries,
1504 unsigned int type)
1505 {
1506 static const u32 funcs[] = {
1507 0, 0x80000000, CENTAUR_CPUID_SIGNATURE, KVM_CPUID_SIGNATURE,
1508 };
1509
1510 struct kvm_cpuid_array array = {
1511 .nent = 0,
1512 };
1513 int r, i;
1514
1515 if (cpuid->nent < 1)
1516 return -E2BIG;
1517 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
1518 cpuid->nent = KVM_MAX_CPUID_ENTRIES;
1519
1520 if (sanity_check_entries(entries, cpuid->nent, type))
1521 return -EINVAL;
1522
1523 array.entries = kvcalloc(cpuid->nent, sizeof(struct kvm_cpuid_entry2), GFP_KERNEL);
1524 if (!array.entries)
1525 return -ENOMEM;
1526
1527 array.maxnent = cpuid->nent;
1528
1529 for (i = 0; i < ARRAY_SIZE(funcs); i++) {
1530 r = get_cpuid_func(&array, funcs[i], type);
1531 if (r)
1532 goto out_free;
1533 }
1534 cpuid->nent = array.nent;
1535
1536 if (copy_to_user(entries, array.entries,
1537 array.nent * sizeof(struct kvm_cpuid_entry2)))
1538 r = -EFAULT;
1539
1540 out_free:
1541 kvfree(array.entries);
1542 return r;
1543 }
1544
kvm_find_cpuid_entry_index(struct kvm_vcpu * vcpu,u32 function,u32 index)1545 struct kvm_cpuid_entry2 *kvm_find_cpuid_entry_index(struct kvm_vcpu *vcpu,
1546 u32 function, u32 index)
1547 {
1548 return cpuid_entry2_find(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
1549 function, index);
1550 }
1551 EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry_index);
1552
kvm_find_cpuid_entry(struct kvm_vcpu * vcpu,u32 function)1553 struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
1554 u32 function)
1555 {
1556 return cpuid_entry2_find(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
1557 function, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
1558 }
1559 EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
1560
1561 /*
1562 * Intel CPUID semantics treats any query for an out-of-range leaf as if the
1563 * highest basic leaf (i.e. CPUID.0H:EAX) were requested. AMD CPUID semantics
1564 * returns all zeroes for any undefined leaf, whether or not the leaf is in
1565 * range. Centaur/VIA follows Intel semantics.
1566 *
1567 * A leaf is considered out-of-range if its function is higher than the maximum
1568 * supported leaf of its associated class or if its associated class does not
1569 * exist.
1570 *
1571 * There are three primary classes to be considered, with their respective
1572 * ranges described as "<base> - <top>[,<base2> - <top2>] inclusive. A primary
1573 * class exists if a guest CPUID entry for its <base> leaf exists. For a given
1574 * class, CPUID.<base>.EAX contains the max supported leaf for the class.
1575 *
1576 * - Basic: 0x00000000 - 0x3fffffff, 0x50000000 - 0x7fffffff
1577 * - Hypervisor: 0x40000000 - 0x4fffffff
1578 * - Extended: 0x80000000 - 0xbfffffff
1579 * - Centaur: 0xc0000000 - 0xcfffffff
1580 *
1581 * The Hypervisor class is further subdivided into sub-classes that each act as
1582 * their own independent class associated with a 0x100 byte range. E.g. if Qemu
1583 * is advertising support for both HyperV and KVM, the resulting Hypervisor
1584 * CPUID sub-classes are:
1585 *
1586 * - HyperV: 0x40000000 - 0x400000ff
1587 * - KVM: 0x40000100 - 0x400001ff
1588 */
1589 static struct kvm_cpuid_entry2 *
get_out_of_range_cpuid_entry(struct kvm_vcpu * vcpu,u32 * fn_ptr,u32 index)1590 get_out_of_range_cpuid_entry(struct kvm_vcpu *vcpu, u32 *fn_ptr, u32 index)
1591 {
1592 struct kvm_cpuid_entry2 *basic, *class;
1593 u32 function = *fn_ptr;
1594
1595 basic = kvm_find_cpuid_entry(vcpu, 0);
1596 if (!basic)
1597 return NULL;
1598
1599 if (is_guest_vendor_amd(basic->ebx, basic->ecx, basic->edx) ||
1600 is_guest_vendor_hygon(basic->ebx, basic->ecx, basic->edx))
1601 return NULL;
1602
1603 if (function >= 0x40000000 && function <= 0x4fffffff)
1604 class = kvm_find_cpuid_entry(vcpu, function & 0xffffff00);
1605 else if (function >= 0xc0000000)
1606 class = kvm_find_cpuid_entry(vcpu, 0xc0000000);
1607 else
1608 class = kvm_find_cpuid_entry(vcpu, function & 0x80000000);
1609
1610 if (class && function <= class->eax)
1611 return NULL;
1612
1613 /*
1614 * Leaf specific adjustments are also applied when redirecting to the
1615 * max basic entry, e.g. if the max basic leaf is 0xb but there is no
1616 * entry for CPUID.0xb.index (see below), then the output value for EDX
1617 * needs to be pulled from CPUID.0xb.1.
1618 */
1619 *fn_ptr = basic->eax;
1620
1621 /*
1622 * The class does not exist or the requested function is out of range;
1623 * the effective CPUID entry is the max basic leaf. Note, the index of
1624 * the original requested leaf is observed!
1625 */
1626 return kvm_find_cpuid_entry_index(vcpu, basic->eax, index);
1627 }
1628
kvm_cpuid(struct kvm_vcpu * vcpu,u32 * eax,u32 * ebx,u32 * ecx,u32 * edx,bool exact_only)1629 bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx,
1630 u32 *ecx, u32 *edx, bool exact_only)
1631 {
1632 u32 orig_function = *eax, function = *eax, index = *ecx;
1633 struct kvm_cpuid_entry2 *entry;
1634 bool exact, used_max_basic = false;
1635
1636 entry = kvm_find_cpuid_entry_index(vcpu, function, index);
1637 exact = !!entry;
1638
1639 if (!entry && !exact_only) {
1640 entry = get_out_of_range_cpuid_entry(vcpu, &function, index);
1641 used_max_basic = !!entry;
1642 }
1643
1644 if (entry) {
1645 *eax = entry->eax;
1646 *ebx = entry->ebx;
1647 *ecx = entry->ecx;
1648 *edx = entry->edx;
1649 if (function == 7 && index == 0) {
1650 u64 data;
1651 if (!__kvm_get_msr(vcpu, MSR_IA32_TSX_CTRL, &data, true) &&
1652 (data & TSX_CTRL_CPUID_CLEAR))
1653 *ebx &= ~(F(RTM) | F(HLE));
1654 } else if (function == 0x80000007) {
1655 if (kvm_hv_invtsc_suppressed(vcpu))
1656 *edx &= ~SF(CONSTANT_TSC);
1657 }
1658 } else {
1659 *eax = *ebx = *ecx = *edx = 0;
1660 /*
1661 * When leaf 0BH or 1FH is defined, CL is pass-through
1662 * and EDX is always the x2APIC ID, even for undefined
1663 * subleaves. Index 1 will exist iff the leaf is
1664 * implemented, so we pass through CL iff leaf 1
1665 * exists. EDX can be copied from any existing index.
1666 */
1667 if (function == 0xb || function == 0x1f) {
1668 entry = kvm_find_cpuid_entry_index(vcpu, function, 1);
1669 if (entry) {
1670 *ecx = index & 0xff;
1671 *edx = entry->edx;
1672 }
1673 }
1674 }
1675 trace_kvm_cpuid(orig_function, index, *eax, *ebx, *ecx, *edx, exact,
1676 used_max_basic);
1677 return exact;
1678 }
1679 EXPORT_SYMBOL_GPL(kvm_cpuid);
1680
kvm_emulate_cpuid(struct kvm_vcpu * vcpu)1681 int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
1682 {
1683 u32 eax, ebx, ecx, edx;
1684
1685 if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
1686 return 1;
1687
1688 eax = kvm_rax_read(vcpu);
1689 ecx = kvm_rcx_read(vcpu);
1690 kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
1691 kvm_rax_write(vcpu, eax);
1692 kvm_rbx_write(vcpu, ebx);
1693 kvm_rcx_write(vcpu, ecx);
1694 kvm_rdx_write(vcpu, edx);
1695 return kvm_skip_emulated_instruction(vcpu);
1696 }
1697 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
1698