1 // SPDX-License-Identifier: GPL-2.0-only
2 /* cpu_feature_enabled() cannot be used this early */
3 #define USE_EARLY_PGTABLE_L5
4
5 #include <linux/memblock.h>
6 #include <linux/linkage.h>
7 #include <linux/bitops.h>
8 #include <linux/kernel.h>
9 #include <linux/export.h>
10 #include <linux/percpu.h>
11 #include <linux/string.h>
12 #include <linux/ctype.h>
13 #include <linux/delay.h>
14 #include <linux/sched/mm.h>
15 #include <linux/sched/clock.h>
16 #include <linux/sched/task.h>
17 #include <linux/sched/smt.h>
18 #include <linux/init.h>
19 #include <linux/kprobes.h>
20 #include <linux/kgdb.h>
21 #include <linux/smp.h>
22 #include <linux/io.h>
23 #include <linux/syscore_ops.h>
24 #include <linux/pgtable.h>
25
26 #include <asm/cmdline.h>
27 #include <asm/stackprotector.h>
28 #include <asm/perf_event.h>
29 #include <asm/mmu_context.h>
30 #include <asm/doublefault.h>
31 #include <asm/archrandom.h>
32 #include <asm/hypervisor.h>
33 #include <asm/processor.h>
34 #include <asm/tlbflush.h>
35 #include <asm/debugreg.h>
36 #include <asm/sections.h>
37 #include <asm/vsyscall.h>
38 #include <linux/topology.h>
39 #include <linux/cpumask.h>
40 #include <linux/atomic.h>
41 #include <asm/proto.h>
42 #include <asm/setup.h>
43 #include <asm/apic.h>
44 #include <asm/desc.h>
45 #include <asm/fpu/internal.h>
46 #include <asm/mtrr.h>
47 #include <asm/hwcap2.h>
48 #include <linux/numa.h>
49 #include <asm/numa.h>
50 #include <asm/asm.h>
51 #include <asm/bugs.h>
52 #include <asm/cpu.h>
53 #include <asm/mce.h>
54 #include <asm/msr.h>
55 #include <asm/memtype.h>
56 #include <asm/microcode.h>
57 #include <asm/microcode_intel.h>
58 #include <asm/intel-family.h>
59 #include <asm/cpu_device_id.h>
60 #include <asm/uv/uv.h>
61
62 #include "cpu.h"
63
64 u32 elf_hwcap2 __read_mostly;
65
66 /* all of these masks are initialized in setup_cpu_local_masks() */
67 cpumask_var_t cpu_initialized_mask;
68 cpumask_var_t cpu_callout_mask;
69 cpumask_var_t cpu_callin_mask;
70
71 /* representing cpus for which sibling maps can be computed */
72 cpumask_var_t cpu_sibling_setup_mask;
73
74 /* Number of siblings per CPU package */
75 int smp_num_siblings = 1;
76 EXPORT_SYMBOL(smp_num_siblings);
77
78 /* Last level cache ID of each logical CPU */
79 DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
80
81 /* correctly size the local cpu masks */
setup_cpu_local_masks(void)82 void __init setup_cpu_local_masks(void)
83 {
84 alloc_bootmem_cpumask_var(&cpu_initialized_mask);
85 alloc_bootmem_cpumask_var(&cpu_callin_mask);
86 alloc_bootmem_cpumask_var(&cpu_callout_mask);
87 alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
88 }
89
default_init(struct cpuinfo_x86 * c)90 static void default_init(struct cpuinfo_x86 *c)
91 {
92 #ifdef CONFIG_X86_64
93 cpu_detect_cache_sizes(c);
94 #else
95 /* Not much we can do here... */
96 /* Check if at least it has cpuid */
97 if (c->cpuid_level == -1) {
98 /* No cpuid. It must be an ancient CPU */
99 if (c->x86 == 4)
100 strcpy(c->x86_model_id, "486");
101 else if (c->x86 == 3)
102 strcpy(c->x86_model_id, "386");
103 }
104 #endif
105 }
106
107 static const struct cpu_dev default_cpu = {
108 .c_init = default_init,
109 .c_vendor = "Unknown",
110 .c_x86_vendor = X86_VENDOR_UNKNOWN,
111 };
112
113 static const struct cpu_dev *this_cpu = &default_cpu;
114
115 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
116 #ifdef CONFIG_X86_64
117 /*
118 * We need valid kernel segments for data and code in long mode too
119 * IRET will check the segment types kkeil 2000/10/28
120 * Also sysret mandates a special GDT layout
121 *
122 * TLS descriptors are currently at a different place compared to i386.
123 * Hopefully nobody expects them at a fixed place (Wine?)
124 */
125 [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
126 [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
127 [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
128 [GDT_ENTRY_DEFAULT_USER32_CS] = GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
129 [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
130 [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
131 #else
132 [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
133 [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
134 [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
135 [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
136 /*
137 * Segments used for calling PnP BIOS have byte granularity.
138 * They code segments and data segments have fixed 64k limits,
139 * the transfer segment sizes are set at run time.
140 */
141 /* 32-bit code */
142 [GDT_ENTRY_PNPBIOS_CS32] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
143 /* 16-bit code */
144 [GDT_ENTRY_PNPBIOS_CS16] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
145 /* 16-bit data */
146 [GDT_ENTRY_PNPBIOS_DS] = GDT_ENTRY_INIT(0x0092, 0, 0xffff),
147 /* 16-bit data */
148 [GDT_ENTRY_PNPBIOS_TS1] = GDT_ENTRY_INIT(0x0092, 0, 0),
149 /* 16-bit data */
150 [GDT_ENTRY_PNPBIOS_TS2] = GDT_ENTRY_INIT(0x0092, 0, 0),
151 /*
152 * The APM segments have byte granularity and their bases
153 * are set at run time. All have 64k limits.
154 */
155 /* 32-bit code */
156 [GDT_ENTRY_APMBIOS_BASE] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
157 /* 16-bit code */
158 [GDT_ENTRY_APMBIOS_BASE+1] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
159 /* data */
160 [GDT_ENTRY_APMBIOS_BASE+2] = GDT_ENTRY_INIT(0x4092, 0, 0xffff),
161
162 [GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
163 [GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
164 GDT_STACK_CANARY_INIT
165 #endif
166 } };
167 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
168
169 #ifdef CONFIG_X86_64
x86_nopcid_setup(char * s)170 static int __init x86_nopcid_setup(char *s)
171 {
172 /* nopcid doesn't accept parameters */
173 if (s)
174 return -EINVAL;
175
176 /* do not emit a message if the feature is not present */
177 if (!boot_cpu_has(X86_FEATURE_PCID))
178 return 0;
179
180 setup_clear_cpu_cap(X86_FEATURE_PCID);
181 pr_info("nopcid: PCID feature disabled\n");
182 return 0;
183 }
184 early_param("nopcid", x86_nopcid_setup);
185 #endif
186
x86_noinvpcid_setup(char * s)187 static int __init x86_noinvpcid_setup(char *s)
188 {
189 /* noinvpcid doesn't accept parameters */
190 if (s)
191 return -EINVAL;
192
193 /* do not emit a message if the feature is not present */
194 if (!boot_cpu_has(X86_FEATURE_INVPCID))
195 return 0;
196
197 setup_clear_cpu_cap(X86_FEATURE_INVPCID);
198 pr_info("noinvpcid: INVPCID feature disabled\n");
199 return 0;
200 }
201 early_param("noinvpcid", x86_noinvpcid_setup);
202
203 #ifdef CONFIG_X86_32
204 static int cachesize_override = -1;
205 static int disable_x86_serial_nr = 1;
206
cachesize_setup(char * str)207 static int __init cachesize_setup(char *str)
208 {
209 get_option(&str, &cachesize_override);
210 return 1;
211 }
212 __setup("cachesize=", cachesize_setup);
213
x86_sep_setup(char * s)214 static int __init x86_sep_setup(char *s)
215 {
216 setup_clear_cpu_cap(X86_FEATURE_SEP);
217 return 1;
218 }
219 __setup("nosep", x86_sep_setup);
220
221 /* Standard macro to see if a specific flag is changeable */
flag_is_changeable_p(u32 flag)222 static inline int flag_is_changeable_p(u32 flag)
223 {
224 u32 f1, f2;
225
226 /*
227 * Cyrix and IDT cpus allow disabling of CPUID
228 * so the code below may return different results
229 * when it is executed before and after enabling
230 * the CPUID. Add "volatile" to not allow gcc to
231 * optimize the subsequent calls to this function.
232 */
233 asm volatile ("pushfl \n\t"
234 "pushfl \n\t"
235 "popl %0 \n\t"
236 "movl %0, %1 \n\t"
237 "xorl %2, %0 \n\t"
238 "pushl %0 \n\t"
239 "popfl \n\t"
240 "pushfl \n\t"
241 "popl %0 \n\t"
242 "popfl \n\t"
243
244 : "=&r" (f1), "=&r" (f2)
245 : "ir" (flag));
246
247 return ((f1^f2) & flag) != 0;
248 }
249
250 /* Probe for the CPUID instruction */
have_cpuid_p(void)251 int have_cpuid_p(void)
252 {
253 return flag_is_changeable_p(X86_EFLAGS_ID);
254 }
255
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)256 static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
257 {
258 unsigned long lo, hi;
259
260 if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
261 return;
262
263 /* Disable processor serial number: */
264
265 rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
266 lo |= 0x200000;
267 wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
268
269 pr_notice("CPU serial number disabled.\n");
270 clear_cpu_cap(c, X86_FEATURE_PN);
271
272 /* Disabling the serial number may affect the cpuid level */
273 c->cpuid_level = cpuid_eax(0);
274 }
275
x86_serial_nr_setup(char * s)276 static int __init x86_serial_nr_setup(char *s)
277 {
278 disable_x86_serial_nr = 0;
279 return 1;
280 }
281 __setup("serialnumber", x86_serial_nr_setup);
282 #else
flag_is_changeable_p(u32 flag)283 static inline int flag_is_changeable_p(u32 flag)
284 {
285 return 1;
286 }
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)287 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
288 {
289 }
290 #endif
291
setup_disable_smep(char * arg)292 static __init int setup_disable_smep(char *arg)
293 {
294 setup_clear_cpu_cap(X86_FEATURE_SMEP);
295 return 1;
296 }
297 __setup("nosmep", setup_disable_smep);
298
setup_smep(struct cpuinfo_x86 * c)299 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
300 {
301 if (cpu_has(c, X86_FEATURE_SMEP))
302 cr4_set_bits(X86_CR4_SMEP);
303 }
304
setup_disable_smap(char * arg)305 static __init int setup_disable_smap(char *arg)
306 {
307 setup_clear_cpu_cap(X86_FEATURE_SMAP);
308 return 1;
309 }
310 __setup("nosmap", setup_disable_smap);
311
setup_smap(struct cpuinfo_x86 * c)312 static __always_inline void setup_smap(struct cpuinfo_x86 *c)
313 {
314 unsigned long eflags = native_save_fl();
315
316 /* This should have been cleared long ago */
317 BUG_ON(eflags & X86_EFLAGS_AC);
318
319 if (cpu_has(c, X86_FEATURE_SMAP)) {
320 #ifdef CONFIG_X86_SMAP
321 cr4_set_bits(X86_CR4_SMAP);
322 #else
323 clear_cpu_cap(c, X86_FEATURE_SMAP);
324 cr4_clear_bits(X86_CR4_SMAP);
325 #endif
326 }
327 }
328
setup_umip(struct cpuinfo_x86 * c)329 static __always_inline void setup_umip(struct cpuinfo_x86 *c)
330 {
331 /* Check the boot processor, plus build option for UMIP. */
332 if (!cpu_feature_enabled(X86_FEATURE_UMIP))
333 goto out;
334
335 /* Check the current processor's cpuid bits. */
336 if (!cpu_has(c, X86_FEATURE_UMIP))
337 goto out;
338
339 cr4_set_bits(X86_CR4_UMIP);
340
341 pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
342
343 return;
344
345 out:
346 /*
347 * Make sure UMIP is disabled in case it was enabled in a
348 * previous boot (e.g., via kexec).
349 */
350 cr4_clear_bits(X86_CR4_UMIP);
351 }
352
353 /* These bits should not change their value after CPU init is finished. */
354 static const unsigned long cr4_pinned_mask =
355 X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | X86_CR4_FSGSBASE;
356 static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
357 static unsigned long cr4_pinned_bits __ro_after_init;
358
native_write_cr0(unsigned long val)359 void native_write_cr0(unsigned long val)
360 {
361 unsigned long bits_missing = 0;
362
363 set_register:
364 asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
365
366 if (static_branch_likely(&cr_pinning)) {
367 if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
368 bits_missing = X86_CR0_WP;
369 val |= bits_missing;
370 goto set_register;
371 }
372 /* Warn after we've set the missing bits. */
373 WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
374 }
375 }
376 EXPORT_SYMBOL(native_write_cr0);
377
native_write_cr4(unsigned long val)378 void native_write_cr4(unsigned long val)
379 {
380 unsigned long bits_changed = 0;
381
382 set_register:
383 asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
384
385 if (static_branch_likely(&cr_pinning)) {
386 if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
387 bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
388 val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
389 goto set_register;
390 }
391 /* Warn after we've corrected the changed bits. */
392 WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
393 bits_changed);
394 }
395 }
396 #if IS_MODULE(CONFIG_LKDTM)
397 EXPORT_SYMBOL_GPL(native_write_cr4);
398 #endif
399
cr4_update_irqsoff(unsigned long set,unsigned long clear)400 void cr4_update_irqsoff(unsigned long set, unsigned long clear)
401 {
402 unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
403
404 lockdep_assert_irqs_disabled();
405
406 newval = (cr4 & ~clear) | set;
407 if (newval != cr4) {
408 this_cpu_write(cpu_tlbstate.cr4, newval);
409 __write_cr4(newval);
410 }
411 }
412 EXPORT_SYMBOL(cr4_update_irqsoff);
413
414 /* Read the CR4 shadow. */
cr4_read_shadow(void)415 unsigned long cr4_read_shadow(void)
416 {
417 return this_cpu_read(cpu_tlbstate.cr4);
418 }
419 EXPORT_SYMBOL_GPL(cr4_read_shadow);
420
cr4_init(void)421 void cr4_init(void)
422 {
423 unsigned long cr4 = __read_cr4();
424
425 if (boot_cpu_has(X86_FEATURE_PCID))
426 cr4 |= X86_CR4_PCIDE;
427 if (static_branch_likely(&cr_pinning))
428 cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
429
430 __write_cr4(cr4);
431
432 /* Initialize cr4 shadow for this CPU. */
433 this_cpu_write(cpu_tlbstate.cr4, cr4);
434 }
435
436 /*
437 * Once CPU feature detection is finished (and boot params have been
438 * parsed), record any of the sensitive CR bits that are set, and
439 * enable CR pinning.
440 */
setup_cr_pinning(void)441 static void __init setup_cr_pinning(void)
442 {
443 cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
444 static_key_enable(&cr_pinning.key);
445 }
446
x86_nofsgsbase_setup(char * arg)447 static __init int x86_nofsgsbase_setup(char *arg)
448 {
449 /* Require an exact match without trailing characters. */
450 if (strlen(arg))
451 return 0;
452
453 /* Do not emit a message if the feature is not present. */
454 if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
455 return 1;
456
457 setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
458 pr_info("FSGSBASE disabled via kernel command line\n");
459 return 1;
460 }
461 __setup("nofsgsbase", x86_nofsgsbase_setup);
462
463 /*
464 * Protection Keys are not available in 32-bit mode.
465 */
466 static bool pku_disabled;
467
setup_pku(struct cpuinfo_x86 * c)468 static __always_inline void setup_pku(struct cpuinfo_x86 *c)
469 {
470 struct pkru_state *pk;
471
472 /* check the boot processor, plus compile options for PKU: */
473 if (!cpu_feature_enabled(X86_FEATURE_PKU))
474 return;
475 /* checks the actual processor's cpuid bits: */
476 if (!cpu_has(c, X86_FEATURE_PKU))
477 return;
478 if (pku_disabled)
479 return;
480
481 cr4_set_bits(X86_CR4_PKE);
482 pk = get_xsave_addr(&init_fpstate.xsave, XFEATURE_PKRU);
483 if (pk)
484 pk->pkru = init_pkru_value;
485 /*
486 * Seting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
487 * cpuid bit to be set. We need to ensure that we
488 * update that bit in this CPU's "cpu_info".
489 */
490 set_cpu_cap(c, X86_FEATURE_OSPKE);
491 }
492
493 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
setup_disable_pku(char * arg)494 static __init int setup_disable_pku(char *arg)
495 {
496 /*
497 * Do not clear the X86_FEATURE_PKU bit. All of the
498 * runtime checks are against OSPKE so clearing the
499 * bit does nothing.
500 *
501 * This way, we will see "pku" in cpuinfo, but not
502 * "ospke", which is exactly what we want. It shows
503 * that the CPU has PKU, but the OS has not enabled it.
504 * This happens to be exactly how a system would look
505 * if we disabled the config option.
506 */
507 pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
508 pku_disabled = true;
509 return 1;
510 }
511 __setup("nopku", setup_disable_pku);
512 #endif /* CONFIG_X86_64 */
513
514 /*
515 * Some CPU features depend on higher CPUID levels, which may not always
516 * be available due to CPUID level capping or broken virtualization
517 * software. Add those features to this table to auto-disable them.
518 */
519 struct cpuid_dependent_feature {
520 u32 feature;
521 u32 level;
522 };
523
524 static const struct cpuid_dependent_feature
525 cpuid_dependent_features[] = {
526 { X86_FEATURE_MWAIT, 0x00000005 },
527 { X86_FEATURE_DCA, 0x00000009 },
528 { X86_FEATURE_XSAVE, 0x0000000d },
529 { 0, 0 }
530 };
531
filter_cpuid_features(struct cpuinfo_x86 * c,bool warn)532 static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
533 {
534 const struct cpuid_dependent_feature *df;
535
536 for (df = cpuid_dependent_features; df->feature; df++) {
537
538 if (!cpu_has(c, df->feature))
539 continue;
540 /*
541 * Note: cpuid_level is set to -1 if unavailable, but
542 * extended_extended_level is set to 0 if unavailable
543 * and the legitimate extended levels are all negative
544 * when signed; hence the weird messing around with
545 * signs here...
546 */
547 if (!((s32)df->level < 0 ?
548 (u32)df->level > (u32)c->extended_cpuid_level :
549 (s32)df->level > (s32)c->cpuid_level))
550 continue;
551
552 clear_cpu_cap(c, df->feature);
553 if (!warn)
554 continue;
555
556 pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
557 x86_cap_flag(df->feature), df->level);
558 }
559 }
560
561 /*
562 * Naming convention should be: <Name> [(<Codename>)]
563 * This table only is used unless init_<vendor>() below doesn't set it;
564 * in particular, if CPUID levels 0x80000002..4 are supported, this
565 * isn't used
566 */
567
568 /* Look up CPU names by table lookup. */
table_lookup_model(struct cpuinfo_x86 * c)569 static const char *table_lookup_model(struct cpuinfo_x86 *c)
570 {
571 #ifdef CONFIG_X86_32
572 const struct legacy_cpu_model_info *info;
573
574 if (c->x86_model >= 16)
575 return NULL; /* Range check */
576
577 if (!this_cpu)
578 return NULL;
579
580 info = this_cpu->legacy_models;
581
582 while (info->family) {
583 if (info->family == c->x86)
584 return info->model_names[c->x86_model];
585 info++;
586 }
587 #endif
588 return NULL; /* Not found */
589 }
590
591 /* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
592 __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
593 __u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
594
load_percpu_segment(int cpu)595 void load_percpu_segment(int cpu)
596 {
597 #ifdef CONFIG_X86_32
598 loadsegment(fs, __KERNEL_PERCPU);
599 #else
600 __loadsegment_simple(gs, 0);
601 wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
602 #endif
603 load_stack_canary_segment();
604 }
605
606 #ifdef CONFIG_X86_32
607 /* The 32-bit entry code needs to find cpu_entry_area. */
608 DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
609 #endif
610
611 /* Load the original GDT from the per-cpu structure */
load_direct_gdt(int cpu)612 void load_direct_gdt(int cpu)
613 {
614 struct desc_ptr gdt_descr;
615
616 gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
617 gdt_descr.size = GDT_SIZE - 1;
618 load_gdt(&gdt_descr);
619 }
620 EXPORT_SYMBOL_GPL(load_direct_gdt);
621
622 /* Load a fixmap remapping of the per-cpu GDT */
load_fixmap_gdt(int cpu)623 void load_fixmap_gdt(int cpu)
624 {
625 struct desc_ptr gdt_descr;
626
627 gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
628 gdt_descr.size = GDT_SIZE - 1;
629 load_gdt(&gdt_descr);
630 }
631 EXPORT_SYMBOL_GPL(load_fixmap_gdt);
632
633 /*
634 * Current gdt points %fs at the "master" per-cpu area: after this,
635 * it's on the real one.
636 */
switch_to_new_gdt(int cpu)637 void switch_to_new_gdt(int cpu)
638 {
639 /* Load the original GDT */
640 load_direct_gdt(cpu);
641 /* Reload the per-cpu base */
642 load_percpu_segment(cpu);
643 }
644
645 static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
646
get_model_name(struct cpuinfo_x86 * c)647 static void get_model_name(struct cpuinfo_x86 *c)
648 {
649 unsigned int *v;
650 char *p, *q, *s;
651
652 if (c->extended_cpuid_level < 0x80000004)
653 return;
654
655 v = (unsigned int *)c->x86_model_id;
656 cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
657 cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
658 cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
659 c->x86_model_id[48] = 0;
660
661 /* Trim whitespace */
662 p = q = s = &c->x86_model_id[0];
663
664 while (*p == ' ')
665 p++;
666
667 while (*p) {
668 /* Note the last non-whitespace index */
669 if (!isspace(*p))
670 s = q;
671
672 *q++ = *p++;
673 }
674
675 *(s + 1) = '\0';
676 }
677
detect_num_cpu_cores(struct cpuinfo_x86 * c)678 void detect_num_cpu_cores(struct cpuinfo_x86 *c)
679 {
680 unsigned int eax, ebx, ecx, edx;
681
682 c->x86_max_cores = 1;
683 if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
684 return;
685
686 cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
687 if (eax & 0x1f)
688 c->x86_max_cores = (eax >> 26) + 1;
689 }
690
cpu_detect_cache_sizes(struct cpuinfo_x86 * c)691 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
692 {
693 unsigned int n, dummy, ebx, ecx, edx, l2size;
694
695 n = c->extended_cpuid_level;
696
697 if (n >= 0x80000005) {
698 cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
699 c->x86_cache_size = (ecx>>24) + (edx>>24);
700 #ifdef CONFIG_X86_64
701 /* On K8 L1 TLB is inclusive, so don't count it */
702 c->x86_tlbsize = 0;
703 #endif
704 }
705
706 if (n < 0x80000006) /* Some chips just has a large L1. */
707 return;
708
709 cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
710 l2size = ecx >> 16;
711
712 #ifdef CONFIG_X86_64
713 c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
714 #else
715 /* do processor-specific cache resizing */
716 if (this_cpu->legacy_cache_size)
717 l2size = this_cpu->legacy_cache_size(c, l2size);
718
719 /* Allow user to override all this if necessary. */
720 if (cachesize_override != -1)
721 l2size = cachesize_override;
722
723 if (l2size == 0)
724 return; /* Again, no L2 cache is possible */
725 #endif
726
727 c->x86_cache_size = l2size;
728 }
729
730 u16 __read_mostly tlb_lli_4k[NR_INFO];
731 u16 __read_mostly tlb_lli_2m[NR_INFO];
732 u16 __read_mostly tlb_lli_4m[NR_INFO];
733 u16 __read_mostly tlb_lld_4k[NR_INFO];
734 u16 __read_mostly tlb_lld_2m[NR_INFO];
735 u16 __read_mostly tlb_lld_4m[NR_INFO];
736 u16 __read_mostly tlb_lld_1g[NR_INFO];
737
cpu_detect_tlb(struct cpuinfo_x86 * c)738 static void cpu_detect_tlb(struct cpuinfo_x86 *c)
739 {
740 if (this_cpu->c_detect_tlb)
741 this_cpu->c_detect_tlb(c);
742
743 pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
744 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
745 tlb_lli_4m[ENTRIES]);
746
747 pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
748 tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
749 tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
750 }
751
detect_ht_early(struct cpuinfo_x86 * c)752 int detect_ht_early(struct cpuinfo_x86 *c)
753 {
754 #ifdef CONFIG_SMP
755 u32 eax, ebx, ecx, edx;
756
757 if (!cpu_has(c, X86_FEATURE_HT))
758 return -1;
759
760 if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
761 return -1;
762
763 if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
764 return -1;
765
766 cpuid(1, &eax, &ebx, &ecx, &edx);
767
768 smp_num_siblings = (ebx & 0xff0000) >> 16;
769 if (smp_num_siblings == 1)
770 pr_info_once("CPU0: Hyper-Threading is disabled\n");
771 #endif
772 return 0;
773 }
774
detect_ht(struct cpuinfo_x86 * c)775 void detect_ht(struct cpuinfo_x86 *c)
776 {
777 #ifdef CONFIG_SMP
778 int index_msb, core_bits;
779
780 if (detect_ht_early(c) < 0)
781 return;
782
783 index_msb = get_count_order(smp_num_siblings);
784 c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
785
786 smp_num_siblings = smp_num_siblings / c->x86_max_cores;
787
788 index_msb = get_count_order(smp_num_siblings);
789
790 core_bits = get_count_order(c->x86_max_cores);
791
792 c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
793 ((1 << core_bits) - 1);
794 #endif
795 }
796
get_cpu_vendor(struct cpuinfo_x86 * c)797 static void get_cpu_vendor(struct cpuinfo_x86 *c)
798 {
799 char *v = c->x86_vendor_id;
800 int i;
801
802 for (i = 0; i < X86_VENDOR_NUM; i++) {
803 if (!cpu_devs[i])
804 break;
805
806 if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
807 (cpu_devs[i]->c_ident[1] &&
808 !strcmp(v, cpu_devs[i]->c_ident[1]))) {
809
810 this_cpu = cpu_devs[i];
811 c->x86_vendor = this_cpu->c_x86_vendor;
812 return;
813 }
814 }
815
816 pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
817 "CPU: Your system may be unstable.\n", v);
818
819 c->x86_vendor = X86_VENDOR_UNKNOWN;
820 this_cpu = &default_cpu;
821 }
822
cpu_detect(struct cpuinfo_x86 * c)823 void cpu_detect(struct cpuinfo_x86 *c)
824 {
825 /* Get vendor name */
826 cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
827 (unsigned int *)&c->x86_vendor_id[0],
828 (unsigned int *)&c->x86_vendor_id[8],
829 (unsigned int *)&c->x86_vendor_id[4]);
830
831 c->x86 = 4;
832 /* Intel-defined flags: level 0x00000001 */
833 if (c->cpuid_level >= 0x00000001) {
834 u32 junk, tfms, cap0, misc;
835
836 cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
837 c->x86 = x86_family(tfms);
838 c->x86_model = x86_model(tfms);
839 c->x86_stepping = x86_stepping(tfms);
840
841 if (cap0 & (1<<19)) {
842 c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
843 c->x86_cache_alignment = c->x86_clflush_size;
844 }
845 }
846 }
847
apply_forced_caps(struct cpuinfo_x86 * c)848 static void apply_forced_caps(struct cpuinfo_x86 *c)
849 {
850 int i;
851
852 for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
853 c->x86_capability[i] &= ~cpu_caps_cleared[i];
854 c->x86_capability[i] |= cpu_caps_set[i];
855 }
856 }
857
init_speculation_control(struct cpuinfo_x86 * c)858 static void init_speculation_control(struct cpuinfo_x86 *c)
859 {
860 /*
861 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
862 * and they also have a different bit for STIBP support. Also,
863 * a hypervisor might have set the individual AMD bits even on
864 * Intel CPUs, for finer-grained selection of what's available.
865 */
866 if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
867 set_cpu_cap(c, X86_FEATURE_IBRS);
868 set_cpu_cap(c, X86_FEATURE_IBPB);
869 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
870 }
871
872 if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
873 set_cpu_cap(c, X86_FEATURE_STIBP);
874
875 if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
876 cpu_has(c, X86_FEATURE_VIRT_SSBD))
877 set_cpu_cap(c, X86_FEATURE_SSBD);
878
879 if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
880 set_cpu_cap(c, X86_FEATURE_IBRS);
881 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
882 }
883
884 if (cpu_has(c, X86_FEATURE_AMD_IBPB))
885 set_cpu_cap(c, X86_FEATURE_IBPB);
886
887 if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
888 set_cpu_cap(c, X86_FEATURE_STIBP);
889 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
890 }
891
892 if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
893 set_cpu_cap(c, X86_FEATURE_SSBD);
894 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
895 clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
896 }
897 }
898
get_cpu_cap(struct cpuinfo_x86 * c)899 void get_cpu_cap(struct cpuinfo_x86 *c)
900 {
901 u32 eax, ebx, ecx, edx;
902
903 /* Intel-defined flags: level 0x00000001 */
904 if (c->cpuid_level >= 0x00000001) {
905 cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
906
907 c->x86_capability[CPUID_1_ECX] = ecx;
908 c->x86_capability[CPUID_1_EDX] = edx;
909 }
910
911 /* Thermal and Power Management Leaf: level 0x00000006 (eax) */
912 if (c->cpuid_level >= 0x00000006)
913 c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
914
915 /* Additional Intel-defined flags: level 0x00000007 */
916 if (c->cpuid_level >= 0x00000007) {
917 cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
918 c->x86_capability[CPUID_7_0_EBX] = ebx;
919 c->x86_capability[CPUID_7_ECX] = ecx;
920 c->x86_capability[CPUID_7_EDX] = edx;
921
922 /* Check valid sub-leaf index before accessing it */
923 if (eax >= 1) {
924 cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
925 c->x86_capability[CPUID_7_1_EAX] = eax;
926 }
927 }
928
929 /* Extended state features: level 0x0000000d */
930 if (c->cpuid_level >= 0x0000000d) {
931 cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
932
933 c->x86_capability[CPUID_D_1_EAX] = eax;
934 }
935
936 /* AMD-defined flags: level 0x80000001 */
937 eax = cpuid_eax(0x80000000);
938 c->extended_cpuid_level = eax;
939
940 if ((eax & 0xffff0000) == 0x80000000) {
941 if (eax >= 0x80000001) {
942 cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
943
944 c->x86_capability[CPUID_8000_0001_ECX] = ecx;
945 c->x86_capability[CPUID_8000_0001_EDX] = edx;
946 }
947 }
948
949 if (c->extended_cpuid_level >= 0x80000007) {
950 cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
951
952 c->x86_capability[CPUID_8000_0007_EBX] = ebx;
953 c->x86_power = edx;
954 }
955
956 if (c->extended_cpuid_level >= 0x80000008) {
957 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
958 c->x86_capability[CPUID_8000_0008_EBX] = ebx;
959 }
960
961 if (c->extended_cpuid_level >= 0x8000000a)
962 c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
963
964 init_scattered_cpuid_features(c);
965 init_speculation_control(c);
966
967 /*
968 * Clear/Set all flags overridden by options, after probe.
969 * This needs to happen each time we re-probe, which may happen
970 * several times during CPU initialization.
971 */
972 apply_forced_caps(c);
973 }
974
get_cpu_address_sizes(struct cpuinfo_x86 * c)975 void get_cpu_address_sizes(struct cpuinfo_x86 *c)
976 {
977 u32 eax, ebx, ecx, edx;
978
979 if (c->extended_cpuid_level >= 0x80000008) {
980 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
981
982 c->x86_virt_bits = (eax >> 8) & 0xff;
983 c->x86_phys_bits = eax & 0xff;
984 }
985 #ifdef CONFIG_X86_32
986 else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
987 c->x86_phys_bits = 36;
988 #endif
989 c->x86_cache_bits = c->x86_phys_bits;
990 }
991
identify_cpu_without_cpuid(struct cpuinfo_x86 * c)992 static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
993 {
994 #ifdef CONFIG_X86_32
995 int i;
996
997 /*
998 * First of all, decide if this is a 486 or higher
999 * It's a 486 if we can modify the AC flag
1000 */
1001 if (flag_is_changeable_p(X86_EFLAGS_AC))
1002 c->x86 = 4;
1003 else
1004 c->x86 = 3;
1005
1006 for (i = 0; i < X86_VENDOR_NUM; i++)
1007 if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1008 c->x86_vendor_id[0] = 0;
1009 cpu_devs[i]->c_identify(c);
1010 if (c->x86_vendor_id[0]) {
1011 get_cpu_vendor(c);
1012 break;
1013 }
1014 }
1015 #endif
1016 }
1017
1018 #define NO_SPECULATION BIT(0)
1019 #define NO_MELTDOWN BIT(1)
1020 #define NO_SSB BIT(2)
1021 #define NO_L1TF BIT(3)
1022 #define NO_MDS BIT(4)
1023 #define MSBDS_ONLY BIT(5)
1024 #define NO_SWAPGS BIT(6)
1025 #define NO_ITLB_MULTIHIT BIT(7)
1026 #define NO_SPECTRE_V2 BIT(8)
1027 #define NO_MMIO BIT(9)
1028 #define NO_EIBRS_PBRSB BIT(10)
1029
1030 #define VULNWL(vendor, family, model, whitelist) \
1031 X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1032
1033 #define VULNWL_INTEL(model, whitelist) \
1034 VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)
1035
1036 #define VULNWL_AMD(family, whitelist) \
1037 VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1038
1039 #define VULNWL_HYGON(family, whitelist) \
1040 VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1041
1042 static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1043 VULNWL(ANY, 4, X86_MODEL_ANY, NO_SPECULATION),
1044 VULNWL(CENTAUR, 5, X86_MODEL_ANY, NO_SPECULATION),
1045 VULNWL(INTEL, 5, X86_MODEL_ANY, NO_SPECULATION),
1046 VULNWL(NSC, 5, X86_MODEL_ANY, NO_SPECULATION),
1047
1048 /* Intel Family 6 */
1049 VULNWL_INTEL(TIGERLAKE, NO_MMIO),
1050 VULNWL_INTEL(TIGERLAKE_L, NO_MMIO),
1051 VULNWL_INTEL(ALDERLAKE, NO_MMIO),
1052 VULNWL_INTEL(ALDERLAKE_L, NO_MMIO),
1053
1054 VULNWL_INTEL(ATOM_SALTWELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
1055 VULNWL_INTEL(ATOM_SALTWELL_TABLET, NO_SPECULATION | NO_ITLB_MULTIHIT),
1056 VULNWL_INTEL(ATOM_SALTWELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
1057 VULNWL_INTEL(ATOM_BONNELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
1058 VULNWL_INTEL(ATOM_BONNELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
1059
1060 VULNWL_INTEL(ATOM_SILVERMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1061 VULNWL_INTEL(ATOM_SILVERMONT_D, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1062 VULNWL_INTEL(ATOM_SILVERMONT_MID, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1063 VULNWL_INTEL(ATOM_AIRMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1064 VULNWL_INTEL(XEON_PHI_KNL, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1065 VULNWL_INTEL(XEON_PHI_KNM, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1066
1067 VULNWL_INTEL(CORE_YONAH, NO_SSB),
1068
1069 VULNWL_INTEL(ATOM_AIRMONT_MID, NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1070 VULNWL_INTEL(ATOM_AIRMONT_NP, NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1071
1072 VULNWL_INTEL(ATOM_GOLDMONT, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1073 VULNWL_INTEL(ATOM_GOLDMONT_D, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1074 VULNWL_INTEL(ATOM_GOLDMONT_PLUS, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1075
1076 /*
1077 * Technically, swapgs isn't serializing on AMD (despite it previously
1078 * being documented as such in the APM). But according to AMD, %gs is
1079 * updated non-speculatively, and the issuing of %gs-relative memory
1080 * operands will be blocked until the %gs update completes, which is
1081 * good enough for our purposes.
1082 */
1083
1084 VULNWL_INTEL(ATOM_TREMONT, NO_EIBRS_PBRSB),
1085 VULNWL_INTEL(ATOM_TREMONT_L, NO_EIBRS_PBRSB),
1086 VULNWL_INTEL(ATOM_TREMONT_D, NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1087
1088 /* AMD Family 0xf - 0x12 */
1089 VULNWL_AMD(0x0f, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1090 VULNWL_AMD(0x10, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1091 VULNWL_AMD(0x11, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1092 VULNWL_AMD(0x12, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1093
1094 /* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1095 VULNWL_AMD(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1096 VULNWL_HYGON(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1097
1098 /* Zhaoxin Family 7 */
1099 VULNWL(CENTAUR, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1100 VULNWL(ZHAOXIN, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1101 {}
1102 };
1103
1104 #define VULNBL(vendor, family, model, blacklist) \
1105 X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1106
1107 #define VULNBL_INTEL_STEPPINGS(model, steppings, issues) \
1108 X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6, \
1109 INTEL_FAM6_##model, steppings, \
1110 X86_FEATURE_ANY, issues)
1111
1112 #define VULNBL_AMD(family, blacklist) \
1113 VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1114
1115 #define VULNBL_HYGON(family, blacklist) \
1116 VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1117
1118 #define SRBDS BIT(0)
1119 /* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1120 #define MMIO BIT(1)
1121 /* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1122 #define MMIO_SBDS BIT(2)
1123 /* CPU is affected by RETbleed, speculating where you would not expect it */
1124 #define RETBLEED BIT(3)
1125
1126 static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1127 VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS),
1128 VULNBL_INTEL_STEPPINGS(HASWELL, X86_STEPPING_ANY, SRBDS),
1129 VULNBL_INTEL_STEPPINGS(HASWELL_L, X86_STEPPING_ANY, SRBDS),
1130 VULNBL_INTEL_STEPPINGS(HASWELL_G, X86_STEPPING_ANY, SRBDS),
1131 VULNBL_INTEL_STEPPINGS(HASWELL_X, X86_STEPPING_ANY, MMIO),
1132 VULNBL_INTEL_STEPPINGS(BROADWELL_D, X86_STEPPING_ANY, MMIO),
1133 VULNBL_INTEL_STEPPINGS(BROADWELL_G, X86_STEPPING_ANY, SRBDS),
1134 VULNBL_INTEL_STEPPINGS(BROADWELL_X, X86_STEPPING_ANY, MMIO),
1135 VULNBL_INTEL_STEPPINGS(BROADWELL, X86_STEPPING_ANY, SRBDS),
1136 VULNBL_INTEL_STEPPINGS(SKYLAKE_L, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED),
1137 VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPING_ANY, MMIO | RETBLEED),
1138 VULNBL_INTEL_STEPPINGS(SKYLAKE, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED),
1139 VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED),
1140 VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPING_ANY, SRBDS | MMIO | RETBLEED),
1141 VULNBL_INTEL_STEPPINGS(CANNONLAKE_L, X86_STEPPING_ANY, RETBLEED),
1142 VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED),
1143 VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPING_ANY, MMIO),
1144 VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO),
1145 VULNBL_INTEL_STEPPINGS(COMETLAKE, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED),
1146 VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x0, 0x0), MMIO | RETBLEED),
1147 VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED),
1148 VULNBL_INTEL_STEPPINGS(LAKEFIELD, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED),
1149 VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED),
1150 VULNBL_INTEL_STEPPINGS(ATOM_TREMONT, X86_STEPPING_ANY, MMIO | MMIO_SBDS),
1151 VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPING_ANY, MMIO),
1152 VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS),
1153
1154 VULNBL_AMD(0x15, RETBLEED),
1155 VULNBL_AMD(0x16, RETBLEED),
1156 VULNBL_AMD(0x17, RETBLEED),
1157 VULNBL_HYGON(0x18, RETBLEED),
1158 {}
1159 };
1160
cpu_matches(const struct x86_cpu_id * table,unsigned long which)1161 static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1162 {
1163 const struct x86_cpu_id *m = x86_match_cpu(table);
1164
1165 return m && !!(m->driver_data & which);
1166 }
1167
x86_read_arch_cap_msr(void)1168 u64 x86_read_arch_cap_msr(void)
1169 {
1170 u64 ia32_cap = 0;
1171
1172 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1173 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
1174
1175 return ia32_cap;
1176 }
1177
arch_cap_mmio_immune(u64 ia32_cap)1178 static bool arch_cap_mmio_immune(u64 ia32_cap)
1179 {
1180 return (ia32_cap & ARCH_CAP_FBSDP_NO &&
1181 ia32_cap & ARCH_CAP_PSDP_NO &&
1182 ia32_cap & ARCH_CAP_SBDR_SSDP_NO);
1183 }
1184
cpu_set_bug_bits(struct cpuinfo_x86 * c)1185 static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1186 {
1187 u64 ia32_cap = x86_read_arch_cap_msr();
1188
1189 /* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1190 if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1191 !(ia32_cap & ARCH_CAP_PSCHANGE_MC_NO))
1192 setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1193
1194 if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1195 return;
1196
1197 setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1198
1199 if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2))
1200 setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1201
1202 if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1203 !(ia32_cap & ARCH_CAP_SSB_NO) &&
1204 !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1205 setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1206
1207 if (ia32_cap & ARCH_CAP_IBRS_ALL)
1208 setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1209
1210 if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1211 !(ia32_cap & ARCH_CAP_MDS_NO)) {
1212 setup_force_cpu_bug(X86_BUG_MDS);
1213 if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1214 setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1215 }
1216
1217 if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1218 setup_force_cpu_bug(X86_BUG_SWAPGS);
1219
1220 /*
1221 * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1222 * - TSX is supported or
1223 * - TSX_CTRL is present
1224 *
1225 * TSX_CTRL check is needed for cases when TSX could be disabled before
1226 * the kernel boot e.g. kexec.
1227 * TSX_CTRL check alone is not sufficient for cases when the microcode
1228 * update is not present or running as guest that don't get TSX_CTRL.
1229 */
1230 if (!(ia32_cap & ARCH_CAP_TAA_NO) &&
1231 (cpu_has(c, X86_FEATURE_RTM) ||
1232 (ia32_cap & ARCH_CAP_TSX_CTRL_MSR)))
1233 setup_force_cpu_bug(X86_BUG_TAA);
1234
1235 /*
1236 * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1237 * in the vulnerability blacklist.
1238 *
1239 * Some of the implications and mitigation of Shared Buffers Data
1240 * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1241 * SRBDS.
1242 */
1243 if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1244 cpu_has(c, X86_FEATURE_RDSEED)) &&
1245 cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1246 setup_force_cpu_bug(X86_BUG_SRBDS);
1247
1248 /*
1249 * Processor MMIO Stale Data bug enumeration
1250 *
1251 * Affected CPU list is generally enough to enumerate the vulnerability,
1252 * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1253 * not want the guest to enumerate the bug.
1254 *
1255 * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist,
1256 * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits.
1257 */
1258 if (!arch_cap_mmio_immune(ia32_cap)) {
1259 if (cpu_matches(cpu_vuln_blacklist, MMIO))
1260 setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1261 else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO))
1262 setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN);
1263 }
1264
1265 if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1266 if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (ia32_cap & ARCH_CAP_RSBA))
1267 setup_force_cpu_bug(X86_BUG_RETBLEED);
1268 }
1269
1270 if (cpu_has(c, X86_FEATURE_IBRS_ENHANCED) &&
1271 !cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1272 !(ia32_cap & ARCH_CAP_PBRSB_NO))
1273 setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1274
1275 if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1276 return;
1277
1278 /* Rogue Data Cache Load? No! */
1279 if (ia32_cap & ARCH_CAP_RDCL_NO)
1280 return;
1281
1282 setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1283
1284 if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1285 return;
1286
1287 setup_force_cpu_bug(X86_BUG_L1TF);
1288 }
1289
1290 /*
1291 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1292 * unfortunately, that's not true in practice because of early VIA
1293 * chips and (more importantly) broken virtualizers that are not easy
1294 * to detect. In the latter case it doesn't even *fail* reliably, so
1295 * probing for it doesn't even work. Disable it completely on 32-bit
1296 * unless we can find a reliable way to detect all the broken cases.
1297 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1298 */
detect_nopl(void)1299 static void detect_nopl(void)
1300 {
1301 #ifdef CONFIG_X86_32
1302 setup_clear_cpu_cap(X86_FEATURE_NOPL);
1303 #else
1304 setup_force_cpu_cap(X86_FEATURE_NOPL);
1305 #endif
1306 }
1307
1308 /*
1309 * We parse cpu parameters early because fpu__init_system() is executed
1310 * before parse_early_param().
1311 */
cpu_parse_early_param(void)1312 static void __init cpu_parse_early_param(void)
1313 {
1314 char arg[128];
1315 char *argptr = arg;
1316 int arglen, res, bit;
1317
1318 #ifdef CONFIG_X86_32
1319 if (cmdline_find_option_bool(boot_command_line, "no387"))
1320 #ifdef CONFIG_MATH_EMULATION
1321 setup_clear_cpu_cap(X86_FEATURE_FPU);
1322 #else
1323 pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1324 #endif
1325
1326 if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1327 setup_clear_cpu_cap(X86_FEATURE_FXSR);
1328 #endif
1329
1330 if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1331 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1332
1333 if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1334 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1335
1336 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1337 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1338
1339 arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1340 if (arglen <= 0)
1341 return;
1342
1343 pr_info("Clearing CPUID bits:");
1344 do {
1345 res = get_option(&argptr, &bit);
1346 if (res == 0 || res == 3)
1347 break;
1348
1349 /* If the argument was too long, the last bit may be cut off */
1350 if (res == 1 && arglen >= sizeof(arg))
1351 break;
1352
1353 if (bit >= 0 && bit < NCAPINTS * 32) {
1354 pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
1355 setup_clear_cpu_cap(bit);
1356 }
1357 } while (res == 2);
1358 pr_cont("\n");
1359 }
1360
1361 /*
1362 * Do minimum CPU detection early.
1363 * Fields really needed: vendor, cpuid_level, family, model, mask,
1364 * cache alignment.
1365 * The others are not touched to avoid unwanted side effects.
1366 *
1367 * WARNING: this function is only called on the boot CPU. Don't add code
1368 * here that is supposed to run on all CPUs.
1369 */
early_identify_cpu(struct cpuinfo_x86 * c)1370 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1371 {
1372 #ifdef CONFIG_X86_64
1373 c->x86_clflush_size = 64;
1374 c->x86_phys_bits = 36;
1375 c->x86_virt_bits = 48;
1376 #else
1377 c->x86_clflush_size = 32;
1378 c->x86_phys_bits = 32;
1379 c->x86_virt_bits = 32;
1380 #endif
1381 c->x86_cache_alignment = c->x86_clflush_size;
1382
1383 memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1384 c->extended_cpuid_level = 0;
1385
1386 if (!have_cpuid_p())
1387 identify_cpu_without_cpuid(c);
1388
1389 /* cyrix could have cpuid enabled via c_identify()*/
1390 if (have_cpuid_p()) {
1391 cpu_detect(c);
1392 get_cpu_vendor(c);
1393 get_cpu_cap(c);
1394 get_cpu_address_sizes(c);
1395 setup_force_cpu_cap(X86_FEATURE_CPUID);
1396 cpu_parse_early_param();
1397
1398 if (this_cpu->c_early_init)
1399 this_cpu->c_early_init(c);
1400
1401 c->cpu_index = 0;
1402 filter_cpuid_features(c, false);
1403
1404 if (this_cpu->c_bsp_init)
1405 this_cpu->c_bsp_init(c);
1406 } else {
1407 setup_clear_cpu_cap(X86_FEATURE_CPUID);
1408 }
1409
1410 setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1411
1412 cpu_set_bug_bits(c);
1413
1414 cpu_set_core_cap_bits(c);
1415
1416 fpu__init_system(c);
1417
1418 #ifdef CONFIG_X86_32
1419 /*
1420 * Regardless of whether PCID is enumerated, the SDM says
1421 * that it can't be enabled in 32-bit mode.
1422 */
1423 setup_clear_cpu_cap(X86_FEATURE_PCID);
1424 #endif
1425
1426 /*
1427 * Later in the boot process pgtable_l5_enabled() relies on
1428 * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1429 * enabled by this point we need to clear the feature bit to avoid
1430 * false-positives at the later stage.
1431 *
1432 * pgtable_l5_enabled() can be false here for several reasons:
1433 * - 5-level paging is disabled compile-time;
1434 * - it's 32-bit kernel;
1435 * - machine doesn't support 5-level paging;
1436 * - user specified 'no5lvl' in kernel command line.
1437 */
1438 if (!pgtable_l5_enabled())
1439 setup_clear_cpu_cap(X86_FEATURE_LA57);
1440
1441 detect_nopl();
1442 }
1443
early_cpu_init(void)1444 void __init early_cpu_init(void)
1445 {
1446 const struct cpu_dev *const *cdev;
1447 int count = 0;
1448
1449 #ifdef CONFIG_PROCESSOR_SELECT
1450 pr_info("KERNEL supported cpus:\n");
1451 #endif
1452
1453 for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1454 const struct cpu_dev *cpudev = *cdev;
1455
1456 if (count >= X86_VENDOR_NUM)
1457 break;
1458 cpu_devs[count] = cpudev;
1459 count++;
1460
1461 #ifdef CONFIG_PROCESSOR_SELECT
1462 {
1463 unsigned int j;
1464
1465 for (j = 0; j < 2; j++) {
1466 if (!cpudev->c_ident[j])
1467 continue;
1468 pr_info(" %s %s\n", cpudev->c_vendor,
1469 cpudev->c_ident[j]);
1470 }
1471 }
1472 #endif
1473 }
1474 early_identify_cpu(&boot_cpu_data);
1475 }
1476
detect_null_seg_behavior(void)1477 static bool detect_null_seg_behavior(void)
1478 {
1479 /*
1480 * Empirically, writing zero to a segment selector on AMD does
1481 * not clear the base, whereas writing zero to a segment
1482 * selector on Intel does clear the base. Intel's behavior
1483 * allows slightly faster context switches in the common case
1484 * where GS is unused by the prev and next threads.
1485 *
1486 * Since neither vendor documents this anywhere that I can see,
1487 * detect it directly instead of hardcoding the choice by
1488 * vendor.
1489 *
1490 * I've designated AMD's behavior as the "bug" because it's
1491 * counterintuitive and less friendly.
1492 */
1493
1494 unsigned long old_base, tmp;
1495 rdmsrl(MSR_FS_BASE, old_base);
1496 wrmsrl(MSR_FS_BASE, 1);
1497 loadsegment(fs, 0);
1498 rdmsrl(MSR_FS_BASE, tmp);
1499 wrmsrl(MSR_FS_BASE, old_base);
1500 return tmp == 0;
1501 }
1502
check_null_seg_clears_base(struct cpuinfo_x86 * c)1503 void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1504 {
1505 /* BUG_NULL_SEG is only relevant with 64bit userspace */
1506 if (!IS_ENABLED(CONFIG_X86_64))
1507 return;
1508
1509 /* Zen3 CPUs advertise Null Selector Clears Base in CPUID. */
1510 if (c->extended_cpuid_level >= 0x80000021 &&
1511 cpuid_eax(0x80000021) & BIT(6))
1512 return;
1513
1514 /*
1515 * CPUID bit above wasn't set. If this kernel is still running
1516 * as a HV guest, then the HV has decided not to advertize
1517 * that CPUID bit for whatever reason. For example, one
1518 * member of the migration pool might be vulnerable. Which
1519 * means, the bug is present: set the BUG flag and return.
1520 */
1521 if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1522 set_cpu_bug(c, X86_BUG_NULL_SEG);
1523 return;
1524 }
1525
1526 /*
1527 * Zen2 CPUs also have this behaviour, but no CPUID bit.
1528 * 0x18 is the respective family for Hygon.
1529 */
1530 if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1531 detect_null_seg_behavior())
1532 return;
1533
1534 /* All the remaining ones are affected */
1535 set_cpu_bug(c, X86_BUG_NULL_SEG);
1536 }
1537
generic_identify(struct cpuinfo_x86 * c)1538 static void generic_identify(struct cpuinfo_x86 *c)
1539 {
1540 c->extended_cpuid_level = 0;
1541
1542 if (!have_cpuid_p())
1543 identify_cpu_without_cpuid(c);
1544
1545 /* cyrix could have cpuid enabled via c_identify()*/
1546 if (!have_cpuid_p())
1547 return;
1548
1549 cpu_detect(c);
1550
1551 get_cpu_vendor(c);
1552
1553 get_cpu_cap(c);
1554
1555 get_cpu_address_sizes(c);
1556
1557 if (c->cpuid_level >= 0x00000001) {
1558 c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1559 #ifdef CONFIG_X86_32
1560 # ifdef CONFIG_SMP
1561 c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1562 # else
1563 c->apicid = c->initial_apicid;
1564 # endif
1565 #endif
1566 c->phys_proc_id = c->initial_apicid;
1567 }
1568
1569 get_model_name(c); /* Default name */
1570
1571 /*
1572 * ESPFIX is a strange bug. All real CPUs have it. Paravirt
1573 * systems that run Linux at CPL > 0 may or may not have the
1574 * issue, but, even if they have the issue, there's absolutely
1575 * nothing we can do about it because we can't use the real IRET
1576 * instruction.
1577 *
1578 * NB: For the time being, only 32-bit kernels support
1579 * X86_BUG_ESPFIX as such. 64-bit kernels directly choose
1580 * whether to apply espfix using paravirt hooks. If any
1581 * non-paravirt system ever shows up that does *not* have the
1582 * ESPFIX issue, we can change this.
1583 */
1584 #ifdef CONFIG_X86_32
1585 set_cpu_bug(c, X86_BUG_ESPFIX);
1586 #endif
1587 }
1588
1589 /*
1590 * Validate that ACPI/mptables have the same information about the
1591 * effective APIC id and update the package map.
1592 */
validate_apic_and_package_id(struct cpuinfo_x86 * c)1593 static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1594 {
1595 #ifdef CONFIG_SMP
1596 unsigned int apicid, cpu = smp_processor_id();
1597
1598 apicid = apic->cpu_present_to_apicid(cpu);
1599
1600 if (apicid != c->apicid) {
1601 pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1602 cpu, apicid, c->initial_apicid);
1603 }
1604 BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1605 BUG_ON(topology_update_die_map(c->cpu_die_id, cpu));
1606 #else
1607 c->logical_proc_id = 0;
1608 #endif
1609 }
1610
1611 /*
1612 * This does the hard work of actually picking apart the CPU stuff...
1613 */
identify_cpu(struct cpuinfo_x86 * c)1614 static void identify_cpu(struct cpuinfo_x86 *c)
1615 {
1616 int i;
1617
1618 c->loops_per_jiffy = loops_per_jiffy;
1619 c->x86_cache_size = 0;
1620 c->x86_vendor = X86_VENDOR_UNKNOWN;
1621 c->x86_model = c->x86_stepping = 0; /* So far unknown... */
1622 c->x86_vendor_id[0] = '\0'; /* Unset */
1623 c->x86_model_id[0] = '\0'; /* Unset */
1624 c->x86_max_cores = 1;
1625 c->x86_coreid_bits = 0;
1626 c->cu_id = 0xff;
1627 #ifdef CONFIG_X86_64
1628 c->x86_clflush_size = 64;
1629 c->x86_phys_bits = 36;
1630 c->x86_virt_bits = 48;
1631 #else
1632 c->cpuid_level = -1; /* CPUID not detected */
1633 c->x86_clflush_size = 32;
1634 c->x86_phys_bits = 32;
1635 c->x86_virt_bits = 32;
1636 #endif
1637 c->x86_cache_alignment = c->x86_clflush_size;
1638 memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1639 #ifdef CONFIG_X86_VMX_FEATURE_NAMES
1640 memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1641 #endif
1642
1643 generic_identify(c);
1644
1645 if (this_cpu->c_identify)
1646 this_cpu->c_identify(c);
1647
1648 /* Clear/Set all flags overridden by options, after probe */
1649 apply_forced_caps(c);
1650
1651 #ifdef CONFIG_X86_64
1652 c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1653 #endif
1654
1655 /*
1656 * Vendor-specific initialization. In this section we
1657 * canonicalize the feature flags, meaning if there are
1658 * features a certain CPU supports which CPUID doesn't
1659 * tell us, CPUID claiming incorrect flags, or other bugs,
1660 * we handle them here.
1661 *
1662 * At the end of this section, c->x86_capability better
1663 * indicate the features this CPU genuinely supports!
1664 */
1665 if (this_cpu->c_init)
1666 this_cpu->c_init(c);
1667
1668 /* Disable the PN if appropriate */
1669 squash_the_stupid_serial_number(c);
1670
1671 /* Set up SMEP/SMAP/UMIP */
1672 setup_smep(c);
1673 setup_smap(c);
1674 setup_umip(c);
1675
1676 /* Enable FSGSBASE instructions if available. */
1677 if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1678 cr4_set_bits(X86_CR4_FSGSBASE);
1679 elf_hwcap2 |= HWCAP2_FSGSBASE;
1680 }
1681
1682 /*
1683 * The vendor-specific functions might have changed features.
1684 * Now we do "generic changes."
1685 */
1686
1687 /* Filter out anything that depends on CPUID levels we don't have */
1688 filter_cpuid_features(c, true);
1689
1690 /* If the model name is still unset, do table lookup. */
1691 if (!c->x86_model_id[0]) {
1692 const char *p;
1693 p = table_lookup_model(c);
1694 if (p)
1695 strcpy(c->x86_model_id, p);
1696 else
1697 /* Last resort... */
1698 sprintf(c->x86_model_id, "%02x/%02x",
1699 c->x86, c->x86_model);
1700 }
1701
1702 #ifdef CONFIG_X86_64
1703 detect_ht(c);
1704 #endif
1705
1706 x86_init_rdrand(c);
1707 setup_pku(c);
1708
1709 /*
1710 * Clear/Set all flags overridden by options, need do it
1711 * before following smp all cpus cap AND.
1712 */
1713 apply_forced_caps(c);
1714
1715 /*
1716 * On SMP, boot_cpu_data holds the common feature set between
1717 * all CPUs; so make sure that we indicate which features are
1718 * common between the CPUs. The first time this routine gets
1719 * executed, c == &boot_cpu_data.
1720 */
1721 if (c != &boot_cpu_data) {
1722 /* AND the already accumulated flags with these */
1723 for (i = 0; i < NCAPINTS; i++)
1724 boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1725
1726 /* OR, i.e. replicate the bug flags */
1727 for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1728 c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1729 }
1730
1731 /* Init Machine Check Exception if available. */
1732 mcheck_cpu_init(c);
1733
1734 select_idle_routine(c);
1735
1736 #ifdef CONFIG_NUMA
1737 numa_add_cpu(smp_processor_id());
1738 #endif
1739 }
1740
1741 /*
1742 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
1743 * on 32-bit kernels:
1744 */
1745 #ifdef CONFIG_X86_32
enable_sep_cpu(void)1746 void enable_sep_cpu(void)
1747 {
1748 struct tss_struct *tss;
1749 int cpu;
1750
1751 if (!boot_cpu_has(X86_FEATURE_SEP))
1752 return;
1753
1754 cpu = get_cpu();
1755 tss = &per_cpu(cpu_tss_rw, cpu);
1756
1757 /*
1758 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
1759 * see the big comment in struct x86_hw_tss's definition.
1760 */
1761
1762 tss->x86_tss.ss1 = __KERNEL_CS;
1763 wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1764 wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1765 wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1766
1767 put_cpu();
1768 }
1769 #endif
1770
identify_boot_cpu(void)1771 void __init identify_boot_cpu(void)
1772 {
1773 identify_cpu(&boot_cpu_data);
1774 #ifdef CONFIG_X86_32
1775 sysenter_setup();
1776 enable_sep_cpu();
1777 #endif
1778 cpu_detect_tlb(&boot_cpu_data);
1779 setup_cr_pinning();
1780
1781 tsx_init();
1782 }
1783
identify_secondary_cpu(struct cpuinfo_x86 * c)1784 void identify_secondary_cpu(struct cpuinfo_x86 *c)
1785 {
1786 BUG_ON(c == &boot_cpu_data);
1787 identify_cpu(c);
1788 #ifdef CONFIG_X86_32
1789 enable_sep_cpu();
1790 #endif
1791 mtrr_ap_init();
1792 validate_apic_and_package_id(c);
1793 x86_spec_ctrl_setup_ap();
1794 update_srbds_msr();
1795 }
1796
setup_noclflush(char * arg)1797 static __init int setup_noclflush(char *arg)
1798 {
1799 setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1800 setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
1801 return 1;
1802 }
1803 __setup("noclflush", setup_noclflush);
1804
print_cpu_info(struct cpuinfo_x86 * c)1805 void print_cpu_info(struct cpuinfo_x86 *c)
1806 {
1807 const char *vendor = NULL;
1808
1809 if (c->x86_vendor < X86_VENDOR_NUM) {
1810 vendor = this_cpu->c_vendor;
1811 } else {
1812 if (c->cpuid_level >= 0)
1813 vendor = c->x86_vendor_id;
1814 }
1815
1816 if (vendor && !strstr(c->x86_model_id, vendor))
1817 pr_cont("%s ", vendor);
1818
1819 if (c->x86_model_id[0])
1820 pr_cont("%s", c->x86_model_id);
1821 else
1822 pr_cont("%d86", c->x86);
1823
1824 pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1825
1826 if (c->x86_stepping || c->cpuid_level >= 0)
1827 pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
1828 else
1829 pr_cont(")\n");
1830 }
1831
1832 /*
1833 * clearcpuid= was already parsed in fpu__init_parse_early_param.
1834 * But we need to keep a dummy __setup around otherwise it would
1835 * show up as an environment variable for init.
1836 */
setup_clearcpuid(char * arg)1837 static __init int setup_clearcpuid(char *arg)
1838 {
1839 return 1;
1840 }
1841 __setup("clearcpuid=", setup_clearcpuid);
1842
1843 #ifdef CONFIG_X86_64
1844 DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
1845 fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
1846 EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
1847
1848 /*
1849 * The following percpu variables are hot. Align current_task to
1850 * cacheline size such that they fall in the same cacheline.
1851 */
1852 DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
1853 &init_task;
1854 EXPORT_PER_CPU_SYMBOL(current_task);
1855
1856 DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
1857 DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1858
1859 DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
1860 EXPORT_PER_CPU_SYMBOL(__preempt_count);
1861
1862 /* May not be marked __init: used by software suspend */
syscall_init(void)1863 void syscall_init(void)
1864 {
1865 wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
1866 wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
1867
1868 #ifdef CONFIG_IA32_EMULATION
1869 wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat);
1870 /*
1871 * This only works on Intel CPUs.
1872 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
1873 * This does not cause SYSENTER to jump to the wrong location, because
1874 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
1875 */
1876 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
1877 wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
1878 (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
1879 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
1880 #else
1881 wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
1882 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1883 wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
1884 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1885 #endif
1886
1887 /* Flags to clear on syscall */
1888 wrmsrl(MSR_SYSCALL_MASK,
1889 X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1890 X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
1891 }
1892
1893 #else /* CONFIG_X86_64 */
1894
1895 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
1896 EXPORT_PER_CPU_SYMBOL(current_task);
1897 DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
1898 EXPORT_PER_CPU_SYMBOL(__preempt_count);
1899
1900 /*
1901 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
1902 * the top of the kernel stack. Use an extra percpu variable to track the
1903 * top of the kernel stack directly.
1904 */
1905 DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
1906 (unsigned long)&init_thread_union + THREAD_SIZE;
1907 EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
1908
1909 #ifdef CONFIG_STACKPROTECTOR
1910 DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1911 #endif
1912
1913 #endif /* CONFIG_X86_64 */
1914
1915 /*
1916 * Clear all 6 debug registers:
1917 */
clear_all_debug_regs(void)1918 static void clear_all_debug_regs(void)
1919 {
1920 int i;
1921
1922 for (i = 0; i < 8; i++) {
1923 /* Ignore db4, db5 */
1924 if ((i == 4) || (i == 5))
1925 continue;
1926
1927 set_debugreg(0, i);
1928 }
1929 }
1930
1931 #ifdef CONFIG_KGDB
1932 /*
1933 * Restore debug regs if using kgdbwait and you have a kernel debugger
1934 * connection established.
1935 */
dbg_restore_debug_regs(void)1936 static void dbg_restore_debug_regs(void)
1937 {
1938 if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
1939 arch_kgdb_ops.correct_hw_break();
1940 }
1941 #else /* ! CONFIG_KGDB */
1942 #define dbg_restore_debug_regs()
1943 #endif /* ! CONFIG_KGDB */
1944
wait_for_master_cpu(int cpu)1945 static void wait_for_master_cpu(int cpu)
1946 {
1947 #ifdef CONFIG_SMP
1948 /*
1949 * wait for ACK from master CPU before continuing
1950 * with AP initialization
1951 */
1952 WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
1953 while (!cpumask_test_cpu(cpu, cpu_callout_mask))
1954 cpu_relax();
1955 #endif
1956 }
1957
1958 #ifdef CONFIG_X86_64
setup_getcpu(int cpu)1959 static inline void setup_getcpu(int cpu)
1960 {
1961 unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
1962 struct desc_struct d = { };
1963
1964 if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
1965 write_rdtscp_aux(cpudata);
1966
1967 /* Store CPU and node number in limit. */
1968 d.limit0 = cpudata;
1969 d.limit1 = cpudata >> 16;
1970
1971 d.type = 5; /* RO data, expand down, accessed */
1972 d.dpl = 3; /* Visible to user code */
1973 d.s = 1; /* Not a system segment */
1974 d.p = 1; /* Present */
1975 d.d = 1; /* 32-bit */
1976
1977 write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
1978 }
1979
ucode_cpu_init(int cpu)1980 static inline void ucode_cpu_init(int cpu)
1981 {
1982 if (cpu)
1983 load_ucode_ap();
1984 }
1985
tss_setup_ist(struct tss_struct * tss)1986 static inline void tss_setup_ist(struct tss_struct *tss)
1987 {
1988 /* Set up the per-CPU TSS IST stacks */
1989 tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
1990 tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
1991 tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
1992 tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
1993 /* Only mapped when SEV-ES is active */
1994 tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
1995 }
1996
1997 #else /* CONFIG_X86_64 */
1998
setup_getcpu(int cpu)1999 static inline void setup_getcpu(int cpu) { }
2000
ucode_cpu_init(int cpu)2001 static inline void ucode_cpu_init(int cpu)
2002 {
2003 show_ucode_info_early();
2004 }
2005
tss_setup_ist(struct tss_struct * tss)2006 static inline void tss_setup_ist(struct tss_struct *tss) { }
2007
2008 #endif /* !CONFIG_X86_64 */
2009
tss_setup_io_bitmap(struct tss_struct * tss)2010 static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2011 {
2012 tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2013
2014 #ifdef CONFIG_X86_IOPL_IOPERM
2015 tss->io_bitmap.prev_max = 0;
2016 tss->io_bitmap.prev_sequence = 0;
2017 memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2018 /*
2019 * Invalidate the extra array entry past the end of the all
2020 * permission bitmap as required by the hardware.
2021 */
2022 tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2023 #endif
2024 }
2025
2026 /*
2027 * Setup everything needed to handle exceptions from the IDT, including the IST
2028 * exceptions which use paranoid_entry().
2029 */
cpu_init_exception_handling(void)2030 void cpu_init_exception_handling(void)
2031 {
2032 struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2033 int cpu = raw_smp_processor_id();
2034
2035 /* paranoid_entry() gets the CPU number from the GDT */
2036 setup_getcpu(cpu);
2037
2038 /* IST vectors need TSS to be set up. */
2039 tss_setup_ist(tss);
2040 tss_setup_io_bitmap(tss);
2041 set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2042
2043 load_TR_desc();
2044
2045 /* Finally load the IDT */
2046 load_current_idt();
2047 }
2048
2049 /*
2050 * cpu_init() initializes state that is per-CPU. Some data is already
2051 * initialized (naturally) in the bootstrap process, such as the GDT
2052 * and IDT. We reload them nevertheless, this function acts as a
2053 * 'CPU state barrier', nothing should get across.
2054 */
cpu_init(void)2055 void cpu_init(void)
2056 {
2057 struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2058 struct task_struct *cur = current;
2059 int cpu = raw_smp_processor_id();
2060
2061 wait_for_master_cpu(cpu);
2062
2063 ucode_cpu_init(cpu);
2064
2065 #ifdef CONFIG_NUMA
2066 if (this_cpu_read(numa_node) == 0 &&
2067 early_cpu_to_node(cpu) != NUMA_NO_NODE)
2068 set_numa_node(early_cpu_to_node(cpu));
2069 #endif
2070 setup_getcpu(cpu);
2071
2072 pr_debug("Initializing CPU#%d\n", cpu);
2073
2074 if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2075 boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2076 cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2077
2078 /*
2079 * Initialize the per-CPU GDT with the boot GDT,
2080 * and set up the GDT descriptor:
2081 */
2082 switch_to_new_gdt(cpu);
2083 load_current_idt();
2084
2085 if (IS_ENABLED(CONFIG_X86_64)) {
2086 loadsegment(fs, 0);
2087 memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2088 syscall_init();
2089
2090 wrmsrl(MSR_FS_BASE, 0);
2091 wrmsrl(MSR_KERNEL_GS_BASE, 0);
2092 barrier();
2093
2094 x2apic_setup();
2095 }
2096
2097 mmgrab(&init_mm);
2098 cur->active_mm = &init_mm;
2099 BUG_ON(cur->mm);
2100 initialize_tlbstate_and_flush();
2101 enter_lazy_tlb(&init_mm, cur);
2102
2103 /* Initialize the TSS. */
2104 tss_setup_ist(tss);
2105 tss_setup_io_bitmap(tss);
2106 set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2107
2108 load_TR_desc();
2109 /*
2110 * sp0 points to the entry trampoline stack regardless of what task
2111 * is running.
2112 */
2113 load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2114
2115 load_mm_ldt(&init_mm);
2116
2117 clear_all_debug_regs();
2118 dbg_restore_debug_regs();
2119
2120 doublefault_init_cpu_tss();
2121
2122 fpu__init_cpu();
2123
2124 if (is_uv_system())
2125 uv_cpu_init();
2126
2127 load_fixmap_gdt(cpu);
2128 }
2129
2130 /*
2131 * The microcode loader calls this upon late microcode load to recheck features,
2132 * only when microcode has been updated. Caller holds microcode_mutex and CPU
2133 * hotplug lock.
2134 */
microcode_check(void)2135 void microcode_check(void)
2136 {
2137 struct cpuinfo_x86 info;
2138
2139 perf_check_microcode();
2140
2141 /* Reload CPUID max function as it might've changed. */
2142 info.cpuid_level = cpuid_eax(0);
2143
2144 /*
2145 * Copy all capability leafs to pick up the synthetic ones so that
2146 * memcmp() below doesn't fail on that. The ones coming from CPUID will
2147 * get overwritten in get_cpu_cap().
2148 */
2149 memcpy(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability));
2150
2151 get_cpu_cap(&info);
2152
2153 if (!memcmp(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability)))
2154 return;
2155
2156 pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2157 pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2158 }
2159
2160 /*
2161 * Invoked from core CPU hotplug code after hotplug operations
2162 */
arch_smt_update(void)2163 void arch_smt_update(void)
2164 {
2165 /* Handle the speculative execution misfeatures */
2166 cpu_bugs_smt_update();
2167 /* Check whether IPI broadcasting can be enabled */
2168 apic_smt_update();
2169 }
2170