1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/export.h>
3 #include <linux/bitops.h>
4 #include <linux/elf.h>
5 #include <linux/mm.h>
6
7 #include <linux/io.h>
8 #include <linux/sched.h>
9 #include <linux/sched/clock.h>
10 #include <linux/random.h>
11 #include <linux/topology.h>
12 #include <asm/processor.h>
13 #include <asm/apic.h>
14 #include <asm/cacheinfo.h>
15 #include <asm/cpu.h>
16 #include <asm/cpu_device_id.h>
17 #include <asm/spec-ctrl.h>
18 #include <asm/smp.h>
19 #include <asm/numa.h>
20 #include <asm/pci-direct.h>
21 #include <asm/delay.h>
22 #include <asm/debugreg.h>
23 #include <asm/resctrl.h>
24 #include <asm/sev.h>
25
26 #ifdef CONFIG_X86_64
27 # include <asm/mmconfig.h>
28 #endif
29
30 #include "cpu.h"
31
rdmsrl_amd_safe(unsigned msr,unsigned long long * p)32 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
33 {
34 u32 gprs[8] = { 0 };
35 int err;
36
37 WARN_ONCE((boot_cpu_data.x86 != 0xf),
38 "%s should only be used on K8!\n", __func__);
39
40 gprs[1] = msr;
41 gprs[7] = 0x9c5a203a;
42
43 err = rdmsr_safe_regs(gprs);
44
45 *p = gprs[0] | ((u64)gprs[2] << 32);
46
47 return err;
48 }
49
wrmsrl_amd_safe(unsigned msr,unsigned long long val)50 static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
51 {
52 u32 gprs[8] = { 0 };
53
54 WARN_ONCE((boot_cpu_data.x86 != 0xf),
55 "%s should only be used on K8!\n", __func__);
56
57 gprs[0] = (u32)val;
58 gprs[1] = msr;
59 gprs[2] = val >> 32;
60 gprs[7] = 0x9c5a203a;
61
62 return wrmsr_safe_regs(gprs);
63 }
64
65 /*
66 * B step AMD K6 before B 9730xxxx have hardware bugs that can cause
67 * misexecution of code under Linux. Owners of such processors should
68 * contact AMD for precise details and a CPU swap.
69 *
70 * See http://www.multimania.com/poulot/k6bug.html
71 * and section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6"
72 * (Publication # 21266 Issue Date: August 1998)
73 *
74 * The following test is erm.. interesting. AMD neglected to up
75 * the chip setting when fixing the bug but they also tweaked some
76 * performance at the same time..
77 */
78
79 #ifdef CONFIG_X86_32
80 extern __visible void vide(void);
81 __asm__(".text\n"
82 ".globl vide\n"
83 ".type vide, @function\n"
84 ".align 4\n"
85 "vide: ret\n");
86 #endif
87
init_amd_k5(struct cpuinfo_x86 * c)88 static void init_amd_k5(struct cpuinfo_x86 *c)
89 {
90 #ifdef CONFIG_X86_32
91 /*
92 * General Systems BIOSen alias the cpu frequency registers
93 * of the Elan at 0x000df000. Unfortunately, one of the Linux
94 * drivers subsequently pokes it, and changes the CPU speed.
95 * Workaround : Remove the unneeded alias.
96 */
97 #define CBAR (0xfffc) /* Configuration Base Address (32-bit) */
98 #define CBAR_ENB (0x80000000)
99 #define CBAR_KEY (0X000000CB)
100 if (c->x86_model == 9 || c->x86_model == 10) {
101 if (inl(CBAR) & CBAR_ENB)
102 outl(0 | CBAR_KEY, CBAR);
103 }
104 #endif
105 }
106
init_amd_k6(struct cpuinfo_x86 * c)107 static void init_amd_k6(struct cpuinfo_x86 *c)
108 {
109 #ifdef CONFIG_X86_32
110 u32 l, h;
111 int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
112
113 if (c->x86_model < 6) {
114 /* Based on AMD doc 20734R - June 2000 */
115 if (c->x86_model == 0) {
116 clear_cpu_cap(c, X86_FEATURE_APIC);
117 set_cpu_cap(c, X86_FEATURE_PGE);
118 }
119 return;
120 }
121
122 if (c->x86_model == 6 && c->x86_stepping == 1) {
123 const int K6_BUG_LOOP = 1000000;
124 int n;
125 void (*f_vide)(void);
126 u64 d, d2;
127
128 pr_info("AMD K6 stepping B detected - ");
129
130 /*
131 * It looks like AMD fixed the 2.6.2 bug and improved indirect
132 * calls at the same time.
133 */
134
135 n = K6_BUG_LOOP;
136 f_vide = vide;
137 OPTIMIZER_HIDE_VAR(f_vide);
138 d = rdtsc();
139 while (n--)
140 f_vide();
141 d2 = rdtsc();
142 d = d2-d;
143
144 if (d > 20*K6_BUG_LOOP)
145 pr_cont("system stability may be impaired when more than 32 MB are used.\n");
146 else
147 pr_cont("probably OK (after B9730xxxx).\n");
148 }
149
150 /* K6 with old style WHCR */
151 if (c->x86_model < 8 ||
152 (c->x86_model == 8 && c->x86_stepping < 8)) {
153 /* We can only write allocate on the low 508Mb */
154 if (mbytes > 508)
155 mbytes = 508;
156
157 rdmsr(MSR_K6_WHCR, l, h);
158 if ((l&0x0000FFFF) == 0) {
159 unsigned long flags;
160 l = (1<<0)|((mbytes/4)<<1);
161 local_irq_save(flags);
162 wbinvd();
163 wrmsr(MSR_K6_WHCR, l, h);
164 local_irq_restore(flags);
165 pr_info("Enabling old style K6 write allocation for %d Mb\n",
166 mbytes);
167 }
168 return;
169 }
170
171 if ((c->x86_model == 8 && c->x86_stepping > 7) ||
172 c->x86_model == 9 || c->x86_model == 13) {
173 /* The more serious chips .. */
174
175 if (mbytes > 4092)
176 mbytes = 4092;
177
178 rdmsr(MSR_K6_WHCR, l, h);
179 if ((l&0xFFFF0000) == 0) {
180 unsigned long flags;
181 l = ((mbytes>>2)<<22)|(1<<16);
182 local_irq_save(flags);
183 wbinvd();
184 wrmsr(MSR_K6_WHCR, l, h);
185 local_irq_restore(flags);
186 pr_info("Enabling new style K6 write allocation for %d Mb\n",
187 mbytes);
188 }
189
190 return;
191 }
192
193 if (c->x86_model == 10) {
194 /* AMD Geode LX is model 10 */
195 /* placeholder for any needed mods */
196 return;
197 }
198 #endif
199 }
200
init_amd_k7(struct cpuinfo_x86 * c)201 static void init_amd_k7(struct cpuinfo_x86 *c)
202 {
203 #ifdef CONFIG_X86_32
204 u32 l, h;
205
206 /*
207 * Bit 15 of Athlon specific MSR 15, needs to be 0
208 * to enable SSE on Palomino/Morgan/Barton CPU's.
209 * If the BIOS didn't enable it already, enable it here.
210 */
211 if (c->x86_model >= 6 && c->x86_model <= 10) {
212 if (!cpu_has(c, X86_FEATURE_XMM)) {
213 pr_info("Enabling disabled K7/SSE Support.\n");
214 msr_clear_bit(MSR_K7_HWCR, 15);
215 set_cpu_cap(c, X86_FEATURE_XMM);
216 }
217 }
218
219 /*
220 * It's been determined by AMD that Athlons since model 8 stepping 1
221 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
222 * As per AMD technical note 27212 0.2
223 */
224 if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) {
225 rdmsr(MSR_K7_CLK_CTL, l, h);
226 if ((l & 0xfff00000) != 0x20000000) {
227 pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
228 l, ((l & 0x000fffff)|0x20000000));
229 wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
230 }
231 }
232
233 /* calling is from identify_secondary_cpu() ? */
234 if (!c->cpu_index)
235 return;
236
237 /*
238 * Certain Athlons might work (for various values of 'work') in SMP
239 * but they are not certified as MP capable.
240 */
241 /* Athlon 660/661 is valid. */
242 if ((c->x86_model == 6) && ((c->x86_stepping == 0) ||
243 (c->x86_stepping == 1)))
244 return;
245
246 /* Duron 670 is valid */
247 if ((c->x86_model == 7) && (c->x86_stepping == 0))
248 return;
249
250 /*
251 * Athlon 662, Duron 671, and Athlon >model 7 have capability
252 * bit. It's worth noting that the A5 stepping (662) of some
253 * Athlon XP's have the MP bit set.
254 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
255 * more.
256 */
257 if (((c->x86_model == 6) && (c->x86_stepping >= 2)) ||
258 ((c->x86_model == 7) && (c->x86_stepping >= 1)) ||
259 (c->x86_model > 7))
260 if (cpu_has(c, X86_FEATURE_MP))
261 return;
262
263 /* If we get here, not a certified SMP capable AMD system. */
264
265 /*
266 * Don't taint if we are running SMP kernel on a single non-MP
267 * approved Athlon
268 */
269 WARN_ONCE(1, "WARNING: This combination of AMD"
270 " processors is not suitable for SMP.\n");
271 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE);
272 #endif
273 }
274
275 #ifdef CONFIG_NUMA
276 /*
277 * To workaround broken NUMA config. Read the comment in
278 * srat_detect_node().
279 */
nearby_node(int apicid)280 static int nearby_node(int apicid)
281 {
282 int i, node;
283
284 for (i = apicid - 1; i >= 0; i--) {
285 node = __apicid_to_node[i];
286 if (node != NUMA_NO_NODE && node_online(node))
287 return node;
288 }
289 for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
290 node = __apicid_to_node[i];
291 if (node != NUMA_NO_NODE && node_online(node))
292 return node;
293 }
294 return first_node(node_online_map); /* Shouldn't happen */
295 }
296 #endif
297
srat_detect_node(struct cpuinfo_x86 * c)298 static void srat_detect_node(struct cpuinfo_x86 *c)
299 {
300 #ifdef CONFIG_NUMA
301 int cpu = smp_processor_id();
302 int node;
303 unsigned apicid = c->topo.apicid;
304
305 node = numa_cpu_node(cpu);
306 if (node == NUMA_NO_NODE)
307 node = per_cpu_llc_id(cpu);
308
309 /*
310 * On multi-fabric platform (e.g. Numascale NumaChip) a
311 * platform-specific handler needs to be called to fixup some
312 * IDs of the CPU.
313 */
314 if (x86_cpuinit.fixup_cpu_id)
315 x86_cpuinit.fixup_cpu_id(c, node);
316
317 if (!node_online(node)) {
318 /*
319 * Two possibilities here:
320 *
321 * - The CPU is missing memory and no node was created. In
322 * that case try picking one from a nearby CPU.
323 *
324 * - The APIC IDs differ from the HyperTransport node IDs
325 * which the K8 northbridge parsing fills in. Assume
326 * they are all increased by a constant offset, but in
327 * the same order as the HT nodeids. If that doesn't
328 * result in a usable node fall back to the path for the
329 * previous case.
330 *
331 * This workaround operates directly on the mapping between
332 * APIC ID and NUMA node, assuming certain relationship
333 * between APIC ID, HT node ID and NUMA topology. As going
334 * through CPU mapping may alter the outcome, directly
335 * access __apicid_to_node[].
336 */
337 int ht_nodeid = c->topo.initial_apicid;
338
339 if (__apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
340 node = __apicid_to_node[ht_nodeid];
341 /* Pick a nearby node */
342 if (!node_online(node))
343 node = nearby_node(apicid);
344 }
345 numa_set_node(cpu, node);
346 #endif
347 }
348
bsp_determine_snp(struct cpuinfo_x86 * c)349 static void bsp_determine_snp(struct cpuinfo_x86 *c)
350 {
351 #ifdef CONFIG_ARCH_HAS_CC_PLATFORM
352 cc_vendor = CC_VENDOR_AMD;
353
354 if (cpu_has(c, X86_FEATURE_SEV_SNP)) {
355 /*
356 * RMP table entry format is not architectural and is defined by the
357 * per-processor PPR. Restrict SNP support on the known CPU models
358 * for which the RMP table entry format is currently defined for.
359 */
360 if (!cpu_has(c, X86_FEATURE_HYPERVISOR) &&
361 c->x86 >= 0x19 && snp_probe_rmptable_info()) {
362 cc_platform_set(CC_ATTR_HOST_SEV_SNP);
363 } else {
364 setup_clear_cpu_cap(X86_FEATURE_SEV_SNP);
365 cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
366 }
367 }
368 #endif
369 }
370
amd_check_tsa_microcode(void)371 static bool amd_check_tsa_microcode(void)
372 {
373 struct cpuinfo_x86 *c = &boot_cpu_data;
374 union zen_patch_rev p;
375 u32 min_rev = 0;
376
377 p.ext_fam = c->x86 - 0xf;
378 p.model = c->x86_model;
379 p.ext_model = c->x86_model >> 4;
380 p.stepping = c->x86_stepping;
381 /* reserved bits are expected to be 0 in test below */
382 p.__reserved = 0;
383
384 if (cpu_has(c, X86_FEATURE_ZEN3) ||
385 cpu_has(c, X86_FEATURE_ZEN4)) {
386 switch (p.ucode_rev >> 8) {
387 case 0xa0011: min_rev = 0x0a0011d7; break;
388 case 0xa0012: min_rev = 0x0a00123b; break;
389 case 0xa0082: min_rev = 0x0a00820d; break;
390 case 0xa1011: min_rev = 0x0a10114c; break;
391 case 0xa1012: min_rev = 0x0a10124c; break;
392 case 0xa1081: min_rev = 0x0a108109; break;
393 case 0xa2010: min_rev = 0x0a20102e; break;
394 case 0xa2012: min_rev = 0x0a201211; break;
395 case 0xa4041: min_rev = 0x0a404108; break;
396 case 0xa5000: min_rev = 0x0a500012; break;
397 case 0xa6012: min_rev = 0x0a60120a; break;
398 case 0xa7041: min_rev = 0x0a704108; break;
399 case 0xa7052: min_rev = 0x0a705208; break;
400 case 0xa7080: min_rev = 0x0a708008; break;
401 case 0xa70c0: min_rev = 0x0a70c008; break;
402 case 0xaa002: min_rev = 0x0aa00216; break;
403 default:
404 pr_debug("%s: ucode_rev: 0x%x, current revision: 0x%x\n",
405 __func__, p.ucode_rev, c->microcode);
406 return false;
407 }
408 }
409
410 if (!min_rev)
411 return false;
412
413 return c->microcode >= min_rev;
414 }
415
tsa_init(struct cpuinfo_x86 * c)416 static void tsa_init(struct cpuinfo_x86 *c)
417 {
418 if (cpu_has(c, X86_FEATURE_HYPERVISOR))
419 return;
420
421 if (cpu_has(c, X86_FEATURE_ZEN3) ||
422 cpu_has(c, X86_FEATURE_ZEN4)) {
423 if (amd_check_tsa_microcode())
424 setup_force_cpu_cap(X86_FEATURE_VERW_CLEAR);
425 } else {
426 setup_force_cpu_cap(X86_FEATURE_TSA_SQ_NO);
427 setup_force_cpu_cap(X86_FEATURE_TSA_L1_NO);
428 }
429 }
430
bsp_init_amd(struct cpuinfo_x86 * c)431 static void bsp_init_amd(struct cpuinfo_x86 *c)
432 {
433 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {
434
435 if (c->x86 > 0x10 ||
436 (c->x86 == 0x10 && c->x86_model >= 0x2)) {
437 u64 val;
438
439 rdmsrl(MSR_K7_HWCR, val);
440 if (!(val & BIT(24)))
441 pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n");
442 }
443 }
444
445 if (c->x86 == 0x15) {
446 unsigned long upperbit;
447 u32 cpuid, assoc;
448
449 cpuid = cpuid_edx(0x80000005);
450 assoc = cpuid >> 16 & 0xff;
451 upperbit = ((cpuid >> 24) << 10) / assoc;
452
453 va_align.mask = (upperbit - 1) & PAGE_MASK;
454 va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
455
456 /* A random value per boot for bit slice [12:upper_bit) */
457 va_align.bits = get_random_u32() & va_align.mask;
458 }
459
460 if (cpu_has(c, X86_FEATURE_MWAITX))
461 use_mwaitx_delay();
462
463 if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) &&
464 !boot_cpu_has(X86_FEATURE_VIRT_SSBD) &&
465 c->x86 >= 0x15 && c->x86 <= 0x17) {
466 unsigned int bit;
467
468 switch (c->x86) {
469 case 0x15: bit = 54; break;
470 case 0x16: bit = 33; break;
471 case 0x17: bit = 10; break;
472 default: return;
473 }
474 /*
475 * Try to cache the base value so further operations can
476 * avoid RMW. If that faults, do not enable SSBD.
477 */
478 if (!rdmsrl_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) {
479 setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD);
480 setup_force_cpu_cap(X86_FEATURE_SSBD);
481 x86_amd_ls_cfg_ssbd_mask = 1ULL << bit;
482 }
483 }
484
485 resctrl_cpu_detect(c);
486
487 /* Figure out Zen generations: */
488 switch (c->x86) {
489 case 0x17:
490 switch (c->x86_model) {
491 case 0x00 ... 0x2f:
492 case 0x50 ... 0x5f:
493 setup_force_cpu_cap(X86_FEATURE_ZEN1);
494 break;
495 case 0x30 ... 0x4f:
496 case 0x60 ... 0x7f:
497 case 0x90 ... 0x91:
498 case 0xa0 ... 0xaf:
499 setup_force_cpu_cap(X86_FEATURE_ZEN2);
500 break;
501 default:
502 goto warn;
503 }
504 break;
505
506 case 0x19:
507 switch (c->x86_model) {
508 case 0x00 ... 0x0f:
509 case 0x20 ... 0x5f:
510 setup_force_cpu_cap(X86_FEATURE_ZEN3);
511 break;
512 case 0x10 ... 0x1f:
513 case 0x60 ... 0xaf:
514 setup_force_cpu_cap(X86_FEATURE_ZEN4);
515 break;
516 default:
517 goto warn;
518 }
519 break;
520
521 case 0x1a:
522 switch (c->x86_model) {
523 case 0x00 ... 0x2f:
524 case 0x40 ... 0x4f:
525 case 0x60 ... 0x7f:
526 setup_force_cpu_cap(X86_FEATURE_ZEN5);
527 break;
528 default:
529 goto warn;
530 }
531 break;
532
533 default:
534 break;
535 }
536
537 bsp_determine_snp(c);
538
539 tsa_init(c);
540
541 return;
542
543 warn:
544 WARN_ONCE(1, "Family 0x%x, model: 0x%x??\n", c->x86, c->x86_model);
545 }
546
early_detect_mem_encrypt(struct cpuinfo_x86 * c)547 static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
548 {
549 u64 msr;
550
551 /*
552 * BIOS support is required for SME and SEV.
553 * For SME: If BIOS has enabled SME then adjust x86_phys_bits by
554 * the SME physical address space reduction value.
555 * If BIOS has not enabled SME then don't advertise the
556 * SME feature (set in scattered.c).
557 * If the kernel has not enabled SME via any means then
558 * don't advertise the SME feature.
559 * For SEV: If BIOS has not enabled SEV then don't advertise SEV and
560 * any additional functionality based on it.
561 *
562 * In all cases, since support for SME and SEV requires long mode,
563 * don't advertise the feature under CONFIG_X86_32.
564 */
565 if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) {
566 /* Check if memory encryption is enabled */
567 rdmsrl(MSR_AMD64_SYSCFG, msr);
568 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
569 goto clear_all;
570
571 /*
572 * Always adjust physical address bits. Even though this
573 * will be a value above 32-bits this is still done for
574 * CONFIG_X86_32 so that accurate values are reported.
575 */
576 c->x86_phys_bits -= (cpuid_ebx(0x8000001f) >> 6) & 0x3f;
577
578 if (IS_ENABLED(CONFIG_X86_32))
579 goto clear_all;
580
581 if (!sme_me_mask)
582 setup_clear_cpu_cap(X86_FEATURE_SME);
583
584 rdmsrl(MSR_K7_HWCR, msr);
585 if (!(msr & MSR_K7_HWCR_SMMLOCK))
586 goto clear_sev;
587
588 return;
589
590 clear_all:
591 setup_clear_cpu_cap(X86_FEATURE_SME);
592 clear_sev:
593 setup_clear_cpu_cap(X86_FEATURE_SEV);
594 setup_clear_cpu_cap(X86_FEATURE_SEV_ES);
595 setup_clear_cpu_cap(X86_FEATURE_SEV_SNP);
596 }
597 }
598
early_init_amd(struct cpuinfo_x86 * c)599 static void early_init_amd(struct cpuinfo_x86 *c)
600 {
601 u32 dummy;
602
603 if (c->x86 >= 0xf)
604 set_cpu_cap(c, X86_FEATURE_K8);
605
606 rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
607
608 /*
609 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
610 * with P/T states and does not stop in deep C-states
611 */
612 if (c->x86_power & (1 << 8)) {
613 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
614 set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
615 }
616
617 /* Bit 12 of 8000_0007 edx is accumulated power mechanism. */
618 if (c->x86_power & BIT(12))
619 set_cpu_cap(c, X86_FEATURE_ACC_POWER);
620
621 /* Bit 14 indicates the Runtime Average Power Limit interface. */
622 if (c->x86_power & BIT(14))
623 set_cpu_cap(c, X86_FEATURE_RAPL);
624
625 #ifdef CONFIG_X86_64
626 set_cpu_cap(c, X86_FEATURE_SYSCALL32);
627 #else
628 /* Set MTRR capability flag if appropriate */
629 if (c->x86 == 5)
630 if (c->x86_model == 13 || c->x86_model == 9 ||
631 (c->x86_model == 8 && c->x86_stepping >= 8))
632 set_cpu_cap(c, X86_FEATURE_K6_MTRR);
633 #endif
634 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
635 /*
636 * ApicID can always be treated as an 8-bit value for AMD APIC versions
637 * >= 0x10, but even old K8s came out of reset with version 0x10. So, we
638 * can safely set X86_FEATURE_EXTD_APICID unconditionally for families
639 * after 16h.
640 */
641 if (boot_cpu_has(X86_FEATURE_APIC)) {
642 if (c->x86 > 0x16)
643 set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
644 else if (c->x86 >= 0xf) {
645 /* check CPU config space for extended APIC ID */
646 unsigned int val;
647
648 val = read_pci_config(0, 24, 0, 0x68);
649 if ((val >> 17 & 0x3) == 0x3)
650 set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
651 }
652 }
653 #endif
654
655 /*
656 * This is only needed to tell the kernel whether to use VMCALL
657 * and VMMCALL. VMMCALL is never executed except under virt, so
658 * we can set it unconditionally.
659 */
660 set_cpu_cap(c, X86_FEATURE_VMMCALL);
661
662 /* F16h erratum 793, CVE-2013-6885 */
663 if (c->x86 == 0x16 && c->x86_model <= 0xf)
664 msr_set_bit(MSR_AMD64_LS_CFG, 15);
665
666 early_detect_mem_encrypt(c);
667
668 if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_IBPB_BRTYPE)) {
669 if (c->x86 == 0x17 && boot_cpu_has(X86_FEATURE_AMD_IBPB))
670 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
671 else if (c->x86 >= 0x19 && !wrmsrl_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) {
672 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
673 setup_force_cpu_cap(X86_FEATURE_SBPB);
674 }
675 }
676 }
677
init_amd_k8(struct cpuinfo_x86 * c)678 static void init_amd_k8(struct cpuinfo_x86 *c)
679 {
680 u32 level;
681 u64 value;
682
683 /* On C+ stepping K8 rep microcode works well for copy/memset */
684 level = cpuid_eax(1);
685 if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
686 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
687
688 /*
689 * Some BIOSes incorrectly force this feature, but only K8 revision D
690 * (model = 0x14) and later actually support it.
691 * (AMD Erratum #110, docId: 25759).
692 */
693 if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM) && !cpu_has(c, X86_FEATURE_HYPERVISOR)) {
694 clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
695 if (!rdmsrl_amd_safe(0xc001100d, &value)) {
696 value &= ~BIT_64(32);
697 wrmsrl_amd_safe(0xc001100d, value);
698 }
699 }
700
701 if (!c->x86_model_id[0])
702 strcpy(c->x86_model_id, "Hammer");
703
704 #ifdef CONFIG_SMP
705 /*
706 * Disable TLB flush filter by setting HWCR.FFDIS on K8
707 * bit 6 of msr C001_0015
708 *
709 * Errata 63 for SH-B3 steppings
710 * Errata 122 for all steppings (F+ have it disabled by default)
711 */
712 msr_set_bit(MSR_K7_HWCR, 6);
713 #endif
714 set_cpu_bug(c, X86_BUG_SWAPGS_FENCE);
715
716 /*
717 * Check models and steppings affected by erratum 400. This is
718 * used to select the proper idle routine and to enable the
719 * check whether the machine is affected in arch_post_acpi_subsys_init()
720 * which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check.
721 */
722 if (c->x86_model > 0x41 ||
723 (c->x86_model == 0x41 && c->x86_stepping >= 0x2))
724 setup_force_cpu_bug(X86_BUG_AMD_E400);
725 }
726
init_amd_gh(struct cpuinfo_x86 * c)727 static void init_amd_gh(struct cpuinfo_x86 *c)
728 {
729 #ifdef CONFIG_MMCONF_FAM10H
730 /* do this for boot cpu */
731 if (c == &boot_cpu_data)
732 check_enable_amd_mmconf_dmi();
733
734 fam10h_check_enable_mmcfg();
735 #endif
736
737 /*
738 * Disable GART TLB Walk Errors on Fam10h. We do this here because this
739 * is always needed when GART is enabled, even in a kernel which has no
740 * MCE support built in. BIOS should disable GartTlbWlk Errors already.
741 * If it doesn't, we do it here as suggested by the BKDG.
742 *
743 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
744 */
745 msr_set_bit(MSR_AMD64_MCx_MASK(4), 10);
746
747 /*
748 * On family 10h BIOS may not have properly enabled WC+ support, causing
749 * it to be converted to CD memtype. This may result in performance
750 * degradation for certain nested-paging guests. Prevent this conversion
751 * by clearing bit 24 in MSR_AMD64_BU_CFG2.
752 *
753 * NOTE: we want to use the _safe accessors so as not to #GP kvm
754 * guests on older kvm hosts.
755 */
756 msr_clear_bit(MSR_AMD64_BU_CFG2, 24);
757
758 set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
759
760 /*
761 * Check models and steppings affected by erratum 400. This is
762 * used to select the proper idle routine and to enable the
763 * check whether the machine is affected in arch_post_acpi_subsys_init()
764 * which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check.
765 */
766 if (c->x86_model > 0x2 ||
767 (c->x86_model == 0x2 && c->x86_stepping >= 0x1))
768 setup_force_cpu_bug(X86_BUG_AMD_E400);
769 }
770
init_amd_ln(struct cpuinfo_x86 * c)771 static void init_amd_ln(struct cpuinfo_x86 *c)
772 {
773 /*
774 * Apply erratum 665 fix unconditionally so machines without a BIOS
775 * fix work.
776 */
777 msr_set_bit(MSR_AMD64_DE_CFG, 31);
778 }
779
780 static bool rdrand_force;
781
rdrand_cmdline(char * str)782 static int __init rdrand_cmdline(char *str)
783 {
784 if (!str)
785 return -EINVAL;
786
787 if (!strcmp(str, "force"))
788 rdrand_force = true;
789 else
790 return -EINVAL;
791
792 return 0;
793 }
794 early_param("rdrand", rdrand_cmdline);
795
clear_rdrand_cpuid_bit(struct cpuinfo_x86 * c)796 static void clear_rdrand_cpuid_bit(struct cpuinfo_x86 *c)
797 {
798 /*
799 * Saving of the MSR used to hide the RDRAND support during
800 * suspend/resume is done by arch/x86/power/cpu.c, which is
801 * dependent on CONFIG_PM_SLEEP.
802 */
803 if (!IS_ENABLED(CONFIG_PM_SLEEP))
804 return;
805
806 /*
807 * The self-test can clear X86_FEATURE_RDRAND, so check for
808 * RDRAND support using the CPUID function directly.
809 */
810 if (!(cpuid_ecx(1) & BIT(30)) || rdrand_force)
811 return;
812
813 msr_clear_bit(MSR_AMD64_CPUID_FN_1, 62);
814
815 /*
816 * Verify that the CPUID change has occurred in case the kernel is
817 * running virtualized and the hypervisor doesn't support the MSR.
818 */
819 if (cpuid_ecx(1) & BIT(30)) {
820 pr_info_once("BIOS may not properly restore RDRAND after suspend, but hypervisor does not support hiding RDRAND via CPUID.\n");
821 return;
822 }
823
824 clear_cpu_cap(c, X86_FEATURE_RDRAND);
825 pr_info_once("BIOS may not properly restore RDRAND after suspend, hiding RDRAND via CPUID. Use rdrand=force to reenable.\n");
826 }
827
init_amd_jg(struct cpuinfo_x86 * c)828 static void init_amd_jg(struct cpuinfo_x86 *c)
829 {
830 /*
831 * Some BIOS implementations do not restore proper RDRAND support
832 * across suspend and resume. Check on whether to hide the RDRAND
833 * instruction support via CPUID.
834 */
835 clear_rdrand_cpuid_bit(c);
836 }
837
init_amd_bd(struct cpuinfo_x86 * c)838 static void init_amd_bd(struct cpuinfo_x86 *c)
839 {
840 u64 value;
841
842 /*
843 * The way access filter has a performance penalty on some workloads.
844 * Disable it on the affected CPUs.
845 */
846 if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
847 if (!rdmsrl_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) {
848 value |= 0x1E;
849 wrmsrl_safe(MSR_F15H_IC_CFG, value);
850 }
851 }
852
853 /*
854 * Some BIOS implementations do not restore proper RDRAND support
855 * across suspend and resume. Check on whether to hide the RDRAND
856 * instruction support via CPUID.
857 */
858 clear_rdrand_cpuid_bit(c);
859 }
860
861 static const struct x86_cpu_desc erratum_1386_microcode[] = {
862 AMD_CPU_DESC(0x17, 0x1, 0x2, 0x0800126e),
863 AMD_CPU_DESC(0x17, 0x31, 0x0, 0x08301052),
864 {},
865 };
866
fix_erratum_1386(struct cpuinfo_x86 * c)867 static void fix_erratum_1386(struct cpuinfo_x86 *c)
868 {
869 /*
870 * Work around Erratum 1386. The XSAVES instruction malfunctions in
871 * certain circumstances on Zen1/2 uarch, and not all parts have had
872 * updated microcode at the time of writing (March 2023).
873 *
874 * Affected parts all have no supervisor XSAVE states, meaning that
875 * the XSAVEC instruction (which works fine) is equivalent.
876 *
877 * Clear the feature flag only on microcode revisions which
878 * don't have the fix.
879 */
880 if (x86_cpu_has_min_microcode_rev(erratum_1386_microcode))
881 return;
882
883 clear_cpu_cap(c, X86_FEATURE_XSAVES);
884 }
885
init_spectral_chicken(struct cpuinfo_x86 * c)886 void init_spectral_chicken(struct cpuinfo_x86 *c)
887 {
888 #ifdef CONFIG_MITIGATION_UNRET_ENTRY
889 u64 value;
890
891 /*
892 * On Zen2 we offer this chicken (bit) on the altar of Speculation.
893 *
894 * This suppresses speculation from the middle of a basic block, i.e. it
895 * suppresses non-branch predictions.
896 */
897 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
898 if (!rdmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) {
899 value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT;
900 wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value);
901 }
902 }
903 #endif
904 }
905
init_amd_zen_common(void)906 static void init_amd_zen_common(void)
907 {
908 setup_force_cpu_cap(X86_FEATURE_ZEN);
909 #ifdef CONFIG_NUMA
910 node_reclaim_distance = 32;
911 #endif
912 }
913
init_amd_zen1(struct cpuinfo_x86 * c)914 static void init_amd_zen1(struct cpuinfo_x86 *c)
915 {
916 fix_erratum_1386(c);
917
918 /* Fix up CPUID bits, but only if not virtualised. */
919 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
920
921 /* Erratum 1076: CPB feature bit not being set in CPUID. */
922 if (!cpu_has(c, X86_FEATURE_CPB))
923 set_cpu_cap(c, X86_FEATURE_CPB);
924 }
925
926 pr_notice_once("AMD Zen1 DIV0 bug detected. Disable SMT for full protection.\n");
927 setup_force_cpu_bug(X86_BUG_DIV0);
928
929 /*
930 * Turn off the Instructions Retired free counter on machines that are
931 * susceptible to erratum #1054 "Instructions Retired Performance
932 * Counter May Be Inaccurate".
933 */
934 if (c->x86_model < 0x30) {
935 msr_clear_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT);
936 clear_cpu_cap(c, X86_FEATURE_IRPERF);
937 }
938 }
939
cpu_has_zenbleed_microcode(void)940 static bool cpu_has_zenbleed_microcode(void)
941 {
942 u32 good_rev = 0;
943
944 switch (boot_cpu_data.x86_model) {
945 case 0x30 ... 0x3f: good_rev = 0x0830107b; break;
946 case 0x60 ... 0x67: good_rev = 0x0860010c; break;
947 case 0x68 ... 0x6f: good_rev = 0x08608107; break;
948 case 0x70 ... 0x7f: good_rev = 0x08701033; break;
949 case 0xa0 ... 0xaf: good_rev = 0x08a00009; break;
950
951 default:
952 return false;
953 }
954
955 if (boot_cpu_data.microcode < good_rev)
956 return false;
957
958 return true;
959 }
960
zen2_zenbleed_check(struct cpuinfo_x86 * c)961 static void zen2_zenbleed_check(struct cpuinfo_x86 *c)
962 {
963 if (cpu_has(c, X86_FEATURE_HYPERVISOR))
964 return;
965
966 if (!cpu_has(c, X86_FEATURE_AVX))
967 return;
968
969 if (!cpu_has_zenbleed_microcode()) {
970 pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n");
971 msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT);
972 } else {
973 msr_clear_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT);
974 }
975 }
976
init_amd_zen2(struct cpuinfo_x86 * c)977 static void init_amd_zen2(struct cpuinfo_x86 *c)
978 {
979 init_spectral_chicken(c);
980 fix_erratum_1386(c);
981 zen2_zenbleed_check(c);
982
983 /* Disable RDSEED on AMD Cyan Skillfish because of an error. */
984 if (c->x86_model == 0x47 && c->x86_stepping == 0x0) {
985 clear_cpu_cap(c, X86_FEATURE_RDSEED);
986 msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18);
987 pr_emerg("RDSEED is not reliable on this platform; disabling.\n");
988 }
989 }
990
init_amd_zen3(struct cpuinfo_x86 * c)991 static void init_amd_zen3(struct cpuinfo_x86 *c)
992 {
993 if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
994 /*
995 * Zen3 (Fam19 model < 0x10) parts are not susceptible to
996 * Branch Type Confusion, but predate the allocation of the
997 * BTC_NO bit.
998 */
999 if (!cpu_has(c, X86_FEATURE_BTC_NO))
1000 set_cpu_cap(c, X86_FEATURE_BTC_NO);
1001 }
1002 }
1003
init_amd_zen4(struct cpuinfo_x86 * c)1004 static void init_amd_zen4(struct cpuinfo_x86 *c)
1005 {
1006 if (!cpu_has(c, X86_FEATURE_HYPERVISOR))
1007 msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT);
1008
1009 /*
1010 * These Zen4 SoCs advertise support for virtualized VMLOAD/VMSAVE
1011 * in some BIOS versions but they can lead to random host reboots.
1012 */
1013 switch (c->x86_model) {
1014 case 0x18 ... 0x1f:
1015 case 0x60 ... 0x7f:
1016 clear_cpu_cap(c, X86_FEATURE_V_VMSAVE_VMLOAD);
1017 break;
1018 }
1019 }
1020
init_amd_zen5(struct cpuinfo_x86 * c)1021 static void init_amd_zen5(struct cpuinfo_x86 *c)
1022 {
1023 }
1024
init_amd(struct cpuinfo_x86 * c)1025 static void init_amd(struct cpuinfo_x86 *c)
1026 {
1027 u64 vm_cr;
1028
1029 early_init_amd(c);
1030
1031 /*
1032 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
1033 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
1034 */
1035 clear_cpu_cap(c, 0*32+31);
1036
1037 if (c->x86 >= 0x10)
1038 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
1039
1040 /* AMD FSRM also implies FSRS */
1041 if (cpu_has(c, X86_FEATURE_FSRM))
1042 set_cpu_cap(c, X86_FEATURE_FSRS);
1043
1044 /* K6s reports MCEs but don't actually have all the MSRs */
1045 if (c->x86 < 6)
1046 clear_cpu_cap(c, X86_FEATURE_MCE);
1047
1048 switch (c->x86) {
1049 case 4: init_amd_k5(c); break;
1050 case 5: init_amd_k6(c); break;
1051 case 6: init_amd_k7(c); break;
1052 case 0xf: init_amd_k8(c); break;
1053 case 0x10: init_amd_gh(c); break;
1054 case 0x12: init_amd_ln(c); break;
1055 case 0x15: init_amd_bd(c); break;
1056 case 0x16: init_amd_jg(c); break;
1057 }
1058
1059 /*
1060 * Save up on some future enablement work and do common Zen
1061 * settings.
1062 */
1063 if (c->x86 >= 0x17)
1064 init_amd_zen_common();
1065
1066 if (boot_cpu_has(X86_FEATURE_ZEN1))
1067 init_amd_zen1(c);
1068 else if (boot_cpu_has(X86_FEATURE_ZEN2))
1069 init_amd_zen2(c);
1070 else if (boot_cpu_has(X86_FEATURE_ZEN3))
1071 init_amd_zen3(c);
1072 else if (boot_cpu_has(X86_FEATURE_ZEN4))
1073 init_amd_zen4(c);
1074 else if (boot_cpu_has(X86_FEATURE_ZEN5))
1075 init_amd_zen5(c);
1076
1077 /*
1078 * Enable workaround for FXSAVE leak on CPUs
1079 * without a XSaveErPtr feature
1080 */
1081 if ((c->x86 >= 6) && (!cpu_has(c, X86_FEATURE_XSAVEERPTR)))
1082 set_cpu_bug(c, X86_BUG_FXSAVE_LEAK);
1083
1084 cpu_detect_cache_sizes(c);
1085
1086 srat_detect_node(c);
1087
1088 init_amd_cacheinfo(c);
1089
1090 if (cpu_has(c, X86_FEATURE_SVM)) {
1091 rdmsrl(MSR_VM_CR, vm_cr);
1092 if (vm_cr & SVM_VM_CR_SVM_DIS_MASK) {
1093 pr_notice_once("SVM disabled (by BIOS) in MSR_VM_CR\n");
1094 clear_cpu_cap(c, X86_FEATURE_SVM);
1095 }
1096 }
1097
1098 if (!cpu_has(c, X86_FEATURE_LFENCE_RDTSC) && cpu_has(c, X86_FEATURE_XMM2)) {
1099 /*
1100 * Use LFENCE for execution serialization. On families which
1101 * don't have that MSR, LFENCE is already serializing.
1102 * msr_set_bit() uses the safe accessors, too, even if the MSR
1103 * is not present.
1104 */
1105 msr_set_bit(MSR_AMD64_DE_CFG,
1106 MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT);
1107
1108 /* A serializing LFENCE stops RDTSC speculation */
1109 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
1110 }
1111
1112 /*
1113 * Family 0x12 and above processors have APIC timer
1114 * running in deep C states.
1115 */
1116 if (c->x86 > 0x11)
1117 set_cpu_cap(c, X86_FEATURE_ARAT);
1118
1119 /* 3DNow or LM implies PREFETCHW */
1120 if (!cpu_has(c, X86_FEATURE_3DNOWPREFETCH))
1121 if (cpu_has(c, X86_FEATURE_3DNOW) || cpu_has(c, X86_FEATURE_LM))
1122 set_cpu_cap(c, X86_FEATURE_3DNOWPREFETCH);
1123
1124 /* AMD CPUs don't reset SS attributes on SYSRET, Xen does. */
1125 if (!cpu_feature_enabled(X86_FEATURE_XENPV))
1126 set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS);
1127
1128 /* Enable the Instructions Retired free counter */
1129 if (cpu_has(c, X86_FEATURE_IRPERF))
1130 msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT);
1131
1132 check_null_seg_clears_base(c);
1133
1134 /*
1135 * Make sure EFER[AIBRSE - Automatic IBRS Enable] is set. The APs are brought up
1136 * using the trampoline code and as part of it, MSR_EFER gets prepared there in
1137 * order to be replicated onto them. Regardless, set it here again, if not set,
1138 * to protect against any future refactoring/code reorganization which might
1139 * miss setting this important bit.
1140 */
1141 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
1142 cpu_has(c, X86_FEATURE_AUTOIBRS))
1143 WARN_ON_ONCE(msr_set_bit(MSR_EFER, _EFER_AUTOIBRS) < 0);
1144
1145 /* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */
1146 clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1147 }
1148
1149 #ifdef CONFIG_X86_32
amd_size_cache(struct cpuinfo_x86 * c,unsigned int size)1150 static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
1151 {
1152 /* AMD errata T13 (order #21922) */
1153 if (c->x86 == 6) {
1154 /* Duron Rev A0 */
1155 if (c->x86_model == 3 && c->x86_stepping == 0)
1156 size = 64;
1157 /* Tbird rev A1/A2 */
1158 if (c->x86_model == 4 &&
1159 (c->x86_stepping == 0 || c->x86_stepping == 1))
1160 size = 256;
1161 }
1162 return size;
1163 }
1164 #endif
1165
cpu_detect_tlb_amd(struct cpuinfo_x86 * c)1166 static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
1167 {
1168 u32 ebx, eax, ecx, edx;
1169 u16 mask = 0xfff;
1170
1171 if (c->x86 < 0xf)
1172 return;
1173
1174 if (c->extended_cpuid_level < 0x80000006)
1175 return;
1176
1177 cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
1178
1179 tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
1180 tlb_lli_4k[ENTRIES] = ebx & mask;
1181
1182 /*
1183 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
1184 * characteristics from the CPUID function 0x80000005 instead.
1185 */
1186 if (c->x86 == 0xf) {
1187 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
1188 mask = 0xff;
1189 }
1190
1191 /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
1192 if (!((eax >> 16) & mask))
1193 tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;
1194 else
1195 tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
1196
1197 /* a 4M entry uses two 2M entries */
1198 tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;
1199
1200 /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
1201 if (!(eax & mask)) {
1202 /* Erratum 658 */
1203 if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
1204 tlb_lli_2m[ENTRIES] = 1024;
1205 } else {
1206 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
1207 tlb_lli_2m[ENTRIES] = eax & 0xff;
1208 }
1209 } else
1210 tlb_lli_2m[ENTRIES] = eax & mask;
1211
1212 tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
1213 }
1214
1215 static const struct cpu_dev amd_cpu_dev = {
1216 .c_vendor = "AMD",
1217 .c_ident = { "AuthenticAMD" },
1218 #ifdef CONFIG_X86_32
1219 .legacy_models = {
1220 { .family = 4, .model_names =
1221 {
1222 [3] = "486 DX/2",
1223 [7] = "486 DX/2-WB",
1224 [8] = "486 DX/4",
1225 [9] = "486 DX/4-WB",
1226 [14] = "Am5x86-WT",
1227 [15] = "Am5x86-WB"
1228 }
1229 },
1230 },
1231 .legacy_cache_size = amd_size_cache,
1232 #endif
1233 .c_early_init = early_init_amd,
1234 .c_detect_tlb = cpu_detect_tlb_amd,
1235 .c_bsp_init = bsp_init_amd,
1236 .c_init = init_amd,
1237 .c_x86_vendor = X86_VENDOR_AMD,
1238 };
1239
1240 cpu_dev_register(amd_cpu_dev);
1241
1242 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long[4], amd_dr_addr_mask);
1243
1244 static unsigned int amd_msr_dr_addr_masks[] = {
1245 MSR_F16H_DR0_ADDR_MASK,
1246 MSR_F16H_DR1_ADDR_MASK,
1247 MSR_F16H_DR1_ADDR_MASK + 1,
1248 MSR_F16H_DR1_ADDR_MASK + 2
1249 };
1250
amd_set_dr_addr_mask(unsigned long mask,unsigned int dr)1251 void amd_set_dr_addr_mask(unsigned long mask, unsigned int dr)
1252 {
1253 int cpu = smp_processor_id();
1254
1255 if (!cpu_feature_enabled(X86_FEATURE_BPEXT))
1256 return;
1257
1258 if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks)))
1259 return;
1260
1261 if (per_cpu(amd_dr_addr_mask, cpu)[dr] == mask)
1262 return;
1263
1264 wrmsr(amd_msr_dr_addr_masks[dr], mask, 0);
1265 per_cpu(amd_dr_addr_mask, cpu)[dr] = mask;
1266 }
1267
amd_get_dr_addr_mask(unsigned int dr)1268 unsigned long amd_get_dr_addr_mask(unsigned int dr)
1269 {
1270 if (!cpu_feature_enabled(X86_FEATURE_BPEXT))
1271 return 0;
1272
1273 if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks)))
1274 return 0;
1275
1276 return per_cpu(amd_dr_addr_mask[dr], smp_processor_id());
1277 }
1278 EXPORT_SYMBOL_GPL(amd_get_dr_addr_mask);
1279
zenbleed_check_cpu(void * unused)1280 static void zenbleed_check_cpu(void *unused)
1281 {
1282 struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
1283
1284 zen2_zenbleed_check(c);
1285 }
1286
amd_check_microcode(void)1287 void amd_check_microcode(void)
1288 {
1289 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
1290 return;
1291
1292 if (cpu_feature_enabled(X86_FEATURE_ZEN2))
1293 on_each_cpu(zenbleed_check_cpu, NULL, 1);
1294 }
1295