1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * (c) 2005-2016 Advanced Micro Devices, Inc.
4 *
5 * Written by Jacob Shin - AMD, Inc.
6 * Maintained by: Borislav Petkov <bp@alien8.de>
7 *
8 * All MC4_MISCi registers are shared between cores on a node.
9 */
10 #include <linux/interrupt.h>
11 #include <linux/notifier.h>
12 #include <linux/kobject.h>
13 #include <linux/percpu.h>
14 #include <linux/errno.h>
15 #include <linux/sched.h>
16 #include <linux/sysfs.h>
17 #include <linux/slab.h>
18 #include <linux/init.h>
19 #include <linux/cpu.h>
20 #include <linux/smp.h>
21 #include <linux/string.h>
22
23 #include <asm/amd_nb.h>
24 #include <asm/traps.h>
25 #include <asm/apic.h>
26 #include <asm/mce.h>
27 #include <asm/msr.h>
28 #include <asm/trace/irq_vectors.h>
29
30 #include "internal.h"
31
32 #define NR_BLOCKS 5
33 #define THRESHOLD_MAX 0xFFF
34 #define INT_TYPE_APIC 0x00020000
35 #define MASK_VALID_HI 0x80000000
36 #define MASK_CNTP_HI 0x40000000
37 #define MASK_LOCKED_HI 0x20000000
38 #define MASK_LVTOFF_HI 0x00F00000
39 #define MASK_COUNT_EN_HI 0x00080000
40 #define MASK_INT_TYPE_HI 0x00060000
41 #define MASK_OVERFLOW_HI 0x00010000
42 #define MASK_ERR_COUNT_HI 0x00000FFF
43 #define MASK_BLKPTR_LO 0xFF000000
44 #define MCG_XBLK_ADDR 0xC0000400
45
46 /* Deferred error settings */
47 #define MSR_CU_DEF_ERR 0xC0000410
48 #define MASK_DEF_LVTOFF 0x000000F0
49 #define MASK_DEF_INT_TYPE 0x00000006
50 #define DEF_LVT_OFF 0x2
51 #define DEF_INT_TYPE_APIC 0x2
52
53 /* Scalable MCA: */
54
55 /* Threshold LVT offset is at MSR0xC0000410[15:12] */
56 #define SMCA_THR_LVT_OFF 0xF000
57
58 static bool thresholding_irq_en;
59
60 static const char * const th_names[] = {
61 "load_store",
62 "insn_fetch",
63 "combined_unit",
64 "decode_unit",
65 "northbridge",
66 "execution_unit",
67 };
68
69 static const char * const smca_umc_block_names[] = {
70 "dram_ecc",
71 "misc_umc"
72 };
73
74 struct smca_bank_name {
75 const char *name; /* Short name for sysfs */
76 const char *long_name; /* Long name for pretty-printing */
77 };
78
79 static struct smca_bank_name smca_names[] = {
80 [SMCA_LS] = { "load_store", "Load Store Unit" },
81 [SMCA_LS_V2] = { "load_store", "Load Store Unit" },
82 [SMCA_IF] = { "insn_fetch", "Instruction Fetch Unit" },
83 [SMCA_L2_CACHE] = { "l2_cache", "L2 Cache" },
84 [SMCA_DE] = { "decode_unit", "Decode Unit" },
85 [SMCA_RESERVED] = { "reserved", "Reserved" },
86 [SMCA_EX] = { "execution_unit", "Execution Unit" },
87 [SMCA_FP] = { "floating_point", "Floating Point Unit" },
88 [SMCA_L3_CACHE] = { "l3_cache", "L3 Cache" },
89 [SMCA_CS] = { "coherent_slave", "Coherent Slave" },
90 [SMCA_CS_V2] = { "coherent_slave", "Coherent Slave" },
91 [SMCA_PIE] = { "pie", "Power, Interrupts, etc." },
92 [SMCA_UMC] = { "umc", "Unified Memory Controller" },
93 [SMCA_PB] = { "param_block", "Parameter Block" },
94 [SMCA_PSP] = { "psp", "Platform Security Processor" },
95 [SMCA_PSP_V2] = { "psp", "Platform Security Processor" },
96 [SMCA_SMU] = { "smu", "System Management Unit" },
97 [SMCA_SMU_V2] = { "smu", "System Management Unit" },
98 [SMCA_MP5] = { "mp5", "Microprocessor 5 Unit" },
99 [SMCA_NBIO] = { "nbio", "Northbridge IO Unit" },
100 [SMCA_PCIE] = { "pcie", "PCI Express Unit" },
101 };
102
smca_get_name(enum smca_bank_types t)103 static const char *smca_get_name(enum smca_bank_types t)
104 {
105 if (t >= N_SMCA_BANK_TYPES)
106 return NULL;
107
108 return smca_names[t].name;
109 }
110
smca_get_long_name(enum smca_bank_types t)111 const char *smca_get_long_name(enum smca_bank_types t)
112 {
113 if (t >= N_SMCA_BANK_TYPES)
114 return NULL;
115
116 return smca_names[t].long_name;
117 }
118 EXPORT_SYMBOL_GPL(smca_get_long_name);
119
smca_get_bank_type(unsigned int bank)120 static enum smca_bank_types smca_get_bank_type(unsigned int bank)
121 {
122 struct smca_bank *b;
123
124 if (bank >= MAX_NR_BANKS)
125 return N_SMCA_BANK_TYPES;
126
127 b = &smca_banks[bank];
128 if (!b->hwid)
129 return N_SMCA_BANK_TYPES;
130
131 return b->hwid->bank_type;
132 }
133
134 static struct smca_hwid smca_hwid_mcatypes[] = {
135 /* { bank_type, hwid_mcatype } */
136
137 /* Reserved type */
138 { SMCA_RESERVED, HWID_MCATYPE(0x00, 0x0) },
139
140 /* ZN Core (HWID=0xB0) MCA types */
141 { SMCA_LS, HWID_MCATYPE(0xB0, 0x0) },
142 { SMCA_LS_V2, HWID_MCATYPE(0xB0, 0x10) },
143 { SMCA_IF, HWID_MCATYPE(0xB0, 0x1) },
144 { SMCA_L2_CACHE, HWID_MCATYPE(0xB0, 0x2) },
145 { SMCA_DE, HWID_MCATYPE(0xB0, 0x3) },
146 /* HWID 0xB0 MCATYPE 0x4 is Reserved */
147 { SMCA_EX, HWID_MCATYPE(0xB0, 0x5) },
148 { SMCA_FP, HWID_MCATYPE(0xB0, 0x6) },
149 { SMCA_L3_CACHE, HWID_MCATYPE(0xB0, 0x7) },
150
151 /* Data Fabric MCA types */
152 { SMCA_CS, HWID_MCATYPE(0x2E, 0x0) },
153 { SMCA_PIE, HWID_MCATYPE(0x2E, 0x1) },
154 { SMCA_CS_V2, HWID_MCATYPE(0x2E, 0x2) },
155
156 /* Unified Memory Controller MCA type */
157 { SMCA_UMC, HWID_MCATYPE(0x96, 0x0) },
158
159 /* Parameter Block MCA type */
160 { SMCA_PB, HWID_MCATYPE(0x05, 0x0) },
161
162 /* Platform Security Processor MCA type */
163 { SMCA_PSP, HWID_MCATYPE(0xFF, 0x0) },
164 { SMCA_PSP_V2, HWID_MCATYPE(0xFF, 0x1) },
165
166 /* System Management Unit MCA type */
167 { SMCA_SMU, HWID_MCATYPE(0x01, 0x0) },
168 { SMCA_SMU_V2, HWID_MCATYPE(0x01, 0x1) },
169
170 /* Microprocessor 5 Unit MCA type */
171 { SMCA_MP5, HWID_MCATYPE(0x01, 0x2) },
172
173 /* Northbridge IO Unit MCA type */
174 { SMCA_NBIO, HWID_MCATYPE(0x18, 0x0) },
175
176 /* PCI Express Unit MCA type */
177 { SMCA_PCIE, HWID_MCATYPE(0x46, 0x0) },
178 };
179
180 struct smca_bank smca_banks[MAX_NR_BANKS];
181 EXPORT_SYMBOL_GPL(smca_banks);
182
183 /*
184 * In SMCA enabled processors, we can have multiple banks for a given IP type.
185 * So to define a unique name for each bank, we use a temp c-string to append
186 * the MCA_IPID[InstanceId] to type's name in get_name().
187 *
188 * InstanceId is 32 bits which is 8 characters. Make sure MAX_MCATYPE_NAME_LEN
189 * is greater than 8 plus 1 (for underscore) plus length of longest type name.
190 */
191 #define MAX_MCATYPE_NAME_LEN 30
192 static char buf_mcatype[MAX_MCATYPE_NAME_LEN];
193
194 static DEFINE_PER_CPU(struct threshold_bank **, threshold_banks);
195
196 /*
197 * A list of the banks enabled on each logical CPU. Controls which respective
198 * descriptors to initialize later in mce_threshold_create_device().
199 */
200 static DEFINE_PER_CPU(u64, bank_map);
201
202 /* Map of banks that have more than MCA_MISC0 available. */
203 static DEFINE_PER_CPU(u64, smca_misc_banks_map);
204
205 static void amd_threshold_interrupt(void);
206 static void amd_deferred_error_interrupt(void);
207
default_deferred_error_interrupt(void)208 static void default_deferred_error_interrupt(void)
209 {
210 pr_err("Unexpected deferred interrupt at vector %x\n", DEFERRED_ERROR_VECTOR);
211 }
212 void (*deferred_error_int_vector)(void) = default_deferred_error_interrupt;
213
smca_set_misc_banks_map(unsigned int bank,unsigned int cpu)214 static void smca_set_misc_banks_map(unsigned int bank, unsigned int cpu)
215 {
216 u32 low, high;
217
218 /*
219 * For SMCA enabled processors, BLKPTR field of the first MISC register
220 * (MCx_MISC0) indicates presence of additional MISC regs set (MISC1-4).
221 */
222 if (rdmsr_safe(MSR_AMD64_SMCA_MCx_CONFIG(bank), &low, &high))
223 return;
224
225 if (!(low & MCI_CONFIG_MCAX))
226 return;
227
228 if (rdmsr_safe(MSR_AMD64_SMCA_MCx_MISC(bank), &low, &high))
229 return;
230
231 if (low & MASK_BLKPTR_LO)
232 per_cpu(smca_misc_banks_map, cpu) |= BIT_ULL(bank);
233
234 }
235
smca_configure(unsigned int bank,unsigned int cpu)236 static void smca_configure(unsigned int bank, unsigned int cpu)
237 {
238 unsigned int i, hwid_mcatype;
239 struct smca_hwid *s_hwid;
240 u32 high, low;
241 u32 smca_config = MSR_AMD64_SMCA_MCx_CONFIG(bank);
242
243 /* Set appropriate bits in MCA_CONFIG */
244 if (!rdmsr_safe(smca_config, &low, &high)) {
245 /*
246 * OS is required to set the MCAX bit to acknowledge that it is
247 * now using the new MSR ranges and new registers under each
248 * bank. It also means that the OS will configure deferred
249 * errors in the new MCx_CONFIG register. If the bit is not set,
250 * uncorrectable errors will cause a system panic.
251 *
252 * MCA_CONFIG[MCAX] is bit 32 (0 in the high portion of the MSR.)
253 */
254 high |= BIT(0);
255
256 /*
257 * SMCA sets the Deferred Error Interrupt type per bank.
258 *
259 * MCA_CONFIG[DeferredIntTypeSupported] is bit 5, and tells us
260 * if the DeferredIntType bit field is available.
261 *
262 * MCA_CONFIG[DeferredIntType] is bits [38:37] ([6:5] in the
263 * high portion of the MSR). OS should set this to 0x1 to enable
264 * APIC based interrupt. First, check that no interrupt has been
265 * set.
266 */
267 if ((low & BIT(5)) && !((high >> 5) & 0x3))
268 high |= BIT(5);
269
270 wrmsr(smca_config, low, high);
271 }
272
273 smca_set_misc_banks_map(bank, cpu);
274
275 /* Return early if this bank was already initialized. */
276 if (smca_banks[bank].hwid && smca_banks[bank].hwid->hwid_mcatype != 0)
277 return;
278
279 if (rdmsr_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &low, &high)) {
280 pr_warn("Failed to read MCA_IPID for bank %d\n", bank);
281 return;
282 }
283
284 hwid_mcatype = HWID_MCATYPE(high & MCI_IPID_HWID,
285 (high & MCI_IPID_MCATYPE) >> 16);
286
287 for (i = 0; i < ARRAY_SIZE(smca_hwid_mcatypes); i++) {
288 s_hwid = &smca_hwid_mcatypes[i];
289 if (hwid_mcatype == s_hwid->hwid_mcatype) {
290 smca_banks[bank].hwid = s_hwid;
291 smca_banks[bank].id = low;
292 smca_banks[bank].sysfs_id = s_hwid->count++;
293 break;
294 }
295 }
296 }
297
298 struct thresh_restart {
299 struct threshold_block *b;
300 int reset;
301 int set_lvt_off;
302 int lvt_off;
303 u16 old_limit;
304 };
305
is_shared_bank(int bank)306 static inline bool is_shared_bank(int bank)
307 {
308 /*
309 * Scalable MCA provides for only one core to have access to the MSRs of
310 * a shared bank.
311 */
312 if (mce_flags.smca)
313 return false;
314
315 /* Bank 4 is for northbridge reporting and is thus shared */
316 return (bank == 4);
317 }
318
bank4_names(const struct threshold_block * b)319 static const char *bank4_names(const struct threshold_block *b)
320 {
321 switch (b->address) {
322 /* MSR4_MISC0 */
323 case 0x00000413:
324 return "dram";
325
326 case 0xc0000408:
327 return "ht_links";
328
329 case 0xc0000409:
330 return "l3_cache";
331
332 default:
333 WARN(1, "Funny MSR: 0x%08x\n", b->address);
334 return "";
335 }
336 };
337
338
lvt_interrupt_supported(unsigned int bank,u32 msr_high_bits)339 static bool lvt_interrupt_supported(unsigned int bank, u32 msr_high_bits)
340 {
341 /*
342 * bank 4 supports APIC LVT interrupts implicitly since forever.
343 */
344 if (bank == 4)
345 return true;
346
347 /*
348 * IntP: interrupt present; if this bit is set, the thresholding
349 * bank can generate APIC LVT interrupts
350 */
351 return msr_high_bits & BIT(28);
352 }
353
lvt_off_valid(struct threshold_block * b,int apic,u32 lo,u32 hi)354 static int lvt_off_valid(struct threshold_block *b, int apic, u32 lo, u32 hi)
355 {
356 int msr = (hi & MASK_LVTOFF_HI) >> 20;
357
358 if (apic < 0) {
359 pr_err(FW_BUG "cpu %d, failed to setup threshold interrupt "
360 "for bank %d, block %d (MSR%08X=0x%x%08x)\n", b->cpu,
361 b->bank, b->block, b->address, hi, lo);
362 return 0;
363 }
364
365 if (apic != msr) {
366 /*
367 * On SMCA CPUs, LVT offset is programmed at a different MSR, and
368 * the BIOS provides the value. The original field where LVT offset
369 * was set is reserved. Return early here:
370 */
371 if (mce_flags.smca)
372 return 0;
373
374 pr_err(FW_BUG "cpu %d, invalid threshold interrupt offset %d "
375 "for bank %d, block %d (MSR%08X=0x%x%08x)\n",
376 b->cpu, apic, b->bank, b->block, b->address, hi, lo);
377 return 0;
378 }
379
380 return 1;
381 };
382
383 /* Reprogram MCx_MISC MSR behind this threshold bank. */
threshold_restart_bank(void * _tr)384 static void threshold_restart_bank(void *_tr)
385 {
386 struct thresh_restart *tr = _tr;
387 u32 hi, lo;
388
389 /* sysfs write might race against an offline operation */
390 if (!this_cpu_read(threshold_banks) && !tr->set_lvt_off)
391 return;
392
393 rdmsr(tr->b->address, lo, hi);
394
395 if (tr->b->threshold_limit < (hi & THRESHOLD_MAX))
396 tr->reset = 1; /* limit cannot be lower than err count */
397
398 if (tr->reset) { /* reset err count and overflow bit */
399 hi =
400 (hi & ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI)) |
401 (THRESHOLD_MAX - tr->b->threshold_limit);
402 } else if (tr->old_limit) { /* change limit w/o reset */
403 int new_count = (hi & THRESHOLD_MAX) +
404 (tr->old_limit - tr->b->threshold_limit);
405
406 hi = (hi & ~MASK_ERR_COUNT_HI) |
407 (new_count & THRESHOLD_MAX);
408 }
409
410 /* clear IntType */
411 hi &= ~MASK_INT_TYPE_HI;
412
413 if (!tr->b->interrupt_capable)
414 goto done;
415
416 if (tr->set_lvt_off) {
417 if (lvt_off_valid(tr->b, tr->lvt_off, lo, hi)) {
418 /* set new lvt offset */
419 hi &= ~MASK_LVTOFF_HI;
420 hi |= tr->lvt_off << 20;
421 }
422 }
423
424 if (tr->b->interrupt_enable)
425 hi |= INT_TYPE_APIC;
426
427 done:
428
429 hi |= MASK_COUNT_EN_HI;
430 wrmsr(tr->b->address, lo, hi);
431 }
432
mce_threshold_block_init(struct threshold_block * b,int offset)433 static void mce_threshold_block_init(struct threshold_block *b, int offset)
434 {
435 struct thresh_restart tr = {
436 .b = b,
437 .set_lvt_off = 1,
438 .lvt_off = offset,
439 };
440
441 b->threshold_limit = THRESHOLD_MAX;
442 threshold_restart_bank(&tr);
443 };
444
setup_APIC_mce_threshold(int reserved,int new)445 static int setup_APIC_mce_threshold(int reserved, int new)
446 {
447 if (reserved < 0 && !setup_APIC_eilvt(new, THRESHOLD_APIC_VECTOR,
448 APIC_EILVT_MSG_FIX, 0))
449 return new;
450
451 return reserved;
452 }
453
setup_APIC_deferred_error(int reserved,int new)454 static int setup_APIC_deferred_error(int reserved, int new)
455 {
456 if (reserved < 0 && !setup_APIC_eilvt(new, DEFERRED_ERROR_VECTOR,
457 APIC_EILVT_MSG_FIX, 0))
458 return new;
459
460 return reserved;
461 }
462
deferred_error_interrupt_enable(struct cpuinfo_x86 * c)463 static void deferred_error_interrupt_enable(struct cpuinfo_x86 *c)
464 {
465 u32 low = 0, high = 0;
466 int def_offset = -1, def_new;
467
468 if (rdmsr_safe(MSR_CU_DEF_ERR, &low, &high))
469 return;
470
471 def_new = (low & MASK_DEF_LVTOFF) >> 4;
472 if (!(low & MASK_DEF_LVTOFF)) {
473 pr_err(FW_BUG "Your BIOS is not setting up LVT offset 0x2 for deferred error IRQs correctly.\n");
474 def_new = DEF_LVT_OFF;
475 low = (low & ~MASK_DEF_LVTOFF) | (DEF_LVT_OFF << 4);
476 }
477
478 def_offset = setup_APIC_deferred_error(def_offset, def_new);
479 if ((def_offset == def_new) &&
480 (deferred_error_int_vector != amd_deferred_error_interrupt))
481 deferred_error_int_vector = amd_deferred_error_interrupt;
482
483 if (!mce_flags.smca)
484 low = (low & ~MASK_DEF_INT_TYPE) | DEF_INT_TYPE_APIC;
485
486 wrmsr(MSR_CU_DEF_ERR, low, high);
487 }
488
smca_get_block_address(unsigned int bank,unsigned int block,unsigned int cpu)489 static u32 smca_get_block_address(unsigned int bank, unsigned int block,
490 unsigned int cpu)
491 {
492 if (!block)
493 return MSR_AMD64_SMCA_MCx_MISC(bank);
494
495 if (!(per_cpu(smca_misc_banks_map, cpu) & BIT_ULL(bank)))
496 return 0;
497
498 return MSR_AMD64_SMCA_MCx_MISCy(bank, block - 1);
499 }
500
get_block_address(u32 current_addr,u32 low,u32 high,unsigned int bank,unsigned int block,unsigned int cpu)501 static u32 get_block_address(u32 current_addr, u32 low, u32 high,
502 unsigned int bank, unsigned int block,
503 unsigned int cpu)
504 {
505 u32 addr = 0, offset = 0;
506
507 if ((bank >= per_cpu(mce_num_banks, cpu)) || (block >= NR_BLOCKS))
508 return addr;
509
510 if (mce_flags.smca)
511 return smca_get_block_address(bank, block, cpu);
512
513 /* Fall back to method we used for older processors: */
514 switch (block) {
515 case 0:
516 addr = mca_msr_reg(bank, MCA_MISC);
517 break;
518 case 1:
519 offset = ((low & MASK_BLKPTR_LO) >> 21);
520 if (offset)
521 addr = MCG_XBLK_ADDR + offset;
522 break;
523 default:
524 addr = ++current_addr;
525 }
526 return addr;
527 }
528
529 static int
prepare_threshold_block(unsigned int bank,unsigned int block,u32 addr,int offset,u32 misc_high)530 prepare_threshold_block(unsigned int bank, unsigned int block, u32 addr,
531 int offset, u32 misc_high)
532 {
533 unsigned int cpu = smp_processor_id();
534 u32 smca_low, smca_high;
535 struct threshold_block b;
536 int new;
537
538 if (!block)
539 per_cpu(bank_map, cpu) |= BIT_ULL(bank);
540
541 memset(&b, 0, sizeof(b));
542 b.cpu = cpu;
543 b.bank = bank;
544 b.block = block;
545 b.address = addr;
546 b.interrupt_capable = lvt_interrupt_supported(bank, misc_high);
547
548 if (!b.interrupt_capable)
549 goto done;
550
551 b.interrupt_enable = 1;
552
553 if (!mce_flags.smca) {
554 new = (misc_high & MASK_LVTOFF_HI) >> 20;
555 goto set_offset;
556 }
557
558 /* Gather LVT offset for thresholding: */
559 if (rdmsr_safe(MSR_CU_DEF_ERR, &smca_low, &smca_high))
560 goto out;
561
562 new = (smca_low & SMCA_THR_LVT_OFF) >> 12;
563
564 set_offset:
565 offset = setup_APIC_mce_threshold(offset, new);
566 if (offset == new)
567 thresholding_irq_en = true;
568
569 done:
570 mce_threshold_block_init(&b, offset);
571
572 out:
573 return offset;
574 }
575
amd_filter_mce(struct mce * m)576 bool amd_filter_mce(struct mce *m)
577 {
578 enum smca_bank_types bank_type = smca_get_bank_type(m->bank);
579 struct cpuinfo_x86 *c = &boot_cpu_data;
580
581 /* See Family 17h Models 10h-2Fh Erratum #1114. */
582 if (c->x86 == 0x17 &&
583 c->x86_model >= 0x10 && c->x86_model <= 0x2F &&
584 bank_type == SMCA_IF && XEC(m->status, 0x3f) == 10)
585 return true;
586
587 /* NB GART TLB error reporting is disabled by default. */
588 if (c->x86 < 0x17) {
589 if (m->bank == 4 && XEC(m->status, 0x1f) == 0x5)
590 return true;
591 }
592
593 return false;
594 }
595
596 /*
597 * Turn off thresholding banks for the following conditions:
598 * - MC4_MISC thresholding is not supported on Family 0x15.
599 * - Prevent possible spurious interrupts from the IF bank on Family 0x17
600 * Models 0x10-0x2F due to Erratum #1114.
601 */
disable_err_thresholding(struct cpuinfo_x86 * c,unsigned int bank)602 static void disable_err_thresholding(struct cpuinfo_x86 *c, unsigned int bank)
603 {
604 int i, num_msrs;
605 u64 hwcr;
606 bool need_toggle;
607 u32 msrs[NR_BLOCKS];
608
609 if (c->x86 == 0x15 && bank == 4) {
610 msrs[0] = 0x00000413; /* MC4_MISC0 */
611 msrs[1] = 0xc0000408; /* MC4_MISC1 */
612 num_msrs = 2;
613 } else if (c->x86 == 0x17 &&
614 (c->x86_model >= 0x10 && c->x86_model <= 0x2F)) {
615
616 if (smca_get_bank_type(bank) != SMCA_IF)
617 return;
618
619 msrs[0] = MSR_AMD64_SMCA_MCx_MISC(bank);
620 num_msrs = 1;
621 } else {
622 return;
623 }
624
625 rdmsrl(MSR_K7_HWCR, hwcr);
626
627 /* McStatusWrEn has to be set */
628 need_toggle = !(hwcr & BIT(18));
629 if (need_toggle)
630 wrmsrl(MSR_K7_HWCR, hwcr | BIT(18));
631
632 /* Clear CntP bit safely */
633 for (i = 0; i < num_msrs; i++)
634 msr_clear_bit(msrs[i], 62);
635
636 /* restore old settings */
637 if (need_toggle)
638 wrmsrl(MSR_K7_HWCR, hwcr);
639 }
640
641 /* cpu init entry point, called from mce.c with preempt off */
mce_amd_feature_init(struct cpuinfo_x86 * c)642 void mce_amd_feature_init(struct cpuinfo_x86 *c)
643 {
644 unsigned int bank, block, cpu = smp_processor_id();
645 u32 low = 0, high = 0, address = 0;
646 int offset = -1;
647
648
649 for (bank = 0; bank < this_cpu_read(mce_num_banks); ++bank) {
650 if (mce_flags.smca)
651 smca_configure(bank, cpu);
652
653 disable_err_thresholding(c, bank);
654
655 for (block = 0; block < NR_BLOCKS; ++block) {
656 address = get_block_address(address, low, high, bank, block, cpu);
657 if (!address)
658 break;
659
660 if (rdmsr_safe(address, &low, &high))
661 break;
662
663 if (!(high & MASK_VALID_HI))
664 continue;
665
666 if (!(high & MASK_CNTP_HI) ||
667 (high & MASK_LOCKED_HI))
668 continue;
669
670 offset = prepare_threshold_block(bank, block, address, offset, high);
671 }
672 }
673
674 if (mce_flags.succor)
675 deferred_error_interrupt_enable(c);
676 }
677
umc_normaddr_to_sysaddr(u64 norm_addr,u16 nid,u8 umc,u64 * sys_addr)678 int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr)
679 {
680 u64 dram_base_addr, dram_limit_addr, dram_hole_base;
681 /* We start from the normalized address */
682 u64 ret_addr = norm_addr;
683
684 u32 tmp;
685
686 u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask;
687 u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets;
688 u8 intlv_addr_sel, intlv_addr_bit;
689 u8 num_intlv_bits, hashed_bit;
690 u8 lgcy_mmio_hole_en, base = 0;
691 u8 cs_mask, cs_id = 0;
692 bool hash_enabled = false;
693
694 /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */
695 if (amd_df_indirect_read(nid, 0, 0x1B4, umc, &tmp))
696 goto out_err;
697
698 /* Remove HiAddrOffset from normalized address, if enabled: */
699 if (tmp & BIT(0)) {
700 u64 hi_addr_offset = (tmp & GENMASK_ULL(31, 20)) << 8;
701
702 if (norm_addr >= hi_addr_offset) {
703 ret_addr -= hi_addr_offset;
704 base = 1;
705 }
706 }
707
708 /* Read D18F0x110 (DramBaseAddress). */
709 if (amd_df_indirect_read(nid, 0, 0x110 + (8 * base), umc, &tmp))
710 goto out_err;
711
712 /* Check if address range is valid. */
713 if (!(tmp & BIT(0))) {
714 pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n",
715 __func__, tmp);
716 goto out_err;
717 }
718
719 lgcy_mmio_hole_en = tmp & BIT(1);
720 intlv_num_chan = (tmp >> 4) & 0xF;
721 intlv_addr_sel = (tmp >> 8) & 0x7;
722 dram_base_addr = (tmp & GENMASK_ULL(31, 12)) << 16;
723
724 /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */
725 if (intlv_addr_sel > 3) {
726 pr_err("%s: Invalid interleave address select %d.\n",
727 __func__, intlv_addr_sel);
728 goto out_err;
729 }
730
731 /* Read D18F0x114 (DramLimitAddress). */
732 if (amd_df_indirect_read(nid, 0, 0x114 + (8 * base), umc, &tmp))
733 goto out_err;
734
735 intlv_num_sockets = (tmp >> 8) & 0x1;
736 intlv_num_dies = (tmp >> 10) & 0x3;
737 dram_limit_addr = ((tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0);
738
739 intlv_addr_bit = intlv_addr_sel + 8;
740
741 /* Re-use intlv_num_chan by setting it equal to log2(#channels) */
742 switch (intlv_num_chan) {
743 case 0: intlv_num_chan = 0; break;
744 case 1: intlv_num_chan = 1; break;
745 case 3: intlv_num_chan = 2; break;
746 case 5: intlv_num_chan = 3; break;
747 case 7: intlv_num_chan = 4; break;
748
749 case 8: intlv_num_chan = 1;
750 hash_enabled = true;
751 break;
752 default:
753 pr_err("%s: Invalid number of interleaved channels %d.\n",
754 __func__, intlv_num_chan);
755 goto out_err;
756 }
757
758 num_intlv_bits = intlv_num_chan;
759
760 if (intlv_num_dies > 2) {
761 pr_err("%s: Invalid number of interleaved nodes/dies %d.\n",
762 __func__, intlv_num_dies);
763 goto out_err;
764 }
765
766 num_intlv_bits += intlv_num_dies;
767
768 /* Add a bit if sockets are interleaved. */
769 num_intlv_bits += intlv_num_sockets;
770
771 /* Assert num_intlv_bits <= 4 */
772 if (num_intlv_bits > 4) {
773 pr_err("%s: Invalid interleave bits %d.\n",
774 __func__, num_intlv_bits);
775 goto out_err;
776 }
777
778 if (num_intlv_bits > 0) {
779 u64 temp_addr_x, temp_addr_i, temp_addr_y;
780 u8 die_id_bit, sock_id_bit, cs_fabric_id;
781
782 /*
783 * Read FabricBlockInstanceInformation3_CS[BlockFabricID].
784 * This is the fabric id for this coherent slave. Use
785 * umc/channel# as instance id of the coherent slave
786 * for FICAA.
787 */
788 if (amd_df_indirect_read(nid, 0, 0x50, umc, &tmp))
789 goto out_err;
790
791 cs_fabric_id = (tmp >> 8) & 0xFF;
792 die_id_bit = 0;
793
794 /* If interleaved over more than 1 channel: */
795 if (intlv_num_chan) {
796 die_id_bit = intlv_num_chan;
797 cs_mask = (1 << die_id_bit) - 1;
798 cs_id = cs_fabric_id & cs_mask;
799 }
800
801 sock_id_bit = die_id_bit;
802
803 /* Read D18F1x208 (SystemFabricIdMask). */
804 if (intlv_num_dies || intlv_num_sockets)
805 if (amd_df_indirect_read(nid, 1, 0x208, umc, &tmp))
806 goto out_err;
807
808 /* If interleaved over more than 1 die. */
809 if (intlv_num_dies) {
810 sock_id_bit = die_id_bit + intlv_num_dies;
811 die_id_shift = (tmp >> 24) & 0xF;
812 die_id_mask = (tmp >> 8) & 0xFF;
813
814 cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit;
815 }
816
817 /* If interleaved over more than 1 socket. */
818 if (intlv_num_sockets) {
819 socket_id_shift = (tmp >> 28) & 0xF;
820 socket_id_mask = (tmp >> 16) & 0xFF;
821
822 cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit;
823 }
824
825 /*
826 * The pre-interleaved address consists of XXXXXXIIIYYYYY
827 * where III is the ID for this CS, and XXXXXXYYYYY are the
828 * address bits from the post-interleaved address.
829 * "num_intlv_bits" has been calculated to tell us how many "I"
830 * bits there are. "intlv_addr_bit" tells us how many "Y" bits
831 * there are (where "I" starts).
832 */
833 temp_addr_y = ret_addr & GENMASK_ULL(intlv_addr_bit-1, 0);
834 temp_addr_i = (cs_id << intlv_addr_bit);
835 temp_addr_x = (ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits;
836 ret_addr = temp_addr_x | temp_addr_i | temp_addr_y;
837 }
838
839 /* Add dram base address */
840 ret_addr += dram_base_addr;
841
842 /* If legacy MMIO hole enabled */
843 if (lgcy_mmio_hole_en) {
844 if (amd_df_indirect_read(nid, 0, 0x104, umc, &tmp))
845 goto out_err;
846
847 dram_hole_base = tmp & GENMASK(31, 24);
848 if (ret_addr >= dram_hole_base)
849 ret_addr += (BIT_ULL(32) - dram_hole_base);
850 }
851
852 if (hash_enabled) {
853 /* Save some parentheses and grab ls-bit at the end. */
854 hashed_bit = (ret_addr >> 12) ^
855 (ret_addr >> 18) ^
856 (ret_addr >> 21) ^
857 (ret_addr >> 30) ^
858 cs_id;
859
860 hashed_bit &= BIT(0);
861
862 if (hashed_bit != ((ret_addr >> intlv_addr_bit) & BIT(0)))
863 ret_addr ^= BIT(intlv_addr_bit);
864 }
865
866 /* Is calculated system address is above DRAM limit address? */
867 if (ret_addr > dram_limit_addr)
868 goto out_err;
869
870 *sys_addr = ret_addr;
871 return 0;
872
873 out_err:
874 return -EINVAL;
875 }
876 EXPORT_SYMBOL_GPL(umc_normaddr_to_sysaddr);
877
amd_mce_is_memory_error(struct mce * m)878 bool amd_mce_is_memory_error(struct mce *m)
879 {
880 /* ErrCodeExt[20:16] */
881 u8 xec = (m->status >> 16) & 0x1f;
882
883 if (mce_flags.smca)
884 return smca_get_bank_type(m->bank) == SMCA_UMC && xec == 0x0;
885
886 return m->bank == 4 && xec == 0x8;
887 }
888
__log_error(unsigned int bank,u64 status,u64 addr,u64 misc)889 static void __log_error(unsigned int bank, u64 status, u64 addr, u64 misc)
890 {
891 struct mce m;
892
893 mce_setup(&m);
894
895 m.status = status;
896 m.misc = misc;
897 m.bank = bank;
898 m.tsc = rdtsc();
899
900 if (m.status & MCI_STATUS_ADDRV) {
901 m.addr = addr;
902
903 /*
904 * Extract [55:<lsb>] where lsb is the least significant
905 * *valid* bit of the address bits.
906 */
907 if (mce_flags.smca) {
908 u8 lsb = (m.addr >> 56) & 0x3f;
909
910 m.addr &= GENMASK_ULL(55, lsb);
911 }
912 }
913
914 if (mce_flags.smca) {
915 rdmsrl(MSR_AMD64_SMCA_MCx_IPID(bank), m.ipid);
916
917 if (m.status & MCI_STATUS_SYNDV)
918 rdmsrl(MSR_AMD64_SMCA_MCx_SYND(bank), m.synd);
919 }
920
921 mce_log(&m);
922 }
923
DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error)924 DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error)
925 {
926 trace_deferred_error_apic_entry(DEFERRED_ERROR_VECTOR);
927 inc_irq_stat(irq_deferred_error_count);
928 deferred_error_int_vector();
929 trace_deferred_error_apic_exit(DEFERRED_ERROR_VECTOR);
930 ack_APIC_irq();
931 }
932
933 /*
934 * Returns true if the logged error is deferred. False, otherwise.
935 */
936 static inline bool
_log_error_bank(unsigned int bank,u32 msr_stat,u32 msr_addr,u64 misc)937 _log_error_bank(unsigned int bank, u32 msr_stat, u32 msr_addr, u64 misc)
938 {
939 u64 status, addr = 0;
940
941 rdmsrl(msr_stat, status);
942 if (!(status & MCI_STATUS_VAL))
943 return false;
944
945 if (status & MCI_STATUS_ADDRV)
946 rdmsrl(msr_addr, addr);
947
948 __log_error(bank, status, addr, misc);
949
950 wrmsrl(msr_stat, 0);
951
952 return status & MCI_STATUS_DEFERRED;
953 }
954
_log_error_deferred(unsigned int bank,u32 misc)955 static bool _log_error_deferred(unsigned int bank, u32 misc)
956 {
957 if (!_log_error_bank(bank, mca_msr_reg(bank, MCA_STATUS),
958 mca_msr_reg(bank, MCA_ADDR), misc))
959 return false;
960
961 /*
962 * Non-SMCA systems don't have MCA_DESTAT/MCA_DEADDR registers.
963 * Return true here to avoid accessing these registers.
964 */
965 if (!mce_flags.smca)
966 return true;
967
968 /* Clear MCA_DESTAT if the deferred error was logged from MCA_STATUS. */
969 wrmsrl(MSR_AMD64_SMCA_MCx_DESTAT(bank), 0);
970 return true;
971 }
972
973 /*
974 * We have three scenarios for checking for Deferred errors:
975 *
976 * 1) Non-SMCA systems check MCA_STATUS and log error if found.
977 * 2) SMCA systems check MCA_STATUS. If error is found then log it and also
978 * clear MCA_DESTAT.
979 * 3) SMCA systems check MCA_DESTAT, if error was not found in MCA_STATUS, and
980 * log it.
981 */
log_error_deferred(unsigned int bank)982 static void log_error_deferred(unsigned int bank)
983 {
984 if (_log_error_deferred(bank, 0))
985 return;
986
987 /*
988 * Only deferred errors are logged in MCA_DE{STAT,ADDR} so just check
989 * for a valid error.
990 */
991 _log_error_bank(bank, MSR_AMD64_SMCA_MCx_DESTAT(bank),
992 MSR_AMD64_SMCA_MCx_DEADDR(bank), 0);
993 }
994
995 /* APIC interrupt handler for deferred errors */
amd_deferred_error_interrupt(void)996 static void amd_deferred_error_interrupt(void)
997 {
998 unsigned int bank;
999
1000 for (bank = 0; bank < this_cpu_read(mce_num_banks); ++bank)
1001 log_error_deferred(bank);
1002 }
1003
log_error_thresholding(unsigned int bank,u64 misc)1004 static void log_error_thresholding(unsigned int bank, u64 misc)
1005 {
1006 _log_error_deferred(bank, misc);
1007 }
1008
log_and_reset_block(struct threshold_block * block)1009 static void log_and_reset_block(struct threshold_block *block)
1010 {
1011 struct thresh_restart tr;
1012 u32 low = 0, high = 0;
1013
1014 if (!block)
1015 return;
1016
1017 if (rdmsr_safe(block->address, &low, &high))
1018 return;
1019
1020 if (!(high & MASK_OVERFLOW_HI))
1021 return;
1022
1023 /* Log the MCE which caused the threshold event. */
1024 log_error_thresholding(block->bank, ((u64)high << 32) | low);
1025
1026 /* Reset threshold block after logging error. */
1027 memset(&tr, 0, sizeof(tr));
1028 tr.b = block;
1029 threshold_restart_bank(&tr);
1030 }
1031
1032 /*
1033 * Threshold interrupt handler will service THRESHOLD_APIC_VECTOR. The interrupt
1034 * goes off when error_count reaches threshold_limit.
1035 */
amd_threshold_interrupt(void)1036 static void amd_threshold_interrupt(void)
1037 {
1038 struct threshold_block *first_block = NULL, *block = NULL, *tmp = NULL;
1039 struct threshold_bank **bp = this_cpu_read(threshold_banks);
1040 unsigned int bank, cpu = smp_processor_id();
1041
1042 /*
1043 * Validate that the threshold bank has been initialized already. The
1044 * handler is installed at boot time, but on a hotplug event the
1045 * interrupt might fire before the data has been initialized.
1046 */
1047 if (!bp)
1048 return;
1049
1050 for (bank = 0; bank < this_cpu_read(mce_num_banks); ++bank) {
1051 if (!(per_cpu(bank_map, cpu) & BIT_ULL(bank)))
1052 continue;
1053
1054 first_block = bp[bank]->blocks;
1055 if (!first_block)
1056 continue;
1057
1058 /*
1059 * The first block is also the head of the list. Check it first
1060 * before iterating over the rest.
1061 */
1062 log_and_reset_block(first_block);
1063 list_for_each_entry_safe(block, tmp, &first_block->miscj, miscj)
1064 log_and_reset_block(block);
1065 }
1066 }
1067
1068 /*
1069 * Sysfs Interface
1070 */
1071
1072 struct threshold_attr {
1073 struct attribute attr;
1074 ssize_t (*show) (struct threshold_block *, char *);
1075 ssize_t (*store) (struct threshold_block *, const char *, size_t count);
1076 };
1077
1078 #define SHOW_FIELDS(name) \
1079 static ssize_t show_ ## name(struct threshold_block *b, char *buf) \
1080 { \
1081 return sprintf(buf, "%lu\n", (unsigned long) b->name); \
1082 }
1083 SHOW_FIELDS(interrupt_enable)
SHOW_FIELDS(threshold_limit)1084 SHOW_FIELDS(threshold_limit)
1085
1086 static ssize_t
1087 store_interrupt_enable(struct threshold_block *b, const char *buf, size_t size)
1088 {
1089 struct thresh_restart tr;
1090 unsigned long new;
1091
1092 if (!b->interrupt_capable)
1093 return -EINVAL;
1094
1095 if (kstrtoul(buf, 0, &new) < 0)
1096 return -EINVAL;
1097
1098 b->interrupt_enable = !!new;
1099
1100 memset(&tr, 0, sizeof(tr));
1101 tr.b = b;
1102
1103 if (smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1))
1104 return -ENODEV;
1105
1106 return size;
1107 }
1108
1109 static ssize_t
store_threshold_limit(struct threshold_block * b,const char * buf,size_t size)1110 store_threshold_limit(struct threshold_block *b, const char *buf, size_t size)
1111 {
1112 struct thresh_restart tr;
1113 unsigned long new;
1114
1115 if (kstrtoul(buf, 0, &new) < 0)
1116 return -EINVAL;
1117
1118 if (new > THRESHOLD_MAX)
1119 new = THRESHOLD_MAX;
1120 if (new < 1)
1121 new = 1;
1122
1123 memset(&tr, 0, sizeof(tr));
1124 tr.old_limit = b->threshold_limit;
1125 b->threshold_limit = new;
1126 tr.b = b;
1127
1128 if (smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1))
1129 return -ENODEV;
1130
1131 return size;
1132 }
1133
show_error_count(struct threshold_block * b,char * buf)1134 static ssize_t show_error_count(struct threshold_block *b, char *buf)
1135 {
1136 u32 lo, hi;
1137
1138 /* CPU might be offline by now */
1139 if (rdmsr_on_cpu(b->cpu, b->address, &lo, &hi))
1140 return -ENODEV;
1141
1142 return sprintf(buf, "%u\n", ((hi & THRESHOLD_MAX) -
1143 (THRESHOLD_MAX - b->threshold_limit)));
1144 }
1145
1146 static struct threshold_attr error_count = {
1147 .attr = {.name = __stringify(error_count), .mode = 0444 },
1148 .show = show_error_count,
1149 };
1150
1151 #define RW_ATTR(val) \
1152 static struct threshold_attr val = { \
1153 .attr = {.name = __stringify(val), .mode = 0644 }, \
1154 .show = show_## val, \
1155 .store = store_## val, \
1156 };
1157
1158 RW_ATTR(interrupt_enable);
1159 RW_ATTR(threshold_limit);
1160
1161 static struct attribute *default_attrs[] = {
1162 &threshold_limit.attr,
1163 &error_count.attr,
1164 NULL, /* possibly interrupt_enable if supported, see below */
1165 NULL,
1166 };
1167
1168 #define to_block(k) container_of(k, struct threshold_block, kobj)
1169 #define to_attr(a) container_of(a, struct threshold_attr, attr)
1170
show(struct kobject * kobj,struct attribute * attr,char * buf)1171 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
1172 {
1173 struct threshold_block *b = to_block(kobj);
1174 struct threshold_attr *a = to_attr(attr);
1175 ssize_t ret;
1176
1177 ret = a->show ? a->show(b, buf) : -EIO;
1178
1179 return ret;
1180 }
1181
store(struct kobject * kobj,struct attribute * attr,const char * buf,size_t count)1182 static ssize_t store(struct kobject *kobj, struct attribute *attr,
1183 const char *buf, size_t count)
1184 {
1185 struct threshold_block *b = to_block(kobj);
1186 struct threshold_attr *a = to_attr(attr);
1187 ssize_t ret;
1188
1189 ret = a->store ? a->store(b, buf, count) : -EIO;
1190
1191 return ret;
1192 }
1193
1194 static const struct sysfs_ops threshold_ops = {
1195 .show = show,
1196 .store = store,
1197 };
1198
1199 static void threshold_block_release(struct kobject *kobj);
1200
1201 static struct kobj_type threshold_ktype = {
1202 .sysfs_ops = &threshold_ops,
1203 .default_attrs = default_attrs,
1204 .release = threshold_block_release,
1205 };
1206
get_name(unsigned int bank,struct threshold_block * b)1207 static const char *get_name(unsigned int bank, struct threshold_block *b)
1208 {
1209 enum smca_bank_types bank_type;
1210
1211 if (!mce_flags.smca) {
1212 if (b && bank == 4)
1213 return bank4_names(b);
1214
1215 return th_names[bank];
1216 }
1217
1218 bank_type = smca_get_bank_type(bank);
1219 if (bank_type >= N_SMCA_BANK_TYPES)
1220 return NULL;
1221
1222 if (b && bank_type == SMCA_UMC) {
1223 if (b->block < ARRAY_SIZE(smca_umc_block_names))
1224 return smca_umc_block_names[b->block];
1225 return NULL;
1226 }
1227
1228 if (smca_banks[bank].hwid->count == 1)
1229 return smca_get_name(bank_type);
1230
1231 snprintf(buf_mcatype, MAX_MCATYPE_NAME_LEN,
1232 "%s_%x", smca_get_name(bank_type),
1233 smca_banks[bank].sysfs_id);
1234 return buf_mcatype;
1235 }
1236
allocate_threshold_blocks(unsigned int cpu,struct threshold_bank * tb,unsigned int bank,unsigned int block,u32 address)1237 static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb,
1238 unsigned int bank, unsigned int block,
1239 u32 address)
1240 {
1241 struct threshold_block *b = NULL;
1242 u32 low, high;
1243 int err;
1244
1245 if ((bank >= this_cpu_read(mce_num_banks)) || (block >= NR_BLOCKS))
1246 return 0;
1247
1248 if (rdmsr_safe(address, &low, &high))
1249 return 0;
1250
1251 if (!(high & MASK_VALID_HI)) {
1252 if (block)
1253 goto recurse;
1254 else
1255 return 0;
1256 }
1257
1258 if (!(high & MASK_CNTP_HI) ||
1259 (high & MASK_LOCKED_HI))
1260 goto recurse;
1261
1262 b = kzalloc(sizeof(struct threshold_block), GFP_KERNEL);
1263 if (!b)
1264 return -ENOMEM;
1265
1266 b->block = block;
1267 b->bank = bank;
1268 b->cpu = cpu;
1269 b->address = address;
1270 b->interrupt_enable = 0;
1271 b->interrupt_capable = lvt_interrupt_supported(bank, high);
1272 b->threshold_limit = THRESHOLD_MAX;
1273
1274 if (b->interrupt_capable) {
1275 threshold_ktype.default_attrs[2] = &interrupt_enable.attr;
1276 b->interrupt_enable = 1;
1277 } else {
1278 threshold_ktype.default_attrs[2] = NULL;
1279 }
1280
1281 INIT_LIST_HEAD(&b->miscj);
1282
1283 /* This is safe as @tb is not visible yet */
1284 if (tb->blocks)
1285 list_add(&b->miscj, &tb->blocks->miscj);
1286 else
1287 tb->blocks = b;
1288
1289 err = kobject_init_and_add(&b->kobj, &threshold_ktype, tb->kobj, get_name(bank, b));
1290 if (err)
1291 goto out_free;
1292 recurse:
1293 address = get_block_address(address, low, high, bank, ++block, cpu);
1294 if (!address)
1295 return 0;
1296
1297 err = allocate_threshold_blocks(cpu, tb, bank, block, address);
1298 if (err)
1299 goto out_free;
1300
1301 if (b)
1302 kobject_uevent(&b->kobj, KOBJ_ADD);
1303
1304 return 0;
1305
1306 out_free:
1307 if (b) {
1308 list_del(&b->miscj);
1309 kobject_put(&b->kobj);
1310 }
1311 return err;
1312 }
1313
__threshold_add_blocks(struct threshold_bank * b)1314 static int __threshold_add_blocks(struct threshold_bank *b)
1315 {
1316 struct list_head *head = &b->blocks->miscj;
1317 struct threshold_block *pos = NULL;
1318 struct threshold_block *tmp = NULL;
1319 int err = 0;
1320
1321 err = kobject_add(&b->blocks->kobj, b->kobj, b->blocks->kobj.name);
1322 if (err)
1323 return err;
1324
1325 list_for_each_entry_safe(pos, tmp, head, miscj) {
1326
1327 err = kobject_add(&pos->kobj, b->kobj, pos->kobj.name);
1328 if (err) {
1329 list_for_each_entry_safe_reverse(pos, tmp, head, miscj)
1330 kobject_del(&pos->kobj);
1331
1332 return err;
1333 }
1334 }
1335 return err;
1336 }
1337
threshold_create_bank(struct threshold_bank ** bp,unsigned int cpu,unsigned int bank)1338 static int threshold_create_bank(struct threshold_bank **bp, unsigned int cpu,
1339 unsigned int bank)
1340 {
1341 struct device *dev = this_cpu_read(mce_device);
1342 struct amd_northbridge *nb = NULL;
1343 struct threshold_bank *b = NULL;
1344 const char *name = get_name(bank, NULL);
1345 int err = 0;
1346
1347 if (!dev)
1348 return -ENODEV;
1349
1350 if (is_shared_bank(bank)) {
1351 nb = node_to_amd_nb(amd_get_nb_id(cpu));
1352
1353 /* threshold descriptor already initialized on this node? */
1354 if (nb && nb->bank4) {
1355 /* yes, use it */
1356 b = nb->bank4;
1357 err = kobject_add(b->kobj, &dev->kobj, name);
1358 if (err)
1359 goto out;
1360
1361 bp[bank] = b;
1362 refcount_inc(&b->cpus);
1363
1364 err = __threshold_add_blocks(b);
1365
1366 goto out;
1367 }
1368 }
1369
1370 b = kzalloc(sizeof(struct threshold_bank), GFP_KERNEL);
1371 if (!b) {
1372 err = -ENOMEM;
1373 goto out;
1374 }
1375
1376 /* Associate the bank with the per-CPU MCE device */
1377 b->kobj = kobject_create_and_add(name, &dev->kobj);
1378 if (!b->kobj) {
1379 err = -EINVAL;
1380 goto out_free;
1381 }
1382
1383 if (is_shared_bank(bank)) {
1384 b->shared = 1;
1385 refcount_set(&b->cpus, 1);
1386
1387 /* nb is already initialized, see above */
1388 if (nb) {
1389 WARN_ON(nb->bank4);
1390 nb->bank4 = b;
1391 }
1392 }
1393
1394 err = allocate_threshold_blocks(cpu, b, bank, 0, mca_msr_reg(bank, MCA_MISC));
1395 if (err)
1396 goto out_kobj;
1397
1398 bp[bank] = b;
1399 return 0;
1400
1401 out_kobj:
1402 kobject_put(b->kobj);
1403 out_free:
1404 kfree(b);
1405 out:
1406 return err;
1407 }
1408
threshold_block_release(struct kobject * kobj)1409 static void threshold_block_release(struct kobject *kobj)
1410 {
1411 kfree(to_block(kobj));
1412 }
1413
deallocate_threshold_blocks(struct threshold_bank * bank)1414 static void deallocate_threshold_blocks(struct threshold_bank *bank)
1415 {
1416 struct threshold_block *pos, *tmp;
1417
1418 list_for_each_entry_safe(pos, tmp, &bank->blocks->miscj, miscj) {
1419 list_del(&pos->miscj);
1420 kobject_put(&pos->kobj);
1421 }
1422
1423 kobject_put(&bank->blocks->kobj);
1424 }
1425
__threshold_remove_blocks(struct threshold_bank * b)1426 static void __threshold_remove_blocks(struct threshold_bank *b)
1427 {
1428 struct threshold_block *pos = NULL;
1429 struct threshold_block *tmp = NULL;
1430
1431 kobject_del(b->kobj);
1432
1433 list_for_each_entry_safe(pos, tmp, &b->blocks->miscj, miscj)
1434 kobject_del(&pos->kobj);
1435 }
1436
threshold_remove_bank(struct threshold_bank * bank)1437 static void threshold_remove_bank(struct threshold_bank *bank)
1438 {
1439 struct amd_northbridge *nb;
1440
1441 if (!bank->blocks)
1442 goto out_free;
1443
1444 if (!bank->shared)
1445 goto out_dealloc;
1446
1447 if (!refcount_dec_and_test(&bank->cpus)) {
1448 __threshold_remove_blocks(bank);
1449 return;
1450 } else {
1451 /*
1452 * The last CPU on this node using the shared bank is going
1453 * away, remove that bank now.
1454 */
1455 nb = node_to_amd_nb(amd_get_nb_id(smp_processor_id()));
1456 nb->bank4 = NULL;
1457 }
1458
1459 out_dealloc:
1460 deallocate_threshold_blocks(bank);
1461
1462 out_free:
1463 kobject_put(bank->kobj);
1464 kfree(bank);
1465 }
1466
__threshold_remove_device(struct threshold_bank ** bp)1467 static void __threshold_remove_device(struct threshold_bank **bp)
1468 {
1469 unsigned int bank, numbanks = this_cpu_read(mce_num_banks);
1470
1471 for (bank = 0; bank < numbanks; bank++) {
1472 if (!bp[bank])
1473 continue;
1474
1475 threshold_remove_bank(bp[bank]);
1476 bp[bank] = NULL;
1477 }
1478 kfree(bp);
1479 }
1480
mce_threshold_remove_device(unsigned int cpu)1481 int mce_threshold_remove_device(unsigned int cpu)
1482 {
1483 struct threshold_bank **bp = this_cpu_read(threshold_banks);
1484
1485 if (!bp)
1486 return 0;
1487
1488 /*
1489 * Clear the pointer before cleaning up, so that the interrupt won't
1490 * touch anything of this.
1491 */
1492 this_cpu_write(threshold_banks, NULL);
1493
1494 __threshold_remove_device(bp);
1495 return 0;
1496 }
1497
1498 /**
1499 * mce_threshold_create_device - Create the per-CPU MCE threshold device
1500 * @cpu: The plugged in CPU
1501 *
1502 * Create directories and files for all valid threshold banks.
1503 *
1504 * This is invoked from the CPU hotplug callback which was installed in
1505 * mcheck_init_device(). The invocation happens in context of the hotplug
1506 * thread running on @cpu. The callback is invoked on all CPUs which are
1507 * online when the callback is installed or during a real hotplug event.
1508 */
mce_threshold_create_device(unsigned int cpu)1509 int mce_threshold_create_device(unsigned int cpu)
1510 {
1511 unsigned int numbanks, bank;
1512 struct threshold_bank **bp;
1513 int err;
1514
1515 if (!mce_flags.amd_threshold)
1516 return 0;
1517
1518 bp = this_cpu_read(threshold_banks);
1519 if (bp)
1520 return 0;
1521
1522 numbanks = this_cpu_read(mce_num_banks);
1523 bp = kcalloc(numbanks, sizeof(*bp), GFP_KERNEL);
1524 if (!bp)
1525 return -ENOMEM;
1526
1527 for (bank = 0; bank < numbanks; ++bank) {
1528 if (!(this_cpu_read(bank_map) & BIT_ULL(bank)))
1529 continue;
1530 err = threshold_create_bank(bp, cpu, bank);
1531 if (err) {
1532 __threshold_remove_device(bp);
1533 return err;
1534 }
1535 }
1536 this_cpu_write(threshold_banks, bp);
1537
1538 if (thresholding_irq_en)
1539 mce_threshold_vector = amd_threshold_interrupt;
1540 return 0;
1541 }
1542