1 /*
2 * edac_mc kernel module
3 * (C) 2005, 2006 Linux Networx (http://lnxi.com)
4 * This file may be distributed under the terms of the
5 * GNU General Public License.
6 *
7 * Written by Thayne Harbaugh
8 * Based on work by Dan Hollis <goemon at anime dot net> and others.
9 * http://www.anime.net/~goemon/linux-ecc/
10 *
11 * Modified by Dave Peterson and Doug Thompson
12 *
13 */
14
15 #include <linux/module.h>
16 #include <linux/proc_fs.h>
17 #include <linux/kernel.h>
18 #include <linux/types.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/sysctl.h>
22 #include <linux/highmem.h>
23 #include <linux/timer.h>
24 #include <linux/slab.h>
25 #include <linux/jiffies.h>
26 #include <linux/spinlock.h>
27 #include <linux/list.h>
28 #include <linux/ctype.h>
29 #include <linux/edac.h>
30 #include <linux/bitops.h>
31 #include <asm/uaccess.h>
32 #include <asm/page.h>
33 #include "edac_core.h"
34 #include "edac_module.h"
35 #include <ras/ras_event.h>
36
37 #ifdef CONFIG_EDAC_ATOMIC_SCRUB
38 #include <asm/edac.h>
39 #else
40 #define edac_atomic_scrub(va, size) do { } while (0)
41 #endif
42
43 /* lock to memory controller's control array */
44 static DEFINE_MUTEX(mem_ctls_mutex);
45 static LIST_HEAD(mc_devices);
46
47 /*
48 * Used to lock EDAC MC to just one module, avoiding two drivers e. g.
49 * apei/ghes and i7core_edac to be used at the same time.
50 */
51 static void const *edac_mc_owner;
52
53 static struct bus_type mc_bus[EDAC_MAX_MCS];
54
edac_dimm_info_location(struct dimm_info * dimm,char * buf,unsigned len)55 unsigned edac_dimm_info_location(struct dimm_info *dimm, char *buf,
56 unsigned len)
57 {
58 struct mem_ctl_info *mci = dimm->mci;
59 int i, n, count = 0;
60 char *p = buf;
61
62 for (i = 0; i < mci->n_layers; i++) {
63 n = snprintf(p, len, "%s %d ",
64 edac_layer_name[mci->layers[i].type],
65 dimm->location[i]);
66 p += n;
67 len -= n;
68 count += n;
69 if (!len)
70 break;
71 }
72
73 return count;
74 }
75
76 #ifdef CONFIG_EDAC_DEBUG
77
edac_mc_dump_channel(struct rank_info * chan)78 static void edac_mc_dump_channel(struct rank_info *chan)
79 {
80 edac_dbg(4, " channel->chan_idx = %d\n", chan->chan_idx);
81 edac_dbg(4, " channel = %p\n", chan);
82 edac_dbg(4, " channel->csrow = %p\n", chan->csrow);
83 edac_dbg(4, " channel->dimm = %p\n", chan->dimm);
84 }
85
edac_mc_dump_dimm(struct dimm_info * dimm,int number)86 static void edac_mc_dump_dimm(struct dimm_info *dimm, int number)
87 {
88 char location[80];
89
90 edac_dimm_info_location(dimm, location, sizeof(location));
91
92 edac_dbg(4, "%s%i: %smapped as virtual row %d, chan %d\n",
93 dimm->mci->csbased ? "rank" : "dimm",
94 number, location, dimm->csrow, dimm->cschannel);
95 edac_dbg(4, " dimm = %p\n", dimm);
96 edac_dbg(4, " dimm->label = '%s'\n", dimm->label);
97 edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm->nr_pages);
98 edac_dbg(4, " dimm->grain = %d\n", dimm->grain);
99 edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm->nr_pages);
100 }
101
edac_mc_dump_csrow(struct csrow_info * csrow)102 static void edac_mc_dump_csrow(struct csrow_info *csrow)
103 {
104 edac_dbg(4, "csrow->csrow_idx = %d\n", csrow->csrow_idx);
105 edac_dbg(4, " csrow = %p\n", csrow);
106 edac_dbg(4, " csrow->first_page = 0x%lx\n", csrow->first_page);
107 edac_dbg(4, " csrow->last_page = 0x%lx\n", csrow->last_page);
108 edac_dbg(4, " csrow->page_mask = 0x%lx\n", csrow->page_mask);
109 edac_dbg(4, " csrow->nr_channels = %d\n", csrow->nr_channels);
110 edac_dbg(4, " csrow->channels = %p\n", csrow->channels);
111 edac_dbg(4, " csrow->mci = %p\n", csrow->mci);
112 }
113
edac_mc_dump_mci(struct mem_ctl_info * mci)114 static void edac_mc_dump_mci(struct mem_ctl_info *mci)
115 {
116 edac_dbg(3, "\tmci = %p\n", mci);
117 edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci->mtype_cap);
118 edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci->edac_ctl_cap);
119 edac_dbg(3, "\tmci->edac_cap = %lx\n", mci->edac_cap);
120 edac_dbg(4, "\tmci->edac_check = %p\n", mci->edac_check);
121 edac_dbg(3, "\tmci->nr_csrows = %d, csrows = %p\n",
122 mci->nr_csrows, mci->csrows);
123 edac_dbg(3, "\tmci->nr_dimms = %d, dimms = %p\n",
124 mci->tot_dimms, mci->dimms);
125 edac_dbg(3, "\tdev = %p\n", mci->pdev);
126 edac_dbg(3, "\tmod_name:ctl_name = %s:%s\n",
127 mci->mod_name, mci->ctl_name);
128 edac_dbg(3, "\tpvt_info = %p\n\n", mci->pvt_info);
129 }
130
131 #endif /* CONFIG_EDAC_DEBUG */
132
133 const char * const edac_mem_types[] = {
134 [MEM_EMPTY] = "Empty csrow",
135 [MEM_RESERVED] = "Reserved csrow type",
136 [MEM_UNKNOWN] = "Unknown csrow type",
137 [MEM_FPM] = "Fast page mode RAM",
138 [MEM_EDO] = "Extended data out RAM",
139 [MEM_BEDO] = "Burst Extended data out RAM",
140 [MEM_SDR] = "Single data rate SDRAM",
141 [MEM_RDR] = "Registered single data rate SDRAM",
142 [MEM_DDR] = "Double data rate SDRAM",
143 [MEM_RDDR] = "Registered Double data rate SDRAM",
144 [MEM_RMBS] = "Rambus DRAM",
145 [MEM_DDR2] = "Unbuffered DDR2 RAM",
146 [MEM_FB_DDR2] = "Fully buffered DDR2",
147 [MEM_RDDR2] = "Registered DDR2 RAM",
148 [MEM_XDR] = "Rambus XDR",
149 [MEM_DDR3] = "Unbuffered DDR3 RAM",
150 [MEM_RDDR3] = "Registered DDR3 RAM",
151 [MEM_LRDDR3] = "Load-Reduced DDR3 RAM",
152 [MEM_DDR4] = "Unbuffered DDR4 RAM",
153 [MEM_RDDR4] = "Registered DDR4 RAM",
154 };
155 EXPORT_SYMBOL_GPL(edac_mem_types);
156
157 /**
158 * edac_align_ptr - Prepares the pointer offsets for a single-shot allocation
159 * @p: pointer to a pointer with the memory offset to be used. At
160 * return, this will be incremented to point to the next offset
161 * @size: Size of the data structure to be reserved
162 * @n_elems: Number of elements that should be reserved
163 *
164 * If 'size' is a constant, the compiler will optimize this whole function
165 * down to either a no-op or the addition of a constant to the value of '*p'.
166 *
167 * The 'p' pointer is absolutely needed to keep the proper advancing
168 * further in memory to the proper offsets when allocating the struct along
169 * with its embedded structs, as edac_device_alloc_ctl_info() does it
170 * above, for example.
171 *
172 * At return, the pointer 'p' will be incremented to be used on a next call
173 * to this function.
174 */
edac_align_ptr(void ** p,unsigned size,int n_elems)175 void *edac_align_ptr(void **p, unsigned size, int n_elems)
176 {
177 unsigned align, r;
178 void *ptr = *p;
179
180 *p += size * n_elems;
181
182 /*
183 * 'p' can possibly be an unaligned item X such that sizeof(X) is
184 * 'size'. Adjust 'p' so that its alignment is at least as
185 * stringent as what the compiler would provide for X and return
186 * the aligned result.
187 * Here we assume that the alignment of a "long long" is the most
188 * stringent alignment that the compiler will ever provide by default.
189 * As far as I know, this is a reasonable assumption.
190 */
191 if (size > sizeof(long))
192 align = sizeof(long long);
193 else if (size > sizeof(int))
194 align = sizeof(long);
195 else if (size > sizeof(short))
196 align = sizeof(int);
197 else if (size > sizeof(char))
198 align = sizeof(short);
199 else
200 return (char *)ptr;
201
202 r = (unsigned long)p % align;
203
204 if (r == 0)
205 return (char *)ptr;
206
207 *p += align - r;
208
209 return (void *)(((unsigned long)ptr) + align - r);
210 }
211
_edac_mc_free(struct mem_ctl_info * mci)212 static void _edac_mc_free(struct mem_ctl_info *mci)
213 {
214 int i, chn, row;
215 struct csrow_info *csr;
216 const unsigned int tot_dimms = mci->tot_dimms;
217 const unsigned int tot_channels = mci->num_cschannel;
218 const unsigned int tot_csrows = mci->nr_csrows;
219
220 if (mci->dimms) {
221 for (i = 0; i < tot_dimms; i++)
222 kfree(mci->dimms[i]);
223 kfree(mci->dimms);
224 }
225 if (mci->csrows) {
226 for (row = 0; row < tot_csrows; row++) {
227 csr = mci->csrows[row];
228 if (csr) {
229 if (csr->channels) {
230 for (chn = 0; chn < tot_channels; chn++)
231 kfree(csr->channels[chn]);
232 kfree(csr->channels);
233 }
234 kfree(csr);
235 }
236 }
237 kfree(mci->csrows);
238 }
239 kfree(mci);
240 }
241
242 /**
243 * edac_mc_alloc: Allocate and partially fill a struct mem_ctl_info structure
244 * @mc_num: Memory controller number
245 * @n_layers: Number of MC hierarchy layers
246 * layers: Describes each layer as seen by the Memory Controller
247 * @size_pvt: size of private storage needed
248 *
249 *
250 * Everything is kmalloc'ed as one big chunk - more efficient.
251 * Only can be used if all structures have the same lifetime - otherwise
252 * you have to allocate and initialize your own structures.
253 *
254 * Use edac_mc_free() to free mc structures allocated by this function.
255 *
256 * NOTE: drivers handle multi-rank memories in different ways: in some
257 * drivers, one multi-rank memory stick is mapped as one entry, while, in
258 * others, a single multi-rank memory stick would be mapped into several
259 * entries. Currently, this function will allocate multiple struct dimm_info
260 * on such scenarios, as grouping the multiple ranks require drivers change.
261 *
262 * Returns:
263 * On failure: NULL
264 * On success: struct mem_ctl_info pointer
265 */
edac_mc_alloc(unsigned mc_num,unsigned n_layers,struct edac_mc_layer * layers,unsigned sz_pvt)266 struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
267 unsigned n_layers,
268 struct edac_mc_layer *layers,
269 unsigned sz_pvt)
270 {
271 struct mem_ctl_info *mci;
272 struct edac_mc_layer *layer;
273 struct csrow_info *csr;
274 struct rank_info *chan;
275 struct dimm_info *dimm;
276 u32 *ce_per_layer[EDAC_MAX_LAYERS], *ue_per_layer[EDAC_MAX_LAYERS];
277 unsigned pos[EDAC_MAX_LAYERS];
278 unsigned size, tot_dimms = 1, count = 1;
279 unsigned tot_csrows = 1, tot_channels = 1, tot_errcount = 0;
280 void *pvt, *p, *ptr = NULL;
281 int i, j, row, chn, n, len, off;
282 bool per_rank = false;
283
284 BUG_ON(n_layers > EDAC_MAX_LAYERS || n_layers == 0);
285 /*
286 * Calculate the total amount of dimms and csrows/cschannels while
287 * in the old API emulation mode
288 */
289 for (i = 0; i < n_layers; i++) {
290 tot_dimms *= layers[i].size;
291 if (layers[i].is_virt_csrow)
292 tot_csrows *= layers[i].size;
293 else
294 tot_channels *= layers[i].size;
295
296 if (layers[i].type == EDAC_MC_LAYER_CHIP_SELECT)
297 per_rank = true;
298 }
299
300 /* Figure out the offsets of the various items from the start of an mc
301 * structure. We want the alignment of each item to be at least as
302 * stringent as what the compiler would provide if we could simply
303 * hardcode everything into a single struct.
304 */
305 mci = edac_align_ptr(&ptr, sizeof(*mci), 1);
306 layer = edac_align_ptr(&ptr, sizeof(*layer), n_layers);
307 for (i = 0; i < n_layers; i++) {
308 count *= layers[i].size;
309 edac_dbg(4, "errcount layer %d size %d\n", i, count);
310 ce_per_layer[i] = edac_align_ptr(&ptr, sizeof(u32), count);
311 ue_per_layer[i] = edac_align_ptr(&ptr, sizeof(u32), count);
312 tot_errcount += 2 * count;
313 }
314
315 edac_dbg(4, "allocating %d error counters\n", tot_errcount);
316 pvt = edac_align_ptr(&ptr, sz_pvt, 1);
317 size = ((unsigned long)pvt) + sz_pvt;
318
319 edac_dbg(1, "allocating %u bytes for mci data (%d %s, %d csrows/channels)\n",
320 size,
321 tot_dimms,
322 per_rank ? "ranks" : "dimms",
323 tot_csrows * tot_channels);
324
325 mci = kzalloc(size, GFP_KERNEL);
326 if (mci == NULL)
327 return NULL;
328
329 /* Adjust pointers so they point within the memory we just allocated
330 * rather than an imaginary chunk of memory located at address 0.
331 */
332 layer = (struct edac_mc_layer *)(((char *)mci) + ((unsigned long)layer));
333 for (i = 0; i < n_layers; i++) {
334 mci->ce_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ce_per_layer[i]));
335 mci->ue_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ue_per_layer[i]));
336 }
337 pvt = sz_pvt ? (((char *)mci) + ((unsigned long)pvt)) : NULL;
338
339 /* setup index and various internal pointers */
340 mci->mc_idx = mc_num;
341 mci->tot_dimms = tot_dimms;
342 mci->pvt_info = pvt;
343 mci->n_layers = n_layers;
344 mci->layers = layer;
345 memcpy(mci->layers, layers, sizeof(*layer) * n_layers);
346 mci->nr_csrows = tot_csrows;
347 mci->num_cschannel = tot_channels;
348 mci->csbased = per_rank;
349
350 /*
351 * Alocate and fill the csrow/channels structs
352 */
353 mci->csrows = kcalloc(tot_csrows, sizeof(*mci->csrows), GFP_KERNEL);
354 if (!mci->csrows)
355 goto error;
356 for (row = 0; row < tot_csrows; row++) {
357 csr = kzalloc(sizeof(**mci->csrows), GFP_KERNEL);
358 if (!csr)
359 goto error;
360 mci->csrows[row] = csr;
361 csr->csrow_idx = row;
362 csr->mci = mci;
363 csr->nr_channels = tot_channels;
364 csr->channels = kcalloc(tot_channels, sizeof(*csr->channels),
365 GFP_KERNEL);
366 if (!csr->channels)
367 goto error;
368
369 for (chn = 0; chn < tot_channels; chn++) {
370 chan = kzalloc(sizeof(**csr->channels), GFP_KERNEL);
371 if (!chan)
372 goto error;
373 csr->channels[chn] = chan;
374 chan->chan_idx = chn;
375 chan->csrow = csr;
376 }
377 }
378
379 /*
380 * Allocate and fill the dimm structs
381 */
382 mci->dimms = kcalloc(tot_dimms, sizeof(*mci->dimms), GFP_KERNEL);
383 if (!mci->dimms)
384 goto error;
385
386 memset(&pos, 0, sizeof(pos));
387 row = 0;
388 chn = 0;
389 for (i = 0; i < tot_dimms; i++) {
390 chan = mci->csrows[row]->channels[chn];
391 off = EDAC_DIMM_OFF(layer, n_layers, pos[0], pos[1], pos[2]);
392 if (off < 0 || off >= tot_dimms) {
393 edac_mc_printk(mci, KERN_ERR, "EDAC core bug: EDAC_DIMM_OFF is trying to do an illegal data access\n");
394 goto error;
395 }
396
397 dimm = kzalloc(sizeof(**mci->dimms), GFP_KERNEL);
398 if (!dimm)
399 goto error;
400 mci->dimms[off] = dimm;
401 dimm->mci = mci;
402
403 /*
404 * Copy DIMM location and initialize it.
405 */
406 len = sizeof(dimm->label);
407 p = dimm->label;
408 n = snprintf(p, len, "mc#%u", mc_num);
409 p += n;
410 len -= n;
411 for (j = 0; j < n_layers; j++) {
412 n = snprintf(p, len, "%s#%u",
413 edac_layer_name[layers[j].type],
414 pos[j]);
415 p += n;
416 len -= n;
417 dimm->location[j] = pos[j];
418
419 if (len <= 0)
420 break;
421 }
422
423 /* Link it to the csrows old API data */
424 chan->dimm = dimm;
425 dimm->csrow = row;
426 dimm->cschannel = chn;
427
428 /* Increment csrow location */
429 if (layers[0].is_virt_csrow) {
430 chn++;
431 if (chn == tot_channels) {
432 chn = 0;
433 row++;
434 }
435 } else {
436 row++;
437 if (row == tot_csrows) {
438 row = 0;
439 chn++;
440 }
441 }
442
443 /* Increment dimm location */
444 for (j = n_layers - 1; j >= 0; j--) {
445 pos[j]++;
446 if (pos[j] < layers[j].size)
447 break;
448 pos[j] = 0;
449 }
450 }
451
452 mci->op_state = OP_ALLOC;
453
454 return mci;
455
456 error:
457 _edac_mc_free(mci);
458
459 return NULL;
460 }
461 EXPORT_SYMBOL_GPL(edac_mc_alloc);
462
463 /**
464 * edac_mc_free
465 * 'Free' a previously allocated 'mci' structure
466 * @mci: pointer to a struct mem_ctl_info structure
467 */
edac_mc_free(struct mem_ctl_info * mci)468 void edac_mc_free(struct mem_ctl_info *mci)
469 {
470 edac_dbg(1, "\n");
471
472 /* If we're not yet registered with sysfs free only what was allocated
473 * in edac_mc_alloc().
474 */
475 if (!device_is_registered(&mci->dev)) {
476 _edac_mc_free(mci);
477 return;
478 }
479
480 /* the mci instance is freed here, when the sysfs object is dropped */
481 edac_unregister_sysfs(mci);
482 }
483 EXPORT_SYMBOL_GPL(edac_mc_free);
484
485
486 /**
487 * find_mci_by_dev
488 *
489 * scan list of controllers looking for the one that manages
490 * the 'dev' device
491 * @dev: pointer to a struct device related with the MCI
492 */
find_mci_by_dev(struct device * dev)493 struct mem_ctl_info *find_mci_by_dev(struct device *dev)
494 {
495 struct mem_ctl_info *mci;
496 struct list_head *item;
497
498 edac_dbg(3, "\n");
499
500 list_for_each(item, &mc_devices) {
501 mci = list_entry(item, struct mem_ctl_info, link);
502
503 if (mci->pdev == dev)
504 return mci;
505 }
506
507 return NULL;
508 }
509 EXPORT_SYMBOL_GPL(find_mci_by_dev);
510
511 /*
512 * handler for EDAC to check if NMI type handler has asserted interrupt
513 */
edac_mc_assert_error_check_and_clear(void)514 static int edac_mc_assert_error_check_and_clear(void)
515 {
516 int old_state;
517
518 if (edac_op_state == EDAC_OPSTATE_POLL)
519 return 1;
520
521 old_state = edac_err_assert;
522 edac_err_assert = 0;
523
524 return old_state;
525 }
526
527 /*
528 * edac_mc_workq_function
529 * performs the operation scheduled by a workq request
530 */
edac_mc_workq_function(struct work_struct * work_req)531 static void edac_mc_workq_function(struct work_struct *work_req)
532 {
533 struct delayed_work *d_work = to_delayed_work(work_req);
534 struct mem_ctl_info *mci = to_edac_mem_ctl_work(d_work);
535
536 mutex_lock(&mem_ctls_mutex);
537
538 if (mci->op_state != OP_RUNNING_POLL) {
539 mutex_unlock(&mem_ctls_mutex);
540 return;
541 }
542
543 if (edac_mc_assert_error_check_and_clear())
544 mci->edac_check(mci);
545
546 mutex_unlock(&mem_ctls_mutex);
547
548 /* Queue ourselves again. */
549 edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
550 }
551
552 /*
553 * edac_mc_reset_delay_period(unsigned long value)
554 *
555 * user space has updated our poll period value, need to
556 * reset our workq delays
557 */
edac_mc_reset_delay_period(unsigned long value)558 void edac_mc_reset_delay_period(unsigned long value)
559 {
560 struct mem_ctl_info *mci;
561 struct list_head *item;
562
563 mutex_lock(&mem_ctls_mutex);
564
565 list_for_each(item, &mc_devices) {
566 mci = list_entry(item, struct mem_ctl_info, link);
567
568 if (mci->op_state == OP_RUNNING_POLL)
569 edac_mod_work(&mci->work, value);
570 }
571 mutex_unlock(&mem_ctls_mutex);
572 }
573
574
575
576 /* Return 0 on success, 1 on failure.
577 * Before calling this function, caller must
578 * assign a unique value to mci->mc_idx.
579 *
580 * locking model:
581 *
582 * called with the mem_ctls_mutex lock held
583 */
add_mc_to_global_list(struct mem_ctl_info * mci)584 static int add_mc_to_global_list(struct mem_ctl_info *mci)
585 {
586 struct list_head *item, *insert_before;
587 struct mem_ctl_info *p;
588
589 insert_before = &mc_devices;
590
591 p = find_mci_by_dev(mci->pdev);
592 if (unlikely(p != NULL))
593 goto fail0;
594
595 list_for_each(item, &mc_devices) {
596 p = list_entry(item, struct mem_ctl_info, link);
597
598 if (p->mc_idx >= mci->mc_idx) {
599 if (unlikely(p->mc_idx == mci->mc_idx))
600 goto fail1;
601
602 insert_before = item;
603 break;
604 }
605 }
606
607 list_add_tail_rcu(&mci->link, insert_before);
608 atomic_inc(&edac_handlers);
609 return 0;
610
611 fail0:
612 edac_printk(KERN_WARNING, EDAC_MC,
613 "%s (%s) %s %s already assigned %d\n", dev_name(p->pdev),
614 edac_dev_name(mci), p->mod_name, p->ctl_name, p->mc_idx);
615 return 1;
616
617 fail1:
618 edac_printk(KERN_WARNING, EDAC_MC,
619 "bug in low-level driver: attempt to assign\n"
620 " duplicate mc_idx %d in %s()\n", p->mc_idx, __func__);
621 return 1;
622 }
623
del_mc_from_global_list(struct mem_ctl_info * mci)624 static int del_mc_from_global_list(struct mem_ctl_info *mci)
625 {
626 int handlers = atomic_dec_return(&edac_handlers);
627 list_del_rcu(&mci->link);
628
629 /* these are for safe removal of devices from global list while
630 * NMI handlers may be traversing list
631 */
632 synchronize_rcu();
633 INIT_LIST_HEAD(&mci->link);
634
635 return handlers;
636 }
637
638 /**
639 * edac_mc_find: Search for a mem_ctl_info structure whose index is 'idx'.
640 *
641 * If found, return a pointer to the structure.
642 * Else return NULL.
643 *
644 * Caller must hold mem_ctls_mutex.
645 */
edac_mc_find(int idx)646 struct mem_ctl_info *edac_mc_find(int idx)
647 {
648 struct list_head *item;
649 struct mem_ctl_info *mci;
650
651 list_for_each(item, &mc_devices) {
652 mci = list_entry(item, struct mem_ctl_info, link);
653
654 if (mci->mc_idx >= idx) {
655 if (mci->mc_idx == idx)
656 return mci;
657
658 break;
659 }
660 }
661
662 return NULL;
663 }
664 EXPORT_SYMBOL(edac_mc_find);
665
666 /**
667 * edac_mc_add_mc_with_groups: Insert the 'mci' structure into the mci
668 * global list and create sysfs entries associated with mci structure
669 * @mci: pointer to the mci structure to be added to the list
670 * @groups: optional attribute groups for the driver-specific sysfs entries
671 *
672 * Return:
673 * 0 Success
674 * !0 Failure
675 */
676
677 /* FIXME - should a warning be printed if no error detection? correction? */
edac_mc_add_mc_with_groups(struct mem_ctl_info * mci,const struct attribute_group ** groups)678 int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
679 const struct attribute_group **groups)
680 {
681 int ret = -EINVAL;
682 edac_dbg(0, "\n");
683
684 if (mci->mc_idx >= EDAC_MAX_MCS) {
685 pr_warn_once("Too many memory controllers: %d\n", mci->mc_idx);
686 return -ENODEV;
687 }
688
689 #ifdef CONFIG_EDAC_DEBUG
690 if (edac_debug_level >= 3)
691 edac_mc_dump_mci(mci);
692
693 if (edac_debug_level >= 4) {
694 int i;
695
696 for (i = 0; i < mci->nr_csrows; i++) {
697 struct csrow_info *csrow = mci->csrows[i];
698 u32 nr_pages = 0;
699 int j;
700
701 for (j = 0; j < csrow->nr_channels; j++)
702 nr_pages += csrow->channels[j]->dimm->nr_pages;
703 if (!nr_pages)
704 continue;
705 edac_mc_dump_csrow(csrow);
706 for (j = 0; j < csrow->nr_channels; j++)
707 if (csrow->channels[j]->dimm->nr_pages)
708 edac_mc_dump_channel(csrow->channels[j]);
709 }
710 for (i = 0; i < mci->tot_dimms; i++)
711 if (mci->dimms[i]->nr_pages)
712 edac_mc_dump_dimm(mci->dimms[i], i);
713 }
714 #endif
715 mutex_lock(&mem_ctls_mutex);
716
717 if (edac_mc_owner && edac_mc_owner != mci->mod_name) {
718 ret = -EPERM;
719 goto fail0;
720 }
721
722 if (add_mc_to_global_list(mci))
723 goto fail0;
724
725 /* set load time so that error rate can be tracked */
726 mci->start_time = jiffies;
727
728 mci->bus = &mc_bus[mci->mc_idx];
729
730 if (edac_create_sysfs_mci_device(mci, groups)) {
731 edac_mc_printk(mci, KERN_WARNING,
732 "failed to create sysfs device\n");
733 goto fail1;
734 }
735
736 if (mci->edac_check) {
737 mci->op_state = OP_RUNNING_POLL;
738
739 INIT_DELAYED_WORK(&mci->work, edac_mc_workq_function);
740 edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
741
742 } else {
743 mci->op_state = OP_RUNNING_INTERRUPT;
744 }
745
746 /* Report action taken */
747 edac_mc_printk(mci, KERN_INFO,
748 "Giving out device to module %s controller %s: DEV %s (%s)\n",
749 mci->mod_name, mci->ctl_name, mci->dev_name,
750 edac_op_state_to_string(mci->op_state));
751
752 edac_mc_owner = mci->mod_name;
753
754 mutex_unlock(&mem_ctls_mutex);
755 return 0;
756
757 fail1:
758 del_mc_from_global_list(mci);
759
760 fail0:
761 mutex_unlock(&mem_ctls_mutex);
762 return ret;
763 }
764 EXPORT_SYMBOL_GPL(edac_mc_add_mc_with_groups);
765
766 /**
767 * edac_mc_del_mc: Remove sysfs entries for specified mci structure and
768 * remove mci structure from global list
769 * @pdev: Pointer to 'struct device' representing mci structure to remove.
770 *
771 * Return pointer to removed mci structure, or NULL if device not found.
772 */
edac_mc_del_mc(struct device * dev)773 struct mem_ctl_info *edac_mc_del_mc(struct device *dev)
774 {
775 struct mem_ctl_info *mci;
776
777 edac_dbg(0, "\n");
778
779 mutex_lock(&mem_ctls_mutex);
780
781 /* find the requested mci struct in the global list */
782 mci = find_mci_by_dev(dev);
783 if (mci == NULL) {
784 mutex_unlock(&mem_ctls_mutex);
785 return NULL;
786 }
787
788 /* mark MCI offline: */
789 mci->op_state = OP_OFFLINE;
790
791 if (!del_mc_from_global_list(mci))
792 edac_mc_owner = NULL;
793
794 mutex_unlock(&mem_ctls_mutex);
795
796 if (mci->edac_check)
797 edac_stop_work(&mci->work);
798
799 /* remove from sysfs */
800 edac_remove_sysfs_mci_device(mci);
801
802 edac_printk(KERN_INFO, EDAC_MC,
803 "Removed device %d for %s %s: DEV %s\n", mci->mc_idx,
804 mci->mod_name, mci->ctl_name, edac_dev_name(mci));
805
806 return mci;
807 }
808 EXPORT_SYMBOL_GPL(edac_mc_del_mc);
809
edac_mc_scrub_block(unsigned long page,unsigned long offset,u32 size)810 static void edac_mc_scrub_block(unsigned long page, unsigned long offset,
811 u32 size)
812 {
813 struct page *pg;
814 void *virt_addr;
815 unsigned long flags = 0;
816
817 edac_dbg(3, "\n");
818
819 /* ECC error page was not in our memory. Ignore it. */
820 if (!pfn_valid(page))
821 return;
822
823 /* Find the actual page structure then map it and fix */
824 pg = pfn_to_page(page);
825
826 if (PageHighMem(pg))
827 local_irq_save(flags);
828
829 virt_addr = kmap_atomic(pg);
830
831 /* Perform architecture specific atomic scrub operation */
832 edac_atomic_scrub(virt_addr + offset, size);
833
834 /* Unmap and complete */
835 kunmap_atomic(virt_addr);
836
837 if (PageHighMem(pg))
838 local_irq_restore(flags);
839 }
840
841 /* FIXME - should return -1 */
edac_mc_find_csrow_by_page(struct mem_ctl_info * mci,unsigned long page)842 int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci, unsigned long page)
843 {
844 struct csrow_info **csrows = mci->csrows;
845 int row, i, j, n;
846
847 edac_dbg(1, "MC%d: 0x%lx\n", mci->mc_idx, page);
848 row = -1;
849
850 for (i = 0; i < mci->nr_csrows; i++) {
851 struct csrow_info *csrow = csrows[i];
852 n = 0;
853 for (j = 0; j < csrow->nr_channels; j++) {
854 struct dimm_info *dimm = csrow->channels[j]->dimm;
855 n += dimm->nr_pages;
856 }
857 if (n == 0)
858 continue;
859
860 edac_dbg(3, "MC%d: first(0x%lx) page(0x%lx) last(0x%lx) mask(0x%lx)\n",
861 mci->mc_idx,
862 csrow->first_page, page, csrow->last_page,
863 csrow->page_mask);
864
865 if ((page >= csrow->first_page) &&
866 (page <= csrow->last_page) &&
867 ((page & csrow->page_mask) ==
868 (csrow->first_page & csrow->page_mask))) {
869 row = i;
870 break;
871 }
872 }
873
874 if (row == -1)
875 edac_mc_printk(mci, KERN_ERR,
876 "could not look up page error address %lx\n",
877 (unsigned long)page);
878
879 return row;
880 }
881 EXPORT_SYMBOL_GPL(edac_mc_find_csrow_by_page);
882
883 const char *edac_layer_name[] = {
884 [EDAC_MC_LAYER_BRANCH] = "branch",
885 [EDAC_MC_LAYER_CHANNEL] = "channel",
886 [EDAC_MC_LAYER_SLOT] = "slot",
887 [EDAC_MC_LAYER_CHIP_SELECT] = "csrow",
888 [EDAC_MC_LAYER_ALL_MEM] = "memory",
889 };
890 EXPORT_SYMBOL_GPL(edac_layer_name);
891
edac_inc_ce_error(struct mem_ctl_info * mci,bool enable_per_layer_report,const int pos[EDAC_MAX_LAYERS],const u16 count)892 static void edac_inc_ce_error(struct mem_ctl_info *mci,
893 bool enable_per_layer_report,
894 const int pos[EDAC_MAX_LAYERS],
895 const u16 count)
896 {
897 int i, index = 0;
898
899 mci->ce_mc += count;
900
901 if (!enable_per_layer_report) {
902 mci->ce_noinfo_count += count;
903 return;
904 }
905
906 for (i = 0; i < mci->n_layers; i++) {
907 if (pos[i] < 0)
908 break;
909 index += pos[i];
910 mci->ce_per_layer[i][index] += count;
911
912 if (i < mci->n_layers - 1)
913 index *= mci->layers[i + 1].size;
914 }
915 }
916
edac_inc_ue_error(struct mem_ctl_info * mci,bool enable_per_layer_report,const int pos[EDAC_MAX_LAYERS],const u16 count)917 static void edac_inc_ue_error(struct mem_ctl_info *mci,
918 bool enable_per_layer_report,
919 const int pos[EDAC_MAX_LAYERS],
920 const u16 count)
921 {
922 int i, index = 0;
923
924 mci->ue_mc += count;
925
926 if (!enable_per_layer_report) {
927 mci->ue_noinfo_count += count;
928 return;
929 }
930
931 for (i = 0; i < mci->n_layers; i++) {
932 if (pos[i] < 0)
933 break;
934 index += pos[i];
935 mci->ue_per_layer[i][index] += count;
936
937 if (i < mci->n_layers - 1)
938 index *= mci->layers[i + 1].size;
939 }
940 }
941
edac_ce_error(struct mem_ctl_info * mci,const u16 error_count,const int pos[EDAC_MAX_LAYERS],const char * msg,const char * location,const char * label,const char * detail,const char * other_detail,const bool enable_per_layer_report,const unsigned long page_frame_number,const unsigned long offset_in_page,long grain)942 static void edac_ce_error(struct mem_ctl_info *mci,
943 const u16 error_count,
944 const int pos[EDAC_MAX_LAYERS],
945 const char *msg,
946 const char *location,
947 const char *label,
948 const char *detail,
949 const char *other_detail,
950 const bool enable_per_layer_report,
951 const unsigned long page_frame_number,
952 const unsigned long offset_in_page,
953 long grain)
954 {
955 unsigned long remapped_page;
956 char *msg_aux = "";
957
958 if (*msg)
959 msg_aux = " ";
960
961 if (edac_mc_get_log_ce()) {
962 if (other_detail && *other_detail)
963 edac_mc_printk(mci, KERN_WARNING,
964 "%d CE %s%son %s (%s %s - %s)\n",
965 error_count, msg, msg_aux, label,
966 location, detail, other_detail);
967 else
968 edac_mc_printk(mci, KERN_WARNING,
969 "%d CE %s%son %s (%s %s)\n",
970 error_count, msg, msg_aux, label,
971 location, detail);
972 }
973 edac_inc_ce_error(mci, enable_per_layer_report, pos, error_count);
974
975 if (mci->scrub_mode == SCRUB_SW_SRC) {
976 /*
977 * Some memory controllers (called MCs below) can remap
978 * memory so that it is still available at a different
979 * address when PCI devices map into memory.
980 * MC's that can't do this, lose the memory where PCI
981 * devices are mapped. This mapping is MC-dependent
982 * and so we call back into the MC driver for it to
983 * map the MC page to a physical (CPU) page which can
984 * then be mapped to a virtual page - which can then
985 * be scrubbed.
986 */
987 remapped_page = mci->ctl_page_to_phys ?
988 mci->ctl_page_to_phys(mci, page_frame_number) :
989 page_frame_number;
990
991 edac_mc_scrub_block(remapped_page,
992 offset_in_page, grain);
993 }
994 }
995
edac_ue_error(struct mem_ctl_info * mci,const u16 error_count,const int pos[EDAC_MAX_LAYERS],const char * msg,const char * location,const char * label,const char * detail,const char * other_detail,const bool enable_per_layer_report)996 static void edac_ue_error(struct mem_ctl_info *mci,
997 const u16 error_count,
998 const int pos[EDAC_MAX_LAYERS],
999 const char *msg,
1000 const char *location,
1001 const char *label,
1002 const char *detail,
1003 const char *other_detail,
1004 const bool enable_per_layer_report)
1005 {
1006 char *msg_aux = "";
1007
1008 if (*msg)
1009 msg_aux = " ";
1010
1011 if (edac_mc_get_log_ue()) {
1012 if (other_detail && *other_detail)
1013 edac_mc_printk(mci, KERN_WARNING,
1014 "%d UE %s%son %s (%s %s - %s)\n",
1015 error_count, msg, msg_aux, label,
1016 location, detail, other_detail);
1017 else
1018 edac_mc_printk(mci, KERN_WARNING,
1019 "%d UE %s%son %s (%s %s)\n",
1020 error_count, msg, msg_aux, label,
1021 location, detail);
1022 }
1023
1024 if (edac_mc_get_panic_on_ue()) {
1025 if (other_detail && *other_detail)
1026 panic("UE %s%son %s (%s%s - %s)\n",
1027 msg, msg_aux, label, location, detail, other_detail);
1028 else
1029 panic("UE %s%son %s (%s%s)\n",
1030 msg, msg_aux, label, location, detail);
1031 }
1032
1033 edac_inc_ue_error(mci, enable_per_layer_report, pos, error_count);
1034 }
1035
1036 /**
1037 * edac_raw_mc_handle_error - reports a memory event to userspace without doing
1038 * anything to discover the error location
1039 *
1040 * @type: severity of the error (CE/UE/Fatal)
1041 * @mci: a struct mem_ctl_info pointer
1042 * @e: error description
1043 *
1044 * This raw function is used internally by edac_mc_handle_error(). It should
1045 * only be called directly when the hardware error come directly from BIOS,
1046 * like in the case of APEI GHES driver.
1047 */
edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,struct mem_ctl_info * mci,struct edac_raw_error_desc * e)1048 void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,
1049 struct mem_ctl_info *mci,
1050 struct edac_raw_error_desc *e)
1051 {
1052 char detail[80];
1053 int pos[EDAC_MAX_LAYERS] = { e->top_layer, e->mid_layer, e->low_layer };
1054
1055 /* Memory type dependent details about the error */
1056 if (type == HW_EVENT_ERR_CORRECTED) {
1057 snprintf(detail, sizeof(detail),
1058 "page:0x%lx offset:0x%lx grain:%ld syndrome:0x%lx",
1059 e->page_frame_number, e->offset_in_page,
1060 e->grain, e->syndrome);
1061 edac_ce_error(mci, e->error_count, pos, e->msg, e->location, e->label,
1062 detail, e->other_detail, e->enable_per_layer_report,
1063 e->page_frame_number, e->offset_in_page, e->grain);
1064 } else {
1065 snprintf(detail, sizeof(detail),
1066 "page:0x%lx offset:0x%lx grain:%ld",
1067 e->page_frame_number, e->offset_in_page, e->grain);
1068
1069 edac_ue_error(mci, e->error_count, pos, e->msg, e->location, e->label,
1070 detail, e->other_detail, e->enable_per_layer_report);
1071 }
1072
1073
1074 }
1075 EXPORT_SYMBOL_GPL(edac_raw_mc_handle_error);
1076
1077 /**
1078 * edac_mc_handle_error - reports a memory event to userspace
1079 *
1080 * @type: severity of the error (CE/UE/Fatal)
1081 * @mci: a struct mem_ctl_info pointer
1082 * @error_count: Number of errors of the same type
1083 * @page_frame_number: mem page where the error occurred
1084 * @offset_in_page: offset of the error inside the page
1085 * @syndrome: ECC syndrome
1086 * @top_layer: Memory layer[0] position
1087 * @mid_layer: Memory layer[1] position
1088 * @low_layer: Memory layer[2] position
1089 * @msg: Message meaningful to the end users that
1090 * explains the event
1091 * @other_detail: Technical details about the event that
1092 * may help hardware manufacturers and
1093 * EDAC developers to analyse the event
1094 */
edac_mc_handle_error(const enum hw_event_mc_err_type type,struct mem_ctl_info * mci,const u16 error_count,const unsigned long page_frame_number,const unsigned long offset_in_page,const unsigned long syndrome,const int top_layer,const int mid_layer,const int low_layer,const char * msg,const char * other_detail)1095 void edac_mc_handle_error(const enum hw_event_mc_err_type type,
1096 struct mem_ctl_info *mci,
1097 const u16 error_count,
1098 const unsigned long page_frame_number,
1099 const unsigned long offset_in_page,
1100 const unsigned long syndrome,
1101 const int top_layer,
1102 const int mid_layer,
1103 const int low_layer,
1104 const char *msg,
1105 const char *other_detail)
1106 {
1107 char *p;
1108 int row = -1, chan = -1;
1109 int pos[EDAC_MAX_LAYERS] = { top_layer, mid_layer, low_layer };
1110 int i, n_labels = 0;
1111 u8 grain_bits;
1112 struct edac_raw_error_desc *e = &mci->error_desc;
1113
1114 edac_dbg(3, "MC%d\n", mci->mc_idx);
1115
1116 /* Fills the error report buffer */
1117 memset(e, 0, sizeof (*e));
1118 e->error_count = error_count;
1119 e->top_layer = top_layer;
1120 e->mid_layer = mid_layer;
1121 e->low_layer = low_layer;
1122 e->page_frame_number = page_frame_number;
1123 e->offset_in_page = offset_in_page;
1124 e->syndrome = syndrome;
1125 e->msg = msg;
1126 e->other_detail = other_detail;
1127
1128 /*
1129 * Check if the event report is consistent and if the memory
1130 * location is known. If it is known, enable_per_layer_report will be
1131 * true, the DIMM(s) label info will be filled and the per-layer
1132 * error counters will be incremented.
1133 */
1134 for (i = 0; i < mci->n_layers; i++) {
1135 if (pos[i] >= (int)mci->layers[i].size) {
1136
1137 edac_mc_printk(mci, KERN_ERR,
1138 "INTERNAL ERROR: %s value is out of range (%d >= %d)\n",
1139 edac_layer_name[mci->layers[i].type],
1140 pos[i], mci->layers[i].size);
1141 /*
1142 * Instead of just returning it, let's use what's
1143 * known about the error. The increment routines and
1144 * the DIMM filter logic will do the right thing by
1145 * pointing the likely damaged DIMMs.
1146 */
1147 pos[i] = -1;
1148 }
1149 if (pos[i] >= 0)
1150 e->enable_per_layer_report = true;
1151 }
1152
1153 /*
1154 * Get the dimm label/grain that applies to the match criteria.
1155 * As the error algorithm may not be able to point to just one memory
1156 * stick, the logic here will get all possible labels that could
1157 * pottentially be affected by the error.
1158 * On FB-DIMM memory controllers, for uncorrected errors, it is common
1159 * to have only the MC channel and the MC dimm (also called "branch")
1160 * but the channel is not known, as the memory is arranged in pairs,
1161 * where each memory belongs to a separate channel within the same
1162 * branch.
1163 */
1164 p = e->label;
1165 *p = '\0';
1166
1167 for (i = 0; i < mci->tot_dimms; i++) {
1168 struct dimm_info *dimm = mci->dimms[i];
1169
1170 if (top_layer >= 0 && top_layer != dimm->location[0])
1171 continue;
1172 if (mid_layer >= 0 && mid_layer != dimm->location[1])
1173 continue;
1174 if (low_layer >= 0 && low_layer != dimm->location[2])
1175 continue;
1176
1177 /* get the max grain, over the error match range */
1178 if (dimm->grain > e->grain)
1179 e->grain = dimm->grain;
1180
1181 /*
1182 * If the error is memory-controller wide, there's no need to
1183 * seek for the affected DIMMs because the whole
1184 * channel/memory controller/... may be affected.
1185 * Also, don't show errors for empty DIMM slots.
1186 */
1187 if (e->enable_per_layer_report && dimm->nr_pages) {
1188 if (n_labels >= EDAC_MAX_LABELS) {
1189 e->enable_per_layer_report = false;
1190 break;
1191 }
1192 n_labels++;
1193 if (p != e->label) {
1194 strcpy(p, OTHER_LABEL);
1195 p += strlen(OTHER_LABEL);
1196 }
1197 strcpy(p, dimm->label);
1198 p += strlen(p);
1199 *p = '\0';
1200
1201 /*
1202 * get csrow/channel of the DIMM, in order to allow
1203 * incrementing the compat API counters
1204 */
1205 edac_dbg(4, "%s csrows map: (%d,%d)\n",
1206 mci->csbased ? "rank" : "dimm",
1207 dimm->csrow, dimm->cschannel);
1208 if (row == -1)
1209 row = dimm->csrow;
1210 else if (row >= 0 && row != dimm->csrow)
1211 row = -2;
1212
1213 if (chan == -1)
1214 chan = dimm->cschannel;
1215 else if (chan >= 0 && chan != dimm->cschannel)
1216 chan = -2;
1217 }
1218 }
1219
1220 if (!e->enable_per_layer_report) {
1221 strcpy(e->label, "any memory");
1222 } else {
1223 edac_dbg(4, "csrow/channel to increment: (%d,%d)\n", row, chan);
1224 if (p == e->label)
1225 strcpy(e->label, "unknown memory");
1226 if (type == HW_EVENT_ERR_CORRECTED) {
1227 if (row >= 0) {
1228 mci->csrows[row]->ce_count += error_count;
1229 if (chan >= 0)
1230 mci->csrows[row]->channels[chan]->ce_count += error_count;
1231 }
1232 } else
1233 if (row >= 0)
1234 mci->csrows[row]->ue_count += error_count;
1235 }
1236
1237 /* Fill the RAM location data */
1238 p = e->location;
1239
1240 for (i = 0; i < mci->n_layers; i++) {
1241 if (pos[i] < 0)
1242 continue;
1243
1244 p += sprintf(p, "%s:%d ",
1245 edac_layer_name[mci->layers[i].type],
1246 pos[i]);
1247 }
1248 if (p > e->location)
1249 *(p - 1) = '\0';
1250
1251 /* Report the error via the trace interface */
1252 grain_bits = fls_long(e->grain) + 1;
1253 trace_mc_event(type, e->msg, e->label, e->error_count,
1254 mci->mc_idx, e->top_layer, e->mid_layer, e->low_layer,
1255 (e->page_frame_number << PAGE_SHIFT) | e->offset_in_page,
1256 grain_bits, e->syndrome, e->other_detail);
1257
1258 edac_raw_mc_handle_error(type, mci, e);
1259 }
1260 EXPORT_SYMBOL_GPL(edac_mc_handle_error);
1261