1 /*
2 * APEI Generic Hardware Error Source support
3 *
4 * Generic Hardware Error Source provides a way to report platform
5 * hardware errors (such as that from chipset). It works in so called
6 * "Firmware First" mode, that is, hardware errors are reported to
7 * firmware firstly, then reported to Linux by firmware. This way,
8 * some non-standard hardware error registers or non-standard hardware
9 * link can be checked by firmware to produce more hardware error
10 * information for Linux.
11 *
12 * For more information about Generic Hardware Error Source, please
13 * refer to ACPI Specification version 4.0, section 17.3.2.6
14 *
15 * Copyright 2010,2011 Intel Corp.
16 * Author: Huang Ying <ying.huang@intel.com>
17 *
18 * This program is free software; you can redistribute it and/or
19 * modify it under the terms of the GNU General Public License version
20 * 2 as published by the Free Software Foundation;
21 *
22 * This program is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
26 *
27 * You should have received a copy of the GNU General Public License
28 * along with this program; if not, write to the Free Software
29 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
30 */
31
32 #include <linux/kernel.h>
33 #include <linux/module.h>
34 #include <linux/init.h>
35 #include <linux/acpi.h>
36 #include <linux/io.h>
37 #include <linux/interrupt.h>
38 #include <linux/timer.h>
39 #include <linux/cper.h>
40 #include <linux/kdebug.h>
41 #include <linux/platform_device.h>
42 #include <linux/mutex.h>
43 #include <linux/ratelimit.h>
44 #include <linux/vmalloc.h>
45 #include <linux/irq_work.h>
46 #include <linux/llist.h>
47 #include <linux/genalloc.h>
48 #include <linux/pci.h>
49 #include <linux/aer.h>
50 #include <linux/nmi.h>
51
52 #include <acpi/ghes.h>
53 #include <acpi/apei.h>
54 #include <asm/tlbflush.h>
55
56 #include "apei-internal.h"
57
58 #define GHES_PFX "GHES: "
59
60 #define GHES_ESTATUS_MAX_SIZE 65536
61 #define GHES_ESOURCE_PREALLOC_MAX_SIZE 65536
62
63 #define GHES_ESTATUS_POOL_MIN_ALLOC_ORDER 3
64
65 /* This is just an estimation for memory pool allocation */
66 #define GHES_ESTATUS_CACHE_AVG_SIZE 512
67
68 #define GHES_ESTATUS_CACHES_SIZE 4
69
70 #define GHES_ESTATUS_IN_CACHE_MAX_NSEC 10000000000ULL
71 /* Prevent too many caches are allocated because of RCU */
72 #define GHES_ESTATUS_CACHE_ALLOCED_MAX (GHES_ESTATUS_CACHES_SIZE * 3 / 2)
73
74 #define GHES_ESTATUS_CACHE_LEN(estatus_len) \
75 (sizeof(struct ghes_estatus_cache) + (estatus_len))
76 #define GHES_ESTATUS_FROM_CACHE(estatus_cache) \
77 ((struct acpi_hest_generic_status *) \
78 ((struct ghes_estatus_cache *)(estatus_cache) + 1))
79
80 #define GHES_ESTATUS_NODE_LEN(estatus_len) \
81 (sizeof(struct ghes_estatus_node) + (estatus_len))
82 #define GHES_ESTATUS_FROM_NODE(estatus_node) \
83 ((struct acpi_hest_generic_status *) \
84 ((struct ghes_estatus_node *)(estatus_node) + 1))
85
86 bool ghes_disable;
87 module_param_named(disable, ghes_disable, bool, 0);
88
89 /*
90 * All error sources notified with SCI shares one notifier function,
91 * so they need to be linked and checked one by one. This is applied
92 * to NMI too.
93 *
94 * RCU is used for these lists, so ghes_list_mutex is only used for
95 * list changing, not for traversing.
96 */
97 static LIST_HEAD(ghes_sci);
98 static DEFINE_MUTEX(ghes_list_mutex);
99
100 /*
101 * Because the memory area used to transfer hardware error information
102 * from BIOS to Linux can be determined only in NMI, IRQ or timer
103 * handler, but general ioremap can not be used in atomic context, so
104 * a special version of atomic ioremap is implemented for that.
105 */
106
107 /*
108 * Two virtual pages are used, one for IRQ/PROCESS context, the other for
109 * NMI context (optionally).
110 */
111 #ifdef CONFIG_HAVE_ACPI_APEI_NMI
112 #define GHES_IOREMAP_PAGES 2
113 #else
114 #define GHES_IOREMAP_PAGES 1
115 #endif
116 #define GHES_IOREMAP_IRQ_PAGE(base) (base)
117 #define GHES_IOREMAP_NMI_PAGE(base) ((base) + PAGE_SIZE)
118
119 /* virtual memory area for atomic ioremap */
120 static struct vm_struct *ghes_ioremap_area;
121 /*
122 * These 2 spinlock is used to prevent atomic ioremap virtual memory
123 * area from being mapped simultaneously.
124 */
125 static DEFINE_RAW_SPINLOCK(ghes_ioremap_lock_nmi);
126 static DEFINE_SPINLOCK(ghes_ioremap_lock_irq);
127
128 static struct gen_pool *ghes_estatus_pool;
129 static unsigned long ghes_estatus_pool_size_request;
130
131 struct ghes_estatus_cache *ghes_estatus_caches[GHES_ESTATUS_CACHES_SIZE];
132 static atomic_t ghes_estatus_cache_alloced;
133
ghes_ioremap_init(void)134 static int ghes_ioremap_init(void)
135 {
136 ghes_ioremap_area = __get_vm_area(PAGE_SIZE * GHES_IOREMAP_PAGES,
137 VM_IOREMAP, VMALLOC_START, VMALLOC_END);
138 if (!ghes_ioremap_area) {
139 pr_err(GHES_PFX "Failed to allocate virtual memory area for atomic ioremap.\n");
140 return -ENOMEM;
141 }
142
143 return 0;
144 }
145
ghes_ioremap_exit(void)146 static void ghes_ioremap_exit(void)
147 {
148 free_vm_area(ghes_ioremap_area);
149 }
150
ghes_ioremap_pfn_nmi(u64 pfn)151 static void __iomem *ghes_ioremap_pfn_nmi(u64 pfn)
152 {
153 unsigned long vaddr;
154
155 vaddr = (unsigned long)GHES_IOREMAP_NMI_PAGE(ghes_ioremap_area->addr);
156 ioremap_page_range(vaddr, vaddr + PAGE_SIZE,
157 pfn << PAGE_SHIFT, PAGE_KERNEL);
158
159 return (void __iomem *)vaddr;
160 }
161
ghes_ioremap_pfn_irq(u64 pfn)162 static void __iomem *ghes_ioremap_pfn_irq(u64 pfn)
163 {
164 unsigned long vaddr;
165
166 vaddr = (unsigned long)GHES_IOREMAP_IRQ_PAGE(ghes_ioremap_area->addr);
167 ioremap_page_range(vaddr, vaddr + PAGE_SIZE,
168 pfn << PAGE_SHIFT, PAGE_KERNEL);
169
170 return (void __iomem *)vaddr;
171 }
172
ghes_iounmap_nmi(void __iomem * vaddr_ptr)173 static void ghes_iounmap_nmi(void __iomem *vaddr_ptr)
174 {
175 unsigned long vaddr = (unsigned long __force)vaddr_ptr;
176 void *base = ghes_ioremap_area->addr;
177
178 BUG_ON(vaddr != (unsigned long)GHES_IOREMAP_NMI_PAGE(base));
179 unmap_kernel_range_noflush(vaddr, PAGE_SIZE);
180 arch_apei_flush_tlb_one(vaddr);
181 }
182
ghes_iounmap_irq(void __iomem * vaddr_ptr)183 static void ghes_iounmap_irq(void __iomem *vaddr_ptr)
184 {
185 unsigned long vaddr = (unsigned long __force)vaddr_ptr;
186 void *base = ghes_ioremap_area->addr;
187
188 BUG_ON(vaddr != (unsigned long)GHES_IOREMAP_IRQ_PAGE(base));
189 unmap_kernel_range_noflush(vaddr, PAGE_SIZE);
190 arch_apei_flush_tlb_one(vaddr);
191 }
192
ghes_estatus_pool_init(void)193 static int ghes_estatus_pool_init(void)
194 {
195 ghes_estatus_pool = gen_pool_create(GHES_ESTATUS_POOL_MIN_ALLOC_ORDER, -1);
196 if (!ghes_estatus_pool)
197 return -ENOMEM;
198 return 0;
199 }
200
ghes_estatus_pool_free_chunk_page(struct gen_pool * pool,struct gen_pool_chunk * chunk,void * data)201 static void ghes_estatus_pool_free_chunk_page(struct gen_pool *pool,
202 struct gen_pool_chunk *chunk,
203 void *data)
204 {
205 free_page(chunk->start_addr);
206 }
207
ghes_estatus_pool_exit(void)208 static void ghes_estatus_pool_exit(void)
209 {
210 gen_pool_for_each_chunk(ghes_estatus_pool,
211 ghes_estatus_pool_free_chunk_page, NULL);
212 gen_pool_destroy(ghes_estatus_pool);
213 }
214
ghes_estatus_pool_expand(unsigned long len)215 static int ghes_estatus_pool_expand(unsigned long len)
216 {
217 unsigned long i, pages, size, addr;
218 int ret;
219
220 ghes_estatus_pool_size_request += PAGE_ALIGN(len);
221 size = gen_pool_size(ghes_estatus_pool);
222 if (size >= ghes_estatus_pool_size_request)
223 return 0;
224 pages = (ghes_estatus_pool_size_request - size) / PAGE_SIZE;
225 for (i = 0; i < pages; i++) {
226 addr = __get_free_page(GFP_KERNEL);
227 if (!addr)
228 return -ENOMEM;
229 ret = gen_pool_add(ghes_estatus_pool, addr, PAGE_SIZE, -1);
230 if (ret)
231 return ret;
232 }
233
234 return 0;
235 }
236
ghes_new(struct acpi_hest_generic * generic)237 static struct ghes *ghes_new(struct acpi_hest_generic *generic)
238 {
239 struct ghes *ghes;
240 unsigned int error_block_length;
241 int rc;
242
243 ghes = kzalloc(sizeof(*ghes), GFP_KERNEL);
244 if (!ghes)
245 return ERR_PTR(-ENOMEM);
246 ghes->generic = generic;
247 rc = apei_map_generic_address(&generic->error_status_address);
248 if (rc)
249 goto err_free;
250 error_block_length = generic->error_block_length;
251 if (error_block_length > GHES_ESTATUS_MAX_SIZE) {
252 pr_warning(FW_WARN GHES_PFX
253 "Error status block length is too long: %u for "
254 "generic hardware error source: %d.\n",
255 error_block_length, generic->header.source_id);
256 error_block_length = GHES_ESTATUS_MAX_SIZE;
257 }
258 ghes->estatus = kmalloc(error_block_length, GFP_KERNEL);
259 if (!ghes->estatus) {
260 rc = -ENOMEM;
261 goto err_unmap;
262 }
263
264 return ghes;
265
266 err_unmap:
267 apei_unmap_generic_address(&generic->error_status_address);
268 err_free:
269 kfree(ghes);
270 return ERR_PTR(rc);
271 }
272
ghes_fini(struct ghes * ghes)273 static void ghes_fini(struct ghes *ghes)
274 {
275 kfree(ghes->estatus);
276 apei_unmap_generic_address(&ghes->generic->error_status_address);
277 }
278
ghes_severity(int severity)279 static inline int ghes_severity(int severity)
280 {
281 switch (severity) {
282 case CPER_SEV_INFORMATIONAL:
283 return GHES_SEV_NO;
284 case CPER_SEV_CORRECTED:
285 return GHES_SEV_CORRECTED;
286 case CPER_SEV_RECOVERABLE:
287 return GHES_SEV_RECOVERABLE;
288 case CPER_SEV_FATAL:
289 return GHES_SEV_PANIC;
290 default:
291 /* Unknown, go panic */
292 return GHES_SEV_PANIC;
293 }
294 }
295
ghes_copy_tofrom_phys(void * buffer,u64 paddr,u32 len,int from_phys)296 static void ghes_copy_tofrom_phys(void *buffer, u64 paddr, u32 len,
297 int from_phys)
298 {
299 void __iomem *vaddr;
300 unsigned long flags = 0;
301 int in_nmi = in_nmi();
302 u64 offset;
303 u32 trunk;
304
305 while (len > 0) {
306 offset = paddr - (paddr & PAGE_MASK);
307 if (in_nmi) {
308 raw_spin_lock(&ghes_ioremap_lock_nmi);
309 vaddr = ghes_ioremap_pfn_nmi(paddr >> PAGE_SHIFT);
310 } else {
311 spin_lock_irqsave(&ghes_ioremap_lock_irq, flags);
312 vaddr = ghes_ioremap_pfn_irq(paddr >> PAGE_SHIFT);
313 }
314 trunk = PAGE_SIZE - offset;
315 trunk = min(trunk, len);
316 if (from_phys)
317 memcpy_fromio(buffer, vaddr + offset, trunk);
318 else
319 memcpy_toio(vaddr + offset, buffer, trunk);
320 len -= trunk;
321 paddr += trunk;
322 buffer += trunk;
323 if (in_nmi) {
324 ghes_iounmap_nmi(vaddr);
325 raw_spin_unlock(&ghes_ioremap_lock_nmi);
326 } else {
327 ghes_iounmap_irq(vaddr);
328 spin_unlock_irqrestore(&ghes_ioremap_lock_irq, flags);
329 }
330 }
331 }
332
ghes_read_estatus(struct ghes * ghes,int silent)333 static int ghes_read_estatus(struct ghes *ghes, int silent)
334 {
335 struct acpi_hest_generic *g = ghes->generic;
336 u64 buf_paddr;
337 u32 len;
338 int rc;
339
340 rc = apei_read(&buf_paddr, &g->error_status_address);
341 if (rc) {
342 if (!silent && printk_ratelimit())
343 pr_warning(FW_WARN GHES_PFX
344 "Failed to read error status block address for hardware error source: %d.\n",
345 g->header.source_id);
346 return -EIO;
347 }
348 if (!buf_paddr)
349 return -ENOENT;
350
351 ghes_copy_tofrom_phys(ghes->estatus, buf_paddr,
352 sizeof(*ghes->estatus), 1);
353 if (!ghes->estatus->block_status)
354 return -ENOENT;
355
356 ghes->buffer_paddr = buf_paddr;
357 ghes->flags |= GHES_TO_CLEAR;
358
359 rc = -EIO;
360 len = cper_estatus_len(ghes->estatus);
361 if (len < sizeof(*ghes->estatus))
362 goto err_read_block;
363 if (len > ghes->generic->error_block_length)
364 goto err_read_block;
365 if (cper_estatus_check_header(ghes->estatus))
366 goto err_read_block;
367 ghes_copy_tofrom_phys(ghes->estatus + 1,
368 buf_paddr + sizeof(*ghes->estatus),
369 len - sizeof(*ghes->estatus), 1);
370 if (cper_estatus_check(ghes->estatus))
371 goto err_read_block;
372 rc = 0;
373
374 err_read_block:
375 if (rc && !silent && printk_ratelimit())
376 pr_warning(FW_WARN GHES_PFX
377 "Failed to read error status block!\n");
378 return rc;
379 }
380
ghes_clear_estatus(struct ghes * ghes)381 static void ghes_clear_estatus(struct ghes *ghes)
382 {
383 ghes->estatus->block_status = 0;
384 if (!(ghes->flags & GHES_TO_CLEAR))
385 return;
386 ghes_copy_tofrom_phys(ghes->estatus, ghes->buffer_paddr,
387 sizeof(ghes->estatus->block_status), 0);
388 ghes->flags &= ~GHES_TO_CLEAR;
389 }
390
ghes_handle_memory_failure(struct acpi_hest_generic_data * gdata,int sev)391 static void ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, int sev)
392 {
393 #ifdef CONFIG_ACPI_APEI_MEMORY_FAILURE
394 unsigned long pfn;
395 int flags = -1;
396 int sec_sev = ghes_severity(gdata->error_severity);
397 struct cper_sec_mem_err *mem_err;
398 mem_err = (struct cper_sec_mem_err *)(gdata + 1);
399
400 if (!(mem_err->validation_bits & CPER_MEM_VALID_PA))
401 return;
402
403 pfn = mem_err->physical_addr >> PAGE_SHIFT;
404 if (!pfn_valid(pfn)) {
405 pr_warn_ratelimited(FW_WARN GHES_PFX
406 "Invalid address in generic error data: %#llx\n",
407 mem_err->physical_addr);
408 return;
409 }
410
411 /* iff following two events can be handled properly by now */
412 if (sec_sev == GHES_SEV_CORRECTED &&
413 (gdata->flags & CPER_SEC_ERROR_THRESHOLD_EXCEEDED))
414 flags = MF_SOFT_OFFLINE;
415 if (sev == GHES_SEV_RECOVERABLE && sec_sev == GHES_SEV_RECOVERABLE)
416 flags = 0;
417
418 if (flags != -1)
419 memory_failure_queue(pfn, 0, flags);
420 #endif
421 }
422
ghes_do_proc(struct ghes * ghes,const struct acpi_hest_generic_status * estatus)423 static void ghes_do_proc(struct ghes *ghes,
424 const struct acpi_hest_generic_status *estatus)
425 {
426 int sev, sec_sev;
427 struct acpi_hest_generic_data *gdata;
428
429 sev = ghes_severity(estatus->error_severity);
430 apei_estatus_for_each_section(estatus, gdata) {
431 sec_sev = ghes_severity(gdata->error_severity);
432 if (!uuid_le_cmp(*(uuid_le *)gdata->section_type,
433 CPER_SEC_PLATFORM_MEM)) {
434 struct cper_sec_mem_err *mem_err;
435 mem_err = (struct cper_sec_mem_err *)(gdata+1);
436 ghes_edac_report_mem_error(ghes, sev, mem_err);
437
438 arch_apei_report_mem_error(sev, mem_err);
439 ghes_handle_memory_failure(gdata, sev);
440 }
441 #ifdef CONFIG_ACPI_APEI_PCIEAER
442 else if (!uuid_le_cmp(*(uuid_le *)gdata->section_type,
443 CPER_SEC_PCIE)) {
444 struct cper_sec_pcie *pcie_err;
445 pcie_err = (struct cper_sec_pcie *)(gdata+1);
446 if (sev == GHES_SEV_RECOVERABLE &&
447 sec_sev == GHES_SEV_RECOVERABLE &&
448 pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID &&
449 pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) {
450 unsigned int devfn;
451 int aer_severity;
452
453 devfn = PCI_DEVFN(pcie_err->device_id.device,
454 pcie_err->device_id.function);
455 aer_severity = cper_severity_to_aer(sev);
456
457 /*
458 * If firmware reset the component to contain
459 * the error, we must reinitialize it before
460 * use, so treat it as a fatal AER error.
461 */
462 if (gdata->flags & CPER_SEC_RESET)
463 aer_severity = AER_FATAL;
464
465 aer_recover_queue(pcie_err->device_id.segment,
466 pcie_err->device_id.bus,
467 devfn, aer_severity,
468 (struct aer_capability_regs *)
469 pcie_err->aer_info);
470 }
471
472 }
473 #endif
474 }
475 }
476
__ghes_print_estatus(const char * pfx,const struct acpi_hest_generic * generic,const struct acpi_hest_generic_status * estatus)477 static void __ghes_print_estatus(const char *pfx,
478 const struct acpi_hest_generic *generic,
479 const struct acpi_hest_generic_status *estatus)
480 {
481 static atomic_t seqno;
482 unsigned int curr_seqno;
483 char pfx_seq[64];
484
485 if (pfx == NULL) {
486 if (ghes_severity(estatus->error_severity) <=
487 GHES_SEV_CORRECTED)
488 pfx = KERN_WARNING;
489 else
490 pfx = KERN_ERR;
491 }
492 curr_seqno = atomic_inc_return(&seqno);
493 snprintf(pfx_seq, sizeof(pfx_seq), "%s{%u}" HW_ERR, pfx, curr_seqno);
494 printk("%s""Hardware error from APEI Generic Hardware Error Source: %d\n",
495 pfx_seq, generic->header.source_id);
496 cper_estatus_print(pfx_seq, estatus);
497 }
498
ghes_print_estatus(const char * pfx,const struct acpi_hest_generic * generic,const struct acpi_hest_generic_status * estatus)499 static int ghes_print_estatus(const char *pfx,
500 const struct acpi_hest_generic *generic,
501 const struct acpi_hest_generic_status *estatus)
502 {
503 /* Not more than 2 messages every 5 seconds */
504 static DEFINE_RATELIMIT_STATE(ratelimit_corrected, 5*HZ, 2);
505 static DEFINE_RATELIMIT_STATE(ratelimit_uncorrected, 5*HZ, 2);
506 struct ratelimit_state *ratelimit;
507
508 if (ghes_severity(estatus->error_severity) <= GHES_SEV_CORRECTED)
509 ratelimit = &ratelimit_corrected;
510 else
511 ratelimit = &ratelimit_uncorrected;
512 if (__ratelimit(ratelimit)) {
513 __ghes_print_estatus(pfx, generic, estatus);
514 return 1;
515 }
516 return 0;
517 }
518
519 /*
520 * GHES error status reporting throttle, to report more kinds of
521 * errors, instead of just most frequently occurred errors.
522 */
ghes_estatus_cached(struct acpi_hest_generic_status * estatus)523 static int ghes_estatus_cached(struct acpi_hest_generic_status *estatus)
524 {
525 u32 len;
526 int i, cached = 0;
527 unsigned long long now;
528 struct ghes_estatus_cache *cache;
529 struct acpi_hest_generic_status *cache_estatus;
530
531 len = cper_estatus_len(estatus);
532 rcu_read_lock();
533 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) {
534 cache = rcu_dereference(ghes_estatus_caches[i]);
535 if (cache == NULL)
536 continue;
537 if (len != cache->estatus_len)
538 continue;
539 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache);
540 if (memcmp(estatus, cache_estatus, len))
541 continue;
542 atomic_inc(&cache->count);
543 now = sched_clock();
544 if (now - cache->time_in < GHES_ESTATUS_IN_CACHE_MAX_NSEC)
545 cached = 1;
546 break;
547 }
548 rcu_read_unlock();
549 return cached;
550 }
551
ghes_estatus_cache_alloc(struct acpi_hest_generic * generic,struct acpi_hest_generic_status * estatus)552 static struct ghes_estatus_cache *ghes_estatus_cache_alloc(
553 struct acpi_hest_generic *generic,
554 struct acpi_hest_generic_status *estatus)
555 {
556 int alloced;
557 u32 len, cache_len;
558 struct ghes_estatus_cache *cache;
559 struct acpi_hest_generic_status *cache_estatus;
560
561 alloced = atomic_add_return(1, &ghes_estatus_cache_alloced);
562 if (alloced > GHES_ESTATUS_CACHE_ALLOCED_MAX) {
563 atomic_dec(&ghes_estatus_cache_alloced);
564 return NULL;
565 }
566 len = cper_estatus_len(estatus);
567 cache_len = GHES_ESTATUS_CACHE_LEN(len);
568 cache = (void *)gen_pool_alloc(ghes_estatus_pool, cache_len);
569 if (!cache) {
570 atomic_dec(&ghes_estatus_cache_alloced);
571 return NULL;
572 }
573 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache);
574 memcpy(cache_estatus, estatus, len);
575 cache->estatus_len = len;
576 atomic_set(&cache->count, 0);
577 cache->generic = generic;
578 cache->time_in = sched_clock();
579 return cache;
580 }
581
ghes_estatus_cache_free(struct ghes_estatus_cache * cache)582 static void ghes_estatus_cache_free(struct ghes_estatus_cache *cache)
583 {
584 u32 len;
585
586 len = cper_estatus_len(GHES_ESTATUS_FROM_CACHE(cache));
587 len = GHES_ESTATUS_CACHE_LEN(len);
588 gen_pool_free(ghes_estatus_pool, (unsigned long)cache, len);
589 atomic_dec(&ghes_estatus_cache_alloced);
590 }
591
ghes_estatus_cache_rcu_free(struct rcu_head * head)592 static void ghes_estatus_cache_rcu_free(struct rcu_head *head)
593 {
594 struct ghes_estatus_cache *cache;
595
596 cache = container_of(head, struct ghes_estatus_cache, rcu);
597 ghes_estatus_cache_free(cache);
598 }
599
ghes_estatus_cache_add(struct acpi_hest_generic * generic,struct acpi_hest_generic_status * estatus)600 static void ghes_estatus_cache_add(
601 struct acpi_hest_generic *generic,
602 struct acpi_hest_generic_status *estatus)
603 {
604 int i, slot = -1, count;
605 unsigned long long now, duration, period, max_period = 0;
606 struct ghes_estatus_cache *cache, *slot_cache = NULL, *new_cache;
607
608 new_cache = ghes_estatus_cache_alloc(generic, estatus);
609 if (new_cache == NULL)
610 return;
611 rcu_read_lock();
612 now = sched_clock();
613 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) {
614 cache = rcu_dereference(ghes_estatus_caches[i]);
615 if (cache == NULL) {
616 slot = i;
617 slot_cache = NULL;
618 break;
619 }
620 duration = now - cache->time_in;
621 if (duration >= GHES_ESTATUS_IN_CACHE_MAX_NSEC) {
622 slot = i;
623 slot_cache = cache;
624 break;
625 }
626 count = atomic_read(&cache->count);
627 period = duration;
628 do_div(period, (count + 1));
629 if (period > max_period) {
630 max_period = period;
631 slot = i;
632 slot_cache = cache;
633 }
634 }
635 /* new_cache must be put into array after its contents are written */
636 smp_wmb();
637 if (slot != -1 && cmpxchg(ghes_estatus_caches + slot,
638 slot_cache, new_cache) == slot_cache) {
639 if (slot_cache)
640 call_rcu(&slot_cache->rcu, ghes_estatus_cache_rcu_free);
641 } else
642 ghes_estatus_cache_free(new_cache);
643 rcu_read_unlock();
644 }
645
ghes_proc(struct ghes * ghes)646 static int ghes_proc(struct ghes *ghes)
647 {
648 int rc;
649
650 rc = ghes_read_estatus(ghes, 0);
651 if (rc)
652 goto out;
653 if (!ghes_estatus_cached(ghes->estatus)) {
654 if (ghes_print_estatus(NULL, ghes->generic, ghes->estatus))
655 ghes_estatus_cache_add(ghes->generic, ghes->estatus);
656 }
657 ghes_do_proc(ghes, ghes->estatus);
658 out:
659 ghes_clear_estatus(ghes);
660 return 0;
661 }
662
ghes_add_timer(struct ghes * ghes)663 static void ghes_add_timer(struct ghes *ghes)
664 {
665 struct acpi_hest_generic *g = ghes->generic;
666 unsigned long expire;
667
668 if (!g->notify.poll_interval) {
669 pr_warning(FW_WARN GHES_PFX "Poll interval is 0 for generic hardware error source: %d, disabled.\n",
670 g->header.source_id);
671 return;
672 }
673 expire = jiffies + msecs_to_jiffies(g->notify.poll_interval);
674 ghes->timer.expires = round_jiffies_relative(expire);
675 add_timer(&ghes->timer);
676 }
677
ghes_poll_func(unsigned long data)678 static void ghes_poll_func(unsigned long data)
679 {
680 struct ghes *ghes = (void *)data;
681
682 ghes_proc(ghes);
683 if (!(ghes->flags & GHES_EXITING))
684 ghes_add_timer(ghes);
685 }
686
ghes_irq_func(int irq,void * data)687 static irqreturn_t ghes_irq_func(int irq, void *data)
688 {
689 struct ghes *ghes = data;
690 int rc;
691
692 rc = ghes_proc(ghes);
693 if (rc)
694 return IRQ_NONE;
695
696 return IRQ_HANDLED;
697 }
698
ghes_notify_sci(struct notifier_block * this,unsigned long event,void * data)699 static int ghes_notify_sci(struct notifier_block *this,
700 unsigned long event, void *data)
701 {
702 struct ghes *ghes;
703 int ret = NOTIFY_DONE;
704
705 rcu_read_lock();
706 list_for_each_entry_rcu(ghes, &ghes_sci, list) {
707 if (!ghes_proc(ghes))
708 ret = NOTIFY_OK;
709 }
710 rcu_read_unlock();
711
712 return ret;
713 }
714
715 static struct notifier_block ghes_notifier_sci = {
716 .notifier_call = ghes_notify_sci,
717 };
718
719 #ifdef CONFIG_HAVE_ACPI_APEI_NMI
720 /*
721 * printk is not safe in NMI context. So in NMI handler, we allocate
722 * required memory from lock-less memory allocator
723 * (ghes_estatus_pool), save estatus into it, put them into lock-less
724 * list (ghes_estatus_llist), then delay printk into IRQ context via
725 * irq_work (ghes_proc_irq_work). ghes_estatus_size_request record
726 * required pool size by all NMI error source.
727 */
728 static struct llist_head ghes_estatus_llist;
729 static struct irq_work ghes_proc_irq_work;
730
731 /*
732 * NMI may be triggered on any CPU, so ghes_nmi_lock is used for
733 * mutual exclusion.
734 */
735 static DEFINE_RAW_SPINLOCK(ghes_nmi_lock);
736
737 static LIST_HEAD(ghes_nmi);
738
739 static int ghes_panic_timeout __read_mostly = 30;
740
llist_nodes_reverse(struct llist_node * llnode)741 static struct llist_node *llist_nodes_reverse(struct llist_node *llnode)
742 {
743 struct llist_node *next, *tail = NULL;
744
745 while (llnode) {
746 next = llnode->next;
747 llnode->next = tail;
748 tail = llnode;
749 llnode = next;
750 }
751
752 return tail;
753 }
754
ghes_proc_in_irq(struct irq_work * irq_work)755 static void ghes_proc_in_irq(struct irq_work *irq_work)
756 {
757 struct llist_node *llnode, *next;
758 struct ghes_estatus_node *estatus_node;
759 struct acpi_hest_generic *generic;
760 struct acpi_hest_generic_status *estatus;
761 u32 len, node_len;
762
763 llnode = llist_del_all(&ghes_estatus_llist);
764 /*
765 * Because the time order of estatus in list is reversed,
766 * revert it back to proper order.
767 */
768 llnode = llist_nodes_reverse(llnode);
769 while (llnode) {
770 next = llnode->next;
771 estatus_node = llist_entry(llnode, struct ghes_estatus_node,
772 llnode);
773 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
774 len = cper_estatus_len(estatus);
775 node_len = GHES_ESTATUS_NODE_LEN(len);
776 ghes_do_proc(estatus_node->ghes, estatus);
777 if (!ghes_estatus_cached(estatus)) {
778 generic = estatus_node->generic;
779 if (ghes_print_estatus(NULL, generic, estatus))
780 ghes_estatus_cache_add(generic, estatus);
781 }
782 gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node,
783 node_len);
784 llnode = next;
785 }
786 }
787
ghes_print_queued_estatus(void)788 static void ghes_print_queued_estatus(void)
789 {
790 struct llist_node *llnode;
791 struct ghes_estatus_node *estatus_node;
792 struct acpi_hest_generic *generic;
793 struct acpi_hest_generic_status *estatus;
794 u32 len, node_len;
795
796 llnode = llist_del_all(&ghes_estatus_llist);
797 /*
798 * Because the time order of estatus in list is reversed,
799 * revert it back to proper order.
800 */
801 llnode = llist_nodes_reverse(llnode);
802 while (llnode) {
803 estatus_node = llist_entry(llnode, struct ghes_estatus_node,
804 llnode);
805 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
806 len = cper_estatus_len(estatus);
807 node_len = GHES_ESTATUS_NODE_LEN(len);
808 generic = estatus_node->generic;
809 ghes_print_estatus(NULL, generic, estatus);
810 llnode = llnode->next;
811 }
812 }
813
ghes_notify_nmi(unsigned int cmd,struct pt_regs * regs)814 static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs)
815 {
816 struct ghes *ghes, *ghes_global = NULL;
817 int sev, sev_global = -1;
818 int ret = NMI_DONE;
819
820 raw_spin_lock(&ghes_nmi_lock);
821 list_for_each_entry_rcu(ghes, &ghes_nmi, list) {
822 if (ghes_read_estatus(ghes, 1)) {
823 ghes_clear_estatus(ghes);
824 continue;
825 }
826 sev = ghes_severity(ghes->estatus->error_severity);
827 if (sev > sev_global) {
828 sev_global = sev;
829 ghes_global = ghes;
830 }
831 ret = NMI_HANDLED;
832 }
833
834 if (ret == NMI_DONE)
835 goto out;
836
837 if (sev_global >= GHES_SEV_PANIC) {
838 oops_begin();
839 ghes_print_queued_estatus();
840 __ghes_print_estatus(KERN_EMERG, ghes_global->generic,
841 ghes_global->estatus);
842 /* reboot to log the error! */
843 if (panic_timeout == 0)
844 panic_timeout = ghes_panic_timeout;
845 panic("Fatal hardware error!");
846 }
847
848 list_for_each_entry_rcu(ghes, &ghes_nmi, list) {
849 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
850 u32 len, node_len;
851 struct ghes_estatus_node *estatus_node;
852 struct acpi_hest_generic_status *estatus;
853 #endif
854 if (!(ghes->flags & GHES_TO_CLEAR))
855 continue;
856 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
857 if (ghes_estatus_cached(ghes->estatus))
858 goto next;
859 /* Save estatus for further processing in IRQ context */
860 len = cper_estatus_len(ghes->estatus);
861 node_len = GHES_ESTATUS_NODE_LEN(len);
862 estatus_node = (void *)gen_pool_alloc(ghes_estatus_pool,
863 node_len);
864 if (estatus_node) {
865 estatus_node->ghes = ghes;
866 estatus_node->generic = ghes->generic;
867 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
868 memcpy(estatus, ghes->estatus, len);
869 llist_add(&estatus_node->llnode, &ghes_estatus_llist);
870 }
871 next:
872 #endif
873 ghes_clear_estatus(ghes);
874 }
875 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
876 irq_work_queue(&ghes_proc_irq_work);
877 #endif
878
879 out:
880 raw_spin_unlock(&ghes_nmi_lock);
881 return ret;
882 }
883
ghes_esource_prealloc_size(const struct acpi_hest_generic * generic)884 static unsigned long ghes_esource_prealloc_size(
885 const struct acpi_hest_generic *generic)
886 {
887 unsigned long block_length, prealloc_records, prealloc_size;
888
889 block_length = min_t(unsigned long, generic->error_block_length,
890 GHES_ESTATUS_MAX_SIZE);
891 prealloc_records = max_t(unsigned long,
892 generic->records_to_preallocate, 1);
893 prealloc_size = min_t(unsigned long, block_length * prealloc_records,
894 GHES_ESOURCE_PREALLOC_MAX_SIZE);
895
896 return prealloc_size;
897 }
898
ghes_estatus_pool_shrink(unsigned long len)899 static void ghes_estatus_pool_shrink(unsigned long len)
900 {
901 ghes_estatus_pool_size_request -= PAGE_ALIGN(len);
902 }
903
ghes_nmi_add(struct ghes * ghes)904 static void ghes_nmi_add(struct ghes *ghes)
905 {
906 unsigned long len;
907
908 len = ghes_esource_prealloc_size(ghes->generic);
909 ghes_estatus_pool_expand(len);
910 mutex_lock(&ghes_list_mutex);
911 if (list_empty(&ghes_nmi))
912 register_nmi_handler(NMI_LOCAL, ghes_notify_nmi, 0, "ghes");
913 list_add_rcu(&ghes->list, &ghes_nmi);
914 mutex_unlock(&ghes_list_mutex);
915 }
916
ghes_nmi_remove(struct ghes * ghes)917 static void ghes_nmi_remove(struct ghes *ghes)
918 {
919 unsigned long len;
920
921 mutex_lock(&ghes_list_mutex);
922 list_del_rcu(&ghes->list);
923 if (list_empty(&ghes_nmi))
924 unregister_nmi_handler(NMI_LOCAL, "ghes");
925 mutex_unlock(&ghes_list_mutex);
926 /*
927 * To synchronize with NMI handler, ghes can only be
928 * freed after NMI handler finishes.
929 */
930 synchronize_rcu();
931 len = ghes_esource_prealloc_size(ghes->generic);
932 ghes_estatus_pool_shrink(len);
933 }
934
ghes_nmi_init_cxt(void)935 static void ghes_nmi_init_cxt(void)
936 {
937 init_irq_work(&ghes_proc_irq_work, ghes_proc_in_irq);
938 }
939 #else /* CONFIG_HAVE_ACPI_APEI_NMI */
ghes_nmi_add(struct ghes * ghes)940 static inline void ghes_nmi_add(struct ghes *ghes)
941 {
942 pr_err(GHES_PFX "ID: %d, trying to add NMI notification which is not supported!\n",
943 ghes->generic->header.source_id);
944 BUG();
945 }
946
ghes_nmi_remove(struct ghes * ghes)947 static inline void ghes_nmi_remove(struct ghes *ghes)
948 {
949 pr_err(GHES_PFX "ID: %d, trying to remove NMI notification which is not supported!\n",
950 ghes->generic->header.source_id);
951 BUG();
952 }
953
ghes_nmi_init_cxt(void)954 static inline void ghes_nmi_init_cxt(void)
955 {
956 }
957 #endif /* CONFIG_HAVE_ACPI_APEI_NMI */
958
ghes_probe(struct platform_device * ghes_dev)959 static int ghes_probe(struct platform_device *ghes_dev)
960 {
961 struct acpi_hest_generic *generic;
962 struct ghes *ghes = NULL;
963
964 int rc = -EINVAL;
965
966 generic = *(struct acpi_hest_generic **)ghes_dev->dev.platform_data;
967 if (!generic->enabled)
968 return -ENODEV;
969
970 switch (generic->notify.type) {
971 case ACPI_HEST_NOTIFY_POLLED:
972 case ACPI_HEST_NOTIFY_EXTERNAL:
973 case ACPI_HEST_NOTIFY_SCI:
974 break;
975 case ACPI_HEST_NOTIFY_NMI:
976 if (!IS_ENABLED(CONFIG_HAVE_ACPI_APEI_NMI)) {
977 pr_warn(GHES_PFX "Generic hardware error source: %d notified via NMI interrupt is not supported!\n",
978 generic->header.source_id);
979 goto err;
980 }
981 break;
982 case ACPI_HEST_NOTIFY_LOCAL:
983 pr_warning(GHES_PFX "Generic hardware error source: %d notified via local interrupt is not supported!\n",
984 generic->header.source_id);
985 goto err;
986 default:
987 pr_warning(FW_WARN GHES_PFX "Unknown notification type: %u for generic hardware error source: %d\n",
988 generic->notify.type, generic->header.source_id);
989 goto err;
990 }
991
992 rc = -EIO;
993 if (generic->error_block_length <
994 sizeof(struct acpi_hest_generic_status)) {
995 pr_warning(FW_BUG GHES_PFX "Invalid error block length: %u for generic hardware error source: %d\n",
996 generic->error_block_length,
997 generic->header.source_id);
998 goto err;
999 }
1000 ghes = ghes_new(generic);
1001 if (IS_ERR(ghes)) {
1002 rc = PTR_ERR(ghes);
1003 ghes = NULL;
1004 goto err;
1005 }
1006
1007 rc = ghes_edac_register(ghes, &ghes_dev->dev);
1008 if (rc < 0)
1009 goto err;
1010
1011 switch (generic->notify.type) {
1012 case ACPI_HEST_NOTIFY_POLLED:
1013 ghes->timer.function = ghes_poll_func;
1014 ghes->timer.data = (unsigned long)ghes;
1015 init_timer_deferrable(&ghes->timer);
1016 ghes_add_timer(ghes);
1017 break;
1018 case ACPI_HEST_NOTIFY_EXTERNAL:
1019 /* External interrupt vector is GSI */
1020 rc = acpi_gsi_to_irq(generic->notify.vector, &ghes->irq);
1021 if (rc) {
1022 pr_err(GHES_PFX "Failed to map GSI to IRQ for generic hardware error source: %d\n",
1023 generic->header.source_id);
1024 goto err_edac_unreg;
1025 }
1026 rc = request_irq(ghes->irq, ghes_irq_func, 0, "GHES IRQ", ghes);
1027 if (rc) {
1028 pr_err(GHES_PFX "Failed to register IRQ for generic hardware error source: %d\n",
1029 generic->header.source_id);
1030 goto err_edac_unreg;
1031 }
1032 break;
1033 case ACPI_HEST_NOTIFY_SCI:
1034 mutex_lock(&ghes_list_mutex);
1035 if (list_empty(&ghes_sci))
1036 register_acpi_hed_notifier(&ghes_notifier_sci);
1037 list_add_rcu(&ghes->list, &ghes_sci);
1038 mutex_unlock(&ghes_list_mutex);
1039 break;
1040 case ACPI_HEST_NOTIFY_NMI:
1041 ghes_nmi_add(ghes);
1042 break;
1043 default:
1044 BUG();
1045 }
1046 platform_set_drvdata(ghes_dev, ghes);
1047
1048 return 0;
1049 err_edac_unreg:
1050 ghes_edac_unregister(ghes);
1051 err:
1052 if (ghes) {
1053 ghes_fini(ghes);
1054 kfree(ghes);
1055 }
1056 return rc;
1057 }
1058
ghes_remove(struct platform_device * ghes_dev)1059 static int ghes_remove(struct platform_device *ghes_dev)
1060 {
1061 struct ghes *ghes;
1062 struct acpi_hest_generic *generic;
1063
1064 ghes = platform_get_drvdata(ghes_dev);
1065 generic = ghes->generic;
1066
1067 ghes->flags |= GHES_EXITING;
1068 switch (generic->notify.type) {
1069 case ACPI_HEST_NOTIFY_POLLED:
1070 del_timer_sync(&ghes->timer);
1071 break;
1072 case ACPI_HEST_NOTIFY_EXTERNAL:
1073 free_irq(ghes->irq, ghes);
1074 break;
1075 case ACPI_HEST_NOTIFY_SCI:
1076 mutex_lock(&ghes_list_mutex);
1077 list_del_rcu(&ghes->list);
1078 if (list_empty(&ghes_sci))
1079 unregister_acpi_hed_notifier(&ghes_notifier_sci);
1080 mutex_unlock(&ghes_list_mutex);
1081 synchronize_rcu();
1082 break;
1083 case ACPI_HEST_NOTIFY_NMI:
1084 ghes_nmi_remove(ghes);
1085 break;
1086 default:
1087 BUG();
1088 break;
1089 }
1090
1091 ghes_fini(ghes);
1092
1093 ghes_edac_unregister(ghes);
1094
1095 kfree(ghes);
1096
1097 platform_set_drvdata(ghes_dev, NULL);
1098
1099 return 0;
1100 }
1101
1102 static struct platform_driver ghes_platform_driver = {
1103 .driver = {
1104 .name = "GHES",
1105 .owner = THIS_MODULE,
1106 },
1107 .probe = ghes_probe,
1108 .remove = ghes_remove,
1109 };
1110
ghes_init(void)1111 static int __init ghes_init(void)
1112 {
1113 int rc;
1114
1115 if (acpi_disabled)
1116 return -ENODEV;
1117
1118 if (hest_disable) {
1119 pr_info(GHES_PFX "HEST is not enabled!\n");
1120 return -EINVAL;
1121 }
1122
1123 if (ghes_disable) {
1124 pr_info(GHES_PFX "GHES is not enabled!\n");
1125 return -EINVAL;
1126 }
1127
1128 ghes_nmi_init_cxt();
1129
1130 rc = ghes_ioremap_init();
1131 if (rc)
1132 goto err;
1133
1134 rc = ghes_estatus_pool_init();
1135 if (rc)
1136 goto err_ioremap_exit;
1137
1138 rc = ghes_estatus_pool_expand(GHES_ESTATUS_CACHE_AVG_SIZE *
1139 GHES_ESTATUS_CACHE_ALLOCED_MAX);
1140 if (rc)
1141 goto err_pool_exit;
1142
1143 rc = platform_driver_register(&ghes_platform_driver);
1144 if (rc)
1145 goto err_pool_exit;
1146
1147 rc = apei_osc_setup();
1148 if (rc == 0 && osc_sb_apei_support_acked)
1149 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit and WHEA _OSC.\n");
1150 else if (rc == 0 && !osc_sb_apei_support_acked)
1151 pr_info(GHES_PFX "APEI firmware first mode is enabled by WHEA _OSC.\n");
1152 else if (rc && osc_sb_apei_support_acked)
1153 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit.\n");
1154 else
1155 pr_info(GHES_PFX "Failed to enable APEI firmware first mode.\n");
1156
1157 return 0;
1158 err_pool_exit:
1159 ghes_estatus_pool_exit();
1160 err_ioremap_exit:
1161 ghes_ioremap_exit();
1162 err:
1163 return rc;
1164 }
1165
ghes_exit(void)1166 static void __exit ghes_exit(void)
1167 {
1168 platform_driver_unregister(&ghes_platform_driver);
1169 ghes_estatus_pool_exit();
1170 ghes_ioremap_exit();
1171 }
1172
1173 module_init(ghes_init);
1174 module_exit(ghes_exit);
1175
1176 MODULE_AUTHOR("Huang Ying");
1177 MODULE_DESCRIPTION("APEI Generic Hardware Error Source support");
1178 MODULE_LICENSE("GPL");
1179 MODULE_ALIAS("platform:GHES");
1180