1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Firmware Assisted dump: A robust mechanism to get reliable kernel crash
4 * dump with assistance from firmware. This approach does not use kexec,
5 * instead firmware assists in booting the kdump kernel while preserving
6 * memory contents. The most of the code implementation has been adapted
7 * from phyp assisted dump implementation written by Linas Vepstas and
8 * Manish Ahuja
9 *
10 * Copyright 2011 IBM Corporation
11 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
12 */
13
14 #undef DEBUG
15 #define pr_fmt(fmt) "fadump: " fmt
16
17 #include <linux/string.h>
18 #include <linux/memblock.h>
19 #include <linux/delay.h>
20 #include <linux/seq_file.h>
21 #include <linux/crash_dump.h>
22 #include <linux/kobject.h>
23 #include <linux/sysfs.h>
24 #include <linux/slab.h>
25 #include <linux/cma.h>
26 #include <linux/hugetlb.h>
27 #include <linux/debugfs.h>
28 #include <linux/of.h>
29 #include <linux/of_fdt.h>
30
31 #include <asm/page.h>
32 #include <asm/fadump.h>
33 #include <asm/fadump-internal.h>
34 #include <asm/setup.h>
35 #include <asm/interrupt.h>
36
37 /*
38 * The CPU who acquired the lock to trigger the fadump crash should
39 * wait for other CPUs to enter.
40 *
41 * The timeout is in milliseconds.
42 */
43 #define CRASH_TIMEOUT 500
44
45 static struct fw_dump fw_dump;
46
47 static void __init fadump_reserve_crash_area(u64 base);
48
49 #ifndef CONFIG_PRESERVE_FA_DUMP
50
51 static struct kobject *fadump_kobj;
52
53 static atomic_t cpus_in_fadump;
54 static DEFINE_MUTEX(fadump_mutex);
55
56 #define RESERVED_RNGS_SZ 16384 /* 16K - 128 entries */
57 #define RESERVED_RNGS_CNT (RESERVED_RNGS_SZ / \
58 sizeof(struct fadump_memory_range))
59 static struct fadump_memory_range rngs[RESERVED_RNGS_CNT];
60 static struct fadump_mrange_info
61 reserved_mrange_info = { "reserved", rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true };
62
63 static void __init early_init_dt_scan_reserved_ranges(unsigned long node);
64
65 #ifdef CONFIG_CMA
66 static struct cma *fadump_cma;
67
68 /*
69 * fadump_cma_init() - Initialize CMA area from a fadump reserved memory
70 *
71 * This function initializes CMA area from fadump reserved memory.
72 * The total size of fadump reserved memory covers for boot memory size
73 * + cpu data size + hpte size and metadata.
74 * Initialize only the area equivalent to boot memory size for CMA use.
75 * The remaining portion of fadump reserved memory will be not given
76 * to CMA and pages for those will stay reserved. boot memory size is
77 * aligned per CMA requirement to satisy cma_init_reserved_mem() call.
78 * But for some reason even if it fails we still have the memory reservation
79 * with us and we can still continue doing fadump.
80 */
fadump_cma_init(void)81 void __init fadump_cma_init(void)
82 {
83 unsigned long long base, size;
84 int rc;
85
86 if (!fw_dump.fadump_supported || !fw_dump.fadump_enabled ||
87 fw_dump.dump_active)
88 return;
89 /*
90 * Do not use CMA if user has provided fadump=nocma kernel parameter.
91 */
92 if (fw_dump.nocma || !fw_dump.boot_memory_size)
93 return;
94
95 base = fw_dump.reserve_dump_area_start;
96 size = fw_dump.boot_memory_size;
97
98 rc = cma_init_reserved_mem(base, size, 0, "fadump_cma", &fadump_cma, false);
99 if (rc) {
100 pr_err("Failed to init cma area for firmware-assisted dump,%d\n", rc);
101 /*
102 * Though the CMA init has failed we still have memory
103 * reservation with us. The reserved memory will be
104 * blocked from production system usage. Hence return 1,
105 * so that we can continue with fadump.
106 */
107 return;
108 }
109
110 /*
111 * If CMA activation fails, keep the pages reserved, instead of
112 * exposing them to buddy allocator. Same as 'fadump=nocma' case.
113 */
114 cma_reserve_pages_on_error(fadump_cma);
115
116 /*
117 * So we now have successfully initialized cma area for fadump.
118 */
119 pr_info("Initialized 0x%lx bytes cma area at %ldMB from 0x%lx "
120 "bytes of memory reserved for firmware-assisted dump\n",
121 cma_get_size(fadump_cma),
122 (unsigned long)cma_get_base(fadump_cma) >> 20,
123 fw_dump.reserve_dump_area_size);
124 }
125 #endif /* CONFIG_CMA */
126
127 /*
128 * Additional parameters meant for capture kernel are placed in a dedicated area.
129 * If this is capture kernel boot, append these parameters to bootargs.
130 */
fadump_append_bootargs(void)131 void __init fadump_append_bootargs(void)
132 {
133 char *append_args;
134 size_t len;
135
136 if (!fw_dump.dump_active || !fw_dump.param_area_supported || !fw_dump.param_area)
137 return;
138
139 if (fw_dump.param_area < fw_dump.boot_mem_top) {
140 if (memblock_reserve(fw_dump.param_area, COMMAND_LINE_SIZE)) {
141 pr_warn("WARNING: Can't use additional parameters area!\n");
142 fw_dump.param_area = 0;
143 return;
144 }
145 }
146
147 append_args = (char *)fw_dump.param_area;
148 len = strlen(boot_command_line);
149
150 /*
151 * Too late to fail even if cmdline size exceeds. Truncate additional parameters
152 * to cmdline size and proceed anyway.
153 */
154 if (len + strlen(append_args) >= COMMAND_LINE_SIZE - 1)
155 pr_warn("WARNING: Appending parameters exceeds cmdline size. Truncating!\n");
156
157 pr_debug("Cmdline: %s\n", boot_command_line);
158 snprintf(boot_command_line + len, COMMAND_LINE_SIZE - len, " %s", append_args);
159 pr_info("Updated cmdline: %s\n", boot_command_line);
160 }
161
162 /* Scan the Firmware Assisted dump configuration details. */
early_init_dt_scan_fw_dump(unsigned long node,const char * uname,int depth,void * data)163 int __init early_init_dt_scan_fw_dump(unsigned long node, const char *uname,
164 int depth, void *data)
165 {
166 if (depth == 0) {
167 early_init_dt_scan_reserved_ranges(node);
168 return 0;
169 }
170
171 if (depth != 1)
172 return 0;
173
174 if (strcmp(uname, "rtas") == 0) {
175 rtas_fadump_dt_scan(&fw_dump, node);
176 return 1;
177 }
178
179 if (strcmp(uname, "ibm,opal") == 0) {
180 opal_fadump_dt_scan(&fw_dump, node);
181 return 1;
182 }
183
184 return 0;
185 }
186
187 /*
188 * If fadump is registered, check if the memory provided
189 * falls within boot memory area and reserved memory area.
190 */
is_fadump_memory_area(u64 addr,unsigned long size)191 int is_fadump_memory_area(u64 addr, unsigned long size)
192 {
193 u64 d_start, d_end;
194
195 if (!fw_dump.dump_registered)
196 return 0;
197
198 if (!size)
199 return 0;
200
201 d_start = fw_dump.reserve_dump_area_start;
202 d_end = d_start + fw_dump.reserve_dump_area_size;
203 if (((addr + size) > d_start) && (addr <= d_end))
204 return 1;
205
206 return (addr <= fw_dump.boot_mem_top);
207 }
208
should_fadump_crash(void)209 int should_fadump_crash(void)
210 {
211 if (!fw_dump.dump_registered || !fw_dump.fadumphdr_addr)
212 return 0;
213 return 1;
214 }
215
is_fadump_active(void)216 int is_fadump_active(void)
217 {
218 return fw_dump.dump_active;
219 }
220
221 /*
222 * Returns true, if there are no holes in memory area between d_start to d_end,
223 * false otherwise.
224 */
is_fadump_mem_area_contiguous(u64 d_start,u64 d_end)225 static bool is_fadump_mem_area_contiguous(u64 d_start, u64 d_end)
226 {
227 phys_addr_t reg_start, reg_end;
228 bool ret = false;
229 u64 i, start, end;
230
231 for_each_mem_range(i, ®_start, ®_end) {
232 start = max_t(u64, d_start, reg_start);
233 end = min_t(u64, d_end, reg_end);
234 if (d_start < end) {
235 /* Memory hole from d_start to start */
236 if (start > d_start)
237 break;
238
239 if (end == d_end) {
240 ret = true;
241 break;
242 }
243
244 d_start = end + 1;
245 }
246 }
247
248 return ret;
249 }
250
251 /*
252 * Returns true, if there are no holes in reserved memory area,
253 * false otherwise.
254 */
is_fadump_reserved_mem_contiguous(void)255 bool is_fadump_reserved_mem_contiguous(void)
256 {
257 u64 d_start, d_end;
258
259 d_start = fw_dump.reserve_dump_area_start;
260 d_end = d_start + fw_dump.reserve_dump_area_size;
261 return is_fadump_mem_area_contiguous(d_start, d_end);
262 }
263
264 /* Print firmware assisted dump configurations for debugging purpose. */
fadump_show_config(void)265 static void __init fadump_show_config(void)
266 {
267 int i;
268
269 pr_debug("Support for firmware-assisted dump (fadump): %s\n",
270 (fw_dump.fadump_supported ? "present" : "no support"));
271
272 if (!fw_dump.fadump_supported)
273 return;
274
275 pr_debug("Fadump enabled : %s\n",
276 (fw_dump.fadump_enabled ? "yes" : "no"));
277 pr_debug("Dump Active : %s\n",
278 (fw_dump.dump_active ? "yes" : "no"));
279 pr_debug("Dump section sizes:\n");
280 pr_debug(" CPU state data size: %lx\n", fw_dump.cpu_state_data_size);
281 pr_debug(" HPTE region size : %lx\n", fw_dump.hpte_region_size);
282 pr_debug(" Boot memory size : %lx\n", fw_dump.boot_memory_size);
283 pr_debug(" Boot memory top : %llx\n", fw_dump.boot_mem_top);
284 pr_debug("Boot memory regions cnt: %llx\n", fw_dump.boot_mem_regs_cnt);
285 for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) {
286 pr_debug("[%03d] base = %llx, size = %llx\n", i,
287 fw_dump.boot_mem_addr[i], fw_dump.boot_mem_sz[i]);
288 }
289 }
290
291 /**
292 * fadump_calculate_reserve_size(): reserve variable boot area 5% of System RAM
293 *
294 * Function to find the largest memory size we need to reserve during early
295 * boot process. This will be the size of the memory that is required for a
296 * kernel to boot successfully.
297 *
298 * This function has been taken from phyp-assisted dump feature implementation.
299 *
300 * returns larger of 256MB or 5% rounded down to multiples of 256MB.
301 *
302 * TODO: Come up with better approach to find out more accurate memory size
303 * that is required for a kernel to boot successfully.
304 *
305 */
fadump_calculate_reserve_size(void)306 static __init u64 fadump_calculate_reserve_size(void)
307 {
308 u64 base, size, bootmem_min;
309 int ret;
310
311 if (fw_dump.reserve_bootvar)
312 pr_warn("'fadump_reserve_mem=' parameter is deprecated in favor of 'crashkernel=' parameter.\n");
313
314 /*
315 * Check if the size is specified through crashkernel= cmdline
316 * option. If yes, then use that but ignore base as fadump reserves
317 * memory at a predefined offset.
318 */
319 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
320 &size, &base, NULL, NULL);
321 if (ret == 0 && size > 0) {
322 unsigned long max_size;
323
324 if (fw_dump.reserve_bootvar)
325 pr_info("Using 'crashkernel=' parameter for memory reservation.\n");
326
327 fw_dump.reserve_bootvar = (unsigned long)size;
328
329 /*
330 * Adjust if the boot memory size specified is above
331 * the upper limit.
332 */
333 max_size = memblock_phys_mem_size() / MAX_BOOT_MEM_RATIO;
334 if (fw_dump.reserve_bootvar > max_size) {
335 fw_dump.reserve_bootvar = max_size;
336 pr_info("Adjusted boot memory size to %luMB\n",
337 (fw_dump.reserve_bootvar >> 20));
338 }
339
340 return fw_dump.reserve_bootvar;
341 } else if (fw_dump.reserve_bootvar) {
342 /*
343 * 'fadump_reserve_mem=' is being used to reserve memory
344 * for firmware-assisted dump.
345 */
346 return fw_dump.reserve_bootvar;
347 }
348
349 /* divide by 20 to get 5% of value */
350 size = memblock_phys_mem_size() / 20;
351
352 /* round it down in multiples of 256 */
353 size = size & ~0x0FFFFFFFUL;
354
355 /* Truncate to memory_limit. We don't want to over reserve the memory.*/
356 if (memory_limit && size > memory_limit)
357 size = memory_limit;
358
359 bootmem_min = fw_dump.ops->fadump_get_bootmem_min();
360 return (size > bootmem_min ? size : bootmem_min);
361 }
362
363 /*
364 * Calculate the total memory size required to be reserved for
365 * firmware-assisted dump registration.
366 */
get_fadump_area_size(void)367 static unsigned long __init get_fadump_area_size(void)
368 {
369 unsigned long size = 0;
370
371 size += fw_dump.cpu_state_data_size;
372 size += fw_dump.hpte_region_size;
373 /*
374 * Account for pagesize alignment of boot memory area destination address.
375 * This faciliates in mmap reading of first kernel's memory.
376 */
377 size = PAGE_ALIGN(size);
378 size += fw_dump.boot_memory_size;
379 size += sizeof(struct fadump_crash_info_header);
380
381 /* This is to hold kernel metadata on platforms that support it */
382 size += (fw_dump.ops->fadump_get_metadata_size ?
383 fw_dump.ops->fadump_get_metadata_size() : 0);
384 return size;
385 }
386
add_boot_mem_region(unsigned long rstart,unsigned long rsize)387 static int __init add_boot_mem_region(unsigned long rstart,
388 unsigned long rsize)
389 {
390 int max_boot_mem_rgns = fw_dump.ops->fadump_max_boot_mem_rgns();
391 int i = fw_dump.boot_mem_regs_cnt++;
392
393 if (fw_dump.boot_mem_regs_cnt > max_boot_mem_rgns) {
394 fw_dump.boot_mem_regs_cnt = max_boot_mem_rgns;
395 return 0;
396 }
397
398 pr_debug("Added boot memory range[%d] [%#016lx-%#016lx)\n",
399 i, rstart, (rstart + rsize));
400 fw_dump.boot_mem_addr[i] = rstart;
401 fw_dump.boot_mem_sz[i] = rsize;
402 return 1;
403 }
404
405 /*
406 * Firmware usually has a hard limit on the data it can copy per region.
407 * Honour that by splitting a memory range into multiple regions.
408 */
add_boot_mem_regions(unsigned long mstart,unsigned long msize)409 static int __init add_boot_mem_regions(unsigned long mstart,
410 unsigned long msize)
411 {
412 unsigned long rstart, rsize, max_size;
413 int ret = 1;
414
415 rstart = mstart;
416 max_size = fw_dump.max_copy_size ? fw_dump.max_copy_size : msize;
417 while (msize) {
418 if (msize > max_size)
419 rsize = max_size;
420 else
421 rsize = msize;
422
423 ret = add_boot_mem_region(rstart, rsize);
424 if (!ret)
425 break;
426
427 msize -= rsize;
428 rstart += rsize;
429 }
430
431 return ret;
432 }
433
fadump_get_boot_mem_regions(void)434 static int __init fadump_get_boot_mem_regions(void)
435 {
436 unsigned long size, cur_size, hole_size, last_end;
437 unsigned long mem_size = fw_dump.boot_memory_size;
438 phys_addr_t reg_start, reg_end;
439 int ret = 1;
440 u64 i;
441
442 fw_dump.boot_mem_regs_cnt = 0;
443
444 last_end = 0;
445 hole_size = 0;
446 cur_size = 0;
447 for_each_mem_range(i, ®_start, ®_end) {
448 size = reg_end - reg_start;
449 hole_size += (reg_start - last_end);
450
451 if ((cur_size + size) >= mem_size) {
452 size = (mem_size - cur_size);
453 ret = add_boot_mem_regions(reg_start, size);
454 break;
455 }
456
457 mem_size -= size;
458 cur_size += size;
459 ret = add_boot_mem_regions(reg_start, size);
460 if (!ret)
461 break;
462
463 last_end = reg_end;
464 }
465 fw_dump.boot_mem_top = PAGE_ALIGN(fw_dump.boot_memory_size + hole_size);
466
467 return ret;
468 }
469
470 /*
471 * Returns true, if the given range overlaps with reserved memory ranges
472 * starting at idx. Also, updates idx to index of overlapping memory range
473 * with the given memory range.
474 * False, otherwise.
475 */
overlaps_reserved_ranges(u64 base,u64 end,int * idx)476 static bool __init overlaps_reserved_ranges(u64 base, u64 end, int *idx)
477 {
478 bool ret = false;
479 int i;
480
481 for (i = *idx; i < reserved_mrange_info.mem_range_cnt; i++) {
482 u64 rbase = reserved_mrange_info.mem_ranges[i].base;
483 u64 rend = rbase + reserved_mrange_info.mem_ranges[i].size;
484
485 if (end <= rbase)
486 break;
487
488 if ((end > rbase) && (base < rend)) {
489 *idx = i;
490 ret = true;
491 break;
492 }
493 }
494
495 return ret;
496 }
497
498 /*
499 * Locate a suitable memory area to reserve memory for FADump. While at it,
500 * lookup reserved-ranges & avoid overlap with them, as they are used by F/W.
501 */
fadump_locate_reserve_mem(u64 base,u64 size)502 static u64 __init fadump_locate_reserve_mem(u64 base, u64 size)
503 {
504 struct fadump_memory_range *mrngs;
505 phys_addr_t mstart, mend;
506 int idx = 0;
507 u64 i, ret = 0;
508
509 mrngs = reserved_mrange_info.mem_ranges;
510 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE,
511 &mstart, &mend, NULL) {
512 pr_debug("%llu) mstart: %llx, mend: %llx, base: %llx\n",
513 i, mstart, mend, base);
514
515 if (mstart > base)
516 base = PAGE_ALIGN(mstart);
517
518 while ((mend > base) && ((mend - base) >= size)) {
519 if (!overlaps_reserved_ranges(base, base+size, &idx)) {
520 ret = base;
521 goto out;
522 }
523
524 base = mrngs[idx].base + mrngs[idx].size;
525 base = PAGE_ALIGN(base);
526 }
527 }
528
529 out:
530 return ret;
531 }
532
fadump_reserve_mem(void)533 int __init fadump_reserve_mem(void)
534 {
535 u64 base, size, mem_boundary, bootmem_min;
536 int ret = 1;
537
538 if (!fw_dump.fadump_enabled)
539 return 0;
540
541 if (!fw_dump.fadump_supported) {
542 pr_info("Firmware-Assisted Dump is not supported on this hardware\n");
543 goto error_out;
544 }
545
546 /*
547 * Initialize boot memory size
548 * If dump is active then we have already calculated the size during
549 * first kernel.
550 */
551 if (!fw_dump.dump_active) {
552 fw_dump.boot_memory_size =
553 PAGE_ALIGN(fadump_calculate_reserve_size());
554 #ifdef CONFIG_CMA
555 if (!fw_dump.nocma) {
556 fw_dump.boot_memory_size =
557 ALIGN(fw_dump.boot_memory_size,
558 CMA_MIN_ALIGNMENT_BYTES);
559 }
560 #endif
561
562 bootmem_min = fw_dump.ops->fadump_get_bootmem_min();
563 if (fw_dump.boot_memory_size < bootmem_min) {
564 pr_err("Can't enable fadump with boot memory size (0x%lx) less than 0x%llx\n",
565 fw_dump.boot_memory_size, bootmem_min);
566 goto error_out;
567 }
568
569 if (!fadump_get_boot_mem_regions()) {
570 pr_err("Too many holes in boot memory area to enable fadump\n");
571 goto error_out;
572 }
573 }
574
575 if (memory_limit)
576 mem_boundary = memory_limit;
577 else
578 mem_boundary = memblock_end_of_DRAM();
579
580 base = fw_dump.boot_mem_top;
581 size = get_fadump_area_size();
582 fw_dump.reserve_dump_area_size = size;
583 if (fw_dump.dump_active) {
584 pr_info("Firmware-assisted dump is active.\n");
585
586 #ifdef CONFIG_HUGETLB_PAGE
587 /*
588 * FADump capture kernel doesn't care much about hugepages.
589 * In fact, handling hugepages in capture kernel is asking for
590 * trouble. So, disable HugeTLB support when fadump is active.
591 */
592 hugetlb_disabled = true;
593 #endif
594 /*
595 * If last boot has crashed then reserve all the memory
596 * above boot memory size so that we don't touch it until
597 * dump is written to disk by userspace tool. This memory
598 * can be released for general use by invalidating fadump.
599 */
600 fadump_reserve_crash_area(base);
601
602 pr_debug("fadumphdr_addr = %#016lx\n", fw_dump.fadumphdr_addr);
603 pr_debug("Reserve dump area start address: 0x%lx\n",
604 fw_dump.reserve_dump_area_start);
605 } else {
606 /*
607 * Reserve memory at an offset closer to bottom of the RAM to
608 * minimize the impact of memory hot-remove operation.
609 */
610 base = fadump_locate_reserve_mem(base, size);
611
612 if (!base || (base + size > mem_boundary)) {
613 pr_err("Failed to find memory chunk for reservation!\n");
614 goto error_out;
615 }
616 fw_dump.reserve_dump_area_start = base;
617
618 /*
619 * Calculate the kernel metadata address and register it with
620 * f/w if the platform supports.
621 */
622 if (fw_dump.ops->fadump_setup_metadata &&
623 (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))
624 goto error_out;
625
626 if (memblock_reserve(base, size)) {
627 pr_err("Failed to reserve memory!\n");
628 goto error_out;
629 }
630
631 pr_info("Reserved %lldMB of memory at %#016llx (System RAM: %lldMB)\n",
632 (size >> 20), base, (memblock_phys_mem_size() >> 20));
633 }
634
635 return ret;
636 error_out:
637 fw_dump.fadump_enabled = 0;
638 fw_dump.reserve_dump_area_size = 0;
639 return 0;
640 }
641
642 /* Look for fadump= cmdline option. */
early_fadump_param(char * p)643 static int __init early_fadump_param(char *p)
644 {
645 if (!p)
646 return 1;
647
648 if (strncmp(p, "on", 2) == 0)
649 fw_dump.fadump_enabled = 1;
650 else if (strncmp(p, "off", 3) == 0)
651 fw_dump.fadump_enabled = 0;
652 else if (strncmp(p, "nocma", 5) == 0) {
653 fw_dump.fadump_enabled = 1;
654 fw_dump.nocma = 1;
655 }
656
657 return 0;
658 }
659 early_param("fadump", early_fadump_param);
660
661 /*
662 * Look for fadump_reserve_mem= cmdline option
663 * TODO: Remove references to 'fadump_reserve_mem=' parameter,
664 * the sooner 'crashkernel=' parameter is accustomed to.
665 */
early_fadump_reserve_mem(char * p)666 static int __init early_fadump_reserve_mem(char *p)
667 {
668 if (p)
669 fw_dump.reserve_bootvar = memparse(p, &p);
670 return 0;
671 }
672 early_param("fadump_reserve_mem", early_fadump_reserve_mem);
673
crash_fadump(struct pt_regs * regs,const char * str)674 void crash_fadump(struct pt_regs *regs, const char *str)
675 {
676 unsigned int msecs;
677 struct fadump_crash_info_header *fdh = NULL;
678 int old_cpu, this_cpu;
679 /* Do not include first CPU */
680 unsigned int ncpus = num_online_cpus() - 1;
681
682 if (!should_fadump_crash())
683 return;
684
685 /*
686 * old_cpu == -1 means this is the first CPU which has come here,
687 * go ahead and trigger fadump.
688 *
689 * old_cpu != -1 means some other CPU has already on its way
690 * to trigger fadump, just keep looping here.
691 */
692 this_cpu = smp_processor_id();
693 old_cpu = cmpxchg(&crashing_cpu, -1, this_cpu);
694
695 if (old_cpu != -1) {
696 atomic_inc(&cpus_in_fadump);
697
698 /*
699 * We can't loop here indefinitely. Wait as long as fadump
700 * is in force. If we race with fadump un-registration this
701 * loop will break and then we go down to normal panic path
702 * and reboot. If fadump is in force the first crashing
703 * cpu will definitely trigger fadump.
704 */
705 while (fw_dump.dump_registered)
706 cpu_relax();
707 return;
708 }
709
710 fdh = __va(fw_dump.fadumphdr_addr);
711 fdh->crashing_cpu = crashing_cpu;
712 crash_save_vmcoreinfo();
713
714 if (regs)
715 fdh->regs = *regs;
716 else
717 ppc_save_regs(&fdh->regs);
718
719 fdh->cpu_mask = *cpu_online_mask;
720
721 /*
722 * If we came in via system reset, wait a while for the secondary
723 * CPUs to enter.
724 */
725 if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) {
726 msecs = CRASH_TIMEOUT;
727 while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0))
728 mdelay(1);
729 }
730
731 fw_dump.ops->fadump_trigger(fdh, str);
732 }
733
fadump_regs_to_elf_notes(u32 * buf,struct pt_regs * regs)734 u32 *__init fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs)
735 {
736 struct elf_prstatus prstatus;
737
738 memset(&prstatus, 0, sizeof(prstatus));
739 /*
740 * FIXME: How do i get PID? Do I really need it?
741 * prstatus.pr_pid = ????
742 */
743 elf_core_copy_regs(&prstatus.pr_reg, regs);
744 buf = append_elf_note(buf, CRASH_CORE_NOTE_NAME, NT_PRSTATUS,
745 &prstatus, sizeof(prstatus));
746 return buf;
747 }
748
fadump_update_elfcore_header(char * bufp)749 void __init fadump_update_elfcore_header(char *bufp)
750 {
751 struct elf_phdr *phdr;
752
753 bufp += sizeof(struct elfhdr);
754
755 /* First note is a place holder for cpu notes info. */
756 phdr = (struct elf_phdr *)bufp;
757
758 if (phdr->p_type == PT_NOTE) {
759 phdr->p_paddr = __pa(fw_dump.cpu_notes_buf_vaddr);
760 phdr->p_offset = phdr->p_paddr;
761 phdr->p_filesz = fw_dump.cpu_notes_buf_size;
762 phdr->p_memsz = fw_dump.cpu_notes_buf_size;
763 }
764 return;
765 }
766
fadump_alloc_buffer(unsigned long size)767 static void *__init fadump_alloc_buffer(unsigned long size)
768 {
769 unsigned long count, i;
770 struct page *page;
771 void *vaddr;
772
773 vaddr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
774 if (!vaddr)
775 return NULL;
776
777 count = PAGE_ALIGN(size) / PAGE_SIZE;
778 page = virt_to_page(vaddr);
779 for (i = 0; i < count; i++)
780 mark_page_reserved(page + i);
781 return vaddr;
782 }
783
fadump_free_buffer(unsigned long vaddr,unsigned long size)784 static void fadump_free_buffer(unsigned long vaddr, unsigned long size)
785 {
786 free_reserved_area((void *)vaddr, (void *)(vaddr + size), -1, NULL);
787 }
788
fadump_setup_cpu_notes_buf(u32 num_cpus)789 s32 __init fadump_setup_cpu_notes_buf(u32 num_cpus)
790 {
791 /* Allocate buffer to hold cpu crash notes. */
792 fw_dump.cpu_notes_buf_size = num_cpus * sizeof(note_buf_t);
793 fw_dump.cpu_notes_buf_size = PAGE_ALIGN(fw_dump.cpu_notes_buf_size);
794 fw_dump.cpu_notes_buf_vaddr =
795 (unsigned long)fadump_alloc_buffer(fw_dump.cpu_notes_buf_size);
796 if (!fw_dump.cpu_notes_buf_vaddr) {
797 pr_err("Failed to allocate %ld bytes for CPU notes buffer\n",
798 fw_dump.cpu_notes_buf_size);
799 return -ENOMEM;
800 }
801
802 pr_debug("Allocated buffer for cpu notes of size %ld at 0x%lx\n",
803 fw_dump.cpu_notes_buf_size,
804 fw_dump.cpu_notes_buf_vaddr);
805 return 0;
806 }
807
fadump_free_cpu_notes_buf(void)808 void fadump_free_cpu_notes_buf(void)
809 {
810 if (!fw_dump.cpu_notes_buf_vaddr)
811 return;
812
813 fadump_free_buffer(fw_dump.cpu_notes_buf_vaddr,
814 fw_dump.cpu_notes_buf_size);
815 fw_dump.cpu_notes_buf_vaddr = 0;
816 fw_dump.cpu_notes_buf_size = 0;
817 }
818
fadump_free_mem_ranges(struct fadump_mrange_info * mrange_info)819 static void fadump_free_mem_ranges(struct fadump_mrange_info *mrange_info)
820 {
821 if (mrange_info->is_static) {
822 mrange_info->mem_range_cnt = 0;
823 return;
824 }
825
826 kfree(mrange_info->mem_ranges);
827 memset((void *)((u64)mrange_info + RNG_NAME_SZ), 0,
828 (sizeof(struct fadump_mrange_info) - RNG_NAME_SZ));
829 }
830
831 /*
832 * Allocate or reallocate mem_ranges array in incremental units
833 * of PAGE_SIZE.
834 */
fadump_alloc_mem_ranges(struct fadump_mrange_info * mrange_info)835 static int fadump_alloc_mem_ranges(struct fadump_mrange_info *mrange_info)
836 {
837 struct fadump_memory_range *new_array;
838 u64 new_size;
839
840 new_size = mrange_info->mem_ranges_sz + PAGE_SIZE;
841 pr_debug("Allocating %llu bytes of memory for %s memory ranges\n",
842 new_size, mrange_info->name);
843
844 new_array = krealloc(mrange_info->mem_ranges, new_size, GFP_KERNEL);
845 if (new_array == NULL) {
846 pr_err("Insufficient memory for setting up %s memory ranges\n",
847 mrange_info->name);
848 fadump_free_mem_ranges(mrange_info);
849 return -ENOMEM;
850 }
851
852 mrange_info->mem_ranges = new_array;
853 mrange_info->mem_ranges_sz = new_size;
854 mrange_info->max_mem_ranges = (new_size /
855 sizeof(struct fadump_memory_range));
856 return 0;
857 }
fadump_add_mem_range(struct fadump_mrange_info * mrange_info,u64 base,u64 end)858 static inline int fadump_add_mem_range(struct fadump_mrange_info *mrange_info,
859 u64 base, u64 end)
860 {
861 struct fadump_memory_range *mem_ranges = mrange_info->mem_ranges;
862 bool is_adjacent = false;
863 u64 start, size;
864
865 if (base == end)
866 return 0;
867
868 /*
869 * Fold adjacent memory ranges to bring down the memory ranges/
870 * PT_LOAD segments count.
871 */
872 if (mrange_info->mem_range_cnt) {
873 start = mem_ranges[mrange_info->mem_range_cnt - 1].base;
874 size = mem_ranges[mrange_info->mem_range_cnt - 1].size;
875
876 /*
877 * Boot memory area needs separate PT_LOAD segment(s) as it
878 * is moved to a different location at the time of crash.
879 * So, fold only if the region is not boot memory area.
880 */
881 if ((start + size) == base && start >= fw_dump.boot_mem_top)
882 is_adjacent = true;
883 }
884 if (!is_adjacent) {
885 /* resize the array on reaching the limit */
886 if (mrange_info->mem_range_cnt == mrange_info->max_mem_ranges) {
887 int ret;
888
889 if (mrange_info->is_static) {
890 pr_err("Reached array size limit for %s memory ranges\n",
891 mrange_info->name);
892 return -ENOSPC;
893 }
894
895 ret = fadump_alloc_mem_ranges(mrange_info);
896 if (ret)
897 return ret;
898
899 /* Update to the new resized array */
900 mem_ranges = mrange_info->mem_ranges;
901 }
902
903 start = base;
904 mem_ranges[mrange_info->mem_range_cnt].base = start;
905 mrange_info->mem_range_cnt++;
906 }
907
908 mem_ranges[mrange_info->mem_range_cnt - 1].size = (end - start);
909 pr_debug("%s_memory_range[%d] [%#016llx-%#016llx], %#llx bytes\n",
910 mrange_info->name, (mrange_info->mem_range_cnt - 1),
911 start, end - 1, (end - start));
912 return 0;
913 }
914
fadump_init_elfcore_header(char * bufp)915 static int fadump_init_elfcore_header(char *bufp)
916 {
917 struct elfhdr *elf;
918
919 elf = (struct elfhdr *) bufp;
920 bufp += sizeof(struct elfhdr);
921 memcpy(elf->e_ident, ELFMAG, SELFMAG);
922 elf->e_ident[EI_CLASS] = ELF_CLASS;
923 elf->e_ident[EI_DATA] = ELF_DATA;
924 elf->e_ident[EI_VERSION] = EV_CURRENT;
925 elf->e_ident[EI_OSABI] = ELF_OSABI;
926 memset(elf->e_ident+EI_PAD, 0, EI_NIDENT-EI_PAD);
927 elf->e_type = ET_CORE;
928 elf->e_machine = ELF_ARCH;
929 elf->e_version = EV_CURRENT;
930 elf->e_entry = 0;
931 elf->e_phoff = sizeof(struct elfhdr);
932 elf->e_shoff = 0;
933
934 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
935 elf->e_flags = 2;
936 else if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1))
937 elf->e_flags = 1;
938 else
939 elf->e_flags = 0;
940
941 elf->e_ehsize = sizeof(struct elfhdr);
942 elf->e_phentsize = sizeof(struct elf_phdr);
943 elf->e_phnum = 0;
944 elf->e_shentsize = 0;
945 elf->e_shnum = 0;
946 elf->e_shstrndx = 0;
947
948 return 0;
949 }
950
951 /*
952 * If the given physical address falls within the boot memory region then
953 * return the relocated address that points to the dump region reserved
954 * for saving initial boot memory contents.
955 */
fadump_relocate(unsigned long paddr)956 static inline unsigned long fadump_relocate(unsigned long paddr)
957 {
958 unsigned long raddr, rstart, rend, rlast, hole_size;
959 int i;
960
961 hole_size = 0;
962 rlast = 0;
963 raddr = paddr;
964 for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) {
965 rstart = fw_dump.boot_mem_addr[i];
966 rend = rstart + fw_dump.boot_mem_sz[i];
967 hole_size += (rstart - rlast);
968
969 if (paddr >= rstart && paddr < rend) {
970 raddr += fw_dump.boot_mem_dest_addr - hole_size;
971 break;
972 }
973
974 rlast = rend;
975 }
976
977 pr_debug("vmcoreinfo: paddr = 0x%lx, raddr = 0x%lx\n", paddr, raddr);
978 return raddr;
979 }
980
populate_elf_pt_load(struct elf_phdr * phdr,u64 start,u64 size,unsigned long long offset)981 static void __init populate_elf_pt_load(struct elf_phdr *phdr, u64 start,
982 u64 size, unsigned long long offset)
983 {
984 phdr->p_align = 0;
985 phdr->p_memsz = size;
986 phdr->p_filesz = size;
987 phdr->p_paddr = start;
988 phdr->p_offset = offset;
989 phdr->p_type = PT_LOAD;
990 phdr->p_flags = PF_R|PF_W|PF_X;
991 phdr->p_vaddr = (unsigned long)__va(start);
992 }
993
fadump_populate_elfcorehdr(struct fadump_crash_info_header * fdh)994 static void __init fadump_populate_elfcorehdr(struct fadump_crash_info_header *fdh)
995 {
996 char *bufp;
997 struct elfhdr *elf;
998 struct elf_phdr *phdr;
999 u64 boot_mem_dest_offset;
1000 unsigned long long i, ra_start, ra_end, ra_size, mstart, mend;
1001
1002 bufp = (char *) fw_dump.elfcorehdr_addr;
1003 fadump_init_elfcore_header(bufp);
1004 elf = (struct elfhdr *)bufp;
1005 bufp += sizeof(struct elfhdr);
1006
1007 /*
1008 * Set up ELF PT_NOTE, a placeholder for CPU notes information.
1009 * The notes info will be populated later by platform-specific code.
1010 * Hence, this PT_NOTE will always be the first ELF note.
1011 *
1012 * NOTE: Any new ELF note addition should be placed after this note.
1013 */
1014 phdr = (struct elf_phdr *)bufp;
1015 bufp += sizeof(struct elf_phdr);
1016 phdr->p_type = PT_NOTE;
1017 phdr->p_flags = 0;
1018 phdr->p_vaddr = 0;
1019 phdr->p_align = 0;
1020 phdr->p_offset = 0;
1021 phdr->p_paddr = 0;
1022 phdr->p_filesz = 0;
1023 phdr->p_memsz = 0;
1024 /* Increment number of program headers. */
1025 (elf->e_phnum)++;
1026
1027 /* setup ELF PT_NOTE for vmcoreinfo */
1028 phdr = (struct elf_phdr *)bufp;
1029 bufp += sizeof(struct elf_phdr);
1030 phdr->p_type = PT_NOTE;
1031 phdr->p_flags = 0;
1032 phdr->p_vaddr = 0;
1033 phdr->p_align = 0;
1034 phdr->p_paddr = phdr->p_offset = fdh->vmcoreinfo_raddr;
1035 phdr->p_memsz = phdr->p_filesz = fdh->vmcoreinfo_size;
1036 /* Increment number of program headers. */
1037 (elf->e_phnum)++;
1038
1039 /*
1040 * Setup PT_LOAD sections. first include boot memory regions
1041 * and then add rest of the memory regions.
1042 */
1043 boot_mem_dest_offset = fw_dump.boot_mem_dest_addr;
1044 for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) {
1045 phdr = (struct elf_phdr *)bufp;
1046 bufp += sizeof(struct elf_phdr);
1047 populate_elf_pt_load(phdr, fw_dump.boot_mem_addr[i],
1048 fw_dump.boot_mem_sz[i],
1049 boot_mem_dest_offset);
1050 /* Increment number of program headers. */
1051 (elf->e_phnum)++;
1052 boot_mem_dest_offset += fw_dump.boot_mem_sz[i];
1053 }
1054
1055 /* Memory reserved for fadump in first kernel */
1056 ra_start = fw_dump.reserve_dump_area_start;
1057 ra_size = get_fadump_area_size();
1058 ra_end = ra_start + ra_size;
1059
1060 phdr = (struct elf_phdr *)bufp;
1061 for_each_mem_range(i, &mstart, &mend) {
1062 /* Boot memory regions already added, skip them now */
1063 if (mstart < fw_dump.boot_mem_top) {
1064 if (mend > fw_dump.boot_mem_top)
1065 mstart = fw_dump.boot_mem_top;
1066 else
1067 continue;
1068 }
1069
1070 /* Handle memblock regions overlaps with fadump reserved area */
1071 if ((ra_start < mend) && (ra_end > mstart)) {
1072 if ((mstart < ra_start) && (mend > ra_end)) {
1073 populate_elf_pt_load(phdr, mstart, ra_start - mstart, mstart);
1074 /* Increment number of program headers. */
1075 (elf->e_phnum)++;
1076 bufp += sizeof(struct elf_phdr);
1077 phdr = (struct elf_phdr *)bufp;
1078 populate_elf_pt_load(phdr, ra_end, mend - ra_end, ra_end);
1079 } else if (mstart < ra_start) {
1080 populate_elf_pt_load(phdr, mstart, ra_start - mstart, mstart);
1081 } else if (ra_end < mend) {
1082 populate_elf_pt_load(phdr, ra_end, mend - ra_end, ra_end);
1083 }
1084 } else {
1085 /* No overlap with fadump reserved memory region */
1086 populate_elf_pt_load(phdr, mstart, mend - mstart, mstart);
1087 }
1088
1089 /* Increment number of program headers. */
1090 (elf->e_phnum)++;
1091 bufp += sizeof(struct elf_phdr);
1092 phdr = (struct elf_phdr *) bufp;
1093 }
1094 }
1095
init_fadump_header(unsigned long addr)1096 static unsigned long init_fadump_header(unsigned long addr)
1097 {
1098 struct fadump_crash_info_header *fdh;
1099
1100 if (!addr)
1101 return 0;
1102
1103 fdh = __va(addr);
1104 addr += sizeof(struct fadump_crash_info_header);
1105
1106 memset(fdh, 0, sizeof(struct fadump_crash_info_header));
1107 fdh->magic_number = FADUMP_CRASH_INFO_MAGIC;
1108 fdh->version = FADUMP_HEADER_VERSION;
1109 /* We will set the crashing cpu id in crash_fadump() during crash. */
1110 fdh->crashing_cpu = FADUMP_CPU_UNKNOWN;
1111
1112 /*
1113 * The physical address and size of vmcoreinfo are required in the
1114 * second kernel to prepare elfcorehdr.
1115 */
1116 fdh->vmcoreinfo_raddr = fadump_relocate(paddr_vmcoreinfo_note());
1117 fdh->vmcoreinfo_size = VMCOREINFO_NOTE_SIZE;
1118
1119
1120 fdh->pt_regs_sz = sizeof(struct pt_regs);
1121 /*
1122 * When LPAR is terminated by PYHP, ensure all possible CPUs'
1123 * register data is processed while exporting the vmcore.
1124 */
1125 fdh->cpu_mask = *cpu_possible_mask;
1126 fdh->cpu_mask_sz = sizeof(struct cpumask);
1127
1128 return addr;
1129 }
1130
register_fadump(void)1131 static int register_fadump(void)
1132 {
1133 unsigned long addr;
1134
1135 /*
1136 * If no memory is reserved then we can not register for firmware-
1137 * assisted dump.
1138 */
1139 if (!fw_dump.reserve_dump_area_size)
1140 return -ENODEV;
1141
1142 addr = fw_dump.fadumphdr_addr;
1143
1144 /* Initialize fadump crash info header. */
1145 addr = init_fadump_header(addr);
1146
1147 /* register the future kernel dump with firmware. */
1148 pr_debug("Registering for firmware-assisted kernel dump...\n");
1149 return fw_dump.ops->fadump_register(&fw_dump);
1150 }
1151
fadump_cleanup(void)1152 void fadump_cleanup(void)
1153 {
1154 if (!fw_dump.fadump_supported)
1155 return;
1156
1157 /* Invalidate the registration only if dump is active. */
1158 if (fw_dump.dump_active) {
1159 pr_debug("Invalidating firmware-assisted dump registration\n");
1160 fw_dump.ops->fadump_invalidate(&fw_dump);
1161 } else if (fw_dump.dump_registered) {
1162 /* Un-register Firmware-assisted dump if it was registered. */
1163 fw_dump.ops->fadump_unregister(&fw_dump);
1164 }
1165
1166 if (fw_dump.ops->fadump_cleanup)
1167 fw_dump.ops->fadump_cleanup(&fw_dump);
1168 }
1169
fadump_free_reserved_memory(unsigned long start_pfn,unsigned long end_pfn)1170 static void fadump_free_reserved_memory(unsigned long start_pfn,
1171 unsigned long end_pfn)
1172 {
1173 unsigned long pfn;
1174 unsigned long time_limit = jiffies + HZ;
1175
1176 pr_info("freeing reserved memory (0x%llx - 0x%llx)\n",
1177 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
1178
1179 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1180 free_reserved_page(pfn_to_page(pfn));
1181
1182 if (time_after(jiffies, time_limit)) {
1183 cond_resched();
1184 time_limit = jiffies + HZ;
1185 }
1186 }
1187 }
1188
1189 /*
1190 * Skip memory holes and free memory that was actually reserved.
1191 */
fadump_release_reserved_area(u64 start,u64 end)1192 static void fadump_release_reserved_area(u64 start, u64 end)
1193 {
1194 unsigned long reg_spfn, reg_epfn;
1195 u64 tstart, tend, spfn, epfn;
1196 int i;
1197
1198 spfn = PHYS_PFN(start);
1199 epfn = PHYS_PFN(end);
1200
1201 for_each_mem_pfn_range(i, MAX_NUMNODES, ®_spfn, ®_epfn, NULL) {
1202 tstart = max_t(u64, spfn, reg_spfn);
1203 tend = min_t(u64, epfn, reg_epfn);
1204
1205 if (tstart < tend) {
1206 fadump_free_reserved_memory(tstart, tend);
1207
1208 if (tend == epfn)
1209 break;
1210
1211 spfn = tend;
1212 }
1213 }
1214 }
1215
1216 /*
1217 * Sort the mem ranges in-place and merge adjacent ranges
1218 * to minimize the memory ranges count.
1219 */
sort_and_merge_mem_ranges(struct fadump_mrange_info * mrange_info)1220 static void sort_and_merge_mem_ranges(struct fadump_mrange_info *mrange_info)
1221 {
1222 struct fadump_memory_range *mem_ranges;
1223 u64 base, size;
1224 int i, j, idx;
1225
1226 if (!reserved_mrange_info.mem_range_cnt)
1227 return;
1228
1229 /* Sort the memory ranges */
1230 mem_ranges = mrange_info->mem_ranges;
1231 for (i = 0; i < mrange_info->mem_range_cnt; i++) {
1232 idx = i;
1233 for (j = (i + 1); j < mrange_info->mem_range_cnt; j++) {
1234 if (mem_ranges[idx].base > mem_ranges[j].base)
1235 idx = j;
1236 }
1237 if (idx != i)
1238 swap(mem_ranges[idx], mem_ranges[i]);
1239 }
1240
1241 /* Merge adjacent reserved ranges */
1242 idx = 0;
1243 for (i = 1; i < mrange_info->mem_range_cnt; i++) {
1244 base = mem_ranges[i-1].base;
1245 size = mem_ranges[i-1].size;
1246 if (mem_ranges[i].base == (base + size))
1247 mem_ranges[idx].size += mem_ranges[i].size;
1248 else {
1249 idx++;
1250 if (i == idx)
1251 continue;
1252
1253 mem_ranges[idx] = mem_ranges[i];
1254 }
1255 }
1256 mrange_info->mem_range_cnt = idx + 1;
1257 }
1258
1259 /*
1260 * Scan reserved-ranges to consider them while reserving/releasing
1261 * memory for FADump.
1262 */
early_init_dt_scan_reserved_ranges(unsigned long node)1263 static void __init early_init_dt_scan_reserved_ranges(unsigned long node)
1264 {
1265 const __be32 *prop;
1266 int len, ret = -1;
1267 unsigned long i;
1268
1269 /* reserved-ranges already scanned */
1270 if (reserved_mrange_info.mem_range_cnt != 0)
1271 return;
1272
1273 prop = of_get_flat_dt_prop(node, "reserved-ranges", &len);
1274 if (!prop)
1275 return;
1276
1277 /*
1278 * Each reserved range is an (address,size) pair, 2 cells each,
1279 * totalling 4 cells per range.
1280 */
1281 for (i = 0; i < len / (sizeof(*prop) * 4); i++) {
1282 u64 base, size;
1283
1284 base = of_read_number(prop + (i * 4) + 0, 2);
1285 size = of_read_number(prop + (i * 4) + 2, 2);
1286
1287 if (size) {
1288 ret = fadump_add_mem_range(&reserved_mrange_info,
1289 base, base + size);
1290 if (ret < 0) {
1291 pr_warn("some reserved ranges are ignored!\n");
1292 break;
1293 }
1294 }
1295 }
1296
1297 /* Compact reserved ranges */
1298 sort_and_merge_mem_ranges(&reserved_mrange_info);
1299 }
1300
1301 /*
1302 * Release the memory that was reserved during early boot to preserve the
1303 * crash'ed kernel's memory contents except reserved dump area (permanent
1304 * reservation) and reserved ranges used by F/W. The released memory will
1305 * be available for general use.
1306 */
fadump_release_memory(u64 begin,u64 end)1307 static void fadump_release_memory(u64 begin, u64 end)
1308 {
1309 u64 ra_start, ra_end, tstart;
1310 int i, ret;
1311
1312 ra_start = fw_dump.reserve_dump_area_start;
1313 ra_end = ra_start + fw_dump.reserve_dump_area_size;
1314
1315 /*
1316 * If reserved ranges array limit is hit, overwrite the last reserved
1317 * memory range with reserved dump area to ensure it is excluded from
1318 * the memory being released (reused for next FADump registration).
1319 */
1320 if (reserved_mrange_info.mem_range_cnt ==
1321 reserved_mrange_info.max_mem_ranges)
1322 reserved_mrange_info.mem_range_cnt--;
1323
1324 ret = fadump_add_mem_range(&reserved_mrange_info, ra_start, ra_end);
1325 if (ret != 0)
1326 return;
1327
1328 /* Get the reserved ranges list in order first. */
1329 sort_and_merge_mem_ranges(&reserved_mrange_info);
1330
1331 /* Exclude reserved ranges and release remaining memory */
1332 tstart = begin;
1333 for (i = 0; i < reserved_mrange_info.mem_range_cnt; i++) {
1334 ra_start = reserved_mrange_info.mem_ranges[i].base;
1335 ra_end = ra_start + reserved_mrange_info.mem_ranges[i].size;
1336
1337 if (tstart >= ra_end)
1338 continue;
1339
1340 if (tstart < ra_start)
1341 fadump_release_reserved_area(tstart, ra_start);
1342 tstart = ra_end;
1343 }
1344
1345 if (tstart < end)
1346 fadump_release_reserved_area(tstart, end);
1347 }
1348
fadump_free_elfcorehdr_buf(void)1349 static void fadump_free_elfcorehdr_buf(void)
1350 {
1351 if (fw_dump.elfcorehdr_addr == 0 || fw_dump.elfcorehdr_size == 0)
1352 return;
1353
1354 /*
1355 * Before freeing the memory of `elfcorehdr`, reset the global
1356 * `elfcorehdr_addr` to prevent modules like `vmcore` from accessing
1357 * invalid memory.
1358 */
1359 elfcorehdr_addr = ELFCORE_ADDR_ERR;
1360 fadump_free_buffer(fw_dump.elfcorehdr_addr, fw_dump.elfcorehdr_size);
1361 fw_dump.elfcorehdr_addr = 0;
1362 fw_dump.elfcorehdr_size = 0;
1363 }
1364
fadump_invalidate_release_mem(void)1365 static void fadump_invalidate_release_mem(void)
1366 {
1367 mutex_lock(&fadump_mutex);
1368 if (!fw_dump.dump_active) {
1369 mutex_unlock(&fadump_mutex);
1370 return;
1371 }
1372
1373 fadump_cleanup();
1374 mutex_unlock(&fadump_mutex);
1375
1376 fadump_free_elfcorehdr_buf();
1377 fadump_release_memory(fw_dump.boot_mem_top, memblock_end_of_DRAM());
1378 fadump_free_cpu_notes_buf();
1379
1380 /*
1381 * Setup kernel metadata and initialize the kernel dump
1382 * memory structure for FADump re-registration.
1383 */
1384 if (fw_dump.ops->fadump_setup_metadata &&
1385 (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))
1386 pr_warn("Failed to setup kernel metadata!\n");
1387 fw_dump.ops->fadump_init_mem_struct(&fw_dump);
1388 }
1389
release_mem_store(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)1390 static ssize_t release_mem_store(struct kobject *kobj,
1391 struct kobj_attribute *attr,
1392 const char *buf, size_t count)
1393 {
1394 int input = -1;
1395
1396 if (!fw_dump.dump_active)
1397 return -EPERM;
1398
1399 if (kstrtoint(buf, 0, &input))
1400 return -EINVAL;
1401
1402 if (input == 1) {
1403 /*
1404 * Take away the '/proc/vmcore'. We are releasing the dump
1405 * memory, hence it will not be valid anymore.
1406 */
1407 #ifdef CONFIG_PROC_VMCORE
1408 vmcore_cleanup();
1409 #endif
1410 fadump_invalidate_release_mem();
1411
1412 } else
1413 return -EINVAL;
1414 return count;
1415 }
1416
1417 /* Release the reserved memory and disable the FADump */
unregister_fadump(void)1418 static void __init unregister_fadump(void)
1419 {
1420 fadump_cleanup();
1421 fadump_release_memory(fw_dump.reserve_dump_area_start,
1422 fw_dump.reserve_dump_area_size);
1423 fw_dump.fadump_enabled = 0;
1424 kobject_put(fadump_kobj);
1425 }
1426
enabled_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)1427 static ssize_t enabled_show(struct kobject *kobj,
1428 struct kobj_attribute *attr,
1429 char *buf)
1430 {
1431 return sprintf(buf, "%d\n", fw_dump.fadump_enabled);
1432 }
1433
1434 /*
1435 * /sys/kernel/fadump/hotplug_ready sysfs node returns 1, which inidcates
1436 * to usersapce that fadump re-registration is not required on memory
1437 * hotplug events.
1438 */
hotplug_ready_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)1439 static ssize_t hotplug_ready_show(struct kobject *kobj,
1440 struct kobj_attribute *attr,
1441 char *buf)
1442 {
1443 return sprintf(buf, "%d\n", 1);
1444 }
1445
mem_reserved_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)1446 static ssize_t mem_reserved_show(struct kobject *kobj,
1447 struct kobj_attribute *attr,
1448 char *buf)
1449 {
1450 return sprintf(buf, "%ld\n", fw_dump.reserve_dump_area_size);
1451 }
1452
registered_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)1453 static ssize_t registered_show(struct kobject *kobj,
1454 struct kobj_attribute *attr,
1455 char *buf)
1456 {
1457 return sprintf(buf, "%d\n", fw_dump.dump_registered);
1458 }
1459
bootargs_append_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)1460 static ssize_t bootargs_append_show(struct kobject *kobj,
1461 struct kobj_attribute *attr,
1462 char *buf)
1463 {
1464 return sprintf(buf, "%s\n", (char *)__va(fw_dump.param_area));
1465 }
1466
bootargs_append_store(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)1467 static ssize_t bootargs_append_store(struct kobject *kobj,
1468 struct kobj_attribute *attr,
1469 const char *buf, size_t count)
1470 {
1471 char *params;
1472
1473 if (!fw_dump.fadump_enabled || fw_dump.dump_active)
1474 return -EPERM;
1475
1476 if (count >= COMMAND_LINE_SIZE)
1477 return -EINVAL;
1478
1479 /*
1480 * Fail here instead of handling this scenario with
1481 * some silly workaround in capture kernel.
1482 */
1483 if (saved_command_line_len + count >= COMMAND_LINE_SIZE) {
1484 pr_err("Appending parameters exceeds cmdline size!\n");
1485 return -ENOSPC;
1486 }
1487
1488 params = __va(fw_dump.param_area);
1489 strscpy_pad(params, buf, COMMAND_LINE_SIZE);
1490 /* Remove newline character at the end. */
1491 if (params[count-1] == '\n')
1492 params[count-1] = '\0';
1493
1494 return count;
1495 }
1496
registered_store(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)1497 static ssize_t registered_store(struct kobject *kobj,
1498 struct kobj_attribute *attr,
1499 const char *buf, size_t count)
1500 {
1501 int ret = 0;
1502 int input = -1;
1503
1504 if (!fw_dump.fadump_enabled || fw_dump.dump_active)
1505 return -EPERM;
1506
1507 if (kstrtoint(buf, 0, &input))
1508 return -EINVAL;
1509
1510 mutex_lock(&fadump_mutex);
1511
1512 switch (input) {
1513 case 0:
1514 if (fw_dump.dump_registered == 0) {
1515 goto unlock_out;
1516 }
1517
1518 /* Un-register Firmware-assisted dump */
1519 pr_debug("Un-register firmware-assisted dump\n");
1520 fw_dump.ops->fadump_unregister(&fw_dump);
1521 break;
1522 case 1:
1523 if (fw_dump.dump_registered == 1) {
1524 /* Un-register Firmware-assisted dump */
1525 fw_dump.ops->fadump_unregister(&fw_dump);
1526 }
1527 /* Register Firmware-assisted dump */
1528 ret = register_fadump();
1529 break;
1530 default:
1531 ret = -EINVAL;
1532 break;
1533 }
1534
1535 unlock_out:
1536 mutex_unlock(&fadump_mutex);
1537 return ret < 0 ? ret : count;
1538 }
1539
fadump_region_show(struct seq_file * m,void * private)1540 static int fadump_region_show(struct seq_file *m, void *private)
1541 {
1542 if (!fw_dump.fadump_enabled)
1543 return 0;
1544
1545 mutex_lock(&fadump_mutex);
1546 fw_dump.ops->fadump_region_show(&fw_dump, m);
1547 mutex_unlock(&fadump_mutex);
1548 return 0;
1549 }
1550
1551 static struct kobj_attribute release_attr = __ATTR_WO(release_mem);
1552 static struct kobj_attribute enable_attr = __ATTR_RO(enabled);
1553 static struct kobj_attribute register_attr = __ATTR_RW(registered);
1554 static struct kobj_attribute mem_reserved_attr = __ATTR_RO(mem_reserved);
1555 static struct kobj_attribute hotplug_ready_attr = __ATTR_RO(hotplug_ready);
1556 static struct kobj_attribute bootargs_append_attr = __ATTR_RW(bootargs_append);
1557
1558 static struct attribute *fadump_attrs[] = {
1559 &enable_attr.attr,
1560 ®ister_attr.attr,
1561 &mem_reserved_attr.attr,
1562 &hotplug_ready_attr.attr,
1563 NULL,
1564 };
1565
1566 ATTRIBUTE_GROUPS(fadump);
1567
1568 DEFINE_SHOW_ATTRIBUTE(fadump_region);
1569
fadump_init_files(void)1570 static void __init fadump_init_files(void)
1571 {
1572 int rc = 0;
1573
1574 fadump_kobj = kobject_create_and_add("fadump", kernel_kobj);
1575 if (!fadump_kobj) {
1576 pr_err("failed to create fadump kobject\n");
1577 return;
1578 }
1579
1580 if (fw_dump.param_area) {
1581 rc = sysfs_create_file(fadump_kobj, &bootargs_append_attr.attr);
1582 if (rc)
1583 pr_err("unable to create bootargs_append sysfs file (%d)\n", rc);
1584 }
1585
1586 debugfs_create_file("fadump_region", 0444, arch_debugfs_dir, NULL,
1587 &fadump_region_fops);
1588
1589 if (fw_dump.dump_active) {
1590 rc = sysfs_create_file(fadump_kobj, &release_attr.attr);
1591 if (rc)
1592 pr_err("unable to create release_mem sysfs file (%d)\n",
1593 rc);
1594 }
1595
1596 rc = sysfs_create_groups(fadump_kobj, fadump_groups);
1597 if (rc) {
1598 pr_err("sysfs group creation failed (%d), unregistering FADump",
1599 rc);
1600 unregister_fadump();
1601 return;
1602 }
1603
1604 /*
1605 * The FADump sysfs are moved from kernel_kobj to fadump_kobj need to
1606 * create symlink at old location to maintain backward compatibility.
1607 *
1608 * - fadump_enabled -> fadump/enabled
1609 * - fadump_registered -> fadump/registered
1610 * - fadump_release_mem -> fadump/release_mem
1611 */
1612 rc = compat_only_sysfs_link_entry_to_kobj(kernel_kobj, fadump_kobj,
1613 "enabled", "fadump_enabled");
1614 if (rc) {
1615 pr_err("unable to create fadump_enabled symlink (%d)", rc);
1616 return;
1617 }
1618
1619 rc = compat_only_sysfs_link_entry_to_kobj(kernel_kobj, fadump_kobj,
1620 "registered",
1621 "fadump_registered");
1622 if (rc) {
1623 pr_err("unable to create fadump_registered symlink (%d)", rc);
1624 sysfs_remove_link(kernel_kobj, "fadump_enabled");
1625 return;
1626 }
1627
1628 if (fw_dump.dump_active) {
1629 rc = compat_only_sysfs_link_entry_to_kobj(kernel_kobj,
1630 fadump_kobj,
1631 "release_mem",
1632 "fadump_release_mem");
1633 if (rc)
1634 pr_err("unable to create fadump_release_mem symlink (%d)",
1635 rc);
1636 }
1637 return;
1638 }
1639
fadump_setup_elfcorehdr_buf(void)1640 static int __init fadump_setup_elfcorehdr_buf(void)
1641 {
1642 int elf_phdr_cnt;
1643 unsigned long elfcorehdr_size;
1644
1645 /*
1646 * Program header for CPU notes comes first, followed by one for
1647 * vmcoreinfo, and the remaining program headers correspond to
1648 * memory regions.
1649 */
1650 elf_phdr_cnt = 2 + fw_dump.boot_mem_regs_cnt + memblock_num_regions(memory);
1651 elfcorehdr_size = sizeof(struct elfhdr) + (elf_phdr_cnt * sizeof(struct elf_phdr));
1652 elfcorehdr_size = PAGE_ALIGN(elfcorehdr_size);
1653
1654 fw_dump.elfcorehdr_addr = (u64)fadump_alloc_buffer(elfcorehdr_size);
1655 if (!fw_dump.elfcorehdr_addr) {
1656 pr_err("Failed to allocate %lu bytes for elfcorehdr\n",
1657 elfcorehdr_size);
1658 return -ENOMEM;
1659 }
1660 fw_dump.elfcorehdr_size = elfcorehdr_size;
1661 return 0;
1662 }
1663
1664 /*
1665 * Check if the fadump header of crashed kernel is compatible with fadump kernel.
1666 *
1667 * It checks the magic number, endianness, and size of non-primitive type
1668 * members of fadump header to ensure safe dump collection.
1669 */
is_fadump_header_compatible(struct fadump_crash_info_header * fdh)1670 static bool __init is_fadump_header_compatible(struct fadump_crash_info_header *fdh)
1671 {
1672 if (fdh->magic_number == FADUMP_CRASH_INFO_MAGIC_OLD) {
1673 pr_err("Old magic number, can't process the dump.\n");
1674 return false;
1675 }
1676
1677 if (fdh->magic_number != FADUMP_CRASH_INFO_MAGIC) {
1678 if (fdh->magic_number == swab64(FADUMP_CRASH_INFO_MAGIC))
1679 pr_err("Endianness mismatch between the crashed and fadump kernels.\n");
1680 else
1681 pr_err("Fadump header is corrupted.\n");
1682
1683 return false;
1684 }
1685
1686 /*
1687 * Dump collection is not safe if the size of non-primitive type members
1688 * of the fadump header do not match between crashed and fadump kernel.
1689 */
1690 if (fdh->pt_regs_sz != sizeof(struct pt_regs) ||
1691 fdh->cpu_mask_sz != sizeof(struct cpumask)) {
1692 pr_err("Fadump header size mismatch.\n");
1693 return false;
1694 }
1695
1696 return true;
1697 }
1698
fadump_process(void)1699 static void __init fadump_process(void)
1700 {
1701 struct fadump_crash_info_header *fdh;
1702
1703 fdh = (struct fadump_crash_info_header *) __va(fw_dump.fadumphdr_addr);
1704 if (!fdh) {
1705 pr_err("Crash info header is empty.\n");
1706 goto err_out;
1707 }
1708
1709 /* Avoid processing the dump if fadump header isn't compatible */
1710 if (!is_fadump_header_compatible(fdh))
1711 goto err_out;
1712
1713 /* Allocate buffer for elfcorehdr */
1714 if (fadump_setup_elfcorehdr_buf())
1715 goto err_out;
1716
1717 fadump_populate_elfcorehdr(fdh);
1718
1719 /* Let platform update the CPU notes in elfcorehdr */
1720 if (fw_dump.ops->fadump_process(&fw_dump) < 0)
1721 goto err_out;
1722
1723 /*
1724 * elfcorehdr is now ready to be exported.
1725 *
1726 * set elfcorehdr_addr so that vmcore module will export the
1727 * elfcorehdr through '/proc/vmcore'.
1728 */
1729 elfcorehdr_addr = virt_to_phys((void *)fw_dump.elfcorehdr_addr);
1730 return;
1731
1732 err_out:
1733 fadump_invalidate_release_mem();
1734 }
1735
1736 /*
1737 * Reserve memory to store additional parameters to be passed
1738 * for fadump/capture kernel.
1739 */
fadump_setup_param_area(void)1740 void __init fadump_setup_param_area(void)
1741 {
1742 phys_addr_t range_start, range_end;
1743
1744 if (!fw_dump.param_area_supported || fw_dump.dump_active)
1745 return;
1746
1747 /* This memory can't be used by PFW or bootloader as it is shared across kernels */
1748 if (early_radix_enabled()) {
1749 /*
1750 * Anywhere in the upper half should be good enough as all memory
1751 * is accessible in real mode.
1752 */
1753 range_start = memblock_end_of_DRAM() / 2;
1754 range_end = memblock_end_of_DRAM();
1755 } else {
1756 /*
1757 * Passing additional parameters is supported for hash MMU only
1758 * if the first memory block size is 768MB or higher.
1759 */
1760 if (ppc64_rma_size < 0x30000000)
1761 return;
1762
1763 /*
1764 * 640 MB to 768 MB is not used by PFW/bootloader. So, try reserving
1765 * memory for passing additional parameters in this range to avoid
1766 * being stomped on by PFW/bootloader.
1767 */
1768 range_start = 0x2A000000;
1769 range_end = range_start + 0x4000000;
1770 }
1771
1772 fw_dump.param_area = memblock_phys_alloc_range(COMMAND_LINE_SIZE,
1773 COMMAND_LINE_SIZE,
1774 range_start,
1775 range_end);
1776 if (!fw_dump.param_area) {
1777 pr_warn("WARNING: Could not setup area to pass additional parameters!\n");
1778 return;
1779 }
1780
1781 memset((void *)fw_dump.param_area, 0, COMMAND_LINE_SIZE);
1782 }
1783
1784 /*
1785 * Prepare for firmware-assisted dump.
1786 */
setup_fadump(void)1787 int __init setup_fadump(void)
1788 {
1789 if (!fw_dump.fadump_supported)
1790 return 0;
1791
1792 fadump_init_files();
1793 fadump_show_config();
1794
1795 if (!fw_dump.fadump_enabled)
1796 return 1;
1797
1798 /*
1799 * If dump data is available then see if it is valid and prepare for
1800 * saving it to the disk.
1801 */
1802 if (fw_dump.dump_active) {
1803 fadump_process();
1804 }
1805 /* Initialize the kernel dump memory structure and register with f/w */
1806 else if (fw_dump.reserve_dump_area_size) {
1807 fw_dump.ops->fadump_init_mem_struct(&fw_dump);
1808 register_fadump();
1809 }
1810
1811 /*
1812 * In case of panic, fadump is triggered via ppc_panic_event()
1813 * panic notifier. Setting crash_kexec_post_notifiers to 'true'
1814 * lets panic() function take crash friendly path before panic
1815 * notifiers are invoked.
1816 */
1817 crash_kexec_post_notifiers = true;
1818
1819 return 1;
1820 }
1821 /*
1822 * Use subsys_initcall_sync() here because there is dependency with
1823 * crash_save_vmcoreinfo_init(), which must run first to ensure vmcoreinfo initialization
1824 * is done before registering with f/w.
1825 */
1826 subsys_initcall_sync(setup_fadump);
1827 #else /* !CONFIG_PRESERVE_FA_DUMP */
1828
1829 /* Scan the Firmware Assisted dump configuration details. */
early_init_dt_scan_fw_dump(unsigned long node,const char * uname,int depth,void * data)1830 int __init early_init_dt_scan_fw_dump(unsigned long node, const char *uname,
1831 int depth, void *data)
1832 {
1833 if ((depth != 1) || (strcmp(uname, "ibm,opal") != 0))
1834 return 0;
1835
1836 opal_fadump_dt_scan(&fw_dump, node);
1837 return 1;
1838 }
1839
1840 /*
1841 * When dump is active but PRESERVE_FA_DUMP is enabled on the kernel,
1842 * preserve crash data. The subsequent memory preserving kernel boot
1843 * is likely to process this crash data.
1844 */
fadump_reserve_mem(void)1845 int __init fadump_reserve_mem(void)
1846 {
1847 if (fw_dump.dump_active) {
1848 /*
1849 * If last boot has crashed then reserve all the memory
1850 * above boot memory to preserve crash data.
1851 */
1852 pr_info("Preserving crash data for processing in next boot.\n");
1853 fadump_reserve_crash_area(fw_dump.boot_mem_top);
1854 } else
1855 pr_debug("FADump-aware kernel..\n");
1856
1857 return 1;
1858 }
1859 #endif /* CONFIG_PRESERVE_FA_DUMP */
1860
1861 /* Preserve everything above the base address */
fadump_reserve_crash_area(u64 base)1862 static void __init fadump_reserve_crash_area(u64 base)
1863 {
1864 u64 i, mstart, mend, msize;
1865
1866 for_each_mem_range(i, &mstart, &mend) {
1867 msize = mend - mstart;
1868
1869 if ((mstart + msize) < base)
1870 continue;
1871
1872 if (mstart < base) {
1873 msize -= (base - mstart);
1874 mstart = base;
1875 }
1876
1877 pr_info("Reserving %lluMB of memory at %#016llx for preserving crash data",
1878 (msize >> 20), mstart);
1879 memblock_reserve(mstart, msize);
1880 }
1881 }
1882