1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Procedures for maintaining information about logical memory blocks.
4 *
5 * Peter Bergner, IBM Corp. June 2001.
6 * Copyright (C) 2001 Peter Bergner.
7 */
8
9 #include <linux/kernel.h>
10 #include <linux/slab.h>
11 #include <linux/init.h>
12 #include <linux/bitops.h>
13 #include <linux/poison.h>
14 #include <linux/pfn.h>
15 #include <linux/debugfs.h>
16 #include <linux/kmemleak.h>
17 #include <linux/seq_file.h>
18 #include <linux/memblock.h>
19
20 #include <asm/sections.h>
21 #include <linux/io.h>
22 #include <linux/sort.h>
23 #include <linux/proc_fs.h>
24
25 #include "internal.h"
26
27 #define INIT_MEMBLOCK_REGIONS 128
28 #define INIT_PHYSMEM_REGIONS 4
29
30 #ifndef INIT_MEMBLOCK_RESERVED_REGIONS
31 # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
32 #endif
33
34 #ifndef INIT_MEMBLOCK_MEMORY_REGIONS
35 #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS
36 #endif
37
38 /**
39 * DOC: memblock overview
40 *
41 * Memblock is a method of managing memory regions during the early
42 * boot period when the usual kernel memory allocators are not up and
43 * running.
44 *
45 * Memblock views the system memory as collections of contiguous
46 * regions. There are several types of these collections:
47 *
48 * * ``memory`` - describes the physical memory available to the
49 * kernel; this may differ from the actual physical memory installed
50 * in the system, for instance when the memory is restricted with
51 * ``mem=`` command line parameter
52 * * ``reserved`` - describes the regions that were allocated
53 * * ``physmem`` - describes the actual physical memory available during
54 * boot regardless of the possible restrictions and memory hot(un)plug;
55 * the ``physmem`` type is only available on some architectures.
56 *
57 * Each region is represented by struct memblock_region that
58 * defines the region extents, its attributes and NUMA node id on NUMA
59 * systems. Every memory type is described by the struct memblock_type
60 * which contains an array of memory regions along with
61 * the allocator metadata. The "memory" and "reserved" types are nicely
62 * wrapped with struct memblock. This structure is statically
63 * initialized at build time. The region arrays are initially sized to
64 * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and
65 * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array
66 * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS.
67 * The memblock_allow_resize() enables automatic resizing of the region
68 * arrays during addition of new regions. This feature should be used
69 * with care so that memory allocated for the region array will not
70 * overlap with areas that should be reserved, for example initrd.
71 *
72 * The early architecture setup should tell memblock what the physical
73 * memory layout is by using memblock_add() or memblock_add_node()
74 * functions. The first function does not assign the region to a NUMA
75 * node and it is appropriate for UMA systems. Yet, it is possible to
76 * use it on NUMA systems as well and assign the region to a NUMA node
77 * later in the setup process using memblock_set_node(). The
78 * memblock_add_node() performs such an assignment directly.
79 *
80 * Once memblock is setup the memory can be allocated using one of the
81 * API variants:
82 *
83 * * memblock_phys_alloc*() - these functions return the **physical**
84 * address of the allocated memory
85 * * memblock_alloc*() - these functions return the **virtual** address
86 * of the allocated memory.
87 *
88 * Note, that both API variants use implicit assumptions about allowed
89 * memory ranges and the fallback methods. Consult the documentation
90 * of memblock_alloc_internal() and memblock_alloc_range_nid()
91 * functions for more elaborate description.
92 *
93 * As the system boot progresses, the architecture specific mem_init()
94 * function frees all the memory to the buddy page allocator.
95 *
96 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
97 * memblock data structures (except "physmem") will be discarded after the
98 * system initialization completes.
99 */
100
101 #ifndef CONFIG_NUMA
102 struct pglist_data __refdata contig_page_data;
103 EXPORT_SYMBOL(contig_page_data);
104 #endif
105
106 unsigned long max_low_pfn;
107 unsigned long min_low_pfn;
108 unsigned long max_pfn;
109 unsigned long long max_possible_pfn;
110
111 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock;
112 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
113 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
114 static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS];
115 #endif
116
117 struct memblock memblock __initdata_memblock = {
118 .memory.regions = memblock_memory_init_regions,
119 .memory.cnt = 1, /* empty dummy entry */
120 .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS,
121 .memory.name = "memory",
122
123 .reserved.regions = memblock_reserved_init_regions,
124 .reserved.cnt = 1, /* empty dummy entry */
125 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS,
126 .reserved.name = "reserved",
127
128 .bottom_up = false,
129 .current_limit = MEMBLOCK_ALLOC_ANYWHERE,
130 };
131
132 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
133 struct memblock_type physmem = {
134 .regions = memblock_physmem_init_regions,
135 .cnt = 1, /* empty dummy entry */
136 .max = INIT_PHYSMEM_REGIONS,
137 .name = "physmem",
138 };
139 #endif
140
141 static long memsize_kinit;
142 static bool memblock_memsize_tracking __initdata_memblock = true;
143
144 /*
145 * keep a pointer to &memblock.memory in the text section to use it in
146 * __next_mem_range() and its helpers.
147 * For architectures that do not keep memblock data after init, this
148 * pointer will be reset to NULL at memblock_discard()
149 */
150 static __refdata struct memblock_type *memblock_memory = &memblock.memory;
151
152 #define for_each_memblock_type(i, memblock_type, rgn) \
153 for (i = 0, rgn = &memblock_type->regions[0]; \
154 i < memblock_type->cnt; \
155 i++, rgn = &memblock_type->regions[i])
156
157 #define memblock_dbg(fmt, ...) \
158 do { \
159 if (memblock_debug) \
160 pr_info(fmt, ##__VA_ARGS__); \
161 } while (0)
162
163 static int memblock_debug __initdata_memblock;
164 static bool system_has_some_mirror __initdata_memblock;
165 static int memblock_can_resize __initdata_memblock;
166 static int memblock_memory_in_slab __initdata_memblock;
167 static int memblock_reserved_in_slab __initdata_memblock;
168
memblock_has_mirror(void)169 bool __init_memblock memblock_has_mirror(void)
170 {
171 return system_has_some_mirror;
172 }
173
choose_memblock_flags(void)174 static enum memblock_flags __init_memblock choose_memblock_flags(void)
175 {
176 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
177 }
178
179 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
memblock_cap_size(phys_addr_t base,phys_addr_t * size)180 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
181 {
182 return *size = min(*size, PHYS_ADDR_MAX - base);
183 }
184
185 /*
186 * Address comparison utilities
187 */
188 unsigned long __init_memblock
memblock_addrs_overlap(phys_addr_t base1,phys_addr_t size1,phys_addr_t base2,phys_addr_t size2)189 memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, phys_addr_t base2,
190 phys_addr_t size2)
191 {
192 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
193 }
194
memblock_overlaps_region(struct memblock_type * type,phys_addr_t base,phys_addr_t size)195 bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
196 phys_addr_t base, phys_addr_t size)
197 {
198 unsigned long i;
199
200 memblock_cap_size(base, &size);
201
202 for (i = 0; i < type->cnt; i++)
203 if (memblock_addrs_overlap(base, size, type->regions[i].base,
204 type->regions[i].size))
205 break;
206 return i < type->cnt;
207 }
208
209 /**
210 * __memblock_find_range_bottom_up - find free area utility in bottom-up
211 * @start: start of candidate range
212 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
213 * %MEMBLOCK_ALLOC_ACCESSIBLE
214 * @size: size of free area to find
215 * @align: alignment of free area to find
216 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
217 * @flags: pick from blocks based on memory attributes
218 *
219 * Utility called from memblock_find_in_range_node(), find free area bottom-up.
220 *
221 * Return:
222 * Found address on success, 0 on failure.
223 */
224 static phys_addr_t __init_memblock
__memblock_find_range_bottom_up(phys_addr_t start,phys_addr_t end,phys_addr_t size,phys_addr_t align,int nid,enum memblock_flags flags)225 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
226 phys_addr_t size, phys_addr_t align, int nid,
227 enum memblock_flags flags)
228 {
229 phys_addr_t this_start, this_end, cand;
230 u64 i;
231
232 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
233 this_start = clamp(this_start, start, end);
234 this_end = clamp(this_end, start, end);
235
236 cand = round_up(this_start, align);
237 if (cand < this_end && this_end - cand >= size)
238 return cand;
239 }
240
241 return 0;
242 }
243
244 /**
245 * __memblock_find_range_top_down - find free area utility, in top-down
246 * @start: start of candidate range
247 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
248 * %MEMBLOCK_ALLOC_ACCESSIBLE
249 * @size: size of free area to find
250 * @align: alignment of free area to find
251 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
252 * @flags: pick from blocks based on memory attributes
253 *
254 * Utility called from memblock_find_in_range_node(), find free area top-down.
255 *
256 * Return:
257 * Found address on success, 0 on failure.
258 */
259 static phys_addr_t __init_memblock
__memblock_find_range_top_down(phys_addr_t start,phys_addr_t end,phys_addr_t size,phys_addr_t align,int nid,enum memblock_flags flags)260 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
261 phys_addr_t size, phys_addr_t align, int nid,
262 enum memblock_flags flags)
263 {
264 phys_addr_t this_start, this_end, cand;
265 u64 i;
266
267 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
268 NULL) {
269 this_start = clamp(this_start, start, end);
270 this_end = clamp(this_end, start, end);
271
272 if (this_end < size)
273 continue;
274
275 cand = round_down(this_end - size, align);
276 if (cand >= this_start)
277 return cand;
278 }
279
280 return 0;
281 }
282
283 /**
284 * memblock_find_in_range_node - find free area in given range and node
285 * @size: size of free area to find
286 * @align: alignment of free area to find
287 * @start: start of candidate range
288 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
289 * %MEMBLOCK_ALLOC_ACCESSIBLE
290 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
291 * @flags: pick from blocks based on memory attributes
292 *
293 * Find @size free area aligned to @align in the specified range and node.
294 *
295 * Return:
296 * Found address on success, 0 on failure.
297 */
memblock_find_in_range_node(phys_addr_t size,phys_addr_t align,phys_addr_t start,phys_addr_t end,int nid,enum memblock_flags flags)298 static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
299 phys_addr_t align, phys_addr_t start,
300 phys_addr_t end, int nid,
301 enum memblock_flags flags)
302 {
303 /* pump up @end */
304 if (end == MEMBLOCK_ALLOC_ACCESSIBLE ||
305 end == MEMBLOCK_ALLOC_NOLEAKTRACE)
306 end = memblock.current_limit;
307
308 /* avoid allocating the first page */
309 start = max_t(phys_addr_t, start, PAGE_SIZE);
310 end = max(start, end);
311
312 if (memblock_bottom_up())
313 return __memblock_find_range_bottom_up(start, end, size, align,
314 nid, flags);
315 else
316 return __memblock_find_range_top_down(start, end, size, align,
317 nid, flags);
318 }
319
320 /**
321 * memblock_find_in_range - find free area in given range
322 * @start: start of candidate range
323 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
324 * %MEMBLOCK_ALLOC_ACCESSIBLE
325 * @size: size of free area to find
326 * @align: alignment of free area to find
327 *
328 * Find @size free area aligned to @align in the specified range.
329 *
330 * Return:
331 * Found address on success, 0 on failure.
332 */
memblock_find_in_range(phys_addr_t start,phys_addr_t end,phys_addr_t size,phys_addr_t align)333 static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
334 phys_addr_t end, phys_addr_t size,
335 phys_addr_t align)
336 {
337 phys_addr_t ret;
338 enum memblock_flags flags = choose_memblock_flags();
339
340 again:
341 ret = memblock_find_in_range_node(size, align, start, end,
342 NUMA_NO_NODE, flags);
343
344 if (!ret && (flags & MEMBLOCK_MIRROR)) {
345 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n",
346 &size);
347 flags &= ~MEMBLOCK_MIRROR;
348 goto again;
349 }
350
351 return ret;
352 }
353
memblock_remove_region(struct memblock_type * type,unsigned long r)354 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
355 {
356 type->total_size -= type->regions[r].size;
357 memmove(&type->regions[r], &type->regions[r + 1],
358 (type->cnt - (r + 1)) * sizeof(type->regions[r]));
359 type->cnt--;
360
361 /* Special case for empty arrays */
362 if (type->cnt == 0) {
363 WARN_ON(type->total_size != 0);
364 type->cnt = 1;
365 type->regions[0].base = 0;
366 type->regions[0].size = 0;
367 type->regions[0].flags = 0;
368 memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
369 }
370 }
371
372 #ifndef CONFIG_ARCH_KEEP_MEMBLOCK
373 /**
374 * memblock_discard - discard memory and reserved arrays if they were allocated
375 */
memblock_discard(void)376 void __init memblock_discard(void)
377 {
378 phys_addr_t addr, size;
379
380 if (memblock.reserved.regions != memblock_reserved_init_regions) {
381 addr = __pa(memblock.reserved.regions);
382 size = PAGE_ALIGN(sizeof(struct memblock_region) *
383 memblock.reserved.max);
384 if (memblock_reserved_in_slab)
385 kfree(memblock.reserved.regions);
386 else
387 memblock_free_late(addr, size);
388 }
389
390 if (memblock.memory.regions != memblock_memory_init_regions) {
391 addr = __pa(memblock.memory.regions);
392 size = PAGE_ALIGN(sizeof(struct memblock_region) *
393 memblock.memory.max);
394 if (memblock_memory_in_slab)
395 kfree(memblock.memory.regions);
396 else
397 memblock_free_late(addr, size);
398 }
399
400 memblock_memory = NULL;
401 }
402 #endif
403
404 /**
405 * memblock_double_array - double the size of the memblock regions array
406 * @type: memblock type of the regions array being doubled
407 * @new_area_start: starting address of memory range to avoid overlap with
408 * @new_area_size: size of memory range to avoid overlap with
409 *
410 * Double the size of the @type regions array. If memblock is being used to
411 * allocate memory for a new reserved regions array and there is a previously
412 * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
413 * waiting to be reserved, ensure the memory used by the new array does
414 * not overlap.
415 *
416 * Return:
417 * 0 on success, -1 on failure.
418 */
memblock_double_array(struct memblock_type * type,phys_addr_t new_area_start,phys_addr_t new_area_size)419 static int __init_memblock memblock_double_array(struct memblock_type *type,
420 phys_addr_t new_area_start,
421 phys_addr_t new_area_size)
422 {
423 struct memblock_region *new_array, *old_array;
424 phys_addr_t old_alloc_size, new_alloc_size;
425 phys_addr_t old_size, new_size, addr, new_end;
426 int use_slab = slab_is_available();
427 int *in_slab;
428
429 /* We don't allow resizing until we know about the reserved regions
430 * of memory that aren't suitable for allocation
431 */
432 if (!memblock_can_resize)
433 return -1;
434
435 /* Calculate new doubled size */
436 old_size = type->max * sizeof(struct memblock_region);
437 new_size = old_size << 1;
438 /*
439 * We need to allocated new one align to PAGE_SIZE,
440 * so we can free them completely later.
441 */
442 old_alloc_size = PAGE_ALIGN(old_size);
443 new_alloc_size = PAGE_ALIGN(new_size);
444
445 /* Retrieve the slab flag */
446 if (type == &memblock.memory)
447 in_slab = &memblock_memory_in_slab;
448 else
449 in_slab = &memblock_reserved_in_slab;
450
451 /* Try to find some space for it */
452 if (use_slab) {
453 new_array = kmalloc(new_size, GFP_KERNEL);
454 addr = new_array ? __pa(new_array) : 0;
455 } else {
456 /* only exclude range when trying to double reserved.regions */
457 if (type != &memblock.reserved)
458 new_area_start = new_area_size = 0;
459
460 addr = memblock_find_in_range(new_area_start + new_area_size,
461 memblock.current_limit,
462 new_alloc_size, PAGE_SIZE);
463 if (!addr && new_area_size)
464 addr = memblock_find_in_range(0,
465 min(new_area_start, memblock.current_limit),
466 new_alloc_size, PAGE_SIZE);
467
468 new_array = addr ? __va(addr) : NULL;
469 }
470 if (!addr) {
471 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
472 type->name, type->max, type->max * 2);
473 return -1;
474 }
475
476 new_end = addr + new_size - 1;
477 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
478 type->name, type->max * 2, &addr, &new_end);
479
480 /*
481 * Found space, we now need to move the array over before we add the
482 * reserved region since it may be our reserved array itself that is
483 * full.
484 */
485 memcpy(new_array, type->regions, old_size);
486 memset(new_array + type->max, 0, old_size);
487 old_array = type->regions;
488 type->regions = new_array;
489 type->max <<= 1;
490
491 /* Free old array. We needn't free it if the array is the static one */
492 if (*in_slab)
493 kfree(old_array);
494 else if (old_array != memblock_memory_init_regions &&
495 old_array != memblock_reserved_init_regions)
496 memblock_free(old_array, old_alloc_size);
497
498 /*
499 * Reserve the new array if that comes from the memblock. Otherwise, we
500 * needn't do it
501 */
502 if (!use_slab)
503 BUG_ON(memblock_reserve(addr, new_alloc_size));
504
505 /* Update slab flag */
506 *in_slab = use_slab;
507
508 return 0;
509 }
510
511 /**
512 * memblock_merge_regions - merge neighboring compatible regions
513 * @type: memblock type to scan
514 * @start_rgn: start scanning from (@start_rgn - 1)
515 * @end_rgn: end scanning at (@end_rgn - 1)
516 * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn)
517 */
memblock_merge_regions(struct memblock_type * type,unsigned long start_rgn,unsigned long end_rgn)518 static void __init_memblock memblock_merge_regions(struct memblock_type *type,
519 unsigned long start_rgn,
520 unsigned long end_rgn)
521 {
522 int i = 0;
523 if (start_rgn)
524 i = start_rgn - 1;
525 end_rgn = min(end_rgn, type->cnt - 1);
526 while (i < end_rgn) {
527 struct memblock_region *this = &type->regions[i];
528 struct memblock_region *next = &type->regions[i + 1];
529
530 if (this->base + this->size != next->base ||
531 memblock_get_region_node(this) !=
532 memblock_get_region_node(next) ||
533 this->flags != next->flags) {
534 BUG_ON(this->base + this->size > next->base);
535 i++;
536 continue;
537 }
538
539 this->size += next->size;
540 /* move forward from next + 1, index of which is i + 2 */
541 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
542 type->cnt--;
543 end_rgn--;
544 }
545 }
546
547 /**
548 * memblock_insert_region - insert new memblock region
549 * @type: memblock type to insert into
550 * @idx: index for the insertion point
551 * @base: base address of the new region
552 * @size: size of the new region
553 * @nid: node id of the new region
554 * @flags: flags of the new region
555 *
556 * Insert new memblock region [@base, @base + @size) into @type at @idx.
557 * @type must already have extra room to accommodate the new region.
558 */
memblock_insert_region(struct memblock_type * type,int idx,phys_addr_t base,phys_addr_t size,int nid,enum memblock_flags flags)559 static void __init_memblock memblock_insert_region(struct memblock_type *type,
560 int idx, phys_addr_t base,
561 phys_addr_t size,
562 int nid,
563 enum memblock_flags flags)
564 {
565 struct memblock_region *rgn = &type->regions[idx];
566
567 BUG_ON(type->cnt >= type->max);
568 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
569 rgn->base = base;
570 rgn->size = size;
571 rgn->flags = flags;
572 memblock_set_region_node(rgn, nid);
573 type->cnt++;
574 type->total_size += size;
575 }
576
577 /**
578 * memblock_add_range - add new memblock region
579 * @type: memblock type to add new region into
580 * @base: base address of the new region
581 * @size: size of the new region
582 * @nid: nid of the new region
583 * @flags: flags of the new region
584 *
585 * Add new memblock region [@base, @base + @size) into @type. The new region
586 * is allowed to overlap with existing ones - overlaps don't affect already
587 * existing regions. @type is guaranteed to be minimal (all neighbouring
588 * compatible regions are merged) after the addition.
589 *
590 * Return:
591 * 0 on success, -errno on failure.
592 */
memblock_add_range(struct memblock_type * type,phys_addr_t base,phys_addr_t size,int nid,enum memblock_flags flags)593 static int __init_memblock memblock_add_range(struct memblock_type *type,
594 phys_addr_t base, phys_addr_t size,
595 int nid, enum memblock_flags flags)
596 {
597 bool insert = false;
598 phys_addr_t obase = base;
599 phys_addr_t end = base + memblock_cap_size(base, &size);
600 int idx, nr_new, start_rgn = -1, end_rgn;
601 struct memblock_region *rgn;
602 phys_addr_t new_size = 0;
603
604 if (!size)
605 return 0;
606
607 /* special case for empty array */
608 if (type->regions[0].size == 0) {
609 WARN_ON(type->cnt != 1 || type->total_size);
610 type->regions[0].base = base;
611 type->regions[0].size = size;
612 type->regions[0].flags = flags;
613 memblock_set_region_node(&type->regions[0], nid);
614 type->total_size = size;
615 new_size = size;
616 goto done;
617 }
618
619 /*
620 * The worst case is when new range overlaps all existing regions,
621 * then we'll need type->cnt + 1 empty regions in @type. So if
622 * type->cnt * 2 + 1 is less than or equal to type->max, we know
623 * that there is enough empty regions in @type, and we can insert
624 * regions directly.
625 */
626 if (type->cnt * 2 + 1 <= type->max)
627 insert = true;
628
629 repeat:
630 /*
631 * The following is executed twice. Once with %false @insert and
632 * then with %true. The first counts the number of regions needed
633 * to accommodate the new area. The second actually inserts them.
634 */
635 base = obase;
636 nr_new = 0;
637
638 for_each_memblock_type(idx, type, rgn) {
639 phys_addr_t rbase = rgn->base;
640 phys_addr_t rend = rbase + rgn->size;
641
642 if (rbase >= end)
643 break;
644 if (rend <= base)
645 continue;
646 /*
647 * @rgn overlaps. If it separates the lower part of new
648 * area, insert that portion.
649 */
650 if (rbase > base) {
651 #ifdef CONFIG_NUMA
652 WARN_ON(nid != memblock_get_region_node(rgn));
653 #endif
654 WARN_ON(flags != rgn->flags);
655 nr_new++;
656 if (insert) {
657 if (start_rgn == -1)
658 start_rgn = idx;
659 end_rgn = idx + 1;
660 memblock_insert_region(type, idx++, base,
661 rbase - base, nid,
662 flags);
663 new_size += rbase - base;
664 }
665 }
666 /* area below @rend is dealt with, forget about it */
667 base = min(rend, end);
668 }
669
670 /* insert the remaining portion */
671 if (base < end) {
672 nr_new++;
673 if (insert) {
674 if (start_rgn == -1)
675 start_rgn = idx;
676 end_rgn = idx + 1;
677 memblock_insert_region(type, idx, base, end - base,
678 nid, flags);
679 new_size += end - base;
680 }
681 }
682
683 if (!nr_new)
684 return 0;
685
686 /*
687 * If this was the first round, resize array and repeat for actual
688 * insertions; otherwise, merge and return.
689 */
690 if (!insert) {
691 while (type->cnt + nr_new > type->max)
692 if (memblock_double_array(type, obase, size) < 0)
693 return -ENOMEM;
694 insert = true;
695 goto repeat;
696 } else {
697 memblock_merge_regions(type, start_rgn, end_rgn);
698 }
699 done:
700 if (memblock_memsize_tracking) {
701 if (new_size && type == &memblock.reserved) {
702 memblock_dbg("%s: kernel %lu %+ld\n", __func__,
703 memsize_kinit, (unsigned long)new_size);
704 memsize_kinit += size;
705 }
706 }
707 return 0;
708 }
709
710 /**
711 * memblock_add_node - add new memblock region within a NUMA node
712 * @base: base address of the new region
713 * @size: size of the new region
714 * @nid: nid of the new region
715 * @flags: flags of the new region
716 *
717 * Add new memblock region [@base, @base + @size) to the "memory"
718 * type. See memblock_add_range() description for mode details
719 *
720 * Return:
721 * 0 on success, -errno on failure.
722 */
memblock_add_node(phys_addr_t base,phys_addr_t size,int nid,enum memblock_flags flags)723 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
724 int nid, enum memblock_flags flags)
725 {
726 phys_addr_t end = base + size - 1;
727
728 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__,
729 &base, &end, nid, flags, (void *)_RET_IP_);
730
731 return memblock_add_range(&memblock.memory, base, size, nid, flags);
732 }
733
734 /**
735 * memblock_add - add new memblock region
736 * @base: base address of the new region
737 * @size: size of the new region
738 *
739 * Add new memblock region [@base, @base + @size) to the "memory"
740 * type. See memblock_add_range() description for mode details
741 *
742 * Return:
743 * 0 on success, -errno on failure.
744 */
memblock_add(phys_addr_t base,phys_addr_t size)745 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
746 {
747 phys_addr_t end = base + size - 1;
748
749 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
750 &base, &end, (void *)_RET_IP_);
751
752 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
753 }
754
755 /**
756 * memblock_isolate_range - isolate given range into disjoint memblocks
757 * @type: memblock type to isolate range for
758 * @base: base of range to isolate
759 * @size: size of range to isolate
760 * @start_rgn: out parameter for the start of isolated region
761 * @end_rgn: out parameter for the end of isolated region
762 *
763 * Walk @type and ensure that regions don't cross the boundaries defined by
764 * [@base, @base + @size). Crossing regions are split at the boundaries,
765 * which may create at most two more regions. The index of the first
766 * region inside the range is returned in *@start_rgn and end in *@end_rgn.
767 *
768 * Return:
769 * 0 on success, -errno on failure.
770 */
memblock_isolate_range(struct memblock_type * type,phys_addr_t base,phys_addr_t size,int * start_rgn,int * end_rgn)771 static int __init_memblock memblock_isolate_range(struct memblock_type *type,
772 phys_addr_t base, phys_addr_t size,
773 int *start_rgn, int *end_rgn)
774 {
775 phys_addr_t end = base + memblock_cap_size(base, &size);
776 int idx;
777 struct memblock_region *rgn;
778
779 *start_rgn = *end_rgn = 0;
780
781 if (!size)
782 return 0;
783
784 /* we'll create at most two more regions */
785 while (type->cnt + 2 > type->max)
786 if (memblock_double_array(type, base, size) < 0)
787 return -ENOMEM;
788
789 for_each_memblock_type(idx, type, rgn) {
790 phys_addr_t rbase = rgn->base;
791 phys_addr_t rend = rbase + rgn->size;
792
793 if (rbase >= end)
794 break;
795 if (rend <= base)
796 continue;
797
798 if (rbase < base) {
799 /*
800 * @rgn intersects from below. Split and continue
801 * to process the next region - the new top half.
802 */
803 rgn->base = base;
804 rgn->size -= base - rbase;
805 type->total_size -= base - rbase;
806 memblock_insert_region(type, idx, rbase, base - rbase,
807 memblock_get_region_node(rgn),
808 rgn->flags);
809 } else if (rend > end) {
810 /*
811 * @rgn intersects from above. Split and redo the
812 * current region - the new bottom half.
813 */
814 rgn->base = end;
815 rgn->size -= end - rbase;
816 type->total_size -= end - rbase;
817 memblock_insert_region(type, idx--, rbase, end - rbase,
818 memblock_get_region_node(rgn),
819 rgn->flags);
820 } else {
821 /* @rgn is fully contained, record it */
822 if (!*end_rgn)
823 *start_rgn = idx;
824 *end_rgn = idx + 1;
825 }
826 }
827
828 return 0;
829 }
830
memblock_remove_range(struct memblock_type * type,phys_addr_t base,phys_addr_t size)831 static int __init_memblock memblock_remove_range(struct memblock_type *type,
832 phys_addr_t base, phys_addr_t size)
833 {
834 int start_rgn, end_rgn;
835 int i, ret;
836
837 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
838 if (ret)
839 return ret;
840
841 for (i = end_rgn - 1; i >= start_rgn; i--)
842 memblock_remove_region(type, i);
843 if (memblock_memsize_tracking) {
844 if (type == &memblock.reserved) {
845 memblock_dbg("%s: kernel %lu %+ld\n", __func__,
846 memsize_kinit, (unsigned long)size);
847 memsize_kinit -= size;
848 }
849 }
850 return 0;
851 }
852
memblock_remove(phys_addr_t base,phys_addr_t size)853 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
854 {
855 phys_addr_t end = base + size - 1;
856
857 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
858 &base, &end, (void *)_RET_IP_);
859
860 return memblock_remove_range(&memblock.memory, base, size);
861 }
862
863 /**
864 * memblock_free - free boot memory allocation
865 * @ptr: starting address of the boot memory allocation
866 * @size: size of the boot memory block in bytes
867 *
868 * Free boot memory block previously allocated by memblock_alloc_xx() API.
869 * The freeing memory will not be released to the buddy allocator.
870 */
memblock_free(void * ptr,size_t size)871 void __init_memblock memblock_free(void *ptr, size_t size)
872 {
873 if (ptr)
874 memblock_phys_free(__pa(ptr), size);
875 }
876
877 /**
878 * memblock_phys_free - free boot memory block
879 * @base: phys starting address of the boot memory block
880 * @size: size of the boot memory block in bytes
881 *
882 * Free boot memory block previously allocated by memblock_phys_alloc_xx() API.
883 * The freeing memory will not be released to the buddy allocator.
884 */
memblock_phys_free(phys_addr_t base,phys_addr_t size)885 int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size)
886 {
887 phys_addr_t end = base + size - 1;
888
889 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
890 &base, &end, (void *)_RET_IP_);
891
892 kmemleak_free_part_phys(base, size);
893 return memblock_remove_range(&memblock.reserved, base, size);
894 }
895
memblock_reserve(phys_addr_t base,phys_addr_t size)896 int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
897 {
898 phys_addr_t end = base + size - 1;
899
900 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
901 &base, &end, (void *)_RET_IP_);
902
903 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
904 }
905
906 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
memblock_physmem_add(phys_addr_t base,phys_addr_t size)907 int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
908 {
909 phys_addr_t end = base + size - 1;
910
911 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
912 &base, &end, (void *)_RET_IP_);
913
914 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0);
915 }
916 #endif
917
918 /**
919 * memblock_setclr_flag - set or clear flag for a memory region
920 * @base: base address of the region
921 * @size: size of the region
922 * @set: set or clear the flag
923 * @flag: the flag to update
924 *
925 * This function isolates region [@base, @base + @size), and sets/clears flag
926 *
927 * Return: 0 on success, -errno on failure.
928 */
memblock_setclr_flag(phys_addr_t base,phys_addr_t size,int set,int flag)929 static int __init_memblock memblock_setclr_flag(phys_addr_t base,
930 phys_addr_t size, int set, int flag)
931 {
932 struct memblock_type *type = &memblock.memory;
933 int i, ret, start_rgn, end_rgn;
934
935 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
936 if (ret)
937 return ret;
938
939 for (i = start_rgn; i < end_rgn; i++) {
940 struct memblock_region *r = &type->regions[i];
941
942 if (set)
943 r->flags |= flag;
944 else
945 r->flags &= ~flag;
946 }
947
948 memblock_merge_regions(type, start_rgn, end_rgn);
949 return 0;
950 }
951
952 /**
953 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
954 * @base: the base phys addr of the region
955 * @size: the size of the region
956 *
957 * Return: 0 on success, -errno on failure.
958 */
memblock_mark_hotplug(phys_addr_t base,phys_addr_t size)959 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
960 {
961 return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
962 }
963
964 /**
965 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
966 * @base: the base phys addr of the region
967 * @size: the size of the region
968 *
969 * Return: 0 on success, -errno on failure.
970 */
memblock_clear_hotplug(phys_addr_t base,phys_addr_t size)971 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
972 {
973 return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
974 }
975
976 /**
977 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
978 * @base: the base phys addr of the region
979 * @size: the size of the region
980 *
981 * Return: 0 on success, -errno on failure.
982 */
memblock_mark_mirror(phys_addr_t base,phys_addr_t size)983 int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
984 {
985 if (!mirrored_kernelcore)
986 return 0;
987
988 system_has_some_mirror = true;
989
990 return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
991 }
992
993 /**
994 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
995 * @base: the base phys addr of the region
996 * @size: the size of the region
997 *
998 * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the
999 * direct mapping of the physical memory. These regions will still be
1000 * covered by the memory map. The struct page representing NOMAP memory
1001 * frames in the memory map will be PageReserved()
1002 *
1003 * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from
1004 * memblock, the caller must inform kmemleak to ignore that memory
1005 *
1006 * Return: 0 on success, -errno on failure.
1007 */
memblock_mark_nomap(phys_addr_t base,phys_addr_t size)1008 int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
1009 {
1010 return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
1011 }
1012
1013 /**
1014 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
1015 * @base: the base phys addr of the region
1016 * @size: the size of the region
1017 *
1018 * Return: 0 on success, -errno on failure.
1019 */
memblock_clear_nomap(phys_addr_t base,phys_addr_t size)1020 int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
1021 {
1022 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
1023 }
1024
should_skip_region(struct memblock_type * type,struct memblock_region * m,int nid,int flags)1025 static bool should_skip_region(struct memblock_type *type,
1026 struct memblock_region *m,
1027 int nid, int flags)
1028 {
1029 int m_nid = memblock_get_region_node(m);
1030
1031 /* we never skip regions when iterating memblock.reserved or physmem */
1032 if (type != memblock_memory)
1033 return false;
1034
1035 /* only memory regions are associated with nodes, check it */
1036 if (nid != NUMA_NO_NODE && nid != m_nid)
1037 return true;
1038
1039 /* skip hotpluggable memory regions if needed */
1040 if (movable_node_is_enabled() && memblock_is_hotpluggable(m) &&
1041 !(flags & MEMBLOCK_HOTPLUG))
1042 return true;
1043
1044 /* if we want mirror memory skip non-mirror memory regions */
1045 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
1046 return true;
1047
1048 /* skip nomap memory unless we were asked for it explicitly */
1049 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
1050 return true;
1051
1052 /* skip driver-managed memory unless we were asked for it explicitly */
1053 if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m))
1054 return true;
1055
1056 return false;
1057 }
1058
1059 /**
1060 * __next_mem_range - next function for for_each_free_mem_range() etc.
1061 * @idx: pointer to u64 loop variable
1062 * @nid: node selector, %NUMA_NO_NODE for all nodes
1063 * @flags: pick from blocks based on memory attributes
1064 * @type_a: pointer to memblock_type from where the range is taken
1065 * @type_b: pointer to memblock_type which excludes memory from being taken
1066 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1067 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1068 * @out_nid: ptr to int for nid of the range, can be %NULL
1069 *
1070 * Find the first area from *@idx which matches @nid, fill the out
1071 * parameters, and update *@idx for the next iteration. The lower 32bit of
1072 * *@idx contains index into type_a and the upper 32bit indexes the
1073 * areas before each region in type_b. For example, if type_b regions
1074 * look like the following,
1075 *
1076 * 0:[0-16), 1:[32-48), 2:[128-130)
1077 *
1078 * The upper 32bit indexes the following regions.
1079 *
1080 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
1081 *
1082 * As both region arrays are sorted, the function advances the two indices
1083 * in lockstep and returns each intersection.
1084 */
__next_mem_range(u64 * idx,int nid,enum memblock_flags flags,struct memblock_type * type_a,struct memblock_type * type_b,phys_addr_t * out_start,phys_addr_t * out_end,int * out_nid)1085 void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
1086 struct memblock_type *type_a,
1087 struct memblock_type *type_b, phys_addr_t *out_start,
1088 phys_addr_t *out_end, int *out_nid)
1089 {
1090 int idx_a = *idx & 0xffffffff;
1091 int idx_b = *idx >> 32;
1092
1093 if (WARN_ONCE(nid == MAX_NUMNODES,
1094 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1095 nid = NUMA_NO_NODE;
1096
1097 for (; idx_a < type_a->cnt; idx_a++) {
1098 struct memblock_region *m = &type_a->regions[idx_a];
1099
1100 phys_addr_t m_start = m->base;
1101 phys_addr_t m_end = m->base + m->size;
1102 int m_nid = memblock_get_region_node(m);
1103
1104 if (should_skip_region(type_a, m, nid, flags))
1105 continue;
1106
1107 if (!type_b) {
1108 if (out_start)
1109 *out_start = m_start;
1110 if (out_end)
1111 *out_end = m_end;
1112 if (out_nid)
1113 *out_nid = m_nid;
1114 idx_a++;
1115 *idx = (u32)idx_a | (u64)idx_b << 32;
1116 return;
1117 }
1118
1119 /* scan areas before each reservation */
1120 for (; idx_b < type_b->cnt + 1; idx_b++) {
1121 struct memblock_region *r;
1122 phys_addr_t r_start;
1123 phys_addr_t r_end;
1124
1125 r = &type_b->regions[idx_b];
1126 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1127 r_end = idx_b < type_b->cnt ?
1128 r->base : PHYS_ADDR_MAX;
1129
1130 /*
1131 * if idx_b advanced past idx_a,
1132 * break out to advance idx_a
1133 */
1134 if (r_start >= m_end)
1135 break;
1136 /* if the two regions intersect, we're done */
1137 if (m_start < r_end) {
1138 if (out_start)
1139 *out_start =
1140 max(m_start, r_start);
1141 if (out_end)
1142 *out_end = min(m_end, r_end);
1143 if (out_nid)
1144 *out_nid = m_nid;
1145 /*
1146 * The region which ends first is
1147 * advanced for the next iteration.
1148 */
1149 if (m_end <= r_end)
1150 idx_a++;
1151 else
1152 idx_b++;
1153 *idx = (u32)idx_a | (u64)idx_b << 32;
1154 return;
1155 }
1156 }
1157 }
1158
1159 /* signal end of iteration */
1160 *idx = ULLONG_MAX;
1161 }
1162
1163 /**
1164 * __next_mem_range_rev - generic next function for for_each_*_range_rev()
1165 *
1166 * @idx: pointer to u64 loop variable
1167 * @nid: node selector, %NUMA_NO_NODE for all nodes
1168 * @flags: pick from blocks based on memory attributes
1169 * @type_a: pointer to memblock_type from where the range is taken
1170 * @type_b: pointer to memblock_type which excludes memory from being taken
1171 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1172 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1173 * @out_nid: ptr to int for nid of the range, can be %NULL
1174 *
1175 * Finds the next range from type_a which is not marked as unsuitable
1176 * in type_b.
1177 *
1178 * Reverse of __next_mem_range().
1179 */
__next_mem_range_rev(u64 * idx,int nid,enum memblock_flags flags,struct memblock_type * type_a,struct memblock_type * type_b,phys_addr_t * out_start,phys_addr_t * out_end,int * out_nid)1180 void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
1181 enum memblock_flags flags,
1182 struct memblock_type *type_a,
1183 struct memblock_type *type_b,
1184 phys_addr_t *out_start,
1185 phys_addr_t *out_end, int *out_nid)
1186 {
1187 int idx_a = *idx & 0xffffffff;
1188 int idx_b = *idx >> 32;
1189
1190 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1191 nid = NUMA_NO_NODE;
1192
1193 if (*idx == (u64)ULLONG_MAX) {
1194 idx_a = type_a->cnt - 1;
1195 if (type_b != NULL)
1196 idx_b = type_b->cnt;
1197 else
1198 idx_b = 0;
1199 }
1200
1201 for (; idx_a >= 0; idx_a--) {
1202 struct memblock_region *m = &type_a->regions[idx_a];
1203
1204 phys_addr_t m_start = m->base;
1205 phys_addr_t m_end = m->base + m->size;
1206 int m_nid = memblock_get_region_node(m);
1207
1208 if (should_skip_region(type_a, m, nid, flags))
1209 continue;
1210
1211 if (!type_b) {
1212 if (out_start)
1213 *out_start = m_start;
1214 if (out_end)
1215 *out_end = m_end;
1216 if (out_nid)
1217 *out_nid = m_nid;
1218 idx_a--;
1219 *idx = (u32)idx_a | (u64)idx_b << 32;
1220 return;
1221 }
1222
1223 /* scan areas before each reservation */
1224 for (; idx_b >= 0; idx_b--) {
1225 struct memblock_region *r;
1226 phys_addr_t r_start;
1227 phys_addr_t r_end;
1228
1229 r = &type_b->regions[idx_b];
1230 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1231 r_end = idx_b < type_b->cnt ?
1232 r->base : PHYS_ADDR_MAX;
1233 /*
1234 * if idx_b advanced past idx_a,
1235 * break out to advance idx_a
1236 */
1237
1238 if (r_end <= m_start)
1239 break;
1240 /* if the two regions intersect, we're done */
1241 if (m_end > r_start) {
1242 if (out_start)
1243 *out_start = max(m_start, r_start);
1244 if (out_end)
1245 *out_end = min(m_end, r_end);
1246 if (out_nid)
1247 *out_nid = m_nid;
1248 if (m_start >= r_start)
1249 idx_a--;
1250 else
1251 idx_b--;
1252 *idx = (u32)idx_a | (u64)idx_b << 32;
1253 return;
1254 }
1255 }
1256 }
1257 /* signal end of iteration */
1258 *idx = ULLONG_MAX;
1259 }
1260
1261 /*
1262 * Common iterator interface used to define for_each_mem_pfn_range().
1263 */
__next_mem_pfn_range(int * idx,int nid,unsigned long * out_start_pfn,unsigned long * out_end_pfn,int * out_nid)1264 void __init_memblock __next_mem_pfn_range(int *idx, int nid,
1265 unsigned long *out_start_pfn,
1266 unsigned long *out_end_pfn, int *out_nid)
1267 {
1268 struct memblock_type *type = &memblock.memory;
1269 struct memblock_region *r;
1270 int r_nid;
1271
1272 while (++*idx < type->cnt) {
1273 r = &type->regions[*idx];
1274 r_nid = memblock_get_region_node(r);
1275
1276 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
1277 continue;
1278 if (nid == MAX_NUMNODES || nid == r_nid)
1279 break;
1280 }
1281 if (*idx >= type->cnt) {
1282 *idx = -1;
1283 return;
1284 }
1285
1286 if (out_start_pfn)
1287 *out_start_pfn = PFN_UP(r->base);
1288 if (out_end_pfn)
1289 *out_end_pfn = PFN_DOWN(r->base + r->size);
1290 if (out_nid)
1291 *out_nid = r_nid;
1292 }
1293
1294 /**
1295 * memblock_set_node - set node ID on memblock regions
1296 * @base: base of area to set node ID for
1297 * @size: size of area to set node ID for
1298 * @type: memblock type to set node ID for
1299 * @nid: node ID to set
1300 *
1301 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
1302 * Regions which cross the area boundaries are split as necessary.
1303 *
1304 * Return:
1305 * 0 on success, -errno on failure.
1306 */
memblock_set_node(phys_addr_t base,phys_addr_t size,struct memblock_type * type,int nid)1307 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
1308 struct memblock_type *type, int nid)
1309 {
1310 #ifdef CONFIG_NUMA
1311 int start_rgn, end_rgn;
1312 int i, ret;
1313
1314 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
1315 if (ret)
1316 return ret;
1317
1318 for (i = start_rgn; i < end_rgn; i++)
1319 memblock_set_region_node(&type->regions[i], nid);
1320
1321 memblock_merge_regions(type, start_rgn, end_rgn);
1322 #endif
1323 return 0;
1324 }
1325
1326 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1327 /**
1328 * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
1329 *
1330 * @idx: pointer to u64 loop variable
1331 * @zone: zone in which all of the memory blocks reside
1332 * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
1333 * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
1334 *
1335 * This function is meant to be a zone/pfn specific wrapper for the
1336 * for_each_mem_range type iterators. Specifically they are used in the
1337 * deferred memory init routines and as such we were duplicating much of
1338 * this logic throughout the code. So instead of having it in multiple
1339 * locations it seemed like it would make more sense to centralize this to
1340 * one new iterator that does everything they need.
1341 */
1342 void __init_memblock
__next_mem_pfn_range_in_zone(u64 * idx,struct zone * zone,unsigned long * out_spfn,unsigned long * out_epfn)1343 __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
1344 unsigned long *out_spfn, unsigned long *out_epfn)
1345 {
1346 int zone_nid = zone_to_nid(zone);
1347 phys_addr_t spa, epa;
1348
1349 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1350 &memblock.memory, &memblock.reserved,
1351 &spa, &epa, NULL);
1352
1353 while (*idx != U64_MAX) {
1354 unsigned long epfn = PFN_DOWN(epa);
1355 unsigned long spfn = PFN_UP(spa);
1356
1357 /*
1358 * Verify the end is at least past the start of the zone and
1359 * that we have at least one PFN to initialize.
1360 */
1361 if (zone->zone_start_pfn < epfn && spfn < epfn) {
1362 /* if we went too far just stop searching */
1363 if (zone_end_pfn(zone) <= spfn) {
1364 *idx = U64_MAX;
1365 break;
1366 }
1367
1368 if (out_spfn)
1369 *out_spfn = max(zone->zone_start_pfn, spfn);
1370 if (out_epfn)
1371 *out_epfn = min(zone_end_pfn(zone), epfn);
1372
1373 return;
1374 }
1375
1376 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1377 &memblock.memory, &memblock.reserved,
1378 &spa, &epa, NULL);
1379 }
1380
1381 /* signal end of iteration */
1382 if (out_spfn)
1383 *out_spfn = ULONG_MAX;
1384 if (out_epfn)
1385 *out_epfn = 0;
1386 }
1387
1388 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1389
1390 /**
1391 * memblock_alloc_range_nid - allocate boot memory block
1392 * @size: size of memory block to be allocated in bytes
1393 * @align: alignment of the region and block's size
1394 * @start: the lower bound of the memory region to allocate (phys address)
1395 * @end: the upper bound of the memory region to allocate (phys address)
1396 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1397 * @exact_nid: control the allocation fall back to other nodes
1398 *
1399 * The allocation is performed from memory region limited by
1400 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
1401 *
1402 * If the specified node can not hold the requested memory and @exact_nid
1403 * is false, the allocation falls back to any node in the system.
1404 *
1405 * For systems with memory mirroring, the allocation is attempted first
1406 * from the regions with mirroring enabled and then retried from any
1407 * memory region.
1408 *
1409 * In addition, function using kmemleak_alloc_phys for allocated boot
1410 * memory block, it is never reported as leaks.
1411 *
1412 * Return:
1413 * Physical address of allocated memory block on success, %0 on failure.
1414 */
memblock_alloc_range_nid(phys_addr_t size,phys_addr_t align,phys_addr_t start,phys_addr_t end,int nid,bool exact_nid)1415 phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
1416 phys_addr_t align, phys_addr_t start,
1417 phys_addr_t end, int nid,
1418 bool exact_nid)
1419 {
1420 enum memblock_flags flags = choose_memblock_flags();
1421 phys_addr_t found;
1422
1423 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1424 nid = NUMA_NO_NODE;
1425
1426 if (!align) {
1427 /* Can't use WARNs this early in boot on powerpc */
1428 dump_stack();
1429 align = SMP_CACHE_BYTES;
1430 }
1431
1432 again:
1433 found = memblock_find_in_range_node(size, align, start, end, nid,
1434 flags);
1435 if (found && !memblock_reserve(found, size))
1436 goto done;
1437
1438 if (nid != NUMA_NO_NODE && !exact_nid) {
1439 found = memblock_find_in_range_node(size, align, start,
1440 end, NUMA_NO_NODE,
1441 flags);
1442 if (found && !memblock_reserve(found, size))
1443 goto done;
1444 }
1445
1446 if (flags & MEMBLOCK_MIRROR) {
1447 flags &= ~MEMBLOCK_MIRROR;
1448 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n",
1449 &size);
1450 goto again;
1451 }
1452
1453 return 0;
1454
1455 done:
1456 /*
1457 * Skip kmemleak for those places like kasan_init() and
1458 * early_pgtable_alloc() due to high volume.
1459 */
1460 if (end != MEMBLOCK_ALLOC_NOLEAKTRACE)
1461 /*
1462 * Memblock allocated blocks are never reported as
1463 * leaks. This is because many of these blocks are
1464 * only referred via the physical address which is
1465 * not looked up by kmemleak.
1466 */
1467 kmemleak_alloc_phys(found, size, 0);
1468
1469 /*
1470 * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP,
1471 * require memory to be accepted before it can be used by the
1472 * guest.
1473 *
1474 * Accept the memory of the allocated buffer.
1475 */
1476 accept_memory(found, found + size);
1477
1478 return found;
1479 }
1480
1481 /**
1482 * memblock_phys_alloc_range - allocate a memory block inside specified range
1483 * @size: size of memory block to be allocated in bytes
1484 * @align: alignment of the region and block's size
1485 * @start: the lower bound of the memory region to allocate (physical address)
1486 * @end: the upper bound of the memory region to allocate (physical address)
1487 *
1488 * Allocate @size bytes in the between @start and @end.
1489 *
1490 * Return: physical address of the allocated memory block on success,
1491 * %0 on failure.
1492 */
memblock_phys_alloc_range(phys_addr_t size,phys_addr_t align,phys_addr_t start,phys_addr_t end)1493 phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
1494 phys_addr_t align,
1495 phys_addr_t start,
1496 phys_addr_t end)
1497 {
1498 memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n",
1499 __func__, (u64)size, (u64)align, &start, &end,
1500 (void *)_RET_IP_);
1501 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1502 false);
1503 }
1504
1505 /**
1506 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node
1507 * @size: size of memory block to be allocated in bytes
1508 * @align: alignment of the region and block's size
1509 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1510 *
1511 * Allocates memory block from the specified NUMA node. If the node
1512 * has no available memory, attempts to allocated from any node in the
1513 * system.
1514 *
1515 * Return: physical address of the allocated memory block on success,
1516 * %0 on failure.
1517 */
memblock_phys_alloc_try_nid(phys_addr_t size,phys_addr_t align,int nid)1518 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
1519 {
1520 return memblock_alloc_range_nid(size, align, 0,
1521 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
1522 }
1523
1524 /**
1525 * memblock_alloc_internal - allocate boot memory block
1526 * @size: size of memory block to be allocated in bytes
1527 * @align: alignment of the region and block's size
1528 * @min_addr: the lower bound of the memory region to allocate (phys address)
1529 * @max_addr: the upper bound of the memory region to allocate (phys address)
1530 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1531 * @exact_nid: control the allocation fall back to other nodes
1532 *
1533 * Allocates memory block using memblock_alloc_range_nid() and
1534 * converts the returned physical address to virtual.
1535 *
1536 * The @min_addr limit is dropped if it can not be satisfied and the allocation
1537 * will fall back to memory below @min_addr. Other constraints, such
1538 * as node and mirrored memory will be handled again in
1539 * memblock_alloc_range_nid().
1540 *
1541 * Return:
1542 * Virtual address of allocated memory block on success, NULL on failure.
1543 */
memblock_alloc_internal(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid,bool exact_nid)1544 static void * __init memblock_alloc_internal(
1545 phys_addr_t size, phys_addr_t align,
1546 phys_addr_t min_addr, phys_addr_t max_addr,
1547 int nid, bool exact_nid)
1548 {
1549 phys_addr_t alloc;
1550
1551 /*
1552 * Detect any accidental use of these APIs after slab is ready, as at
1553 * this moment memblock may be deinitialized already and its
1554 * internal data may be destroyed (after execution of memblock_free_all)
1555 */
1556 if (WARN_ON_ONCE(slab_is_available()))
1557 return kzalloc_node(size, GFP_NOWAIT, nid);
1558
1559 if (max_addr > memblock.current_limit)
1560 max_addr = memblock.current_limit;
1561
1562 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
1563 exact_nid);
1564
1565 /* retry allocation without lower limit */
1566 if (!alloc && min_addr)
1567 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
1568 exact_nid);
1569
1570 if (!alloc)
1571 return NULL;
1572
1573 return phys_to_virt(alloc);
1574 }
1575
1576 /**
1577 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
1578 * without zeroing memory
1579 * @size: size of memory block to be allocated in bytes
1580 * @align: alignment of the region and block's size
1581 * @min_addr: the lower bound of the memory region from where the allocation
1582 * is preferred (phys address)
1583 * @max_addr: the upper bound of the memory region from where the allocation
1584 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1585 * allocate only from memory limited by memblock.current_limit value
1586 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1587 *
1588 * Public function, provides additional debug information (including caller
1589 * info), if enabled. Does not zero allocated memory.
1590 *
1591 * Return:
1592 * Virtual address of allocated memory block on success, NULL on failure.
1593 */
memblock_alloc_exact_nid_raw(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid)1594 void * __init memblock_alloc_exact_nid_raw(
1595 phys_addr_t size, phys_addr_t align,
1596 phys_addr_t min_addr, phys_addr_t max_addr,
1597 int nid)
1598 {
1599 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1600 __func__, (u64)size, (u64)align, nid, &min_addr,
1601 &max_addr, (void *)_RET_IP_);
1602
1603 return memblock_alloc_internal(size, align, min_addr, max_addr, nid,
1604 true);
1605 }
1606
1607 /**
1608 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
1609 * memory and without panicking
1610 * @size: size of memory block to be allocated in bytes
1611 * @align: alignment of the region and block's size
1612 * @min_addr: the lower bound of the memory region from where the allocation
1613 * is preferred (phys address)
1614 * @max_addr: the upper bound of the memory region from where the allocation
1615 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1616 * allocate only from memory limited by memblock.current_limit value
1617 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1618 *
1619 * Public function, provides additional debug information (including caller
1620 * info), if enabled. Does not zero allocated memory, does not panic if request
1621 * cannot be satisfied.
1622 *
1623 * Return:
1624 * Virtual address of allocated memory block on success, NULL on failure.
1625 */
memblock_alloc_try_nid_raw(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid)1626 void * __init memblock_alloc_try_nid_raw(
1627 phys_addr_t size, phys_addr_t align,
1628 phys_addr_t min_addr, phys_addr_t max_addr,
1629 int nid)
1630 {
1631 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1632 __func__, (u64)size, (u64)align, nid, &min_addr,
1633 &max_addr, (void *)_RET_IP_);
1634
1635 return memblock_alloc_internal(size, align, min_addr, max_addr, nid,
1636 false);
1637 }
1638
1639 /**
1640 * memblock_alloc_try_nid - allocate boot memory block
1641 * @size: size of memory block to be allocated in bytes
1642 * @align: alignment of the region and block's size
1643 * @min_addr: the lower bound of the memory region from where the allocation
1644 * is preferred (phys address)
1645 * @max_addr: the upper bound of the memory region from where the allocation
1646 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1647 * allocate only from memory limited by memblock.current_limit value
1648 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1649 *
1650 * Public function, provides additional debug information (including caller
1651 * info), if enabled. This function zeroes the allocated memory.
1652 *
1653 * Return:
1654 * Virtual address of allocated memory block on success, NULL on failure.
1655 */
memblock_alloc_try_nid(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid)1656 void * __init memblock_alloc_try_nid(
1657 phys_addr_t size, phys_addr_t align,
1658 phys_addr_t min_addr, phys_addr_t max_addr,
1659 int nid)
1660 {
1661 void *ptr;
1662
1663 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1664 __func__, (u64)size, (u64)align, nid, &min_addr,
1665 &max_addr, (void *)_RET_IP_);
1666 ptr = memblock_alloc_internal(size, align,
1667 min_addr, max_addr, nid, false);
1668 if (ptr)
1669 memset(ptr, 0, size);
1670
1671 return ptr;
1672 }
1673
1674 /**
1675 * memblock_free_late - free pages directly to buddy allocator
1676 * @base: phys starting address of the boot memory block
1677 * @size: size of the boot memory block in bytes
1678 *
1679 * This is only useful when the memblock allocator has already been torn
1680 * down, but we are still initializing the system. Pages are released directly
1681 * to the buddy allocator.
1682 */
memblock_free_late(phys_addr_t base,phys_addr_t size)1683 void __init memblock_free_late(phys_addr_t base, phys_addr_t size)
1684 {
1685 phys_addr_t cursor, end;
1686
1687 end = base + size - 1;
1688 memblock_dbg("%s: [%pa-%pa] %pS\n",
1689 __func__, &base, &end, (void *)_RET_IP_);
1690 kmemleak_free_part_phys(base, size);
1691 cursor = PFN_UP(base);
1692 end = PFN_DOWN(base + size);
1693
1694 memblock_memsize_mod_kernel_size(-1 * ((long)(end - cursor) << PAGE_SHIFT));
1695
1696 for (; cursor < end; cursor++) {
1697 memblock_free_pages(pfn_to_page(cursor), cursor, 0);
1698 totalram_pages_inc();
1699 }
1700 }
1701
1702 /*
1703 * Remaining API functions
1704 */
1705
memblock_phys_mem_size(void)1706 phys_addr_t __init_memblock memblock_phys_mem_size(void)
1707 {
1708 return memblock.memory.total_size;
1709 }
1710
memblock_reserved_size(void)1711 phys_addr_t __init_memblock memblock_reserved_size(void)
1712 {
1713 return memblock.reserved.total_size;
1714 }
1715
1716 /* lowest address */
memblock_start_of_DRAM(void)1717 phys_addr_t __init_memblock memblock_start_of_DRAM(void)
1718 {
1719 return memblock.memory.regions[0].base;
1720 }
1721
memblock_end_of_DRAM(void)1722 phys_addr_t __init_memblock memblock_end_of_DRAM(void)
1723 {
1724 int idx = memblock.memory.cnt - 1;
1725
1726 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
1727 }
1728 EXPORT_SYMBOL_GPL(memblock_end_of_DRAM);
1729
__find_max_addr(phys_addr_t limit)1730 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
1731 {
1732 phys_addr_t max_addr = PHYS_ADDR_MAX;
1733 struct memblock_region *r;
1734
1735 /*
1736 * translate the memory @limit size into the max address within one of
1737 * the memory memblock regions, if the @limit exceeds the total size
1738 * of those regions, max_addr will keep original value PHYS_ADDR_MAX
1739 */
1740 for_each_mem_region(r) {
1741 if (limit <= r->size) {
1742 max_addr = r->base + limit;
1743 break;
1744 }
1745 limit -= r->size;
1746 }
1747
1748 return max_addr;
1749 }
1750
memblock_enforce_memory_limit(phys_addr_t limit)1751 void __init memblock_enforce_memory_limit(phys_addr_t limit)
1752 {
1753 phys_addr_t max_addr;
1754
1755 if (!limit)
1756 return;
1757
1758 max_addr = __find_max_addr(limit);
1759
1760 /* @limit exceeds the total size of the memory, do nothing */
1761 if (max_addr == PHYS_ADDR_MAX)
1762 return;
1763
1764 /* truncate both memory and reserved regions */
1765 memblock_remove_range(&memblock.memory, max_addr,
1766 PHYS_ADDR_MAX);
1767 memblock_remove_range(&memblock.reserved, max_addr,
1768 PHYS_ADDR_MAX);
1769 }
1770
memblock_cap_memory_range(phys_addr_t base,phys_addr_t size)1771 void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
1772 {
1773 int start_rgn, end_rgn;
1774 int i, ret;
1775
1776 if (!size)
1777 return;
1778
1779 if (!memblock_memory->total_size) {
1780 pr_warn("%s: No memory registered yet\n", __func__);
1781 return;
1782 }
1783
1784 ret = memblock_isolate_range(&memblock.memory, base, size,
1785 &start_rgn, &end_rgn);
1786 if (ret)
1787 return;
1788
1789 /* remove all the MAP regions */
1790 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
1791 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1792 memblock_remove_region(&memblock.memory, i);
1793
1794 for (i = start_rgn - 1; i >= 0; i--)
1795 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1796 memblock_remove_region(&memblock.memory, i);
1797
1798 /* truncate the reserved regions */
1799 memblock_remove_range(&memblock.reserved, 0, base);
1800 memblock_remove_range(&memblock.reserved,
1801 base + size, PHYS_ADDR_MAX);
1802 }
1803
memblock_mem_limit_remove_map(phys_addr_t limit)1804 void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1805 {
1806 phys_addr_t max_addr;
1807
1808 if (!limit)
1809 return;
1810
1811 max_addr = __find_max_addr(limit);
1812
1813 /* @limit exceeds the total size of the memory, do nothing */
1814 if (max_addr == PHYS_ADDR_MAX)
1815 return;
1816
1817 memblock_cap_memory_range(0, max_addr);
1818 }
1819
memblock_search(struct memblock_type * type,phys_addr_t addr)1820 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
1821 {
1822 unsigned int left = 0, right = type->cnt;
1823
1824 do {
1825 unsigned int mid = (right + left) / 2;
1826
1827 if (addr < type->regions[mid].base)
1828 right = mid;
1829 else if (addr >= (type->regions[mid].base +
1830 type->regions[mid].size))
1831 left = mid + 1;
1832 else
1833 return mid;
1834 } while (left < right);
1835 return -1;
1836 }
1837
memblock_is_reserved(phys_addr_t addr)1838 bool __init_memblock memblock_is_reserved(phys_addr_t addr)
1839 {
1840 return memblock_search(&memblock.reserved, addr) != -1;
1841 }
1842
memblock_is_memory(phys_addr_t addr)1843 bool __init_memblock memblock_is_memory(phys_addr_t addr)
1844 {
1845 return memblock_search(&memblock.memory, addr) != -1;
1846 }
1847
memblock_is_map_memory(phys_addr_t addr)1848 bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
1849 {
1850 int i = memblock_search(&memblock.memory, addr);
1851
1852 if (i == -1)
1853 return false;
1854 return !memblock_is_nomap(&memblock.memory.regions[i]);
1855 }
1856
memblock_search_pfn_nid(unsigned long pfn,unsigned long * start_pfn,unsigned long * end_pfn)1857 int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
1858 unsigned long *start_pfn, unsigned long *end_pfn)
1859 {
1860 struct memblock_type *type = &memblock.memory;
1861 int mid = memblock_search(type, PFN_PHYS(pfn));
1862
1863 if (mid == -1)
1864 return -1;
1865
1866 *start_pfn = PFN_DOWN(type->regions[mid].base);
1867 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
1868
1869 return memblock_get_region_node(&type->regions[mid]);
1870 }
1871
1872 /**
1873 * memblock_is_region_memory - check if a region is a subset of memory
1874 * @base: base of region to check
1875 * @size: size of region to check
1876 *
1877 * Check if the region [@base, @base + @size) is a subset of a memory block.
1878 *
1879 * Return:
1880 * 0 if false, non-zero if true
1881 */
memblock_is_region_memory(phys_addr_t base,phys_addr_t size)1882 bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
1883 {
1884 int idx = memblock_search(&memblock.memory, base);
1885 phys_addr_t end = base + memblock_cap_size(base, &size);
1886
1887 if (idx == -1)
1888 return false;
1889 return (memblock.memory.regions[idx].base +
1890 memblock.memory.regions[idx].size) >= end;
1891 }
1892
1893 /**
1894 * memblock_is_region_reserved - check if a region intersects reserved memory
1895 * @base: base of region to check
1896 * @size: size of region to check
1897 *
1898 * Check if the region [@base, @base + @size) intersects a reserved
1899 * memory block.
1900 *
1901 * Return:
1902 * True if they intersect, false if not.
1903 */
memblock_is_region_reserved(phys_addr_t base,phys_addr_t size)1904 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
1905 {
1906 return memblock_overlaps_region(&memblock.reserved, base, size);
1907 }
1908
memblock_trim_memory(phys_addr_t align)1909 void __init_memblock memblock_trim_memory(phys_addr_t align)
1910 {
1911 phys_addr_t start, end, orig_start, orig_end;
1912 struct memblock_region *r;
1913
1914 for_each_mem_region(r) {
1915 orig_start = r->base;
1916 orig_end = r->base + r->size;
1917 start = round_up(orig_start, align);
1918 end = round_down(orig_end, align);
1919
1920 if (start == orig_start && end == orig_end)
1921 continue;
1922
1923 if (start < end) {
1924 r->base = start;
1925 r->size = end - start;
1926 } else {
1927 memblock_remove_region(&memblock.memory,
1928 r - memblock.memory.regions);
1929 r--;
1930 }
1931 }
1932 }
1933
memblock_set_current_limit(phys_addr_t limit)1934 void __init_memblock memblock_set_current_limit(phys_addr_t limit)
1935 {
1936 memblock.current_limit = limit;
1937 }
1938
memblock_get_current_limit(void)1939 phys_addr_t __init_memblock memblock_get_current_limit(void)
1940 {
1941 return memblock.current_limit;
1942 }
1943
memblock_dump(struct memblock_type * type)1944 static void __init_memblock memblock_dump(struct memblock_type *type)
1945 {
1946 phys_addr_t base, end, size;
1947 enum memblock_flags flags;
1948 int idx;
1949 struct memblock_region *rgn;
1950
1951 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt);
1952
1953 for_each_memblock_type(idx, type, rgn) {
1954 char nid_buf[32] = "";
1955
1956 base = rgn->base;
1957 size = rgn->size;
1958 end = base + size - 1;
1959 flags = rgn->flags;
1960 #ifdef CONFIG_NUMA
1961 if (memblock_get_region_node(rgn) != MAX_NUMNODES)
1962 snprintf(nid_buf, sizeof(nid_buf), " on node %d",
1963 memblock_get_region_node(rgn));
1964 #endif
1965 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
1966 type->name, idx, &base, &end, &size, nid_buf, flags);
1967 }
1968 }
1969
__memblock_dump_all(void)1970 static void __init_memblock __memblock_dump_all(void)
1971 {
1972 pr_info("MEMBLOCK configuration:\n");
1973 pr_info(" memory size = %pa reserved size = %pa\n",
1974 &memblock.memory.total_size,
1975 &memblock.reserved.total_size);
1976
1977 memblock_dump(&memblock.memory);
1978 memblock_dump(&memblock.reserved);
1979 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
1980 memblock_dump(&physmem);
1981 #endif
1982 }
1983
memblock_dump_all(void)1984 void __init_memblock memblock_dump_all(void)
1985 {
1986 if (memblock_debug)
1987 __memblock_dump_all();
1988 }
1989
memblock_allow_resize(void)1990 void __init memblock_allow_resize(void)
1991 {
1992 memblock_can_resize = 1;
1993 }
1994
early_memblock(char * p)1995 static int __init early_memblock(char *p)
1996 {
1997 if (p && strstr(p, "debug"))
1998 memblock_debug = 1;
1999 return 0;
2000 }
2001 early_param("memblock", early_memblock);
2002
2003 #define NAME_SIZE 100
2004 struct memsize_rgn_struct {
2005 phys_addr_t base;
2006 long size;
2007 bool nomap; /* 1/32 byte */
2008 bool reusable; /* 1/32 byte */
2009 char name[NAME_SIZE]; /* 30/32 byte */
2010 };
2011
2012 #define MAX_MEMBLOCK_MEMSIZE 100
2013
2014 static struct memsize_rgn_struct memsize_rgn[MAX_MEMBLOCK_MEMSIZE] __initdata_memblock;
2015 static int memsize_rgn_count __initdata_memblock;
2016 static long memsize_memmap;
2017 static unsigned long memsize_code __initdata_memblock;
2018 static unsigned long memsize_data __initdata_memblock;
2019 static unsigned long memsize_ro __initdata_memblock;
2020 static unsigned long memsize_bss __initdata_memblock;
2021 static long memsize_reusable_size;
2022
2023 enum memblock_memsize_state {
2024 MEMBLOCK_MEMSIZE_NONE = 0,
2025 MEMBLOCK_MEMSIZE_DEBUGFS,
2026 MEMBLOCK_MEMSIZE_PROCFS,
2027 };
2028
2029 static enum memblock_memsize_state memsize_state __initdata_memblock = MEMBLOCK_MEMSIZE_NONE;
2030
early_memblock_memsize(char * str)2031 static int __init early_memblock_memsize(char *str)
2032 {
2033 if (!str)
2034 return -EINVAL;
2035 if (strcmp(str, "none") == 0)
2036 memsize_state = MEMBLOCK_MEMSIZE_NONE;
2037 else if (strcmp(str, "debugfs") == 0)
2038 memsize_state = MEMBLOCK_MEMSIZE_DEBUGFS;
2039 else if (strcmp(str, "procfs") == 0)
2040 memsize_state = MEMBLOCK_MEMSIZE_PROCFS;
2041 else
2042 return -EINVAL;
2043 return 0;
2044 }
2045 early_param("memblock_memsize", early_memblock_memsize);
2046
memblock_memsize_enable_tracking(void)2047 void __init memblock_memsize_enable_tracking(void)
2048 {
2049 memblock_memsize_tracking = true;
2050 }
2051
memblock_memsize_disable_tracking(void)2052 void __init memblock_memsize_disable_tracking(void)
2053 {
2054 memblock_memsize_tracking = false;
2055 }
2056
memblock_memsize_mod_memmap_size(long size)2057 void __init memblock_memsize_mod_memmap_size(long size)
2058 {
2059 memsize_memmap += size;
2060 }
2061
memblock_memsize_mod_kernel_size(long size)2062 void memblock_memsize_mod_kernel_size(long size)
2063 {
2064 memsize_kinit += size;
2065 }
2066
memblock_memsize_kernel_code_data(unsigned long code,unsigned long data,unsigned long ro,unsigned long bss)2067 void __init memblock_memsize_kernel_code_data(unsigned long code, unsigned long data,
2068 unsigned long ro, unsigned long bss)
2069 {
2070 memsize_code = code;
2071 memsize_data = data;
2072 memsize_ro = ro;
2073 memsize_bss = bss;
2074 }
2075
memsize_get_valid_name(char * valid_name,const char * name)2076 static void __init_memblock memsize_get_valid_name(char *valid_name, const char *name)
2077 {
2078 char *head, *tail, *found;
2079 int valid_size;
2080
2081 head = (char *)name;
2082 tail = head + strlen(name);
2083
2084 /* get tail position after valid char */
2085 found = strchr(name, '@');
2086 if (found)
2087 tail = found;
2088
2089 valid_size = tail - head + 1;
2090 if (valid_size > NAME_SIZE)
2091 valid_size = NAME_SIZE;
2092 strscpy(valid_name, head, valid_size);
2093 }
2094
memblock_memsize_mod_reusable_size(long size)2095 void memblock_memsize_mod_reusable_size(long size)
2096 {
2097 memsize_reusable_size += size;
2098 }
2099
memsize_get_new_rgn(void)2100 static inline struct memsize_rgn_struct * __init_memblock memsize_get_new_rgn(void)
2101 {
2102 if (memsize_rgn_count == ARRAY_SIZE(memsize_rgn)) {
2103 pr_err("not enough space on memsize_rgn\n");
2104 return NULL;
2105 }
2106 return &memsize_rgn[memsize_rgn_count++];
2107 }
2108
memsize_update_nomap_region(const char * name,phys_addr_t base,phys_addr_t size,bool nomap)2109 static bool __init_memblock memsize_update_nomap_region(const char *name, phys_addr_t base,
2110 phys_addr_t size, bool nomap)
2111 {
2112 int i;
2113 struct memsize_rgn_struct *rmem_rgn, *new_rgn;
2114
2115 if (!name)
2116 return false;
2117
2118 for (i = 0; i < memsize_rgn_count; i++) {
2119 rmem_rgn = &memsize_rgn[i];
2120
2121 /* skip either !nomap, !unknown, !overlap */
2122 if (!rmem_rgn->nomap)
2123 continue;
2124 if (strcmp(rmem_rgn->name, "unknown"))
2125 continue;
2126 if (base + size <= rmem_rgn->base)
2127 continue;
2128 if (base >= rmem_rgn->base + rmem_rgn->size)
2129 continue;
2130
2131 /* exactly same */
2132 if (base == rmem_rgn->base && size == rmem_rgn->size) {
2133 memsize_get_valid_name(rmem_rgn->name, name);
2134 return true;
2135 }
2136
2137 /* bigger */
2138 if (base <= rmem_rgn->base &&
2139 base + size >= rmem_rgn->base + rmem_rgn->size) {
2140 memsize_get_valid_name(rmem_rgn->name, name);
2141 rmem_rgn->base = base;
2142 rmem_rgn->size = size;
2143 return true;
2144 }
2145
2146 /* intersect */
2147 if (base < rmem_rgn->base ||
2148 base + size > rmem_rgn->base + rmem_rgn->size) {
2149 new_rgn = memsize_get_new_rgn();
2150 if (!new_rgn)
2151 return true;
2152 new_rgn->base = base;
2153 new_rgn->size = size;
2154 new_rgn->nomap = nomap;
2155 new_rgn->reusable = false;
2156 memsize_get_valid_name(new_rgn->name, name);
2157
2158 if (base < rmem_rgn->base) {
2159 rmem_rgn->size -= base + size - rmem_rgn->base;
2160 rmem_rgn->base = base + size;
2161 } else {
2162 rmem_rgn->size -= rmem_rgn->base
2163 + rmem_rgn->size - base;
2164 }
2165 return true;
2166 }
2167
2168 /* smaller */
2169 new_rgn = memsize_get_new_rgn();
2170 if (!new_rgn)
2171 return true;
2172 new_rgn->base = base;
2173 new_rgn->size = size;
2174 new_rgn->nomap = nomap;
2175 new_rgn->reusable = false;
2176 memsize_get_valid_name(new_rgn->name, name);
2177
2178 if (base == rmem_rgn->base && size < rmem_rgn->size) {
2179 rmem_rgn->base = base + size;
2180 rmem_rgn->size -= size;
2181 } else if (base + size == rmem_rgn->base + rmem_rgn->size) {
2182 rmem_rgn->size -= size;
2183 } else {
2184 new_rgn = memsize_get_new_rgn();
2185 if (!new_rgn)
2186 return true;
2187 new_rgn->base = base + size;
2188 new_rgn->size = (rmem_rgn->base + rmem_rgn->size)
2189 - (base + size);
2190 new_rgn->nomap = nomap;
2191 new_rgn->reusable = false;
2192 strscpy(new_rgn->name, "unknown", sizeof(new_rgn->name));
2193 rmem_rgn->size = base - rmem_rgn->base;
2194 }
2195 return true;
2196 }
2197
2198 return false;
2199 }
2200
memblock_memsize_record(const char * name,phys_addr_t base,phys_addr_t size,bool nomap,bool reusable)2201 void __init_memblock memblock_memsize_record(const char *name, phys_addr_t base,
2202 phys_addr_t size, bool nomap, bool reusable)
2203 {
2204 struct memsize_rgn_struct *rgn;
2205 phys_addr_t end;
2206
2207 if (name && memsize_state == MEMBLOCK_MEMSIZE_NONE)
2208 return;
2209
2210 if (memsize_rgn_count == MAX_MEMBLOCK_MEMSIZE) {
2211 pr_err("not enough space on memsize_rgn\n");
2212 return;
2213 }
2214
2215 if (memsize_update_nomap_region(name, base, size, nomap))
2216 return;
2217
2218 rgn = memsize_get_new_rgn();
2219 if (!rgn)
2220 return;
2221
2222 rgn->base = base;
2223 rgn->size = size;
2224 rgn->nomap = nomap;
2225 rgn->reusable = reusable;
2226
2227 if (!name)
2228 strscpy(rgn->name, "unknown", sizeof(rgn->name));
2229 else
2230 memsize_get_valid_name(rgn->name, name);
2231 end = base + size - 1;
2232 memblock_dbg("%s %pa..%pa nomap:%d reusable:%d\n",
2233 __func__, &base, &end, nomap, reusable);
2234 }
2235
memblock_memsize_detect_hole(void)2236 void __init memblock_memsize_detect_hole(void)
2237 {
2238 phys_addr_t base, end;
2239 phys_addr_t prev_end, hole_sz;
2240 int idx;
2241 struct memblock_region *rgn;
2242 int memblock_cnt = (int)memblock.memory.cnt;
2243
2244 /* assume that the hole size is less than 1 GB */
2245 for_each_memblock_type(idx, (&memblock.memory), rgn) {
2246 prev_end = (idx == 0) ? round_down(rgn->base, SZ_1G) : end;
2247 base = rgn->base;
2248 end = rgn->base + rgn->size;
2249
2250 /* only for the last region, check a hole after the region */
2251 if (idx + 1 == memblock_cnt) {
2252 hole_sz = round_up(end, SZ_1G) - end;
2253 if (hole_sz)
2254 memblock_memsize_record(NULL, end, hole_sz,
2255 true, false);
2256 }
2257
2258 /* for each region, check a hole prior to the region */
2259 hole_sz = base - prev_end;
2260 if (!hole_sz)
2261 continue;
2262 if (hole_sz < SZ_1G) {
2263 memblock_memsize_record(NULL, prev_end, hole_sz, true,
2264 false);
2265 } else {
2266 phys_addr_t hole_sz1, hole_sz2;
2267
2268 hole_sz1 = round_up(prev_end, SZ_1G) - prev_end;
2269 if (hole_sz1)
2270 memblock_memsize_record(NULL, prev_end,
2271 hole_sz1, true, false);
2272 hole_sz2 = base % SZ_1G;
2273 if (hole_sz2)
2274 memblock_memsize_record(NULL, base - hole_sz2,
2275 hole_sz2, true, false);
2276 }
2277 }
2278 }
2279
free_memmap(unsigned long start_pfn,unsigned long end_pfn)2280 static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn)
2281 {
2282 struct page *start_pg, *end_pg;
2283 phys_addr_t pg, pgend;
2284
2285 /*
2286 * Convert start_pfn/end_pfn to a struct page pointer.
2287 */
2288 start_pg = pfn_to_page(start_pfn - 1) + 1;
2289 end_pg = pfn_to_page(end_pfn - 1) + 1;
2290
2291 /*
2292 * Convert to physical addresses, and round start upwards and end
2293 * downwards.
2294 */
2295 pg = PAGE_ALIGN(__pa(start_pg));
2296 pgend = __pa(end_pg) & PAGE_MASK;
2297
2298 /*
2299 * If there are free pages between these, free the section of the
2300 * memmap array.
2301 */
2302 if (pg < pgend)
2303 memblock_phys_free(pg, pgend - pg);
2304 }
2305
2306 /*
2307 * The mem_map array can get very big. Free the unused area of the memory map.
2308 */
free_unused_memmap(void)2309 static void __init free_unused_memmap(void)
2310 {
2311 unsigned long start, end, prev_end = 0;
2312 int i;
2313
2314 if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) ||
2315 IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))
2316 return;
2317
2318 /*
2319 * This relies on each bank being in address order.
2320 * The banks are sorted previously in bootmem_init().
2321 */
2322 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) {
2323 #ifdef CONFIG_SPARSEMEM
2324 /*
2325 * Take care not to free memmap entries that don't exist
2326 * due to SPARSEMEM sections which aren't present.
2327 */
2328 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION));
2329 #endif
2330 /*
2331 * Align down here since many operations in VM subsystem
2332 * presume that there are no holes in the memory map inside
2333 * a pageblock
2334 */
2335 start = pageblock_start_pfn(start);
2336
2337 /*
2338 * If we had a previous bank, and there is a space
2339 * between the current bank and the previous, free it.
2340 */
2341 if (prev_end && prev_end < start)
2342 free_memmap(prev_end, start);
2343
2344 /*
2345 * Align up here since many operations in VM subsystem
2346 * presume that there are no holes in the memory map inside
2347 * a pageblock
2348 */
2349 prev_end = pageblock_align(end);
2350 }
2351
2352 #ifdef CONFIG_SPARSEMEM
2353 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) {
2354 prev_end = pageblock_align(end);
2355 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
2356 }
2357 #endif
2358 }
2359
__free_pages_memory(unsigned long start,unsigned long end)2360 static void __init __free_pages_memory(unsigned long start, unsigned long end)
2361 {
2362 int order;
2363
2364 while (start < end) {
2365 /*
2366 * Free the pages in the largest chunks alignment allows.
2367 *
2368 * __ffs() behaviour is undefined for 0. start == 0 is
2369 * MAX_ORDER-aligned, set order to MAX_ORDER for the case.
2370 */
2371 if (start)
2372 order = min_t(int, MAX_ORDER, __ffs(start));
2373 else
2374 order = MAX_ORDER;
2375
2376 while (start + (1UL << order) > end)
2377 order--;
2378
2379 memblock_free_pages(pfn_to_page(start), start, order);
2380
2381 start += (1UL << order);
2382 }
2383 }
2384
__free_memory_core(phys_addr_t start,phys_addr_t end)2385 static unsigned long __init __free_memory_core(phys_addr_t start,
2386 phys_addr_t end)
2387 {
2388 unsigned long start_pfn = PFN_UP(start);
2389 unsigned long end_pfn = min_t(unsigned long,
2390 PFN_DOWN(end), max_low_pfn);
2391
2392 unsigned long start_align_up = PFN_ALIGN(start);
2393 unsigned long end_align_down = PFN_PHYS(end_pfn);
2394
2395 if (start_pfn >= end_pfn) {
2396 memblock_memsize_mod_kernel_size(end - start);
2397 } else {
2398 if (start_align_up > start)
2399 memblock_memsize_mod_kernel_size(start_align_up - start);
2400 if (end_pfn != max_low_pfn && end_align_down < end)
2401 memblock_memsize_mod_kernel_size(end - end_align_down);
2402 }
2403 if (start_pfn >= end_pfn)
2404 return 0;
2405
2406 __free_pages_memory(start_pfn, end_pfn);
2407
2408 return end_pfn - start_pfn;
2409 }
2410
memmap_init_reserved_pages(void)2411 static void __init memmap_init_reserved_pages(void)
2412 {
2413 struct memblock_region *region;
2414 phys_addr_t start, end;
2415 int nid;
2416
2417 /*
2418 * set nid on all reserved pages and also treat struct
2419 * pages for the NOMAP regions as PageReserved
2420 */
2421 for_each_mem_region(region) {
2422 nid = memblock_get_region_node(region);
2423 start = region->base;
2424 end = start + region->size;
2425
2426 if (memblock_is_nomap(region))
2427 reserve_bootmem_region(start, end, nid);
2428
2429 memblock_set_node(start, end, &memblock.reserved, nid);
2430 }
2431
2432 /* initialize struct pages for the reserved regions */
2433 for_each_reserved_mem_region(region) {
2434 nid = memblock_get_region_node(region);
2435 start = region->base;
2436 end = start + region->size;
2437
2438 if (nid == NUMA_NO_NODE || nid >= MAX_NUMNODES)
2439 nid = early_pfn_to_nid(PFN_DOWN(start));
2440
2441 reserve_bootmem_region(start, end, nid);
2442 }
2443 }
2444
free_low_memory_core_early(void)2445 static unsigned long __init free_low_memory_core_early(void)
2446 {
2447 unsigned long count = 0;
2448 phys_addr_t start, end;
2449 u64 i;
2450
2451 memblock_clear_hotplug(0, -1);
2452
2453 memmap_init_reserved_pages();
2454
2455 /*
2456 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
2457 * because in some case like Node0 doesn't have RAM installed
2458 * low ram will be on Node1
2459 */
2460 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
2461 NULL)
2462 count += __free_memory_core(start, end);
2463
2464 return count;
2465 }
2466
2467 static int reset_managed_pages_done __initdata;
2468
reset_node_managed_pages(pg_data_t * pgdat)2469 static void __init reset_node_managed_pages(pg_data_t *pgdat)
2470 {
2471 struct zone *z;
2472
2473 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
2474 atomic_long_set(&z->managed_pages, 0);
2475 }
2476
reset_all_zones_managed_pages(void)2477 void __init reset_all_zones_managed_pages(void)
2478 {
2479 struct pglist_data *pgdat;
2480
2481 if (reset_managed_pages_done)
2482 return;
2483
2484 for_each_online_pgdat(pgdat)
2485 reset_node_managed_pages(pgdat);
2486
2487 reset_managed_pages_done = 1;
2488 }
2489
2490 /**
2491 * memblock_free_all - release free pages to the buddy allocator
2492 */
memblock_free_all(void)2493 void __init memblock_free_all(void)
2494 {
2495 unsigned long pages;
2496
2497 free_unused_memmap();
2498 reset_all_zones_managed_pages();
2499
2500 pages = free_low_memory_core_early();
2501 totalram_pages_add(pages);
2502
2503 memblock_memsize_disable_tracking();
2504 }
2505
2506 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
2507 static const char * const flagname[] = {
2508 [ilog2(MEMBLOCK_HOTPLUG)] = "HOTPLUG",
2509 [ilog2(MEMBLOCK_MIRROR)] = "MIRROR",
2510 [ilog2(MEMBLOCK_NOMAP)] = "NOMAP",
2511 [ilog2(MEMBLOCK_DRIVER_MANAGED)] = "DRV_MNG",
2512 };
2513
memblock_debug_show(struct seq_file * m,void * private)2514 static int memblock_debug_show(struct seq_file *m, void *private)
2515 {
2516 struct memblock_type *type = m->private;
2517 struct memblock_region *reg;
2518 int i, j, nid;
2519 unsigned int count = ARRAY_SIZE(flagname);
2520 phys_addr_t end;
2521
2522 for (i = 0; i < type->cnt; i++) {
2523 reg = &type->regions[i];
2524 end = reg->base + reg->size - 1;
2525 nid = memblock_get_region_node(reg);
2526
2527 seq_printf(m, "%4d: ", i);
2528 seq_printf(m, "%pa..%pa ", ®->base, &end);
2529 if (nid != MAX_NUMNODES)
2530 seq_printf(m, "%4d ", nid);
2531 else
2532 seq_printf(m, "%4c ", 'x');
2533 if (reg->flags) {
2534 for (j = 0; j < count; j++) {
2535 if (reg->flags & (1U << j)) {
2536 seq_printf(m, "%s\n", flagname[j]);
2537 break;
2538 }
2539 }
2540 if (j == count)
2541 seq_printf(m, "%s\n", "UNKNOWN");
2542 } else {
2543 seq_printf(m, "%s\n", "NONE");
2544 }
2545 }
2546 return 0;
2547 }
2548 DEFINE_SHOW_ATTRIBUTE(memblock_debug);
2549
2550 /* assume that freeing region is NOT bigger than the previous region */
memblock_memsize_free(phys_addr_t free_base,phys_addr_t free_size)2551 static void memblock_memsize_free(phys_addr_t free_base,
2552 phys_addr_t free_size)
2553 {
2554 int i;
2555 struct memsize_rgn_struct *rgn;
2556 phys_addr_t free_end, end;
2557
2558 free_end = free_base + free_size - 1;
2559 memblock_dbg("%s %pa..%pa\n",
2560 __func__, &free_base, &free_end);
2561
2562 for (i = 0; i < memsize_rgn_count; i++) {
2563 rgn = &memsize_rgn[i];
2564
2565 end = rgn->base + rgn->size;
2566 if (free_base < rgn->base ||
2567 free_base >= end)
2568 continue;
2569
2570 free_end = free_base + free_size;
2571 if (free_base == rgn->base) {
2572 rgn->size -= free_size;
2573 if (rgn->size != 0)
2574 rgn->base += free_size;
2575 } else if (free_end == end) {
2576 rgn->size -= free_size;
2577 } else {
2578 memblock_memsize_record(rgn->name, free_end,
2579 end - free_end, rgn->nomap, rgn->reusable);
2580 rgn->size = free_base - rgn->base;
2581 }
2582 }
2583 }
2584
memsize_rgn_cmp(const void * a,const void * b)2585 static int memsize_rgn_cmp(const void *a, const void *b)
2586 {
2587 const struct memsize_rgn_struct *ra = a, *rb = b;
2588
2589 if (ra->base > rb->base)
2590 return -1;
2591
2592 if (ra->base < rb->base)
2593 return 1;
2594
2595 return 0;
2596 }
2597
2598 /* assume that freed size is always 64 KB aligned */
memblock_memsize_check_size(struct memsize_rgn_struct * rgn)2599 static inline void memblock_memsize_check_size(struct memsize_rgn_struct *rgn)
2600 {
2601 phys_addr_t phy, end, freed = 0;
2602 bool has_freed = false;
2603 struct page *page;
2604
2605 if (rgn->reusable || rgn->nomap)
2606 return;
2607
2608 phy = rgn->base;
2609 end = rgn->base + rgn->size;
2610 while (phy < end) {
2611 unsigned long pfn = __phys_to_pfn(phy);
2612
2613 if (!pfn_valid(pfn))
2614 return;
2615 page = pfn_to_page(pfn);
2616 if (!has_freed && !PageReserved(page)) {
2617 has_freed = true;
2618 freed = phy;
2619 } else if (has_freed && PageReserved(page)) {
2620 has_freed = false;
2621 memblock_memsize_free(freed, phy - freed);
2622 }
2623
2624 if (has_freed && (phy + SZ_64K >= end))
2625 memblock_memsize_free(freed, end - freed);
2626
2627 /* check the first page only */
2628 phy += SZ_64K;
2629 }
2630 }
2631
memblock_memsize_show(struct seq_file * m,void * private)2632 static int memblock_memsize_show(struct seq_file *m, void *private)
2633 {
2634 int i;
2635 struct memsize_rgn_struct *rgn;
2636 unsigned long reserved = 0, reusable = 0, total;
2637 unsigned long system = totalram_pages() << PAGE_SHIFT;
2638 unsigned long etc;
2639
2640 etc = memsize_kinit;
2641 etc -= memsize_code + memsize_data + memsize_ro + memsize_bss +
2642 memsize_memmap;
2643
2644 system += memsize_reusable_size;
2645 sort(memsize_rgn, memsize_rgn_count,
2646 sizeof(memsize_rgn[0]), memsize_rgn_cmp, NULL);
2647 for (i = 0; i < memsize_rgn_count; i++) {
2648 phys_addr_t base, end;
2649 long size;
2650
2651 rgn = &memsize_rgn[i];
2652 memblock_memsize_check_size(rgn);
2653 base = rgn->base;
2654 size = rgn->size;
2655 end = base + size;
2656
2657 seq_printf(m, "0x%pK-0x%pK 0x%08lx ( %7lu KB ) %s %s %s\n",
2658 (void *)base, (void *)end,
2659 size, DIV_ROUND_UP(size, SZ_1K),
2660 rgn->nomap ? "nomap" : " map",
2661 rgn->reusable ? "reusable" : "unusable",
2662 rgn->name);
2663 if (rgn->reusable)
2664 reusable += (unsigned long)rgn->size;
2665 else
2666 reserved += (unsigned long)rgn->size;
2667 }
2668
2669 total = memsize_kinit + reserved + system;
2670
2671 seq_puts(m, "\n");
2672 seq_printf(m, "Reserved : %7lu KB\n",
2673 DIV_ROUND_UP(memsize_kinit + reserved, SZ_1K));
2674 seq_printf(m, " .kernel : %7lu KB\n",
2675 DIV_ROUND_UP(memsize_kinit, SZ_1K));
2676 seq_printf(m, " .text : %7lu KB\n"
2677 " .rwdata : %7lu KB\n"
2678 " .rodata : %7lu KB\n"
2679 " .bss : %7lu KB\n"
2680 " .memmap : %7lu KB\n"
2681 " .etc : %7lu KB\n",
2682 DIV_ROUND_UP(memsize_code, SZ_1K),
2683 DIV_ROUND_UP(memsize_data, SZ_1K),
2684 DIV_ROUND_UP(memsize_ro, SZ_1K),
2685 DIV_ROUND_UP(memsize_bss, SZ_1K),
2686 DIV_ROUND_UP(memsize_memmap, SZ_1K),
2687 DIV_ROUND_UP(etc, SZ_1K));
2688 seq_printf(m, " .unusable : %7lu KB\n",
2689 DIV_ROUND_UP(reserved, SZ_1K));
2690 seq_printf(m, "System : %7lu KB\n",
2691 DIV_ROUND_UP(system, SZ_1K));
2692 seq_printf(m, " .common : %7lu KB\n",
2693 DIV_ROUND_UP(system - reusable, SZ_1K));
2694 seq_printf(m, " .reusable : %7lu KB\n",
2695 DIV_ROUND_UP(reusable, SZ_1K));
2696 seq_printf(m, "Total : %7lu KB ( %5lu.%02lu MB )\n",
2697 DIV_ROUND_UP(total, SZ_1K),
2698 total >> 20, ((total % SZ_1M) * 100) >> 20);
2699 return 0;
2700 }
2701
2702 DEFINE_SHOW_ATTRIBUTE(memblock_memsize);
2703
memblock_init_debugfs(void)2704 static int __init memblock_init_debugfs(void)
2705 {
2706 struct dentry *root = debugfs_create_dir("memblock", NULL);
2707
2708 debugfs_create_file("memory", 0444, root,
2709 &memblock.memory, &memblock_debug_fops);
2710 debugfs_create_file("reserved", 0444, root,
2711 &memblock.reserved, &memblock_debug_fops);
2712 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
2713 debugfs_create_file("physmem", 0444, root, &physmem,
2714 &memblock_debug_fops);
2715 #endif
2716 if (memsize_state == MEMBLOCK_MEMSIZE_DEBUGFS)
2717 debugfs_create_file("memsize", 0444, root, NULL,
2718 &memblock_memsize_fops);
2719 else if (memsize_state == MEMBLOCK_MEMSIZE_PROCFS)
2720 proc_create_single("memsize", 0, NULL,
2721 memblock_memsize_show);
2722
2723 return 0;
2724 }
2725 __initcall(memblock_init_debugfs);
2726
2727 #endif /* CONFIG_DEBUG_FS */
2728