1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 Regents of the University of California
4 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
5 * Copyright (C) 2020 FORTH-ICS/CARV
6 * Nick Kossifidis <mick@ics.forth.gr>
7 */
8
9 #include <linux/init.h>
10 #include <linux/mm.h>
11 #include <linux/memblock.h>
12 #include <linux/initrd.h>
13 #include <linux/swap.h>
14 #include <linux/swiotlb.h>
15 #include <linux/sizes.h>
16 #include <linux/of_fdt.h>
17 #include <linux/of_reserved_mem.h>
18 #include <linux/libfdt.h>
19 #include <linux/set_memory.h>
20 #include <linux/dma-map-ops.h>
21 #include <linux/crash_dump.h>
22 #include <linux/hugetlb.h>
23
24 #include <asm/fixmap.h>
25 #include <asm/tlbflush.h>
26 #include <asm/sections.h>
27 #include <asm/soc.h>
28 #include <asm/io.h>
29 #include <asm/ptdump.h>
30 #include <asm/numa.h>
31
32 #include "../kernel/head.h"
33
34 struct kernel_mapping kernel_map __ro_after_init;
35 EXPORT_SYMBOL(kernel_map);
36 #ifdef CONFIG_XIP_KERNEL
37 #define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map))
38 #endif
39
40 phys_addr_t phys_ram_base __ro_after_init;
41 EXPORT_SYMBOL(phys_ram_base);
42
43 #ifdef CONFIG_XIP_KERNEL
44 extern char _xiprom[], _exiprom[];
45 #endif
46
47 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
48 __page_aligned_bss;
49 EXPORT_SYMBOL(empty_zero_page);
50
51 extern char _start[];
52 void *_dtb_early_va __initdata;
53 uintptr_t _dtb_early_pa __initdata;
54
55 struct pt_alloc_ops {
56 pte_t *(*get_pte_virt)(phys_addr_t pa);
57 phys_addr_t (*alloc_pte)(uintptr_t va);
58 #ifndef __PAGETABLE_PMD_FOLDED
59 pmd_t *(*get_pmd_virt)(phys_addr_t pa);
60 phys_addr_t (*alloc_pmd)(uintptr_t va);
61 #endif
62 };
63
64 static phys_addr_t dma32_phys_limit __initdata;
65
zone_sizes_init(void)66 static void __init zone_sizes_init(void)
67 {
68 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, };
69
70 #ifdef CONFIG_ZONE_DMA32
71 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit);
72 #endif
73 max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
74
75 free_area_init(max_zone_pfns);
76 }
77
78 #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM)
print_mlk(char * name,unsigned long b,unsigned long t)79 static inline void print_mlk(char *name, unsigned long b, unsigned long t)
80 {
81 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t,
82 (((t) - (b)) >> 10));
83 }
84
print_mlm(char * name,unsigned long b,unsigned long t)85 static inline void print_mlm(char *name, unsigned long b, unsigned long t)
86 {
87 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t,
88 (((t) - (b)) >> 20));
89 }
90
print_vm_layout(void)91 static void __init print_vm_layout(void)
92 {
93 pr_notice("Virtual kernel memory layout:\n");
94 print_mlk("fixmap", (unsigned long)FIXADDR_START,
95 (unsigned long)FIXADDR_TOP);
96 print_mlm("pci io", (unsigned long)PCI_IO_START,
97 (unsigned long)PCI_IO_END);
98 print_mlm("vmemmap", (unsigned long)VMEMMAP_START,
99 (unsigned long)VMEMMAP_END);
100 print_mlm("vmalloc", (unsigned long)VMALLOC_START,
101 (unsigned long)VMALLOC_END);
102 #ifdef CONFIG_64BIT
103 print_mlm("modules", (unsigned long)MODULES_VADDR,
104 (unsigned long)MODULES_END);
105 #endif
106 print_mlm("lowmem", (unsigned long)PAGE_OFFSET,
107 (unsigned long)high_memory);
108 #ifdef CONFIG_64BIT
109 print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR,
110 (unsigned long)ADDRESS_SPACE_END);
111 #endif
112 }
113 #else
print_vm_layout(void)114 static void print_vm_layout(void) { }
115 #endif /* CONFIG_DEBUG_VM */
116
mem_init(void)117 void __init mem_init(void)
118 {
119 #ifdef CONFIG_FLATMEM
120 BUG_ON(!mem_map);
121 #endif /* CONFIG_FLATMEM */
122
123 #ifdef CONFIG_SWIOTLB
124 if (swiotlb_force == SWIOTLB_FORCE ||
125 max_pfn > PFN_DOWN(dma32_phys_limit))
126 swiotlb_init(1);
127 else
128 swiotlb_force = SWIOTLB_NO_FORCE;
129 #endif
130 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn)));
131 memblock_free_all();
132
133 print_vm_layout();
134 }
135
136 /*
137 * The default maximal physical memory size is -PAGE_OFFSET for 32-bit kernel,
138 * whereas for 64-bit kernel, the end of the virtual address space is occupied
139 * by the modules/BPF/kernel mappings which reduces the available size of the
140 * linear mapping.
141 * Limit the memory size via mem.
142 */
143 #ifdef CONFIG_64BIT
144 static phys_addr_t memory_limit = -PAGE_OFFSET - SZ_4G;
145 #else
146 static phys_addr_t memory_limit = -PAGE_OFFSET;
147 #endif
148
early_mem(char * p)149 static int __init early_mem(char *p)
150 {
151 u64 size;
152
153 if (!p)
154 return 1;
155
156 size = memparse(p, &p) & PAGE_MASK;
157 memory_limit = min_t(u64, size, memory_limit);
158
159 pr_notice("Memory limited to %lldMB\n", (u64)memory_limit >> 20);
160
161 return 0;
162 }
163 early_param("mem", early_mem);
164
setup_bootmem(void)165 static void __init setup_bootmem(void)
166 {
167 phys_addr_t vmlinux_end = __pa_symbol(&_end);
168 phys_addr_t vmlinux_start = __pa_symbol(&_start);
169 phys_addr_t __maybe_unused max_mapped_addr;
170 phys_addr_t phys_ram_end;
171
172 #ifdef CONFIG_XIP_KERNEL
173 vmlinux_start = __pa_symbol(&_sdata);
174 #endif
175
176 memblock_enforce_memory_limit(memory_limit);
177
178 /*
179 * Reserve from the start of the kernel to the end of the kernel
180 */
181 #if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX)
182 /*
183 * Make sure we align the reservation on PMD_SIZE since we will
184 * map the kernel in the linear mapping as read-only: we do not want
185 * any allocation to happen between _end and the next pmd aligned page.
186 */
187 vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK;
188 #endif
189 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start);
190
191
192 phys_ram_end = memblock_end_of_DRAM();
193 #ifndef CONFIG_XIP_KERNEL
194 phys_ram_base = memblock_start_of_DRAM();
195 #endif
196 #ifndef CONFIG_64BIT
197 /*
198 * memblock allocator is not aware of the fact that last 4K bytes of
199 * the addressable memory can not be mapped because of IS_ERR_VALUE
200 * macro. Make sure that last 4k bytes are not usable by memblock
201 * if end of dram is equal to maximum addressable memory. For 64-bit
202 * kernel, this problem can't happen here as the end of the virtual
203 * address space is occupied by the kernel mapping then this check must
204 * be done as soon as the kernel mapping base address is determined.
205 */
206 max_mapped_addr = __pa(~(ulong)0);
207 if (max_mapped_addr == (phys_ram_end - 1))
208 memblock_set_current_limit(max_mapped_addr - 4096);
209 #endif
210
211 min_low_pfn = PFN_UP(phys_ram_base);
212 max_low_pfn = max_pfn = PFN_DOWN(phys_ram_end);
213
214 dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn));
215 set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET);
216
217 reserve_initrd_mem();
218
219 /*
220 * No allocation should be done before reserving the memory as defined
221 * in the device tree, otherwise the allocation could end up in a
222 * reserved region.
223 */
224 early_init_fdt_scan_reserved_mem();
225
226 /*
227 * If DTB is built in, no need to reserve its memblock.
228 * Otherwise, do reserve it but avoid using
229 * early_init_fdt_reserve_self() since __pa() does
230 * not work for DTB pointers that are fixmap addresses
231 */
232 if (!IS_ENABLED(CONFIG_BUILTIN_DTB))
233 memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va));
234
235 dma_contiguous_reserve(dma32_phys_limit);
236 if (IS_ENABLED(CONFIG_64BIT))
237 hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);
238 }
239
240 #ifdef CONFIG_MMU
241 static struct pt_alloc_ops _pt_ops __initdata;
242
243 #ifdef CONFIG_XIP_KERNEL
244 #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&_pt_ops))
245 #else
246 #define pt_ops _pt_ops
247 #endif
248
249 unsigned long riscv_pfn_base __ro_after_init;
250 EXPORT_SYMBOL(riscv_pfn_base);
251
252 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
253 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
254 static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;
255
256 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
257
258 #ifdef CONFIG_XIP_KERNEL
259 #define riscv_pfn_base (*(unsigned long *)XIP_FIXUP(&riscv_pfn_base))
260 #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir))
261 #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte))
262 #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir))
263 #endif /* CONFIG_XIP_KERNEL */
264
__set_fixmap(enum fixed_addresses idx,phys_addr_t phys,pgprot_t prot)265 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
266 {
267 unsigned long addr = __fix_to_virt(idx);
268 pte_t *ptep;
269
270 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
271
272 ptep = &fixmap_pte[pte_index(addr)];
273
274 if (pgprot_val(prot))
275 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot));
276 else
277 pte_clear(&init_mm, addr, ptep);
278 local_flush_tlb_page(addr);
279 }
280
get_pte_virt_early(phys_addr_t pa)281 static inline pte_t *__init get_pte_virt_early(phys_addr_t pa)
282 {
283 return (pte_t *)((uintptr_t)pa);
284 }
285
get_pte_virt_fixmap(phys_addr_t pa)286 static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa)
287 {
288 clear_fixmap(FIX_PTE);
289 return (pte_t *)set_fixmap_offset(FIX_PTE, pa);
290 }
291
get_pte_virt_late(phys_addr_t pa)292 static inline pte_t *__init get_pte_virt_late(phys_addr_t pa)
293 {
294 return (pte_t *) __va(pa);
295 }
296
alloc_pte_early(uintptr_t va)297 static inline phys_addr_t __init alloc_pte_early(uintptr_t va)
298 {
299 /*
300 * We only create PMD or PGD early mappings so we
301 * should never reach here with MMU disabled.
302 */
303 BUG();
304 }
305
alloc_pte_fixmap(uintptr_t va)306 static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va)
307 {
308 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
309 }
310
alloc_pte_late(uintptr_t va)311 static phys_addr_t __init alloc_pte_late(uintptr_t va)
312 {
313 unsigned long vaddr;
314
315 vaddr = __get_free_page(GFP_KERNEL);
316 BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr)));
317
318 return __pa(vaddr);
319 }
320
create_pte_mapping(pte_t * ptep,uintptr_t va,phys_addr_t pa,phys_addr_t sz,pgprot_t prot)321 static void __init create_pte_mapping(pte_t *ptep,
322 uintptr_t va, phys_addr_t pa,
323 phys_addr_t sz, pgprot_t prot)
324 {
325 uintptr_t pte_idx = pte_index(va);
326
327 BUG_ON(sz != PAGE_SIZE);
328
329 if (pte_none(ptep[pte_idx]))
330 ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot);
331 }
332
333 #ifndef __PAGETABLE_PMD_FOLDED
334
335 static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss;
336 static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss;
337 static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
338
339 #ifdef CONFIG_XIP_KERNEL
340 #define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd))
341 #define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd))
342 #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd))
343 #endif /* CONFIG_XIP_KERNEL */
344
get_pmd_virt_early(phys_addr_t pa)345 static pmd_t *__init get_pmd_virt_early(phys_addr_t pa)
346 {
347 /* Before MMU is enabled */
348 return (pmd_t *)((uintptr_t)pa);
349 }
350
get_pmd_virt_fixmap(phys_addr_t pa)351 static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa)
352 {
353 clear_fixmap(FIX_PMD);
354 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa);
355 }
356
get_pmd_virt_late(phys_addr_t pa)357 static pmd_t *__init get_pmd_virt_late(phys_addr_t pa)
358 {
359 return (pmd_t *) __va(pa);
360 }
361
alloc_pmd_early(uintptr_t va)362 static phys_addr_t __init alloc_pmd_early(uintptr_t va)
363 {
364 BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);
365
366 return (uintptr_t)early_pmd;
367 }
368
alloc_pmd_fixmap(uintptr_t va)369 static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va)
370 {
371 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
372 }
373
alloc_pmd_late(uintptr_t va)374 static phys_addr_t __init alloc_pmd_late(uintptr_t va)
375 {
376 unsigned long vaddr;
377
378 vaddr = __get_free_page(GFP_KERNEL);
379 BUG_ON(!vaddr);
380 return __pa(vaddr);
381 }
382
create_pmd_mapping(pmd_t * pmdp,uintptr_t va,phys_addr_t pa,phys_addr_t sz,pgprot_t prot)383 static void __init create_pmd_mapping(pmd_t *pmdp,
384 uintptr_t va, phys_addr_t pa,
385 phys_addr_t sz, pgprot_t prot)
386 {
387 pte_t *ptep;
388 phys_addr_t pte_phys;
389 uintptr_t pmd_idx = pmd_index(va);
390
391 if (sz == PMD_SIZE) {
392 if (pmd_none(pmdp[pmd_idx]))
393 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot);
394 return;
395 }
396
397 if (pmd_none(pmdp[pmd_idx])) {
398 pte_phys = pt_ops.alloc_pte(va);
399 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE);
400 ptep = pt_ops.get_pte_virt(pte_phys);
401 memset(ptep, 0, PAGE_SIZE);
402 } else {
403 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx]));
404 ptep = pt_ops.get_pte_virt(pte_phys);
405 }
406
407 create_pte_mapping(ptep, va, pa, sz, prot);
408 }
409
410 #define pgd_next_t pmd_t
411 #define alloc_pgd_next(__va) pt_ops.alloc_pmd(__va)
412 #define get_pgd_next_virt(__pa) pt_ops.get_pmd_virt(__pa)
413 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
414 create_pmd_mapping(__nextp, __va, __pa, __sz, __prot)
415 #define fixmap_pgd_next fixmap_pmd
416 #else
417 #define pgd_next_t pte_t
418 #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va)
419 #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa)
420 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
421 create_pte_mapping(__nextp, __va, __pa, __sz, __prot)
422 #define fixmap_pgd_next fixmap_pte
423 #define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot)
424 #endif
425
create_pgd_mapping(pgd_t * pgdp,uintptr_t va,phys_addr_t pa,phys_addr_t sz,pgprot_t prot)426 void __init create_pgd_mapping(pgd_t *pgdp,
427 uintptr_t va, phys_addr_t pa,
428 phys_addr_t sz, pgprot_t prot)
429 {
430 pgd_next_t *nextp;
431 phys_addr_t next_phys;
432 uintptr_t pgd_idx = pgd_index(va);
433
434 if (sz == PGDIR_SIZE) {
435 if (pgd_val(pgdp[pgd_idx]) == 0)
436 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot);
437 return;
438 }
439
440 if (pgd_val(pgdp[pgd_idx]) == 0) {
441 next_phys = alloc_pgd_next(va);
442 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE);
443 nextp = get_pgd_next_virt(next_phys);
444 memset(nextp, 0, PAGE_SIZE);
445 } else {
446 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx]));
447 nextp = get_pgd_next_virt(next_phys);
448 }
449
450 create_pgd_next_mapping(nextp, va, pa, sz, prot);
451 }
452
best_map_size(phys_addr_t base,phys_addr_t size)453 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)
454 {
455 /* Upgrade to PMD_SIZE mappings whenever possible */
456 if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1)))
457 return PAGE_SIZE;
458
459 return PMD_SIZE;
460 }
461
462 #ifdef CONFIG_XIP_KERNEL
463 #define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base))
464 /* called from head.S with MMU off */
__copy_data(void)465 asmlinkage void __init __copy_data(void)
466 {
467 void *from = (void *)(&_sdata);
468 void *end = (void *)(&_end);
469 void *to = (void *)CONFIG_PHYS_RAM_BASE;
470 size_t sz = (size_t)(end - from + 1);
471
472 memcpy(to, from, sz);
473 }
474 #endif
475
476 #ifdef CONFIG_STRICT_KERNEL_RWX
pgprot_from_va(uintptr_t va)477 static __init pgprot_t pgprot_from_va(uintptr_t va)
478 {
479 if (is_va_kernel_text(va))
480 return PAGE_KERNEL_READ_EXEC;
481
482 /*
483 * In 64-bit kernel, the kernel mapping is outside the linear mapping so
484 * we must protect its linear mapping alias from being executed and
485 * written.
486 * And rodata section is marked readonly in mark_rodata_ro.
487 */
488 if (IS_ENABLED(CONFIG_64BIT) && is_va_kernel_lm_alias_text(va))
489 return PAGE_KERNEL_READ;
490
491 return PAGE_KERNEL;
492 }
493
mark_rodata_ro(void)494 void mark_rodata_ro(void)
495 {
496 set_kernel_memory(__start_rodata, _data, set_memory_ro);
497 if (IS_ENABLED(CONFIG_64BIT))
498 set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data),
499 set_memory_ro);
500
501 debug_checkwx();
502 }
503 #else
pgprot_from_va(uintptr_t va)504 static __init pgprot_t pgprot_from_va(uintptr_t va)
505 {
506 if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va))
507 return PAGE_KERNEL;
508
509 return PAGE_KERNEL_EXEC;
510 }
511 #endif /* CONFIG_STRICT_KERNEL_RWX */
512
513 /*
514 * setup_vm() is called from head.S with MMU-off.
515 *
516 * Following requirements should be honoured for setup_vm() to work
517 * correctly:
518 * 1) It should use PC-relative addressing for accessing kernel symbols.
519 * To achieve this we always use GCC cmodel=medany.
520 * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
521 * so disable compiler instrumentation when FTRACE is enabled.
522 *
523 * Currently, the above requirements are honoured by using custom CFLAGS
524 * for init.o in mm/Makefile.
525 */
526
527 #ifndef __riscv_cmodel_medany
528 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."
529 #endif
530
531 #ifdef CONFIG_XIP_KERNEL
create_kernel_page_table(pgd_t * pgdir,__always_unused bool early)532 static void __init create_kernel_page_table(pgd_t *pgdir,
533 __always_unused bool early)
534 {
535 uintptr_t va, end_va;
536
537 /* Map the flash resident part */
538 end_va = kernel_map.virt_addr + kernel_map.xiprom_sz;
539 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE)
540 create_pgd_mapping(pgdir, va,
541 kernel_map.xiprom + (va - kernel_map.virt_addr),
542 PMD_SIZE, PAGE_KERNEL_EXEC);
543
544 /* Map the data in RAM */
545 end_va = kernel_map.virt_addr + XIP_OFFSET + kernel_map.size;
546 for (va = kernel_map.virt_addr + XIP_OFFSET; va < end_va; va += PMD_SIZE)
547 create_pgd_mapping(pgdir, va,
548 kernel_map.phys_addr + (va - (kernel_map.virt_addr + XIP_OFFSET)),
549 PMD_SIZE, PAGE_KERNEL);
550 }
551 #else
create_kernel_page_table(pgd_t * pgdir,bool early)552 static void __init create_kernel_page_table(pgd_t *pgdir, bool early)
553 {
554 uintptr_t va, end_va;
555
556 end_va = kernel_map.virt_addr + kernel_map.size;
557 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE)
558 create_pgd_mapping(pgdir, va,
559 kernel_map.phys_addr + (va - kernel_map.virt_addr),
560 PMD_SIZE,
561 early ?
562 PAGE_KERNEL_EXEC : pgprot_from_va(va));
563 }
564 #endif
565
566 /*
567 * Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel,
568 * this means 2 PMD entries whereas for 32-bit kernel, this is only 1 PGDIR
569 * entry.
570 */
create_fdt_early_page_table(uintptr_t fix_fdt_va,uintptr_t dtb_pa)571 static void __init create_fdt_early_page_table(uintptr_t fix_fdt_va,
572 uintptr_t dtb_pa)
573 {
574 #ifndef CONFIG_BUILTIN_DTB
575 uintptr_t pa = dtb_pa & ~(PMD_SIZE - 1);
576
577 /* Make sure the fdt fixmap address is always aligned on PMD size */
578 BUILD_BUG_ON(FIX_FDT % (PMD_SIZE / PAGE_SIZE));
579
580 /* In 32-bit only, the fdt lies in its own PGD */
581 if (!IS_ENABLED(CONFIG_64BIT)) {
582 create_pgd_mapping(early_pg_dir, fix_fdt_va,
583 pa, MAX_FDT_SIZE, PAGE_KERNEL);
584 } else {
585 create_pmd_mapping(fixmap_pmd, fix_fdt_va,
586 pa, PMD_SIZE, PAGE_KERNEL);
587 create_pmd_mapping(fixmap_pmd, fix_fdt_va + PMD_SIZE,
588 pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL);
589 }
590
591 dtb_early_va = (void *)fix_fdt_va + (dtb_pa & (PMD_SIZE - 1));
592 #else
593 /*
594 * For 64-bit kernel, __va can't be used since it would return a linear
595 * mapping address whereas dtb_early_va will be used before
596 * setup_vm_final installs the linear mapping. For 32-bit kernel, as the
597 * kernel is mapped in the linear mapping, that makes no difference.
598 */
599 dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa));
600 #endif
601
602 dtb_early_pa = dtb_pa;
603 }
604
setup_vm(uintptr_t dtb_pa)605 asmlinkage void __init setup_vm(uintptr_t dtb_pa)
606 {
607 pmd_t __maybe_unused fix_bmap_spmd, fix_bmap_epmd;
608
609 kernel_map.virt_addr = KERNEL_LINK_ADDR;
610
611 #ifdef CONFIG_XIP_KERNEL
612 kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR;
613 kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom);
614
615 phys_ram_base = CONFIG_PHYS_RAM_BASE;
616 kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE;
617 kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_sdata);
618
619 kernel_map.va_kernel_xip_pa_offset = kernel_map.virt_addr - kernel_map.xiprom;
620 #else
621 kernel_map.phys_addr = (uintptr_t)(&_start);
622 kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr;
623 #endif
624 kernel_map.va_pa_offset = PAGE_OFFSET - kernel_map.phys_addr;
625 kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr;
626
627 riscv_pfn_base = PFN_DOWN(kernel_map.phys_addr);
628
629 /* Sanity check alignment and size */
630 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0);
631 BUG_ON((kernel_map.phys_addr % PMD_SIZE) != 0);
632
633 #ifdef CONFIG_64BIT
634 /*
635 * The last 4K bytes of the addressable memory can not be mapped because
636 * of IS_ERR_VALUE macro.
637 */
638 BUG_ON((kernel_map.virt_addr + kernel_map.size) > ADDRESS_SPACE_END - SZ_4K);
639 #endif
640
641 pt_ops.alloc_pte = alloc_pte_early;
642 pt_ops.get_pte_virt = get_pte_virt_early;
643 #ifndef __PAGETABLE_PMD_FOLDED
644 pt_ops.alloc_pmd = alloc_pmd_early;
645 pt_ops.get_pmd_virt = get_pmd_virt_early;
646 #endif
647 /* Setup early PGD for fixmap */
648 create_pgd_mapping(early_pg_dir, FIXADDR_START,
649 (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE);
650
651 #ifndef __PAGETABLE_PMD_FOLDED
652 /* Setup fixmap PMD */
653 create_pmd_mapping(fixmap_pmd, FIXADDR_START,
654 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE);
655 /* Setup trampoline PGD and PMD */
656 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr,
657 (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE);
658 #ifdef CONFIG_XIP_KERNEL
659 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr,
660 kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC);
661 #else
662 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr,
663 kernel_map.phys_addr, PMD_SIZE, PAGE_KERNEL_EXEC);
664 #endif
665 #else
666 /* Setup trampoline PGD */
667 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr,
668 kernel_map.phys_addr, PGDIR_SIZE, PAGE_KERNEL_EXEC);
669 #endif
670
671 /*
672 * Setup early PGD covering entire kernel which will allow
673 * us to reach paging_init(). We map all memory banks later
674 * in setup_vm_final() below.
675 */
676 create_kernel_page_table(early_pg_dir, true);
677
678 /* Setup early mapping for FDT early scan */
679 create_fdt_early_page_table(__fix_to_virt(FIX_FDT), dtb_pa);
680
681 /*
682 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap
683 * range can not span multiple pmds.
684 */
685 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
686 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
687
688 #ifndef __PAGETABLE_PMD_FOLDED
689 /*
690 * Early ioremap fixmap is already created as it lies within first 2MB
691 * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END
692 * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn
693 * the user if not.
694 */
695 fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))];
696 fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))];
697 if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) {
698 WARN_ON(1);
699 pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n",
700 pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd));
701 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
702 fix_to_virt(FIX_BTMAP_BEGIN));
703 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n",
704 fix_to_virt(FIX_BTMAP_END));
705
706 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
707 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN);
708 }
709 #endif
710 }
711
setup_vm_final(void)712 static void __init setup_vm_final(void)
713 {
714 uintptr_t va, map_size;
715 phys_addr_t pa, start, end;
716 unsigned long idx __maybe_unused;
717 u64 i;
718
719 /**
720 * MMU is enabled at this point. But page table setup is not complete yet.
721 * fixmap page table alloc functions should be used at this point
722 */
723 pt_ops.alloc_pte = alloc_pte_fixmap;
724 pt_ops.get_pte_virt = get_pte_virt_fixmap;
725 #ifndef __PAGETABLE_PMD_FOLDED
726 pt_ops.alloc_pmd = alloc_pmd_fixmap;
727 pt_ops.get_pmd_virt = get_pmd_virt_fixmap;
728 #endif
729 /* Setup swapper PGD for fixmap */
730 #if !defined(CONFIG_64BIT)
731 /*
732 * In 32-bit, the device tree lies in a pgd entry, so it must be copied
733 * directly in swapper_pg_dir in addition to the pgd entry that points
734 * to fixmap_pte.
735 */
736 idx = pgd_index(__fix_to_virt(FIX_FDT));
737
738 set_pgd(&swapper_pg_dir[idx], early_pg_dir[idx]);
739 #endif
740 create_pgd_mapping(swapper_pg_dir, FIXADDR_START,
741 __pa_symbol(fixmap_pgd_next),
742 PGDIR_SIZE, PAGE_TABLE);
743
744 /* Map all memory banks in the linear mapping */
745 for_each_mem_range(i, &start, &end) {
746 if (start >= end)
747 break;
748 if (start <= __pa(PAGE_OFFSET) &&
749 __pa(PAGE_OFFSET) < end)
750 start = __pa(PAGE_OFFSET);
751 if (end >= __pa(PAGE_OFFSET) + memory_limit)
752 end = __pa(PAGE_OFFSET) + memory_limit;
753
754 map_size = best_map_size(start, end - start);
755 for (pa = start; pa < end; pa += map_size) {
756 va = (uintptr_t)__va(pa);
757
758 create_pgd_mapping(swapper_pg_dir, va, pa, map_size,
759 pgprot_from_va(va));
760 }
761 }
762
763 #ifdef CONFIG_64BIT
764 /* Map the kernel */
765 create_kernel_page_table(swapper_pg_dir, false);
766 #endif
767
768 /* Clear fixmap PTE and PMD mappings */
769 clear_fixmap(FIX_PTE);
770 clear_fixmap(FIX_PMD);
771
772 /* Move to swapper page table */
773 csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE);
774 local_flush_tlb_all();
775
776 /* generic page allocation functions must be used to setup page table */
777 pt_ops.alloc_pte = alloc_pte_late;
778 pt_ops.get_pte_virt = get_pte_virt_late;
779 #ifndef __PAGETABLE_PMD_FOLDED
780 pt_ops.alloc_pmd = alloc_pmd_late;
781 pt_ops.get_pmd_virt = get_pmd_virt_late;
782 #endif
783 }
784 #else
setup_vm(uintptr_t dtb_pa)785 asmlinkage void __init setup_vm(uintptr_t dtb_pa)
786 {
787 dtb_early_va = (void *)dtb_pa;
788 dtb_early_pa = dtb_pa;
789 }
790
setup_vm_final(void)791 static inline void setup_vm_final(void)
792 {
793 }
794 #endif /* CONFIG_MMU */
795
796 #ifdef CONFIG_KEXEC_CORE
797 /*
798 * reserve_crashkernel() - reserves memory for crash kernel
799 *
800 * This function reserves memory area given in "crashkernel=" kernel command
801 * line parameter. The memory reserved is used by dump capture kernel when
802 * primary kernel is crashing.
803 */
reserve_crashkernel(void)804 static void __init reserve_crashkernel(void)
805 {
806 unsigned long long crash_base = 0;
807 unsigned long long crash_size = 0;
808 unsigned long search_start = memblock_start_of_DRAM();
809 unsigned long search_end = memblock_end_of_DRAM();
810
811 int ret = 0;
812
813 /*
814 * Don't reserve a region for a crash kernel on a crash kernel
815 * since it doesn't make much sense and we have limited memory
816 * resources.
817 */
818 #ifdef CONFIG_CRASH_DUMP
819 if (is_kdump_kernel()) {
820 pr_info("crashkernel: ignoring reservation request\n");
821 return;
822 }
823 #endif
824
825 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
826 &crash_size, &crash_base);
827 if (ret || !crash_size)
828 return;
829
830 crash_size = PAGE_ALIGN(crash_size);
831
832 if (crash_base) {
833 search_start = crash_base;
834 search_end = crash_base + crash_size;
835 }
836
837 /*
838 * Current riscv boot protocol requires 2MB alignment for
839 * RV64 and 4MB alignment for RV32 (hugepage size)
840 *
841 * Try to alloc from 32bit addressible physical memory so that
842 * swiotlb can work on the crash kernel.
843 */
844 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE,
845 search_start,
846 min(search_end, (unsigned long)(SZ_4G - 1)));
847 if (crash_base == 0) {
848 /* Try again without restricting region to 32bit addressible memory */
849 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE,
850 search_start, search_end);
851 if (crash_base == 0) {
852 pr_warn("crashkernel: couldn't allocate %lldKB\n",
853 crash_size >> 10);
854 return;
855 }
856 }
857
858 pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n",
859 crash_base, crash_base + crash_size, crash_size >> 20);
860
861 crashk_res.start = crash_base;
862 crashk_res.end = crash_base + crash_size - 1;
863 }
864 #endif /* CONFIG_KEXEC_CORE */
865
paging_init(void)866 void __init paging_init(void)
867 {
868 setup_bootmem();
869 setup_vm_final();
870
871 /* Depend on that Linear Mapping is ready */
872 memblock_allow_resize();
873 }
874
misc_mem_init(void)875 void __init misc_mem_init(void)
876 {
877 early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT);
878 arch_numa_init();
879 sparse_init();
880 zone_sizes_init();
881 #ifdef CONFIG_KEXEC_CORE
882 reserve_crashkernel();
883 #endif
884 memblock_dump_all();
885 }
886
887 #ifdef CONFIG_SPARSEMEM_VMEMMAP
vmemmap_populate(unsigned long start,unsigned long end,int node,struct vmem_altmap * altmap)888 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
889 struct vmem_altmap *altmap)
890 {
891 return vmemmap_populate_basepages(start, end, node, NULL);
892 }
893 #endif
894