1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2020 FORTH-ICS/CARV
20 #include <linux/dma-map-ops.h>
91 	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld kB)\n", name, b, t,  in print_mlk()
92 		  (((t) - (b)) >> LOG2_SZ_1K));  in print_mlk()
97 	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld MB)\n", name, b, t,  in print_mlm()
98 		  (((t) - (b)) >> LOG2_SZ_1M));  in print_mlm()
103 	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld GB)\n", name, b, t,  in print_mlg()
104 		   (((t) - (b)) >> LOG2_SZ_1G));  in print_mlg()
110 	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld TB)\n", name, b, t,  in print_mlt()
111 		   (((t) - (b)) >> LOG2_SZ_1T));  in print_mlt()
119 	unsigned long diff = t - b;  in print_ml()
210 	 * map the kernel in the linear mapping as read-only: we do not want  in setup_bootmem()
214 		vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK;  in setup_bootmem()
218 	memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start);  in setup_bootmem()
224 	 * at worst, we map the linear mapping with PMD mappings.  in setup_bootmem()
230 	 * In 64-bit, any use of __va/__pa before this point is wrong as we  in setup_bootmem()
234 		kernel_map.va_pa_offset = PAGE_OFFSET - phys_ram_base;  in setup_bootmem()
240 	 * if end of dram is equal to maximum addressable memory.  For 64-bit  in setup_bootmem()
242 	 * address space is occupied by the kernel mapping then this check must  in setup_bootmem()
243 	 * be done as soon as the kernel mapping base address is determined.  in setup_bootmem()
247 		if (max_mapped_addr == (phys_ram_end - 1))  in setup_bootmem()
248 			memblock_set_current_limit(max_mapped_addr - 4096);  in setup_bootmem()
256 	set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET);  in setup_bootmem()
278 		hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);  in setup_bootmem()
434 	BUG_ON((va - kernel_map.virt_addr) >> PUD_SHIFT);  in alloc_pmd_early()
497 	/* Only one PUD is available for early mapping */  in alloc_pud_early()
498 	BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);  in alloc_pud_early()
535 	/* Only one P4D is available for early mapping */  in alloc_p4d_early()
536 	BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);  in alloc_p4d_early()
670 	if (!(pa & (PGDIR_SIZE - 1)) && !(va & (PGDIR_SIZE - 1)) && size >= PGDIR_SIZE)  in best_map_size()
673 	if (!(pa & (P4D_SIZE - 1)) && !(va & (P4D_SIZE - 1)) && size >= P4D_SIZE)  in best_map_size()
676 	if (!(pa & (PUD_SIZE - 1)) && !(va & (PUD_SIZE - 1)) && size >= PUD_SIZE)  in best_map_size()
679 	if (!(pa & (PMD_SIZE - 1)) && !(va & (PMD_SIZE - 1)) && size >= PMD_SIZE)  in best_map_size()
694 	size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata));  in __copy_data()
707 	 * In 64-bit kernel, the kernel mapping is outside the linear mapping so  in pgprot_from_va()
708 	 * we must protect its linear mapping alias from being executed and  in pgprot_from_va()
756 	pr_info("Disabled 4-level and 5-level paging");  in print_no4lvl()
763 	pr_info("Disabled 5-level paging");  in print_no5lvl()
769  * There is a simple way to determine if 4-level is supported by the
770  * underlying hardware: establish 1:1 mapping in 4-level page table mode
833  * setup_vm() is called from head.S with MMU-off.
837  * 1) It should use PC-relative addressing for accessing kernel symbols.
860 	uintptr_t reloc_offset = kernel_map.virt_addr - KERNEL_LINK_ADDR;  in relocate_kernel()
865 	uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr;  in relocate_kernel()
868 		Elf64_Addr addr = (rela->r_offset - va_kernel_link_pa_offset);  in relocate_kernel()
869 		Elf64_Addr relocated_addr = rela->r_addend;  in relocate_kernel()
871 		if (rela->r_info != R_RISCV_RELATIVE)  in relocate_kernel()
878 		 * mm->context.vdso in VDSO_OFFSET macro.  in relocate_kernel()
898 				   kernel_map.xiprom + (va - kernel_map.virt_addr),  in create_kernel_page_table()
905 				   kernel_map.phys_addr + (va - (kernel_map.virt_addr + XIP_OFFSET)),  in create_kernel_page_table()
916 				   kernel_map.phys_addr + (va - kernel_map.virt_addr),  in create_kernel_page_table()
924  * Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel,
925  * this means 2 PMD entries whereas for 32-bit kernel, this is only 1 PGDIR
932 	uintptr_t pa = dtb_pa & ~(PMD_SIZE - 1);  in create_fdt_early_page_table()
937 	/* In 32-bit only, the fdt lies in its own PGD */  in create_fdt_early_page_table()
948 	dtb_early_va = (void *)fix_fdt_va + (dtb_pa & (PMD_SIZE - 1));  in create_fdt_early_page_table()
951 	 * For 64-bit kernel, __va can't be used since it would return a linear  in create_fdt_early_page_table()
952 	 * mapping address whereas dtb_early_va will be used before  in create_fdt_early_page_table()
953 	 * setup_vm_final installs the linear mapping. For 32-bit kernel, as the  in create_fdt_early_page_table()
954 	 * kernel is mapped in the linear mapping, that makes no difference.  in create_fdt_early_page_table()
983  * map the allocated physical pages since the linear mapping does not exist yet.
1044 		u32 kernel_size = (uintptr_t)(&_end) - (uintptr_t)(&_start);  in setup_vm()
1052 		nr_pos = (PUD_SIZE - kernel_size) / PMD_SIZE;  in setup_vm()
1067 	kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom);  in setup_vm()
1071 	kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_sdata);  in setup_vm()
1073 	kernel_map.va_kernel_xip_pa_offset = kernel_map.virt_addr - kernel_map.xiprom;  in setup_vm()
1077 	kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr;  in setup_vm()
1085 	 * In 64-bit, we defer the setup of va_pa_offset to setup_bootmem,  in setup_vm()
1088 	 * for the linear mapping. This is only possible because the kernel  in setup_vm()
1089 	 * mapping lies outside the linear mapping.  in setup_vm()
1090 	 * In 32-bit however, as the kernel resides in the linear mapping,  in setup_vm()
1091 	 * setup_vm_final can not change the mapping established here,  in setup_vm()
1097 				0UL : PAGE_OFFSET - kernel_map.phys_addr;  in setup_vm()
1098 	kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr;  in setup_vm()
1101 	 * The default maximal physical memory size is KERN_VIRT_SIZE for 32-bit  in setup_vm()
1102 	 * kernel, whereas for 64-bit kernel, the end of the virtual address  in setup_vm()
1104 	 * the available size of the linear mapping.  in setup_vm()
1106 	memory_limit = KERN_VIRT_SIZE - (IS_ENABLED(CONFIG_64BIT) ? SZ_4G : 0);  in setup_vm()
1117 	BUG_ON((kernel_map.virt_addr + kernel_map.size) > ADDRESS_SPACE_END - SZ_4K);  in setup_vm()
1127 	BUG_ON(PUD_SIZE - (kernel_map.virt_addr & (PUD_SIZE - 1)) < kernel_map.size);  in setup_vm()
1178 	/* Setup early mapping for FDT early scan */  in setup_vm()
1182 	 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap  in setup_vm()
1224 					    best_map_size(pa, va, end - pa);  in create_linear_mapping_range()
1239 	phys_addr_t ktext_size = __init_data_begin - _start;  in create_linear_mapping_page_table()
1241 	phys_addr_t krodata_size = _data - __start_rodata;  in create_linear_mapping_page_table()
1251 	 *  before we setup the linear mapping so that we avoid using hugepages  in create_linear_mapping_page_table()
1261 	/* Map all memory banks in the linear mapping */  in create_linear_mapping_page_table()
1297 	 * In 32-bit, the device tree lies in a pgd entry, so it must be copied  in setup_vm_final()
1309 	/* Map the linear mapping */  in setup_vm_final()
1354 		return -ENOMEM;  in reserve_crashkernel_low()
1357 	pr_info("crashkernel low memory reserved: 0x%016llx - 0x%016llx (%lld MB)\n",  in reserve_crashkernel_low()
1361 	crashk_low_res.end = low_base + low_size - 1;  in reserve_crashkernel_low()
1367  * reserve_crashkernel() - reserves memory for crash kernel
1400 	if (ret == -ENOENT) {  in reserve_crashkernel()
1411 		if (ret == -ENOENT)  in reserve_crashkernel()
1484 	pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n",  in reserve_crashkernel()
1488 	crashk_res.end = crash_base + crash_size - 1;  in reserve_crashkernel()
1496 	/* Depend on that Linear Mapping is ready */  in paging_init()
1524  * Pre-allocates page-table pages for a specific area in the kernel
1525  * page-table. Only the level which needs to be synchronized between
1526  * all page-tables is allocated because the synchronization can be
1567 	 * process page-tables later.  in preallocate_pgd_pages_range()
1569 	panic("Failed to pre-allocate %s pages for %s area\n", lvl, area);  in preallocate_pgd_pages_range()