• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Based on arch/arm/mm/mmu.c
4  *
5  * Copyright (C) 1995-2005 Russell King
6  * Copyright (C) 2012 ARM Ltd.
7  */
8 
9 #include <linux/cache.h>
10 #include <linux/export.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/init.h>
14 #include <linux/ioport.h>
15 #include <linux/kexec.h>
16 #include <linux/libfdt.h>
17 #include <linux/mman.h>
18 #include <linux/nodemask.h>
19 #include <linux/memblock.h>
20 #include <linux/fs.h>
21 #include <linux/io.h>
22 #include <linux/mm.h>
23 #include <linux/vmalloc.h>
24 
25 #include <asm/barrier.h>
26 #include <asm/cputype.h>
27 #include <asm/fixmap.h>
28 #include <asm/kasan.h>
29 #include <asm/kernel-pgtable.h>
30 #include <asm/sections.h>
31 #include <asm/setup.h>
32 #include <linux/sizes.h>
33 #include <asm/tlb.h>
34 #include <asm/mmu_context.h>
35 #include <asm/ptdump.h>
36 #include <asm/tlbflush.h>
37 
38 #define NO_BLOCK_MAPPINGS	BIT(0)
39 #define NO_CONT_MAPPINGS	BIT(1)
40 
41 u64 idmap_t0sz = TCR_T0SZ(VA_BITS_MIN);
42 u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
43 
44 u64 __section(".mmuoff.data.write") vabits_actual;
45 EXPORT_SYMBOL(vabits_actual);
46 
47 u64 kimage_voffset __ro_after_init;
48 EXPORT_SYMBOL(kimage_voffset);
49 
50 /*
51  * Empty_zero_page is a special page that is used for zero-initialized data
52  * and COW.
53  */
54 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
55 EXPORT_SYMBOL(empty_zero_page);
56 
57 static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
58 static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
59 static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
60 
61 static DEFINE_SPINLOCK(swapper_pgdir_lock);
62 
set_swapper_pgd(pgd_t * pgdp,pgd_t pgd)63 void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
64 {
65 	pgd_t *fixmap_pgdp;
66 
67 	spin_lock(&swapper_pgdir_lock);
68 	fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp));
69 	WRITE_ONCE(*fixmap_pgdp, pgd);
70 	/*
71 	 * We need dsb(ishst) here to ensure the page-table-walker sees
72 	 * our new entry before set_p?d() returns. The fixmap's
73 	 * flush_tlb_kernel_range() via clear_fixmap() does this for us.
74 	 */
75 	pgd_clear_fixmap();
76 	spin_unlock(&swapper_pgdir_lock);
77 }
78 
phys_mem_access_prot(struct file * file,unsigned long pfn,unsigned long size,pgprot_t vma_prot)79 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
80 			      unsigned long size, pgprot_t vma_prot)
81 {
82 	if (!pfn_valid(pfn))
83 		return pgprot_noncached(vma_prot);
84 	else if (file->f_flags & O_SYNC)
85 		return pgprot_writecombine(vma_prot);
86 	return vma_prot;
87 }
88 EXPORT_SYMBOL(phys_mem_access_prot);
89 
early_pgtable_alloc(int shift)90 static phys_addr_t __init early_pgtable_alloc(int shift)
91 {
92 	phys_addr_t phys;
93 	void *ptr;
94 
95 	phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
96 	if (!phys)
97 		panic("Failed to allocate page table page\n");
98 
99 	/*
100 	 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
101 	 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
102 	 * any level of table.
103 	 */
104 	ptr = pte_set_fixmap(phys);
105 
106 	memset(ptr, 0, PAGE_SIZE);
107 
108 	/*
109 	 * Implicit barriers also ensure the zeroed page is visible to the page
110 	 * table walker
111 	 */
112 	pte_clear_fixmap();
113 
114 	return phys;
115 }
116 
pgattr_change_is_safe(u64 old,u64 new)117 static bool pgattr_change_is_safe(u64 old, u64 new)
118 {
119 	/*
120 	 * The following mapping attributes may be updated in live
121 	 * kernel mappings without the need for break-before-make.
122 	 */
123 	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG;
124 
125 	/* creating or taking down mappings is always safe */
126 	if (old == 0 || new == 0)
127 		return true;
128 
129 	/* live contiguous mappings may not be manipulated at all */
130 	if ((old | new) & PTE_CONT)
131 		return false;
132 
133 	/* Transitioning from Non-Global to Global is unsafe */
134 	if (old & ~new & PTE_NG)
135 		return false;
136 
137 	return ((old ^ new) & ~mask) == 0;
138 }
139 
init_pte(pmd_t * pmdp,unsigned long addr,unsigned long end,phys_addr_t phys,pgprot_t prot)140 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
141 		     phys_addr_t phys, pgprot_t prot)
142 {
143 	pte_t *ptep;
144 
145 	ptep = pte_set_fixmap_offset(pmdp, addr);
146 	do {
147 		pte_t old_pte = READ_ONCE(*ptep);
148 
149 		set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot));
150 
151 		/*
152 		 * After the PTE entry has been populated once, we
153 		 * only allow updates to the permission attributes.
154 		 */
155 		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte),
156 					      READ_ONCE(pte_val(*ptep))));
157 
158 		phys += PAGE_SIZE;
159 	} while (ptep++, addr += PAGE_SIZE, addr != end);
160 
161 	pte_clear_fixmap();
162 }
163 
alloc_init_cont_pte(pmd_t * pmdp,unsigned long addr,unsigned long end,phys_addr_t phys,pgprot_t prot,phys_addr_t (* pgtable_alloc)(int),int flags)164 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
165 				unsigned long end, phys_addr_t phys,
166 				pgprot_t prot,
167 				phys_addr_t (*pgtable_alloc)(int),
168 				int flags)
169 {
170 	unsigned long next;
171 	pmd_t pmd = READ_ONCE(*pmdp);
172 
173 	BUG_ON(pmd_sect(pmd));
174 	if (pmd_none(pmd)) {
175 		phys_addr_t pte_phys;
176 		BUG_ON(!pgtable_alloc);
177 		pte_phys = pgtable_alloc(PAGE_SHIFT);
178 		__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
179 		pmd = READ_ONCE(*pmdp);
180 	}
181 	BUG_ON(pmd_bad(pmd));
182 
183 	do {
184 		pgprot_t __prot = prot;
185 
186 		next = pte_cont_addr_end(addr, end);
187 
188 		/* use a contiguous mapping if the range is suitably aligned */
189 		if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
190 		    (flags & NO_CONT_MAPPINGS) == 0)
191 			__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
192 
193 		init_pte(pmdp, addr, next, phys, __prot);
194 
195 		phys += next - addr;
196 	} while (addr = next, addr != end);
197 }
198 
init_pmd(pud_t * pudp,unsigned long addr,unsigned long end,phys_addr_t phys,pgprot_t prot,phys_addr_t (* pgtable_alloc)(int),int flags)199 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
200 		     phys_addr_t phys, pgprot_t prot,
201 		     phys_addr_t (*pgtable_alloc)(int), int flags)
202 {
203 	unsigned long next;
204 	pmd_t *pmdp;
205 
206 	pmdp = pmd_set_fixmap_offset(pudp, addr);
207 	do {
208 		pmd_t old_pmd = READ_ONCE(*pmdp);
209 
210 		next = pmd_addr_end(addr, end);
211 
212 		/* try section mapping first */
213 		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
214 		    (flags & NO_BLOCK_MAPPINGS) == 0) {
215 			pmd_set_huge(pmdp, phys, prot);
216 
217 			/*
218 			 * After the PMD entry has been populated once, we
219 			 * only allow updates to the permission attributes.
220 			 */
221 			BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
222 						      READ_ONCE(pmd_val(*pmdp))));
223 		} else {
224 			alloc_init_cont_pte(pmdp, addr, next, phys, prot,
225 					    pgtable_alloc, flags);
226 
227 			BUG_ON(pmd_val(old_pmd) != 0 &&
228 			       pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
229 		}
230 		phys += next - addr;
231 	} while (pmdp++, addr = next, addr != end);
232 
233 	pmd_clear_fixmap();
234 }
235 
alloc_init_cont_pmd(pud_t * pudp,unsigned long addr,unsigned long end,phys_addr_t phys,pgprot_t prot,phys_addr_t (* pgtable_alloc)(int),int flags)236 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
237 				unsigned long end, phys_addr_t phys,
238 				pgprot_t prot,
239 				phys_addr_t (*pgtable_alloc)(int), int flags)
240 {
241 	unsigned long next;
242 	pud_t pud = READ_ONCE(*pudp);
243 
244 	/*
245 	 * Check for initial section mappings in the pgd/pud.
246 	 */
247 	BUG_ON(pud_sect(pud));
248 	if (pud_none(pud)) {
249 		phys_addr_t pmd_phys;
250 		BUG_ON(!pgtable_alloc);
251 		pmd_phys = pgtable_alloc(PMD_SHIFT);
252 		__pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);
253 		pud = READ_ONCE(*pudp);
254 	}
255 	BUG_ON(pud_bad(pud));
256 
257 	do {
258 		pgprot_t __prot = prot;
259 
260 		next = pmd_cont_addr_end(addr, end);
261 
262 		/* use a contiguous mapping if the range is suitably aligned */
263 		if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
264 		    (flags & NO_CONT_MAPPINGS) == 0)
265 			__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
266 
267 		init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags);
268 
269 		phys += next - addr;
270 	} while (addr = next, addr != end);
271 }
272 
use_1G_block(unsigned long addr,unsigned long next,unsigned long phys)273 static inline bool use_1G_block(unsigned long addr, unsigned long next,
274 			unsigned long phys)
275 {
276 	if (PAGE_SHIFT != 12)
277 		return false;
278 
279 	if (((addr | next | phys) & ~PUD_MASK) != 0)
280 		return false;
281 
282 	return true;
283 }
284 
alloc_init_pud(pgd_t * pgdp,unsigned long addr,unsigned long end,phys_addr_t phys,pgprot_t prot,phys_addr_t (* pgtable_alloc)(int),int flags)285 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
286 			   phys_addr_t phys, pgprot_t prot,
287 			   phys_addr_t (*pgtable_alloc)(int),
288 			   int flags)
289 {
290 	unsigned long next;
291 	pud_t *pudp;
292 	pgd_t pgd = READ_ONCE(*pgdp);
293 
294 	if (pgd_none(pgd)) {
295 		phys_addr_t pud_phys;
296 		BUG_ON(!pgtable_alloc);
297 		pud_phys = pgtable_alloc(PUD_SHIFT);
298 		__pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE);
299 		pgd = READ_ONCE(*pgdp);
300 	}
301 	BUG_ON(pgd_bad(pgd));
302 
303 	pudp = pud_set_fixmap_offset(pgdp, addr);
304 	do {
305 		pud_t old_pud = READ_ONCE(*pudp);
306 
307 		next = pud_addr_end(addr, end);
308 
309 		/*
310 		 * For 4K granule only, attempt to put down a 1GB block
311 		 */
312 		if (use_1G_block(addr, next, phys) &&
313 		    (flags & NO_BLOCK_MAPPINGS) == 0) {
314 			pud_set_huge(pudp, phys, prot);
315 
316 			/*
317 			 * After the PUD entry has been populated once, we
318 			 * only allow updates to the permission attributes.
319 			 */
320 			BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
321 						      READ_ONCE(pud_val(*pudp))));
322 		} else {
323 			alloc_init_cont_pmd(pudp, addr, next, phys, prot,
324 					    pgtable_alloc, flags);
325 
326 			BUG_ON(pud_val(old_pud) != 0 &&
327 			       pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
328 		}
329 		phys += next - addr;
330 	} while (pudp++, addr = next, addr != end);
331 
332 	pud_clear_fixmap();
333 }
334 
__create_pgd_mapping(pgd_t * pgdir,phys_addr_t phys,unsigned long virt,phys_addr_t size,pgprot_t prot,phys_addr_t (* pgtable_alloc)(int),int flags)335 static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
336 				 unsigned long virt, phys_addr_t size,
337 				 pgprot_t prot,
338 				 phys_addr_t (*pgtable_alloc)(int),
339 				 int flags)
340 {
341 	unsigned long addr, length, end, next;
342 	pgd_t *pgdp = pgd_offset_raw(pgdir, virt);
343 
344 	/*
345 	 * If the virtual and physical address don't have the same offset
346 	 * within a page, we cannot map the region as the caller expects.
347 	 */
348 	if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
349 		return;
350 
351 	phys &= PAGE_MASK;
352 	addr = virt & PAGE_MASK;
353 	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));
354 
355 	end = addr + length;
356 	do {
357 		next = pgd_addr_end(addr, end);
358 		alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,
359 			       flags);
360 		phys += next - addr;
361 	} while (pgdp++, addr = next, addr != end);
362 }
363 
__pgd_pgtable_alloc(int shift)364 static phys_addr_t __pgd_pgtable_alloc(int shift)
365 {
366 	void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL);
367 	BUG_ON(!ptr);
368 
369 	/* Ensure the zeroed page is visible to the page table walker */
370 	dsb(ishst);
371 	return __pa(ptr);
372 }
373 
pgd_pgtable_alloc(int shift)374 static phys_addr_t pgd_pgtable_alloc(int shift)
375 {
376 	phys_addr_t pa = __pgd_pgtable_alloc(shift);
377 
378 	/*
379 	 * Call proper page table ctor in case later we need to
380 	 * call core mm functions like apply_to_page_range() on
381 	 * this pre-allocated page table.
382 	 *
383 	 * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
384 	 * folded, and if so pgtable_pmd_page_ctor() becomes nop.
385 	 */
386 	if (shift == PAGE_SHIFT)
387 		BUG_ON(!pgtable_pte_page_ctor(phys_to_page(pa)));
388 	else if (shift == PMD_SHIFT)
389 		BUG_ON(!pgtable_pmd_page_ctor(phys_to_page(pa)));
390 
391 	return pa;
392 }
393 
394 /*
395  * This function can only be used to modify existing table entries,
396  * without allocating new levels of table. Note that this permits the
397  * creation of new section or page entries.
398  */
create_mapping_noalloc(phys_addr_t phys,unsigned long virt,phys_addr_t size,pgprot_t prot)399 static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
400 				  phys_addr_t size, pgprot_t prot)
401 {
402 	if (virt < PAGE_OFFSET) {
403 		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
404 			&phys, virt);
405 		return;
406 	}
407 	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
408 			     NO_CONT_MAPPINGS);
409 }
410 
create_pgd_mapping(struct mm_struct * mm,phys_addr_t phys,unsigned long virt,phys_addr_t size,pgprot_t prot,bool page_mappings_only)411 void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
412 			       unsigned long virt, phys_addr_t size,
413 			       pgprot_t prot, bool page_mappings_only)
414 {
415 	int flags = 0;
416 
417 	BUG_ON(mm == &init_mm);
418 
419 	if (page_mappings_only)
420 		flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
421 
422 	__create_pgd_mapping(mm->pgd, phys, virt, size, prot,
423 			     pgd_pgtable_alloc, flags);
424 }
425 
update_mapping_prot(phys_addr_t phys,unsigned long virt,phys_addr_t size,pgprot_t prot)426 static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
427 				phys_addr_t size, pgprot_t prot)
428 {
429 	if (virt < PAGE_OFFSET) {
430 		pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
431 			&phys, virt);
432 		return;
433 	}
434 
435 	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
436 			     NO_CONT_MAPPINGS);
437 
438 	/* flush the TLBs after updating live kernel mappings */
439 	flush_tlb_kernel_range(virt, virt + size);
440 }
441 
__map_memblock(pgd_t * pgdp,phys_addr_t start,phys_addr_t end,pgprot_t prot,int flags)442 static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start,
443 				  phys_addr_t end, pgprot_t prot, int flags)
444 {
445 	__create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
446 			     prot, early_pgtable_alloc, flags);
447 }
448 
mark_linear_text_alias_ro(void)449 void __init mark_linear_text_alias_ro(void)
450 {
451 	/*
452 	 * Remove the write permissions from the linear alias of .text/.rodata
453 	 */
454 	update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text),
455 			    (unsigned long)__init_begin - (unsigned long)_text,
456 			    PAGE_KERNEL_RO);
457 }
458 
map_mem(pgd_t * pgdp)459 static void __init map_mem(pgd_t *pgdp)
460 {
461 	phys_addr_t kernel_start = __pa_symbol(_text);
462 	phys_addr_t kernel_end = __pa_symbol(__init_begin);
463 	struct memblock_region *reg;
464 	int flags = 0;
465 
466 	if (rodata_full || debug_pagealloc_enabled())
467 		flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
468 
469 	/*
470 	 * Take care not to create a writable alias for the
471 	 * read-only text and rodata sections of the kernel image.
472 	 * So temporarily mark them as NOMAP to skip mappings in
473 	 * the following for-loop
474 	 */
475 	memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
476 #ifdef CONFIG_KEXEC_CORE
477 	if (crashk_res.end)
478 		memblock_mark_nomap(crashk_res.start,
479 				    resource_size(&crashk_res));
480 #endif
481 
482 	/* map all the memory banks */
483 	for_each_memblock(memory, reg) {
484 		phys_addr_t start = reg->base;
485 		phys_addr_t end = start + reg->size;
486 
487 		if (start >= end)
488 			break;
489 		if (memblock_is_nomap(reg))
490 			continue;
491 
492 		__map_memblock(pgdp, start, end, PAGE_KERNEL, flags);
493 	}
494 
495 	/*
496 	 * Map the linear alias of the [_text, __init_begin) interval
497 	 * as non-executable now, and remove the write permission in
498 	 * mark_linear_text_alias_ro() below (which will be called after
499 	 * alternative patching has completed). This makes the contents
500 	 * of the region accessible to subsystems such as hibernate,
501 	 * but protects it from inadvertent modification or execution.
502 	 * Note that contiguous mappings cannot be remapped in this way,
503 	 * so we should avoid them here.
504 	 */
505 	__map_memblock(pgdp, kernel_start, kernel_end,
506 		       PAGE_KERNEL, NO_CONT_MAPPINGS);
507 	memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
508 
509 #ifdef CONFIG_KEXEC_CORE
510 	/*
511 	 * Use page-level mappings here so that we can shrink the region
512 	 * in page granularity and put back unused memory to buddy system
513 	 * through /sys/kernel/kexec_crash_size interface.
514 	 */
515 	if (crashk_res.end) {
516 		__map_memblock(pgdp, crashk_res.start, crashk_res.end + 1,
517 			       PAGE_KERNEL,
518 			       NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
519 		memblock_clear_nomap(crashk_res.start,
520 				     resource_size(&crashk_res));
521 	}
522 #endif
523 }
524 
mark_rodata_ro(void)525 void mark_rodata_ro(void)
526 {
527 	unsigned long section_size;
528 
529 	/*
530 	 * mark .rodata as read only. Use __init_begin rather than __end_rodata
531 	 * to cover NOTES and EXCEPTION_TABLE.
532 	 */
533 	section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
534 	update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
535 			    section_size, PAGE_KERNEL_RO);
536 
537 	debug_checkwx();
538 }
539 
map_kernel_segment(pgd_t * pgdp,void * va_start,void * va_end,pgprot_t prot,struct vm_struct * vma,int flags,unsigned long vm_flags)540 static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end,
541 				      pgprot_t prot, struct vm_struct *vma,
542 				      int flags, unsigned long vm_flags)
543 {
544 	phys_addr_t pa_start = __pa_symbol(va_start);
545 	unsigned long size = va_end - va_start;
546 
547 	BUG_ON(!PAGE_ALIGNED(pa_start));
548 	BUG_ON(!PAGE_ALIGNED(size));
549 
550 	__create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot,
551 			     early_pgtable_alloc, flags);
552 
553 	if (!(vm_flags & VM_NO_GUARD))
554 		size += PAGE_SIZE;
555 
556 	vma->addr	= va_start;
557 	vma->phys_addr	= pa_start;
558 	vma->size	= size;
559 	vma->flags	= VM_MAP | vm_flags;
560 	vma->caller	= __builtin_return_address(0);
561 
562 	vm_area_add_early(vma);
563 }
564 
parse_rodata(char * arg)565 static int __init parse_rodata(char *arg)
566 {
567 	int ret = strtobool(arg, &rodata_enabled);
568 	if (!ret) {
569 		rodata_full = false;
570 		return 0;
571 	}
572 
573 	/* permit 'full' in addition to boolean options */
574 	if (strcmp(arg, "full"))
575 		return -EINVAL;
576 
577 	rodata_enabled = true;
578 	rodata_full = true;
579 	return 0;
580 }
581 early_param("rodata", parse_rodata);
582 
583 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
map_entry_trampoline(void)584 static int __init map_entry_trampoline(void)
585 {
586 	int i;
587 
588 	pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
589 	phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
590 
591 	/* The trampoline is always mapped and can therefore be global */
592 	pgprot_val(prot) &= ~PTE_NG;
593 
594 	/* Map only the text into the trampoline page table */
595 	memset(tramp_pg_dir, 0, PGD_SIZE);
596 	__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
597 			     entry_tramp_text_size(), prot,
598 			     __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
599 
600 	/* Map both the text and data into the kernel page table */
601 	for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
602 		__set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
603 			     pa_start + i * PAGE_SIZE, prot);
604 
605 	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
606 		extern char __entry_tramp_data_start[];
607 
608 		__set_fixmap(FIX_ENTRY_TRAMP_DATA,
609 			     __pa_symbol(__entry_tramp_data_start),
610 			     PAGE_KERNEL_RO);
611 	}
612 
613 	return 0;
614 }
615 core_initcall(map_entry_trampoline);
616 #endif
617 
618 /*
619  * Create fine-grained mappings for the kernel.
620  */
map_kernel(pgd_t * pgdp)621 static void __init map_kernel(pgd_t *pgdp)
622 {
623 	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext,
624 				vmlinux_initdata, vmlinux_data;
625 
626 	/*
627 	 * External debuggers may need to write directly to the text
628 	 * mapping to install SW breakpoints. Allow this (only) when
629 	 * explicitly requested with rodata=off.
630 	 */
631 	pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
632 
633 	/*
634 	 * Only rodata will be remapped with different permissions later on,
635 	 * all other segments are allowed to use contiguous mappings.
636 	 */
637 	map_kernel_segment(pgdp, _text, _etext, text_prot, &vmlinux_text, 0,
638 			   VM_NO_GUARD);
639 	map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL,
640 			   &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD);
641 	map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot,
642 			   &vmlinux_inittext, 0, VM_NO_GUARD);
643 	map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL,
644 			   &vmlinux_initdata, 0, VM_NO_GUARD);
645 	map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0);
646 
647 	if (!READ_ONCE(pgd_val(*pgd_offset_raw(pgdp, FIXADDR_START)))) {
648 		/*
649 		 * The fixmap falls in a separate pgd to the kernel, and doesn't
650 		 * live in the carveout for the swapper_pg_dir. We can simply
651 		 * re-use the existing dir for the fixmap.
652 		 */
653 		set_pgd(pgd_offset_raw(pgdp, FIXADDR_START),
654 			READ_ONCE(*pgd_offset_k(FIXADDR_START)));
655 	} else if (CONFIG_PGTABLE_LEVELS > 3) {
656 		pgd_t *bm_pgdp;
657 		pud_t *bm_pudp;
658 		/*
659 		 * The fixmap shares its top level pgd entry with the kernel
660 		 * mapping. This can really only occur when we are running
661 		 * with 16k/4 levels, so we can simply reuse the pud level
662 		 * entry instead.
663 		 */
664 		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
665 		bm_pgdp = pgd_offset_raw(pgdp, FIXADDR_START);
666 		bm_pudp = pud_set_fixmap_offset(bm_pgdp, FIXADDR_START);
667 		pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd));
668 		pud_clear_fixmap();
669 	} else {
670 		BUG();
671 	}
672 
673 	kasan_copy_shadow(pgdp);
674 }
675 
paging_init(void)676 void __init paging_init(void)
677 {
678 	pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir));
679 
680 	map_kernel(pgdp);
681 	map_mem(pgdp);
682 
683 	pgd_clear_fixmap();
684 
685 	cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
686 	init_mm.pgd = swapper_pg_dir;
687 
688 	memblock_free(__pa_symbol(init_pg_dir),
689 		      __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir));
690 
691 	memblock_allow_resize();
692 }
693 
694 /*
695  * Check whether a kernel address is valid (derived from arch/x86/).
696  */
kern_addr_valid(unsigned long addr)697 int kern_addr_valid(unsigned long addr)
698 {
699 	pgd_t *pgdp;
700 	pud_t *pudp, pud;
701 	pmd_t *pmdp, pmd;
702 	pte_t *ptep, pte;
703 
704 	if ((((long)addr) >> VA_BITS) != -1UL)
705 		return 0;
706 
707 	pgdp = pgd_offset_k(addr);
708 	if (pgd_none(READ_ONCE(*pgdp)))
709 		return 0;
710 
711 	pudp = pud_offset(pgdp, addr);
712 	pud = READ_ONCE(*pudp);
713 	if (pud_none(pud))
714 		return 0;
715 
716 	if (pud_sect(pud))
717 		return pfn_valid(pud_pfn(pud));
718 
719 	pmdp = pmd_offset(pudp, addr);
720 	pmd = READ_ONCE(*pmdp);
721 	if (pmd_none(pmd))
722 		return 0;
723 
724 	if (pmd_sect(pmd))
725 		return pfn_valid(pmd_pfn(pmd));
726 
727 	ptep = pte_offset_kernel(pmdp, addr);
728 	pte = READ_ONCE(*ptep);
729 	if (pte_none(pte))
730 		return 0;
731 
732 	return pfn_valid(pte_pfn(pte));
733 }
734 #ifdef CONFIG_SPARSEMEM_VMEMMAP
735 #if !ARM64_SWAPPER_USES_SECTION_MAPS
vmemmap_populate(unsigned long start,unsigned long end,int node,struct vmem_altmap * altmap)736 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
737 		struct vmem_altmap *altmap)
738 {
739 	return vmemmap_populate_basepages(start, end, node);
740 }
741 #else	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
vmemmap_populate(unsigned long start,unsigned long end,int node,struct vmem_altmap * altmap)742 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
743 		struct vmem_altmap *altmap)
744 {
745 	unsigned long addr = start;
746 	unsigned long next;
747 	pgd_t *pgdp;
748 	pud_t *pudp;
749 	pmd_t *pmdp;
750 
751 	do {
752 		next = pmd_addr_end(addr, end);
753 
754 		pgdp = vmemmap_pgd_populate(addr, node);
755 		if (!pgdp)
756 			return -ENOMEM;
757 
758 		pudp = vmemmap_pud_populate(pgdp, addr, node);
759 		if (!pudp)
760 			return -ENOMEM;
761 
762 		pmdp = pmd_offset(pudp, addr);
763 		if (pmd_none(READ_ONCE(*pmdp))) {
764 			void *p = NULL;
765 
766 			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
767 			if (!p)
768 				return -ENOMEM;
769 
770 			pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));
771 		} else
772 			vmemmap_verify((pte_t *)pmdp, node, addr, next);
773 	} while (addr = next, addr != end);
774 
775 	return 0;
776 }
777 #endif	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
vmemmap_free(unsigned long start,unsigned long end,struct vmem_altmap * altmap)778 void vmemmap_free(unsigned long start, unsigned long end,
779 		struct vmem_altmap *altmap)
780 {
781 }
782 #endif	/* CONFIG_SPARSEMEM_VMEMMAP */
783 
fixmap_pud(unsigned long addr)784 static inline pud_t * fixmap_pud(unsigned long addr)
785 {
786 	pgd_t *pgdp = pgd_offset_k(addr);
787 	pgd_t pgd = READ_ONCE(*pgdp);
788 
789 	BUG_ON(pgd_none(pgd) || pgd_bad(pgd));
790 
791 	return pud_offset_kimg(pgdp, addr);
792 }
793 
fixmap_pmd(unsigned long addr)794 static inline pmd_t * fixmap_pmd(unsigned long addr)
795 {
796 	pud_t *pudp = fixmap_pud(addr);
797 	pud_t pud = READ_ONCE(*pudp);
798 
799 	BUG_ON(pud_none(pud) || pud_bad(pud));
800 
801 	return pmd_offset_kimg(pudp, addr);
802 }
803 
fixmap_pte(unsigned long addr)804 static inline pte_t * fixmap_pte(unsigned long addr)
805 {
806 	return &bm_pte[pte_index(addr)];
807 }
808 
809 /*
810  * The p*d_populate functions call virt_to_phys implicitly so they can't be used
811  * directly on kernel symbols (bm_p*d). This function is called too early to use
812  * lm_alias so __p*d_populate functions must be used to populate with the
813  * physical address from __pa_symbol.
814  */
early_fixmap_init(void)815 void __init early_fixmap_init(void)
816 {
817 	pgd_t *pgdp, pgd;
818 	pud_t *pudp;
819 	pmd_t *pmdp;
820 	unsigned long addr = FIXADDR_START;
821 
822 	pgdp = pgd_offset_k(addr);
823 	pgd = READ_ONCE(*pgdp);
824 	if (CONFIG_PGTABLE_LEVELS > 3 &&
825 	    !(pgd_none(pgd) || pgd_page_paddr(pgd) == __pa_symbol(bm_pud))) {
826 		/*
827 		 * We only end up here if the kernel mapping and the fixmap
828 		 * share the top level pgd entry, which should only happen on
829 		 * 16k/4 levels configurations.
830 		 */
831 		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
832 		pudp = pud_offset_kimg(pgdp, addr);
833 	} else {
834 		if (pgd_none(pgd))
835 			__pgd_populate(pgdp, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
836 		pudp = fixmap_pud(addr);
837 	}
838 	if (pud_none(READ_ONCE(*pudp)))
839 		__pud_populate(pudp, __pa_symbol(bm_pmd), PMD_TYPE_TABLE);
840 	pmdp = fixmap_pmd(addr);
841 	__pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
842 
843 	/*
844 	 * The boot-ioremap range spans multiple pmds, for which
845 	 * we are not prepared:
846 	 */
847 	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
848 		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
849 
850 	if ((pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
851 	     || pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
852 		WARN_ON(1);
853 		pr_warn("pmdp %p != %p, %p\n",
854 			pmdp, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
855 			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
856 		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
857 			fix_to_virt(FIX_BTMAP_BEGIN));
858 		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
859 			fix_to_virt(FIX_BTMAP_END));
860 
861 		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
862 		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
863 	}
864 }
865 
866 /*
867  * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we
868  * ever need to use IPIs for TLB broadcasting, then we're in trouble here.
869  */
__set_fixmap(enum fixed_addresses idx,phys_addr_t phys,pgprot_t flags)870 void __set_fixmap(enum fixed_addresses idx,
871 			       phys_addr_t phys, pgprot_t flags)
872 {
873 	unsigned long addr = __fix_to_virt(idx);
874 	pte_t *ptep;
875 
876 	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
877 
878 	ptep = fixmap_pte(addr);
879 
880 	if (pgprot_val(flags)) {
881 		set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
882 	} else {
883 		pte_clear(&init_mm, addr, ptep);
884 		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
885 	}
886 }
887 
fixmap_remap_fdt(phys_addr_t dt_phys,int * size,pgprot_t prot)888 void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
889 {
890 	const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
891 	int offset;
892 	void *dt_virt;
893 
894 	/*
895 	 * Check whether the physical FDT address is set and meets the minimum
896 	 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
897 	 * at least 8 bytes so that we can always access the magic and size
898 	 * fields of the FDT header after mapping the first chunk, double check
899 	 * here if that is indeed the case.
900 	 */
901 	BUILD_BUG_ON(MIN_FDT_ALIGN < 8);
902 	if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
903 		return NULL;
904 
905 	/*
906 	 * Make sure that the FDT region can be mapped without the need to
907 	 * allocate additional translation table pages, so that it is safe
908 	 * to call create_mapping_noalloc() this early.
909 	 *
910 	 * On 64k pages, the FDT will be mapped using PTEs, so we need to
911 	 * be in the same PMD as the rest of the fixmap.
912 	 * On 4k pages, we'll use section mappings for the FDT so we only
913 	 * have to be in the same PUD.
914 	 */
915 	BUILD_BUG_ON(dt_virt_base % SZ_2M);
916 
917 	BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
918 		     __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
919 
920 	offset = dt_phys % SWAPPER_BLOCK_SIZE;
921 	dt_virt = (void *)dt_virt_base + offset;
922 
923 	/* map the first chunk so we can read the size from the header */
924 	create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE),
925 			dt_virt_base, SWAPPER_BLOCK_SIZE, prot);
926 
927 	if (fdt_magic(dt_virt) != FDT_MAGIC)
928 		return NULL;
929 
930 	*size = fdt_totalsize(dt_virt);
931 	if (*size > MAX_FDT_SIZE)
932 		return NULL;
933 
934 	if (offset + *size > SWAPPER_BLOCK_SIZE)
935 		create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
936 			       round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot);
937 
938 	return dt_virt;
939 }
940 
arch_ioremap_p4d_supported(void)941 int __init arch_ioremap_p4d_supported(void)
942 {
943 	return 0;
944 }
945 
arch_ioremap_pud_supported(void)946 int __init arch_ioremap_pud_supported(void)
947 {
948 	/*
949 	 * Only 4k granule supports level 1 block mappings.
950 	 * SW table walks can't handle removal of intermediate entries.
951 	 */
952 	return IS_ENABLED(CONFIG_ARM64_4K_PAGES) &&
953 	       !IS_ENABLED(CONFIG_ARM64_PTDUMP_DEBUGFS);
954 }
955 
arch_ioremap_pmd_supported(void)956 int __init arch_ioremap_pmd_supported(void)
957 {
958 	/* See arch_ioremap_pud_supported() */
959 	return !IS_ENABLED(CONFIG_ARM64_PTDUMP_DEBUGFS);
960 }
961 
pud_set_huge(pud_t * pudp,phys_addr_t phys,pgprot_t prot)962 int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
963 {
964 	pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot));
965 
966 	/* Only allow permission changes for now */
967 	if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)),
968 				   pud_val(new_pud)))
969 		return 0;
970 
971 	VM_BUG_ON(phys & ~PUD_MASK);
972 	set_pud(pudp, new_pud);
973 	return 1;
974 }
975 
pmd_set_huge(pmd_t * pmdp,phys_addr_t phys,pgprot_t prot)976 int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
977 {
978 	pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot));
979 
980 	/* Only allow permission changes for now */
981 	if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)),
982 				   pmd_val(new_pmd)))
983 		return 0;
984 
985 	VM_BUG_ON(phys & ~PMD_MASK);
986 	set_pmd(pmdp, new_pmd);
987 	return 1;
988 }
989 
pud_clear_huge(pud_t * pudp)990 int pud_clear_huge(pud_t *pudp)
991 {
992 	if (!pud_sect(READ_ONCE(*pudp)))
993 		return 0;
994 	pud_clear(pudp);
995 	return 1;
996 }
997 
pmd_clear_huge(pmd_t * pmdp)998 int pmd_clear_huge(pmd_t *pmdp)
999 {
1000 	if (!pmd_sect(READ_ONCE(*pmdp)))
1001 		return 0;
1002 	pmd_clear(pmdp);
1003 	return 1;
1004 }
1005 
pmd_free_pte_page(pmd_t * pmdp,unsigned long addr)1006 int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
1007 {
1008 	pte_t *table;
1009 	pmd_t pmd;
1010 
1011 	pmd = READ_ONCE(*pmdp);
1012 
1013 	if (!pmd_table(pmd)) {
1014 		VM_WARN_ON(1);
1015 		return 1;
1016 	}
1017 
1018 	table = pte_offset_kernel(pmdp, addr);
1019 	pmd_clear(pmdp);
1020 	__flush_tlb_kernel_pgtable(addr);
1021 	pte_free_kernel(NULL, table);
1022 	return 1;
1023 }
1024 
pud_free_pmd_page(pud_t * pudp,unsigned long addr)1025 int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
1026 {
1027 	pmd_t *table;
1028 	pmd_t *pmdp;
1029 	pud_t pud;
1030 	unsigned long next, end;
1031 
1032 	pud = READ_ONCE(*pudp);
1033 
1034 	if (!pud_table(pud)) {
1035 		VM_WARN_ON(1);
1036 		return 1;
1037 	}
1038 
1039 	table = pmd_offset(pudp, addr);
1040 	pmdp = table;
1041 	next = addr;
1042 	end = addr + PUD_SIZE;
1043 	do {
1044 		pmd_free_pte_page(pmdp, next);
1045 	} while (pmdp++, next += PMD_SIZE, next != end);
1046 
1047 	pud_clear(pudp);
1048 	__flush_tlb_kernel_pgtable(addr);
1049 	pmd_free(NULL, table);
1050 	return 1;
1051 }
1052 
p4d_free_pud_page(p4d_t * p4d,unsigned long addr)1053 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr)
1054 {
1055 	return 0;	/* Don't attempt a block mapping */
1056 }
1057 
1058 #ifdef CONFIG_MEMORY_HOTPLUG
arch_add_memory(int nid,u64 start,u64 size,struct mhp_restrictions * restrictions)1059 int arch_add_memory(int nid, u64 start, u64 size,
1060 			struct mhp_restrictions *restrictions)
1061 {
1062 	int flags = 0;
1063 
1064 	if (rodata_full || debug_pagealloc_enabled())
1065 		flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
1066 
1067 	__create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
1068 			     size, PAGE_KERNEL, __pgd_pgtable_alloc, flags);
1069 
1070 	return __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
1071 			   restrictions);
1072 }
arch_remove_memory(int nid,u64 start,u64 size,struct vmem_altmap * altmap)1073 void arch_remove_memory(int nid, u64 start, u64 size,
1074 			struct vmem_altmap *altmap)
1075 {
1076 	unsigned long start_pfn = start >> PAGE_SHIFT;
1077 	unsigned long nr_pages = size >> PAGE_SHIFT;
1078 
1079 	/*
1080 	 * FIXME: Cleanup page tables (also in arch_add_memory() in case
1081 	 * adding fails). Until then, this function should only be used
1082 	 * during memory hotplug (adding memory), not for memory
1083 	 * unplug. ARCH_ENABLE_MEMORY_HOTREMOVE must not be
1084 	 * unlocked yet.
1085 	 */
1086 	__remove_pages(start_pfn, nr_pages, altmap);
1087 }
1088 #endif
1089