• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * This file contains some kasan initialization code.
3  *
4  * Copyright (c) 2015 Samsung Electronics Co., Ltd.
5  * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  *
11  */
12 
13 #include <linux/bootmem.h>
14 #include <linux/init.h>
15 #include <linux/kasan.h>
16 #include <linux/kernel.h>
17 #include <linux/memblock.h>
18 #include <linux/mm.h>
19 #include <linux/pfn.h>
20 
21 #include <asm/page.h>
22 #include <asm/pgalloc.h>
23 
24 /*
25  * This page serves two purposes:
26  *   - It used as early shadow memory. The entire shadow region populated
27  *     with this page, before we will be able to setup normal shadow memory.
28  *   - Latter it reused it as zero shadow to cover large ranges of memory
29  *     that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
30  */
31 unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
32 
33 #if CONFIG_PGTABLE_LEVELS > 4
34 p4d_t kasan_zero_p4d[PTRS_PER_P4D] __page_aligned_bss;
35 #endif
36 #if CONFIG_PGTABLE_LEVELS > 3
37 pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
38 #endif
39 #if CONFIG_PGTABLE_LEVELS > 2
40 pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
41 #endif
42 pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
43 
early_alloc(size_t size,int node)44 static __init void *early_alloc(size_t size, int node)
45 {
46 	return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
47 					BOOTMEM_ALLOC_ACCESSIBLE, node);
48 }
49 
zero_pte_populate(pmd_t * pmd,unsigned long addr,unsigned long end)50 static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
51 				unsigned long end)
52 {
53 	pte_t *pte = pte_offset_kernel(pmd, addr);
54 	pte_t zero_pte;
55 
56 	zero_pte = pfn_pte(PFN_DOWN(__pa_symbol(kasan_zero_page)), PAGE_KERNEL);
57 	zero_pte = pte_wrprotect(zero_pte);
58 
59 	while (addr + PAGE_SIZE <= end) {
60 		set_pte_at(&init_mm, addr, pte, zero_pte);
61 		addr += PAGE_SIZE;
62 		pte = pte_offset_kernel(pmd, addr);
63 	}
64 }
65 
zero_pmd_populate(pud_t * pud,unsigned long addr,unsigned long end)66 static void __init zero_pmd_populate(pud_t *pud, unsigned long addr,
67 				unsigned long end)
68 {
69 	pmd_t *pmd = pmd_offset(pud, addr);
70 	unsigned long next;
71 
72 	do {
73 		next = pmd_addr_end(addr, end);
74 
75 		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
76 			pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
77 			continue;
78 		}
79 
80 		if (pmd_none(*pmd)) {
81 			pmd_populate_kernel(&init_mm, pmd,
82 					early_alloc(PAGE_SIZE, NUMA_NO_NODE));
83 		}
84 		zero_pte_populate(pmd, addr, next);
85 	} while (pmd++, addr = next, addr != end);
86 }
87 
zero_pud_populate(p4d_t * p4d,unsigned long addr,unsigned long end)88 static void __init zero_pud_populate(p4d_t *p4d, unsigned long addr,
89 				unsigned long end)
90 {
91 	pud_t *pud = pud_offset(p4d, addr);
92 	unsigned long next;
93 
94 	do {
95 		next = pud_addr_end(addr, end);
96 		if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
97 			pmd_t *pmd;
98 
99 			pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
100 			pmd = pmd_offset(pud, addr);
101 			pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
102 			continue;
103 		}
104 
105 		if (pud_none(*pud)) {
106 			pud_populate(&init_mm, pud,
107 				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
108 		}
109 		zero_pmd_populate(pud, addr, next);
110 	} while (pud++, addr = next, addr != end);
111 }
112 
zero_p4d_populate(pgd_t * pgd,unsigned long addr,unsigned long end)113 static void __init zero_p4d_populate(pgd_t *pgd, unsigned long addr,
114 				unsigned long end)
115 {
116 	p4d_t *p4d = p4d_offset(pgd, addr);
117 	unsigned long next;
118 
119 	do {
120 		next = p4d_addr_end(addr, end);
121 		if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
122 			pud_t *pud;
123 			pmd_t *pmd;
124 
125 			p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
126 			pud = pud_offset(p4d, addr);
127 			pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
128 			pmd = pmd_offset(pud, addr);
129 			pmd_populate_kernel(&init_mm, pmd,
130 						lm_alias(kasan_zero_pte));
131 			continue;
132 		}
133 
134 		if (p4d_none(*p4d)) {
135 			p4d_populate(&init_mm, p4d,
136 				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
137 		}
138 		zero_pud_populate(p4d, addr, next);
139 	} while (p4d++, addr = next, addr != end);
140 }
141 
142 /**
143  * kasan_populate_zero_shadow - populate shadow memory region with
144  *                               kasan_zero_page
145  * @shadow_start - start of the memory range to populate
146  * @shadow_end   - end of the memory range to populate
147  */
kasan_populate_zero_shadow(const void * shadow_start,const void * shadow_end)148 void __init kasan_populate_zero_shadow(const void *shadow_start,
149 				const void *shadow_end)
150 {
151 	unsigned long addr = (unsigned long)shadow_start;
152 	unsigned long end = (unsigned long)shadow_end;
153 	pgd_t *pgd = pgd_offset_k(addr);
154 	unsigned long next;
155 
156 	do {
157 		next = pgd_addr_end(addr, end);
158 
159 		if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
160 			p4d_t *p4d;
161 			pud_t *pud;
162 			pmd_t *pmd;
163 
164 			/*
165 			 * kasan_zero_pud should be populated with pmds
166 			 * at this moment.
167 			 * [pud,pmd]_populate*() below needed only for
168 			 * 3,2 - level page tables where we don't have
169 			 * puds,pmds, so pgd_populate(), pud_populate()
170 			 * is noops.
171 			 *
172 			 * The ifndef is required to avoid build breakage.
173 			 *
174 			 * With 5level-fixup.h, pgd_populate() is not nop and
175 			 * we reference kasan_zero_p4d. It's not defined
176 			 * unless 5-level paging enabled.
177 			 *
178 			 * The ifndef can be dropped once all KASAN-enabled
179 			 * architectures will switch to pgtable-nop4d.h.
180 			 */
181 #ifndef __ARCH_HAS_5LEVEL_HACK
182 			pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_p4d));
183 #endif
184 			p4d = p4d_offset(pgd, addr);
185 			p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
186 			pud = pud_offset(p4d, addr);
187 			pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
188 			pmd = pmd_offset(pud, addr);
189 			pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
190 			continue;
191 		}
192 
193 		if (pgd_none(*pgd)) {
194 			pgd_populate(&init_mm, pgd,
195 				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
196 		}
197 		zero_p4d_populate(pgd, addr, next);
198 	} while (pgd++, addr = next, addr != end);
199 }
200