1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * This file contains core hardware tag-based KASAN code.
4 *
5 * Copyright (c) 2020 Google, Inc.
6 * Author: Andrey Konovalov <andreyknvl@google.com>
7 */
8
9 #define pr_fmt(fmt) "kasan: " fmt
10
11 #include <linux/init.h>
12 #include <linux/kasan.h>
13 #include <linux/kernel.h>
14 #include <linux/memory.h>
15 #include <linux/mm.h>
16 #include <linux/static_key.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19
20 #include "kasan.h"
21
22 enum kasan_arg {
23 KASAN_ARG_DEFAULT,
24 KASAN_ARG_OFF,
25 KASAN_ARG_ON,
26 };
27
28 enum kasan_arg_mode {
29 KASAN_ARG_MODE_DEFAULT,
30 KASAN_ARG_MODE_SYNC,
31 KASAN_ARG_MODE_ASYNC,
32 KASAN_ARG_MODE_ASYMM,
33 };
34
35 enum kasan_arg_vmalloc {
36 KASAN_ARG_VMALLOC_DEFAULT,
37 KASAN_ARG_VMALLOC_OFF,
38 KASAN_ARG_VMALLOC_ON,
39 };
40
41 enum kasan_arg_stacktrace {
42 KASAN_ARG_STACKTRACE_DEFAULT,
43 KASAN_ARG_STACKTRACE_OFF,
44 KASAN_ARG_STACKTRACE_ON,
45 };
46
47 static enum kasan_arg kasan_arg __ro_after_init;
48 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
49 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
50 static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;
51
52 /*
53 * Whether KASAN is enabled at all.
54 * The value remains false until KASAN is initialized by kasan_init_hw_tags().
55 */
56 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
57 EXPORT_SYMBOL(kasan_flag_enabled);
58
59 /*
60 * Whether the selected mode is synchronous, asynchronous, or asymmetric.
61 * Defaults to KASAN_MODE_SYNC.
62 */
63 enum kasan_mode kasan_mode __ro_after_init;
64 EXPORT_SYMBOL_GPL(kasan_mode);
65
66 /* Whether to enable vmalloc tagging. */
67 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
68
69 /* Whether to collect alloc/free stack traces. */
70 DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
71
72 /* kasan=off/on */
early_kasan_flag(char * arg)73 static int __init early_kasan_flag(char *arg)
74 {
75 if (!arg)
76 return -EINVAL;
77
78 if (!strcmp(arg, "off"))
79 kasan_arg = KASAN_ARG_OFF;
80 else if (!strcmp(arg, "on"))
81 kasan_arg = KASAN_ARG_ON;
82 else
83 return -EINVAL;
84
85 return 0;
86 }
87 early_param("kasan", early_kasan_flag);
88
89 /* kasan.mode=sync/async/asymm */
early_kasan_mode(char * arg)90 static int __init early_kasan_mode(char *arg)
91 {
92 if (!arg)
93 return -EINVAL;
94
95 if (!strcmp(arg, "sync"))
96 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
97 else if (!strcmp(arg, "async"))
98 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
99 else if (!strcmp(arg, "asymm"))
100 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
101 else
102 return -EINVAL;
103
104 return 0;
105 }
106 early_param("kasan.mode", early_kasan_mode);
107
108 /* kasan.vmalloc=off/on */
early_kasan_flag_vmalloc(char * arg)109 static int __init early_kasan_flag_vmalloc(char *arg)
110 {
111 if (!arg)
112 return -EINVAL;
113
114 if (!strcmp(arg, "off"))
115 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
116 else if (!strcmp(arg, "on"))
117 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
118 else
119 return -EINVAL;
120
121 return 0;
122 }
123 early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
124
125 /* kasan.stacktrace=off/on */
early_kasan_flag_stacktrace(char * arg)126 static int __init early_kasan_flag_stacktrace(char *arg)
127 {
128 if (!arg)
129 return -EINVAL;
130
131 if (!strcmp(arg, "off"))
132 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
133 else if (!strcmp(arg, "on"))
134 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
135 else
136 return -EINVAL;
137
138 return 0;
139 }
140 early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
141
kasan_mode_info(void)142 static inline const char *kasan_mode_info(void)
143 {
144 if (kasan_mode == KASAN_MODE_ASYNC)
145 return "async";
146 else if (kasan_mode == KASAN_MODE_ASYMM)
147 return "asymm";
148 else
149 return "sync";
150 }
151
152 /*
153 * kasan_init_hw_tags_cpu() is called for each CPU.
154 * Not marked as __init as a CPU can be hot-plugged after boot.
155 */
kasan_init_hw_tags_cpu(void)156 void kasan_init_hw_tags_cpu(void)
157 {
158 /*
159 * There's no need to check that the hardware is MTE-capable here,
160 * as this function is only called for MTE-capable hardware.
161 */
162
163 /*
164 * If KASAN is disabled via command line, don't initialize it.
165 * When this function is called, kasan_flag_enabled is not yet
166 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
167 */
168 if (kasan_arg == KASAN_ARG_OFF)
169 return;
170
171 /*
172 * Enable async or asymm modes only when explicitly requested
173 * through the command line.
174 */
175 kasan_enable_tagging();
176 }
177
178 /* kasan_init_hw_tags() is called once on boot CPU. */
kasan_init_hw_tags(void)179 void __init kasan_init_hw_tags(void)
180 {
181 /* If hardware doesn't support MTE, don't initialize KASAN. */
182 if (!system_supports_mte())
183 return;
184
185 /* If KASAN is disabled via command line, don't initialize it. */
186 if (kasan_arg == KASAN_ARG_OFF)
187 return;
188
189 switch (kasan_arg_mode) {
190 case KASAN_ARG_MODE_DEFAULT:
191 /* Default is specified by kasan_mode definition. */
192 break;
193 case KASAN_ARG_MODE_SYNC:
194 kasan_mode = KASAN_MODE_SYNC;
195 break;
196 case KASAN_ARG_MODE_ASYNC:
197 kasan_mode = KASAN_MODE_ASYNC;
198 break;
199 case KASAN_ARG_MODE_ASYMM:
200 kasan_mode = KASAN_MODE_ASYMM;
201 break;
202 }
203
204 switch (kasan_arg_vmalloc) {
205 case KASAN_ARG_VMALLOC_DEFAULT:
206 /* Default is specified by kasan_flag_vmalloc definition. */
207 break;
208 case KASAN_ARG_VMALLOC_OFF:
209 static_branch_disable(&kasan_flag_vmalloc);
210 break;
211 case KASAN_ARG_VMALLOC_ON:
212 static_branch_enable(&kasan_flag_vmalloc);
213 break;
214 }
215
216 switch (kasan_arg_stacktrace) {
217 case KASAN_ARG_STACKTRACE_DEFAULT:
218 /* Default is specified by kasan_flag_stacktrace definition. */
219 break;
220 case KASAN_ARG_STACKTRACE_OFF:
221 static_branch_disable(&kasan_flag_stacktrace);
222 break;
223 case KASAN_ARG_STACKTRACE_ON:
224 static_branch_enable(&kasan_flag_stacktrace);
225 break;
226 }
227
228 /* KASAN is now initialized, enable it. */
229 static_branch_enable(&kasan_flag_enabled);
230
231 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n",
232 kasan_mode_info(),
233 kasan_vmalloc_enabled() ? "on" : "off",
234 kasan_stack_collection_enabled() ? "on" : "off");
235 }
236
237 #ifdef CONFIG_KASAN_VMALLOC
238
unpoison_vmalloc_pages(const void * addr,u8 tag)239 static void unpoison_vmalloc_pages(const void *addr, u8 tag)
240 {
241 struct vm_struct *area;
242 int i;
243
244 /*
245 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
246 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
247 * should belong to a single area.
248 */
249 area = find_vm_area((void *)addr);
250 if (WARN_ON(!area))
251 return;
252
253 for (i = 0; i < area->nr_pages; i++) {
254 struct page *page = area->pages[i];
255
256 page_kasan_tag_set(page, tag);
257 }
258 }
259
__kasan_unpoison_vmalloc(const void * start,unsigned long size,kasan_vmalloc_flags_t flags)260 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
261 kasan_vmalloc_flags_t flags)
262 {
263 u8 tag;
264 unsigned long redzone_start, redzone_size;
265
266 if (!kasan_vmalloc_enabled())
267 return (void *)start;
268
269 if (!is_vmalloc_or_module_addr(start))
270 return (void *)start;
271
272 /*
273 * Skip unpoisoning and assigning a pointer tag for non-VM_ALLOC
274 * mappings as:
275 *
276 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
277 * supports tagging physical memory. Therefore, it can only tag a
278 * single mapping of normal physical pages.
279 * 2. Hardware tag-based KASAN can only tag memory mapped with special
280 * mapping protection bits, see arch_vmalloc_pgprot_modify().
281 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
282 * providing these bits would require tracking all non-VM_ALLOC
283 * mappers.
284 *
285 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
286 * the first virtual mapping, which is created by vmalloc().
287 * Tagging the page_alloc memory backing that vmalloc() allocation is
288 * skipped, see ___GFP_SKIP_KASAN_UNPOISON.
289 *
290 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
291 */
292 if (!(flags & KASAN_VMALLOC_VM_ALLOC))
293 return (void *)start;
294
295 /*
296 * Don't tag executable memory.
297 * The kernel doesn't tolerate having the PC register tagged.
298 */
299 if (!(flags & KASAN_VMALLOC_PROT_NORMAL))
300 return (void *)start;
301
302 tag = kasan_random_tag();
303 start = set_tag(start, tag);
304
305 /* Unpoison and initialize memory up to size. */
306 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
307
308 /*
309 * Explicitly poison and initialize the in-page vmalloc() redzone.
310 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
311 * unpoison memory when populating shadow for vmalloc() space.
312 */
313 redzone_start = round_up((unsigned long)start + size,
314 KASAN_GRANULE_SIZE);
315 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
316 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
317 flags & KASAN_VMALLOC_INIT);
318
319 /*
320 * Set per-page tag flags to allow accessing physical memory for the
321 * vmalloc() mapping through page_address(vmalloc_to_page()).
322 */
323 unpoison_vmalloc_pages(start, tag);
324
325 return (void *)start;
326 }
327
__kasan_poison_vmalloc(const void * start,unsigned long size)328 void __kasan_poison_vmalloc(const void *start, unsigned long size)
329 {
330 /*
331 * No tagging here.
332 * The physical pages backing the vmalloc() allocation are poisoned
333 * through the usual page_alloc paths.
334 */
335 }
336
337 #endif
338
kasan_enable_tagging(void)339 void kasan_enable_tagging(void)
340 {
341 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
342 hw_enable_tagging_async();
343 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
344 hw_enable_tagging_asymm();
345 else
346 hw_enable_tagging_sync();
347 }
348
349 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
350
351 EXPORT_SYMBOL_GPL(kasan_enable_tagging);
352
kasan_force_async_fault(void)353 void kasan_force_async_fault(void)
354 {
355 hw_force_async_tag_fault();
356 }
357 EXPORT_SYMBOL_GPL(kasan_force_async_fault);
358
359 #endif
360