1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * This file contains core hardware tag-based KASAN code.
4 *
5 * Copyright (c) 2020 Google, Inc.
6 * Author: Andrey Konovalov <andreyknvl@google.com>
7 */
8
9 #define pr_fmt(fmt) "kasan: " fmt
10
11 #include <linux/init.h>
12 #include <linux/kasan.h>
13 #include <linux/kernel.h>
14 #include <linux/memory.h>
15 #include <linux/mm.h>
16 #include <linux/static_key.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19
20 #include "kasan.h"
21
22 enum kasan_arg {
23 KASAN_ARG_DEFAULT,
24 KASAN_ARG_OFF,
25 KASAN_ARG_ON,
26 };
27
28 enum kasan_arg_mode {
29 KASAN_ARG_MODE_DEFAULT,
30 KASAN_ARG_MODE_SYNC,
31 KASAN_ARG_MODE_ASYNC,
32 };
33
34 enum kasan_arg_stacktrace {
35 KASAN_ARG_STACKTRACE_DEFAULT,
36 KASAN_ARG_STACKTRACE_OFF,
37 KASAN_ARG_STACKTRACE_ON,
38 };
39
40 enum kasan_arg_fault {
41 KASAN_ARG_FAULT_DEFAULT,
42 KASAN_ARG_FAULT_REPORT,
43 KASAN_ARG_FAULT_PANIC,
44 };
45
46 static enum kasan_arg kasan_arg __ro_after_init;
47 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
48 static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init;
49 static enum kasan_arg_fault kasan_arg_fault __ro_after_init;
50
51 /* Whether KASAN is enabled at all. */
52 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
53 EXPORT_SYMBOL(kasan_flag_enabled);
54
55 /* Whether the asynchronous mode is enabled. */
56 bool kasan_flag_async __ro_after_init;
57 EXPORT_SYMBOL_GPL(kasan_flag_async);
58
59 /* Whether to collect alloc/free stack traces. */
60 DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
61
62 /* Whether to panic or print a report and disable tag checking on fault. */
63 bool kasan_flag_panic __ro_after_init;
64
65 /* kasan=off/on */
early_kasan_flag(char * arg)66 static int __init early_kasan_flag(char *arg)
67 {
68 if (!arg)
69 return -EINVAL;
70
71 if (!strcmp(arg, "off"))
72 kasan_arg = KASAN_ARG_OFF;
73 else if (!strcmp(arg, "on"))
74 kasan_arg = KASAN_ARG_ON;
75 else
76 return -EINVAL;
77
78 return 0;
79 }
80 early_param("kasan", early_kasan_flag);
81
82 /* kasan.mode=sync/async */
early_kasan_mode(char * arg)83 static int __init early_kasan_mode(char *arg)
84 {
85 if (!arg)
86 return -EINVAL;
87
88 if (!strcmp(arg, "sync"))
89 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
90 else if (!strcmp(arg, "async"))
91 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
92 else
93 return -EINVAL;
94
95 return 0;
96 }
97 early_param("kasan.mode", early_kasan_mode);
98
99 /* kasan.stacktrace=off/on */
early_kasan_flag_stacktrace(char * arg)100 static int __init early_kasan_flag_stacktrace(char *arg)
101 {
102 if (!arg)
103 return -EINVAL;
104
105 if (!strcmp(arg, "off"))
106 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
107 else if (!strcmp(arg, "on"))
108 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
109 else
110 return -EINVAL;
111
112 return 0;
113 }
114 early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
115
116 /* kasan.fault=report/panic */
early_kasan_fault(char * arg)117 static int __init early_kasan_fault(char *arg)
118 {
119 if (!arg)
120 return -EINVAL;
121
122 if (!strcmp(arg, "report"))
123 kasan_arg_fault = KASAN_ARG_FAULT_REPORT;
124 else if (!strcmp(arg, "panic"))
125 kasan_arg_fault = KASAN_ARG_FAULT_PANIC;
126 else
127 return -EINVAL;
128
129 return 0;
130 }
131 early_param("kasan.fault", early_kasan_fault);
132
133 /* kasan_init_hw_tags_cpu() is called for each CPU. */
kasan_init_hw_tags_cpu(void)134 void kasan_init_hw_tags_cpu(void)
135 {
136 /*
137 * There's no need to check that the hardware is MTE-capable here,
138 * as this function is only called for MTE-capable hardware.
139 */
140
141 /* If KASAN is disabled via command line, don't initialize it. */
142 if (kasan_arg == KASAN_ARG_OFF)
143 return;
144
145 /*
146 * Enable async mode only when explicitly requested through
147 * the command line.
148 */
149 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
150 hw_enable_tagging_async();
151 else
152 hw_enable_tagging_sync();
153 }
154
155 /* kasan_init_hw_tags() is called once on boot CPU. */
kasan_init_hw_tags(void)156 void __init kasan_init_hw_tags(void)
157 {
158 /* If hardware doesn't support MTE, don't initialize KASAN. */
159 if (!system_supports_mte())
160 return;
161
162 /* If KASAN is disabled via command line, don't initialize it. */
163 if (kasan_arg == KASAN_ARG_OFF)
164 return;
165
166 /* Enable KASAN. */
167 static_branch_enable(&kasan_flag_enabled);
168
169 switch (kasan_arg_mode) {
170 case KASAN_ARG_MODE_DEFAULT:
171 /*
172 * Default to sync mode.
173 * Do nothing, kasan_flag_async keeps its default value.
174 */
175 break;
176 case KASAN_ARG_MODE_SYNC:
177 /* Do nothing, kasan_flag_async keeps its default value. */
178 break;
179 case KASAN_ARG_MODE_ASYNC:
180 /* Async mode enabled. */
181 kasan_flag_async = true;
182 break;
183 }
184
185 switch (kasan_arg_stacktrace) {
186 case KASAN_ARG_STACKTRACE_DEFAULT:
187 /* Default to enabling stack trace collection. */
188 static_branch_enable(&kasan_flag_stacktrace);
189 break;
190 case KASAN_ARG_STACKTRACE_OFF:
191 /* Do nothing, kasan_flag_stacktrace keeps its default value. */
192 break;
193 case KASAN_ARG_STACKTRACE_ON:
194 static_branch_enable(&kasan_flag_stacktrace);
195 break;
196 }
197
198 switch (kasan_arg_fault) {
199 case KASAN_ARG_FAULT_DEFAULT:
200 /*
201 * Default to no panic on report.
202 * Do nothing, kasan_flag_panic keeps its default value.
203 */
204 break;
205 case KASAN_ARG_FAULT_REPORT:
206 /* Do nothing, kasan_flag_panic keeps its default value. */
207 break;
208 case KASAN_ARG_FAULT_PANIC:
209 /* Enable panic on report. */
210 kasan_flag_panic = true;
211 break;
212 }
213
214 pr_info("KernelAddressSanitizer initialized\n");
215 }
216
kasan_set_free_info(struct kmem_cache * cache,void * object,u8 tag)217 void kasan_set_free_info(struct kmem_cache *cache,
218 void *object, u8 tag)
219 {
220 struct kasan_alloc_meta *alloc_meta;
221
222 alloc_meta = kasan_get_alloc_meta(cache, object);
223 if (alloc_meta)
224 kasan_set_track(&alloc_meta->free_track[0], GFP_NOWAIT);
225 }
226
kasan_get_free_track(struct kmem_cache * cache,void * object,u8 tag)227 struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
228 void *object, u8 tag)
229 {
230 struct kasan_alloc_meta *alloc_meta;
231
232 alloc_meta = kasan_get_alloc_meta(cache, object);
233 if (!alloc_meta)
234 return NULL;
235
236 return &alloc_meta->free_track[0];
237 }
238
kasan_alloc_pages(struct page * page,unsigned int order,gfp_t flags)239 void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags)
240 {
241 /*
242 * This condition should match the one in post_alloc_hook() in
243 * page_alloc.c.
244 */
245 bool init = !want_init_on_free() && want_init_on_alloc(flags);
246
247 if (flags & __GFP_SKIP_KASAN_POISON)
248 SetPageSkipKASanPoison(page);
249
250 if (flags & __GFP_ZEROTAGS) {
251 int i;
252
253 for (i = 0; i != 1 << order; ++i)
254 tag_clear_highpage(page + i);
255 } else {
256 kasan_unpoison_pages(page, order, init);
257 }
258 }
259
kasan_free_pages(struct page * page,unsigned int order)260 void kasan_free_pages(struct page *page, unsigned int order)
261 {
262 /*
263 * This condition should match the one in free_pages_prepare() in
264 * page_alloc.c.
265 */
266 bool init = want_init_on_free();
267
268 kasan_poison_pages(page, order, init);
269 }
270
271 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
272
kasan_set_tagging_report_once(bool state)273 void kasan_set_tagging_report_once(bool state)
274 {
275 hw_set_tagging_report_once(state);
276 }
277 EXPORT_SYMBOL_GPL(kasan_set_tagging_report_once);
278
kasan_enable_tagging_sync(void)279 void kasan_enable_tagging_sync(void)
280 {
281 hw_enable_tagging_sync();
282 }
283 EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync);
284
kasan_force_async_fault(void)285 void kasan_force_async_fault(void)
286 {
287 hw_force_async_tag_fault();
288 }
289 EXPORT_SYMBOL_GPL(kasan_force_async_fault);
290
291 #endif
292