1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_KASAN_H
3 #define _LINUX_KASAN_H
4
5 #include <linux/bug.h>
6 #include <linux/kasan-enabled.h>
7 #include <linux/kernel.h>
8 #include <linux/static_key.h>
9 #include <linux/types.h>
10
11 struct kmem_cache;
12 struct page;
13 struct vm_struct;
14 struct task_struct;
15
16 #ifdef CONFIG_KASAN
17
18 #include <linux/linkage.h>
19 #include <asm/kasan.h>
20
21 #endif
22
23 typedef unsigned int __bitwise kasan_vmalloc_flags_t;
24
25 #define KASAN_VMALLOC_NONE ((__force kasan_vmalloc_flags_t)0x00u)
26 #define KASAN_VMALLOC_INIT ((__force kasan_vmalloc_flags_t)0x01u)
27 #define KASAN_VMALLOC_VM_ALLOC ((__force kasan_vmalloc_flags_t)0x02u)
28 #define KASAN_VMALLOC_PROT_NORMAL ((__force kasan_vmalloc_flags_t)0x04u)
29
30 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
31
32 #include <linux/pgtable.h>
33
34 /* Software KASAN implementations use shadow memory. */
35
36 #ifdef CONFIG_KASAN_SW_TAGS
37 /* This matches KASAN_TAG_INVALID. */
38 #define KASAN_SHADOW_INIT 0xFE
39 #else
40 #define KASAN_SHADOW_INIT 0
41 #endif
42
43 #ifndef PTE_HWTABLE_PTRS
44 #define PTE_HWTABLE_PTRS 0
45 #endif
46
47 extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
48 extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
49 extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
50 extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
51 extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];
52
53 int kasan_populate_early_shadow(const void *shadow_start,
54 const void *shadow_end);
55
kasan_mem_to_shadow(const void * addr)56 static inline void *kasan_mem_to_shadow(const void *addr)
57 {
58 return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
59 + KASAN_SHADOW_OFFSET;
60 }
61
62 int kasan_add_zero_shadow(void *start, unsigned long size);
63 void kasan_remove_zero_shadow(void *start, unsigned long size);
64
65 /* Enable reporting bugs after kasan_disable_current() */
66 extern void kasan_enable_current(void);
67
68 /* Disable reporting bugs for current task */
69 extern void kasan_disable_current(void);
70
71 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
72
kasan_add_zero_shadow(void * start,unsigned long size)73 static inline int kasan_add_zero_shadow(void *start, unsigned long size)
74 {
75 return 0;
76 }
kasan_remove_zero_shadow(void * start,unsigned long size)77 static inline void kasan_remove_zero_shadow(void *start,
78 unsigned long size)
79 {}
80
kasan_enable_current(void)81 static inline void kasan_enable_current(void) {}
kasan_disable_current(void)82 static inline void kasan_disable_current(void) {}
83
84 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
85
kasan_has_integrated_init(void)86 static inline bool kasan_has_integrated_init(void)
87 {
88 return kasan_hw_tags_enabled();
89 }
90
91 #ifdef CONFIG_KASAN
92
93 struct kasan_cache {
94 #ifdef CONFIG_KASAN_GENERIC
95 int alloc_meta_offset;
96 int free_meta_offset;
97 #endif
98 bool is_kmalloc;
99 };
100
101 void __kasan_unpoison_range(const void *addr, size_t size);
kasan_unpoison_range(const void * addr,size_t size)102 static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
103 {
104 if (kasan_enabled())
105 __kasan_unpoison_range(addr, size);
106 }
107
108 void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
kasan_poison_pages(struct page * page,unsigned int order,bool init)109 static __always_inline void kasan_poison_pages(struct page *page,
110 unsigned int order, bool init)
111 {
112 if (kasan_enabled())
113 __kasan_poison_pages(page, order, init);
114 }
115
116 bool __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
kasan_unpoison_pages(struct page * page,unsigned int order,bool init)117 static __always_inline bool kasan_unpoison_pages(struct page *page,
118 unsigned int order, bool init)
119 {
120 if (kasan_enabled())
121 return __kasan_unpoison_pages(page, order, init);
122 return false;
123 }
124
125 void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
kasan_cache_create_kmalloc(struct kmem_cache * cache)126 static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
127 {
128 if (kasan_enabled())
129 __kasan_cache_create_kmalloc(cache);
130 }
131
132 void __kasan_poison_slab(struct page *page);
kasan_poison_slab(struct page * page)133 static __always_inline void kasan_poison_slab(struct page *page)
134 {
135 if (kasan_enabled())
136 __kasan_poison_slab(page);
137 }
138
139 void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
kasan_unpoison_object_data(struct kmem_cache * cache,void * object)140 static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
141 void *object)
142 {
143 if (kasan_enabled())
144 __kasan_unpoison_object_data(cache, object);
145 }
146
147 void __kasan_poison_object_data(struct kmem_cache *cache, void *object);
kasan_poison_object_data(struct kmem_cache * cache,void * object)148 static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
149 void *object)
150 {
151 if (kasan_enabled())
152 __kasan_poison_object_data(cache, object);
153 }
154
155 void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
156 const void *object);
kasan_init_slab_obj(struct kmem_cache * cache,const void * object)157 static __always_inline void * __must_check kasan_init_slab_obj(
158 struct kmem_cache *cache, const void *object)
159 {
160 if (kasan_enabled())
161 return __kasan_init_slab_obj(cache, object);
162 return (void *)object;
163 }
164
165 bool __kasan_slab_free(struct kmem_cache *s, void *object,
166 unsigned long ip, bool init);
kasan_slab_free(struct kmem_cache * s,void * object,bool init)167 static __always_inline bool kasan_slab_free(struct kmem_cache *s,
168 void *object, bool init)
169 {
170 if (kasan_enabled())
171 return __kasan_slab_free(s, object, _RET_IP_, init);
172 return false;
173 }
174
175 void __kasan_kfree_large(void *ptr, unsigned long ip);
kasan_kfree_large(void * ptr)176 static __always_inline void kasan_kfree_large(void *ptr)
177 {
178 if (kasan_enabled())
179 __kasan_kfree_large(ptr, _RET_IP_);
180 }
181
182 void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
kasan_slab_free_mempool(void * ptr)183 static __always_inline void kasan_slab_free_mempool(void *ptr)
184 {
185 if (kasan_enabled())
186 __kasan_slab_free_mempool(ptr, _RET_IP_);
187 }
188
189 void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
190 void *object, gfp_t flags, bool init);
kasan_slab_alloc(struct kmem_cache * s,void * object,gfp_t flags,bool init)191 static __always_inline void * __must_check kasan_slab_alloc(
192 struct kmem_cache *s, void *object, gfp_t flags, bool init)
193 {
194 if (kasan_enabled())
195 return __kasan_slab_alloc(s, object, flags, init);
196 return object;
197 }
198
199 void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object,
200 size_t size, gfp_t flags);
kasan_kmalloc(struct kmem_cache * s,const void * object,size_t size,gfp_t flags)201 static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
202 const void *object, size_t size, gfp_t flags)
203 {
204 if (kasan_enabled())
205 return __kasan_kmalloc(s, object, size, flags);
206 return (void *)object;
207 }
208
209 void * __must_check __kasan_kmalloc_large(const void *ptr,
210 size_t size, gfp_t flags);
kasan_kmalloc_large(const void * ptr,size_t size,gfp_t flags)211 static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
212 size_t size, gfp_t flags)
213 {
214 if (kasan_enabled())
215 return __kasan_kmalloc_large(ptr, size, flags);
216 return (void *)ptr;
217 }
218
219 void * __must_check __kasan_krealloc(const void *object,
220 size_t new_size, gfp_t flags);
kasan_krealloc(const void * object,size_t new_size,gfp_t flags)221 static __always_inline void * __must_check kasan_krealloc(const void *object,
222 size_t new_size, gfp_t flags)
223 {
224 if (kasan_enabled())
225 return __kasan_krealloc(object, new_size, flags);
226 return (void *)object;
227 }
228
229 /*
230 * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
231 * the hardware tag-based mode that doesn't rely on compiler instrumentation.
232 */
233 bool __kasan_check_byte(const void *addr, unsigned long ip);
kasan_check_byte(const void * addr)234 static __always_inline bool kasan_check_byte(const void *addr)
235 {
236 if (kasan_enabled())
237 return __kasan_check_byte(addr, _RET_IP_);
238 return true;
239 }
240
241 #else /* CONFIG_KASAN */
242
kasan_unpoison_range(const void * address,size_t size)243 static inline void kasan_unpoison_range(const void *address, size_t size) {}
kasan_poison_pages(struct page * page,unsigned int order,bool init)244 static inline void kasan_poison_pages(struct page *page, unsigned int order,
245 bool init) {}
kasan_unpoison_pages(struct page * page,unsigned int order,bool init)246 static inline bool kasan_unpoison_pages(struct page *page, unsigned int order,
247 bool init)
248 {
249 return false;
250 }
kasan_cache_create_kmalloc(struct kmem_cache * cache)251 static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
kasan_poison_slab(struct page * page)252 static inline void kasan_poison_slab(struct page *page) {}
kasan_unpoison_object_data(struct kmem_cache * cache,void * object)253 static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
254 void *object) {}
kasan_poison_object_data(struct kmem_cache * cache,void * object)255 static inline void kasan_poison_object_data(struct kmem_cache *cache,
256 void *object) {}
kasan_init_slab_obj(struct kmem_cache * cache,const void * object)257 static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
258 const void *object)
259 {
260 return (void *)object;
261 }
kasan_slab_free(struct kmem_cache * s,void * object,bool init)262 static inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init)
263 {
264 return false;
265 }
kasan_kfree_large(void * ptr)266 static inline void kasan_kfree_large(void *ptr) {}
kasan_slab_free_mempool(void * ptr)267 static inline void kasan_slab_free_mempool(void *ptr) {}
kasan_slab_alloc(struct kmem_cache * s,void * object,gfp_t flags,bool init)268 static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object,
269 gfp_t flags, bool init)
270 {
271 return object;
272 }
kasan_kmalloc(struct kmem_cache * s,const void * object,size_t size,gfp_t flags)273 static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object,
274 size_t size, gfp_t flags)
275 {
276 return (void *)object;
277 }
kasan_kmalloc_large(const void * ptr,size_t size,gfp_t flags)278 static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
279 {
280 return (void *)ptr;
281 }
kasan_krealloc(const void * object,size_t new_size,gfp_t flags)282 static inline void *kasan_krealloc(const void *object, size_t new_size,
283 gfp_t flags)
284 {
285 return (void *)object;
286 }
kasan_check_byte(const void * address)287 static inline bool kasan_check_byte(const void *address)
288 {
289 return true;
290 }
291
292 #endif /* CONFIG_KASAN */
293
294 #if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
295 void kasan_unpoison_task_stack(struct task_struct *task);
296 #else
kasan_unpoison_task_stack(struct task_struct * task)297 static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
298 #endif
299
300 #ifdef CONFIG_KASAN_GENERIC
301
302 size_t kasan_metadata_size(struct kmem_cache *cache);
303 slab_flags_t kasan_never_merge(void);
304 void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
305 slab_flags_t *flags);
306
307 void kasan_cache_shrink(struct kmem_cache *cache);
308 void kasan_cache_shutdown(struct kmem_cache *cache);
309 void kasan_record_aux_stack(void *ptr);
310 void kasan_record_aux_stack_noalloc(void *ptr);
311
312 #else /* CONFIG_KASAN_GENERIC */
313
314 /* Tag-based KASAN modes do not use per-object metadata. */
kasan_metadata_size(struct kmem_cache * cache)315 static inline size_t kasan_metadata_size(struct kmem_cache *cache)
316 {
317 return 0;
318 }
319 /* And thus nothing prevents cache merging. */
kasan_never_merge(void)320 static inline slab_flags_t kasan_never_merge(void)
321 {
322 return 0;
323 }
324 /* And no cache-related metadata initialization is required. */
kasan_cache_create(struct kmem_cache * cache,unsigned int * size,slab_flags_t * flags)325 static inline void kasan_cache_create(struct kmem_cache *cache,
326 unsigned int *size,
327 slab_flags_t *flags) {}
328
kasan_cache_shrink(struct kmem_cache * cache)329 static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
kasan_cache_shutdown(struct kmem_cache * cache)330 static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
kasan_record_aux_stack(void * ptr)331 static inline void kasan_record_aux_stack(void *ptr) {}
kasan_record_aux_stack_noalloc(void * ptr)332 static inline void kasan_record_aux_stack_noalloc(void *ptr) {}
333
334 #endif /* CONFIG_KASAN_GENERIC */
335
336 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
337
kasan_reset_tag(const void * addr)338 static inline void *kasan_reset_tag(const void *addr)
339 {
340 return (void *)arch_kasan_reset_tag(addr);
341 }
342
343 /**
344 * kasan_report - print a report about a bad memory access detected by KASAN
345 * @addr: address of the bad access
346 * @size: size of the bad access
347 * @is_write: whether the bad access is a write or a read
348 * @ip: instruction pointer for the accessibility check or the bad access itself
349 */
350 bool kasan_report(unsigned long addr, size_t size,
351 bool is_write, unsigned long ip);
352
353 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
354
kasan_reset_tag(const void * addr)355 static inline void *kasan_reset_tag(const void *addr)
356 {
357 return (void *)addr;
358 }
359
360 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/
361
362 #ifdef CONFIG_KASAN_HW_TAGS
363
364 void kasan_report_async(void);
365
366 #endif /* CONFIG_KASAN_HW_TAGS */
367
368 #ifdef CONFIG_KASAN_SW_TAGS
369 void __init kasan_init_sw_tags(void);
370 #else
kasan_init_sw_tags(void)371 static inline void kasan_init_sw_tags(void) { }
372 #endif
373
374 #ifdef CONFIG_KASAN_HW_TAGS
375 void kasan_init_hw_tags_cpu(void);
376 void __init kasan_init_hw_tags(void);
377 #else
kasan_init_hw_tags_cpu(void)378 static inline void kasan_init_hw_tags_cpu(void) { }
kasan_init_hw_tags(void)379 static inline void kasan_init_hw_tags(void) { }
380 #endif
381
382 #ifdef CONFIG_KASAN_VMALLOC
383
384 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
385
386 void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);
387 int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
388 void kasan_release_vmalloc(unsigned long start, unsigned long end,
389 unsigned long free_region_start,
390 unsigned long free_region_end);
391
392 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
393
kasan_populate_early_vm_area_shadow(void * start,unsigned long size)394 static inline void kasan_populate_early_vm_area_shadow(void *start,
395 unsigned long size)
396 { }
kasan_populate_vmalloc(unsigned long start,unsigned long size)397 static inline int kasan_populate_vmalloc(unsigned long start,
398 unsigned long size)
399 {
400 return 0;
401 }
kasan_release_vmalloc(unsigned long start,unsigned long end,unsigned long free_region_start,unsigned long free_region_end)402 static inline void kasan_release_vmalloc(unsigned long start,
403 unsigned long end,
404 unsigned long free_region_start,
405 unsigned long free_region_end) { }
406
407 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
408
409 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
410 kasan_vmalloc_flags_t flags);
kasan_unpoison_vmalloc(const void * start,unsigned long size,kasan_vmalloc_flags_t flags)411 static __always_inline void *kasan_unpoison_vmalloc(const void *start,
412 unsigned long size,
413 kasan_vmalloc_flags_t flags)
414 {
415 if (kasan_enabled())
416 return __kasan_unpoison_vmalloc(start, size, flags);
417 return (void *)start;
418 }
419
420 void __kasan_poison_vmalloc(const void *start, unsigned long size);
kasan_poison_vmalloc(const void * start,unsigned long size)421 static __always_inline void kasan_poison_vmalloc(const void *start,
422 unsigned long size)
423 {
424 if (kasan_enabled())
425 __kasan_poison_vmalloc(start, size);
426 }
427
428 #else /* CONFIG_KASAN_VMALLOC */
429
kasan_populate_early_vm_area_shadow(void * start,unsigned long size)430 static inline void kasan_populate_early_vm_area_shadow(void *start,
431 unsigned long size) { }
kasan_populate_vmalloc(unsigned long start,unsigned long size)432 static inline int kasan_populate_vmalloc(unsigned long start,
433 unsigned long size)
434 {
435 return 0;
436 }
kasan_release_vmalloc(unsigned long start,unsigned long end,unsigned long free_region_start,unsigned long free_region_end)437 static inline void kasan_release_vmalloc(unsigned long start,
438 unsigned long end,
439 unsigned long free_region_start,
440 unsigned long free_region_end) { }
441
kasan_unpoison_vmalloc(const void * start,unsigned long size,kasan_vmalloc_flags_t flags)442 static inline void *kasan_unpoison_vmalloc(const void *start,
443 unsigned long size,
444 kasan_vmalloc_flags_t flags)
445 {
446 return (void *)start;
447 }
kasan_poison_vmalloc(const void * start,unsigned long size)448 static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
449 { }
450
451 #endif /* CONFIG_KASAN_VMALLOC */
452
453 #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
454 !defined(CONFIG_KASAN_VMALLOC)
455
456 /*
457 * These functions allocate and free shadow memory for kernel modules.
458 * They are only required when KASAN_VMALLOC is not supported, as otherwise
459 * shadow memory is allocated by the generic vmalloc handlers.
460 */
461 int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask);
462 void kasan_free_module_shadow(const struct vm_struct *vm);
463
464 #else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
465
kasan_alloc_module_shadow(void * addr,size_t size,gfp_t gfp_mask)466 static inline int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
kasan_free_module_shadow(const struct vm_struct * vm)467 static inline void kasan_free_module_shadow(const struct vm_struct *vm) {}
468
469 #endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
470
471 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
472 void kasan_non_canonical_hook(unsigned long addr);
473 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
kasan_non_canonical_hook(unsigned long addr)474 static inline void kasan_non_canonical_hook(unsigned long addr) { }
475 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
476
477 #endif /* LINUX_KASAN_H */
478