17 static inline void *asan_mem_to_shadow(const void *addr)  in asan_mem_to_shadow()  argument
20 	return (void *)((uintptr_t)&_asan_shadow + (((uintptr_t)addr -  in asan_mem_to_shadow()
23 	return (void *)((uintptr_t)&_asan_shadow + (((uintptr_t)addr -  in asan_mem_to_shadow()
59 static __always_inline bool memory_is_poisoned_1(unsigned long addr)  in memory_is_poisoned_1()  argument
61 	s8 shadow_value = *(s8 *)asan_mem_to_shadow((void *)addr);  in memory_is_poisoned_1()
64 		s8 last_accessible_byte = addr & ASAN_SHADOW_MASK;  in memory_is_poisoned_1()
71 static __always_inline bool memory_is_poisoned_2_4_8(unsigned long addr,  in memory_is_poisoned_2_4_8()  argument
74 	u8 *shadow_addr = (u8 *)asan_mem_to_shadow((void *)addr);  in memory_is_poisoned_2_4_8()
76 	if (unlikely(((addr + size - 1) & ASAN_SHADOW_MASK) < size - 1))  in memory_is_poisoned_2_4_8()
77 		return *shadow_addr || memory_is_poisoned_1(addr + size - 1);  in memory_is_poisoned_2_4_8()
79 	return memory_is_poisoned_1(addr + size - 1);  in memory_is_poisoned_2_4_8()
82 static __always_inline bool memory_is_poisoned_16(unsigned long addr)  in memory_is_poisoned_16()  argument
84 	u16 *shadow_addr = (u16 *)asan_mem_to_shadow((void *)addr);  in memory_is_poisoned_16()
86 	if (unlikely(!IS_ALIGNED(addr, ASAN_SHADOW_SCALE_SIZE)))  in memory_is_poisoned_16()
87 		return *shadow_addr || memory_is_poisoned_1(addr + 15);  in memory_is_poisoned_16()
134 static __always_inline bool memory_is_poisoned_n(unsigned long addr,  in memory_is_poisoned_n()  argument
139 	ret = memory_is_nonzero(asan_mem_to_shadow((void *)addr),  in memory_is_poisoned_n()
140 			asan_mem_to_shadow((void *)addr + size - 1) + 1);  in memory_is_poisoned_n()
143 		unsigned long last_byte = addr + size - 1;  in memory_is_poisoned_n()
153 static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)  in memory_is_poisoned()  argument
158 			return memory_is_poisoned_1(addr);  in memory_is_poisoned()
162 			return memory_is_poisoned_2_4_8(addr, size);  in memory_is_poisoned()
164 			return memory_is_poisoned_16(addr);  in memory_is_poisoned()
170 	return memory_is_poisoned_n(addr, size);  in memory_is_poisoned()
173 static const void *find_first_bad_addr(const void *addr, size_t size)  in find_first_bad_addr()  argument
175 	u8 shadow_val = *(u8 *)asan_mem_to_shadow(addr);  in find_first_bad_addr()
176 	const void *first_bad_addr = addr;  in find_first_bad_addr()
178 	while (!shadow_val && first_bad_addr < addr + size) {  in find_first_bad_addr()
185 static const char *get_shadow_bug_type(const void *addr, size_t size)  in get_shadow_bug_type()  argument
191 	if (addr < asan_shadow_to_mem((void *) &_asan_shadow))  in get_shadow_bug_type()
194 	first_bad_addr = find_first_bad_addr(addr, size);  in get_shadow_bug_type()
224 void asan_report(unsigned long addr, size_t size, bool is_write,  in asan_report()  argument
227 	const char *bug_type = get_shadow_bug_type((void *) addr, size);  in asan_report()
230 	printk(BIOS_ERR, "%s of %zu byte%s at addr %p\n",  in asan_report()
232 		(void *) addr);  in asan_report()
236 static __always_inline void check_memory_region_inline(unsigned long addr,  in check_memory_region_inline()  argument
241 	if (((uintptr_t)addr < (uintptr_t)&_car_region_start) ||  in check_memory_region_inline()
242 		((uintptr_t)addr > (uintptr_t)&_ebss))  in check_memory_region_inline()
245 	if (((uintptr_t)addr < (uintptr_t)&_data) ||  in check_memory_region_inline()
246 		((uintptr_t)addr > (uintptr_t)&_eheap))  in check_memory_region_inline()
252 	if (unlikely((void *)addr <  in check_memory_region_inline()
254 		asan_report(addr, size, write, ret_ip);  in check_memory_region_inline()
258 	if (likely(!memory_is_poisoned(addr, size)))  in check_memory_region_inline()
261 	asan_report(addr, size, write, ret_ip);  in check_memory_region_inline()
264 void check_memory_region(unsigned long addr, size_t size, bool write,  in check_memory_region()  argument
267 	check_memory_region_inline(addr, size, write, ret_ip);  in check_memory_region()
270 uintptr_t __asan_shadow_offset(uintptr_t addr)  in __asan_shadow_offset()  argument
336 void __asan_poison_stack_memory(const void *addr, size_t size)  in __asan_poison_stack_memory()  argument
338 	asan_poison_shadow(addr, ALIGN_UP(size, ASAN_SHADOW_SCALE_SIZE),  in __asan_poison_stack_memory()
342 void __asan_unpoison_stack_memory(const void *addr, size_t size)  in __asan_unpoison_stack_memory()  argument
344 	asan_unpoison_shadow(addr, size);  in __asan_unpoison_stack_memory()
348 	void __asan_load##size(unsigned long addr)	\
350 		check_memory_region_inline(addr, size, false, _RET_IP_);\
352 	void __asan_load##size##_noabort(unsigned long addr)	\
354 		check_memory_region_inline(addr, size, false, _RET_IP_);\
356 	void __asan_store##size(unsigned long addr)	\
358 		check_memory_region_inline(addr, size, true, _RET_IP_);	\
360 	void __asan_store##size##_noabort(unsigned long addr)	\
362 		check_memory_region_inline(addr, size, true, _RET_IP_);	\
371 void __asan_loadN(unsigned long addr, size_t size)  in __asan_loadN()  argument
373 	check_memory_region(addr, size, false, _RET_IP_);  in __asan_loadN()
376 void __asan_storeN(unsigned long addr, size_t size)  in __asan_storeN()  argument
378 	check_memory_region(addr, size, true, _RET_IP_);  in __asan_storeN()
381 void __asan_loadN_noabort(unsigned long addr, size_t size)  in __asan_loadN_noabort()  argument
383 	check_memory_region(addr, size, false, _RET_IP_);  in __asan_loadN_noabort()
386 void __asan_storeN_noabort(unsigned long addr, size_t size)  in __asan_storeN_noabort()  argument
388 	check_memory_region(addr, size, true, _RET_IP_);  in __asan_storeN_noabort()
396 	void __asan_set_shadow_##byte(const void *addr, size_t size)	\
398 		__builtin_memset((void *)addr, 0x##byte, size);	\
409 void __asan_report_load##size##_noabort(unsigned long addr)	\
411 	asan_report(addr, size, false, _RET_IP_);	\
415 void __asan_report_store##size##_noabort(unsigned long addr)	\
417 	asan_report(addr, size, true, _RET_IP_);	\
431 void __asan_report_load_n_noabort(unsigned long addr, size_t size)  in __asan_report_load_n_noabort()  argument
433 	asan_report(addr, size, false, _RET_IP_);  in __asan_report_load_n_noabort()
436 void __asan_report_store_n_noabort(unsigned long addr, size_t size)  in __asan_report_store_n_noabort()  argument
438 	asan_report(addr, size, true, _RET_IP_);  in __asan_report_store_n_noabort()