43 static __always_inline bool memory_is_poisoned_1(unsigned long addr)  in memory_is_poisoned_1()  argument
45 	s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr);  in memory_is_poisoned_1()
48 		s8 last_accessible_byte = addr & KASAN_GRANULE_MASK;  in memory_is_poisoned_1()
55 static __always_inline bool memory_is_poisoned_2_4_8(unsigned long addr,  in memory_is_poisoned_2_4_8()  argument
58 	u8 *shadow_addr = (u8 *)kasan_mem_to_shadow((void *)addr);  in memory_is_poisoned_2_4_8()
64 	if (unlikely(((addr + size - 1) & KASAN_GRANULE_MASK) < size - 1))  in memory_is_poisoned_2_4_8()
65 		return *shadow_addr || memory_is_poisoned_1(addr + size - 1);  in memory_is_poisoned_2_4_8()
67 	return memory_is_poisoned_1(addr + size - 1);  in memory_is_poisoned_2_4_8()
70 static __always_inline bool memory_is_poisoned_16(unsigned long addr)  in memory_is_poisoned_16()  argument
72 	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);  in memory_is_poisoned_16()
75 	if (unlikely(!IS_ALIGNED(addr, KASAN_GRANULE_SIZE)))  in memory_is_poisoned_16()
76 		return *shadow_addr || memory_is_poisoned_1(addr + 15);  in memory_is_poisoned_16()
123 static __always_inline bool memory_is_poisoned_n(unsigned long addr,  in memory_is_poisoned_n()  argument
128 	ret = memory_is_nonzero(kasan_mem_to_shadow((void *)addr),  in memory_is_poisoned_n()
129 			kasan_mem_to_shadow((void *)addr + size - 1) + 1);  in memory_is_poisoned_n()
132 		unsigned long last_byte = addr + size - 1;  in memory_is_poisoned_n()
142 static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)  in memory_is_poisoned()  argument
147 			return memory_is_poisoned_1(addr);  in memory_is_poisoned()
151 			return memory_is_poisoned_2_4_8(addr, size);  in memory_is_poisoned()
153 			return memory_is_poisoned_16(addr);  in memory_is_poisoned()
159 	return memory_is_poisoned_n(addr, size);  in memory_is_poisoned()
162 static __always_inline bool check_region_inline(unsigned long addr,  in check_region_inline()  argument
169 	if (unlikely(addr + size < addr))  in check_region_inline()
170 		return !kasan_report(addr, size, write, ret_ip);  in check_region_inline()
172 	if (unlikely((void *)addr <  in check_region_inline()
174 		return !kasan_report(addr, size, write, ret_ip);  in check_region_inline()
177 	if (likely(!memory_is_poisoned(addr, size)))  in check_region_inline()
180 	return !kasan_report(addr, size, write, ret_ip);  in check_region_inline()
183 bool kasan_check_range(unsigned long addr, size_t size, bool write,  in kasan_check_range()  argument
186 	return check_region_inline(addr, size, write, ret_ip);  in kasan_check_range()
189 bool kasan_byte_accessible(const void *addr)  in kasan_byte_accessible()  argument
191 	s8 shadow_byte = READ_ONCE(*(s8 *)kasan_mem_to_shadow(addr));  in kasan_byte_accessible()
233 	void __asan_load##size(unsigned long addr)			\
235 		check_region_inline(addr, size, false, _RET_IP_);	\
241 	void __asan_store##size(unsigned long addr)			\
243 		check_region_inline(addr, size, true, _RET_IP_);	\
256 void __asan_loadN(unsigned long addr, size_t size)  in __asan_loadN()  argument
258 	kasan_check_range(addr, size, false, _RET_IP_);  in __asan_loadN()
266 void __asan_storeN(unsigned long addr, size_t size)  in __asan_storeN()  argument
268 	kasan_check_range(addr, size, true, _RET_IP_);  in __asan_storeN()
281 void __asan_alloca_poison(unsigned long addr, size_t size)  in __asan_alloca_poison()  argument
288 	const void *left_redzone = (const void *)(addr -  in __asan_alloca_poison()
290 	const void *right_redzone = (const void *)(addr + rounded_up_size);  in __asan_alloca_poison()
292 	WARN_ON(!IS_ALIGNED(addr, KASAN_ALLOCA_REDZONE_SIZE));  in __asan_alloca_poison()
294 	kasan_unpoison((const void *)(addr + rounded_down_size),  in __asan_alloca_poison()
315 	void __asan_set_shadow_##byte(const void *addr, size_t size)	\
317 		__memset((void *)addr, 0x##byte, size);			\
328 void kasan_record_aux_stack(void *addr)  in kasan_record_aux_stack()  argument
330 	struct page *page = kasan_addr_to_page(addr);  in kasan_record_aux_stack()
335 	if (is_kfence_address(addr) || !(page && PageSlab(page)))  in kasan_record_aux_stack()
339 	object = nearest_obj(cache, page, addr);  in kasan_record_aux_stack()