137 void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,  in __kasan_cache_create()  argument
158 	cache->kasan_info.alloc_meta_offset = *size;  in __kasan_cache_create()
168 		cache->kasan_info.alloc_meta_offset = 0;  in __kasan_cache_create()
175 		cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;  in __kasan_cache_create()
189 	if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor ||  in __kasan_cache_create()
190 	    cache->object_size < sizeof(struct kasan_free_meta)) {  in __kasan_cache_create()
193 		cache->kasan_info.free_meta_offset = *size;  in __kasan_cache_create()
198 			cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;  in __kasan_cache_create()
204 	optimal_size = cache->object_size + optimal_redzone(cache->object_size);  in __kasan_cache_create()
213 void __kasan_cache_create_kmalloc(struct kmem_cache *cache)  in __kasan_cache_create_kmalloc()  argument
215 	cache->kasan_info.is_kmalloc = true;  in __kasan_cache_create_kmalloc()
218 size_t __kasan_metadata_size(struct kmem_cache *cache)  in __kasan_metadata_size()  argument
222 	return (cache->kasan_info.alloc_meta_offset ?  in __kasan_metadata_size()
224 		(cache->kasan_info.free_meta_offset ?  in __kasan_metadata_size()
228 struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,  in kasan_get_alloc_meta()  argument
231 	if (!cache->kasan_info.alloc_meta_offset)  in kasan_get_alloc_meta()
233 	return kasan_reset_tag(object) + cache->kasan_info.alloc_meta_offset;  in kasan_get_alloc_meta()
237 struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,  in kasan_get_free_meta()  argument
241 	if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META)  in kasan_get_free_meta()
243 	return kasan_reset_tag(object) + cache->kasan_info.free_meta_offset;  in kasan_get_free_meta()
257 void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object)  in __kasan_unpoison_object_data()  argument
259 	kasan_unpoison(object, cache->object_size, false);  in __kasan_unpoison_object_data()
262 void __kasan_poison_object_data(struct kmem_cache *cache, void *object)  in __kasan_poison_object_data()  argument
264 	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),  in __kasan_poison_object_data()
282 static inline u8 assign_tag(struct kmem_cache *cache,  in assign_tag()  argument
292 	if (!cache->ctor && !(cache->flags & SLAB_TYPESAFE_BY_RCU))  in assign_tag()
298 	return (u8)obj_to_index(cache, virt_to_page(object), (void *)object);  in assign_tag()
308 void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,  in __kasan_init_slab_obj()  argument
314 		alloc_meta = kasan_get_alloc_meta(cache, object);  in __kasan_init_slab_obj()
320 	object = set_tag(object, assign_tag(cache, object, true));  in __kasan_init_slab_obj()
325 static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object,  in ____kasan_slab_free()  argument
338 	if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) !=  in ____kasan_slab_free()
345 	if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU))  in ____kasan_slab_free()
353 	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),  in ____kasan_slab_free()
360 		kasan_set_free_info(cache, object, tag);  in ____kasan_slab_free()
362 	return kasan_quarantine_put(cache, object);  in ____kasan_slab_free()
365 bool __kasan_slab_free(struct kmem_cache *cache, void *object,  in __kasan_slab_free()  argument
368 	return ____kasan_slab_free(cache, object, ip, true, init);  in __kasan_slab_free()
417 static void set_alloc_info(struct kmem_cache *cache, void *object,  in set_alloc_info()  argument
423 	if (cache->kasan_info.is_kmalloc && !is_kmalloc)  in set_alloc_info()
426 	alloc_meta = kasan_get_alloc_meta(cache, object);  in set_alloc_info()
431 void * __must_check __kasan_slab_alloc(struct kmem_cache *cache,  in __kasan_slab_alloc()  argument
450 	tag = assign_tag(cache, object, false);  in __kasan_slab_alloc()
457 	kasan_unpoison(tagged_object, cache->object_size, init);  in __kasan_slab_alloc()
461 		set_alloc_info(cache, (void *)object, flags, false);  in __kasan_slab_alloc()
466 static inline void *____kasan_kmalloc(struct kmem_cache *cache,  in ____kasan_kmalloc()  argument
497 	redzone_end = round_up((unsigned long)(object + cache->object_size),  in ____kasan_kmalloc()
507 		set_alloc_info(cache, (void *)object, flags, true);  in ____kasan_kmalloc()
513 void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object,  in __kasan_kmalloc()  argument
516 	return ____kasan_kmalloc(cache, object, size, flags);  in __kasan_kmalloc()