25 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
28 	const int nr = pool->curr_nr;  in poison_error()
34 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
42 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
51 			poison_error(pool, element, size, i);  in __check_element()
58 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
61 	if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {  in check_element()
62 		__check_element(pool, element, ksize(element));  in check_element()
63 	} else if (pool->free == mempool_free_pages) {  in check_element()
65 		int order = (int)(long)pool->pool_data;  in check_element()
68 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
81 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
84 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {  in poison_element()
86 	} else if (pool->alloc == mempool_alloc_pages) {  in poison_element()
88 		int order = (int)(long)pool->pool_data;  in poison_element()
96 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
99 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
104 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
106 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_poison_element()
108 	else if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
109 		kasan_poison_pages(element, (unsigned long)pool->pool_data,  in kasan_poison_element()
113 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
115 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
117 	else if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
118 		kasan_unpoison_pages(element, (unsigned long)pool->pool_data,  in kasan_unpoison_element()
122 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
124 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
125 	poison_element(pool, element);  in add_element()
126 	kasan_poison_element(pool, element);  in add_element()
127 	pool->elements[pool->curr_nr++] = element;  in add_element()
130 static void *remove_element(mempool_t *pool)  in remove_element()  argument
132 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
134 	BUG_ON(pool->curr_nr < 0);  in remove_element()
135 	kasan_unpoison_element(pool, element);  in remove_element()
136 	check_element(pool, element);  in remove_element()
151 void mempool_exit(mempool_t *pool)  in mempool_exit()  argument
153 	while (pool->curr_nr) {  in mempool_exit()
154 		void *element = remove_element(pool);  in mempool_exit()
155 		pool->free(element, pool->pool_data);  in mempool_exit()
157 	kfree(pool->elements);  in mempool_exit()
158 	pool->elements = NULL;  in mempool_exit()
170 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
172 	if (unlikely(!pool))  in mempool_destroy()
175 	mempool_exit(pool);  in mempool_destroy()
176 	kfree(pool);  in mempool_destroy()
180 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init_node()  argument
184 	spin_lock_init(&pool->lock);  in mempool_init_node()
185 	pool->min_nr	= min_nr;  in mempool_init_node()
186 	pool->pool_data = pool_data;  in mempool_init_node()
187 	pool->alloc	= alloc_fn;  in mempool_init_node()
188 	pool->free	= free_fn;  in mempool_init_node()
189 	init_waitqueue_head(&pool->wait);  in mempool_init_node()
191 	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),  in mempool_init_node()
193 	if (!pool->elements)  in mempool_init_node()
199 	while (pool->curr_nr < pool->min_nr) {  in mempool_init_node()
202 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
204 			mempool_exit(pool);  in mempool_init_node()
207 		add_element(pool, element);  in mempool_init_node()
228 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init()  argument
231 	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,  in mempool_init()
265 	mempool_t *pool;  in mempool_create_node()  local
267 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
268 	if (!pool)  in mempool_create_node()
271 	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,  in mempool_create_node()
273 		kfree(pool);  in mempool_create_node()
277 	return pool;  in mempool_create_node()
299 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
308 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
309 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
310 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
311 			element = remove_element(pool);  in mempool_resize()
312 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
313 			pool->free(element, pool->pool_data);  in mempool_resize()
314 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
316 		pool->min_nr = new_min_nr;  in mempool_resize()
319 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
327 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
328 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
330 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
334 	memcpy(new_elements, pool->elements,  in mempool_resize()
335 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
336 	kfree(pool->elements);  in mempool_resize()
337 	pool->elements = new_elements;  in mempool_resize()
338 	pool->min_nr = new_min_nr;  in mempool_resize()
340 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
341 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
342 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
345 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
346 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
347 			add_element(pool, element);  in mempool_resize()
349 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
350 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
355 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
375 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
393 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
397 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
398 	if (likely(pool->curr_nr)) {  in mempool_alloc()
399 		element = remove_element(pool);  in mempool_alloc()
400 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
416 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
423 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
429 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
431 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
439 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
452 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
492 	if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {  in mempool_free()
493 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
494 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
495 			add_element(pool, element);  in mempool_free()
496 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
497 			wake_up(&pool->wait);  in mempool_free()
500 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
502 	pool->free(element, pool->pool_data);  in mempool_free()