19 static void add_element(mempool_t *pool, void *element)  in add_element()  argument
21 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
22 	pool->elements[pool->curr_nr++] = element;  in add_element()
25 static void *remove_element(mempool_t *pool)  in remove_element()  argument
27 	BUG_ON(pool->curr_nr <= 0);  in remove_element()
28 	return pool->elements[--pool->curr_nr];  in remove_element()
39 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
41 	while (pool->curr_nr) {  in mempool_destroy()
42 		void *element = remove_element(pool);  in mempool_destroy()
43 		pool->free(element, pool->pool_data);  in mempool_destroy()
45 	kfree(pool->elements);  in mempool_destroy()
46 	kfree(pool);  in mempool_destroy()
76 	mempool_t *pool;  in mempool_create_node()  local
77 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
78 	if (!pool)  in mempool_create_node()
80 	pool->elements = kmalloc_node(min_nr * sizeof(void *),  in mempool_create_node()
82 	if (!pool->elements) {  in mempool_create_node()
83 		kfree(pool);  in mempool_create_node()
86 	spin_lock_init(&pool->lock);  in mempool_create_node()
87 	pool->min_nr = min_nr;  in mempool_create_node()
88 	pool->pool_data = pool_data;  in mempool_create_node()
89 	init_waitqueue_head(&pool->wait);  in mempool_create_node()
90 	pool->alloc = alloc_fn;  in mempool_create_node()
91 	pool->free = free_fn;  in mempool_create_node()
96 	while (pool->curr_nr < pool->min_nr) {  in mempool_create_node()
99 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_create_node()
101 			mempool_destroy(pool);  in mempool_create_node()
104 		add_element(pool, element);  in mempool_create_node()
106 	return pool;  in mempool_create_node()
126 int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask)  in mempool_resize()  argument
134 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
135 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
136 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
137 			element = remove_element(pool);  in mempool_resize()
138 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
139 			pool->free(element, pool->pool_data);  in mempool_resize()
140 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
142 		pool->min_nr = new_min_nr;  in mempool_resize()
145 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
152 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
153 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
155 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
159 	memcpy(new_elements, pool->elements,  in mempool_resize()
160 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
161 	kfree(pool->elements);  in mempool_resize()
162 	pool->elements = new_elements;  in mempool_resize()
163 	pool->min_nr = new_min_nr;  in mempool_resize()
165 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
166 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
167 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_resize()
170 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
171 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
172 			add_element(pool, element);  in mempool_resize()
174 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
175 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
180 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
198 void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
216 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
220 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
221 	if (likely(pool->curr_nr)) {  in mempool_alloc()
222 		element = remove_element(pool);  in mempool_alloc()
223 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
239 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
246 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
252 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
254 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
262 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
275 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
315 	if (unlikely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
316 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
317 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
318 			add_element(pool, element);  in mempool_free()
319 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
320 			wake_up(&pool->wait);  in mempool_free()
323 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
325 	pool->free(element, pool->pool_data);  in mempool_free()