42 struct pool {  struct
59 static struct pool mp[MAX_POOLS];  argument
63 static inline int ptr_valid(struct pool *pool, void *ptr)  in ptr_valid()  argument
65 	unsigned int pool_size = pool->nr_blocks * SMALLOC_BPL;  in ptr_valid()
67 	return (ptr >= pool->map) && (ptr < pool->map + pool_size);  in ptr_valid()
75 static int blocks_iter(struct pool *pool, unsigned int pool_idx,  in blocks_iter()  argument
84 		if (pool_idx >= pool->nr_blocks)  in blocks_iter()
87 		map = &pool->bitmap[pool_idx];  in blocks_iter()
130 static int blocks_free(struct pool *pool, unsigned int pool_idx,  in blocks_free()  argument
133 	return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);  in blocks_free()
136 static void set_blocks(struct pool *pool, unsigned int pool_idx,  in set_blocks()  argument
139 	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);  in set_blocks()
142 static void clear_blocks(struct pool *pool, unsigned int pool_idx,  in clear_blocks()  argument
145 	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);  in clear_blocks()
155 static bool add_pool(struct pool *pool, unsigned int alloc_size)  in add_pool()  argument
175 	pool->mmap_size = alloc_size;  in add_pool()
177 	pool->nr_blocks = bitmap_blocks;  in add_pool()
178 	pool->free_blocks = bitmap_blocks * SMALLOC_BPB;  in add_pool()
191 	pool->map = ptr;  in add_pool()
192 	pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL);  in add_pool()
193 	memset(pool->bitmap, 0, bitmap_blocks * sizeof(unsigned int));  in add_pool()
195 	pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);  in add_pool()
196 	if (!pool->lock)  in add_pool()
203 	if (pool->map)  in add_pool()
204 		munmap(pool->map, pool->mmap_size);  in add_pool()
226 static void cleanup_pool(struct pool *pool)  in cleanup_pool()  argument
232 	munmap(pool->map, pool->mmap_size);  in cleanup_pool()
234 	if (pool->lock)  in cleanup_pool()
235 		fio_mutex_remove(pool->lock);  in cleanup_pool()
292 static void sfree_pool(struct pool *pool, void *ptr)  in sfree_pool()  argument
304 	assert(ptr_valid(pool, ptr));  in sfree_pool()
308 	offset = ptr - pool->map;  in sfree_pool()
312 	fio_mutex_down(pool->lock);  in sfree_pool()
313 	clear_blocks(pool, i, idx, size_to_blocks(hdr->size));  in sfree_pool()
314 	if (i < pool->next_non_full)  in sfree_pool()
315 		pool->next_non_full = i;  in sfree_pool()
316 	pool->free_blocks += size_to_blocks(hdr->size);  in sfree_pool()
317 	fio_mutex_up(pool->lock);  in sfree_pool()
322 	struct pool *pool = NULL;  in sfree()  local
330 			pool = &mp[i];  in sfree()
335 	if (pool) {  in sfree()
336 		sfree_pool(pool, ptr);  in sfree()
343 static void *__smalloc_pool(struct pool *pool, size_t size)  in __smalloc_pool()  argument
351 	fio_mutex_down(pool->lock);  in __smalloc_pool()
354 	if (nr_blocks > pool->free_blocks)  in __smalloc_pool()
357 	i = pool->next_non_full;  in __smalloc_pool()
360 	while (i < pool->nr_blocks) {  in __smalloc_pool()
363 		if (pool->bitmap[i] == -1U) {  in __smalloc_pool()
365 			pool->next_non_full = i;  in __smalloc_pool()
370 		idx = find_next_zero(pool->bitmap[i], last_idx);  in __smalloc_pool()
371 		if (!blocks_free(pool, i, idx, nr_blocks)) {  in __smalloc_pool()
384 		set_blocks(pool, i, idx, nr_blocks);  in __smalloc_pool()
389 	if (i < pool->nr_blocks) {  in __smalloc_pool()
390 		pool->free_blocks -= nr_blocks;  in __smalloc_pool()
391 		ret = pool->map + offset;  in __smalloc_pool()
394 	fio_mutex_up(pool->lock);  in __smalloc_pool()
398 static void *smalloc_pool(struct pool *pool, size_t size)  in smalloc_pool()  argument
412 	ptr = __smalloc_pool(pool, alloc_size);  in smalloc_pool()