49 	struct rds_ib_mr_pool	*pool;  member
86 static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool, int free_all, struct rds_ib_mr **);
217 	struct rds_ib_mr_pool *pool;  in rds_ib_create_mr_pool()  local
219 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);  in rds_ib_create_mr_pool()
220 	if (!pool)  in rds_ib_create_mr_pool()
223 	init_llist_head(&pool->free_list);  in rds_ib_create_mr_pool()
224 	init_llist_head(&pool->drop_list);  in rds_ib_create_mr_pool()
225 	init_llist_head(&pool->clean_list);  in rds_ib_create_mr_pool()
226 	mutex_init(&pool->flush_lock);  in rds_ib_create_mr_pool()
227 	init_waitqueue_head(&pool->flush_wait);  in rds_ib_create_mr_pool()
228 	INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker);  in rds_ib_create_mr_pool()
230 	pool->fmr_attr.max_pages = fmr_message_size;  in rds_ib_create_mr_pool()
231 	pool->fmr_attr.max_maps = rds_ibdev->fmr_max_remaps;  in rds_ib_create_mr_pool()
232 	pool->fmr_attr.page_shift = PAGE_SHIFT;  in rds_ib_create_mr_pool()
233 	pool->max_free_pinned = rds_ibdev->max_fmrs * fmr_message_size / 4;  in rds_ib_create_mr_pool()
240 	pool->max_items_soft = rds_ibdev->max_fmrs * 3 / 4;  in rds_ib_create_mr_pool()
241 	pool->max_items = rds_ibdev->max_fmrs;  in rds_ib_create_mr_pool()
243 	return pool;  in rds_ib_create_mr_pool()
248 	struct rds_ib_mr_pool *pool = rds_ibdev->mr_pool;  in rds_ib_get_mr_info()  local
250 	iinfo->rdma_mr_max = pool->max_items;  in rds_ib_get_mr_info()
251 	iinfo->rdma_mr_size = pool->fmr_attr.max_pages;  in rds_ib_get_mr_info()
254 void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool)  in rds_ib_destroy_mr_pool()  argument
256 	cancel_delayed_work_sync(&pool->flush_worker);  in rds_ib_destroy_mr_pool()
257 	rds_ib_flush_mr_pool(pool, 1, NULL);  in rds_ib_destroy_mr_pool()
258 	WARN_ON(atomic_read(&pool->item_count));  in rds_ib_destroy_mr_pool()
259 	WARN_ON(atomic_read(&pool->free_pinned));  in rds_ib_destroy_mr_pool()
260 	kfree(pool);  in rds_ib_destroy_mr_pool()
263 static inline struct rds_ib_mr *rds_ib_reuse_fmr(struct rds_ib_mr_pool *pool)  in rds_ib_reuse_fmr()  argument
272 	ret = llist_del_first(&pool->clean_list);  in rds_ib_reuse_fmr()
295 	struct rds_ib_mr_pool *pool = rds_ibdev->mr_pool;  in rds_ib_alloc_fmr()  local
299 	if (atomic_read(&pool->dirty_count) >= pool->max_items / 10)  in rds_ib_alloc_fmr()
300 		schedule_delayed_work(&pool->flush_worker, 10);  in rds_ib_alloc_fmr()
303 		ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_alloc_fmr()
316 		if (atomic_inc_return(&pool->item_count) <= pool->max_items)  in rds_ib_alloc_fmr()
319 		atomic_dec(&pool->item_count);  in rds_ib_alloc_fmr()
328 		rds_ib_flush_mr_pool(pool, 0, &ibmr);  in rds_ib_alloc_fmr()
346 			&pool->fmr_attr);  in rds_ib_alloc_fmr()
363 	atomic_dec(&pool->item_count);  in rds_ib_alloc_fmr()
506 		struct rds_ib_mr_pool *pool = rds_ibdev->mr_pool;  in rds_ib_teardown_mr()  local
508 		atomic_sub(pinned, &pool->free_pinned);  in rds_ib_teardown_mr()
512 static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int free_all)  in rds_ib_flush_goal()  argument
516 	item_count = atomic_read(&pool->item_count);  in rds_ib_flush_goal()
546 static void list_to_llist_nodes(struct rds_ib_mr_pool *pool,  in list_to_llist_nodes()  argument
570 static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,  in rds_ib_flush_mr_pool()  argument
586 		while(!mutex_trylock(&pool->flush_lock)) {  in rds_ib_flush_mr_pool()
587 			ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_flush_mr_pool()
590 				finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
594 			prepare_to_wait(&pool->flush_wait, &wait,  in rds_ib_flush_mr_pool()
596 			if (llist_empty(&pool->clean_list))  in rds_ib_flush_mr_pool()
599 			ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_flush_mr_pool()
602 				finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
606 		finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
608 		mutex_lock(&pool->flush_lock);  in rds_ib_flush_mr_pool()
611 		ibmr = rds_ib_reuse_fmr(pool);  in rds_ib_flush_mr_pool()
621 	llist_append_to_list(&pool->drop_list, &unmap_list);  in rds_ib_flush_mr_pool()
622 	llist_append_to_list(&pool->free_list, &unmap_list);  in rds_ib_flush_mr_pool()
624 		llist_append_to_list(&pool->clean_list, &unmap_list);  in rds_ib_flush_mr_pool()
626 	free_goal = rds_ib_flush_goal(pool, free_all);  in rds_ib_flush_mr_pool()
643 		if (nfreed < free_goal || ibmr->remap_count >= pool->fmr_attr.max_maps) {  in rds_ib_flush_mr_pool()
665 		list_to_llist_nodes(pool, &unmap_list, &clean_nodes, &clean_tail);  in rds_ib_flush_mr_pool()
671 			llist_add_batch(clean_nodes->next, clean_tail, &pool->clean_list);  in rds_ib_flush_mr_pool()
675 	atomic_sub(unpinned, &pool->free_pinned);  in rds_ib_flush_mr_pool()
676 	atomic_sub(ncleaned, &pool->dirty_count);  in rds_ib_flush_mr_pool()
677 	atomic_sub(nfreed, &pool->item_count);  in rds_ib_flush_mr_pool()
680 	mutex_unlock(&pool->flush_lock);  in rds_ib_flush_mr_pool()
681 	if (waitqueue_active(&pool->flush_wait))  in rds_ib_flush_mr_pool()
682 		wake_up(&pool->flush_wait);  in rds_ib_flush_mr_pool()
689 	struct rds_ib_mr_pool *pool = container_of(work, struct rds_ib_mr_pool, flush_worker.work);  in rds_ib_mr_pool_flush_worker()  local
691 	rds_ib_flush_mr_pool(pool, 0, NULL);  in rds_ib_mr_pool_flush_worker()
698 	struct rds_ib_mr_pool *pool = rds_ibdev->mr_pool;  in rds_ib_free_mr()  local
703 	if (ibmr->remap_count >= pool->fmr_attr.max_maps)  in rds_ib_free_mr()
704 		llist_add(&ibmr->llnode, &pool->drop_list);  in rds_ib_free_mr()
706 		llist_add(&ibmr->llnode, &pool->free_list);  in rds_ib_free_mr()
708 	atomic_add(ibmr->sg_len, &pool->free_pinned);  in rds_ib_free_mr()
709 	atomic_inc(&pool->dirty_count);  in rds_ib_free_mr()
712 	if (atomic_read(&pool->free_pinned) >= pool->max_free_pinned ||  in rds_ib_free_mr()
713 	    atomic_read(&pool->dirty_count) >= pool->max_items / 10)  in rds_ib_free_mr()
714 		schedule_delayed_work(&pool->flush_worker, 10);  in rds_ib_free_mr()
718 			rds_ib_flush_mr_pool(pool, 0, NULL);  in rds_ib_free_mr()
722 			schedule_delayed_work(&pool->flush_worker, 10);  in rds_ib_free_mr()
735 		struct rds_ib_mr_pool *pool = rds_ibdev->mr_pool;  in rds_ib_flush_mrs()  local
737 		if (pool)  in rds_ib_flush_mrs()
738 			rds_ib_flush_mr_pool(pool, 0, NULL);  in rds_ib_flush_mrs()