121 static void bio_put_slab(struct bio_set *bs)  in bio_put_slab()  argument
129 		if (bs->bio_slab == bio_slabs[i].slab) {  in bio_put_slab()
253 	struct bio_set *bs = bio->bi_pool;  in bio_free()  local
259 	if (bs) {  in bio_free()
260 		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));  in bio_free()
266 		p -= bs->front_pad;  in bio_free()
268 		mempool_free(p, &bs->bio_pool);  in bio_free()
352 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);  in bio_alloc_rescue()  local
356 		spin_lock(&bs->rescue_lock);  in bio_alloc_rescue()
357 		bio = bio_list_pop(&bs->rescue_list);  in bio_alloc_rescue()
358 		spin_unlock(&bs->rescue_lock);  in bio_alloc_rescue()
367 static void punt_bios_to_rescuer(struct bio_set *bs)  in punt_bios_to_rescuer()  argument
372 	if (WARN_ON_ONCE(!bs->rescue_workqueue))  in punt_bios_to_rescuer()
389 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
394 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
397 	spin_lock(&bs->rescue_lock);  in punt_bios_to_rescuer()
398 	bio_list_merge(&bs->rescue_list, &punt);  in punt_bios_to_rescuer()
399 	spin_unlock(&bs->rescue_lock);  in punt_bios_to_rescuer()
401 	queue_work(bs->rescue_workqueue, &bs->rescue_work);  in punt_bios_to_rescuer()
440 			     struct bio_set *bs)  in bio_alloc_bioset()  argument
449 	if (!bs) {  in bio_alloc_bioset()
458 		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&  in bio_alloc_bioset()
485 		    bs->rescue_workqueue)  in bio_alloc_bioset()
488 		p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
490 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
492 			p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
495 		front_pad = bs->front_pad;  in bio_alloc_bioset()
508 		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
510 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
512 			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
523 	bio->bi_pool = bs;  in bio_alloc_bioset()
529 	mempool_free(p, &bs->bio_pool);  in bio_alloc_bioset()
709 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)  in bio_clone_fast()  argument
713 	b = bio_alloc_bioset(gfp_mask, 0, bs);  in bio_clone_fast()
1479 		      gfp_t gfp, struct bio_set *bs)  in bio_split()  argument
1490 	split = bio_clone_fast(bio, gfp, bs);  in bio_split()
1550 void bioset_exit(struct bio_set *bs)  in bioset_exit()  argument
1552 	if (bs->rescue_workqueue)  in bioset_exit()
1553 		destroy_workqueue(bs->rescue_workqueue);  in bioset_exit()
1554 	bs->rescue_workqueue = NULL;  in bioset_exit()
1556 	mempool_exit(&bs->bio_pool);  in bioset_exit()
1557 	mempool_exit(&bs->bvec_pool);  in bioset_exit()
1559 	bioset_integrity_free(bs);  in bioset_exit()
1560 	if (bs->bio_slab)  in bioset_exit()
1561 		bio_put_slab(bs);  in bioset_exit()
1562 	bs->bio_slab = NULL;  in bioset_exit()
1587 int bioset_init(struct bio_set *bs,  in bioset_init()  argument
1594 	bs->front_pad = front_pad;  in bioset_init()
1596 	spin_lock_init(&bs->rescue_lock);  in bioset_init()
1597 	bio_list_init(&bs->rescue_list);  in bioset_init()
1598 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);  in bioset_init()
1600 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);  in bioset_init()
1601 	if (!bs->bio_slab)  in bioset_init()
1604 	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))  in bioset_init()
1608 	    biovec_init_pool(&bs->bvec_pool, pool_size))  in bioset_init()
1614 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);  in bioset_init()
1615 	if (!bs->rescue_workqueue)  in bioset_init()
1620 	bioset_exit(bs);  in bioset_init()
1629 int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)  in bioset_init_from_src()  argument
1639 	return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);  in bioset_init_from_src()