565 static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,  in alloc_tio()  argument
570 	if (!ci->io->tio.io) {  in alloc_tio()
572 		tio = &ci->io->tio;  in alloc_tio()
574 		struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);  in alloc_tio()
583 	tio->io = ci->io;  in alloc_tio()
1322 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,  in alloc_multiple_bios()  argument
1332 		tio = alloc_tio(ci, ti, 0, GFP_NOIO);  in alloc_multiple_bios()
1342 			mutex_lock(&ci->io->md->table_devices_lock);  in alloc_multiple_bios()
1344 			tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);  in alloc_multiple_bios()
1351 			mutex_unlock(&ci->io->md->table_devices_lock);  in alloc_multiple_bios()
1362 static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,  in __clone_and_map_simple_bio()  argument
1369 	__bio_clone_fast(clone, ci->bio);  in __clone_and_map_simple_bio()
1371 		bio_setup_sector(clone, ci->sector, *len);  in __clone_and_map_simple_bio()
1376 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,  in __send_duplicate_bios()  argument
1383 	alloc_multiple_bios(&blist, ci, ti, num_bios);  in __send_duplicate_bios()
1387 		(void) __clone_and_map_simple_bio(ci, tio, len);  in __send_duplicate_bios()
1391 static int __send_empty_flush(struct clone_info *ci)  in __send_empty_flush()  argument
1404 	bio_set_dev(&flush_bio, ci->io->md->disk->part0);  in __send_empty_flush()
1406 	ci->bio = &flush_bio;  in __send_empty_flush()
1407 	ci->sector_count = 0;  in __send_empty_flush()
1409 	BUG_ON(bio_has_data(ci->bio));  in __send_empty_flush()
1410 	while ((ti = dm_table_get_target(ci->map, target_nr++)))  in __send_empty_flush()
1411 		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);  in __send_empty_flush()
1413 	bio_uninit(ci->bio);  in __send_empty_flush()
1417 static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,  in __clone_and_map_data_bio()  argument
1420 	struct bio *bio = ci->bio;  in __clone_and_map_data_bio()
1424 	tio = alloc_tio(ci, ti, 0, GFP_NOIO);  in __clone_and_map_data_bio()
1436 static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,  in __send_changing_extent_only()  argument
1450 	len = min_t(sector_t, ci->sector_count,  in __send_changing_extent_only()
1451 		    max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));  in __send_changing_extent_only()
1453 	__send_duplicate_bios(ci, ti, num_bios, &len);  in __send_changing_extent_only()
1455 	ci->sector += len;  in __send_changing_extent_only()
1456 	ci->sector_count -= len;  in __send_changing_extent_only()
1477 static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,  in __process_abnormal_io()  argument
1480 	struct bio *bio = ci->bio;  in __process_abnormal_io()
1500 	*result = __send_changing_extent_only(ci, ti, num_bios);  in __process_abnormal_io()
1507 static int __split_and_process_non_flush(struct clone_info *ci)  in __split_and_process_non_flush()  argument
1513 	ti = dm_table_find_target(ci->map, ci->sector);  in __split_and_process_non_flush()
1517 	if (__process_abnormal_io(ci, ti, &r))  in __split_and_process_non_flush()
1520 	len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);  in __split_and_process_non_flush()
1522 	r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);  in __split_and_process_non_flush()
1526 	ci->sector += len;  in __split_and_process_non_flush()
1527 	ci->sector_count -= len;  in __split_and_process_non_flush()
1532 static void init_clone_info(struct clone_info *ci, struct mapped_device *md,  in init_clone_info()  argument
1535 	ci->map = map;  in init_clone_info()
1536 	ci->io = alloc_io(md, bio);  in init_clone_info()
1537 	ci->sector = bio->bi_iter.bi_sector;  in init_clone_info()
1546 	struct clone_info ci;  in __split_and_process_bio()  local
1550 	init_clone_info(&ci, md, map, bio);  in __split_and_process_bio()
1553 		error = __send_empty_flush(&ci);  in __split_and_process_bio()
1556 		ci.bio = bio;  in __split_and_process_bio()
1557 		ci.sector_count = 0;  in __split_and_process_bio()
1558 		error = __split_and_process_non_flush(&ci);  in __split_and_process_bio()
1560 		ci.bio = bio;  in __split_and_process_bio()
1561 		ci.sector_count = bio_sectors(bio);  in __split_and_process_bio()
1562 		error = __split_and_process_non_flush(&ci);  in __split_and_process_bio()
1563 		if (ci.sector_count && !error) {  in __split_and_process_bio()
1572 			struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,  in __split_and_process_bio()
1574 			ci.io->orig_bio = b;  in __split_and_process_bio()
1581 	start_io_acct(ci.io);  in __split_and_process_bio()
1584 	dm_io_dec_pending(ci.io, errno_to_blk_status(error));  in __split_and_process_bio()