80 	struct request *orig, clone;  member
96 	struct bio clone;  member
560 	bio_put(&tio->clone);  in free_tio()
903 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);  in clone_endio()
939 static void end_clone_bio(struct bio *clone, int error)  in end_clone_bio()  argument
942 		container_of(clone, struct dm_rq_clone_bio_info, clone);  in end_clone_bio()
947 	bio_put(clone);  in end_clone_bio()
1015 static void free_rq_clone(struct request *clone)  in free_rq_clone()  argument
1017 	struct dm_rq_target_io *tio = clone->end_io_data;  in free_rq_clone()
1019 	blk_rq_unprep_clone(clone);  in free_rq_clone()
1027 static void dm_end_request(struct request *clone, int error)  in dm_end_request()  argument
1029 	int rw = rq_data_dir(clone);  in dm_end_request()
1030 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_end_request()
1035 		rq->errors = clone->errors;  in dm_end_request()
1036 		rq->resid_len = clone->resid_len;  in dm_end_request()
1044 			rq->sense_len = clone->sense_len;  in dm_end_request()
1047 	free_rq_clone(clone);  in dm_end_request()
1054 	struct request *clone = rq->special;  in dm_unprep_request()  local
1059 	free_rq_clone(clone);  in dm_unprep_request()
1065 void dm_requeue_unmapped_request(struct request *clone)  in dm_requeue_unmapped_request()  argument
1067 	int rw = rq_data_dir(clone);  in dm_requeue_unmapped_request()
1068 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_requeue_unmapped_request()
1113 static void dm_done(struct request *clone, int error, bool mapped)  in dm_done()  argument
1116 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_done()
1123 			r = rq_end_io(tio->ti, clone, error, &tio->info);  in dm_done()
1126 	if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) &&  in dm_done()
1127 		     !clone->q->limits.max_write_same_sectors))  in dm_done()
1132 		dm_end_request(clone, r);  in dm_done()
1138 		dm_requeue_unmapped_request(clone);  in dm_done()
1151 	struct request *clone = rq->completion_data;  in dm_softirq_done()  local
1152 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_softirq_done()
1157 	dm_done(clone, tio->error, mapped);  in dm_softirq_done()
1164 static void dm_complete_request(struct request *clone, int error)  in dm_complete_request()  argument
1166 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_complete_request()
1170 	rq->completion_data = clone;  in dm_complete_request()
1180 void dm_kill_unmapped_request(struct request *clone, int error)  in dm_kill_unmapped_request()  argument
1182 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_kill_unmapped_request()
1186 	dm_complete_request(clone, error);  in dm_kill_unmapped_request()
1193 static void end_clone_request(struct request *clone, int error)  in end_clone_request()  argument
1201 	__blk_put_request(clone->q, clone);  in end_clone_request()
1211 	dm_complete_request(clone, error);  in end_clone_request()
1293 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);  in dm_accept_partial_bio()
1359 	struct bio *clone = &tio->clone;  in __map_bio()  local
1362 	clone->bi_end_io = clone_endio;  in __map_bio()
1370 	sector = clone->bi_iter.bi_sector;  in __map_bio()
1373 	r = ti->type->map(ti, clone);  in __map_bio()
1379 		trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,  in __map_bio()
1382 		generic_make_request(clone);  in __map_bio()
1415 	struct bio *clone = &tio->clone;  in clone_bio()  local
1417 	__bio_clone_fast(clone, bio);  in clone_bio()
1420 		bio_integrity_clone(clone, bio, GFP_NOIO);  in clone_bio()
1422 	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));  in clone_bio()
1423 	clone->bi_iter.bi_size = to_bytes(len);  in clone_bio()
1426 		bio_integrity_trim(clone, 0, len);  in clone_bio()
1434 	struct bio *clone;  in alloc_tio()  local
1436 	clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);  in alloc_tio()
1437 	tio = container_of(clone, struct dm_target_io, clone);  in alloc_tio()
1451 	struct bio *clone = &tio->clone;  in __clone_and_map_simple_bio()  local
1455 	__bio_clone_fast(clone, ci->bio);  in __clone_and_map_simple_bio()
1457 		bio_setup_sector(clone, ci->sector, *len);  in __clone_and_map_simple_bio()
1767 		container_of(bio, struct dm_rq_clone_bio_info, clone);  in dm_rq_bio_constructor()
1776 static int setup_clone(struct request *clone, struct request *rq,  in setup_clone()  argument
1781 	r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,  in setup_clone()
1786 	clone->cmd = rq->cmd;  in setup_clone()
1787 	clone->cmd_len = rq->cmd_len;  in setup_clone()
1788 	clone->sense = rq->sense;  in setup_clone()
1789 	clone->end_io = end_clone_request;  in setup_clone()
1790 	clone->end_io_data = tio;  in setup_clone()
1798 	struct request *clone;  in clone_rq()  local
1811 	clone = &tio->clone;  in clone_rq()
1812 	if (setup_clone(clone, rq, tio)) {  in clone_rq()
1818 	return clone;  in clone_rq()
1827 	struct request *clone;  in dm_prep_fn()  local
1834 	clone = clone_rq(rq, md, GFP_ATOMIC);  in dm_prep_fn()
1835 	if (!clone)  in dm_prep_fn()
1838 	rq->special = clone;  in dm_prep_fn()
1849 static int map_request(struct dm_target *ti, struct request *clone,  in map_request()  argument
1853 	struct dm_rq_target_io *tio = clone->end_io_data;  in map_request()
1856 	r = ti->type->map_rq(ti, clone, &tio->info);  in map_request()
1863 		trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),  in map_request()
1865 		dm_dispatch_request(clone);  in map_request()
1869 		dm_requeue_unmapped_request(clone);  in map_request()
1879 		dm_kill_unmapped_request(clone, r);  in map_request()
1888 	struct request *clone;  in dm_start_request()  local
1891 	clone = orig->special;  in dm_start_request()
1892 	atomic_inc(&md->pending[rq_data_dir(clone)]);  in dm_start_request()
1903 	return clone;  in dm_start_request()
1916 	struct request *rq, *clone;  in dm_request_fn()  local
1942 			clone = dm_start_request(md, rq);  in dm_request_fn()
1943 			dm_kill_unmapped_request(clone, -EIO);  in dm_request_fn()
1950 		clone = dm_start_request(md, rq);  in dm_request_fn()
1953 		if (map_request(ti, clone, md))  in dm_request_fn()
3084 …oundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);  in dm_alloc_md_mempools()
3088 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);  in dm_alloc_md_mempools()