49 	struct nvmet_req *req = bio->bi_private;  in nvmet_bio_done()  local
51 	nvmet_req_complete(req,  in nvmet_bio_done()
54 	if (bio != &req->b.inline_bio)  in nvmet_bio_done()
58 static void nvmet_bdev_execute_rw(struct nvmet_req *req)  in nvmet_bdev_execute_rw()  argument
60 	int sg_cnt = req->sg_cnt;  in nvmet_bdev_execute_rw()
61 	struct bio *bio = &req->b.inline_bio;  in nvmet_bdev_execute_rw()
67 	if (!req->sg_cnt) {  in nvmet_bdev_execute_rw()
68 		nvmet_req_complete(req, 0);  in nvmet_bdev_execute_rw()
72 	if (req->cmd->rw.opcode == nvme_cmd_write) {  in nvmet_bdev_execute_rw()
75 		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))  in nvmet_bdev_execute_rw()
81 	sector = le64_to_cpu(req->cmd->rw.slba);  in nvmet_bdev_execute_rw()
82 	sector <<= (req->ns->blksize_shift - 9);  in nvmet_bdev_execute_rw()
84 	bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));  in nvmet_bdev_execute_rw()
85 	bio_set_dev(bio, req->ns->bdev);  in nvmet_bdev_execute_rw()
87 	bio->bi_private = req;  in nvmet_bdev_execute_rw()
91 	for_each_sg(req->sg, sg, req->sg_cnt, i) {  in nvmet_bdev_execute_rw()
97 			bio_set_dev(bio, req->ns->bdev);  in nvmet_bdev_execute_rw()
111 	blk_poll(bdev_get_queue(req->ns->bdev), cookie);  in nvmet_bdev_execute_rw()
114 static void nvmet_bdev_execute_flush(struct nvmet_req *req)  in nvmet_bdev_execute_flush()  argument
116 	struct bio *bio = &req->b.inline_bio;  in nvmet_bdev_execute_flush()
118 	bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));  in nvmet_bdev_execute_flush()
119 	bio_set_dev(bio, req->ns->bdev);  in nvmet_bdev_execute_flush()
120 	bio->bi_private = req;  in nvmet_bdev_execute_flush()
127 u16 nvmet_bdev_flush(struct nvmet_req *req)  in nvmet_bdev_flush()  argument
129 	if (blkdev_issue_flush(req->ns->bdev, GFP_KERNEL, NULL))  in nvmet_bdev_flush()
148 static void nvmet_bdev_execute_discard(struct nvmet_req *req)  in nvmet_bdev_execute_discard()  argument
155 	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {  in nvmet_bdev_execute_discard()
156 		status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,  in nvmet_bdev_execute_discard()
161 		status = nvmet_bdev_discard_range(req->ns, &range, &bio);  in nvmet_bdev_execute_discard()
167 		bio->bi_private = req;  in nvmet_bdev_execute_discard()
176 		nvmet_req_complete(req, status);  in nvmet_bdev_execute_discard()
180 static void nvmet_bdev_execute_dsm(struct nvmet_req *req)  in nvmet_bdev_execute_dsm()  argument
182 	switch (le32_to_cpu(req->cmd->dsm.attributes)) {  in nvmet_bdev_execute_dsm()
184 		nvmet_bdev_execute_discard(req);  in nvmet_bdev_execute_dsm()
190 		nvmet_req_complete(req, 0);  in nvmet_bdev_execute_dsm()
195 static void nvmet_bdev_execute_write_zeroes(struct nvmet_req *req)  in nvmet_bdev_execute_write_zeroes()  argument
197 	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;  in nvmet_bdev_execute_write_zeroes()
204 		(req->ns->blksize_shift - 9);  in nvmet_bdev_execute_write_zeroes()
206 		(req->ns->blksize_shift - 9));  in nvmet_bdev_execute_write_zeroes()
208 	if (__blkdev_issue_zeroout(req->ns->bdev, sector, nr_sector,  in nvmet_bdev_execute_write_zeroes()
213 		bio->bi_private = req;  in nvmet_bdev_execute_write_zeroes()
217 		nvmet_req_complete(req, status);  in nvmet_bdev_execute_write_zeroes()
221 u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req)  in nvmet_bdev_parse_io_cmd()  argument
223 	struct nvme_command *cmd = req->cmd;  in nvmet_bdev_parse_io_cmd()
228 		req->execute = nvmet_bdev_execute_rw;  in nvmet_bdev_parse_io_cmd()
229 		req->data_len = nvmet_rw_len(req);  in nvmet_bdev_parse_io_cmd()
232 		req->execute = nvmet_bdev_execute_flush;  in nvmet_bdev_parse_io_cmd()
233 		req->data_len = 0;  in nvmet_bdev_parse_io_cmd()
236 		req->execute = nvmet_bdev_execute_dsm;  in nvmet_bdev_parse_io_cmd()
237 		req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *  in nvmet_bdev_parse_io_cmd()
241 		req->execute = nvmet_bdev_execute_write_zeroes;  in nvmet_bdev_parse_io_cmd()
242 		req->data_len = 0;  in nvmet_bdev_parse_io_cmd()
246 		       req->sq->qid);  in nvmet_bdev_parse_io_cmd()