193 static void blkdev_bio_end_io_simple(struct bio *bio)  in blkdev_bio_end_io_simple()  argument
195 	struct task_struct *waiter = bio->bi_private;  in blkdev_bio_end_io_simple()
197 	WRITE_ONCE(bio->bi_private, NULL);  in blkdev_bio_end_io_simple()
210 	struct bio bio;  in __blkdev_direct_IO_simple()  local
227 	bio_init(&bio, vecs, nr_pages);  in __blkdev_direct_IO_simple()
228 	bio_set_dev(&bio, bdev);  in __blkdev_direct_IO_simple()
229 	bio.bi_iter.bi_sector = pos >> 9;  in __blkdev_direct_IO_simple()
230 	bio.bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO_simple()
231 	bio.bi_private = current;  in __blkdev_direct_IO_simple()
232 	bio.bi_end_io = blkdev_bio_end_io_simple;  in __blkdev_direct_IO_simple()
233 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
235 	ret = bio_iov_iter_get_pages(&bio, iter);  in __blkdev_direct_IO_simple()
238 	ret = bio.bi_iter.bi_size;  in __blkdev_direct_IO_simple()
241 		bio.bi_opf = REQ_OP_READ;  in __blkdev_direct_IO_simple()
245 		bio.bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO_simple()
249 		bio_set_polled(&bio, iocb);  in __blkdev_direct_IO_simple()
251 	qc = submit_bio(&bio);  in __blkdev_direct_IO_simple()
254 		if (!READ_ONCE(bio.bi_private))  in __blkdev_direct_IO_simple()
262 	bio_release_pages(&bio, should_dirty);  in __blkdev_direct_IO_simple()
263 	if (unlikely(bio.bi_status))  in __blkdev_direct_IO_simple()
264 		ret = blk_status_to_errno(bio.bi_status);  in __blkdev_direct_IO_simple()
270 	bio_uninit(&bio);  in __blkdev_direct_IO_simple()
285 	struct bio		bio;  member
298 static void blkdev_bio_end_io(struct bio *bio)  in blkdev_bio_end_io()  argument
300 	struct blkdev_dio *dio = bio->bi_private;  in blkdev_bio_end_io()
303 	if (bio->bi_status && !dio->bio.bi_status)  in blkdev_bio_end_io()
304 		dio->bio.bi_status = bio->bi_status;  in blkdev_bio_end_io()
311 			if (likely(!dio->bio.bi_status)) {  in blkdev_bio_end_io()
315 				ret = blk_status_to_errno(dio->bio.bi_status);  in blkdev_bio_end_io()
320 				bio_put(&dio->bio);  in blkdev_bio_end_io()
330 		bio_check_pages_dirty(bio);  in blkdev_bio_end_io()
332 		bio_release_pages(bio, false);  in blkdev_bio_end_io()
333 		bio_put(bio);  in blkdev_bio_end_io()
345 	struct bio *bio;  in __blkdev_direct_IO()  local
356 	bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool);  in __blkdev_direct_IO()
358 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO()
362 		bio_get(bio);  in __blkdev_direct_IO()
379 		bio_set_dev(bio, bdev);  in __blkdev_direct_IO()
380 		bio->bi_iter.bi_sector = pos >> 9;  in __blkdev_direct_IO()
381 		bio->bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO()
382 		bio->bi_private = dio;  in __blkdev_direct_IO()
383 		bio->bi_end_io = blkdev_bio_end_io;  in __blkdev_direct_IO()
384 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
386 		ret = bio_iov_iter_get_pages(bio, iter);  in __blkdev_direct_IO()
388 			bio->bi_status = BLK_STS_IOERR;  in __blkdev_direct_IO()
389 			bio_endio(bio);  in __blkdev_direct_IO()
394 			bio->bi_opf = REQ_OP_READ;  in __blkdev_direct_IO()
396 				bio_set_pages_dirty(bio);  in __blkdev_direct_IO()
398 			bio->bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO()
399 			task_io_account_write(bio->bi_iter.bi_size);  in __blkdev_direct_IO()
402 		dio->size += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
403 		pos += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
410 				bio_set_polled(bio, iocb);  in __blkdev_direct_IO()
414 			qc = submit_bio(bio);  in __blkdev_direct_IO()
428 				bio_get(bio);  in __blkdev_direct_IO()
435 		submit_bio(bio);  in __blkdev_direct_IO()
436 		bio = bio_alloc(GFP_KERNEL, nr_pages);  in __blkdev_direct_IO()
457 		ret = blk_status_to_errno(dio->bio.bi_status);  in __blkdev_direct_IO()
461 	bio_put(&dio->bio);  in __blkdev_direct_IO()
481 	return bioset_init(&blkdev_dio_pool, 4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS);  in blkdev_init()