216 static unsigned int dio_bio_write_op(struct kiocb *iocb)  in dio_bio_write_op()  argument
221 	if (iocb->ki_flags & IOCB_DSYNC)  in dio_bio_write_op()
237 __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,  in __blkdev_direct_IO_simple()  argument
240 	struct file *file = iocb->ki_filp;  in __blkdev_direct_IO_simple()
243 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_simple()
265 	bio.bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO_simple()
268 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
280 		bio.bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO_simple()
283 	if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_simple()
285 	if (iocb->ki_flags & IOCB_HIPRI)  in __blkdev_direct_IO_simple()
286 		bio_set_polled(&bio, iocb);  in __blkdev_direct_IO_simple()
293 		if (!(iocb->ki_flags & IOCB_HIPRI) ||  in __blkdev_direct_IO_simple()
314 		struct kiocb		*iocb;  member
345 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()  local
350 				iocb->ki_pos += ret;  in blkdev_bio_end_io()
355 			dio->iocb->ki_complete(iocb, ret, 0);  in blkdev_bio_end_io()
375 __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)  in __blkdev_direct_IO()  argument
377 	struct file *file = iocb->ki_filp;  in __blkdev_direct_IO()
383 	bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;  in __blkdev_direct_IO()
385 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO()
396 	dio->is_sync = is_sync = is_sync_kiocb(iocb);  in __blkdev_direct_IO()
401 		dio->iocb = iocb;  in __blkdev_direct_IO()
418 		bio->bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO()
421 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
435 			bio->bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO()
438 		if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO()
448 			if (iocb->ki_flags & IOCB_HIPRI) {  in __blkdev_direct_IO()
449 				bio_set_polled(bio, iocb);  in __blkdev_direct_IO()
456 				WRITE_ONCE(iocb->ki_cookie, qc);  in __blkdev_direct_IO()
489 		if (!(iocb->ki_flags & IOCB_HIPRI) ||  in __blkdev_direct_IO()
505 blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)  in blkdev_direct_IO()  argument
512 	if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)  in blkdev_direct_IO()
513 		return __blkdev_direct_IO_simple(iocb, iter, nr_pages);  in blkdev_direct_IO()
515 	return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES));  in blkdev_direct_IO()
1903 ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)  in blkdev_write_iter()  argument
1905 	struct file *file = iocb->ki_filp;  in blkdev_write_iter()
1921 	if (iocb->ki_pos >= size)  in blkdev_write_iter()
1924 	if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)  in blkdev_write_iter()
1927 	size -= iocb->ki_pos;  in blkdev_write_iter()
1934 	ret = __generic_file_write_iter(iocb, from);  in blkdev_write_iter()
1936 		ret = generic_write_sync(iocb, ret);  in blkdev_write_iter()
1943 ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)  in blkdev_read_iter()  argument
1945 	struct file *file = iocb->ki_filp;  in blkdev_read_iter()
1948 	loff_t pos = iocb->ki_pos;  in blkdev_read_iter()
1961 	ret = generic_file_read_iter(iocb, to);  in blkdev_read_iter()