39 static bool ext4_dio_supported(struct kiocb *iocb, struct iov_iter *iter)  in ext4_dio_supported()  argument
41 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_supported()
43 	if (!fscrypt_dio_supported(iocb, iter))  in ext4_dio_supported()
54 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dio_read_iter()  argument
57 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_read_iter()
59 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_read_iter()
66 	if (!ext4_dio_supported(iocb, to)) {  in ext4_dio_read_iter()
75 		iocb->ki_flags &= ~IOCB_DIRECT;  in ext4_dio_read_iter()
76 		return generic_file_read_iter(iocb, to);  in ext4_dio_read_iter()
79 	ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL,  in ext4_dio_read_iter()
80 			   is_sync_kiocb(iocb));  in ext4_dio_read_iter()
83 	file_accessed(iocb->ki_filp);  in ext4_dio_read_iter()
88 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dax_read_iter()  argument
90 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_read_iter()
93 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_read_iter()
106 		return generic_file_read_iter(iocb, to);  in ext4_dax_read_iter()
108 	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);  in ext4_dax_read_iter()
111 	file_accessed(iocb->ki_filp);  in ext4_dax_read_iter()
116 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_file_read_iter()  argument
118 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_read_iter()
128 		return ext4_dax_read_iter(iocb, to);  in ext4_file_read_iter()
130 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_read_iter()
131 		return ext4_dio_read_iter(iocb, to);  in ext4_file_read_iter()
133 	return generic_file_read_iter(iocb, to);  in ext4_file_read_iter()
214 static ssize_t ext4_generic_write_checks(struct kiocb *iocb,  in ext4_generic_write_checks()  argument
217 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_generic_write_checks()
223 	ret = generic_write_checks(iocb, from);  in ext4_generic_write_checks()
234 		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)  in ext4_generic_write_checks()
236 		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);  in ext4_generic_write_checks()
242 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)  in ext4_write_checks()  argument
246 	count = ext4_generic_write_checks(iocb, from);  in ext4_write_checks()
250 	ret = file_modified(iocb->ki_filp);  in ext4_write_checks()
256 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,  in ext4_buffered_write_iter()  argument
260 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_buffered_write_iter()
262 	if (iocb->ki_flags & IOCB_NOWAIT)  in ext4_buffered_write_iter()
266 	ret = ext4_write_checks(iocb, from);  in ext4_buffered_write_iter()
271 	ret = generic_perform_write(iocb->ki_filp, from, iocb->ki_pos);  in ext4_buffered_write_iter()
277 		iocb->ki_pos += ret;  in ext4_buffered_write_iter()
278 		ret = generic_write_sync(iocb, ret);  in ext4_buffered_write_iter()
372 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,  in ext4_dio_write_end_io()  argument
375 	loff_t pos = iocb->ki_pos;  in ext4_dio_write_end_io()
376 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_end_io()
423 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,  in ext4_dio_write_checks()  argument
426 	struct file *file = iocb->ki_filp;  in ext4_dio_write_checks()
433 	ret = ext4_generic_write_checks(iocb, from);  in ext4_dio_write_checks()
437 	offset = iocb->ki_pos;  in ext4_dio_write_checks()
449 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
472 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dio_write_iter()  argument
476 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_iter()
477 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_iter()
499 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_iter()
515 	if (!ext4_dio_supported(iocb, from)) {  in ext4_dio_write_iter()
520 		return ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
523 	ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend);  in ext4_dio_write_iter()
528 	if ((iocb->ki_flags & IOCB_NOWAIT) && (unaligned_io || extend)) {  in ext4_dio_write_iter()
539 	offset = iocb->ki_pos;  in ext4_dio_write_iter()
573 	ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,  in ext4_dio_write_iter()
574 			   is_sync_kiocb(iocb) || unaligned_io || extend);  in ext4_dio_write_iter()
591 		offset = iocb->ki_pos;  in ext4_dio_write_iter()
592 		err = ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
605 		err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
608 			invalidate_mapping_pages(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
618 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dax_write_iter()  argument
625 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_write_iter()
627 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_write_iter()
634 	ret = ext4_write_checks(iocb, from);  in ext4_dax_write_iter()
638 	offset = iocb->ki_pos;  in ext4_dax_write_iter()
658 	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);  in ext4_dax_write_iter()
665 		ret = generic_write_sync(iocb, ret);  in ext4_dax_write_iter()
671 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_file_write_iter()  argument
673 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_write_iter()
680 		return ext4_dax_write_iter(iocb, from);  in ext4_file_write_iter()
682 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_write_iter()
683 		return ext4_dio_write_iter(iocb, from);  in ext4_file_write_iter()
685 		return ext4_buffered_write_iter(iocb, from);  in ext4_file_write_iter()