1108 				unsigned int issue_flags);
1117 				 unsigned int issue_flags, u32 slot_index);
1118 static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags);
1904 static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,  in __io_req_complete()  argument
1907 	if (issue_flags & IO_URING_F_COMPLETE_DEFER)  in __io_req_complete()
3081 		       unsigned int issue_flags)  in kiocb_done()  argument
3094 			__io_req_complete(req, issue_flags,  in kiocb_done()
3112 			if (!(issue_flags & IO_URING_F_NONBLOCK)) {  in kiocb_done()
3114 				__io_req_complete(req, issue_flags, ret, cflags);  in kiocb_done()
3117 				__io_req_complete(req, issue_flags, ret, cflags);  in kiocb_done()
3607 static int io_read(struct io_kiocb *req, unsigned int issue_flags)  in io_read()  argument
3613 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_read()
3731 	kiocb_done(kiocb, ret, issue_flags);  in io_read()
3746 static int io_write(struct io_kiocb *req, unsigned int issue_flags)  in io_write()  argument
3752 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_write()
3832 		kiocb_done(kiocb, ret2, issue_flags);  in io_write()
3884 static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)  in io_renameat()  argument
3889 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_renameat()
3931 static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)  in io_unlinkat()  argument
3936 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_unlinkat()
3968 static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)  in io_shutdown()  argument
3974 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_shutdown()
4016 static int io_tee(struct io_kiocb *req, unsigned int issue_flags)  in io_tee()  argument
4024 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_tee()
4028 			 (sp->flags & SPLICE_F_FD_IN_FIXED), issue_flags);  in io_tee()
4055 static int io_splice(struct io_kiocb *req, unsigned int issue_flags)  in io_splice()  argument
4064 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_splice()
4068 			 (sp->flags & SPLICE_F_FD_IN_FIXED), issue_flags);  in io_splice()
4092 static int io_nop(struct io_kiocb *req, unsigned int issue_flags)  in io_nop()  argument
4099 	__io_req_complete(req, issue_flags, 0, 0);  in io_nop()
4122 static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)  in io_fsync()  argument
4128 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_fsync()
4155 static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)  in io_fallocate()  argument
4160 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_fallocate()
4234 static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)  in io_openat2()  argument
4247 	if (issue_flags & IO_URING_F_NONBLOCK) {  in io_openat2()
4279 		    (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)))  in io_openat2()
4284 	if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)  in io_openat2()
4291 		ret = io_install_fixed_file(req, file, issue_flags,  in io_openat2()
4298 	__io_req_complete(req, issue_flags, ret, 0);  in io_openat2()
4302 static int io_openat(struct io_kiocb *req, unsigned int issue_flags)  in io_openat()  argument
4304 	return io_openat2(req, issue_flags);  in io_openat()
4354 static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)  in io_remove_buffers()  argument
4360 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_remove_buffers()
4374 	__io_req_complete(req, issue_flags, ret, 0);  in io_remove_buffers()
4442 static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)  in io_provide_buffers()  argument
4448 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_provide_buffers()
4466 	__io_req_complete(req, issue_flags, ret, 0);  in io_provide_buffers()
4498 static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)  in io_epoll_ctl()  argument
4503 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_epoll_ctl()
4511 	__io_req_complete(req, issue_flags, ret, 0);  in io_epoll_ctl()
4535 static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)  in io_madvise()  argument
4541 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_madvise()
4567 static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)  in io_fadvise()  argument
4572 	if (issue_flags & IO_URING_F_NONBLOCK) {  in io_fadvise()
4586 	__io_req_complete(req, issue_flags, ret, 0);  in io_fadvise()
4608 static int io_statx(struct io_kiocb *req, unsigned int issue_flags)  in io_statx()  argument
4613 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_statx()
4643 static int io_close(struct io_kiocb *req, unsigned int issue_flags)  in io_close()  argument
4652 		ret = io_close_fixed(req, issue_flags);  in io_close()
4670 	if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {  in io_close()
4690 	__io_req_complete(req, issue_flags, ret, 0);  in io_close()
4710 static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)  in io_sync_file_range()  argument
4715 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_sync_file_range()
4809 static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)  in io_sendmsg()  argument
4833 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_sendmsg()
4841 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))  in io_sendmsg()
4860 	__io_req_complete(req, issue_flags, ret, 0);  in io_sendmsg()
4864 static int io_send(struct io_kiocb *req, unsigned int issue_flags)  in io_send()  argument
4888 	if (issue_flags & IO_URING_F_NONBLOCK)  in io_send()
4896 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))  in io_send()
4913 	__io_req_complete(req, issue_flags, ret, 0);  in io_send()
5060 static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)  in io_recvmsg()  argument
5069 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_recvmsg()
5128 	__io_req_complete(req, issue_flags, ret, cflags);  in io_recvmsg()
5132 static int io_recv(struct io_kiocb *req, unsigned int issue_flags)  in io_recv()  argument
5143 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_recv()
5197 	__io_req_complete(req, issue_flags, ret, cflags);  in io_recv()
5225 static int io_accept(struct io_kiocb *req, unsigned int issue_flags)  in io_accept()  argument
5228 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_accept()
5257 		ret = io_install_fixed_file(req, file, issue_flags,  in io_accept()
5260 	__io_req_complete(req, issue_flags, ret, 0);  in io_accept()
5287 static int io_connect(struct io_kiocb *req, unsigned int issue_flags)  in io_connect()  argument
5292 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_connect()
5324 	__io_req_complete(req, issue_flags, ret, 0);  in io_connect()
5329 static int io_##op(struct io_kiocb *req, unsigned int issue_flags)	\
5998 static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_add()  argument
6011 		__io_req_complete(req, issue_flags, ret, 0);  in io_poll_add()
6015 static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_update()  argument
6021 	io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_poll_update()
6042 		ret2 = io_poll_add(preq, issue_flags);  in io_poll_update()
6054 	io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_poll_update()
6222 static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout_remove()  argument
6304 static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout()  argument
6431 static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)  in io_async_cancel()  argument
6443 	io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_async_cancel()
6452 	io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_async_cancel()
6476 static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)  in io_files_update()  argument
6489 	io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_files_update()
6492 	io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_files_update()
6496 	__io_req_complete(req, issue_flags, ret, 0);  in io_files_update()
6755 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)  in io_issue_sqe()  argument
6766 		ret = io_nop(req, issue_flags);  in io_issue_sqe()
6771 		ret = io_read(req, issue_flags);  in io_issue_sqe()
6776 		ret = io_write(req, issue_flags);  in io_issue_sqe()
6779 		ret = io_fsync(req, issue_flags);  in io_issue_sqe()
6782 		ret = io_poll_add(req, issue_flags);  in io_issue_sqe()
6785 		ret = io_poll_update(req, issue_flags);  in io_issue_sqe()
6788 		ret = io_sync_file_range(req, issue_flags);  in io_issue_sqe()
6791 		ret = io_sendmsg(req, issue_flags);  in io_issue_sqe()
6794 		ret = io_send(req, issue_flags);  in io_issue_sqe()
6797 		ret = io_recvmsg(req, issue_flags);  in io_issue_sqe()
6800 		ret = io_recv(req, issue_flags);  in io_issue_sqe()
6803 		ret = io_timeout(req, issue_flags);  in io_issue_sqe()
6806 		ret = io_timeout_remove(req, issue_flags);  in io_issue_sqe()
6809 		ret = io_accept(req, issue_flags);  in io_issue_sqe()
6812 		ret = io_connect(req, issue_flags);  in io_issue_sqe()
6815 		ret = io_async_cancel(req, issue_flags);  in io_issue_sqe()
6818 		ret = io_fallocate(req, issue_flags);  in io_issue_sqe()
6821 		ret = io_openat(req, issue_flags);  in io_issue_sqe()
6824 		ret = io_close(req, issue_flags);  in io_issue_sqe()
6827 		ret = io_files_update(req, issue_flags);  in io_issue_sqe()
6830 		ret = io_statx(req, issue_flags);  in io_issue_sqe()
6833 		ret = io_fadvise(req, issue_flags);  in io_issue_sqe()
6836 		ret = io_madvise(req, issue_flags);  in io_issue_sqe()
6839 		ret = io_openat2(req, issue_flags);  in io_issue_sqe()
6842 		ret = io_epoll_ctl(req, issue_flags);  in io_issue_sqe()
6845 		ret = io_splice(req, issue_flags);  in io_issue_sqe()
6848 		ret = io_provide_buffers(req, issue_flags);  in io_issue_sqe()
6851 		ret = io_remove_buffers(req, issue_flags);  in io_issue_sqe()
6854 		ret = io_tee(req, issue_flags);  in io_issue_sqe()
6857 		ret = io_shutdown(req, issue_flags);  in io_issue_sqe()
6860 		ret = io_renameat(req, issue_flags);  in io_issue_sqe()
6863 		ret = io_unlinkat(req, issue_flags);  in io_issue_sqe()
6966 					     unsigned int issue_flags)  in io_file_get_fixed()  argument
6971 	io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_file_get_fixed()
6983 	io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_file_get_fixed()
7002 				       unsigned int issue_flags)  in io_file_get()  argument
7005 		return io_file_get_fixed(ctx, req, fd, issue_flags);  in io_file_get()
8486 				 unsigned int issue_flags, u32 slot_index)  in io_install_fixed_file()  argument
8489 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;  in io_install_fixed_file()
8535 static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)  in io_close_fixed()  argument
8543 	io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_close_fixed()
8569 	io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));  in io_close_fixed()