Lines Matching full:req
31 static inline bool io_file_supports_nowait(struct io_kiocb *req) in io_file_supports_nowait() argument
33 return req->flags & REQ_F_SUPPORT_NOWAIT; in io_file_supports_nowait()
55 static int io_iov_buffer_select_prep(struct io_kiocb *req) in io_iov_buffer_select_prep() argument
59 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_iov_buffer_select_prep()
65 if (req->ctx->compat) in io_iov_buffer_select_prep()
76 int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_rw() argument
78 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_prep_rw()
84 req->buf_index = READ_ONCE(sqe->buf_index); in io_prep_rw()
86 if (req->opcode == IORING_OP_READ_FIXED || in io_prep_rw()
87 req->opcode == IORING_OP_WRITE_FIXED) { in io_prep_rw()
88 struct io_ring_ctx *ctx = req->ctx; in io_prep_rw()
91 if (unlikely(req->buf_index >= ctx->nr_user_bufs)) in io_prep_rw()
93 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); in io_prep_rw()
94 req->imu = ctx->user_bufs[index]; in io_prep_rw()
95 io_req_set_rsrc_node(req, ctx, 0); in io_prep_rw()
117 if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) { in io_prep_rw()
118 ret = io_iov_buffer_select_prep(req); in io_prep_rw()
126 void io_readv_writev_cleanup(struct io_kiocb *req) in io_readv_writev_cleanup() argument
128 struct io_async_rw *io = req->async_data; in io_readv_writev_cleanup()
154 static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req) in io_kiocb_update_pos() argument
156 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_kiocb_update_pos()
161 if (!(req->file->f_mode & FMODE_STREAM)) { in io_kiocb_update_pos()
162 req->flags |= REQ_F_CUR_POS; in io_kiocb_update_pos()
163 rw->kiocb.ki_pos = req->file->f_pos; in io_kiocb_update_pos()
172 static bool io_resubmit_prep(struct io_kiocb *req) in io_resubmit_prep() argument
174 struct io_async_rw *io = req->async_data; in io_resubmit_prep()
176 if (!req_has_async_data(req)) in io_resubmit_prep()
177 return !io_req_prep_async(req); in io_resubmit_prep()
182 static bool io_rw_should_reissue(struct io_kiocb *req) in io_rw_should_reissue() argument
184 umode_t mode = file_inode(req->file)->i_mode; in io_rw_should_reissue()
185 struct io_ring_ctx *ctx = req->ctx; in io_rw_should_reissue()
189 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && in io_rw_should_reissue()
203 if (!same_thread_group(req->task, current) || !in_task()) in io_rw_should_reissue()
208 static bool io_resubmit_prep(struct io_kiocb *req) in io_resubmit_prep() argument
212 static bool io_rw_should_reissue(struct io_kiocb *req) in io_rw_should_reissue() argument
218 static void io_req_end_write(struct io_kiocb *req) in io_req_end_write() argument
220 if (req->flags & REQ_F_ISREG) { in io_req_end_write()
221 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_end_write()
231 static void io_req_io_end(struct io_kiocb *req) in io_req_io_end() argument
233 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_io_end()
236 io_req_end_write(req); in io_req_io_end()
237 fsnotify_modify(req->file); in io_req_io_end()
239 fsnotify_access(req->file); in io_req_io_end()
243 static bool __io_complete_rw_common(struct io_kiocb *req, long res) in __io_complete_rw_common() argument
245 if (unlikely(res != req->cqe.res)) { in __io_complete_rw_common()
247 io_rw_should_reissue(req)) { in __io_complete_rw_common()
252 io_req_io_end(req); in __io_complete_rw_common()
253 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in __io_complete_rw_common()
256 req_set_fail(req); in __io_complete_rw_common()
257 req->cqe.res = res; in __io_complete_rw_common()
262 static inline int io_fixup_rw_res(struct io_kiocb *req, long res) in io_fixup_rw_res() argument
264 struct io_async_rw *io = req->async_data; in io_fixup_rw_res()
267 if (req_has_async_data(req) && io->bytes_done > 0) { in io_fixup_rw_res()
276 void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts) in io_req_rw_complete() argument
278 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_rw_complete()
284 io_req_set_res(req, io_fixup_rw_res(req, res), 0); in io_req_rw_complete()
287 io_req_io_end(req); in io_req_rw_complete()
289 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { in io_req_rw_complete()
292 req->cqe.flags |= io_put_kbuf(req, issue_flags); in io_req_rw_complete()
294 io_req_task_complete(req, ts); in io_req_rw_complete()
300 struct io_kiocb *req = cmd_to_io_kiocb(rw); in io_complete_rw() local
303 if (__io_complete_rw_common(req, res)) in io_complete_rw()
305 io_req_set_res(req, io_fixup_rw_res(req, res), 0); in io_complete_rw()
307 req->io_task_work.func = io_req_rw_complete; in io_complete_rw()
308 __io_req_task_work_add(req, IOU_F_TWQ_LAZY_WAKE); in io_complete_rw()
314 struct io_kiocb *req = cmd_to_io_kiocb(rw); in io_complete_rw_iopoll() local
317 io_req_end_write(req); in io_complete_rw_iopoll()
318 if (unlikely(res != req->cqe.res)) { in io_complete_rw_iopoll()
319 if (res == -EAGAIN && io_rw_should_reissue(req)) { in io_complete_rw_iopoll()
320 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in io_complete_rw_iopoll()
323 req->cqe.res = res; in io_complete_rw_iopoll()
327 smp_store_release(&req->iopoll_completed, 1); in io_complete_rw_iopoll()
330 static int kiocb_done(struct io_kiocb *req, ssize_t ret, in kiocb_done() argument
333 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in kiocb_done()
334 unsigned final_ret = io_fixup_rw_res(req, ret); in kiocb_done()
336 if (ret >= 0 && req->flags & REQ_F_CUR_POS) in kiocb_done()
337 req->file->f_pos = rw->kiocb.ki_pos; in kiocb_done()
339 if (!__io_complete_rw_common(req, ret)) { in kiocb_done()
344 io_req_io_end(req); in kiocb_done()
345 io_req_set_res(req, final_ret, in kiocb_done()
346 io_put_kbuf(req, issue_flags)); in kiocb_done()
353 if (req->flags & REQ_F_REISSUE) { in kiocb_done()
354 req->flags &= ~REQ_F_REISSUE; in kiocb_done()
355 if (io_resubmit_prep(req)) in kiocb_done()
358 io_req_task_queue_fail(req, final_ret); in kiocb_done()
363 static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, in __io_import_iovec() argument
367 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in __io_import_iovec()
369 u8 opcode = req->opcode; in __io_import_iovec()
376 ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len); in __io_import_iovec()
386 (req->flags & REQ_F_BUFFER_SELECT)) { in __io_import_iovec()
387 if (io_do_buffer_select(req)) { in __io_import_iovec()
388 buf = io_buffer_select(req, &sqe_len, issue_flags); in __io_import_iovec()
403 req->ctx->compat); in __io_import_iovec()
409 static inline int io_import_iovec(int rw, struct io_kiocb *req, in io_import_iovec() argument
413 *iovec = __io_import_iovec(rw, req, s, issue_flags); in io_import_iovec()
492 static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, in io_req_map_rw() argument
495 struct io_async_rw *io = req->async_data; in io_req_map_rw()
515 req->flags |= REQ_F_NEED_CLEANUP; in io_req_map_rw()
519 static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, in io_setup_async_rw() argument
522 if (!force && !io_cold_defs[req->opcode].prep_async) in io_setup_async_rw()
524 if (!req_has_async_data(req)) { in io_setup_async_rw()
527 if (io_alloc_async_data(req)) { in io_setup_async_rw()
532 io_req_map_rw(req, iovec, s->fast_iov, &s->iter); in io_setup_async_rw()
533 iorw = req->async_data; in io_setup_async_rw()
540 static inline int io_rw_prep_async(struct io_kiocb *req, int rw) in io_rw_prep_async() argument
542 struct io_async_rw *iorw = req->async_data; in io_rw_prep_async()
550 ret = io_import_iovec(rw, req, &iov, &iorw->s, 0); in io_rw_prep_async()
556 req->flags |= REQ_F_NEED_CLEANUP; in io_rw_prep_async()
562 int io_readv_prep_async(struct io_kiocb *req) in io_readv_prep_async() argument
564 return io_rw_prep_async(req, ITER_DEST); in io_readv_prep_async()
567 int io_writev_prep_async(struct io_kiocb *req) in io_writev_prep_async() argument
569 return io_rw_prep_async(req, ITER_SOURCE); in io_writev_prep_async()
586 struct io_kiocb *req = wait->private; in io_async_buf_func() local
587 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_async_buf_func()
597 io_req_task_queue(req); in io_async_buf_func()
613 static bool io_rw_should_retry(struct io_kiocb *req) in io_rw_should_retry() argument
615 struct io_async_rw *io = req->async_data; in io_rw_should_retry()
617 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_should_retry()
621 if (req->flags & REQ_F_NOWAIT) in io_rw_should_retry()
632 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) in io_rw_should_retry()
636 wait->wait.private = req; in io_rw_should_retry()
657 static bool need_complete_io(struct io_kiocb *req) in need_complete_io() argument
659 return req->flags & REQ_F_ISREG || in need_complete_io()
660 S_ISBLK(file_inode(req->file)->i_mode); in need_complete_io()
663 static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) in io_rw_init_file() argument
665 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_init_file()
667 struct io_ring_ctx *ctx = req->ctx; in io_rw_init_file()
668 struct file *file = req->file; in io_rw_init_file()
674 if (!(req->flags & REQ_F_FIXED_FILE)) in io_rw_init_file()
675 req->flags |= io_file_get_flags(file); in io_rw_init_file()
689 ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) in io_rw_init_file()
690 req->flags |= REQ_F_NOWAIT; in io_rw_init_file()
699 req->iopoll_completed = 0; in io_rw_init_file()
709 static int __io_read(struct io_kiocb *req, unsigned int issue_flags) in __io_read() argument
711 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in __io_read()
720 if (!req_has_async_data(req)) { in __io_read()
721 ret = io_import_iovec(ITER_DEST, req, &iovec, s, issue_flags); in __io_read()
725 io = req->async_data; in __io_read()
732 if (io_do_buffer_select(req)) { in __io_read()
733 ret = io_import_iovec(ITER_DEST, req, &iovec, s, issue_flags); in __io_read()
746 ret = io_rw_init_file(req, FMODE_READ); in __io_read()
751 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
755 if (unlikely(!io_file_supports_nowait(req))) { in __io_read()
756 ret = io_setup_async_rw(req, iovec, s, true); in __io_read()
765 ppos = io_kiocb_update_pos(req); in __io_read()
767 ret = rw_verify_area(READ, req->file, ppos, req->cqe.res); in __io_read()
783 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { in __io_read()
784 req->flags &= ~REQ_F_REISSUE; in __io_read()
786 if (req->opcode == IORING_OP_READ && file_can_poll(req->file)) in __io_read()
789 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) in __io_read()
792 if (req->flags & REQ_F_NOWAIT) in __io_read()
796 req->flags |= REQ_F_PARTIAL_IO; in __io_read()
797 io_kbuf_recycle(req, issue_flags); in __io_read()
801 } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock || in __io_read()
802 (req->flags & REQ_F_NOWAIT) || !need_complete_io(req)) { in __io_read()
814 ret2 = io_setup_async_rw(req, iovec, s, true); in __io_read()
821 req->flags |= REQ_F_PARTIAL_IO; in __io_read()
822 io_kbuf_recycle(req, issue_flags); in __io_read()
824 io = req->async_data; in __io_read()
844 if (!io_rw_should_retry(req)) { in __io_read()
849 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
870 int io_read(struct io_kiocb *req, unsigned int issue_flags) in io_read() argument
874 ret = __io_read(req, issue_flags); in io_read()
876 return kiocb_done(req, ret, issue_flags); in io_read()
881 static bool io_kiocb_start_write(struct io_kiocb *req, struct kiocb *kiocb) in io_kiocb_start_write() argument
886 if (!(req->flags & REQ_F_ISREG)) in io_kiocb_start_write()
900 int io_write(struct io_kiocb *req, unsigned int issue_flags) in io_write() argument
902 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_write()
910 if (!req_has_async_data(req)) { in io_write()
911 ret = io_import_iovec(ITER_SOURCE, req, &iovec, s, issue_flags); in io_write()
915 struct io_async_rw *io = req->async_data; in io_write()
921 ret = io_rw_init_file(req, FMODE_WRITE); in io_write()
926 req->cqe.res = iov_iter_count(&s->iter); in io_write()
930 if (unlikely(!io_file_supports_nowait(req))) in io_write()
936 (req->flags & REQ_F_ISREG)) in io_write()
945 ppos = io_kiocb_update_pos(req); in io_write()
947 ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res); in io_write()
953 if (unlikely(!io_kiocb_start_write(req, kiocb))) in io_write()
957 if (likely(req->file->f_op->write_iter)) in io_write()
958 ret2 = call_write_iter(req->file, kiocb, &s->iter); in io_write()
959 else if (req->file->f_op->write) in io_write()
965 req->flags |= REQ_F_PARTIAL_IO; in io_write()
966 io_kbuf_recycle(req, issue_flags); in io_write()
969 if (req->flags & REQ_F_REISSUE) { in io_write()
970 req->flags &= ~REQ_F_REISSUE; in io_write()
981 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) in io_write()
985 if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) in io_write()
988 if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) { in io_write()
991 trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2, in io_write()
992 req->cqe.res, ret2); in io_write()
1000 ret = io_setup_async_rw(req, iovec, s, true); in io_write()
1002 io = req->async_data; in io_write()
1007 io_req_end_write(req); in io_write()
1011 ret = kiocb_done(req, ret2, issue_flags); in io_write()
1015 ret = io_setup_async_rw(req, iovec, s, false); in io_write()
1018 io_req_end_write(req); in io_write()
1029 void io_rw_fail(struct io_kiocb *req) in io_rw_fail() argument
1033 res = io_fixup_rw_res(req, req->cqe.res); in io_rw_fail()
1034 io_req_set_res(req, res, req->cqe.flags); in io_rw_fail()
1052 struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); in io_do_iopoll() local
1053 struct file *file = req->file; in io_do_iopoll()
1061 if (READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1064 if (req->opcode == IORING_OP_URING_CMD) { in io_do_iopoll()
1067 ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); in io_do_iopoll()
1071 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_do_iopoll()
1080 /* iopoll may have completed current req */ in io_do_iopoll()
1082 READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1093 struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); in io_do_iopoll() local
1096 if (!smp_load_acquire(&req->iopoll_completed)) in io_do_iopoll()
1099 req->cqe.flags = io_put_kbuf(req, 0); in io_do_iopoll()