Lines Matching +full:blocking +full:- +full:io
1 // SPDX-License-Identifier: GPL-2.0
6 #include <linux/blk-mq.h>
34 if (req->flags & REQ_F_SUPPORT_NOWAIT) in io_file_supports_nowait()
40 return vfs_poll(req->file, &pt) & mask; in io_file_supports_nowait()
52 uiov = u64_to_user_ptr(rw->addr); in io_iov_compat_buffer_select_prep()
54 return -EFAULT; in io_iov_compat_buffer_select_prep()
55 if (__get_user(clen, &uiov->iov_len)) in io_iov_compat_buffer_select_prep()
56 return -EFAULT; in io_iov_compat_buffer_select_prep()
58 return -EINVAL; in io_iov_compat_buffer_select_prep()
60 rw->len = clen; in io_iov_compat_buffer_select_prep()
71 if (rw->len != 1) in io_iov_buffer_select_prep()
72 return -EINVAL; in io_iov_buffer_select_prep()
75 if (req->ctx->compat) in io_iov_buffer_select_prep()
79 uiov = u64_to_user_ptr(rw->addr); in io_iov_buffer_select_prep()
81 return -EFAULT; in io_iov_buffer_select_prep()
82 rw->len = iov.iov_len; in io_iov_buffer_select_prep()
92 rw->kiocb.ki_pos = READ_ONCE(sqe->off); in io_prep_rw()
93 /* used for fixed read/write too - just read unconditionally */ in io_prep_rw()
94 req->buf_index = READ_ONCE(sqe->buf_index); in io_prep_rw()
96 if (req->opcode == IORING_OP_READ_FIXED || in io_prep_rw()
97 req->opcode == IORING_OP_WRITE_FIXED) { in io_prep_rw()
98 struct io_ring_ctx *ctx = req->ctx; in io_prep_rw()
101 if (unlikely(req->buf_index >= ctx->nr_user_bufs)) in io_prep_rw()
102 return -EFAULT; in io_prep_rw()
103 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); in io_prep_rw()
104 req->imu = ctx->user_bufs[index]; in io_prep_rw()
108 ioprio = READ_ONCE(sqe->ioprio); in io_prep_rw()
114 rw->kiocb.ki_ioprio = ioprio; in io_prep_rw()
116 rw->kiocb.ki_ioprio = get_current_ioprio(); in io_prep_rw()
118 rw->kiocb.dio_complete = NULL; in io_prep_rw()
120 rw->addr = READ_ONCE(sqe->addr); in io_prep_rw()
121 rw->len = READ_ONCE(sqe->len); in io_prep_rw()
122 rw->flags = READ_ONCE(sqe->rw_flags); in io_prep_rw()
124 /* Have to do this validation here, as this is in io_read() rw->len might in io_prep_rw()
127 if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) { in io_prep_rw()
138 struct io_async_rw *io = req->async_data; in io_readv_writev_cleanup() local
140 kfree(io->free_iovec); in io_readv_writev_cleanup()
146 case -EIOCBQUEUED: in io_rw_done()
148 case -ERESTARTSYS: in io_rw_done()
149 case -ERESTARTNOINTR: in io_rw_done()
150 case -ERESTARTNOHAND: in io_rw_done()
151 case -ERESTART_RESTARTBLOCK: in io_rw_done()
155 * IO with EINTR. in io_rw_done()
157 ret = -EINTR; in io_rw_done()
160 kiocb->ki_complete(kiocb, ret); in io_rw_done()
168 if (rw->kiocb.ki_pos != -1) in io_kiocb_update_pos()
169 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
171 if (!(req->file->f_mode & FMODE_STREAM)) { in io_kiocb_update_pos()
172 req->flags |= REQ_F_CUR_POS; in io_kiocb_update_pos()
173 rw->kiocb.ki_pos = req->file->f_pos; in io_kiocb_update_pos()
174 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
177 rw->kiocb.ki_pos = 0; in io_kiocb_update_pos()
184 struct io_async_rw *io = req->async_data; in io_resubmit_prep() local
188 iov_iter_restore(&io->s.iter, &io->s.iter_state); in io_resubmit_prep()
194 umode_t mode = file_inode(req->file)->i_mode; in io_rw_should_reissue()
195 struct io_ring_ctx *ctx = req->ctx; in io_rw_should_reissue()
199 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && in io_rw_should_reissue()
200 !(ctx->flags & IORING_SETUP_IOPOLL))) in io_rw_should_reissue()
205 * -EAGAIN. in io_rw_should_reissue()
207 if (percpu_ref_is_dying(&ctx->refs)) in io_rw_should_reissue()
210 * Play it safe and assume not safe to re-import and reissue if we're in io_rw_should_reissue()
213 if (!same_thread_group(req->task, current) || !in_task()) in io_rw_should_reissue()
230 if (req->flags & REQ_F_ISREG) { in io_req_end_write()
233 kiocb_end_write(&rw->kiocb); in io_req_end_write()
238 * Trigger the notifications after having done some IO, and finish the write
245 if (rw->kiocb.ki_flags & IOCB_WRITE) { in io_req_io_end()
247 fsnotify_modify(req->file); in io_req_io_end()
249 fsnotify_access(req->file); in io_req_io_end()
255 if (unlikely(res != req->cqe.res)) { in __io_complete_rw_common()
256 if ((res == -EAGAIN || res == -EOPNOTSUPP) && in __io_complete_rw_common()
263 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in __io_complete_rw_common()
267 req->cqe.res = res; in __io_complete_rw_common()
274 struct io_async_rw *io = req->async_data; in io_fixup_rw_res() local
276 /* add previously done IO, if any */ in io_fixup_rw_res()
277 if (req_has_async_data(req) && io->bytes_done > 0) { in io_fixup_rw_res()
279 res = io->bytes_done; in io_fixup_rw_res()
281 res += io->bytes_done; in io_fixup_rw_res()
289 struct kiocb *kiocb = &rw->kiocb; in io_req_rw_complete()
291 if ((kiocb->ki_flags & IOCB_DIO_CALLER_COMP) && kiocb->dio_complete) { in io_req_rw_complete()
292 long res = kiocb->dio_complete(rw->kiocb.private); in io_req_rw_complete()
299 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { in io_req_rw_complete()
300 unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED; in io_req_rw_complete()
302 req->cqe.flags |= io_put_kbuf(req, issue_flags); in io_req_rw_complete()
312 if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) { in io_complete_rw()
317 req->io_task_work.func = io_req_rw_complete; in io_complete_rw()
326 if (kiocb->ki_flags & IOCB_WRITE) in io_complete_rw_iopoll()
328 if (unlikely(res != req->cqe.res)) { in io_complete_rw_iopoll()
329 if (res == -EAGAIN && io_rw_should_reissue(req)) { in io_complete_rw_iopoll()
330 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in io_complete_rw_iopoll()
333 req->cqe.res = res; in io_complete_rw_iopoll()
336 /* order with io_iopoll_complete() checking ->iopoll_completed */ in io_complete_rw_iopoll()
337 smp_store_release(&req->iopoll_completed, 1); in io_complete_rw_iopoll()
346 if (ret >= 0 && req->flags & REQ_F_CUR_POS) in kiocb_done()
347 req->file->f_pos = rw->kiocb.ki_pos; in kiocb_done()
348 if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) { in kiocb_done()
360 io_rw_done(&rw->kiocb, ret); in kiocb_done()
363 if (req->flags & REQ_F_REISSUE) { in kiocb_done()
364 req->flags &= ~REQ_F_REISSUE; in kiocb_done()
366 return -EAGAIN; in kiocb_done()
378 struct iov_iter *iter = &s->iter; in __io_import_iovec()
379 u8 opcode = req->opcode; in __io_import_iovec()
386 ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len); in __io_import_iovec()
392 buf = u64_to_user_ptr(rw->addr); in __io_import_iovec()
393 sqe_len = rw->len; in __io_import_iovec()
396 (req->flags & REQ_F_BUFFER_SELECT)) { in __io_import_iovec()
400 return ERR_PTR(-ENOBUFS); in __io_import_iovec()
401 rw->addr = (unsigned long) buf; in __io_import_iovec()
402 rw->len = sqe_len; in __io_import_iovec()
411 iovec = s->fast_iov; in __io_import_iovec()
413 req->ctx->compat); in __io_import_iovec()
427 iov_iter_save_state(&s->iter, &s->iter_state); in io_import_iovec()
433 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; in io_kiocb_ppos()
437 * For files that don't have ->read_iter() and ->write_iter(), handle them
438 * by looping over ->read() or ->write() manually.
442 struct kiocb *kiocb = &rw->kiocb; in loop_rw_iter()
443 struct file *file = kiocb->ki_filp; in loop_rw_iter()
448 * Don't support polled IO through this interface, and we can't in loop_rw_iter()
449 * support non-blocking either. For the latter, this just causes in loop_rw_iter()
452 if (kiocb->ki_flags & IOCB_HIPRI) in loop_rw_iter()
453 return -EOPNOTSUPP; in loop_rw_iter()
454 if ((kiocb->ki_flags & IOCB_NOWAIT) && in loop_rw_iter()
455 !(kiocb->ki_filp->f_flags & O_NONBLOCK)) in loop_rw_iter()
456 return -EAGAIN; in loop_rw_iter()
466 addr = iter->ubuf + iter->iov_offset; in loop_rw_iter()
472 addr = u64_to_user_ptr(rw->addr); in loop_rw_iter()
473 len = rw->len; in loop_rw_iter()
477 nr = file->f_op->read(file, addr, len, ppos); in loop_rw_iter()
479 nr = file->f_op->write(file, addr, len, ppos); in loop_rw_iter()
490 rw->addr += nr; in loop_rw_iter()
491 rw->len -= nr; in loop_rw_iter()
492 if (!rw->len) in loop_rw_iter()
505 struct io_async_rw *io = req->async_data; in io_req_map_rw() local
507 memcpy(&io->s.iter, iter, sizeof(*iter)); in io_req_map_rw()
508 io->free_iovec = iovec; in io_req_map_rw()
509 io->bytes_done = 0; in io_req_map_rw()
516 io->s.iter.__iov = io->s.fast_iov; in io_req_map_rw()
517 if (iter->__iov != fast_iov) { in io_req_map_rw()
518 iov_off = iter_iov(iter) - fast_iov; in io_req_map_rw()
519 io->s.iter.__iov += iov_off; in io_req_map_rw()
521 if (io->s.fast_iov != fast_iov) in io_req_map_rw()
522 memcpy(io->s.fast_iov + iov_off, fast_iov + iov_off, in io_req_map_rw()
523 sizeof(struct iovec) * iter->nr_segs); in io_req_map_rw()
525 req->flags |= REQ_F_NEED_CLEANUP; in io_req_map_rw()
532 if (!force && !io_cold_defs[req->opcode].prep_async) in io_setup_async_rw()
539 return -ENOMEM; in io_setup_async_rw()
542 io_req_map_rw(req, iovec, s->fast_iov, &s->iter); in io_setup_async_rw()
543 iorw = req->async_data; in io_setup_async_rw()
545 iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); in io_setup_async_rw()
552 struct io_async_rw *iorw = req->async_data; in io_rw_prep_async()
556 iorw->bytes_done = 0; in io_rw_prep_async()
557 iorw->free_iovec = NULL; in io_rw_prep_async()
559 /* submission path, ->uring_lock should already be taken */ in io_rw_prep_async()
560 ret = io_import_iovec(rw, req, &iov, &iorw->s, 0); in io_rw_prep_async()
565 iorw->free_iovec = iov; in io_rw_prep_async()
566 req->flags |= REQ_F_NEED_CLEANUP; in io_rw_prep_async()
584 * when we initially tried to do the IO with the iocb armed our waitqueue.
586 * happen when the page IO is completed and the page is now uptodate. This will
589 * do a thread based blocking retry of the operation. That's the unexpected
596 struct io_kiocb *req = wait->private; in io_async_buf_func()
605 rw->kiocb.ki_flags &= ~IOCB_WAITQ; in io_async_buf_func()
606 list_del_init(&wait->entry); in io_async_buf_func()
612 * This controls whether a given IO request should be armed for async page
617 * will register a callback when the page is unlocked at IO completion. Through
621 * async worker threads for a blocking retry.
625 struct io_async_rw *io = req->async_data; in io_rw_should_retry() local
626 struct wait_page_queue *wait = &io->wpq; in io_rw_should_retry()
628 struct kiocb *kiocb = &rw->kiocb; in io_rw_should_retry()
630 /* never retry for NOWAIT, we just complete with -EAGAIN */ in io_rw_should_retry()
631 if (req->flags & REQ_F_NOWAIT) in io_rw_should_retry()
634 /* Only for buffered IO */ in io_rw_should_retry()
635 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) in io_rw_should_retry()
642 if (io_file_can_poll(req) || !(req->file->f_mode & FMODE_BUF_RASYNC)) in io_rw_should_retry()
645 wait->wait.func = io_async_buf_func; in io_rw_should_retry()
646 wait->wait.private = req; in io_rw_should_retry()
647 wait->wait.flags = 0; in io_rw_should_retry()
648 INIT_LIST_HEAD(&wait->wait.entry); in io_rw_should_retry()
649 kiocb->ki_flags |= IOCB_WAITQ; in io_rw_should_retry()
650 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_rw_should_retry()
651 kiocb->ki_waitq = wait; in io_rw_should_retry()
657 struct file *file = rw->kiocb.ki_filp; in io_iter_do_read()
659 if (likely(file->f_op->read_iter)) in io_iter_do_read()
660 return call_read_iter(file, &rw->kiocb, iter); in io_iter_do_read()
661 else if (file->f_op->read) in io_iter_do_read()
664 return -EINVAL; in io_iter_do_read()
669 return req->flags & REQ_F_ISREG || in need_complete_io()
670 S_ISBLK(file_inode(req->file)->i_mode); in need_complete_io()
676 struct kiocb *kiocb = &rw->kiocb; in io_rw_init_file()
677 struct io_ring_ctx *ctx = req->ctx; in io_rw_init_file()
678 struct file *file = req->file; in io_rw_init_file()
681 if (unlikely(!file || !(file->f_mode & mode))) in io_rw_init_file()
682 return -EBADF; in io_rw_init_file()
684 if (!(req->flags & REQ_F_FIXED_FILE)) in io_rw_init_file()
685 req->flags |= io_file_get_flags(file); in io_rw_init_file()
687 kiocb->ki_flags = file->f_iocb_flags; in io_rw_init_file()
688 ret = kiocb_set_rw_flags(kiocb, rw->flags); in io_rw_init_file()
691 kiocb->ki_flags |= IOCB_ALLOC_CACHE; in io_rw_init_file()
698 if (kiocb->ki_flags & IOCB_NOWAIT || in io_rw_init_file()
699 ((file->f_flags & O_NONBLOCK && !(req->flags & REQ_F_SUPPORT_NOWAIT)))) in io_rw_init_file()
700 req->flags |= REQ_F_NOWAIT; in io_rw_init_file()
702 if (ctx->flags & IORING_SETUP_IOPOLL) { in io_rw_init_file()
703 if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) in io_rw_init_file()
704 return -EOPNOTSUPP; in io_rw_init_file()
706 kiocb->private = NULL; in io_rw_init_file()
707 kiocb->ki_flags |= IOCB_HIPRI; in io_rw_init_file()
708 kiocb->ki_complete = io_complete_rw_iopoll; in io_rw_init_file()
709 req->iopoll_completed = 0; in io_rw_init_file()
711 if (kiocb->ki_flags & IOCB_HIPRI) in io_rw_init_file()
712 return -EINVAL; in io_rw_init_file()
713 kiocb->ki_complete = io_complete_rw; in io_rw_init_file()
724 struct kiocb *kiocb = &rw->kiocb; in __io_read()
726 struct io_async_rw *io; in __io_read() local
735 io = req->async_data; in __io_read()
736 s = &io->s; in __io_read()
739 * Safe and required to re-import if we're using provided in __io_read()
753 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
761 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
767 return ret ?: -EAGAIN; in __io_read()
769 kiocb->ki_flags |= IOCB_NOWAIT; in __io_read()
771 /* Ensure we clear previously set non-block flag */ in __io_read()
772 kiocb->ki_flags &= ~IOCB_NOWAIT; in __io_read()
777 ret = rw_verify_area(READ, req->file, ppos, req->cqe.res); in __io_read()
783 ret = io_iter_do_read(rw, &s->iter); in __io_read()
786 * Some file systems like to return -EOPNOTSUPP for an IOCB_NOWAIT in __io_read()
787 * issue, even though they should be returning -EAGAIN. To be safe, in __io_read()
788 * retry from blocking context for either. in __io_read()
790 if (ret == -EOPNOTSUPP && force_nonblock) in __io_read()
791 ret = -EAGAIN; in __io_read()
793 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { in __io_read()
794 req->flags &= ~REQ_F_REISSUE; in __io_read()
796 if (req->opcode == IORING_OP_READ && io_file_can_poll(req)) in __io_read()
797 return -EAGAIN; in __io_read()
798 /* IOPOLL retry should happen for io-wq threads */ in __io_read()
799 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) in __io_read()
802 if (req->flags & REQ_F_NOWAIT) in __io_read()
805 } else if (ret == -EIOCBQUEUED) { in __io_read()
806 req->flags |= REQ_F_PARTIAL_IO; in __io_read()
811 } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock || in __io_read()
812 (req->flags & REQ_F_NOWAIT) || !need_complete_io(req)) { in __io_read()
822 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
831 req->flags |= REQ_F_PARTIAL_IO; in __io_read()
834 io = req->async_data; in __io_read()
835 s = &io->s; in __io_read()
847 iov_iter_advance(&s->iter, ret); in __io_read()
848 if (!iov_iter_count(&s->iter)) in __io_read()
850 io->bytes_done += ret; in __io_read()
851 iov_iter_save_state(&s->iter, &s->iter_state); in __io_read()
855 kiocb->ki_flags &= ~IOCB_WAITQ; in __io_read()
856 return -EAGAIN; in __io_read()
859 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
862 * we get -EIOCBQUEUED, then we'll get a notification when the in __io_read()
866 ret = io_iter_do_read(rw, &s->iter); in __io_read()
867 if (ret == -EIOCBQUEUED) in __io_read()
870 kiocb->ki_flags &= ~IOCB_WAITQ; in __io_read()
871 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
896 if (!(req->flags & REQ_F_ISREG)) in io_kiocb_start_write()
898 if (!(kiocb->ki_flags & IOCB_NOWAIT)) { in io_kiocb_start_write()
903 inode = file_inode(kiocb->ki_filp); in io_kiocb_start_write()
904 ret = sb_start_write_trylock(inode->i_sb); in io_kiocb_start_write()
906 __sb_writers_release(inode->i_sb, SB_FREEZE_WRITE); in io_kiocb_start_write()
915 struct kiocb *kiocb = &rw->kiocb; in io_write()
925 struct io_async_rw *io = req->async_data; in io_write() local
927 s = &io->s; in io_write()
928 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
936 req->cqe.res = iov_iter_count(&s->iter); in io_write()
943 /* File path supports NOWAIT for non-direct_IO only for block devices. */ in io_write()
944 if (!(kiocb->ki_flags & IOCB_DIRECT) && in io_write()
945 !(kiocb->ki_filp->f_mode & FMODE_BUF_WASYNC) && in io_write()
946 (req->flags & REQ_F_ISREG)) in io_write()
949 kiocb->ki_flags |= IOCB_NOWAIT; in io_write()
951 /* Ensure we clear previously set non-block flag */ in io_write()
952 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_write()
957 ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res); in io_write()
964 return -EAGAIN; in io_write()
965 kiocb->ki_flags |= IOCB_WRITE; in io_write()
967 if (likely(req->file->f_op->write_iter)) in io_write()
968 ret2 = call_write_iter(req->file, kiocb, &s->iter); in io_write()
969 else if (req->file->f_op->write) in io_write()
970 ret2 = loop_rw_iter(WRITE, rw, &s->iter); in io_write()
972 ret2 = -EINVAL; in io_write()
974 if (ret2 == -EIOCBQUEUED) { in io_write()
975 req->flags |= REQ_F_PARTIAL_IO; in io_write()
979 if (req->flags & REQ_F_REISSUE) { in io_write()
980 req->flags &= ~REQ_F_REISSUE; in io_write()
981 ret2 = -EAGAIN; in io_write()
985 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just in io_write()
988 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) in io_write()
989 ret2 = -EAGAIN; in io_write()
991 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) in io_write()
993 if (!force_nonblock || ret2 != -EAGAIN) { in io_write()
994 /* IOPOLL retry should happen for io-wq threads */ in io_write()
995 if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) in io_write()
998 if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) { in io_write()
999 struct io_async_rw *io; in io_write() local
1001 trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2, in io_write()
1002 req->cqe.res, ret2); in io_write()
1009 iov_iter_save_state(&s->iter, &s->iter_state); in io_write()
1012 io = req->async_data; in io_write()
1013 if (io) in io_write()
1014 io->bytes_done += ret2; in io_write()
1016 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
1018 return ret ? ret : -EAGAIN; in io_write()
1024 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
1027 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
1029 return -EAGAIN; in io_write()
1043 res = io_fixup_rw_res(req, req->cqe.res); in io_rw_fail()
1044 io_req_set_res(req, res, req->cqe.flags); in io_rw_fail()
1058 if (ctx->poll_multi_queue || force_nonspin) in io_do_iopoll()
1061 wq_list_for_each(pos, start, &ctx->iopoll_list) { in io_do_iopoll()
1063 struct file *file = req->file; in io_do_iopoll()
1071 if (READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1074 if (req->opcode == IORING_OP_URING_CMD) { in io_do_iopoll()
1078 ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob, in io_do_iopoll()
1083 ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags); in io_do_iopoll()
1092 READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1105 /* order with io_complete_rw_iopoll(), e.g. ->result updates */ in io_do_iopoll()
1106 if (!smp_load_acquire(&req->iopoll_completed)) in io_do_iopoll()
1109 req->cqe.flags = io_put_kbuf(req, 0); in io_do_iopoll()
1114 pos = start ? start->next : ctx->iopoll_list.first; in io_do_iopoll()
1115 wq_list_cut(&ctx->iopoll_list, prev, start); in io_do_iopoll()
1117 if (WARN_ON_ONCE(!wq_list_empty(&ctx->submit_state.compl_reqs))) in io_do_iopoll()
1119 ctx->submit_state.compl_reqs.first = pos; in io_do_iopoll()