509 	struct io_kiocb			*req;  member
905 	struct io_kiocb		*req;  member
1147 #define req_ref_zero_or_close_to_overflow(req)	\  argument
1150 static inline bool req_ref_inc_not_zero(struct io_kiocb *req)  in req_ref_inc_not_zero()
1156 static inline bool req_ref_put_and_test(struct io_kiocb *req)  in req_ref_put_and_test()
1165 static inline void req_ref_get(struct io_kiocb *req)  in req_ref_get()
1172 static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)  in __io_req_set_refcount()
1180 static inline void io_req_set_refcount(struct io_kiocb *req)  in io_req_set_refcount()
1185 static inline void io_req_set_rsrc_node(struct io_kiocb *req)  in io_req_set_rsrc_node()
1211 	struct io_kiocb *req;  in io_match_task()  local
1227 	struct io_kiocb *req;  in io_match_linked()  local
1263 static inline void req_set_fail(struct io_kiocb *req)  in req_set_fail()
1268 static inline void req_fail_link_node(struct io_kiocb *req, int res)  in req_fail_link_node()
1281 static inline bool io_is_timeout_noseq(struct io_kiocb *req)  in io_is_timeout_noseq()
1291 	struct io_kiocb *req, *tmp;  in io_fallback_req_func()  local
1381 static bool req_need_defer(struct io_kiocb *req, u32 seq)  in req_need_defer()
1401 static inline bool io_req_ffs_set(struct io_kiocb *req)  in io_req_ffs_set()
1406 static void io_req_track_inflight(struct io_kiocb *req)  in io_req_track_inflight()
1414 static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)  in __io_prep_linked_timeout()
1428 static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)  in io_prep_linked_timeout()
1435 static void io_prep_async_work(struct io_kiocb *req)  in io_prep_async_work()
1459 static void io_prep_async_link(struct io_kiocb *req)  in io_prep_async_link()
1476 static void io_queue_async_work(struct io_kiocb *req, bool *locked)  in io_queue_async_work()
1508 static void io_kill_timeout(struct io_kiocb *req, int status)  in io_kill_timeout()
1545 	struct io_kiocb *req, *tmp;  in io_flush_timeouts()  local
1833 static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)  in io_fill_cqe_req()
1845 static void io_req_complete_post(struct io_kiocb *req, s32 res,  in io_req_complete_post()
1882 static inline bool io_req_needs_clean(struct io_kiocb *req)  in io_req_needs_clean()
1887 static inline void io_req_complete_state(struct io_kiocb *req, s32 res,  in io_req_complete_state()
1897 static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,  in __io_req_complete()
1906 static inline void io_req_complete(struct io_kiocb *req, s32 res)  in io_req_complete()
1911 static void io_req_complete_failed(struct io_kiocb *req, s32 res)  in io_req_complete_failed()
1917 static void io_req_complete_fail_submit(struct io_kiocb *req)  in io_req_complete_fail_submit()
1932 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_preinit_req()
1966 		struct io_kiocb *req = list_first_entry(&state->free_list,  in io_flush_cached_reqs()  local
2025 static void io_dismantle_req(struct io_kiocb *req)  in io_dismantle_req()
2041 static void __io_free_req(struct io_kiocb *req)  in __io_free_req()
2056 static inline void io_remove_next_linked(struct io_kiocb *req)  in io_remove_next_linked()
2064 static bool io_kill_linked_timeout(struct io_kiocb *req)  in io_kill_linked_timeout()
2085 static void io_fail_links(struct io_kiocb *req)  in io_fail_links()
2107 static bool io_disarm_next(struct io_kiocb *req)  in io_disarm_next()
2137 static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)  in __io_req_find_next()
2164 static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)  in io_req_find_next()
2208 			struct io_kiocb *req = container_of(node, struct io_kiocb,  in tctx_task_work()  local
2235 static void io_req_task_work_add(struct io_kiocb *req)  in io_req_task_work_add()
2284 static void io_req_task_cancel(struct io_kiocb *req, bool *locked)  in io_req_task_cancel()
2293 static void io_req_task_submit(struct io_kiocb *req, bool *locked)  in io_req_task_submit()
2305 static void io_req_task_queue_fail(struct io_kiocb *req, int ret)  in io_req_task_queue_fail()
2312 static void io_req_task_queue(struct io_kiocb *req)  in io_req_task_queue()
2318 static void io_req_task_queue_reissue(struct io_kiocb *req)  in io_req_task_queue_reissue()
2324 static inline void io_queue_next(struct io_kiocb *req)  in io_queue_next()
2332 static void io_free_req(struct io_kiocb *req)  in io_free_req()
2338 static void io_free_req_work(struct io_kiocb *req, bool *locked)  in io_free_req_work()
2365 static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,  in io_req_free_batch()
2395 		struct io_kiocb *req = state->compl_reqs[i];  in io_submit_flush_completions()  local
2406 		struct io_kiocb *req = state->compl_reqs[i];  in io_submit_flush_completions()  local
2420 static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)  in io_put_req_find_next()
2431 static inline void io_put_req(struct io_kiocb *req)  in io_put_req()
2437 static inline void io_put_req_deferred(struct io_kiocb *req)  in io_put_req_deferred()
2460 static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)  in io_put_kbuf()
2471 static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)  in io_put_rw_kbuf()
2508 	struct io_kiocb *req;  in io_iopoll_complete()  local
2552 	struct io_kiocb *req, *tmp;  in io_do_iopoll()  local
2684 static void kiocb_end_write(struct io_kiocb *req)  in kiocb_end_write()
2699 static bool io_resubmit_prep(struct io_kiocb *req)  in io_resubmit_prep()
2709 static bool io_rw_should_reissue(struct io_kiocb *req)  in io_rw_should_reissue()
2735 static bool io_resubmit_prep(struct io_kiocb *req)  in io_resubmit_prep()
2739 static bool io_rw_should_reissue(struct io_kiocb *req)  in io_rw_should_reissue()
2749 static void io_req_io_end(struct io_kiocb *req)  in io_req_io_end()
2761 static bool __io_complete_rw_common(struct io_kiocb *req, long res)  in __io_complete_rw_common()
2780 static inline int io_fixup_rw_res(struct io_kiocb *req, long res)  in io_fixup_rw_res()
2794 static void io_req_task_complete(struct io_kiocb *req, bool *locked)  in io_req_task_complete()
2812 static void io_req_rw_complete(struct io_kiocb *req, bool *locked)  in io_req_rw_complete()
2820 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw()  local
2831 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw_iopoll()  local
2854 static void io_iopoll_req_issued(struct io_kiocb *req)  in io_iopoll_req_issued()
2954 static bool io_file_supports_nowait(struct io_kiocb *req, int rw)  in io_file_supports_nowait()
2964 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,  in io_prep_rw()
3058 static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req)  in io_kiocb_update_pos()
3078 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in kiocb_done()  local
3118 static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,  in __io_import_fixed()
3176 static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)  in io_import_fixed()
3201 static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,  in io_buffer_select()
3235 static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,  in io_rw_buffer_select()
3252 static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,  in io_compat_import()
3278 static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,  in __io_iov_buffer_select()
3299 static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,  in io_iov_buffer_select()
3321 static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,  in io_import_iovec()
3372 static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)  in loop_rw_iter()
3431 static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,  in io_req_map_rw()
3458 static inline int io_alloc_async_data(struct io_kiocb *req)  in io_alloc_async_data()
3465 static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,  in io_setup_async_rw()
3487 static inline int io_rw_prep_async(struct io_kiocb *req, int rw)  in io_rw_prep_async()
3508 static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_read_prep()
3529 	struct io_kiocb *req = wait->private;  in io_async_buf_func()  local
3555 static bool io_rw_should_retry(struct io_kiocb *req)  in io_rw_should_retry()
3586 static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)  in io_iter_do_read()
3596 static bool need_read_all(struct io_kiocb *req)  in need_read_all()
3602 static int io_read(struct io_kiocb *req, unsigned int issue_flags)  in io_read()
3734 static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_write_prep()
3741 static int io_write(struct io_kiocb *req, unsigned int issue_flags)  in io_write()
3846 static int io_renameat_prep(struct io_kiocb *req,  in io_renameat_prep()
3879 static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)  in io_renameat()
3897 static int io_unlinkat_prep(struct io_kiocb *req,  in io_unlinkat_prep()
3926 static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)  in io_unlinkat()
3946 static int io_mkdirat_prep(struct io_kiocb *req,  in io_mkdirat_prep()
3972 static int io_mkdirat(struct io_kiocb *req, int issue_flags)  in io_mkdirat()
3989 static int io_symlinkat_prep(struct io_kiocb *req,  in io_symlinkat_prep()
4021 static int io_symlinkat(struct io_kiocb *req, int issue_flags)  in io_symlinkat()
4038 static int io_linkat_prep(struct io_kiocb *req,  in io_linkat_prep()
4071 static int io_linkat(struct io_kiocb *req, int issue_flags)  in io_linkat()
4089 static int io_shutdown_prep(struct io_kiocb *req,  in io_shutdown_prep()
4106 static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)  in io_shutdown()
4129 static int __io_splice_prep(struct io_kiocb *req,  in __io_splice_prep()
4146 static int io_tee_prep(struct io_kiocb *req,  in io_tee_prep()
4154 static int io_tee(struct io_kiocb *req, unsigned int issue_flags)  in io_tee()
4184 static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_splice_prep()
4193 static int io_splice(struct io_kiocb *req, unsigned int issue_flags)  in io_splice()
4230 static int io_nop(struct io_kiocb *req, unsigned int issue_flags)  in io_nop()
4241 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_fsync_prep()
4260 static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)  in io_fsync()
4278 static int io_fallocate_prep(struct io_kiocb *req,  in io_fallocate_prep()
4293 static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)  in io_fallocate()
4310 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in __io_openat_prep()
4344 static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_openat_prep()
4353 static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_openat2_prep()
4372 static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)  in io_openat2()
4440 static int io_openat(struct io_kiocb *req, unsigned int issue_flags)  in io_openat()
4445 static int io_remove_buffers_prep(struct io_kiocb *req,  in io_remove_buffers_prep()
4492 static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)  in io_remove_buffers()
4517 static int io_provide_buffers_prep(struct io_kiocb *req,  in io_provide_buffers_prep()
4580 static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)  in io_provide_buffers()
4609 static int io_epoll_ctl_prep(struct io_kiocb *req,  in io_epoll_ctl_prep()
4636 static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)  in io_epoll_ctl()
4656 static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_madvise_prep()
4673 static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)  in io_madvise()
4692 static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_fadvise_prep()
4705 static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)  in io_fadvise()
4728 static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_statx_prep()
4746 static int io_statx(struct io_kiocb *req, unsigned int issue_flags)  in io_statx()
4763 static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_close_prep()
4781 static int io_close(struct io_kiocb *req, unsigned int issue_flags)  in io_close()
4832 static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_sfr_prep()
4848 static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)  in io_sync_file_range()
4872 static int io_setup_async_msg(struct io_kiocb *req,  in io_setup_async_msg()
4897 static int io_sendmsg_copy_hdr(struct io_kiocb *req,  in io_sendmsg_copy_hdr()
4912 static int io_sendmsg_prep_async(struct io_kiocb *req)  in io_sendmsg_prep_async()
4922 static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_sendmsg_prep()
4947 static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)  in io_sendmsg()
5004 static int io_send(struct io_kiocb *req, unsigned int issue_flags)  in io_send()
5057 static int __io_recvmsg_copy_hdr(struct io_kiocb *req,  in __io_recvmsg_copy_hdr()
5090 static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,  in __io_compat_recvmsg_copy_hdr()
5131 static int io_recvmsg_copy_hdr(struct io_kiocb *req,  in io_recvmsg_copy_hdr()
5144 static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,  in io_recv_buffer_select()
5159 static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)  in io_put_recv_kbuf()
5164 static int io_recvmsg_prep_async(struct io_kiocb *req)  in io_recvmsg_prep_async()
5174 static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_recvmsg_prep()
5200 static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)  in io_recvmsg()
5270 static int io_recv(struct io_kiocb *req, unsigned int issue_flags)  in io_recv()
5339 static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_accept_prep()
5363 static int io_accept(struct io_kiocb *req, unsigned int issue_flags)  in io_accept()
5401 static int io_connect_prep_async(struct io_kiocb *req)  in io_connect_prep_async()
5409 static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_connect_prep()
5424 static int io_connect(struct io_kiocb *req, unsigned int issue_flags)  in io_connect()
5495 	struct io_kiocb *req;  member
5510 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)  in io_poll_get_ownership_slowpath()
5531 static inline bool io_poll_get_ownership(struct io_kiocb *req)  in io_poll_get_ownership()
5538 static void io_poll_mark_cancelled(struct io_kiocb *req)  in io_poll_mark_cancelled()
5543 static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)  in io_poll_get_double()
5551 static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)  in io_poll_get_single()
5558 static void io_poll_req_insert(struct io_kiocb *req)  in io_poll_req_insert()
5590 static void io_poll_remove_entries(struct io_kiocb *req)  in io_poll_remove_entries()
5625 static int io_poll_check_events(struct io_kiocb *req)  in io_poll_check_events()
5697 static void io_poll_task_func(struct io_kiocb *req, bool *locked)  in io_poll_task_func()
5720 static void io_apoll_task_func(struct io_kiocb *req, bool *locked)  in io_apoll_task_func()
5741 static void __io_poll_execute(struct io_kiocb *req, int mask)  in __io_poll_execute()
5753 static inline void io_poll_execute(struct io_kiocb *req, int res)  in io_poll_execute()
5759 static void io_poll_cancel_req(struct io_kiocb *req)  in io_poll_cancel_req()
5769 	struct io_kiocb *req = wait->private;  in io_poll_wake()  local
5820 	struct io_kiocb *req = pt->req;  in __io_queue_proc()  local
5869 static int __io_arm_poll_handler(struct io_kiocb *req,  in __io_arm_poll_handler()
5950 static int io_arm_poll_handler(struct io_kiocb *req)  in io_arm_poll_handler()
6009 	struct io_kiocb *req;  in io_poll_remove_all()  local
6035 	struct io_kiocb *req;  in io_poll_find()  local
6048 static bool io_poll_disarm(struct io_kiocb *req)  in io_poll_disarm()
6062 	struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only);  in io_poll_cancel()  local
6084 static int io_poll_update_prep(struct io_kiocb *req,  in io_poll_update_prep()
6117 static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_poll_add_prep()
6135 static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_add()
6152 static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_update()
6195 static void io_req_task_timeout(struct io_kiocb *req, bool *locked)  in io_req_task_timeout()
6205 	struct io_kiocb *req = data->req;  in io_timeout_fn()  local
6225 	struct io_kiocb *req;  in io_timeout_extract()  local
6247 	struct io_kiocb *req = io_timeout_extract(ctx, user_data);  in io_timeout_cancel()  local
6279 	struct io_kiocb *req;  in io_linked_timeout_update()  local
6303 	struct io_kiocb *req = io_timeout_extract(ctx, user_data);  in io_timeout_update()  local
6318 static int io_timeout_remove_prep(struct io_kiocb *req,  in io_timeout_remove_prep()
6359 static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout_remove()
6388 static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,  in io_timeout_prep()
6441 static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout()
6498 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_cancel_cb()  local
6530 static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)  in io_try_cancel_userdata()
6553 static int io_async_cancel_prep(struct io_kiocb *req,  in io_async_cancel_prep()
6568 static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)  in io_async_cancel()
6597 static int io_rsrc_update_prep(struct io_kiocb *req,  in io_rsrc_update_prep()
6613 static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)  in io_files_update()
6637 static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_req_prep()
6721 static int io_req_prep_async(struct io_kiocb *req)  in io_req_prep_async()
6747 static u32 io_get_sequence(struct io_kiocb *req)  in io_get_sequence()
6757 static bool io_drain_req(struct io_kiocb *req)  in io_drain_req()
6832 static void io_clean_op(struct io_kiocb *req)  in io_clean_op()
6909 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)  in io_issue_sqe()
7046 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_wq_free_work()  local
7054 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_wq_submit_work()  local
7128 					     struct io_kiocb *req, int fd,  in io_file_get_fixed()
7151 				       struct io_kiocb *req, int fd)  in io_file_get_normal()
7164 				       struct io_kiocb *req, int fd, bool fixed,  in io_file_get()
7173 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)  in io_req_task_link_timeout()
7192 	struct io_kiocb *prev, *req = data->req;  in io_link_timeout_fn()  local
7218 static void io_queue_linked_timeout(struct io_kiocb *req)  in io_queue_linked_timeout()
7240 static void __io_queue_sqe(struct io_kiocb *req)  in __io_queue_sqe()
7291 static inline void io_queue_sqe(struct io_kiocb *req)  in io_queue_sqe()
7317 					struct io_kiocb *req,  in io_check_restriction()
7337 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_init_req()
7400 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_submit_sqe()
7568 		struct io_kiocb *req;  in io_submit_sqes()  local
8650 static int io_install_fixed_file(struct io_kiocb *req, struct file *file,  in io_install_fixed_file()
8700 static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)  in io_close_fixed()
9512 	struct io_kiocb *req, *nxt;  in io_req_cache_free()  local
9670 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_cancel_ctx_cb()  local
9758 	struct io_kiocb *req, *tmp;  in io_kill_timeouts()  local
9827 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_cancel_task_cb()  local
10413 		struct io_kiocb *req;  in __io_uring_show_fdinfo()  local