210 struct io_ring_ctx;
245 typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
248 	struct io_ring_ctx		*ctx;
330 struct io_ring_ctx {  struct
478 	const struct io_ring_ctx *last;
882 	struct io_ring_ctx		*ctx;
910 	struct io_ring_ctx	*ctx;
1091 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1102 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
1106 static struct file *io_file_get(struct io_ring_ctx *ctx,
1113 static void io_submit_flush_completions(struct io_ring_ctx *ctx);
1130 		struct io_ring_ctx *ctx = file->private_data;  in io_uring_get_socket()
1139 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)  in io_tw_lock()
1194 	struct io_ring_ctx *ctx = req->ctx;  in io_req_set_rsrc_node()
1258 		struct io_ring_ctx *ctx = head->ctx;  in io_match_task_safe()
1283 	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);  in io_ring_ctx_ref_free()
1295 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,  in io_fallback_req_func()
1314 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)  in io_ring_ctx_alloc()
1316 	struct io_ring_ctx *ctx;  in io_ring_ctx_alloc()
1380 static void io_account_cq_overflow(struct io_ring_ctx *ctx)  in io_account_cq_overflow()
1391 		struct io_ring_ctx *ctx = req->ctx;  in req_need_defer()
1445 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_work()
1471 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_link()
1485 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_async_work()
1532 static void io_queue_deferred(struct io_ring_ctx *ctx)  in io_queue_deferred()
1548 static void io_flush_timeouts(struct io_ring_ctx *ctx)  in io_flush_timeouts()
1579 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)  in __io_commit_cqring_flush()
1587 static inline bool io_commit_needs_flush(struct io_ring_ctx *ctx)  in io_commit_needs_flush()
1592 static inline void __io_commit_cqring(struct io_ring_ctx *ctx)  in __io_commit_cqring()
1598 static inline void io_commit_cqring(struct io_ring_ctx *ctx)  in io_commit_cqring()
1605 static inline bool io_sqring_full(struct io_ring_ctx *ctx)  in io_sqring_full()
1612 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)  in __io_cqring_events()
1617 static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)  in io_get_cqe()
1634 static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)  in io_should_trigger_evfd()
1650 static void io_cqring_ev_posted(struct io_ring_ctx *ctx)  in io_cqring_ev_posted()
1669 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)  in io_cqring_ev_posted_iopoll()
1687 static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)  in __io_cqring_overflow_flush()
1729 static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)  in io_cqring_overflow_flush()
1790 static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,  in io_cqring_event_overflow()
1818 static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data,  in __io_fill_cqe()
1845 static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,  in io_fill_cqe_aux()
1855 	struct io_ring_ctx *ctx = req->ctx;  in io_req_complete_post()
1939 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_preinit_req()
1948 static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,  in io_flush_cached_locked_reqs()
1958 static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)  in io_flush_cached_reqs()
1992 static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)  in io_alloc_req()
2050 	struct io_ring_ctx *ctx = req->ctx;  in __io_free_req()
2130 		struct io_ring_ctx *ctx = req->ctx;  in io_disarm_next()
2155 		struct io_ring_ctx *ctx = req->ctx;  in __io_req_find_next()
2178 static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)  in ctx_flush_and_put()
2194 	struct io_ring_ctx *ctx = NULL;  in tctx_task_work()
2293 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_cancel()
2302 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_submit()
2363 static void io_req_free_batch_finish(struct io_ring_ctx *ctx,  in io_req_free_batch_finish()
2393 static void io_submit_flush_completions(struct io_ring_ctx *ctx)  in io_submit_flush_completions()
2452 static unsigned io_cqring_events(struct io_ring_ctx *ctx)  in io_cqring_events()
2459 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)  in io_sqring_entries()
2511 static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_iopoll_complete()
2556 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_do_iopoll()
2606 static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)  in io_iopoll_try_reap_events()
2634 static int io_iopoll_check(struct io_ring_ctx *ctx, long min)  in io_iopoll_check()
2717 	struct io_ring_ctx *ctx = req->ctx;  in io_rw_should_reissue()
2805 		struct io_ring_ctx *ctx = req->ctx;  in io_req_task_complete()
2861 	struct io_ring_ctx *ctx = req->ctx;  in io_iopoll_req_issued()
2972 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
3027 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
3108 			struct io_ring_ctx *ctx = req->ctx;  in kiocb_done()
3188 static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_unlock()
3194 static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_lock()
4094 	struct io_ring_ctx *ctx = req->ctx;  in io_nop()
4105 	struct io_ring_ctx *ctx = req->ctx;  in io_fsync_prep()
4327 static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,  in __io_remove_buffers()
4357 	struct io_ring_ctx *ctx = req->ctx;  in io_remove_buffers()
4445 	struct io_ring_ctx *ctx = req->ctx;  in io_provide_buffers()
4696 	struct io_ring_ctx *ctx = req->ctx;  in io_sfr_prep()
5423 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_req_insert()
5490 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_check_events()
5562 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_task_func()
5585 	struct io_ring_ctx *ctx = req->ctx;  in io_apoll_task_func()
5736 	struct io_ring_ctx *ctx = req->ctx;  in __io_arm_poll_handler()
5816 	struct io_ring_ctx *ctx = req->ctx;  in io_arm_poll_handler()
5868 static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_poll_remove_all()
5893 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_find()
5921 static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_cancel()
6017 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_update()
6069 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_fn()
6083 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,  in io_timeout_extract()
6106 static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)  in io_timeout_cancel()
6137 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_linked_timeout_update()
6162 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_timeout_update()
6225 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_remove()
6306 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout()
6355 	struct io_ring_ctx *ctx;
6368 			       struct io_ring_ctx *ctx)  in io_async_cancel_one()
6395 	struct io_ring_ctx *ctx = req->ctx;  in io_try_cancel_userdata()
6433 	struct io_ring_ctx *ctx = req->ctx;  in io_async_cancel()
6478 	struct io_ring_ctx *ctx = req->ctx;  in io_files_update()
6617 	struct io_ring_ctx *ctx = req->ctx;  in io_drain_req()
6757 	struct io_ring_ctx *ctx = req->ctx;  in io_issue_sqe()
6943 static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,  in io_file_from_index()
6964 static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,  in io_file_get_fixed()
6987 static struct file *io_file_get_normal(struct io_ring_ctx *ctx,  in io_file_get_normal()
7000 static inline struct file *io_file_get(struct io_ring_ctx *ctx,  in io_file_get()
7030 	struct io_ring_ctx *ctx = req->ctx;  in io_link_timeout_fn()
7057 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_linked_timeout()
7092 			struct io_ring_ctx *ctx = req->ctx;  in __io_queue_sqe()
7153 static inline bool io_check_restriction(struct io_ring_ctx *ctx,  in io_check_restriction()
7174 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_init_req()
7237 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_submit_sqe()
7323 				struct io_ring_ctx *ctx)  in io_submit_state_end()
7345 static void io_commit_sqring(struct io_ring_ctx *ctx)  in io_commit_sqring()
7365 static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)  in io_get_sqe()
7389 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)  in io_submit_sqes()
7442 static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_set_wakeup_flag()
7451 static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_clear_wakeup_flag()
7459 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)  in __io_sq_thread()
7500 	struct io_ring_ctx *ctx;  in io_sqd_update_thread_idle()
7527 	struct io_ring_ctx *ctx;  in io_sq_thread()
7612 	struct io_ring_ctx *ctx;
7619 	struct io_ring_ctx *ctx = iowq->ctx;  in io_should_wake()
7666 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,  in io_cqring_wait_schedule()
7699 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,  in io_cqring_wait()
7804 	struct io_ring_ctx *ctx = node->rsrc_data->ctx;  in io_rsrc_node_ref_zero()
7831 static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)  in io_rsrc_node_alloc()
7850 static void io_rsrc_node_switch(struct io_ring_ctx *ctx,  in io_rsrc_node_switch()
7875 static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)  in io_rsrc_node_switch_start()
7883 static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)  in io_rsrc_ref_quiesce()
7947 static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,  in io_rsrc_data_alloc()
8000 static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)  in __io_sqe_files_unregister()
8027 static int io_sqe_files_unregister(struct io_ring_ctx *ctx)  in io_sqe_files_unregister()
8097 static void io_sq_thread_finish(struct io_ring_ctx *ctx)  in io_sq_thread_finish()
8114 	struct io_ring_ctx *ctx_attach;  in io_attach_sq_data()
8178 static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)  in __io_sqe_files_scm()
8238 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8269 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8275 static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_file_put()
8341 	struct io_ring_ctx *ctx = rsrc_data->ctx;  in __io_rsrc_put_work()
8370 	struct io_ring_ctx *ctx;  in io_rsrc_put_work()
8373 	ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);  in io_rsrc_put_work()
8386 static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_files_register()
8488 	struct io_ring_ctx *ctx = req->ctx;  in io_install_fixed_file()
8538 	struct io_ring_ctx *ctx = req->ctx;  in io_close_fixed()
8573 static int __io_sqe_files_update(struct io_ring_ctx *ctx,  in __io_sqe_files_update()
8646 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,  in io_init_wq_offload()
8679 				       struct io_ring_ctx *ctx)  in io_uring_alloc_task_context()
8726 static int io_sq_offload_create(struct io_ring_ctx *ctx,  in io_sq_offload_create()
8837 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_unaccount_mem()
8846 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_account_mem()
8910 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)  in io_buffer_unmap()
8925 static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_buf_put()
8931 static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in __io_sqe_buffers_unregister()
8944 static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in io_sqe_buffers_unregister()
8964 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,  in io_copy_iov()
8998 static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,  in headpage_already_acct()
9026 static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,  in io_buffer_account_pin()
9058 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,  in io_sqe_buffer_register()
9161 static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)  in io_buffers_map_alloc()
9191 static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_buffers_register()
9243 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,  in __io_sqe_buffers_update()
9303 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)  in io_eventfd_register()
9325 static int io_eventfd_unregister(struct io_ring_ctx *ctx)  in io_eventfd_unregister()
9336 static void io_destroy_buffers(struct io_ring_ctx *ctx)  in io_destroy_buffers()
9355 static void io_req_caches_free(struct io_ring_ctx *ctx)  in io_req_caches_free()
9377 static void io_ring_ctx_free(struct io_ring_ctx *ctx)  in io_ring_ctx_free()
9436 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_poll()
9467 static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)  in io_unregister_personality()
9483 	struct io_ring_ctx		*ctx;
9512 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);  in io_ring_exit_work()
9595 static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_kill_timeouts()
9618 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)  in io_ring_ctx_wait_and_kill()
9653 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_release()
9673 static bool io_cancel_defer_files(struct io_ring_ctx *ctx,  in io_cancel_defer_files()
9699 static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)  in io_uring_try_cancel_iowq()
9723 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,  in io_uring_try_cancel_requests()
9767 static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in __io_uring_add_tctx_node()
9813 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in io_uring_add_tctx_node()
9882 	struct io_ring_ctx *ctx;  in io_uring_cancel_generic()
9952 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_validate_mmap_request()
10019 static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)  in io_sqpoll_wait_sq()
10073 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE6()
10197 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)  in __io_uring_show_fdinfo()
10266 	struct io_ring_ctx *ctx = f->private_data;  in io_uring_show_fdinfo()
10288 static int io_allocate_scq_urings(struct io_ring_ctx *ctx,  in io_allocate_scq_urings()
10330 static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)  in io_uring_install_fd()
10353 static struct file *io_uring_get_file(struct io_ring_ctx *ctx)  in io_uring_get_file()
10381 	struct io_ring_ctx *ctx;  in io_uring_create()
10537 static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)  in io_probe()
10576 static int io_register_personality(struct io_ring_ctx *ctx)  in io_register_personality()
10593 static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,  in io_register_restrictions()
10663 static int io_register_enable_rings(struct io_ring_ctx *ctx)  in io_register_enable_rings()
10677 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,  in __io_register_rsrc_update()
10699 static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_files_update()
10714 static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc_update()
10728 static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc()
10754 static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,  in io_register_iowq_aff()
10793 static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)  in io_unregister_iowq_aff()
10803 static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,  in io_register_iowq_max_workers()
10911 static int io_ctx_quiesce(struct io_ring_ctx *ctx)  in io_ctx_quiesce()
10938 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,  in __io_uring_register()
11081 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE4()