210 struct io_ring_ctx;
245 typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
248 	struct io_ring_ctx		*ctx;
330 struct io_ring_ctx {  struct
469 	const struct io_ring_ctx *last;
872 	struct io_ring_ctx		*ctx;
900 	struct io_ring_ctx	*ctx;
1084 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1095 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
1099 static struct file *io_file_get(struct io_ring_ctx *ctx,
1106 static void io_submit_flush_completions(struct io_ring_ctx *ctx);
1123 		struct io_ring_ctx *ctx = file->private_data;  in io_uring_get_socket()
1132 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)  in io_tw_lock()
1187 	struct io_ring_ctx *ctx = req->ctx;  in io_req_set_rsrc_node()
1251 		struct io_ring_ctx *ctx = head->ctx;  in io_match_task_safe()
1276 	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);  in io_ring_ctx_ref_free()
1288 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,  in io_fallback_req_func()
1307 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)  in io_ring_ctx_alloc()
1309 	struct io_ring_ctx *ctx;  in io_ring_ctx_alloc()
1373 static void io_account_cq_overflow(struct io_ring_ctx *ctx)  in io_account_cq_overflow()
1384 		struct io_ring_ctx *ctx = req->ctx;  in req_need_defer()
1438 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_work()
1464 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_async_link()
1478 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_async_work()
1525 static void io_queue_deferred(struct io_ring_ctx *ctx)  in io_queue_deferred()
1541 static void io_flush_timeouts(struct io_ring_ctx *ctx)  in io_flush_timeouts()
1572 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)  in __io_commit_cqring_flush()
1580 static inline bool io_commit_needs_flush(struct io_ring_ctx *ctx)  in io_commit_needs_flush()
1585 static inline void __io_commit_cqring(struct io_ring_ctx *ctx)  in __io_commit_cqring()
1591 static inline void io_commit_cqring(struct io_ring_ctx *ctx)  in io_commit_cqring()
1598 static inline bool io_sqring_full(struct io_ring_ctx *ctx)  in io_sqring_full()
1605 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)  in __io_cqring_events()
1610 static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)  in io_get_cqe()
1627 static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)  in io_should_trigger_evfd()
1643 static void io_cqring_ev_posted(struct io_ring_ctx *ctx)  in io_cqring_ev_posted()
1662 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)  in io_cqring_ev_posted_iopoll()
1680 static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)  in __io_cqring_overflow_flush()
1722 static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)  in io_cqring_overflow_flush()
1783 static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,  in io_cqring_event_overflow()
1811 static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data,  in __io_fill_cqe()
1838 static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,  in io_fill_cqe_aux()
1848 	struct io_ring_ctx *ctx = req->ctx;  in io_req_complete_post()
1932 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_preinit_req()
1941 static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,  in io_flush_cached_locked_reqs()
1951 static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)  in io_flush_cached_reqs()
1985 static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)  in io_alloc_req()
2043 	struct io_ring_ctx *ctx = req->ctx;  in __io_free_req()
2123 		struct io_ring_ctx *ctx = req->ctx;  in io_disarm_next()
2148 		struct io_ring_ctx *ctx = req->ctx;  in __io_req_find_next()
2171 static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)  in ctx_flush_and_put()
2187 	struct io_ring_ctx *ctx = NULL;  in tctx_task_work()
2286 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_cancel()
2295 	struct io_ring_ctx *ctx = req->ctx;  in io_req_task_submit()
2356 static void io_req_free_batch_finish(struct io_ring_ctx *ctx,  in io_req_free_batch_finish()
2386 static void io_submit_flush_completions(struct io_ring_ctx *ctx)  in io_submit_flush_completions()
2445 static unsigned io_cqring_events(struct io_ring_ctx *ctx)  in io_cqring_events()
2452 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)  in io_sqring_entries()
2504 static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_iopoll_complete()
2549 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,  in io_do_iopoll()
2599 static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)  in io_iopoll_try_reap_events()
2629 static int io_iopoll_check(struct io_ring_ctx *ctx, long min)  in io_iopoll_check()
2712 	struct io_ring_ctx *ctx = req->ctx;  in io_rw_should_reissue()
2800 		struct io_ring_ctx *ctx = req->ctx;  in io_req_task_complete()
2856 	struct io_ring_ctx *ctx = req->ctx;  in io_iopoll_req_issued()
2967 	struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
3022 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
3103 			struct io_ring_ctx *ctx = req->ctx;  in kiocb_done()
3183 static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_unlock()
3189 static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)  in io_ring_submit_lock()
4232 	struct io_ring_ctx *ctx = req->ctx;  in io_nop()
4243 	struct io_ring_ctx *ctx = req->ctx;  in io_fsync_prep()
4465 static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,  in __io_remove_buffers()
4495 	struct io_ring_ctx *ctx = req->ctx;  in io_remove_buffers()
4583 	struct io_ring_ctx *ctx = req->ctx;  in io_provide_buffers()
4834 	struct io_ring_ctx *ctx = req->ctx;  in io_sfr_prep()
5560 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_req_insert()
5627 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_check_events()
5699 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_task_func()
5722 	struct io_ring_ctx *ctx = req->ctx;  in io_apoll_task_func()
5873 	struct io_ring_ctx *ctx = req->ctx;  in __io_arm_poll_handler()
5953 	struct io_ring_ctx *ctx = req->ctx;  in io_arm_poll_handler()
6005 static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_poll_remove_all()
6030 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_find()
6058 static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,  in io_poll_cancel()
6154 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_update()
6206 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_fn()
6220 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,  in io_timeout_extract()
6243 static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)  in io_timeout_cancel()
6274 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_linked_timeout_update()
6299 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,  in io_timeout_update()
6362 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_remove()
6443 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout()
6492 	struct io_ring_ctx *ctx;
6505 			       struct io_ring_ctx *ctx)  in io_async_cancel_one()
6532 	struct io_ring_ctx *ctx = req->ctx;  in io_try_cancel_userdata()
6570 	struct io_ring_ctx *ctx = req->ctx;  in io_async_cancel()
6615 	struct io_ring_ctx *ctx = req->ctx;  in io_files_update()
6760 	struct io_ring_ctx *ctx = req->ctx;  in io_drain_req()
6911 	struct io_ring_ctx *ctx = req->ctx;  in io_issue_sqe()
7106 static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,  in io_file_from_index()
7127 static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,  in io_file_get_fixed()
7150 static struct file *io_file_get_normal(struct io_ring_ctx *ctx,  in io_file_get_normal()
7163 static inline struct file *io_file_get(struct io_ring_ctx *ctx,  in io_file_get()
7193 	struct io_ring_ctx *ctx = req->ctx;  in io_link_timeout_fn()
7220 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_linked_timeout()
7255 			struct io_ring_ctx *ctx = req->ctx;  in __io_queue_sqe()
7316 static inline bool io_check_restriction(struct io_ring_ctx *ctx,  in io_check_restriction()
7337 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_init_req()
7400 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_submit_sqe()
7486 				struct io_ring_ctx *ctx)  in io_submit_state_end()
7508 static void io_commit_sqring(struct io_ring_ctx *ctx)  in io_commit_sqring()
7528 static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)  in io_get_sqe()
7554 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)  in io_submit_sqes()
7607 static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_set_wakeup_flag()
7616 static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)  in io_ring_clear_wakeup_flag()
7624 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)  in __io_sq_thread()
7665 	struct io_ring_ctx *ctx;  in io_sqd_update_thread_idle()
7692 	struct io_ring_ctx *ctx;  in io_sq_thread()
7777 	struct io_ring_ctx *ctx;
7784 	struct io_ring_ctx *ctx = iowq->ctx;  in io_should_wake()
7831 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,  in io_cqring_wait_schedule()
7864 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,  in io_cqring_wait()
7969 	struct io_ring_ctx *ctx = node->rsrc_data->ctx;  in io_rsrc_node_ref_zero()
7996 static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)  in io_rsrc_node_alloc()
8015 static void io_rsrc_node_switch(struct io_ring_ctx *ctx,  in io_rsrc_node_switch()
8040 static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)  in io_rsrc_node_switch_start()
8048 static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)  in io_rsrc_ref_quiesce()
8112 static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,  in io_rsrc_data_alloc()
8165 static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)  in __io_sqe_files_unregister()
8192 static int io_sqe_files_unregister(struct io_ring_ctx *ctx)  in io_sqe_files_unregister()
8262 static void io_sq_thread_finish(struct io_ring_ctx *ctx)  in io_sq_thread_finish()
8279 	struct io_ring_ctx *ctx_attach;  in io_attach_sq_data()
8343 static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)  in __io_sqe_files_scm()
8403 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8434 static int io_sqe_files_scm(struct io_ring_ctx *ctx)  in io_sqe_files_scm()
8440 static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_file_put()
8506 	struct io_ring_ctx *ctx = rsrc_data->ctx;  in __io_rsrc_put_work()
8535 	struct io_ring_ctx *ctx;  in io_rsrc_put_work()
8538 	ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);  in io_rsrc_put_work()
8551 static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_files_register()
8653 	struct io_ring_ctx *ctx = req->ctx;  in io_install_fixed_file()
8703 	struct io_ring_ctx *ctx = req->ctx;  in io_close_fixed()
8738 static int __io_sqe_files_update(struct io_ring_ctx *ctx,  in __io_sqe_files_update()
8811 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,  in io_init_wq_offload()
8844 				       struct io_ring_ctx *ctx)  in io_uring_alloc_task_context()
8891 static int io_sq_offload_create(struct io_ring_ctx *ctx,  in io_sq_offload_create()
9002 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_unaccount_mem()
9011 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)  in io_account_mem()
9075 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)  in io_buffer_unmap()
9090 static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)  in io_rsrc_buf_put()
9096 static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in __io_sqe_buffers_unregister()
9109 static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)  in io_sqe_buffers_unregister()
9129 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,  in io_copy_iov()
9163 static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,  in headpage_already_acct()
9191 static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,  in io_buffer_account_pin()
9223 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,  in io_sqe_buffer_register()
9326 static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)  in io_buffers_map_alloc()
9356 static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,  in io_sqe_buffers_register()
9408 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,  in __io_sqe_buffers_update()
9468 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)  in io_eventfd_register()
9490 static int io_eventfd_unregister(struct io_ring_ctx *ctx)  in io_eventfd_unregister()
9501 static void io_destroy_buffers(struct io_ring_ctx *ctx)  in io_destroy_buffers()
9520 static void io_req_caches_free(struct io_ring_ctx *ctx)  in io_req_caches_free()
9542 static void io_ring_ctx_free(struct io_ring_ctx *ctx)  in io_ring_ctx_free()
9601 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_poll()
9632 static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)  in io_unregister_personality()
9648 	struct io_ring_ctx		*ctx;
9677 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);  in io_ring_exit_work()
9755 static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,  in io_kill_timeouts()
9778 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)  in io_ring_ctx_wait_and_kill()
9813 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_release()
9833 static bool io_cancel_defer_files(struct io_ring_ctx *ctx,  in io_cancel_defer_files()
9859 static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)  in io_uring_try_cancel_iowq()
9883 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,  in io_uring_try_cancel_requests()
9927 static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in __io_uring_add_tctx_node()
9973 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)  in io_uring_add_tctx_node()
10042 	struct io_ring_ctx *ctx;  in io_uring_cancel_generic()
10112 	struct io_ring_ctx *ctx = file->private_data;  in io_uring_validate_mmap_request()
10179 static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)  in io_sqpoll_wait_sq()
10233 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE6()
10357 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)  in __io_uring_show_fdinfo()
10426 	struct io_ring_ctx *ctx = f->private_data;  in io_uring_show_fdinfo()
10448 static int io_allocate_scq_urings(struct io_ring_ctx *ctx,  in io_allocate_scq_urings()
10490 static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)  in io_uring_install_fd()
10513 static struct file *io_uring_get_file(struct io_ring_ctx *ctx)  in io_uring_get_file()
10541 	struct io_ring_ctx *ctx;  in io_uring_create()
10697 static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)  in io_probe()
10736 static int io_register_personality(struct io_ring_ctx *ctx)  in io_register_personality()
10753 static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,  in io_register_restrictions()
10823 static int io_register_enable_rings(struct io_ring_ctx *ctx)  in io_register_enable_rings()
10837 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,  in __io_register_rsrc_update()
10859 static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_files_update()
10874 static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc_update()
10888 static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,  in io_register_rsrc()
10914 static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,  in io_register_iowq_aff()
10949 static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)  in io_unregister_iowq_aff()
10959 static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,  in io_register_iowq_max_workers()
11067 static int io_ctx_quiesce(struct io_ring_ctx *ctx)  in io_ctx_quiesce()
11094 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,  in __io_uring_register()
11237 	struct io_ring_ctx *ctx;  in SYSCALL_DEFINE4()