6066 	struct io_timeout_data *data = container_of(timer,  in io_timeout_fn()  local
6068 	struct io_kiocb *req = data->req;  in io_timeout_fn()
6121 static clockid_t io_timeout_get_clock(struct io_timeout_data *data)  in io_timeout_get_clock()  argument
6123 	switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {  in io_timeout_get_clock()
6167 	struct io_timeout_data *data;  in io_timeout_update()  local
6173 	data = req->async_data;  in io_timeout_update()
6175 	hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);  in io_timeout_update()
6176 	data->timer.function = io_timeout_fn;  in io_timeout_update()
6177 	hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);  in io_timeout_update()
6254 	struct io_timeout_data *data;  in io_timeout_prep()  local
6280 	data = req->async_data;  in io_timeout_prep()
6281 	data->req = req;  in io_timeout_prep()
6282 	data->flags = flags;  in io_timeout_prep()
6284 	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))  in io_timeout_prep()
6288 	data->mode = io_translate_timeout_mode(flags);  in io_timeout_prep()
6289 	hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);  in io_timeout_prep()
6307 	struct io_timeout_data *data = req->async_data;  in io_timeout()  local
6348 	data->timer.function = io_timeout_fn;  in io_timeout()
6349 	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);  in io_timeout()
6359 static bool io_cancel_cb(struct io_wq_work *work, void *data)  in io_cancel_cb()  argument
6362 	struct io_cancel_data *cd = data;  in io_cancel_cb()
6370 	struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };  in io_async_cancel_one()  local
6377 	cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);  in io_async_cancel_one()
6483 	up.data = req->rsrc_update.arg;  in io_files_update()
7027 	struct io_timeout_data *data = container_of(timer,  in io_link_timeout_fn()  local
7029 	struct io_kiocb *prev, *req = data->req;  in io_link_timeout_fn()
7065 		struct io_timeout_data *data = req->async_data;  in io_queue_linked_timeout()  local
7067 		data->timer.function = io_link_timeout_fn;  in io_queue_linked_timeout()
7068 		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),  in io_queue_linked_timeout()
7069 				data->mode);  in io_queue_linked_timeout()
7524 static int io_sq_thread(void *data)  in io_sq_thread()  argument
7526 	struct io_sq_data *sqd = data;  in io_sq_thread()
7883 static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)  in io_rsrc_ref_quiesce()  argument
7888 	if (data->quiesce)  in io_rsrc_ref_quiesce()
7891 	data->quiesce = true;  in io_rsrc_ref_quiesce()
7896 		io_rsrc_node_switch(ctx, data);  in io_rsrc_ref_quiesce()
7899 		if (atomic_dec_and_test(&data->refs))  in io_rsrc_ref_quiesce()
7903 		ret = wait_for_completion_interruptible(&data->done);  in io_rsrc_ref_quiesce()
7906 			if (atomic_read(&data->refs) > 0) {  in io_rsrc_ref_quiesce()
7917 		atomic_inc(&data->refs);  in io_rsrc_ref_quiesce()
7920 		reinit_completion(&data->done);  in io_rsrc_ref_quiesce()
7925 	data->quiesce = false;  in io_rsrc_ref_quiesce()
7930 static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)  in io_get_tag_slot()  argument
7935 	return &data->tags[table_idx][off];  in io_get_tag_slot()
7938 static void io_rsrc_data_free(struct io_rsrc_data *data)  in io_rsrc_data_free()  argument
7940 	size_t size = data->nr * sizeof(data->tags[0][0]);  in io_rsrc_data_free()
7942 	if (data->tags)  in io_rsrc_data_free()
7943 		io_free_page_table((void **)data->tags, size);  in io_rsrc_data_free()
7944 	kfree(data);  in io_rsrc_data_free()
7951 	struct io_rsrc_data *data;  in io_rsrc_data_alloc()  local
7955 	data = kzalloc(sizeof(*data), GFP_KERNEL);  in io_rsrc_data_alloc()
7956 	if (!data)  in io_rsrc_data_alloc()
7958 	data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));  in io_rsrc_data_alloc()
7959 	if (!data->tags) {  in io_rsrc_data_alloc()
7960 		kfree(data);  in io_rsrc_data_alloc()
7964 	data->nr = nr;  in io_rsrc_data_alloc()
7965 	data->ctx = ctx;  in io_rsrc_data_alloc()
7966 	data->do_put = do_put;  in io_rsrc_data_alloc()
7970 			u64 *tag_slot = io_get_tag_slot(data, i);  in io_rsrc_data_alloc()
7978 	atomic_set(&data->refs, 1);  in io_rsrc_data_alloc()
7979 	init_completion(&data->done);  in io_rsrc_data_alloc()
7980 	*pdata = data;  in io_rsrc_data_alloc()
7983 	io_rsrc_data_free(data);  in io_rsrc_data_alloc()
8468 static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,  in io_queue_rsrc_removal()  argument
8471 	u64 *tag_slot = io_get_tag_slot(data, idx);  in io_queue_rsrc_removal()
8578 	__s32 __user *fds = u64_to_user_ptr(up->data);  in __io_sqe_files_update()
8579 	struct io_rsrc_data *data = ctx->file_data;  in __io_sqe_files_update()  local
8611 			err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file);  in __io_sqe_files_update()
8636 			*io_get_tag_slot(data, i) = tag;  in __io_sqe_files_update()
8642 		io_rsrc_node_switch(ctx, data);  in __io_sqe_files_update()
8650 	struct io_wq_data data;  in io_init_wq_offload()  local
8667 	data.hash = hash;  in io_init_wq_offload()
8668 	data.task = task;  in io_init_wq_offload()
8669 	data.free_work = io_wq_free_work;  in io_init_wq_offload()
8670 	data.do_work = io_wq_submit_work;  in io_init_wq_offload()
8675 	return io_wq_create(concurrency, &data);  in io_init_wq_offload()
9195 	struct io_rsrc_data *data;  in io_sqe_buffers_register()  local
9206 	ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);  in io_sqe_buffers_register()
9211 		io_rsrc_data_free(data);  in io_sqe_buffers_register()
9222 		if (!iov.iov_base && *io_get_tag_slot(data, i)) {  in io_sqe_buffers_register()
9235 	ctx->buf_data = data;  in io_sqe_buffers_register()
9248 	struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);  in __io_sqe_buffers_update()
9371 static void io_wait_rsrc_data(struct io_rsrc_data *data)  in io_wait_rsrc_data()  argument
9373 	if (data && !atomic_dec_and_test(&data->refs))  in io_wait_rsrc_data()
9374 		wait_for_completion(&data->done);  in io_wait_rsrc_data()
9503 static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)  in io_cancel_ctx_cb()  argument
9507 	return req->ctx == data;  in io_cancel_ctx_cb()
9665 static bool io_cancel_task_cb(struct io_wq_work *work, void *data)  in io_cancel_task_cb()  argument
9668 	struct io_task_cancel *cancel = data;  in io_cancel_task_cb()
10745 		return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),  in io_register_rsrc()
10748 		return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),  in io_register_rsrc()