1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/io_uring.h>
6
7 #include <trace/events/io_uring.h>
8
9 #include <uapi/linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "refs.h"
13 #include "cancel.h"
14 #include "timeout.h"
15
16 struct io_timeout {
17 struct file *file;
18 u32 off;
19 u32 target_seq;
20 u32 repeats;
21 struct list_head list;
22 /* head of the link, used by linked timeouts only */
23 struct io_kiocb *head;
24 /* for linked completions */
25 struct io_kiocb *prev;
26 };
27
28 struct io_timeout_rem {
29 struct file *file;
30 u64 addr;
31
32 /* timeout update */
33 struct timespec64 ts;
34 u32 flags;
35 bool ltimeout;
36 };
37
io_is_timeout_noseq(struct io_kiocb * req)38 static inline bool io_is_timeout_noseq(struct io_kiocb *req)
39 {
40 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
41 struct io_timeout_data *data = req->async_data;
42
43 return !timeout->off || data->flags & IORING_TIMEOUT_MULTISHOT;
44 }
45
io_put_req(struct io_kiocb * req)46 static inline void io_put_req(struct io_kiocb *req)
47 {
48 if (req_ref_put_and_test(req)) {
49 io_queue_next(req);
50 io_free_req(req);
51 }
52 }
53
io_timeout_finish(struct io_timeout * timeout,struct io_timeout_data * data)54 static inline bool io_timeout_finish(struct io_timeout *timeout,
55 struct io_timeout_data *data)
56 {
57 if (!(data->flags & IORING_TIMEOUT_MULTISHOT))
58 return true;
59
60 if (!timeout->off || (timeout->repeats && --timeout->repeats))
61 return false;
62
63 return true;
64 }
65
66 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
67
io_timeout_complete(struct io_kiocb * req,struct io_tw_state * ts)68 static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
69 {
70 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
71 struct io_timeout_data *data = req->async_data;
72 struct io_ring_ctx *ctx = req->ctx;
73
74 if (!io_timeout_finish(timeout, data)) {
75 if (io_req_post_cqe(req, -ETIME, IORING_CQE_F_MORE)) {
76 /* re-arm timer */
77 spin_lock_irq(&ctx->timeout_lock);
78 list_add(&timeout->list, ctx->timeout_list.prev);
79 data->timer.function = io_timeout_fn;
80 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
81 spin_unlock_irq(&ctx->timeout_lock);
82 return;
83 }
84 }
85
86 io_req_task_complete(req, ts);
87 }
88
io_kill_timeout(struct io_kiocb * req,int status)89 static bool io_kill_timeout(struct io_kiocb *req, int status)
90 __must_hold(&req->ctx->timeout_lock)
91 {
92 struct io_timeout_data *io = req->async_data;
93
94 if (hrtimer_try_to_cancel(&io->timer) != -1) {
95 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
96
97 if (status)
98 req_set_fail(req);
99 atomic_set(&req->ctx->cq_timeouts,
100 atomic_read(&req->ctx->cq_timeouts) + 1);
101 list_del_init(&timeout->list);
102 io_req_queue_tw_complete(req, status);
103 return true;
104 }
105 return false;
106 }
107
io_flush_timeouts(struct io_ring_ctx * ctx)108 __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
109 {
110 u32 seq;
111 struct io_timeout *timeout, *tmp;
112
113 spin_lock_irq(&ctx->timeout_lock);
114 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
115
116 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
117 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
118 u32 events_needed, events_got;
119
120 if (io_is_timeout_noseq(req))
121 break;
122
123 /*
124 * Since seq can easily wrap around over time, subtract
125 * the last seq at which timeouts were flushed before comparing.
126 * Assuming not more than 2^31-1 events have happened since,
127 * these subtractions won't have wrapped, so we can check if
128 * target is in [last_seq, current_seq] by comparing the two.
129 */
130 events_needed = timeout->target_seq - ctx->cq_last_tm_flush;
131 events_got = seq - ctx->cq_last_tm_flush;
132 if (events_got < events_needed)
133 break;
134
135 io_kill_timeout(req, 0);
136 }
137 ctx->cq_last_tm_flush = seq;
138 spin_unlock_irq(&ctx->timeout_lock);
139 }
140
io_req_tw_fail_links(struct io_kiocb * link,struct io_tw_state * ts)141 static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
142 {
143 io_tw_lock(link->ctx, ts);
144 while (link) {
145 struct io_kiocb *nxt = link->link;
146 long res = -ECANCELED;
147
148 if (link->flags & REQ_F_FAIL)
149 res = link->cqe.res;
150 link->link = NULL;
151 io_req_set_res(link, res, 0);
152 io_req_task_complete(link, ts);
153 link = nxt;
154 }
155 }
156
io_fail_links(struct io_kiocb * req)157 static void io_fail_links(struct io_kiocb *req)
158 __must_hold(&req->ctx->completion_lock)
159 {
160 struct io_kiocb *link = req->link;
161 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;
162
163 if (!link)
164 return;
165
166 while (link) {
167 if (ignore_cqes)
168 link->flags |= REQ_F_CQE_SKIP;
169 else
170 link->flags &= ~REQ_F_CQE_SKIP;
171 trace_io_uring_fail_link(req, link);
172 link = link->link;
173 }
174
175 link = req->link;
176 link->io_task_work.func = io_req_tw_fail_links;
177 io_req_task_work_add(link);
178 req->link = NULL;
179 }
180
io_remove_next_linked(struct io_kiocb * req)181 static inline void io_remove_next_linked(struct io_kiocb *req)
182 {
183 struct io_kiocb *nxt = req->link;
184
185 req->link = nxt->link;
186 nxt->link = NULL;
187 }
188
io_disarm_next(struct io_kiocb * req)189 void io_disarm_next(struct io_kiocb *req)
190 __must_hold(&req->ctx->completion_lock)
191 {
192 struct io_kiocb *link = NULL;
193
194 if (req->flags & REQ_F_ARM_LTIMEOUT) {
195 link = req->link;
196 req->flags &= ~REQ_F_ARM_LTIMEOUT;
197 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
198 io_remove_next_linked(req);
199 io_req_queue_tw_complete(link, -ECANCELED);
200 }
201 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
202 struct io_ring_ctx *ctx = req->ctx;
203
204 spin_lock_irq(&ctx->timeout_lock);
205 link = io_disarm_linked_timeout(req);
206 spin_unlock_irq(&ctx->timeout_lock);
207 if (link)
208 io_req_queue_tw_complete(link, -ECANCELED);
209 }
210 if (unlikely((req->flags & REQ_F_FAIL) &&
211 !(req->flags & REQ_F_HARDLINK)))
212 io_fail_links(req);
213 }
214
__io_disarm_linked_timeout(struct io_kiocb * req,struct io_kiocb * link)215 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
216 struct io_kiocb *link)
217 __must_hold(&req->ctx->completion_lock)
218 __must_hold(&req->ctx->timeout_lock)
219 {
220 struct io_timeout_data *io = link->async_data;
221 struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout);
222
223 io_remove_next_linked(req);
224 timeout->head = NULL;
225 if (hrtimer_try_to_cancel(&io->timer) != -1) {
226 list_del(&timeout->list);
227 return link;
228 }
229
230 return NULL;
231 }
232
io_timeout_fn(struct hrtimer * timer)233 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
234 {
235 struct io_timeout_data *data = container_of(timer,
236 struct io_timeout_data, timer);
237 struct io_kiocb *req = data->req;
238 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
239 struct io_ring_ctx *ctx = req->ctx;
240 unsigned long flags;
241
242 spin_lock_irqsave(&ctx->timeout_lock, flags);
243 list_del_init(&timeout->list);
244 atomic_set(&req->ctx->cq_timeouts,
245 atomic_read(&req->ctx->cq_timeouts) + 1);
246 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
247
248 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
249 req_set_fail(req);
250
251 io_req_set_res(req, -ETIME, 0);
252 req->io_task_work.func = io_timeout_complete;
253 io_req_task_work_add(req);
254 return HRTIMER_NORESTART;
255 }
256
io_timeout_extract(struct io_ring_ctx * ctx,struct io_cancel_data * cd)257 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
258 struct io_cancel_data *cd)
259 __must_hold(&ctx->timeout_lock)
260 {
261 struct io_timeout *timeout;
262 struct io_timeout_data *io;
263 struct io_kiocb *req = NULL;
264
265 list_for_each_entry(timeout, &ctx->timeout_list, list) {
266 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
267
268 if (io_cancel_req_match(tmp, cd)) {
269 req = tmp;
270 break;
271 }
272 }
273 if (!req)
274 return ERR_PTR(-ENOENT);
275
276 io = req->async_data;
277 if (hrtimer_try_to_cancel(&io->timer) == -1)
278 return ERR_PTR(-EALREADY);
279 timeout = io_kiocb_to_cmd(req, struct io_timeout);
280 list_del_init(&timeout->list);
281 return req;
282 }
283
io_timeout_cancel(struct io_ring_ctx * ctx,struct io_cancel_data * cd)284 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
285 __must_hold(&ctx->completion_lock)
286 {
287 struct io_kiocb *req;
288
289 spin_lock_irq(&ctx->timeout_lock);
290 req = io_timeout_extract(ctx, cd);
291 spin_unlock_irq(&ctx->timeout_lock);
292
293 if (IS_ERR(req))
294 return PTR_ERR(req);
295 io_req_task_queue_fail(req, -ECANCELED);
296 return 0;
297 }
298
io_req_task_link_timeout(struct io_kiocb * req,struct io_tw_state * ts)299 static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)
300 {
301 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
302 struct io_kiocb *prev = timeout->prev;
303 int ret = -ENOENT;
304
305 if (prev) {
306 if (!io_should_terminate_tw(req->ctx)) {
307 struct io_cancel_data cd = {
308 .ctx = req->ctx,
309 .data = prev->cqe.user_data,
310 };
311
312 ret = io_try_cancel(req->task->io_uring, &cd, 0);
313 }
314 io_req_set_res(req, ret ?: -ETIME, 0);
315 io_req_task_complete(req, ts);
316 io_put_req(prev);
317 } else {
318 io_req_set_res(req, -ETIME, 0);
319 io_req_task_complete(req, ts);
320 }
321 }
322
io_link_timeout_fn(struct hrtimer * timer)323 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
324 {
325 struct io_timeout_data *data = container_of(timer,
326 struct io_timeout_data, timer);
327 struct io_kiocb *prev, *req = data->req;
328 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
329 struct io_ring_ctx *ctx = req->ctx;
330 unsigned long flags;
331
332 spin_lock_irqsave(&ctx->timeout_lock, flags);
333 prev = timeout->head;
334 timeout->head = NULL;
335
336 /*
337 * We don't expect the list to be empty, that will only happen if we
338 * race with the completion of the linked work.
339 */
340 if (prev) {
341 io_remove_next_linked(prev);
342 if (!req_ref_inc_not_zero(prev))
343 prev = NULL;
344 }
345 list_del(&timeout->list);
346 timeout->prev = prev;
347 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
348
349 req->io_task_work.func = io_req_task_link_timeout;
350 io_req_task_work_add(req);
351 return HRTIMER_NORESTART;
352 }
353
io_timeout_get_clock(struct io_timeout_data * data)354 static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
355 {
356 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
357 case IORING_TIMEOUT_BOOTTIME:
358 return CLOCK_BOOTTIME;
359 case IORING_TIMEOUT_REALTIME:
360 return CLOCK_REALTIME;
361 default:
362 /* can't happen, vetted at prep time */
363 WARN_ON_ONCE(1);
364 fallthrough;
365 case 0:
366 return CLOCK_MONOTONIC;
367 }
368 }
369
io_linked_timeout_update(struct io_ring_ctx * ctx,__u64 user_data,struct timespec64 * ts,enum hrtimer_mode mode)370 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
371 struct timespec64 *ts, enum hrtimer_mode mode)
372 __must_hold(&ctx->timeout_lock)
373 {
374 struct io_timeout_data *io;
375 struct io_timeout *timeout;
376 struct io_kiocb *req = NULL;
377
378 list_for_each_entry(timeout, &ctx->ltimeout_list, list) {
379 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
380
381 if (user_data == tmp->cqe.user_data) {
382 req = tmp;
383 break;
384 }
385 }
386 if (!req)
387 return -ENOENT;
388
389 io = req->async_data;
390 if (hrtimer_try_to_cancel(&io->timer) == -1)
391 return -EALREADY;
392 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode);
393 io->timer.function = io_link_timeout_fn;
394 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
395 return 0;
396 }
397
io_timeout_update(struct io_ring_ctx * ctx,__u64 user_data,struct timespec64 * ts,enum hrtimer_mode mode)398 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
399 struct timespec64 *ts, enum hrtimer_mode mode)
400 __must_hold(&ctx->timeout_lock)
401 {
402 struct io_cancel_data cd = { .ctx = ctx, .data = user_data, };
403 struct io_kiocb *req = io_timeout_extract(ctx, &cd);
404 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
405 struct io_timeout_data *data;
406
407 if (IS_ERR(req))
408 return PTR_ERR(req);
409
410 timeout->off = 0; /* noseq */
411 data = req->async_data;
412 data->ts = *ts;
413
414 list_add_tail(&timeout->list, &ctx->timeout_list);
415 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
416 data->timer.function = io_timeout_fn;
417 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), mode);
418 return 0;
419 }
420
io_timeout_remove_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)421 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
422 {
423 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
424
425 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
426 return -EINVAL;
427 if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
428 return -EINVAL;
429
430 tr->ltimeout = false;
431 tr->addr = READ_ONCE(sqe->addr);
432 tr->flags = READ_ONCE(sqe->timeout_flags);
433 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
434 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
435 return -EINVAL;
436 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
437 tr->ltimeout = true;
438 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
439 return -EINVAL;
440 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
441 return -EFAULT;
442 if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0)
443 return -EINVAL;
444 } else if (tr->flags) {
445 /* timeout removal doesn't support flags */
446 return -EINVAL;
447 }
448
449 return 0;
450 }
451
io_translate_timeout_mode(unsigned int flags)452 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
453 {
454 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
455 : HRTIMER_MODE_REL;
456 }
457
458 /*
459 * Remove or update an existing timeout command
460 */
io_timeout_remove(struct io_kiocb * req,unsigned int issue_flags)461 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
462 {
463 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
464 struct io_ring_ctx *ctx = req->ctx;
465 int ret;
466
467 if (!(tr->flags & IORING_TIMEOUT_UPDATE)) {
468 struct io_cancel_data cd = { .ctx = ctx, .data = tr->addr, };
469
470 spin_lock(&ctx->completion_lock);
471 ret = io_timeout_cancel(ctx, &cd);
472 spin_unlock(&ctx->completion_lock);
473 } else {
474 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
475
476 spin_lock_irq(&ctx->timeout_lock);
477 if (tr->ltimeout)
478 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
479 else
480 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
481 spin_unlock_irq(&ctx->timeout_lock);
482 }
483
484 if (ret < 0)
485 req_set_fail(req);
486 io_req_set_res(req, ret, 0);
487 return IOU_OK;
488 }
489
__io_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe,bool is_timeout_link)490 static int __io_timeout_prep(struct io_kiocb *req,
491 const struct io_uring_sqe *sqe,
492 bool is_timeout_link)
493 {
494 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
495 struct io_timeout_data *data;
496 unsigned flags;
497 u32 off = READ_ONCE(sqe->off);
498
499 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
500 return -EINVAL;
501 if (off && is_timeout_link)
502 return -EINVAL;
503 flags = READ_ONCE(sqe->timeout_flags);
504 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK |
505 IORING_TIMEOUT_ETIME_SUCCESS |
506 IORING_TIMEOUT_MULTISHOT))
507 return -EINVAL;
508 /* more than one clock specified is invalid, obviously */
509 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
510 return -EINVAL;
511 /* multishot requests only make sense with rel values */
512 if (!(~flags & (IORING_TIMEOUT_MULTISHOT | IORING_TIMEOUT_ABS)))
513 return -EINVAL;
514
515 INIT_LIST_HEAD(&timeout->list);
516 timeout->off = off;
517 if (unlikely(off && !req->ctx->off_timeout_used))
518 req->ctx->off_timeout_used = true;
519 /*
520 * for multishot reqs w/ fixed nr of repeats, repeats tracks the
521 * remaining nr
522 */
523 timeout->repeats = 0;
524 if ((flags & IORING_TIMEOUT_MULTISHOT) && off > 0)
525 timeout->repeats = off;
526
527 if (WARN_ON_ONCE(req_has_async_data(req)))
528 return -EFAULT;
529 if (io_alloc_async_data(req))
530 return -ENOMEM;
531
532 data = req->async_data;
533 data->req = req;
534 data->flags = flags;
535
536 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
537 return -EFAULT;
538
539 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0)
540 return -EINVAL;
541
542 data->mode = io_translate_timeout_mode(flags);
543 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
544
545 if (is_timeout_link) {
546 struct io_submit_link *link = &req->ctx->submit_state.link;
547
548 if (!link->head)
549 return -EINVAL;
550 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
551 return -EINVAL;
552 timeout->head = link->last;
553 link->last->flags |= REQ_F_ARM_LTIMEOUT;
554 }
555 return 0;
556 }
557
io_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)558 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
559 {
560 return __io_timeout_prep(req, sqe, false);
561 }
562
io_link_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)563 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
564 {
565 return __io_timeout_prep(req, sqe, true);
566 }
567
io_timeout(struct io_kiocb * req,unsigned int issue_flags)568 int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
569 {
570 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
571 struct io_ring_ctx *ctx = req->ctx;
572 struct io_timeout_data *data = req->async_data;
573 struct list_head *entry;
574 u32 tail, off = timeout->off;
575
576 spin_lock_irq(&ctx->timeout_lock);
577
578 /*
579 * sqe->off holds how many events that need to occur for this
580 * timeout event to be satisfied. If it isn't set, then this is
581 * a pure timeout request, sequence isn't used.
582 */
583 if (io_is_timeout_noseq(req)) {
584 entry = ctx->timeout_list.prev;
585 goto add;
586 }
587
588 tail = data_race(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts);
589 timeout->target_seq = tail + off;
590
591 /* Update the last seq here in case io_flush_timeouts() hasn't.
592 * This is safe because ->completion_lock is held, and submissions
593 * and completions are never mixed in the same ->completion_lock section.
594 */
595 ctx->cq_last_tm_flush = tail;
596
597 /*
598 * Insertion sort, ensuring the first entry in the list is always
599 * the one we need first.
600 */
601 list_for_each_prev(entry, &ctx->timeout_list) {
602 struct io_timeout *nextt = list_entry(entry, struct io_timeout, list);
603 struct io_kiocb *nxt = cmd_to_io_kiocb(nextt);
604
605 if (io_is_timeout_noseq(nxt))
606 continue;
607 /* nxt.seq is behind @tail, otherwise would've been completed */
608 if (off >= nextt->target_seq - tail)
609 break;
610 }
611 add:
612 list_add(&timeout->list, entry);
613 data->timer.function = io_timeout_fn;
614 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
615 spin_unlock_irq(&ctx->timeout_lock);
616 return IOU_ISSUE_SKIP_COMPLETE;
617 }
618
io_queue_linked_timeout(struct io_kiocb * req)619 void io_queue_linked_timeout(struct io_kiocb *req)
620 {
621 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
622 struct io_ring_ctx *ctx = req->ctx;
623
624 spin_lock_irq(&ctx->timeout_lock);
625 /*
626 * If the back reference is NULL, then our linked request finished
627 * before we got a chance to setup the timer
628 */
629 if (timeout->head) {
630 struct io_timeout_data *data = req->async_data;
631
632 data->timer.function = io_link_timeout_fn;
633 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
634 data->mode);
635 list_add_tail(&timeout->list, &ctx->ltimeout_list);
636 }
637 spin_unlock_irq(&ctx->timeout_lock);
638 /* drop submission reference */
639 io_put_req(req);
640 }
641
io_match_task(struct io_kiocb * head,struct task_struct * task,bool cancel_all)642 static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
643 bool cancel_all)
644 __must_hold(&head->ctx->timeout_lock)
645 {
646 struct io_kiocb *req;
647
648 if (task && head->task != task)
649 return false;
650 if (cancel_all)
651 return true;
652
653 io_for_each_link(req, head) {
654 if (req->flags & REQ_F_INFLIGHT)
655 return true;
656 }
657 return false;
658 }
659
660 /* Returns true if we found and killed one or more timeouts */
io_kill_timeouts(struct io_ring_ctx * ctx,struct task_struct * tsk,bool cancel_all)661 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
662 bool cancel_all)
663 {
664 struct io_timeout *timeout, *tmp;
665 int canceled = 0;
666
667 /*
668 * completion_lock is needed for io_match_task(). Take it before
669 * timeout_lockfirst to keep locking ordering.
670 */
671 spin_lock(&ctx->completion_lock);
672 spin_lock_irq(&ctx->timeout_lock);
673 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
674 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
675
676 if (io_match_task(req, tsk, cancel_all) &&
677 io_kill_timeout(req, -ECANCELED))
678 canceled++;
679 }
680 spin_unlock_irq(&ctx->timeout_lock);
681 spin_unlock(&ctx->completion_lock);
682 return canceled != 0;
683 }
684