Lines Matching refs:ctx
314 struct io_ring_ctx *ctx; member
371 static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
381 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); in io_ring_ctx_ref_free() local
383 complete(&ctx->ctx_done); in io_ring_ctx_ref_free()
388 struct io_ring_ctx *ctx; in io_ring_ctx_alloc() local
391 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); in io_ring_ctx_alloc()
392 if (!ctx) in io_ring_ctx_alloc()
395 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, in io_ring_ctx_alloc()
397 kfree(ctx); in io_ring_ctx_alloc()
401 ctx->flags = p->flags; in io_ring_ctx_alloc()
402 init_waitqueue_head(&ctx->sqo_wait); in io_ring_ctx_alloc()
403 init_waitqueue_head(&ctx->cq_wait); in io_ring_ctx_alloc()
404 init_completion(&ctx->ctx_done); in io_ring_ctx_alloc()
405 init_completion(&ctx->sqo_thread_started); in io_ring_ctx_alloc()
406 mutex_init(&ctx->uring_lock); in io_ring_ctx_alloc()
407 init_waitqueue_head(&ctx->wait); in io_ring_ctx_alloc()
408 for (i = 0; i < ARRAY_SIZE(ctx->pending_async); i++) { in io_ring_ctx_alloc()
409 spin_lock_init(&ctx->pending_async[i].lock); in io_ring_ctx_alloc()
410 INIT_LIST_HEAD(&ctx->pending_async[i].list); in io_ring_ctx_alloc()
411 atomic_set(&ctx->pending_async[i].cnt, 0); in io_ring_ctx_alloc()
413 spin_lock_init(&ctx->completion_lock); in io_ring_ctx_alloc()
414 INIT_LIST_HEAD(&ctx->poll_list); in io_ring_ctx_alloc()
415 INIT_LIST_HEAD(&ctx->cancel_list); in io_ring_ctx_alloc()
416 INIT_LIST_HEAD(&ctx->defer_list); in io_ring_ctx_alloc()
417 INIT_LIST_HEAD(&ctx->timeout_list); in io_ring_ctx_alloc()
418 INIT_LIST_HEAD(&ctx->task_list); in io_ring_ctx_alloc()
419 spin_lock_init(&ctx->task_lock); in io_ring_ctx_alloc()
420 return ctx; in io_ring_ctx_alloc()
439 static inline bool __io_sequence_defer(struct io_ring_ctx *ctx, in __io_sequence_defer() argument
442 return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped in __io_sequence_defer()
443 + atomic_read(&ctx->cached_cq_overflow); in __io_sequence_defer()
446 static inline bool io_sequence_defer(struct io_ring_ctx *ctx, in io_sequence_defer() argument
452 return __io_sequence_defer(ctx, req); in io_sequence_defer()
455 static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx) in io_get_deferred_req() argument
459 req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list); in io_get_deferred_req()
460 if (req && !io_sequence_defer(ctx, req)) { in io_get_deferred_req()
468 static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx) in io_get_timeout_req() argument
472 req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list); in io_get_timeout_req()
476 if (!__io_sequence_defer(ctx, req)) { in io_get_timeout_req()
485 static void __io_commit_cqring(struct io_ring_ctx *ctx) in __io_commit_cqring() argument
487 struct io_rings *rings = ctx->rings; in __io_commit_cqring()
489 if (ctx->cached_cq_tail != READ_ONCE(rings->cq.tail)) { in __io_commit_cqring()
491 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail); in __io_commit_cqring()
493 if (wq_has_sleeper(&ctx->cq_wait)) { in __io_commit_cqring()
494 wake_up_interruptible(&ctx->cq_wait); in __io_commit_cqring()
495 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN); in __io_commit_cqring()
500 static inline void io_queue_async_work(struct io_ring_ctx *ctx, in io_queue_async_work() argument
518 spin_lock_irqsave(&ctx->task_lock, flags); in io_queue_async_work()
519 list_add(&req->task_list, &ctx->task_list); in io_queue_async_work()
521 spin_unlock_irqrestore(&ctx->task_lock, flags); in io_queue_async_work()
524 queue_work(ctx->sqo_wq[rw], &req->work); in io_queue_async_work()
533 atomic_inc(&req->ctx->cq_timeouts); in io_kill_timeout()
535 io_cqring_fill_event(req->ctx, req->user_data, 0); in io_kill_timeout()
541 static void io_kill_timeouts(struct io_ring_ctx *ctx) in io_kill_timeouts() argument
545 spin_lock_irq(&ctx->completion_lock); in io_kill_timeouts()
546 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list) in io_kill_timeouts()
548 spin_unlock_irq(&ctx->completion_lock); in io_kill_timeouts()
551 static void io_commit_cqring(struct io_ring_ctx *ctx) in io_commit_cqring() argument
555 while ((req = io_get_timeout_req(ctx)) != NULL) in io_commit_cqring()
558 __io_commit_cqring(ctx); in io_commit_cqring()
560 while ((req = io_get_deferred_req(ctx)) != NULL) { in io_commit_cqring()
567 io_queue_async_work(ctx, req); in io_commit_cqring()
571 static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) in io_get_cqring() argument
573 struct io_rings *rings = ctx->rings; in io_get_cqring()
576 tail = ctx->cached_cq_tail; in io_get_cqring()
585 ctx->cached_cq_tail++; in io_get_cqring()
586 return &rings->cqes[tail & ctx->cq_mask]; in io_get_cqring()
589 static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data, in io_cqring_fill_event() argument
599 cqe = io_get_cqring(ctx); in io_cqring_fill_event()
605 WRITE_ONCE(ctx->rings->cq_overflow, in io_cqring_fill_event()
606 atomic_inc_return(&ctx->cached_cq_overflow)); in io_cqring_fill_event()
610 static void io_cqring_ev_posted(struct io_ring_ctx *ctx) in io_cqring_ev_posted() argument
612 if (waitqueue_active(&ctx->wait)) in io_cqring_ev_posted()
613 wake_up(&ctx->wait); in io_cqring_ev_posted()
614 if (waitqueue_active(&ctx->sqo_wait)) in io_cqring_ev_posted()
615 wake_up(&ctx->sqo_wait); in io_cqring_ev_posted()
616 if (ctx->cq_ev_fd) in io_cqring_ev_posted()
617 eventfd_signal(ctx->cq_ev_fd, 1); in io_cqring_ev_posted()
620 static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data, in io_cqring_add_event() argument
625 spin_lock_irqsave(&ctx->completion_lock, flags); in io_cqring_add_event()
626 io_cqring_fill_event(ctx, user_data, res); in io_cqring_add_event()
627 io_commit_cqring(ctx); in io_cqring_add_event()
628 spin_unlock_irqrestore(&ctx->completion_lock, flags); in io_cqring_add_event()
630 io_cqring_ev_posted(ctx); in io_cqring_add_event()
633 static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx, in io_get_req() argument
639 if (!percpu_ref_tryget(&ctx->refs)) in io_get_req()
674 req->ctx = ctx; in io_get_req()
682 percpu_ref_put(&ctx->refs); in io_get_req()
686 static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr) in io_free_req_many() argument
690 percpu_ref_put_many(&ctx->refs, *nr); in io_free_req_many()
700 percpu_ref_put(&req->ctx->refs); in __io_free_req()
724 io_queue_async_work(req->ctx, nxt); in io_req_link_next()
739 io_cqring_add_event(req->ctx, link->user_data, -ECANCELED); in io_fail_links()
775 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) in io_sqring_entries() argument
777 struct io_rings *rings = ctx->rings; in io_sqring_entries()
780 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; in io_sqring_entries()
786 static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, in io_iopoll_complete() argument
798 io_cqring_fill_event(ctx, req->user_data, req->result); in io_iopoll_complete()
811 io_free_req_many(ctx, reqs, &to_free); in io_iopoll_complete()
818 io_commit_cqring(ctx); in io_iopoll_complete()
819 io_free_req_many(ctx, reqs, &to_free); in io_iopoll_complete()
822 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, in io_do_iopoll() argument
834 spin = !ctx->poll_multi_file && *nr_events < min; in io_do_iopoll()
837 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) { in io_do_iopoll()
862 io_iopoll_complete(ctx, nr_events, &done); in io_do_iopoll()
872 static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events, in io_iopoll_getevents() argument
875 while (!list_empty(&ctx->poll_list) && !need_resched()) { in io_iopoll_getevents()
878 ret = io_do_iopoll(ctx, nr_events, min); in io_iopoll_getevents()
892 static void io_iopoll_reap_events(struct io_ring_ctx *ctx) in io_iopoll_reap_events() argument
894 if (!(ctx->flags & IORING_SETUP_IOPOLL)) in io_iopoll_reap_events()
897 mutex_lock(&ctx->uring_lock); in io_iopoll_reap_events()
898 while (!list_empty(&ctx->poll_list)) { in io_iopoll_reap_events()
901 io_iopoll_getevents(ctx, &nr_events, 1); in io_iopoll_reap_events()
909 mutex_unlock(&ctx->uring_lock); in io_iopoll_reap_events()
912 static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, in io_iopoll_check() argument
922 mutex_lock(&ctx->uring_lock); in io_iopoll_check()
931 if (io_cqring_events(ctx->rings)) in io_iopoll_check()
945 mutex_unlock(&ctx->uring_lock); in io_iopoll_check()
946 mutex_lock(&ctx->uring_lock); in io_iopoll_check()
952 ret = io_iopoll_getevents(ctx, nr_events, tmin); in io_iopoll_check()
958 mutex_unlock(&ctx->uring_lock); in io_iopoll_check()
985 io_cqring_add_event(req->ctx, req->user_data, res); in io_complete_rw()
1011 struct io_ring_ctx *ctx = req->ctx; in io_iopoll_req_issued() local
1018 if (list_empty(&ctx->poll_list)) { in io_iopoll_req_issued()
1019 ctx->poll_multi_file = false; in io_iopoll_req_issued()
1020 } else if (!ctx->poll_multi_file) { in io_iopoll_req_issued()
1023 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb, in io_iopoll_req_issued()
1026 ctx->poll_multi_file = true; in io_iopoll_req_issued()
1034 list_add(&req->list, &ctx->poll_list); in io_iopoll_req_issued()
1036 list_add_tail(&req->list, &ctx->poll_list); in io_iopoll_req_issued()
1100 struct io_ring_ctx *ctx = req->ctx; in io_prep_rw() local
1149 if (ctx->flags & IORING_SETUP_IOPOLL) { in io_prep_rw()
1186 static int io_import_fixed(struct io_ring_ctx *ctx, int rw, in io_import_fixed() argument
1197 if (unlikely(!ctx->user_bufs)) in io_import_fixed()
1201 if (unlikely(buf_index >= ctx->nr_user_bufs)) in io_import_fixed()
1204 index = array_index_nospec(buf_index, ctx->nr_user_bufs); in io_import_fixed()
1205 imu = &ctx->user_bufs[index]; in io_import_fixed()
1260 static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw, in io_import_iovec() argument
1272 ssize_t ret = io_import_fixed(ctx, rw, sqe, iter); in io_import_iovec()
1281 if (ctx->compat) in io_import_iovec()
1319 struct async_list *async_list = &req->ctx->pending_async[rw]; in io_async_list_note()
1424 ret = io_import_iovec(req->ctx, READ, req, &iovec, &iter); in io_read()
1491 ret = io_import_iovec(req->ctx, WRITE, req, &iovec, &iter); in io_write()
1562 struct io_ring_ctx *ctx = req->ctx; in io_nop() local
1565 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) in io_nop()
1568 io_cqring_add_event(ctx, user_data, err); in io_nop()
1575 struct io_ring_ctx *ctx = req->ctx; in io_prep_fsync() local
1580 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) in io_prep_fsync()
1615 io_cqring_add_event(req->ctx, sqe->user_data, ret); in io_fsync()
1622 struct io_ring_ctx *ctx = req->ctx; in io_prep_sfr() local
1628 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) in io_prep_sfr()
1661 io_cqring_add_event(req->ctx, sqe->user_data, ret); in io_sync_file_range()
1675 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) in io_send_recvmsg()
1690 if (req->ctx->compat) in io_send_recvmsg()
1705 io_cqring_add_event(req->ctx, sqe->user_data, ret); in io_send_recvmsg()
1739 io_queue_async_work(req->ctx, req); in io_poll_remove_one()
1746 static void io_poll_remove_all(struct io_ring_ctx *ctx) in io_poll_remove_all() argument
1750 spin_lock_irq(&ctx->completion_lock); in io_poll_remove_all()
1751 while (!list_empty(&ctx->cancel_list)) { in io_poll_remove_all()
1752 req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list); in io_poll_remove_all()
1755 spin_unlock_irq(&ctx->completion_lock); in io_poll_remove_all()
1764 struct io_ring_ctx *ctx = req->ctx; in io_poll_remove() local
1768 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) in io_poll_remove()
1774 spin_lock_irq(&ctx->completion_lock); in io_poll_remove()
1775 list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) { in io_poll_remove()
1782 spin_unlock_irq(&ctx->completion_lock); in io_poll_remove()
1784 io_cqring_add_event(req->ctx, sqe->user_data, ret); in io_poll_remove()
1789 static void io_poll_complete(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_poll_complete() argument
1793 io_cqring_fill_event(ctx, req->user_data, mangle_poll(mask)); in io_poll_complete()
1794 io_commit_cqring(ctx); in io_poll_complete()
1802 struct io_ring_ctx *ctx = req->ctx; in io_poll_complete_work() local
1806 old_cred = override_creds(ctx->creds); in io_poll_complete_work()
1818 spin_lock_irq(&ctx->completion_lock); in io_poll_complete_work()
1821 spin_unlock_irq(&ctx->completion_lock); in io_poll_complete_work()
1825 io_poll_complete(ctx, req, mask); in io_poll_complete_work()
1826 spin_unlock_irq(&ctx->completion_lock); in io_poll_complete_work()
1828 io_cqring_ev_posted(ctx); in io_poll_complete_work()
1840 struct io_ring_ctx *ctx = req->ctx; in io_poll_wake() local
1850 if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) { in io_poll_wake()
1852 io_poll_complete(ctx, req, mask); in io_poll_wake()
1853 spin_unlock_irqrestore(&ctx->completion_lock, flags); in io_poll_wake()
1855 io_cqring_ev_posted(ctx); in io_poll_wake()
1858 io_queue_async_work(ctx, req); in io_poll_wake()
1888 struct io_ring_ctx *ctx = req->ctx; in io_poll_add() local
1894 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) in io_poll_add()
1923 spin_lock_irq(&ctx->completion_lock); in io_poll_add()
1937 list_add_tail(&req->list, &ctx->cancel_list); in io_poll_add()
1942 io_poll_complete(ctx, req, mask); in io_poll_add()
1944 spin_unlock_irq(&ctx->completion_lock); in io_poll_add()
1947 io_cqring_ev_posted(ctx); in io_poll_add()
1955 struct io_ring_ctx *ctx; in io_timeout_fn() local
1960 ctx = req->ctx; in io_timeout_fn()
1961 atomic_inc(&ctx->cq_timeouts); in io_timeout_fn()
1963 spin_lock_irqsave(&ctx->completion_lock, flags); in io_timeout_fn()
1971 list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list) in io_timeout_fn()
1975 io_cqring_fill_event(ctx, req->user_data, -ETIME); in io_timeout_fn()
1976 io_commit_cqring(ctx); in io_timeout_fn()
1977 spin_unlock_irqrestore(&ctx->completion_lock, flags); in io_timeout_fn()
1979 io_cqring_ev_posted(ctx); in io_timeout_fn()
1988 struct io_ring_ctx *ctx = req->ctx; in io_timeout() local
1993 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) in io_timeout()
2012 spin_lock_irq(&ctx->completion_lock); in io_timeout()
2013 entry = ctx->timeout_list.prev; in io_timeout()
2017 req->sequence = ctx->cached_sq_head + count - 1; in io_timeout()
2025 spin_lock_irq(&ctx->completion_lock); in io_timeout()
2026 list_for_each_prev(entry, &ctx->timeout_list) { in io_timeout()
2038 tmp = (long long)ctx->cached_sq_head + count - 1; in io_timeout()
2046 if (ctx->cached_sq_head < nxt_sq_head) in io_timeout()
2067 spin_unlock_irq(&ctx->completion_lock); in io_timeout()
2071 static int io_req_defer(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_req_defer() argument
2076 if (!io_sequence_defer(ctx, req) && list_empty(&ctx->defer_list)) in io_req_defer()
2083 spin_lock_irq(&ctx->completion_lock); in io_req_defer()
2084 if (!io_sequence_defer(ctx, req) && list_empty(&ctx->defer_list)) { in io_req_defer()
2085 spin_unlock_irq(&ctx->completion_lock); in io_req_defer()
2095 list_add_tail(&req->list, &ctx->defer_list); in io_req_defer()
2096 spin_unlock_irq(&ctx->completion_lock); in io_req_defer()
2100 static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, in __io_submit_sqe() argument
2107 if (unlikely(s->index >= ctx->sq_entries)) in __io_submit_sqe()
2159 if (ctx->flags & IORING_SETUP_IOPOLL) { in __io_submit_sqe()
2165 mutex_lock(&ctx->uring_lock); in __io_submit_sqe()
2168 mutex_unlock(&ctx->uring_lock); in __io_submit_sqe()
2174 static struct async_list *io_async_list_from_req(struct io_ring_ctx *ctx, in io_async_list_from_req() argument
2180 return &ctx->pending_async[READ]; in io_async_list_from_req()
2183 return &ctx->pending_async[WRITE]; in io_async_list_from_req()
2199 struct io_ring_ctx *ctx = req->ctx; in io_sq_wq_submit_work() local
2207 old_cred = override_creds(ctx->creds); in io_sq_wq_submit_work()
2208 async_list = io_async_list_from_req(ctx, req); in io_sq_wq_submit_work()
2232 if (!mmget_not_zero(ctx->sqo_mm)) { in io_sq_wq_submit_work()
2236 cur_mm = ctx->sqo_mm; in io_sq_wq_submit_work()
2259 ret = __io_submit_sqe(ctx, req, s, false); in io_sq_wq_submit_work()
2272 spin_lock_irq(&ctx->task_lock); in io_sq_wq_submit_work()
2274 spin_unlock_irq(&ctx->task_lock); in io_sq_wq_submit_work()
2280 io_cqring_add_event(ctx, sqe->user_data, ret); in io_sq_wq_submit_work()
2382 struct io_ring_ctx *ctx = req->ctx; in io_add_to_prev_work() local
2386 spin_lock_irq(&ctx->task_lock); in io_add_to_prev_work()
2387 list_add(&req->task_list, &ctx->task_list); in io_add_to_prev_work()
2389 spin_unlock_irq(&ctx->task_lock); in io_add_to_prev_work()
2407 static int io_req_set_file(struct io_ring_ctx *ctx, const struct sqe_submit *s, in io_req_set_file() argument
2429 if (unlikely(!ctx->user_files || in io_req_set_file()
2430 (unsigned) fd >= ctx->nr_user_files)) in io_req_set_file()
2432 req->file = ctx->user_files[fd]; in io_req_set_file()
2445 static int __io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, in __io_queue_sqe() argument
2450 ret = __io_submit_sqe(ctx, req, s, true); in __io_queue_sqe()
2466 list = io_async_list_from_req(ctx, req); in __io_queue_sqe()
2471 io_queue_async_work(ctx, req); in __io_queue_sqe()
2487 io_cqring_add_event(ctx, req->user_data, ret); in __io_queue_sqe()
2496 static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_queue_sqe() argument
2501 ret = io_req_defer(ctx, req, s); in io_queue_sqe()
2505 io_cqring_add_event(ctx, s->sqe->user_data, ret); in io_queue_sqe()
2510 return __io_queue_sqe(ctx, req, s); in io_queue_sqe()
2513 static int io_queue_link_head(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_queue_link_head() argument
2520 return io_queue_sqe(ctx, req, s); in io_queue_link_head()
2528 ret = io_req_defer(ctx, req, s); in io_queue_link_head()
2533 io_cqring_add_event(ctx, s->sqe->user_data, ret); in io_queue_link_head()
2545 spin_lock_irq(&ctx->completion_lock); in io_queue_link_head()
2546 list_add_tail(&shadow->list, &ctx->defer_list); in io_queue_link_head()
2547 spin_unlock_irq(&ctx->completion_lock); in io_queue_link_head()
2550 return __io_queue_sqe(ctx, req, s); in io_queue_link_head()
2557 static void io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, in io_submit_sqe() argument
2570 req = io_get_req(ctx, state); in io_submit_sqe()
2577 ret = io_req_set_file(ctx, s, state, req); in io_submit_sqe()
2582 io_cqring_add_event(ctx, s->sqe->user_data, ret); in io_submit_sqe()
2631 io_queue_sqe(ctx, req, s); in io_submit_sqe()
2651 struct io_ring_ctx *ctx, unsigned max_ios) in io_submit_state_start() argument
2659 static void io_commit_sqring(struct io_ring_ctx *ctx) in io_commit_sqring() argument
2661 struct io_rings *rings = ctx->rings; in io_commit_sqring()
2663 if (ctx->cached_sq_head != READ_ONCE(rings->sq.head)) { in io_commit_sqring()
2669 smp_store_release(&rings->sq.head, ctx->cached_sq_head); in io_commit_sqring()
2681 static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s) in io_get_sqring() argument
2683 struct io_rings *rings = ctx->rings; in io_get_sqring()
2684 u32 *sq_array = ctx->sq_array; in io_get_sqring()
2695 head = ctx->cached_sq_head; in io_get_sqring()
2700 head = READ_ONCE(sq_array[head & ctx->sq_mask]); in io_get_sqring()
2701 if (head < ctx->sq_entries) { in io_get_sqring()
2703 s->sqe = &ctx->sq_sqes[head]; in io_get_sqring()
2705 s->sequence = ctx->cached_sq_head; in io_get_sqring()
2706 ctx->cached_sq_head++; in io_get_sqring()
2711 ctx->cached_sq_head++; in io_get_sqring()
2712 ctx->cached_sq_dropped++; in io_get_sqring()
2713 WRITE_ONCE(rings->sq_dropped, ctx->cached_sq_dropped); in io_get_sqring()
2717 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, in io_submit_sqes() argument
2727 io_submit_state_start(&state, ctx, nr); in io_submit_sqes()
2734 if (!io_get_sqring(ctx, &s)) in io_submit_sqes()
2742 io_queue_link_head(ctx, link, &link->submit, shadow_req); in io_submit_sqes()
2750 shadow_req = io_get_req(ctx, NULL); in io_submit_sqes()
2761 io_cqring_add_event(ctx, s.sqe->user_data, in io_submit_sqes()
2767 io_submit_sqe(ctx, &s, statep, &link); in io_submit_sqes()
2773 io_queue_link_head(ctx, link, &link->submit, shadow_req); in io_submit_sqes()
2782 struct io_ring_ctx *ctx = data; in io_sq_thread() local
2790 complete(&ctx->sqo_thread_started); in io_sq_thread()
2794 old_cred = override_creds(ctx->creds); in io_sq_thread()
2804 if (ctx->flags & IORING_SETUP_IOPOLL) { in io_sq_thread()
2813 mutex_lock(&ctx->uring_lock); in io_sq_thread()
2814 if (!list_empty(&ctx->poll_list)) in io_sq_thread()
2815 io_iopoll_getevents(ctx, &nr_events, 0); in io_sq_thread()
2818 mutex_unlock(&ctx->uring_lock); in io_sq_thread()
2829 timeout = jiffies + ctx->sq_thread_idle; in io_sq_thread()
2832 to_submit = io_sqring_entries(ctx); in io_sq_thread()
2856 prepare_to_wait(&ctx->sqo_wait, &wait, in io_sq_thread()
2860 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP; in io_sq_thread()
2864 to_submit = io_sqring_entries(ctx); in io_sq_thread()
2867 finish_wait(&ctx->sqo_wait, &wait); in io_sq_thread()
2873 finish_wait(&ctx->sqo_wait, &wait); in io_sq_thread()
2875 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; in io_sq_thread()
2878 finish_wait(&ctx->sqo_wait, &wait); in io_sq_thread()
2880 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; in io_sq_thread()
2885 mm_fault = !mmget_not_zero(ctx->sqo_mm); in io_sq_thread()
2887 use_mm(ctx->sqo_mm); in io_sq_thread()
2888 cur_mm = ctx->sqo_mm; in io_sq_thread()
2892 to_submit = min(to_submit, ctx->sq_entries); in io_sq_thread()
2893 inflight += io_submit_sqes(ctx, to_submit, cur_mm != NULL, in io_sq_thread()
2897 io_commit_sqring(ctx); in io_sq_thread()
2912 static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) in io_ring_submit() argument
2921 io_submit_state_start(&state, ctx, to_submit); in io_ring_submit()
2928 if (!io_get_sqring(ctx, &s)) in io_ring_submit()
2936 io_queue_link_head(ctx, link, &link->submit, shadow_req); in io_ring_submit()
2944 shadow_req = io_get_req(ctx, NULL); in io_ring_submit()
2958 io_submit_sqe(ctx, &s, statep, &link); in io_ring_submit()
2962 io_queue_link_head(ctx, link, &link->submit, shadow_req); in io_ring_submit()
2966 io_commit_sqring(ctx); in io_ring_submit()
2973 struct io_ring_ctx *ctx; member
2980 struct io_ring_ctx *ctx = iowq->ctx; in io_should_wake() local
2987 return io_cqring_events(ctx->rings) >= iowq->to_wait || in io_should_wake()
2988 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; in io_should_wake()
3007 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, in io_cqring_wait() argument
3016 .ctx = ctx, in io_cqring_wait()
3019 struct io_rings *rings = ctx->rings; in io_cqring_wait()
3039 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); in io_cqring_wait()
3041 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, in io_cqring_wait()
3051 finish_wait(&ctx->wait, &iowq.wq); in io_cqring_wait()
3060 static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) in __io_sqe_files_unregister() argument
3064 for (i = 0; i < ctx->nr_user_files; i++) in __io_sqe_files_unregister()
3065 fput(ctx->user_files[i]); in __io_sqe_files_unregister()
3068 static int io_sqe_files_unregister(struct io_ring_ctx *ctx) in io_sqe_files_unregister() argument
3070 if (!ctx->user_files) in io_sqe_files_unregister()
3073 __io_sqe_files_unregister(ctx); in io_sqe_files_unregister()
3074 kfree(ctx->user_files); in io_sqe_files_unregister()
3075 ctx->user_files = NULL; in io_sqe_files_unregister()
3076 ctx->nr_user_files = 0; in io_sqe_files_unregister()
3080 static void io_sq_thread_stop(struct io_ring_ctx *ctx) in io_sq_thread_stop() argument
3082 if (ctx->sqo_thread) { in io_sq_thread_stop()
3083 wait_for_completion(&ctx->sqo_thread_started); in io_sq_thread_stop()
3089 kthread_park(ctx->sqo_thread); in io_sq_thread_stop()
3090 kthread_stop(ctx->sqo_thread); in io_sq_thread_stop()
3091 ctx->sqo_thread = NULL; in io_sq_thread_stop()
3095 static void io_finish_async(struct io_ring_ctx *ctx) in io_finish_async() argument
3099 io_sq_thread_stop(ctx); in io_finish_async()
3101 for (i = 0; i < ARRAY_SIZE(ctx->sqo_wq); i++) { in io_finish_async()
3102 if (ctx->sqo_wq[i]) { in io_finish_async()
3103 destroy_workqueue(ctx->sqo_wq[i]); in io_finish_async()
3104 ctx->sqo_wq[i] = NULL; in io_finish_async()
3109 static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, in io_sqe_files_register() argument
3116 if (ctx->user_files) in io_sqe_files_register()
3123 ctx->user_files = kcalloc(nr_args, sizeof(struct file *), GFP_KERNEL); in io_sqe_files_register()
3124 if (!ctx->user_files) in io_sqe_files_register()
3132 ctx->user_files[i] = fget(fd); in io_sqe_files_register()
3135 if (!ctx->user_files[i]) in io_sqe_files_register()
3144 if (ctx->user_files[i]->f_op == &io_uring_fops) { in io_sqe_files_register()
3145 fput(ctx->user_files[i]); in io_sqe_files_register()
3148 ctx->nr_user_files++; in io_sqe_files_register()
3153 for (i = 0; i < ctx->nr_user_files; i++) in io_sqe_files_register()
3154 fput(ctx->user_files[i]); in io_sqe_files_register()
3156 kfree(ctx->user_files); in io_sqe_files_register()
3157 ctx->user_files = NULL; in io_sqe_files_register()
3158 ctx->nr_user_files = 0; in io_sqe_files_register()
3165 static int io_sq_offload_start(struct io_ring_ctx *ctx, in io_sq_offload_start() argument
3171 ctx->sqo_mm = current->mm; in io_sq_offload_start()
3173 if (ctx->flags & IORING_SETUP_SQPOLL) { in io_sq_offload_start()
3178 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle); in io_sq_offload_start()
3179 if (!ctx->sq_thread_idle) in io_sq_offload_start()
3180 ctx->sq_thread_idle = HZ; in io_sq_offload_start()
3191 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread, in io_sq_offload_start()
3192 ctx, cpu, in io_sq_offload_start()
3195 ctx->sqo_thread = kthread_create(io_sq_thread, ctx, in io_sq_offload_start()
3198 if (IS_ERR(ctx->sqo_thread)) { in io_sq_offload_start()
3199 ret = PTR_ERR(ctx->sqo_thread); in io_sq_offload_start()
3200 ctx->sqo_thread = NULL; in io_sq_offload_start()
3203 wake_up_process(ctx->sqo_thread); in io_sq_offload_start()
3211 ctx->sqo_wq[0] = alloc_workqueue("io_ring-wq", in io_sq_offload_start()
3213 min(ctx->sq_entries - 1, 2 * num_online_cpus())); in io_sq_offload_start()
3214 if (!ctx->sqo_wq[0]) { in io_sq_offload_start()
3226 ctx->sqo_wq[1] = alloc_workqueue("io_ring-write-wq", in io_sq_offload_start()
3228 if (!ctx->sqo_wq[1]) { in io_sq_offload_start()
3235 io_finish_async(ctx); in io_sq_offload_start()
3236 mmdrop(ctx->sqo_mm); in io_sq_offload_start()
3237 ctx->sqo_mm = NULL; in io_sq_offload_start()
3325 static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx) in io_sqe_buffer_unregister() argument
3329 if (!ctx->user_bufs) in io_sqe_buffer_unregister()
3332 for (i = 0; i < ctx->nr_user_bufs; i++) { in io_sqe_buffer_unregister()
3333 struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; in io_sqe_buffer_unregister()
3338 if (ctx->account_mem) in io_sqe_buffer_unregister()
3339 io_unaccount_mem(ctx->user, imu->nr_bvecs); in io_sqe_buffer_unregister()
3344 kfree(ctx->user_bufs); in io_sqe_buffer_unregister()
3345 ctx->user_bufs = NULL; in io_sqe_buffer_unregister()
3346 ctx->nr_user_bufs = 0; in io_sqe_buffer_unregister()
3350 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst, in io_copy_iov() argument
3356 if (ctx->compat) { in io_copy_iov()
3375 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, in io_sqe_buffer_register() argument
3383 if (ctx->user_bufs) in io_sqe_buffer_register()
3388 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf), in io_sqe_buffer_register()
3390 if (!ctx->user_bufs) in io_sqe_buffer_register()
3394 struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; in io_sqe_buffer_register()
3400 ret = io_copy_iov(ctx, &iov, arg, i); in io_sqe_buffer_register()
3422 if (ctx->account_mem) { in io_sqe_buffer_register()
3423 ret = io_account_mem(ctx->user, nr_pages); in io_sqe_buffer_register()
3439 if (ctx->account_mem) in io_sqe_buffer_register()
3440 io_unaccount_mem(ctx->user, nr_pages); in io_sqe_buffer_register()
3450 if (ctx->account_mem) in io_sqe_buffer_register()
3451 io_unaccount_mem(ctx->user, nr_pages); in io_sqe_buffer_register()
3482 if (ctx->account_mem) in io_sqe_buffer_register()
3483 io_unaccount_mem(ctx->user, nr_pages); in io_sqe_buffer_register()
3505 ctx->nr_user_bufs++; in io_sqe_buffer_register()
3513 io_sqe_buffer_unregister(ctx); in io_sqe_buffer_register()
3517 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) in io_eventfd_register() argument
3522 if (ctx->cq_ev_fd) in io_eventfd_register()
3528 ctx->cq_ev_fd = eventfd_ctx_fdget(fd); in io_eventfd_register()
3529 if (IS_ERR(ctx->cq_ev_fd)) { in io_eventfd_register()
3530 int ret = PTR_ERR(ctx->cq_ev_fd); in io_eventfd_register()
3531 ctx->cq_ev_fd = NULL; in io_eventfd_register()
3538 static int io_eventfd_unregister(struct io_ring_ctx *ctx) in io_eventfd_unregister() argument
3540 if (ctx->cq_ev_fd) { in io_eventfd_unregister()
3541 eventfd_ctx_put(ctx->cq_ev_fd); in io_eventfd_unregister()
3542 ctx->cq_ev_fd = NULL; in io_eventfd_unregister()
3549 static void io_ring_ctx_free(struct io_ring_ctx *ctx) in io_ring_ctx_free() argument
3551 io_finish_async(ctx); in io_ring_ctx_free()
3552 if (ctx->sqo_mm) in io_ring_ctx_free()
3553 mmdrop(ctx->sqo_mm); in io_ring_ctx_free()
3555 io_iopoll_reap_events(ctx); in io_ring_ctx_free()
3556 io_sqe_buffer_unregister(ctx); in io_ring_ctx_free()
3557 io_sqe_files_unregister(ctx); in io_ring_ctx_free()
3558 io_eventfd_unregister(ctx); in io_ring_ctx_free()
3560 io_mem_free(ctx->rings); in io_ring_ctx_free()
3561 io_mem_free(ctx->sq_sqes); in io_ring_ctx_free()
3563 percpu_ref_exit(&ctx->refs); in io_ring_ctx_free()
3564 if (ctx->account_mem) in io_ring_ctx_free()
3565 io_unaccount_mem(ctx->user, in io_ring_ctx_free()
3566 ring_pages(ctx->sq_entries, ctx->cq_entries)); in io_ring_ctx_free()
3567 free_uid(ctx->user); in io_ring_ctx_free()
3568 if (ctx->creds) in io_ring_ctx_free()
3569 put_cred(ctx->creds); in io_ring_ctx_free()
3570 kfree(ctx); in io_ring_ctx_free()
3575 struct io_ring_ctx *ctx = file->private_data; in io_uring_poll() local
3578 poll_wait(file, &ctx->cq_wait, wait); in io_uring_poll()
3584 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head != in io_uring_poll()
3585 ctx->rings->sq_ring_entries) in io_uring_poll()
3587 if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail) in io_uring_poll()
3595 struct io_ring_ctx *ctx = file->private_data; in io_uring_fasync() local
3597 return fasync_helper(fd, file, on, &ctx->cq_fasync); in io_uring_fasync()
3600 static void io_cancel_async_work(struct io_ring_ctx *ctx, in io_cancel_async_work() argument
3605 spin_lock_irq(&ctx->task_lock); in io_cancel_async_work()
3607 list_for_each_entry(req, &ctx->task_list, task_list) { in io_cancel_async_work()
3622 spin_unlock_irq(&ctx->task_lock); in io_cancel_async_work()
3625 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) in io_ring_ctx_wait_and_kill() argument
3627 mutex_lock(&ctx->uring_lock); in io_ring_ctx_wait_and_kill()
3628 percpu_ref_kill(&ctx->refs); in io_ring_ctx_wait_and_kill()
3629 mutex_unlock(&ctx->uring_lock); in io_ring_ctx_wait_and_kill()
3631 io_cancel_async_work(ctx, NULL); in io_ring_ctx_wait_and_kill()
3632 io_kill_timeouts(ctx); in io_ring_ctx_wait_and_kill()
3633 io_poll_remove_all(ctx); in io_ring_ctx_wait_and_kill()
3634 io_iopoll_reap_events(ctx); in io_ring_ctx_wait_and_kill()
3635 wait_for_completion(&ctx->ctx_done); in io_ring_ctx_wait_and_kill()
3636 io_ring_ctx_free(ctx); in io_ring_ctx_wait_and_kill()
3641 struct io_ring_ctx *ctx = file->private_data; in io_uring_flush() local
3644 io_cancel_async_work(ctx, data); in io_uring_flush()
3651 struct io_ring_ctx *ctx = file->private_data; in io_uring_release() local
3654 io_ring_ctx_wait_and_kill(ctx); in io_uring_release()
3662 struct io_ring_ctx *ctx = file->private_data; in io_uring_mmap() local
3670 ptr = ctx->rings; in io_uring_mmap()
3673 ptr = ctx->sq_sqes; in io_uring_mmap()
3691 struct io_ring_ctx *ctx; in SYSCALL_DEFINE6() local
3708 ctx = f.file->private_data; in SYSCALL_DEFINE6()
3709 if (!percpu_ref_tryget(&ctx->refs)) in SYSCALL_DEFINE6()
3718 if (ctx->flags & IORING_SETUP_SQPOLL) { in SYSCALL_DEFINE6()
3720 wake_up(&ctx->sqo_wait); in SYSCALL_DEFINE6()
3723 to_submit = min(to_submit, ctx->sq_entries); in SYSCALL_DEFINE6()
3725 mutex_lock(&ctx->uring_lock); in SYSCALL_DEFINE6()
3726 submitted = io_ring_submit(ctx, to_submit); in SYSCALL_DEFINE6()
3727 mutex_unlock(&ctx->uring_lock); in SYSCALL_DEFINE6()
3735 min_complete = min(min_complete, ctx->cq_entries); in SYSCALL_DEFINE6()
3737 if (ctx->flags & IORING_SETUP_IOPOLL) { in SYSCALL_DEFINE6()
3738 ret = io_iopoll_check(ctx, &nr_events, min_complete); in SYSCALL_DEFINE6()
3740 ret = io_cqring_wait(ctx, min_complete, sig, sigsz); in SYSCALL_DEFINE6()
3745 percpu_ref_put(&ctx->refs); in SYSCALL_DEFINE6()
3764 static int io_allocate_scq_urings(struct io_ring_ctx *ctx, in io_allocate_scq_urings() argument
3771 ctx->sq_entries = p->sq_entries; in io_allocate_scq_urings()
3772 ctx->cq_entries = p->cq_entries; in io_allocate_scq_urings()
3782 ctx->rings = rings; in io_allocate_scq_urings()
3783 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); in io_allocate_scq_urings()
3788 ctx->sq_mask = rings->sq_ring_mask; in io_allocate_scq_urings()
3789 ctx->cq_mask = rings->cq_ring_mask; in io_allocate_scq_urings()
3793 io_mem_free(ctx->rings); in io_allocate_scq_urings()
3794 ctx->rings = NULL; in io_allocate_scq_urings()
3798 ctx->sq_sqes = io_mem_alloc(size); in io_allocate_scq_urings()
3799 if (!ctx->sq_sqes) { in io_allocate_scq_urings()
3800 io_mem_free(ctx->rings); in io_allocate_scq_urings()
3801 ctx->rings = NULL; in io_allocate_scq_urings()
3813 static int io_uring_get_fd(struct io_ring_ctx *ctx) in io_uring_get_fd() argument
3822 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx, in io_uring_get_fd()
3836 struct io_ring_ctx *ctx; in io_uring_create() local
3864 ctx = io_ring_ctx_alloc(p); in io_uring_create()
3865 if (!ctx) { in io_uring_create()
3872 ctx->compat = in_compat_syscall(); in io_uring_create()
3873 ctx->account_mem = account_mem; in io_uring_create()
3874 ctx->user = user; in io_uring_create()
3876 ctx->creds = get_current_cred(); in io_uring_create()
3877 if (!ctx->creds) { in io_uring_create()
3882 ret = io_allocate_scq_urings(ctx, p); in io_uring_create()
3886 ret = io_sq_offload_start(ctx, p); in io_uring_create()
3897 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; in io_uring_create()
3911 ret = io_uring_get_fd(ctx); in io_uring_create()
3918 io_ring_ctx_wait_and_kill(ctx); in io_uring_create()
3960 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, in __io_uring_register() argument
3962 __releases(ctx->uring_lock) in __io_uring_register()
3963 __acquires(ctx->uring_lock) in __io_uring_register()
3972 if (percpu_ref_is_dying(&ctx->refs)) in __io_uring_register()
3975 percpu_ref_kill(&ctx->refs); in __io_uring_register()
3984 mutex_unlock(&ctx->uring_lock); in __io_uring_register()
3985 wait_for_completion(&ctx->ctx_done); in __io_uring_register()
3986 mutex_lock(&ctx->uring_lock); in __io_uring_register()
3990 ret = io_sqe_buffer_register(ctx, arg, nr_args); in __io_uring_register()
3996 ret = io_sqe_buffer_unregister(ctx); in __io_uring_register()
3999 ret = io_sqe_files_register(ctx, arg, nr_args); in __io_uring_register()
4005 ret = io_sqe_files_unregister(ctx); in __io_uring_register()
4011 ret = io_eventfd_register(ctx, arg); in __io_uring_register()
4017 ret = io_eventfd_unregister(ctx); in __io_uring_register()
4025 reinit_completion(&ctx->ctx_done); in __io_uring_register()
4026 percpu_ref_reinit(&ctx->refs); in __io_uring_register()
4033 struct io_ring_ctx *ctx; in SYSCALL_DEFINE4() local
4045 ctx = f.file->private_data; in SYSCALL_DEFINE4()
4047 mutex_lock(&ctx->uring_lock); in SYSCALL_DEFINE4()
4048 ret = __io_uring_register(ctx, opcode, arg, nr_args); in SYSCALL_DEFINE4()
4049 mutex_unlock(&ctx->uring_lock); in SYSCALL_DEFINE4()