Lines Matching refs:req
563 struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw); in kiocb_set_cancel_fn() local
564 struct kioctx *ctx = req->ki_ctx; in kiocb_set_cancel_fn()
567 if (WARN_ON_ONCE(!list_empty(&req->ki_list))) in kiocb_set_cancel_fn()
571 list_add_tail(&req->ki_list, &ctx->active_reqs); in kiocb_set_cancel_fn()
572 req->ki_cancel = cancel; in kiocb_set_cancel_fn()
616 struct aio_kiocb *req; in free_ioctx_users() local
621 req = list_first_entry(&ctx->active_reqs, in free_ioctx_users()
623 req->ki_cancel(&req->rw); in free_ioctx_users()
624 list_del_init(&req->ki_list); in free_ioctx_users()
1028 struct aio_kiocb *req; in aio_get_req() local
1030 req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL); in aio_get_req()
1031 if (unlikely(!req)) in aio_get_req()
1035 kmem_cache_free(kiocb_cachep, req); in aio_get_req()
1040 req->ki_ctx = ctx; in aio_get_req()
1041 INIT_LIST_HEAD(&req->ki_list); in aio_get_req()
1042 refcount_set(&req->ki_refcnt, 2); in aio_get_req()
1043 req->ki_eventfd = NULL; in aio_get_req()
1044 return req; in aio_get_req()
1445 static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) in aio_prep_rw() argument
1449 req->ki_complete = aio_complete_rw; in aio_prep_rw()
1450 req->private = NULL; in aio_prep_rw()
1451 req->ki_pos = iocb->aio_offset; in aio_prep_rw()
1452 req->ki_flags = iocb_flags(req->ki_filp); in aio_prep_rw()
1454 req->ki_flags |= IOCB_EVENTFD; in aio_prep_rw()
1455 req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp)); in aio_prep_rw()
1468 req->ki_ioprio = iocb->aio_reqprio; in aio_prep_rw()
1470 req->ki_ioprio = get_current_ioprio(); in aio_prep_rw()
1472 ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); in aio_prep_rw()
1476 req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ in aio_prep_rw()
1500 static inline void aio_rw_done(struct kiocb *req, ssize_t ret) in aio_rw_done() argument
1516 req->ki_complete(req, ret, 0); in aio_rw_done()
1520 static int aio_read(struct kiocb *req, const struct iocb *iocb, in aio_read() argument
1528 ret = aio_prep_rw(req, iocb); in aio_read()
1531 file = req->ki_filp; in aio_read()
1541 ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); in aio_read()
1543 aio_rw_done(req, call_read_iter(file, req, &iter)); in aio_read()
1548 static int aio_write(struct kiocb *req, const struct iocb *iocb, in aio_write() argument
1556 ret = aio_prep_rw(req, iocb); in aio_write()
1559 file = req->ki_filp; in aio_write()
1569 ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); in aio_write()
1582 req->ki_flags |= IOCB_WRITE; in aio_write()
1583 aio_rw_done(req, call_write_iter(file, req, &iter)); in aio_write()
1597 static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, in aio_fsync() argument
1604 if (unlikely(!req->file->f_op->fsync)) in aio_fsync()
1607 req->datasync = datasync; in aio_fsync()
1608 INIT_WORK(&req->work, aio_fsync_work); in aio_fsync()
1609 schedule_work(&req->work); in aio_fsync()
1615 struct poll_iocb *req = container_of(work, struct poll_iocb, work); in aio_poll_complete_work() local
1616 struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); in aio_poll_complete_work()
1617 struct poll_table_struct pt = { ._key = req->events }; in aio_poll_complete_work()
1621 if (!READ_ONCE(req->cancelled)) in aio_poll_complete_work()
1622 mask = vfs_poll(req->file, &pt) & req->events; in aio_poll_complete_work()
1632 if (!mask && !READ_ONCE(req->cancelled)) { in aio_poll_complete_work()
1633 add_wait_queue(req->head, &req->wait); in aio_poll_complete_work()
1639 req->done = true; in aio_poll_complete_work()
1649 struct poll_iocb *req = &aiocb->poll; in aio_poll_cancel() local
1651 spin_lock(&req->head->lock); in aio_poll_cancel()
1652 WRITE_ONCE(req->cancelled, true); in aio_poll_cancel()
1653 if (!list_empty(&req->wait.entry)) { in aio_poll_cancel()
1654 list_del_init(&req->wait.entry); in aio_poll_cancel()
1657 spin_unlock(&req->head->lock); in aio_poll_cancel()
1665 struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); in aio_poll_wake() local
1666 struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); in aio_poll_wake()
1671 if (mask && !(mask & req->events)) in aio_poll_wake()
1674 list_del_init(&req->wait.entry); in aio_poll_wake()
1685 req->done = true; in aio_poll_wake()
1689 schedule_work(&req->work); in aio_poll_wake()
1720 struct poll_iocb *req = &aiocb->poll; in aio_poll() local
1732 INIT_WORK(&req->work, aio_poll_complete_work); in aio_poll()
1733 req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; in aio_poll()
1735 req->head = NULL; in aio_poll()
1736 req->done = false; in aio_poll()
1737 req->cancelled = false; in aio_poll()
1740 apt.pt._key = req->events; in aio_poll()
1745 INIT_LIST_HEAD(&req->wait.entry); in aio_poll()
1746 init_waitqueue_func_entry(&req->wait, aio_poll_wake); in aio_poll()
1748 mask = vfs_poll(req->file, &apt.pt) & req->events; in aio_poll()
1750 if (likely(req->head)) { in aio_poll()
1751 spin_lock(&req->head->lock); in aio_poll()
1752 if (unlikely(list_empty(&req->wait.entry))) { in aio_poll()
1759 list_del_init(&req->wait.entry); in aio_poll()
1761 WRITE_ONCE(req->cancelled, true); in aio_poll()
1762 } else if (!req->done) { /* actually waiting for an event */ in aio_poll()
1766 spin_unlock(&req->head->lock); in aio_poll()
1779 struct iocb __user *user_iocb, struct aio_kiocb *req, in __io_submit_one() argument
1782 req->ki_filp = fget(iocb->aio_fildes); in __io_submit_one()
1783 if (unlikely(!req->ki_filp)) in __io_submit_one()
1798 req->ki_eventfd = eventfd; in __io_submit_one()
1806 req->ki_res.obj = (u64)(unsigned long)user_iocb; in __io_submit_one()
1807 req->ki_res.data = iocb->aio_data; in __io_submit_one()
1808 req->ki_res.res = 0; in __io_submit_one()
1809 req->ki_res.res2 = 0; in __io_submit_one()
1813 return aio_read(&req->rw, iocb, false, compat); in __io_submit_one()
1815 return aio_write(&req->rw, iocb, false, compat); in __io_submit_one()
1817 return aio_read(&req->rw, iocb, true, compat); in __io_submit_one()
1819 return aio_write(&req->rw, iocb, true, compat); in __io_submit_one()
1821 return aio_fsync(&req->fsync, iocb, false); in __io_submit_one()
1823 return aio_fsync(&req->fsync, iocb, true); in __io_submit_one()
1825 return aio_poll(req, iocb); in __io_submit_one()
1835 struct aio_kiocb *req; in io_submit_one() local
1858 req = aio_get_req(ctx); in io_submit_one()
1859 if (unlikely(!req)) in io_submit_one()
1862 err = __io_submit_one(ctx, &iocb, user_iocb, req, compat); in io_submit_one()
1865 iocb_put(req); in io_submit_one()
1873 iocb_destroy(req); in io_submit_one()