Lines Matching refs:sqe
2964 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, in io_prep_rw() argument
2976 kiocb->ki_pos = READ_ONCE(sqe->off); in io_prep_rw()
2979 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); in io_prep_rw()
2992 ioprio = READ_ONCE(sqe->ioprio); in io_prep_rw()
3017 req->buf_index = READ_ONCE(sqe->buf_index); in io_prep_rw()
3032 req->rw.addr = READ_ONCE(sqe->addr); in io_prep_rw()
3033 req->rw.len = READ_ONCE(sqe->len); in io_prep_rw()
3508 static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_read_prep() argument
3512 return io_prep_rw(req, sqe, READ); in io_read_prep()
3734 static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_write_prep() argument
3738 return io_prep_rw(req, sqe, WRITE); in io_write_prep()
3847 const struct io_uring_sqe *sqe) in io_renameat_prep() argument
3854 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) in io_renameat_prep()
3859 ren->old_dfd = READ_ONCE(sqe->fd); in io_renameat_prep()
3860 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_renameat_prep()
3861 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2)); in io_renameat_prep()
3862 ren->new_dfd = READ_ONCE(sqe->len); in io_renameat_prep()
3863 ren->flags = READ_ONCE(sqe->rename_flags); in io_renameat_prep()
3898 const struct io_uring_sqe *sqe) in io_unlinkat_prep() argument
3905 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index || in io_unlinkat_prep()
3906 sqe->splice_fd_in) in io_unlinkat_prep()
3911 un->dfd = READ_ONCE(sqe->fd); in io_unlinkat_prep()
3913 un->flags = READ_ONCE(sqe->unlink_flags); in io_unlinkat_prep()
3917 fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_unlinkat_prep()
3947 const struct io_uring_sqe *sqe) in io_mkdirat_prep() argument
3954 if (sqe->ioprio || sqe->off || sqe->rw_flags || sqe->buf_index || in io_mkdirat_prep()
3955 sqe->splice_fd_in) in io_mkdirat_prep()
3960 mkd->dfd = READ_ONCE(sqe->fd); in io_mkdirat_prep()
3961 mkd->mode = READ_ONCE(sqe->len); in io_mkdirat_prep()
3963 fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_mkdirat_prep()
3990 const struct io_uring_sqe *sqe) in io_symlinkat_prep() argument
3997 if (sqe->ioprio || sqe->len || sqe->rw_flags || sqe->buf_index || in io_symlinkat_prep()
3998 sqe->splice_fd_in) in io_symlinkat_prep()
4003 sl->new_dfd = READ_ONCE(sqe->fd); in io_symlinkat_prep()
4004 oldpath = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_symlinkat_prep()
4005 newpath = u64_to_user_ptr(READ_ONCE(sqe->addr2)); in io_symlinkat_prep()
4039 const struct io_uring_sqe *sqe) in io_linkat_prep() argument
4046 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) in io_linkat_prep()
4051 lnk->old_dfd = READ_ONCE(sqe->fd); in io_linkat_prep()
4052 lnk->new_dfd = READ_ONCE(sqe->len); in io_linkat_prep()
4053 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_linkat_prep()
4054 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2)); in io_linkat_prep()
4055 lnk->flags = READ_ONCE(sqe->hardlink_flags); in io_linkat_prep()
4090 const struct io_uring_sqe *sqe) in io_shutdown_prep() argument
4095 if (unlikely(sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags || in io_shutdown_prep()
4096 sqe->buf_index || sqe->splice_fd_in)) in io_shutdown_prep()
4099 req->shutdown.how = READ_ONCE(sqe->len); in io_shutdown_prep()
4130 const struct io_uring_sqe *sqe) in __io_splice_prep() argument
4138 sp->len = READ_ONCE(sqe->len); in __io_splice_prep()
4139 sp->flags = READ_ONCE(sqe->splice_flags); in __io_splice_prep()
4142 sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); in __io_splice_prep()
4147 const struct io_uring_sqe *sqe) in io_tee_prep() argument
4149 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) in io_tee_prep()
4151 return __io_splice_prep(req, sqe); in io_tee_prep()
4184 static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_splice_prep() argument
4188 sp->off_in = READ_ONCE(sqe->splice_off_in); in io_splice_prep()
4189 sp->off_out = READ_ONCE(sqe->off); in io_splice_prep()
4190 return __io_splice_prep(req, sqe); in io_splice_prep()
4241 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_fsync_prep() argument
4247 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || in io_fsync_prep()
4248 sqe->splice_fd_in)) in io_fsync_prep()
4251 req->sync.flags = READ_ONCE(sqe->fsync_flags); in io_fsync_prep()
4255 req->sync.off = READ_ONCE(sqe->off); in io_fsync_prep()
4256 req->sync.len = READ_ONCE(sqe->len); in io_fsync_prep()
4279 const struct io_uring_sqe *sqe) in io_fallocate_prep() argument
4281 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags || in io_fallocate_prep()
4282 sqe->splice_fd_in) in io_fallocate_prep()
4287 req->sync.off = READ_ONCE(sqe->off); in io_fallocate_prep()
4288 req->sync.len = READ_ONCE(sqe->addr); in io_fallocate_prep()
4289 req->sync.mode = READ_ONCE(sqe->len); in io_fallocate_prep()
4310 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in __io_openat_prep() argument
4317 if (unlikely(sqe->ioprio || sqe->buf_index)) in __io_openat_prep()
4326 req->open.dfd = READ_ONCE(sqe->fd); in __io_openat_prep()
4327 fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); in __io_openat_prep()
4335 req->open.file_slot = READ_ONCE(sqe->file_index); in __io_openat_prep()
4344 static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_openat_prep() argument
4346 u64 mode = READ_ONCE(sqe->len); in io_openat_prep()
4347 u64 flags = READ_ONCE(sqe->open_flags); in io_openat_prep()
4350 return __io_openat_prep(req, sqe); in io_openat_prep()
4353 static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_openat2_prep() argument
4359 how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); in io_openat2_prep()
4360 len = READ_ONCE(sqe->len); in io_openat2_prep()
4369 return __io_openat_prep(req, sqe); in io_openat2_prep()
4446 const struct io_uring_sqe *sqe) in io_remove_buffers_prep() argument
4451 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off || in io_remove_buffers_prep()
4452 sqe->splice_fd_in) in io_remove_buffers_prep()
4455 tmp = READ_ONCE(sqe->fd); in io_remove_buffers_prep()
4461 p->bgid = READ_ONCE(sqe->buf_group); in io_remove_buffers_prep()
4518 const struct io_uring_sqe *sqe) in io_provide_buffers_prep() argument
4524 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) in io_provide_buffers_prep()
4527 tmp = READ_ONCE(sqe->fd); in io_provide_buffers_prep()
4531 p->addr = READ_ONCE(sqe->addr); in io_provide_buffers_prep()
4532 p->len = READ_ONCE(sqe->len); in io_provide_buffers_prep()
4544 p->bgid = READ_ONCE(sqe->buf_group); in io_provide_buffers_prep()
4545 tmp = READ_ONCE(sqe->off); in io_provide_buffers_prep()
4610 const struct io_uring_sqe *sqe) in io_epoll_ctl_prep() argument
4613 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) in io_epoll_ctl_prep()
4618 req->epoll.epfd = READ_ONCE(sqe->fd); in io_epoll_ctl_prep()
4619 req->epoll.op = READ_ONCE(sqe->len); in io_epoll_ctl_prep()
4620 req->epoll.fd = READ_ONCE(sqe->off); in io_epoll_ctl_prep()
4625 ev = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_epoll_ctl_prep()
4656 static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_madvise_prep() argument
4659 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in) in io_madvise_prep()
4664 req->madvise.addr = READ_ONCE(sqe->addr); in io_madvise_prep()
4665 req->madvise.len = READ_ONCE(sqe->len); in io_madvise_prep()
4666 req->madvise.advice = READ_ONCE(sqe->fadvise_advice); in io_madvise_prep()
4692 static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_fadvise_prep() argument
4694 if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in) in io_fadvise_prep()
4699 req->fadvise.offset = READ_ONCE(sqe->off); in io_fadvise_prep()
4700 req->fadvise.len = READ_ONCE(sqe->len); in io_fadvise_prep()
4701 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice); in io_fadvise_prep()
4728 static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_statx_prep() argument
4732 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) in io_statx_prep()
4737 req->statx.dfd = READ_ONCE(sqe->fd); in io_statx_prep()
4738 req->statx.mask = READ_ONCE(sqe->len); in io_statx_prep()
4739 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_statx_prep()
4740 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); in io_statx_prep()
4741 req->statx.flags = READ_ONCE(sqe->statx_flags); in io_statx_prep()
4763 static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_close_prep() argument
4767 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || in io_close_prep()
4768 sqe->rw_flags || sqe->buf_index) in io_close_prep()
4773 req->close.fd = READ_ONCE(sqe->fd); in io_close_prep()
4774 req->close.file_slot = READ_ONCE(sqe->file_index); in io_close_prep()
4832 static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_sfr_prep() argument
4838 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || in io_sfr_prep()
4839 sqe->splice_fd_in)) in io_sfr_prep()
4842 req->sync.off = READ_ONCE(sqe->off); in io_sfr_prep()
4843 req->sync.len = READ_ONCE(sqe->len); in io_sfr_prep()
4844 req->sync.flags = READ_ONCE(sqe->sync_range_flags); in io_sfr_prep()
4922 static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_sendmsg_prep() argument
4928 if (unlikely(sqe->addr2 || sqe->file_index)) in io_sendmsg_prep()
4930 if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio)) in io_sendmsg_prep()
4933 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_sendmsg_prep()
4934 sr->len = READ_ONCE(sqe->len); in io_sendmsg_prep()
4935 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; in io_sendmsg_prep()
5174 static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_recvmsg_prep() argument
5180 if (unlikely(sqe->addr2 || sqe->file_index)) in io_recvmsg_prep()
5182 if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio)) in io_recvmsg_prep()
5185 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_recvmsg_prep()
5186 sr->len = READ_ONCE(sqe->len); in io_recvmsg_prep()
5187 sr->bgid = READ_ONCE(sqe->buf_group); in io_recvmsg_prep()
5188 sr->msg_flags = READ_ONCE(sqe->msg_flags); in io_recvmsg_prep()
5339 static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_accept_prep() argument
5345 if (sqe->ioprio || sqe->len || sqe->buf_index) in io_accept_prep()
5348 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_accept_prep()
5349 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); in io_accept_prep()
5350 accept->flags = READ_ONCE(sqe->accept_flags); in io_accept_prep()
5353 accept->file_slot = READ_ONCE(sqe->file_index); in io_accept_prep()
5409 static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_connect_prep() argument
5415 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags || in io_connect_prep()
5416 sqe->splice_fd_in) in io_connect_prep()
5419 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_connect_prep()
5420 conn->addr_len = READ_ONCE(sqe->addr2); in io_connect_prep()
5473 static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
6070 static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, in io_poll_parse_events() argument
6075 events = READ_ONCE(sqe->poll32_events); in io_poll_parse_events()
6085 const struct io_uring_sqe *sqe) in io_poll_update_prep() argument
6092 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) in io_poll_update_prep()
6094 flags = READ_ONCE(sqe->len); in io_poll_update_prep()
6102 upd->old_user_data = READ_ONCE(sqe->addr); in io_poll_update_prep()
6106 upd->new_user_data = READ_ONCE(sqe->off); in io_poll_update_prep()
6110 upd->events = io_poll_parse_events(sqe, flags); in io_poll_update_prep()
6111 else if (sqe->poll32_events) in io_poll_update_prep()
6117 static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_poll_add_prep() argument
6124 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr) in io_poll_add_prep()
6126 flags = READ_ONCE(sqe->len); in io_poll_add_prep()
6131 poll->events = io_poll_parse_events(sqe, flags); in io_poll_add_prep()
6319 const struct io_uring_sqe *sqe) in io_timeout_remove_prep() argument
6327 if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in) in io_timeout_remove_prep()
6331 tr->addr = READ_ONCE(sqe->addr); in io_timeout_remove_prep()
6332 tr->flags = READ_ONCE(sqe->timeout_flags); in io_timeout_remove_prep()
6340 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) in io_timeout_remove_prep()
6388 static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, in io_timeout_prep() argument
6393 u32 off = READ_ONCE(sqe->off); in io_timeout_prep()
6397 if (sqe->ioprio || sqe->buf_index || sqe->len != 1 || in io_timeout_prep()
6398 sqe->splice_fd_in) in io_timeout_prep()
6402 flags = READ_ONCE(sqe->timeout_flags); in io_timeout_prep()
6421 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) in io_timeout_prep()
6554 const struct io_uring_sqe *sqe) in io_async_cancel_prep() argument
6560 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags || in io_async_cancel_prep()
6561 sqe->splice_fd_in) in io_async_cancel_prep()
6564 req->cancel.addr = READ_ONCE(sqe->addr); in io_async_cancel_prep()
6598 const struct io_uring_sqe *sqe) in io_rsrc_update_prep() argument
6602 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) in io_rsrc_update_prep()
6605 req->rsrc_update.offset = READ_ONCE(sqe->off); in io_rsrc_update_prep()
6606 req->rsrc_update.nr_args = READ_ONCE(sqe->len); in io_rsrc_update_prep()
6609 req->rsrc_update.arg = READ_ONCE(sqe->addr); in io_rsrc_update_prep()
6637 static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_req_prep() argument
6645 return io_read_prep(req, sqe); in io_req_prep()
6649 return io_write_prep(req, sqe); in io_req_prep()
6651 return io_poll_add_prep(req, sqe); in io_req_prep()
6653 return io_poll_update_prep(req, sqe); in io_req_prep()
6655 return io_fsync_prep(req, sqe); in io_req_prep()
6657 return io_sfr_prep(req, sqe); in io_req_prep()
6660 return io_sendmsg_prep(req, sqe); in io_req_prep()
6663 return io_recvmsg_prep(req, sqe); in io_req_prep()
6665 return io_connect_prep(req, sqe); in io_req_prep()
6667 return io_timeout_prep(req, sqe, false); in io_req_prep()
6669 return io_timeout_remove_prep(req, sqe); in io_req_prep()
6671 return io_async_cancel_prep(req, sqe); in io_req_prep()
6673 return io_timeout_prep(req, sqe, true); in io_req_prep()
6675 return io_accept_prep(req, sqe); in io_req_prep()
6677 return io_fallocate_prep(req, sqe); in io_req_prep()
6679 return io_openat_prep(req, sqe); in io_req_prep()
6681 return io_close_prep(req, sqe); in io_req_prep()
6683 return io_rsrc_update_prep(req, sqe); in io_req_prep()
6685 return io_statx_prep(req, sqe); in io_req_prep()
6687 return io_fadvise_prep(req, sqe); in io_req_prep()
6689 return io_madvise_prep(req, sqe); in io_req_prep()
6691 return io_openat2_prep(req, sqe); in io_req_prep()
6693 return io_epoll_ctl_prep(req, sqe); in io_req_prep()
6695 return io_splice_prep(req, sqe); in io_req_prep()
6697 return io_provide_buffers_prep(req, sqe); in io_req_prep()
6699 return io_remove_buffers_prep(req, sqe); in io_req_prep()
6701 return io_tee_prep(req, sqe); in io_req_prep()
6703 return io_shutdown_prep(req, sqe); in io_req_prep()
6705 return io_renameat_prep(req, sqe); in io_req_prep()
6707 return io_unlinkat_prep(req, sqe); in io_req_prep()
6709 return io_mkdirat_prep(req, sqe); in io_req_prep()
6711 return io_symlinkat_prep(req, sqe); in io_req_prep()
6713 return io_linkat_prep(req, sqe); in io_req_prep()
7338 const struct io_uring_sqe *sqe) in io_init_req() argument
7346 req->opcode = READ_ONCE(sqe->opcode); in io_init_req()
7348 req->flags = sqe_flags = READ_ONCE(sqe->flags); in io_init_req()
7349 req->user_data = READ_ONCE(sqe->user_data); in io_init_req()
7368 personality = READ_ONCE(sqe->personality); in io_init_req()
7389 req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd), in io_init_req()
7401 const struct io_uring_sqe *sqe) in io_submit_sqe() argument
7407 ret = io_init_req(ctx, req, sqe); in io_submit_sqe()
7433 ret = io_req_prep(req, sqe); in io_submit_sqe()
7567 const struct io_uring_sqe *sqe; in io_submit_sqes() local
7576 sqe = io_get_sqe(ctx); in io_submit_sqes()
7577 if (unlikely(!sqe)) { in io_submit_sqes()
7583 if (io_submit_sqe(ctx, req, sqe)) in io_submit_sqes()