36 void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
40 int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
41 int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);
43 int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
44 int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);
49 unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
51 void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
53 static inline void io_kbuf_recycle_ring(struct io_kiocb *req)  in io_kbuf_recycle_ring()  argument
62 	if (req->buf_list) {  in io_kbuf_recycle_ring()
63 		if (req->flags & REQ_F_PARTIAL_IO) {  in io_kbuf_recycle_ring()
71 			req->buf_list->head++;  in io_kbuf_recycle_ring()
72 			req->buf_list = NULL;  in io_kbuf_recycle_ring()
74 			req->buf_index = req->buf_list->bgid;  in io_kbuf_recycle_ring()
75 			req->flags &= ~REQ_F_BUFFER_RING;  in io_kbuf_recycle_ring()
80 static inline bool io_do_buffer_select(struct io_kiocb *req)  in io_do_buffer_select()  argument
82 	if (!(req->flags & REQ_F_BUFFER_SELECT))  in io_do_buffer_select()
84 	return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));  in io_do_buffer_select()
87 static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)  in io_kbuf_recycle()  argument
89 	if (req->flags & REQ_F_BUFFER_SELECTED)  in io_kbuf_recycle()
90 		io_kbuf_recycle_legacy(req, issue_flags);  in io_kbuf_recycle()
91 	if (req->flags & REQ_F_BUFFER_RING)  in io_kbuf_recycle()
92 		io_kbuf_recycle_ring(req);  in io_kbuf_recycle()
95 static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req,  in __io_put_kbuf_list()  argument
98 	unsigned int ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);  in __io_put_kbuf_list()
100 	if (req->flags & REQ_F_BUFFER_RING) {  in __io_put_kbuf_list()
101 		if (req->buf_list) {  in __io_put_kbuf_list()
102 			req->buf_index = req->buf_list->bgid;  in __io_put_kbuf_list()
103 			req->buf_list->head++;  in __io_put_kbuf_list()
105 		req->flags &= ~REQ_F_BUFFER_RING;  in __io_put_kbuf_list()
107 		req->buf_index = req->kbuf->bgid;  in __io_put_kbuf_list()
108 		list_add(&req->kbuf->list, list);  in __io_put_kbuf_list()
109 		req->flags &= ~REQ_F_BUFFER_SELECTED;  in __io_put_kbuf_list()
115 static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)  in io_put_kbuf_comp()  argument
117 	lockdep_assert_held(&req->ctx->completion_lock);  in io_put_kbuf_comp()
119 	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))  in io_put_kbuf_comp()
121 	return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp);  in io_put_kbuf_comp()
124 static inline unsigned int io_put_kbuf(struct io_kiocb *req,  in io_put_kbuf()  argument
128 	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))  in io_put_kbuf()
130 	return __io_put_kbuf(req, issue_flags);  in io_put_kbuf()