Lines Matching +full:rpc +full:- +full:if
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
9 * COPYING in the main directory of this source tree, or the BSD-type
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * This file contains the guts of the RPC RDMA protocol, and
47 * to the Linux RPC framework lives.
57 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
61 /* Returns size of largest RPC-over-RDMA header in a Call message
63 * The largest Call header contains a full-size Read list and a
84 /* Returns size of largest RPC-over-RDMA header in a Reply message
105 * rpcrdma_set_max_header_sizes - Initialize inline payload sizes
108 * The max_inline fields contain the maximum size of an RPC message
110 * for every RPC.
114 unsigned int maxsegs = ep->re_max_rdma_segs; in rpcrdma_set_max_header_sizes()
116 ep->re_max_inline_send = in rpcrdma_set_max_header_sizes()
117 ep->re_inline_send - rpcrdma_max_call_header_size(maxsegs); in rpcrdma_set_max_header_sizes()
118 ep->re_max_inline_recv = in rpcrdma_set_max_header_sizes()
119 ep->re_inline_recv - rpcrdma_max_reply_header_size(maxsegs); in rpcrdma_set_max_header_sizes()
123 * plus the RPC call fit under the transport's inline limit. If the
127 * A Read chunk is also required if sending the RPC call inline would
133 struct xdr_buf *xdr = &rqst->rq_snd_buf; in rpcrdma_args_inline()
134 struct rpcrdma_ep *ep = r_xprt->rx_ep; in rpcrdma_args_inline()
137 if (xdr->len > ep->re_max_inline_send) in rpcrdma_args_inline()
140 if (xdr->page_len) { in rpcrdma_args_inline()
141 remaining = xdr->page_len; in rpcrdma_args_inline()
142 offset = offset_in_page(xdr->page_base); in rpcrdma_args_inline()
145 remaining -= min_t(unsigned int, in rpcrdma_args_inline()
146 PAGE_SIZE - offset, remaining); in rpcrdma_args_inline()
148 if (++count > ep->re_attr.cap.max_send_sge) in rpcrdma_args_inline()
158 * operation. If the maximum combined reply message size exceeds that
165 return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_results_inline()
168 /* The client is required to provide a Reply chunk if the maximum
169 * size of the non-payload part of the RPC Reply is larger than
176 const struct xdr_buf *buf = &rqst->rq_rcv_buf; in rpcrdma_nonpayload_inline()
178 return (buf->head[0].iov_len + buf->tail[0].iov_len) < in rpcrdma_nonpayload_inline()
179 r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_nonpayload_inline()
193 len = buf->page_len; in rpcrdma_alloc_sparse_pages()
194 ppages = buf->pages + (buf->page_base >> PAGE_SHIFT); in rpcrdma_alloc_sparse_pages()
196 if (!*ppages) in rpcrdma_alloc_sparse_pages()
198 if (!*ppages) in rpcrdma_alloc_sparse_pages()
199 return -ENOBUFS; in rpcrdma_alloc_sparse_pages()
201 len -= PAGE_SIZE; in rpcrdma_alloc_sparse_pages()
221 base = vec->iov_base; in rpcrdma_convert_kvec()
223 remaining = vec->iov_len; in rpcrdma_convert_kvec()
225 seg->mr_page = NULL; in rpcrdma_convert_kvec()
226 seg->mr_offset = base; in rpcrdma_convert_kvec()
227 seg->mr_len = min_t(u32, PAGE_SIZE - page_offset, remaining); in rpcrdma_convert_kvec()
228 remaining -= seg->mr_len; in rpcrdma_convert_kvec()
229 base += seg->mr_len; in rpcrdma_convert_kvec()
254 if (pos == 0) in rpcrdma_convert_iovs()
255 seg = rpcrdma_convert_kvec(&xdrbuf->head[0], seg, &n); in rpcrdma_convert_iovs()
257 len = xdrbuf->page_len; in rpcrdma_convert_iovs()
258 ppages = xdrbuf->pages + (xdrbuf->page_base >> PAGE_SHIFT); in rpcrdma_convert_iovs()
259 page_base = offset_in_page(xdrbuf->page_base); in rpcrdma_convert_iovs()
261 seg->mr_page = *ppages; in rpcrdma_convert_iovs()
262 seg->mr_offset = (char *)page_base; in rpcrdma_convert_iovs()
263 seg->mr_len = min_t(u32, PAGE_SIZE - page_base, len); in rpcrdma_convert_iovs()
264 len -= seg->mr_len; in rpcrdma_convert_iovs()
274 if (type == rpcrdma_readch && r_xprt->rx_ep->re_implicit_roundup) in rpcrdma_convert_iovs()
278 * extra segment for non-XDR-aligned Write chunks. The upper in rpcrdma_convert_iovs()
282 if (type == rpcrdma_writech && r_xprt->rx_ep->re_implicit_roundup) in rpcrdma_convert_iovs()
285 if (xdrbuf->tail[0].iov_len) in rpcrdma_convert_iovs()
286 seg = rpcrdma_convert_kvec(&xdrbuf->tail[0], seg, &n); in rpcrdma_convert_iovs()
289 if (unlikely(n > RPCRDMA_MAX_SEGS)) in rpcrdma_convert_iovs()
290 return -EIO; in rpcrdma_convert_iovs()
300 if (unlikely(!p)) in encode_rdma_segment()
301 return -EMSGSIZE; in encode_rdma_segment()
303 xdr_encode_rdma_segment(p, mr->mr_handle, mr->mr_length, mr->mr_offset); in encode_rdma_segment()
314 if (unlikely(!p)) in encode_read_segment()
315 return -EMSGSIZE; in encode_read_segment()
318 xdr_encode_read_segment(p, position, mr->mr_handle, mr->mr_length, in encode_read_segment()
319 mr->mr_offset); in encode_read_segment()
329 *mr = rpcrdma_mr_pop(&req->rl_free_mrs); in rpcrdma_mr_prepare()
330 if (!*mr) { in rpcrdma_mr_prepare()
332 if (!*mr) in rpcrdma_mr_prepare()
335 (*mr)->mr_req = req; in rpcrdma_mr_prepare()
338 rpcrdma_mr_push(*mr, &req->rl_registered); in rpcrdma_mr_prepare()
339 return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr); in rpcrdma_mr_prepare()
343 xprt_wait_for_buffer_space(&r_xprt->rx_xprt); in rpcrdma_mr_prepare()
345 return ERR_PTR(-EAGAIN); in rpcrdma_mr_prepare()
351 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
355 * 1 - PHLOO - 1 - PHLOO - ... - 1 - PHLOO - 0
357 * Returns zero on success, or a negative errno if a failure occurred.
367 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_read_list()
373 if (rtype == rpcrdma_noch_pullup || rtype == rpcrdma_noch_mapped) in rpcrdma_encode_read_list()
376 pos = rqst->rq_snd_buf.head[0].iov_len; in rpcrdma_encode_read_list()
377 if (rtype == rpcrdma_areadch) in rpcrdma_encode_read_list()
379 seg = req->rl_segments; in rpcrdma_encode_read_list()
380 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_snd_buf, pos, in rpcrdma_encode_read_list()
382 if (nsegs < 0) in rpcrdma_encode_read_list()
387 if (IS_ERR(seg)) in rpcrdma_encode_read_list()
390 if (encode_read_segment(xdr, mr, pos) < 0) in rpcrdma_encode_read_list()
391 return -EMSGSIZE; in rpcrdma_encode_read_list()
393 trace_xprtrdma_chunk_read(rqst->rq_task, pos, mr, nsegs); in rpcrdma_encode_read_list()
394 r_xprt->rx_stats.read_chunk_count++; in rpcrdma_encode_read_list()
395 nsegs -= mr->mr_nents; in rpcrdma_encode_read_list()
399 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_read_list()
400 return -EMSGSIZE; in rpcrdma_encode_read_list()
408 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
412 * 1 - N - HLOO - HLOO - ... - HLOO - 0
414 * Returns zero on success, or a negative errno if a failure occurred.
424 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_write_list()
430 if (wtype != rpcrdma_writech) in rpcrdma_encode_write_list()
433 seg = req->rl_segments; in rpcrdma_encode_write_list()
434 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, in rpcrdma_encode_write_list()
435 rqst->rq_rcv_buf.head[0].iov_len, in rpcrdma_encode_write_list()
437 if (nsegs < 0) in rpcrdma_encode_write_list()
440 if (xdr_stream_encode_item_present(xdr) < 0) in rpcrdma_encode_write_list()
441 return -EMSGSIZE; in rpcrdma_encode_write_list()
443 if (unlikely(!segcount)) in rpcrdma_encode_write_list()
444 return -EMSGSIZE; in rpcrdma_encode_write_list()
450 if (IS_ERR(seg)) in rpcrdma_encode_write_list()
453 if (encode_rdma_segment(xdr, mr) < 0) in rpcrdma_encode_write_list()
454 return -EMSGSIZE; in rpcrdma_encode_write_list()
456 trace_xprtrdma_chunk_write(rqst->rq_task, mr, nsegs); in rpcrdma_encode_write_list()
457 r_xprt->rx_stats.write_chunk_count++; in rpcrdma_encode_write_list()
458 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_write_list()
460 nsegs -= mr->mr_nents; in rpcrdma_encode_write_list()
467 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_write_list()
468 return -EMSGSIZE; in rpcrdma_encode_write_list()
475 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
479 * 1 - N - HLOO - HLOO - ... - HLOO
481 * Returns zero on success, or a negative errno if a failure occurred.
489 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_reply_chunk()
495 if (wtype != rpcrdma_replych) { in rpcrdma_encode_reply_chunk()
496 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_reply_chunk()
497 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
501 seg = req->rl_segments; in rpcrdma_encode_reply_chunk()
502 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 0, wtype, seg); in rpcrdma_encode_reply_chunk()
503 if (nsegs < 0) in rpcrdma_encode_reply_chunk()
506 if (xdr_stream_encode_item_present(xdr) < 0) in rpcrdma_encode_reply_chunk()
507 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
509 if (unlikely(!segcount)) in rpcrdma_encode_reply_chunk()
510 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
516 if (IS_ERR(seg)) in rpcrdma_encode_reply_chunk()
519 if (encode_rdma_segment(xdr, mr) < 0) in rpcrdma_encode_reply_chunk()
520 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
522 trace_xprtrdma_chunk_reply(rqst->rq_task, mr, nsegs); in rpcrdma_encode_reply_chunk()
523 r_xprt->rx_stats.reply_chunk_count++; in rpcrdma_encode_reply_chunk()
524 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_reply_chunk()
526 nsegs -= mr->mr_nents; in rpcrdma_encode_reply_chunk()
539 struct rpcrdma_rep *rep = req->rl_reply; in rpcrdma_sendctx_done()
542 rep->rr_rxprt->rx_stats.reply_waits_for_send++; in rpcrdma_sendctx_done()
546 * rpcrdma_sendctx_unmap - DMA-unmap Send buffer
552 struct rpcrdma_regbuf *rb = sc->sc_req->rl_sendbuf; in rpcrdma_sendctx_unmap()
555 if (!sc->sc_unmap_count) in rpcrdma_sendctx_unmap()
560 * they can be cheaply re-used. in rpcrdma_sendctx_unmap()
562 for (sge = &sc->sc_sges[2]; sc->sc_unmap_count; in rpcrdma_sendctx_unmap()
563 ++sge, --sc->sc_unmap_count) in rpcrdma_sendctx_unmap()
564 ib_dma_unmap_page(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_sendctx_unmap()
567 kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done); in rpcrdma_sendctx_unmap()
570 /* Prepare an SGE for the RPC-over-RDMA transport header.
575 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_hdr_sge()
576 struct rpcrdma_regbuf *rb = req->rl_rdmabuf; in rpcrdma_prepare_hdr_sge()
577 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_hdr_sge()
579 sge->addr = rdmab_addr(rb); in rpcrdma_prepare_hdr_sge()
580 sge->length = len; in rpcrdma_prepare_hdr_sge()
581 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_hdr_sge()
583 ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_prepare_hdr_sge()
588 * DMA-mapped. Sync the content that has changed.
593 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_head_iov()
594 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_head_iov()
595 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_head_iov()
597 if (!rpcrdma_regbuf_dma_map(r_xprt, rb)) in rpcrdma_prepare_head_iov()
600 sge->addr = rdmab_addr(rb); in rpcrdma_prepare_head_iov()
601 sge->length = len; in rpcrdma_prepare_head_iov()
602 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_head_iov()
604 ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_prepare_head_iov()
609 /* If there is a page list present, DMA map and prepare an
615 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_pagelist()
616 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_pagelist()
621 ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT); in rpcrdma_prepare_pagelist()
622 page_base = offset_in_page(xdr->page_base); in rpcrdma_prepare_pagelist()
623 remaining = xdr->page_len; in rpcrdma_prepare_pagelist()
625 sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_pagelist()
626 len = min_t(unsigned int, PAGE_SIZE - page_base, remaining); in rpcrdma_prepare_pagelist()
627 sge->addr = ib_dma_map_page(rdmab_device(rb), *ppages, in rpcrdma_prepare_pagelist()
629 if (ib_dma_mapping_error(rdmab_device(rb), sge->addr)) in rpcrdma_prepare_pagelist()
632 sge->length = len; in rpcrdma_prepare_pagelist()
633 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_pagelist()
635 sc->sc_unmap_count++; in rpcrdma_prepare_pagelist()
637 remaining -= len; in rpcrdma_prepare_pagelist()
644 trace_xprtrdma_dma_maperr(sge->addr); in rpcrdma_prepare_pagelist()
656 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_tail_iov()
657 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_tail_iov()
658 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_tail_iov()
659 struct page *page = virt_to_page(xdr->tail[0].iov_base); in rpcrdma_prepare_tail_iov()
661 sge->addr = ib_dma_map_page(rdmab_device(rb), page, page_base, len, in rpcrdma_prepare_tail_iov()
663 if (ib_dma_mapping_error(rdmab_device(rb), sge->addr)) in rpcrdma_prepare_tail_iov()
666 sge->length = len; in rpcrdma_prepare_tail_iov()
667 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_tail_iov()
668 ++sc->sc_unmap_count; in rpcrdma_prepare_tail_iov()
672 trace_xprtrdma_dma_maperr(sge->addr); in rpcrdma_prepare_tail_iov()
684 dst = (unsigned char *)xdr->head[0].iov_base; in rpcrdma_pullup_tail_iov()
685 dst += xdr->head[0].iov_len + xdr->page_len; in rpcrdma_pullup_tail_iov()
686 memmove(dst, xdr->tail[0].iov_base, xdr->tail[0].iov_len); in rpcrdma_pullup_tail_iov()
687 r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len; in rpcrdma_pullup_tail_iov()
700 dst = (unsigned char *)xdr->head[0].iov_base; in rpcrdma_pullup_pagelist()
701 dst += xdr->head[0].iov_len; in rpcrdma_pullup_pagelist()
702 ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT); in rpcrdma_pullup_pagelist()
703 page_base = offset_in_page(xdr->page_base); in rpcrdma_pullup_pagelist()
704 remaining = xdr->page_len; in rpcrdma_pullup_pagelist()
708 len = min_t(unsigned int, PAGE_SIZE - page_base, remaining); in rpcrdma_pullup_pagelist()
710 r_xprt->rx_stats.pullup_copy_count += len; in rpcrdma_pullup_pagelist()
714 remaining -= len; in rpcrdma_pullup_pagelist()
720 * When the head, pagelist, and tail are small, a pull-up copy
725 * - the caller has already verified that the total length
726 * of the RPC Call body will fit into @rl_sendbuf.
732 if (unlikely(xdr->tail[0].iov_len)) in rpcrdma_prepare_noch_pullup()
735 if (unlikely(xdr->page_len)) in rpcrdma_prepare_noch_pullup()
738 /* The whole RPC message resides in the head iovec now */ in rpcrdma_prepare_noch_pullup()
739 return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len); in rpcrdma_prepare_noch_pullup()
746 struct kvec *tail = &xdr->tail[0]; in rpcrdma_prepare_noch_mapped()
748 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_noch_mapped()
750 if (xdr->page_len) in rpcrdma_prepare_noch_mapped()
751 if (!rpcrdma_prepare_pagelist(req, xdr)) in rpcrdma_prepare_noch_mapped()
753 if (tail->iov_len) in rpcrdma_prepare_noch_mapped()
754 if (!rpcrdma_prepare_tail_iov(req, xdr, in rpcrdma_prepare_noch_mapped()
755 offset_in_page(tail->iov_base), in rpcrdma_prepare_noch_mapped()
756 tail->iov_len)) in rpcrdma_prepare_noch_mapped()
759 if (req->rl_sendctx->sc_unmap_count) in rpcrdma_prepare_noch_mapped()
760 kref_get(&req->rl_kref); in rpcrdma_prepare_noch_mapped()
768 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_readch()
771 /* If there is a Read chunk, the page list is being handled in rpcrdma_prepare_readch()
775 /* Do not include the tail if it is only an XDR pad */ in rpcrdma_prepare_readch()
776 if (xdr->tail[0].iov_len > 3) { in rpcrdma_prepare_readch()
779 /* If the content in the page list is an odd length, in rpcrdma_prepare_readch()
781 * the tail iovec. Force the tail's non-pad content to in rpcrdma_prepare_readch()
784 page_base = offset_in_page(xdr->tail[0].iov_base); in rpcrdma_prepare_readch()
785 len = xdr->tail[0].iov_len; in rpcrdma_prepare_readch()
787 len -= len & 3; in rpcrdma_prepare_readch()
788 if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len)) in rpcrdma_prepare_readch()
790 kref_get(&req->rl_kref); in rpcrdma_prepare_readch()
797 * rpcrdma_prepare_send_sges - Construct SGEs for a Send WR
799 * @req: context of RPC Call being marshalled
801 * @xdr: xdr_buf containing RPC Call
813 ret = -EAGAIN; in rpcrdma_prepare_send_sges()
814 req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt); in rpcrdma_prepare_send_sges()
815 if (!req->rl_sendctx) in rpcrdma_prepare_send_sges()
817 req->rl_sendctx->sc_unmap_count = 0; in rpcrdma_prepare_send_sges()
818 req->rl_sendctx->sc_req = req; in rpcrdma_prepare_send_sges()
819 kref_init(&req->rl_kref); in rpcrdma_prepare_send_sges()
820 req->rl_wr.wr_cqe = &req->rl_sendctx->sc_cqe; in rpcrdma_prepare_send_sges()
821 req->rl_wr.sg_list = req->rl_sendctx->sc_sges; in rpcrdma_prepare_send_sges()
822 req->rl_wr.num_sge = 0; in rpcrdma_prepare_send_sges()
823 req->rl_wr.opcode = IB_WR_SEND; in rpcrdma_prepare_send_sges()
827 ret = -EIO; in rpcrdma_prepare_send_sges()
830 if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
834 if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
838 if (!rpcrdma_prepare_readch(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
850 rpcrdma_sendctx_unmap(req->rl_sendctx); in rpcrdma_prepare_send_sges()
852 trace_xprtrdma_prepsend_failed(&req->rl_slot, ret); in rpcrdma_prepare_send_sges()
857 * rpcrdma_marshal_req - Marshal and send one RPC request
859 * @rqst: RPC request to be marshaled
861 * For the RPC in "rqst", this function:
862 * - Chooses the transfer mode (eg., RDMA_MSG or RDMA_NOMSG)
863 * - Registers Read, Write, and Reply chunks
864 * - Constructs the transport header
865 * - Posts a Send WR to send the transport header and request
868 * %0 if the RPC was sent successfully,
869 * %-ENOTCONN if the connection was lost,
870 * %-EAGAIN if the caller should call again with the same arguments,
871 * %-ENOBUFS if the caller should call again after a delay,
872 * %-EMSGSIZE if the transport header is too small,
873 * %-EIO if a permanent problem occurred while marshaling.
879 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_marshal_req()
881 struct xdr_buf *buf = &rqst->rq_snd_buf; in rpcrdma_marshal_req()
886 if (unlikely(rqst->rq_rcv_buf.flags & XDRBUF_SPARSE_PAGES)) { in rpcrdma_marshal_req()
887 ret = rpcrdma_alloc_sparse_pages(&rqst->rq_rcv_buf); in rpcrdma_marshal_req()
888 if (ret) in rpcrdma_marshal_req()
892 rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0); in rpcrdma_marshal_req()
893 xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf), in rpcrdma_marshal_req()
897 ret = -EMSGSIZE; in rpcrdma_marshal_req()
899 if (!p) in rpcrdma_marshal_req()
901 *p++ = rqst->rq_xid; in rpcrdma_marshal_req()
903 *p++ = r_xprt->rx_buf.rb_max_requests; in rpcrdma_marshal_req()
910 &rqst->rq_cred->cr_auth->au_flags); in rpcrdma_marshal_req()
915 * o If the expected result is under the inline threshold, all ops in rpcrdma_marshal_req()
919 * o Large non-read ops return as a single reply chunk. in rpcrdma_marshal_req()
921 if (rpcrdma_results_inline(r_xprt, rqst)) in rpcrdma_marshal_req()
923 else if ((ddp_allowed && rqst->rq_rcv_buf.flags & XDRBUF_READ) && in rpcrdma_marshal_req()
932 * o If the total request is under the inline threshold, all ops in rpcrdma_marshal_req()
936 * o Large non-write ops are sent with the entire message as a in rpcrdma_marshal_req()
937 * single read chunk (protocol 0-position special case). in rpcrdma_marshal_req()
940 * that both has a data payload, and whose non-data arguments in rpcrdma_marshal_req()
943 if (rpcrdma_args_inline(r_xprt, rqst)) { in rpcrdma_marshal_req()
945 rtype = buf->len < rdmab_length(req->rl_sendbuf) ? in rpcrdma_marshal_req()
947 } else if (ddp_allowed && buf->flags & XDRBUF_WRITE) { in rpcrdma_marshal_req()
951 r_xprt->rx_stats.nomsg_call_count++; in rpcrdma_marshal_req()
957 * of chunk lists in one RPC-over-RDMA Call message: in rpcrdma_marshal_req()
959 * - Read list in rpcrdma_marshal_req()
960 * - Write list in rpcrdma_marshal_req()
961 * - Reply chunk in rpcrdma_marshal_req()
962 * - Read list + Reply chunk in rpcrdma_marshal_req()
966 * - Read list + Write list in rpcrdma_marshal_req()
970 * - Write list + Reply chunk in rpcrdma_marshal_req()
971 * - Read list + Write list + Reply chunk in rpcrdma_marshal_req()
979 if (ret) in rpcrdma_marshal_req()
982 if (ret) in rpcrdma_marshal_req()
985 if (ret) in rpcrdma_marshal_req()
988 ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len, in rpcrdma_marshal_req()
990 if (ret) in rpcrdma_marshal_req()
998 r_xprt->rx_stats.failed_marshal_count++; in rpcrdma_marshal_req()
1007 buf->rb_credits = grant; in __rpcrdma_update_cwnd_locked()
1008 xprt->cwnd = grant << RPC_CWNDSHIFT; in __rpcrdma_update_cwnd_locked()
1013 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_update_cwnd()
1015 spin_lock(&xprt->transport_lock); in rpcrdma_update_cwnd()
1016 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, grant); in rpcrdma_update_cwnd()
1017 spin_unlock(&xprt->transport_lock); in rpcrdma_update_cwnd()
1021 * rpcrdma_reset_cwnd - Reset the xprt's congestion window
1029 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reset_cwnd()
1031 spin_lock(&xprt->transport_lock); in rpcrdma_reset_cwnd()
1032 xprt->cong = 0; in rpcrdma_reset_cwnd()
1033 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, 1); in rpcrdma_reset_cwnd()
1034 spin_unlock(&xprt->transport_lock); in rpcrdma_reset_cwnd()
1038 * rpcrdma_inline_fixup - Scatter inline received data into rqst's iovecs
1039 * @rqst: controlling RPC request
1040 * @srcp: points to RPC message payload in receive buffer
1064 /* The head iovec is redirected to the RPC reply message in rpcrdma_inline_fixup()
1067 rqst->rq_rcv_buf.head[0].iov_base = srcp; in rpcrdma_inline_fixup()
1068 rqst->rq_private_buf.head[0].iov_base = srcp; in rpcrdma_inline_fixup()
1073 curlen = rqst->rq_rcv_buf.head[0].iov_len; in rpcrdma_inline_fixup()
1074 if (curlen > copy_len) in rpcrdma_inline_fixup()
1077 copy_len -= curlen; in rpcrdma_inline_fixup()
1079 ppages = rqst->rq_rcv_buf.pages + in rpcrdma_inline_fixup()
1080 (rqst->rq_rcv_buf.page_base >> PAGE_SHIFT); in rpcrdma_inline_fixup()
1081 page_base = offset_in_page(rqst->rq_rcv_buf.page_base); in rpcrdma_inline_fixup()
1083 if (copy_len && rqst->rq_rcv_buf.page_len) { in rpcrdma_inline_fixup()
1086 pagelist_len = rqst->rq_rcv_buf.page_len; in rpcrdma_inline_fixup()
1087 if (pagelist_len > copy_len) in rpcrdma_inline_fixup()
1091 curlen = PAGE_SIZE - page_base; in rpcrdma_inline_fixup()
1092 if (curlen > pagelist_len) in rpcrdma_inline_fixup()
1100 copy_len -= curlen; in rpcrdma_inline_fixup()
1102 pagelist_len -= curlen; in rpcrdma_inline_fixup()
1103 if (!pagelist_len) in rpcrdma_inline_fixup()
1114 if (pad) in rpcrdma_inline_fixup()
1115 srcp -= pad; in rpcrdma_inline_fixup()
1121 if (copy_len || pad) { in rpcrdma_inline_fixup()
1122 rqst->rq_rcv_buf.tail[0].iov_base = srcp; in rpcrdma_inline_fixup()
1123 rqst->rq_private_buf.tail[0].iov_base = srcp; in rpcrdma_inline_fixup()
1126 if (fixup_copy_count) in rpcrdma_inline_fixup()
1133 * the RPC/RDMA header small and fixed in size, so it is
1134 * straightforward to check the RPC header's direction field.
1138 #if defined(CONFIG_SUNRPC_BACKCHANNEL) in rpcrdma_is_bcall()
1140 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_is_bcall()
1143 if (rep->rr_proc != rdma_msg) in rpcrdma_is_bcall()
1150 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1152 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1154 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1157 /* RPC header */ in rpcrdma_is_bcall()
1158 if (*p++ != rep->rr_xid) in rpcrdma_is_bcall()
1160 if (*p != cpu_to_be32(RPC_CALL)) in rpcrdma_is_bcall()
1164 * advance to the RPC header. in rpcrdma_is_bcall()
1167 if (unlikely(!p)) in rpcrdma_is_bcall()
1174 pr_warn("RPC/RDMA short backward direction call\n"); in rpcrdma_is_bcall()
1190 if (unlikely(!p)) in decode_rdma_segment()
1191 return -EIO; in decode_rdma_segment()
1204 if (unlikely(!p)) in decode_write_chunk()
1205 return -EIO; in decode_write_chunk()
1209 while (segcount--) { in decode_write_chunk()
1210 if (decode_rdma_segment(xdr, &seglength)) in decode_write_chunk()
1211 return -EIO; in decode_write_chunk()
1218 /* In RPC-over-RDMA Version One replies, a Read list is never
1219 * expected. This decoder is a stub that returns an error if
1227 if (unlikely(!p)) in decode_read_list()
1228 return -EIO; in decode_read_list()
1229 if (unlikely(xdr_item_is_present(p))) in decode_read_list()
1230 return -EIO; in decode_read_list()
1246 if (unlikely(!p)) in decode_write_list()
1247 return -EIO; in decode_write_list()
1248 if (xdr_item_is_absent(p)) in decode_write_list()
1250 if (!first) in decode_write_list()
1251 return -EIO; in decode_write_list()
1253 if (decode_write_chunk(xdr, &chunklen)) in decode_write_list()
1254 return -EIO; in decode_write_list()
1266 if (unlikely(!p)) in decode_reply_chunk()
1267 return -EIO; in decode_reply_chunk()
1270 if (xdr_item_is_present(p)) in decode_reply_chunk()
1271 if (decode_write_chunk(xdr, length)) in decode_reply_chunk()
1272 return -EIO; in decode_reply_chunk()
1280 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_msg()
1285 if (decode_read_list(xdr)) in rpcrdma_decode_msg()
1286 return -EIO; in rpcrdma_decode_msg()
1287 if (decode_write_list(xdr, &writelist)) in rpcrdma_decode_msg()
1288 return -EIO; in rpcrdma_decode_msg()
1289 if (decode_reply_chunk(xdr, &replychunk)) in rpcrdma_decode_msg()
1290 return -EIO; in rpcrdma_decode_msg()
1293 if (unlikely(replychunk)) in rpcrdma_decode_msg()
1294 return -EIO; in rpcrdma_decode_msg()
1296 /* Build the RPC reply's Payload stream in rqst->rq_rcv_buf */ in rpcrdma_decode_msg()
1299 r_xprt->rx_stats.fixup_copy_count += in rpcrdma_decode_msg()
1302 r_xprt->rx_stats.total_rdma_reply += writelist; in rpcrdma_decode_msg()
1309 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_nomsg()
1313 if (decode_read_list(xdr)) in rpcrdma_decode_nomsg()
1314 return -EIO; in rpcrdma_decode_nomsg()
1315 if (decode_write_list(xdr, &writelist)) in rpcrdma_decode_nomsg()
1316 return -EIO; in rpcrdma_decode_nomsg()
1317 if (decode_reply_chunk(xdr, &replychunk)) in rpcrdma_decode_nomsg()
1318 return -EIO; in rpcrdma_decode_nomsg()
1321 if (unlikely(writelist)) in rpcrdma_decode_nomsg()
1322 return -EIO; in rpcrdma_decode_nomsg()
1323 if (unlikely(!replychunk)) in rpcrdma_decode_nomsg()
1324 return -EIO; in rpcrdma_decode_nomsg()
1327 r_xprt->rx_stats.total_rdma_reply += replychunk; in rpcrdma_decode_nomsg()
1335 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_error()
1339 if (unlikely(!p)) in rpcrdma_decode_error()
1340 return -EIO; in rpcrdma_decode_error()
1345 if (!p) in rpcrdma_decode_error()
1347 dprintk("RPC: %s: server reports " in rpcrdma_decode_error()
1348 "version error (%u-%u), xid %08x\n", __func__, in rpcrdma_decode_error()
1350 be32_to_cpu(rep->rr_xid)); in rpcrdma_decode_error()
1353 dprintk("RPC: %s: server reports " in rpcrdma_decode_error()
1355 be32_to_cpu(rep->rr_xid)); in rpcrdma_decode_error()
1358 dprintk("RPC: %s: server reports " in rpcrdma_decode_error()
1360 be32_to_cpup(p), be32_to_cpu(rep->rr_xid)); in rpcrdma_decode_error()
1363 return -EIO; in rpcrdma_decode_error()
1366 /* Perform XID lookup, reconstruction of the RPC reply, and
1367 * RPC completion while holding the transport lock to ensure
1372 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_complete_rqst()
1373 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_complete_rqst()
1374 struct rpc_rqst *rqst = rep->rr_rqst; in rpcrdma_complete_rqst()
1377 switch (rep->rr_proc) { in rpcrdma_complete_rqst()
1388 status = -EIO; in rpcrdma_complete_rqst()
1390 if (status < 0) in rpcrdma_complete_rqst()
1394 spin_lock(&xprt->queue_lock); in rpcrdma_complete_rqst()
1395 xprt_complete_rqst(rqst->rq_task, status); in rpcrdma_complete_rqst()
1397 spin_unlock(&xprt->queue_lock); in rpcrdma_complete_rqst()
1402 r_xprt->rx_stats.bad_reply_count++; in rpcrdma_complete_rqst()
1403 rqst->rq_task->tk_status = status; in rpcrdma_complete_rqst()
1413 rpcrdma_complete_rqst(req->rl_reply); in rpcrdma_reply_done()
1417 * rpcrdma_reply_handler - Process received RPC/RDMA messages
1420 * Errors must result in the RPC task either being awakened, or
1425 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_reply_handler()
1426 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reply_handler()
1427 struct rpcrdma_buffer *buf = &r_xprt->rx_buf; in rpcrdma_reply_handler()
1436 if (xprt->reestablish_timeout) in rpcrdma_reply_handler()
1437 xprt->reestablish_timeout = 0; in rpcrdma_reply_handler()
1440 xdr_init_decode(&rep->rr_stream, &rep->rr_hdrbuf, in rpcrdma_reply_handler()
1441 rep->rr_hdrbuf.head[0].iov_base, NULL); in rpcrdma_reply_handler()
1442 p = xdr_inline_decode(&rep->rr_stream, 4 * sizeof(*p)); in rpcrdma_reply_handler()
1443 if (unlikely(!p)) in rpcrdma_reply_handler()
1445 rep->rr_xid = *p++; in rpcrdma_reply_handler()
1446 rep->rr_vers = *p++; in rpcrdma_reply_handler()
1448 rep->rr_proc = *p++; in rpcrdma_reply_handler()
1450 if (rep->rr_vers != rpcrdma_version) in rpcrdma_reply_handler()
1453 if (rpcrdma_is_bcall(r_xprt, rep)) in rpcrdma_reply_handler()
1459 spin_lock(&xprt->queue_lock); in rpcrdma_reply_handler()
1460 rqst = xprt_lookup_rqst(xprt, rep->rr_xid); in rpcrdma_reply_handler()
1461 if (!rqst) in rpcrdma_reply_handler()
1464 spin_unlock(&xprt->queue_lock); in rpcrdma_reply_handler()
1466 if (credits == 0) in rpcrdma_reply_handler()
1468 else if (credits > r_xprt->rx_ep->re_max_requests) in rpcrdma_reply_handler()
1469 credits = r_xprt->rx_ep->re_max_requests; in rpcrdma_reply_handler()
1470 rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1), in rpcrdma_reply_handler()
1472 if (buf->rb_credits != credits) in rpcrdma_reply_handler()
1476 if (req->rl_reply) { in rpcrdma_reply_handler()
1477 trace_xprtrdma_leaked_rep(rqst, req->rl_reply); in rpcrdma_reply_handler()
1478 rpcrdma_recv_buffer_put(req->rl_reply); in rpcrdma_reply_handler()
1480 req->rl_reply = rep; in rpcrdma_reply_handler()
1481 rep->rr_rqst = rqst; in rpcrdma_reply_handler()
1483 trace_xprtrdma_reply(rqst->rq_task, rep, req, credits); in rpcrdma_reply_handler()
1485 if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) in rpcrdma_reply_handler()
1486 frwr_reminv(rep, &req->rl_registered); in rpcrdma_reply_handler()
1487 if (!list_empty(&req->rl_registered)) in rpcrdma_reply_handler()
1489 /* LocalInv completion will complete the RPC */ in rpcrdma_reply_handler()
1491 kref_put(&req->rl_kref, rpcrdma_reply_done); in rpcrdma_reply_handler()
1499 spin_unlock(&xprt->queue_lock); in rpcrdma_reply_handler()