57 void frwr_release_mr(struct rpcrdma_mr *mr)  in frwr_release_mr()  argument
61 	rc = ib_dereg_mr(mr->frwr.fr_mr);  in frwr_release_mr()
63 		trace_xprtrdma_frwr_dereg(mr, rc);  in frwr_release_mr()
64 	kfree(mr->mr_sg);  in frwr_release_mr()
65 	kfree(mr);  in frwr_release_mr()
68 static void frwr_mr_recycle(struct rpcrdma_mr *mr)  in frwr_mr_recycle()  argument
70 	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;  in frwr_mr_recycle()
72 	trace_xprtrdma_mr_recycle(mr);  in frwr_mr_recycle()
74 	if (mr->mr_dir != DMA_NONE) {  in frwr_mr_recycle()
75 		trace_xprtrdma_mr_unmap(mr);  in frwr_mr_recycle()
77 				mr->mr_sg, mr->mr_nents, mr->mr_dir);  in frwr_mr_recycle()
78 		mr->mr_dir = DMA_NONE;  in frwr_mr_recycle()
82 	list_del(&mr->mr_all);  in frwr_mr_recycle()
86 	frwr_release_mr(mr);  in frwr_mr_recycle()
101 	struct rpcrdma_mr *mr;  in frwr_reset()  local
103 	while ((mr = rpcrdma_mr_pop(&req->rl_registered)))  in frwr_reset()
104 		rpcrdma_mr_put(mr);  in frwr_reset()
115 int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr)  in frwr_mr_init()  argument
131 	mr->mr_xprt = r_xprt;  in frwr_mr_init()
132 	mr->frwr.fr_mr = frmr;  in frwr_mr_init()
133 	mr->mr_dir = DMA_NONE;  in frwr_mr_init()
134 	INIT_LIST_HEAD(&mr->mr_list);  in frwr_mr_init()
135 	init_completion(&mr->frwr.fr_linv_done);  in frwr_mr_init()
138 	mr->mr_sg = sg;  in frwr_mr_init()
143 	trace_xprtrdma_frwr_alloc(mr, rc);  in frwr_mr_init()
284 				struct rpcrdma_mr *mr)  in frwr_map()  argument
296 			sg_set_page(&mr->mr_sg[i],  in frwr_map()
301 			sg_set_buf(&mr->mr_sg[i], seg->mr_offset,  in frwr_map()
312 	mr->mr_dir = rpcrdma_data_dir(writing);  in frwr_map()
313 	mr->mr_nents = i;  in frwr_map()
315 	dma_nents = ib_dma_map_sg(ep->re_id->device, mr->mr_sg, mr->mr_nents,  in frwr_map()
316 				  mr->mr_dir);  in frwr_map()
320 	ibmr = mr->frwr.fr_mr;  in frwr_map()
321 	n = ib_map_mr_sg(ibmr, mr->mr_sg, dma_nents, NULL, PAGE_SIZE);  in frwr_map()
330 	reg_wr = &mr->frwr.fr_regwr;  in frwr_map()
331 	reg_wr->mr = ibmr;  in frwr_map()
337 	mr->mr_handle = ibmr->rkey;  in frwr_map()
338 	mr->mr_length = ibmr->length;  in frwr_map()
339 	mr->mr_offset = ibmr->iova;  in frwr_map()
340 	trace_xprtrdma_mr_map(mr);  in frwr_map()
345 	mr->mr_dir = DMA_NONE;  in frwr_map()
346 	trace_xprtrdma_frwr_sgerr(mr, i);  in frwr_map()
350 	trace_xprtrdma_frwr_maperr(mr, n);  in frwr_map()
390 	struct rpcrdma_mr *mr;  in frwr_send()  local
393 	list_for_each_entry(mr, &req->rl_registered, mr_list) {  in frwr_send()
396 		frwr = &mr->frwr;  in frwr_send()
419 	struct rpcrdma_mr *mr;  in frwr_reminv()  local
421 	list_for_each_entry(mr, mrs, mr_list)  in frwr_reminv()
422 		if (mr->mr_handle == rep->rr_inv_rkey) {  in frwr_reminv()
423 			list_del_init(&mr->mr_list);  in frwr_reminv()
424 			trace_xprtrdma_mr_reminv(mr);  in frwr_reminv()
425 			rpcrdma_mr_put(mr);  in frwr_reminv()
430 static void __frwr_release_mr(struct ib_wc *wc, struct rpcrdma_mr *mr)  in __frwr_release_mr()  argument
433 		frwr_mr_recycle(mr);  in __frwr_release_mr()
435 		rpcrdma_mr_put(mr);  in __frwr_release_mr()
449 	struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);  in frwr_wc_localinv()  local
453 	__frwr_release_mr(wc, mr);  in frwr_wc_localinv()
470 	struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);  in frwr_wc_localinv_wake()  local
474 	__frwr_release_mr(wc, mr);  in frwr_wc_localinv_wake()
496 	struct rpcrdma_mr *mr;  in frwr_unmap_sync()  local
506 	while ((mr = rpcrdma_mr_pop(&req->rl_registered))) {  in frwr_unmap_sync()
508 		trace_xprtrdma_mr_localinv(mr);  in frwr_unmap_sync()
511 		frwr = &mr->frwr;  in frwr_unmap_sync()
520 		last->ex.invalidate_rkey = mr->mr_handle;  in frwr_unmap_sync()
555 		mr = container_of(frwr, struct rpcrdma_mr, frwr);  in frwr_unmap_sync()
558 		frwr_mr_recycle(mr);  in frwr_unmap_sync()
573 	struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);  in frwr_wc_localinv_done()  local
574 	struct rpcrdma_rep *rep = mr->mr_req->rl_reply;  in frwr_wc_localinv_done()
578 	__frwr_release_mr(wc, mr);  in frwr_wc_localinv_done()
602 	struct rpcrdma_mr *mr;  in frwr_unmap_async()  local
610 	while ((mr = rpcrdma_mr_pop(&req->rl_registered))) {  in frwr_unmap_async()
612 		trace_xprtrdma_mr_localinv(mr);  in frwr_unmap_async()
615 		frwr = &mr->frwr;  in frwr_unmap_async()
624 		last->ex.invalidate_rkey = mr->mr_handle;  in frwr_unmap_async()
651 		mr = container_of(frwr, struct rpcrdma_mr, frwr);  in frwr_unmap_async()
654 		frwr_mr_recycle(mr);  in frwr_unmap_async()