97 	struct rds_sock *rs = mr->r_sock;  in rds_destroy_mr()  local
104 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_destroy_mr()
106 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_destroy_mr()
109 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_destroy_mr()
127 void rds_rdma_drop_keys(struct rds_sock *rs)  in rds_rdma_drop_keys()  argument
134 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
135 	while ((node = rb_first(&rs->rs_rdma_keys))) {  in rds_rdma_drop_keys()
137 		if (mr->r_trans == rs->rs_transport)  in rds_rdma_drop_keys()
139 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_drop_keys()
141 		spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
143 		spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
145 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
147 	if (rs->rs_transport && rs->rs_transport->flush_mrs)  in rds_rdma_drop_keys()
148 		rs->rs_transport->flush_mrs();  in rds_rdma_drop_keys()
172 static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args,  in __rds_rdma_map()  argument
188 	if (ipv6_addr_any(&rs->rs_bound_addr) || !rs->rs_transport) {  in __rds_rdma_map()
193 	if (!rs->rs_transport->get_mr) {  in __rds_rdma_map()
245 	mr->r_trans = rs->rs_transport;  in __rds_rdma_map()
246 	mr->r_sock = rs;  in __rds_rdma_map()
290 	trans_private = rs->rs_transport->get_mr(  in __rds_rdma_map()
291 		sg, nents, rs, &mr->r_key, cp ? cp->cp_conn : NULL,  in __rds_rdma_map()
336 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in __rds_rdma_map()
337 	found = rds_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr);  in __rds_rdma_map()
338 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in __rds_rdma_map()
356 int rds_get_mr(struct rds_sock *rs, sockptr_t optval, int optlen)  in rds_get_mr()  argument
366 	return __rds_rdma_map(rs, &args, NULL, NULL, NULL);  in rds_get_mr()
369 int rds_get_mr_for_dest(struct rds_sock *rs, sockptr_t optval, int optlen)  in rds_get_mr_for_dest()  argument
390 	return __rds_rdma_map(rs, &new_args, NULL, NULL, NULL);  in rds_get_mr_for_dest()
396 int rds_free_mr(struct rds_sock *rs, sockptr_t optval, int optlen)  in rds_free_mr()  argument
410 		if (!rs->rs_transport || !rs->rs_transport->flush_mrs)  in rds_free_mr()
412 		rs->rs_transport->flush_mrs();  in rds_free_mr()
420 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_free_mr()
421 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);  in rds_free_mr()
423 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_free_mr()
428 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_free_mr()
442 void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force)  in rds_rdma_unuse()  argument
448 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
449 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_rdma_unuse()
453 		spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
466 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_unuse()
470 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
608 int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_args()  argument
628 	if (ipv6_addr_any(&rs->rs_bound_addr)) {  in rds_cmsg_rdma_args()
665 	op->op_recverr = rs->rs_recverr;  in rds_cmsg_rdma_args()
712 		rs->rs_user_addr = iov->addr;  in rds_cmsg_rdma_args()
713 		rs->rs_user_bytes = iov->bytes;  in rds_cmsg_rdma_args()
726 			if (!rs->rs_transport->get_mr) {  in rds_cmsg_rdma_args()
738 			local_odp_mr->r_trans = rs->rs_transport;  in rds_cmsg_rdma_args()
739 			local_odp_mr->r_sock = rs;  in rds_cmsg_rdma_args()
741 				rs->rs_transport->get_mr(  in rds_cmsg_rdma_args()
742 					NULL, 0, rs, &local_odp_mr->r_key, NULL,  in rds_cmsg_rdma_args()
808 int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_dest()  argument
829 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_cmsg_rdma_dest()
830 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_cmsg_rdma_dest()
835 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_cmsg_rdma_dest()
851 int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_map()  argument
858 	return __rds_rdma_map(rs, CMSG_DATA(cmsg), &rm->m_rdma_cookie,  in rds_cmsg_rdma_map()
865 int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_atomic()  argument
911 	rm->atomic.op_recverr = rs->rs_recverr;  in rds_cmsg_atomic()