173 static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args,  in __rds_rdma_map()  argument
198 	nr_pages = rds_pages_in_vec(&args->vec);  in __rds_rdma_map()
213 		args->vec.addr, args->vec.bytes, nr_pages);  in __rds_rdma_map()
233 	if (args->flags & RDS_RDMA_USE_ONCE)  in __rds_rdma_map()
235 	if (args->flags & RDS_RDMA_INVALIDATE)  in __rds_rdma_map()
237 	if (args->flags & RDS_RDMA_READWRITE)  in __rds_rdma_map()
250 	ret = rds_pin_pages(args->vec.addr, nr_pages, pages, 1);  in __rds_rdma_map()
288 	       mr->r_key, (void *)(unsigned long) args->cookie_addr);  in __rds_rdma_map()
294 	cookie = rds_rdma_make_cookie(mr->r_key, args->vec.addr & ~PAGE_MASK);  in __rds_rdma_map()
298 	if (args->cookie_addr && put_user(cookie, (u64 __user *)(unsigned long) args->cookie_addr)) {  in __rds_rdma_map()
327 	struct rds_get_mr_args args;  in rds_get_mr()  local
332 	if (copy_from_user(&args, (struct rds_get_mr_args __user *)optval,  in rds_get_mr()
336 	return __rds_rdma_map(rs, &args, NULL, NULL, NULL);  in rds_get_mr()
341 	struct rds_get_mr_for_dest_args args;  in rds_get_mr_for_dest()  local
347 	if (copy_from_user(&args, (struct rds_get_mr_for_dest_args __user *)optval,  in rds_get_mr_for_dest()
356 	new_args.vec = args.vec;  in rds_get_mr_for_dest()
357 	new_args.cookie_addr = args.cookie_addr;  in rds_get_mr_for_dest()
358 	new_args.flags = args.flags;  in rds_get_mr_for_dest()
368 	struct rds_free_mr_args args;  in rds_free_mr()  local
375 	if (copy_from_user(&args, (struct rds_free_mr_args __user *)optval,  in rds_free_mr()
380 	if (args.cookie == 0) {  in rds_free_mr()
392 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);  in rds_free_mr()
396 		if (args.flags & RDS_RDMA_INVALIDATE)  in rds_free_mr()
521 int rds_rdma_extra_size(struct rds_rdma_args *args,  in rds_rdma_extra_size()  argument
530 	local_vec = (struct rds_iovec __user *)(unsigned long) args->local_vec_addr;  in rds_rdma_extra_size()
532 	if (args->nr_local == 0)  in rds_rdma_extra_size()
535 	iov->iov = kcalloc(args->nr_local,  in rds_rdma_extra_size()
543 	if (copy_from_user(vec, local_vec, args->nr_local *  in rds_rdma_extra_size()
546 	iov->len = args->nr_local;  in rds_rdma_extra_size()
549 	for (i = 0; i < args->nr_local; i++, vec++) {  in rds_rdma_extra_size()
576 	struct rds_rdma_args *args;  in rds_cmsg_rdma_args()  local
589 	args = CMSG_DATA(cmsg);  in rds_cmsg_rdma_args()
596 	if (args->nr_local > UIO_MAXIOV) {  in rds_cmsg_rdma_args()
601 	if (vec->len != args->nr_local) {  in rds_cmsg_rdma_args()
608 	nr_pages = rds_rdma_pages(iovs, args->nr_local);  in rds_cmsg_rdma_args()
620 	op->op_write = !!(args->flags & RDS_RDMA_READWRITE);  in rds_cmsg_rdma_args()
621 	op->op_fence = !!(args->flags & RDS_RDMA_FENCE);  in rds_cmsg_rdma_args()
622 	op->op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);  in rds_cmsg_rdma_args()
623 	op->op_silent = !!(args->flags & RDS_RDMA_SILENT);  in rds_cmsg_rdma_args()
642 		op->op_notifier->n_user_token = args->user_token;  in rds_cmsg_rdma_args()
653 	op->op_rkey = rds_rdma_cookie_key(args->cookie);  in rds_cmsg_rdma_args()
654 	op->op_remote_addr = args->remote_vec.addr + rds_rdma_cookie_offset(args->cookie);  in rds_cmsg_rdma_args()
659 	       (unsigned long long)args->nr_local,  in rds_cmsg_rdma_args()
660 	       (unsigned long long)args->remote_vec.addr,  in rds_cmsg_rdma_args()
663 	for (i = 0; i < args->nr_local; i++) {  in rds_cmsg_rdma_args()
704 	if (nr_bytes > args->remote_vec.bytes) {  in rds_cmsg_rdma_args()
707 				(unsigned int) args->remote_vec.bytes);  in rds_cmsg_rdma_args()
788 	struct rds_atomic_args *args;  in rds_cmsg_atomic()  local
795 	args = CMSG_DATA(cmsg);  in rds_cmsg_atomic()
801 		rm->atomic.op_m_fadd.add = args->fadd.add;  in rds_cmsg_atomic()
806 		rm->atomic.op_m_fadd.add = args->m_fadd.add;  in rds_cmsg_atomic()
807 		rm->atomic.op_m_fadd.nocarry_mask = args->m_fadd.nocarry_mask;  in rds_cmsg_atomic()
811 		rm->atomic.op_m_cswp.compare = args->cswp.compare;  in rds_cmsg_atomic()
812 		rm->atomic.op_m_cswp.swap = args->cswp.swap;  in rds_cmsg_atomic()
818 		rm->atomic.op_m_cswp.compare = args->m_cswp.compare;  in rds_cmsg_atomic()
819 		rm->atomic.op_m_cswp.swap = args->m_cswp.swap;  in rds_cmsg_atomic()
820 		rm->atomic.op_m_cswp.compare_mask = args->m_cswp.compare_mask;  in rds_cmsg_atomic()
821 		rm->atomic.op_m_cswp.swap_mask = args->m_cswp.swap_mask;  in rds_cmsg_atomic()
827 	rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);  in rds_cmsg_atomic()
828 	rm->atomic.op_silent = !!(args->flags & RDS_RDMA_SILENT);  in rds_cmsg_atomic()
836 	if (args->local_addr & 0x7) {  in rds_cmsg_atomic()
841 	ret = rds_pin_pages(args->local_addr, 1, &page, 1);  in rds_cmsg_atomic()
846 	sg_set_page(rm->atomic.op_sg, page, 8, offset_in_page(args->local_addr));  in rds_cmsg_atomic()
860 		rm->atomic.op_notifier->n_user_token = args->user_token;  in rds_cmsg_atomic()
864 	rm->atomic.op_rkey = rds_rdma_cookie_key(args->cookie);  in rds_cmsg_atomic()
865 	rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie);  in rds_cmsg_atomic()