74 				   struct rm_data_op *op,  in rds_ib_send_unmap_data()  argument
77 	if (op->op_nents)  in rds_ib_send_unmap_data()
79 				op->op_sg, op->op_nents,  in rds_ib_send_unmap_data()
84 				   struct rm_rdma_op *op,  in rds_ib_send_unmap_rdma()  argument
87 	if (op->op_mapped) {  in rds_ib_send_unmap_rdma()
89 				op->op_sg, op->op_nents,  in rds_ib_send_unmap_rdma()
90 				op->op_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);  in rds_ib_send_unmap_rdma()
91 		op->op_mapped = 0;  in rds_ib_send_unmap_rdma()
114 	rds_ib_send_complete(container_of(op, struct rds_message, rdma),  in rds_ib_send_unmap_rdma()
117 	if (op->op_write)  in rds_ib_send_unmap_rdma()
118 		rds_stats_add(s_send_rdma_bytes, op->op_bytes);  in rds_ib_send_unmap_rdma()
120 		rds_stats_add(s_recv_rdma_bytes, op->op_bytes);  in rds_ib_send_unmap_rdma()
124 				     struct rm_atomic_op *op,  in rds_ib_send_unmap_atomic()  argument
128 	if (op->op_mapped) {  in rds_ib_send_unmap_atomic()
129 		ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,  in rds_ib_send_unmap_atomic()
131 		op->op_mapped = 0;  in rds_ib_send_unmap_atomic()
134 	rds_ib_send_complete(container_of(op, struct rds_message, atomic),  in rds_ib_send_unmap_atomic()
137 	if (op->op_type == RDS_ATOMIC_TYPE_CSWP)  in rds_ib_send_unmap_atomic()
767 int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)  in rds_ib_xmit_atomic()  argument
788 	if (op->op_type == RDS_ATOMIC_TYPE_CSWP) {  in rds_ib_xmit_atomic()
790 		send->s_atomic_wr.compare_add = op->op_m_cswp.compare;  in rds_ib_xmit_atomic()
791 		send->s_atomic_wr.swap = op->op_m_cswp.swap;  in rds_ib_xmit_atomic()
792 		send->s_atomic_wr.compare_add_mask = op->op_m_cswp.compare_mask;  in rds_ib_xmit_atomic()
793 		send->s_atomic_wr.swap_mask = op->op_m_cswp.swap_mask;  in rds_ib_xmit_atomic()
796 		send->s_atomic_wr.compare_add = op->op_m_fadd.add;  in rds_ib_xmit_atomic()
798 		send->s_atomic_wr.compare_add_mask = op->op_m_fadd.nocarry_mask;  in rds_ib_xmit_atomic()
802 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
805 	send->s_atomic_wr.remote_addr = op->op_remote_addr;  in rds_ib_xmit_atomic()
806 	send->s_atomic_wr.rkey = op->op_rkey;  in rds_ib_xmit_atomic()
807 	send->s_op = op;  in rds_ib_xmit_atomic()
811 	ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);  in rds_ib_xmit_atomic()
812 	rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);  in rds_ib_xmit_atomic()
821 	send->s_sge[0].addr = sg_dma_address(op->op_sg);  in rds_ib_xmit_atomic()
822 	send->s_sge[0].length = sg_dma_len(op->op_sg);  in rds_ib_xmit_atomic()
825 	rdsdebug("rva %Lx rpa %Lx len %u\n", op->op_remote_addr,  in rds_ib_xmit_atomic()
853 int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)  in rds_ib_xmit_rdma()  argument
862 	u64 remote_addr = op->op_remote_addr;  in rds_ib_xmit_rdma()
872 	u64 odp_addr = op->op_odp_addr;  in rds_ib_xmit_rdma()
876 	if (!op->op_odp_mr) {  in rds_ib_xmit_rdma()
877 		if (!op->op_mapped) {  in rds_ib_xmit_rdma()
878 			op->op_count =  in rds_ib_xmit_rdma()
879 				ib_dma_map_sg(ic->i_cm_id->device, op->op_sg,  in rds_ib_xmit_rdma()
880 					      op->op_nents,  in rds_ib_xmit_rdma()
881 					      (op->op_write) ? DMA_TO_DEVICE :  in rds_ib_xmit_rdma()
883 			rdsdebug("ic %p mapping op %p: %d\n", ic, op,  in rds_ib_xmit_rdma()
884 				 op->op_count);  in rds_ib_xmit_rdma()
885 			if (op->op_count == 0) {  in rds_ib_xmit_rdma()
890 			op->op_mapped = 1;  in rds_ib_xmit_rdma()
893 		op->op_count = op->op_nents;  in rds_ib_xmit_rdma()
894 		odp_lkey = rds_ib_get_lkey(op->op_odp_mr->r_trans_private);  in rds_ib_xmit_rdma()
901 	i = DIV_ROUND_UP(op->op_count, max_sge);  in rds_ib_xmit_rdma()
914 	scat = &op->op_sg[0];  in rds_ib_xmit_rdma()
916 	num_sge = op->op_count;  in rds_ib_xmit_rdma()
918 	for (i = 0; i < work_alloc && scat != &op->op_sg[op->op_count]; i++) {  in rds_ib_xmit_rdma()
923 		if (!op->op_notify)  in rds_ib_xmit_rdma()
925 							     op->op_notify);  in rds_ib_xmit_rdma()
927 		send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;  in rds_ib_xmit_rdma()
929 		send->s_rdma_wr.rkey = op->op_rkey;  in rds_ib_xmit_rdma()
944 		     scat != &op->op_sg[op->op_count]; j++) {  in rds_ib_xmit_rdma()
946 			if (!op->op_odp_mr) {  in rds_ib_xmit_rdma()
974 	if (scat == &op->op_sg[op->op_count]) {  in rds_ib_xmit_rdma()
975 		prev->s_op = op;  in rds_ib_xmit_rdma()
976 		rds_message_addref(container_of(op, struct rds_message, rdma));  in rds_ib_xmit_rdma()