106 				   struct rm_rdma_op *op,  in rds_ib_send_unmap_rdma()  argument
109 	if (op->op_mapped) {  in rds_ib_send_unmap_rdma()
111 				op->op_sg, op->op_nents,  in rds_ib_send_unmap_rdma()
112 				op->op_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);  in rds_ib_send_unmap_rdma()
113 		op->op_mapped = 0;  in rds_ib_send_unmap_rdma()
136 	rds_ib_send_complete(container_of(op, struct rds_message, rdma),  in rds_ib_send_unmap_rdma()
139 	if (op->op_write)  in rds_ib_send_unmap_rdma()
140 		rds_stats_add(s_send_rdma_bytes, op->op_bytes);  in rds_ib_send_unmap_rdma()
142 		rds_stats_add(s_recv_rdma_bytes, op->op_bytes);  in rds_ib_send_unmap_rdma()
146 				     struct rm_atomic_op *op,  in rds_ib_send_unmap_atomic()  argument
150 	if (op->op_mapped) {  in rds_ib_send_unmap_atomic()
151 		ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,  in rds_ib_send_unmap_atomic()
153 		op->op_mapped = 0;  in rds_ib_send_unmap_atomic()
156 	rds_ib_send_complete(container_of(op, struct rds_message, atomic),  in rds_ib_send_unmap_atomic()
159 	if (op->op_type == RDS_ATOMIC_TYPE_CSWP)  in rds_ib_send_unmap_atomic()
166 				   struct rm_data_op *op,  in rds_ib_send_unmap_data()  argument
169 	struct rds_message *rm = container_of(op, struct rds_message, data);  in rds_ib_send_unmap_data()
171 	if (op->op_nents)  in rds_ib_send_unmap_data()
173 				op->op_sg, op->op_nents,  in rds_ib_send_unmap_data()
791 int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)  in rds_ib_xmit_atomic()  argument
816 	if (op->op_type == RDS_ATOMIC_TYPE_CSWP) {  in rds_ib_xmit_atomic()
818 		send->s_wr.wr.atomic.compare_add = op->op_m_cswp.compare;  in rds_ib_xmit_atomic()
819 		send->s_wr.wr.atomic.swap = op->op_m_cswp.swap;  in rds_ib_xmit_atomic()
820 		send->s_wr.wr.atomic.compare_add_mask = op->op_m_cswp.compare_mask;  in rds_ib_xmit_atomic()
821 		send->s_wr.wr.atomic.swap_mask = op->op_m_cswp.swap_mask;  in rds_ib_xmit_atomic()
824 		send->s_wr.wr.atomic.compare_add = op->op_m_fadd.add;  in rds_ib_xmit_atomic()
826 		send->s_wr.wr.atomic.compare_add_mask = op->op_m_fadd.nocarry_mask;  in rds_ib_xmit_atomic()
829 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
832 	send->s_wr.wr.atomic.remote_addr = op->op_remote_addr;  in rds_ib_xmit_atomic()
833 	send->s_wr.wr.atomic.rkey = op->op_rkey;  in rds_ib_xmit_atomic()
834 	send->s_op = op;  in rds_ib_xmit_atomic()
838 	ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);  in rds_ib_xmit_atomic()
839 	rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);  in rds_ib_xmit_atomic()
848 	send->s_sge[0].addr = ib_sg_dma_address(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
849 	send->s_sge[0].length = ib_sg_dma_len(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
852 	rdsdebug("rva %Lx rpa %Lx len %u\n", op->op_remote_addr,  in rds_ib_xmit_atomic()
880 int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)  in rds_ib_xmit_rdma()  argument
889 	u64 remote_addr = op->op_remote_addr;  in rds_ib_xmit_rdma()
901 	if (!op->op_mapped) {  in rds_ib_xmit_rdma()
902 		op->op_count = ib_dma_map_sg(ic->i_cm_id->device,  in rds_ib_xmit_rdma()
903 					     op->op_sg, op->op_nents, (op->op_write) ?  in rds_ib_xmit_rdma()
905 		rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count);  in rds_ib_xmit_rdma()
906 		if (op->op_count == 0) {  in rds_ib_xmit_rdma()
912 		op->op_mapped = 1;  in rds_ib_xmit_rdma()
919 	i = ceil(op->op_count, max_sge);  in rds_ib_xmit_rdma()
932 	scat = &op->op_sg[0];  in rds_ib_xmit_rdma()
934 	num_sge = op->op_count;  in rds_ib_xmit_rdma()
936 	for (i = 0; i < work_alloc && scat != &op->op_sg[op->op_count]; i++) {  in rds_ib_xmit_rdma()
941 		nr_sig += rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_rdma()
943 		send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;  in rds_ib_xmit_rdma()
945 		send->s_wr.wr.rdma.rkey = op->op_rkey;  in rds_ib_xmit_rdma()
959 		for (j = 0; j < send->s_wr.num_sge && scat != &op->op_sg[op->op_count]; j++) {  in rds_ib_xmit_rdma()
982 	if (scat == &op->op_sg[op->op_count]) {  in rds_ib_xmit_rdma()
983 		prev->s_op = op;  in rds_ib_xmit_rdma()
984 		rds_message_addref(container_of(op, struct rds_message, rdma));  in rds_ib_xmit_rdma()