150 						struct rds_ib_send_work *send,  in rds_ib_send_unmap_op()  argument
156 	switch (send->s_wr.opcode) {  in rds_ib_send_unmap_op()
158 		if (send->s_op) {  in rds_ib_send_unmap_op()
159 			rm = container_of(send->s_op, struct rds_message, data);  in rds_ib_send_unmap_op()
160 			rds_ib_send_unmap_data(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
165 		if (send->s_op) {  in rds_ib_send_unmap_op()
166 			rm = container_of(send->s_op, struct rds_message, rdma);  in rds_ib_send_unmap_op()
167 			rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
172 		if (send->s_op) {  in rds_ib_send_unmap_op()
173 			rm = container_of(send->s_op, struct rds_message, atomic);  in rds_ib_send_unmap_op()
174 			rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
180 			       __func__, send->s_wr.opcode);  in rds_ib_send_unmap_op()
184 	send->s_wr.opcode = 0xdead;  in rds_ib_send_unmap_op()
191 	struct rds_ib_send_work *send;  in rds_ib_send_init_ring()  local
194 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_init_ring()
197 		send->s_op = NULL;  in rds_ib_send_init_ring()
199 		send->s_wr.wr_id = i;  in rds_ib_send_init_ring()
200 		send->s_wr.sg_list = send->s_sge;  in rds_ib_send_init_ring()
201 		send->s_wr.ex.imm_data = 0;  in rds_ib_send_init_ring()
203 		sge = &send->s_sge[0];  in rds_ib_send_init_ring()
208 		send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
214 	struct rds_ib_send_work *send;  in rds_ib_send_clear_ring()  local
217 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_clear_ring()
218 		if (send->s_op && send->s_wr.opcode != 0xdead)  in rds_ib_send_clear_ring()
219 			rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_ib_send_clear_ring()
245 	struct rds_ib_send_work *send;  in rds_ib_send_cqe_handler()  local
270 		send = &ic->i_sends[oldest];  in rds_ib_send_cqe_handler()
271 		if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_send_cqe_handler()
274 		rm = rds_ib_send_unmap_op(ic, send, wc->status);  in rds_ib_send_cqe_handler()
276 		if (time_after(jiffies, send->s_queued + HZ / 2))  in rds_ib_send_cqe_handler()
279 		if (send->s_op) {  in rds_ib_send_cqe_handler()
280 			if (send->s_op == rm->m_final_op) {  in rds_ib_send_cqe_handler()
287 			send->s_op = NULL;  in rds_ib_send_cqe_handler()
454 					     struct rds_ib_send_work *send,  in rds_ib_set_wr_signal_state()  argument
464 		send->s_wr.send_flags |= IB_SEND_SIGNALED;  in rds_ib_set_wr_signal_state()
488 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit()  local
618 	send = &ic->i_sends[pos];  in rds_ib_xmit()
619 	first = send;  in rds_ib_xmit()
627 		send->s_wr.send_flags = send_flags;  in rds_ib_xmit()
628 		send->s_wr.opcode = IB_WR_SEND;  in rds_ib_xmit()
629 		send->s_wr.num_sge = 1;  in rds_ib_xmit()
630 		send->s_wr.next = NULL;  in rds_ib_xmit()
631 		send->s_queued = jiffies;  in rds_ib_xmit()
632 		send->s_op = NULL;  in rds_ib_xmit()
634 		send->s_sge[0].addr = ic->i_send_hdrs_dma  in rds_ib_xmit()
636 		send->s_sge[0].length = sizeof(struct rds_header);  in rds_ib_xmit()
645 			send->s_wr.num_sge = 2;  in rds_ib_xmit()
647 			send->s_sge[1].addr = sg_dma_address(scat);  in rds_ib_xmit()
648 			send->s_sge[1].addr += rm->data.op_dmaoff;  in rds_ib_xmit()
649 			send->s_sge[1].length = len;  in rds_ib_xmit()
660 		rds_ib_set_wr_signal_state(ic, send, false);  in rds_ib_xmit()
666 			rds_ib_set_wr_signal_state(ic, send, true);  in rds_ib_xmit()
667 			send->s_wr.send_flags |= IB_SEND_SOLICITED;  in rds_ib_xmit()
670 		if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_xmit()
673 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit()
674 			 &send->s_wr, send->s_wr.num_sge, send->s_wr.next);  in rds_ib_xmit()
687 			prev->s_wr.next = &send->s_wr;  in rds_ib_xmit()
688 		prev = send;  in rds_ib_xmit()
691 		send = &ic->i_sends[pos];  in rds_ib_xmit()
756 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_atomic()  local
771 	send = &ic->i_sends[pos];  in rds_ib_xmit_atomic()
772 	send->s_queued = jiffies;  in rds_ib_xmit_atomic()
775 		send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;  in rds_ib_xmit_atomic()
776 		send->s_atomic_wr.compare_add = op->op_m_cswp.compare;  in rds_ib_xmit_atomic()
777 		send->s_atomic_wr.swap = op->op_m_cswp.swap;  in rds_ib_xmit_atomic()
778 		send->s_atomic_wr.compare_add_mask = op->op_m_cswp.compare_mask;  in rds_ib_xmit_atomic()
779 		send->s_atomic_wr.swap_mask = op->op_m_cswp.swap_mask;  in rds_ib_xmit_atomic()
781 		send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;  in rds_ib_xmit_atomic()
782 		send->s_atomic_wr.compare_add = op->op_m_fadd.add;  in rds_ib_xmit_atomic()
783 		send->s_atomic_wr.swap = 0;  in rds_ib_xmit_atomic()
784 		send->s_atomic_wr.compare_add_mask = op->op_m_fadd.nocarry_mask;  in rds_ib_xmit_atomic()
785 		send->s_atomic_wr.swap_mask = 0;  in rds_ib_xmit_atomic()
787 	send->s_wr.send_flags = 0;  in rds_ib_xmit_atomic()
788 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
789 	send->s_atomic_wr.wr.num_sge = 1;  in rds_ib_xmit_atomic()
790 	send->s_atomic_wr.wr.next = NULL;  in rds_ib_xmit_atomic()
791 	send->s_atomic_wr.remote_addr = op->op_remote_addr;  in rds_ib_xmit_atomic()
792 	send->s_atomic_wr.rkey = op->op_rkey;  in rds_ib_xmit_atomic()
793 	send->s_op = op;  in rds_ib_xmit_atomic()
794 	rds_message_addref(container_of(send->s_op, struct rds_message, atomic));  in rds_ib_xmit_atomic()
807 	send->s_sge[0].addr = sg_dma_address(op->op_sg);  in rds_ib_xmit_atomic()
808 	send->s_sge[0].length = sg_dma_len(op->op_sg);  in rds_ib_xmit_atomic()
809 	send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_atomic()
812 		 send->s_sge[0].addr, send->s_sge[0].length);  in rds_ib_xmit_atomic()
817 	failed_wr = &send->s_atomic_wr.wr;  in rds_ib_xmit_atomic()
818 	ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);  in rds_ib_xmit_atomic()
820 		 send, &send->s_atomic_wr, ret, failed_wr);  in rds_ib_xmit_atomic()
821 	BUG_ON(failed_wr != &send->s_atomic_wr.wr);  in rds_ib_xmit_atomic()
830 	if (unlikely(failed_wr != &send->s_atomic_wr.wr)) {  in rds_ib_xmit_atomic()
832 		BUG_ON(failed_wr != &send->s_atomic_wr.wr);  in rds_ib_xmit_atomic()
842 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_rdma()  local
888 	send = &ic->i_sends[pos];  in rds_ib_xmit_rdma()
889 	first = send;  in rds_ib_xmit_rdma()
896 		send->s_wr.send_flags = 0;  in rds_ib_xmit_rdma()
897 		send->s_queued = jiffies;  in rds_ib_xmit_rdma()
898 		send->s_op = NULL;  in rds_ib_xmit_rdma()
901 			nr_sig += rds_ib_set_wr_signal_state(ic, send,  in rds_ib_xmit_rdma()
904 		send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;  in rds_ib_xmit_rdma()
905 		send->s_rdma_wr.remote_addr = remote_addr;  in rds_ib_xmit_rdma()
906 		send->s_rdma_wr.rkey = op->op_rkey;  in rds_ib_xmit_rdma()
909 			send->s_rdma_wr.wr.num_sge = max_sge;  in rds_ib_xmit_rdma()
912 			send->s_rdma_wr.wr.num_sge = num_sge;  in rds_ib_xmit_rdma()
915 		send->s_rdma_wr.wr.next = NULL;  in rds_ib_xmit_rdma()
918 			prev->s_rdma_wr.wr.next = &send->s_rdma_wr.wr;  in rds_ib_xmit_rdma()
920 		for (j = 0; j < send->s_rdma_wr.wr.num_sge &&  in rds_ib_xmit_rdma()
923 			send->s_sge[j].addr = sg_dma_address(scat);  in rds_ib_xmit_rdma()
924 			send->s_sge[j].length = len;  in rds_ib_xmit_rdma()
925 			send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_rdma()
934 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit_rdma()
935 			&send->s_rdma_wr.wr,  in rds_ib_xmit_rdma()
936 			send->s_rdma_wr.wr.num_sge,  in rds_ib_xmit_rdma()
937 			send->s_rdma_wr.wr.next);  in rds_ib_xmit_rdma()
939 		prev = send;  in rds_ib_xmit_rdma()
940 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_ib_xmit_rdma()
941 			send = ic->i_sends;  in rds_ib_xmit_rdma()