139 	struct rxe_send_wqe *wqe;  in get_wqe()  local
144 	wqe = queue_head(qp->sq.queue);  in get_wqe()
145 	*wqe_p = wqe;  in get_wqe()
148 	if (!wqe || wqe->state == wqe_state_posted)  in get_wqe()
152 	if (wqe->state == wqe_state_done)  in get_wqe()
156 	if (wqe->state == wqe_state_error)  in get_wqe()
172 					struct rxe_send_wqe *wqe)  in check_psn()  argument
179 	diff = psn_compare(pkt->psn, wqe->last_psn);  in check_psn()
181 		if (wqe->state == wqe_state_pending) {  in check_psn()
182 			if (wqe->mask & WR_ATOMIC_OR_READ_MASK)  in check_psn()
198 		if (pkt->psn == wqe->last_psn)  in check_psn()
202 	} else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) {  in check_psn()
211 					struct rxe_send_wqe *wqe)  in check_ack()  argument
233 			if ((pkt->psn == wqe->first_psn &&  in check_ack()
236 			    (wqe->first_psn == wqe->last_psn &&  in check_ack()
262 		if (wqe->wr.opcode != IB_WR_RDMA_READ &&  in check_ack()
263 		    wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) {  in check_ack()
264 			wqe->status = IB_WC_FATAL_ERR;  in check_ack()
276 		if (wqe->wr.opcode != IB_WR_ATOMIC_CMP_AND_SWP &&  in check_ack()
277 		    wqe->wr.opcode != IB_WR_ATOMIC_FETCH_AND_ADD)  in check_ack()
311 				wqe->status = IB_WC_REM_INV_REQ_ERR;  in check_ack()
315 				wqe->status = IB_WC_REM_ACCESS_ERR;  in check_ack()
319 				wqe->status = IB_WC_REM_OP_ERR;  in check_ack()
324 				wqe->status = IB_WC_REM_OP_ERR;  in check_ack()
342 				      struct rxe_send_wqe *wqe)  in do_read()  argument
347 			&wqe->dma, payload_addr(pkt),  in do_read()
352 	if (wqe->dma.resid == 0 && (pkt->mask & RXE_END_MASK))  in do_read()
360 					struct rxe_send_wqe *wqe)  in do_atomic()  argument
367 			&wqe->dma, &atomic_orig,  in do_atomic()
375 static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,  in make_send_cqe()  argument
383 		wc->wr_id		= wqe->wr.wr_id;  in make_send_cqe()
384 		wc->status		= wqe->status;  in make_send_cqe()
385 		wc->opcode		= wr_to_wc_opcode(wqe->wr.opcode);  in make_send_cqe()
386 		if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||  in make_send_cqe()
387 		    wqe->wr.opcode == IB_WR_SEND_WITH_IMM)  in make_send_cqe()
389 		wc->byte_len		= wqe->dma.length;  in make_send_cqe()
394 		uwc->wr_id		= wqe->wr.wr_id;  in make_send_cqe()
395 		uwc->status		= wqe->status;  in make_send_cqe()
396 		uwc->opcode		= wr_to_wc_opcode(wqe->wr.opcode);  in make_send_cqe()
397 		if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||  in make_send_cqe()
398 		    wqe->wr.opcode == IB_WR_SEND_WITH_IMM)  in make_send_cqe()
400 		uwc->byte_len		= wqe->dma.length;  in make_send_cqe()
413 static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)  in do_complete()  argument
419 	    (wqe->wr.send_flags & IB_SEND_SIGNALED) ||  in do_complete()
420 	    wqe->status != IB_WC_SUCCESS) {  in do_complete()
421 		make_send_cqe(qp, wqe, &cqe);  in do_complete()
428 	if (wqe->wr.opcode == IB_WR_SEND ||  in do_complete()
429 	    wqe->wr.opcode == IB_WR_SEND_WITH_IMM ||  in do_complete()
430 	    wqe->wr.opcode == IB_WR_SEND_WITH_INV)  in do_complete()
445 					   struct rxe_send_wqe *wqe)  in complete_ack()  argument
449 	if (wqe->has_rd_atomic) {  in complete_ack()
450 		wqe->has_rd_atomic = 0;  in complete_ack()
481 	do_complete(qp, wqe);  in complete_ack()
491 					   struct rxe_send_wqe *wqe)  in complete_wqe()  argument
493 	if (pkt && wqe->state == wqe_state_pending) {  in complete_wqe()
494 		if (psn_compare(wqe->last_psn, qp->comp.psn) >= 0) {  in complete_wqe()
495 			qp->comp.psn = (wqe->last_psn + 1) & BTH_PSN_MASK;  in complete_wqe()
505 	do_complete(qp, wqe);  in complete_wqe()
513 	struct rxe_send_wqe *wqe;  in rxe_drain_resp_pkts()  local
520 	while ((wqe = queue_head(qp->sq.queue))) {  in rxe_drain_resp_pkts()
522 			wqe->status = IB_WC_WR_FLUSH_ERR;  in rxe_drain_resp_pkts()
523 			do_complete(qp, wqe);  in rxe_drain_resp_pkts()
534 	struct rxe_send_wqe *wqe = NULL;  in rxe_completer()  local
574 			state = get_wqe(qp, pkt, &wqe);  in rxe_completer()
578 			state = check_psn(qp, pkt, wqe);  in rxe_completer()
582 			state = check_ack(qp, pkt, wqe);  in rxe_completer()
586 			state = do_read(qp, pkt, wqe);  in rxe_completer()
590 			state = do_atomic(qp, pkt, wqe);  in rxe_completer()
594 			if (wqe->state == wqe_state_pending &&  in rxe_completer()
595 			    wqe->last_psn == pkt->psn)  in rxe_completer()
602 			state = complete_ack(qp, pkt, wqe);  in rxe_completer()
606 			state = complete_wqe(qp, pkt, wqe);  in rxe_completer()
635 			if (qp->comp.timeout_retry && wqe) {  in rxe_completer()
666 			if (!wqe || (wqe->state == wqe_state_posted))  in rxe_completer()
713 				wqe->status = IB_WC_RETRY_EXC_ERR;  in rxe_completer()
736 				wqe->status = IB_WC_RNR_RETRY_EXC_ERR;  in rxe_completer()
742 			WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS);  in rxe_completer()
743 			do_complete(qp, wqe);  in rxe_completer()