52 	struct ipath_qp *qp;  in ipath_ud_loopback()  local
67 	qp = ipath_lookup_qpn(&dev->qp_table, swqe->wr.wr.ud.remote_qpn);  in ipath_ud_loopback()
68 	if (!qp || !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {  in ipath_ud_loopback()
78 	if (unlikely(qp->ibqp.qp_num &&  in ipath_ud_loopback()
80 		      sqp->qkey : swqe->wr.wr.ud.remote_qkey) != qp->qkey)) {  in ipath_ud_loopback()
106 	if (qp->ibqp.srq) {  in ipath_ud_loopback()
107 		srq = to_isrq(qp->ibqp.srq);  in ipath_ud_loopback()
113 		rq = &qp->r_rq;  in ipath_ud_loopback()
133 	rsge.sg_list = qp->r_ud_sg_list;  in ipath_ud_loopback()
134 	if (!ipath_init_sge(qp, wqe, &rlen, &rsge)) {  in ipath_ud_loopback()
168 			ev.device = qp->ibqp.device;  in ipath_ud_loopback()
169 			ev.element.srq = qp->ibqp.srq;  in ipath_ud_loopback()
214 	wc.qp = &qp->ibqp;  in ipath_ud_loopback()
226 	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,  in ipath_ud_loopback()
229 	if (atomic_dec_and_test(&qp->refcount))  in ipath_ud_loopback()
230 		wake_up(&qp->wait);  in ipath_ud_loopback()
240 int ipath_make_ud_req(struct ipath_qp *qp)  in ipath_make_ud_req()  argument
242 	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);  in ipath_make_ud_req()
255 	spin_lock_irqsave(&qp->s_lock, flags);  in ipath_make_ud_req()
257 	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_NEXT_SEND_OK)) {  in ipath_make_ud_req()
258 		if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))  in ipath_make_ud_req()
261 		if (qp->s_last == qp->s_head)  in ipath_make_ud_req()
264 		if (atomic_read(&qp->s_dma_busy)) {  in ipath_make_ud_req()
265 			qp->s_flags |= IPATH_S_WAIT_DMA;  in ipath_make_ud_req()
268 		wqe = get_swqe_ptr(qp, qp->s_last);  in ipath_make_ud_req()
269 		ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);  in ipath_make_ud_req()
273 	if (qp->s_cur == qp->s_head)  in ipath_make_ud_req()
276 	wqe = get_swqe_ptr(qp, qp->s_cur);  in ipath_make_ud_req()
277 	next_cur = qp->s_cur + 1;  in ipath_make_ud_req()
278 	if (next_cur >= qp->s_size)  in ipath_make_ud_req()
299 			if (atomic_read(&qp->s_dma_busy)) {  in ipath_make_ud_req()
300 				qp->s_flags |= IPATH_S_WAIT_DMA;  in ipath_make_ud_req()
303 			qp->s_cur = next_cur;  in ipath_make_ud_req()
304 			spin_unlock_irqrestore(&qp->s_lock, flags);  in ipath_make_ud_req()
305 			ipath_ud_loopback(qp, wqe);  in ipath_make_ud_req()
306 			spin_lock_irqsave(&qp->s_lock, flags);  in ipath_make_ud_req()
307 			ipath_send_complete(qp, wqe, IB_WC_SUCCESS);  in ipath_make_ud_req()
312 	qp->s_cur = next_cur;  in ipath_make_ud_req()
317 	qp->s_hdrwords = 7;  in ipath_make_ud_req()
318 	qp->s_cur_size = wqe->length;  in ipath_make_ud_req()
319 	qp->s_cur_sge = &qp->s_sge;  in ipath_make_ud_req()
320 	qp->s_dmult = ah_attr->static_rate;  in ipath_make_ud_req()
321 	qp->s_wqe = wqe;  in ipath_make_ud_req()
322 	qp->s_sge.sge = wqe->sg_list[0];  in ipath_make_ud_req()
323 	qp->s_sge.sg_list = wqe->sg_list + 1;  in ipath_make_ud_req()
324 	qp->s_sge.num_sge = wqe->wr.num_sge;  in ipath_make_ud_req()
328 		qp->s_hdrwords += ipath_make_grh(dev, &qp->s_hdr.u.l.grh,  in ipath_make_ud_req()
330 						 qp->s_hdrwords, nwords);  in ipath_make_ud_req()
332 		ohdr = &qp->s_hdr.u.l.oth;  in ipath_make_ud_req()
340 		ohdr = &qp->s_hdr.u.oth;  in ipath_make_ud_req()
343 		qp->s_hdrwords++;  in ipath_make_ud_req()
349 	if (qp->ibqp.qp_type == IB_QPT_SMI)  in ipath_make_ud_req()
351 	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);  in ipath_make_ud_req()
352 	qp->s_hdr.lrh[1] = cpu_to_be16(ah_attr->dlid);	/* DEST LID */  in ipath_make_ud_req()
353 	qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords +  in ipath_make_ud_req()
359 		qp->s_hdr.lrh[3] = cpu_to_be16(lid);  in ipath_make_ud_req()
361 		qp->s_hdr.lrh[3] = IB_LID_PERMISSIVE;  in ipath_make_ud_req()
365 	bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? IPATH_DEFAULT_P_KEY :  in ipath_make_ud_req()
366 		ipath_get_pkey(dev->dd, qp->s_pkey_index);  in ipath_make_ud_req()
375 	ohdr->bth[2] = cpu_to_be32(qp->s_next_psn++ & IPATH_PSN_MASK);  in ipath_make_ud_req()
381 					 qp->qkey : wqe->wr.wr.ud.remote_qkey);  in ipath_make_ud_req()
382 	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);  in ipath_make_ud_req()
389 	qp->s_flags &= ~IPATH_S_BUSY;  in ipath_make_ud_req()
391 	spin_unlock_irqrestore(&qp->s_lock, flags);  in ipath_make_ud_req()
409 		  int has_grh, void *data, u32 tlen, struct ipath_qp *qp)  in ipath_ud_rcv()  argument
452 	if (qp->ibqp.qp_num) {  in ipath_ud_rcv()
458 		if (unlikely(qkey != qp->qkey)) {  in ipath_ud_rcv()
479 	if (qp->ibqp.qp_num > 1 &&  in ipath_ud_rcv()
506 	if (unlikely((qp->ibqp.qp_num == 0 &&  in ipath_ud_rcv()
509 		     (qp->ibqp.qp_num == 1 &&  in ipath_ud_rcv()
525 	if (qp->r_flags & IPATH_R_REUSE_SGE)  in ipath_ud_rcv()
526 		qp->r_flags &= ~IPATH_R_REUSE_SGE;  in ipath_ud_rcv()
527 	else if (!ipath_get_rwqe(qp, 0)) {  in ipath_ud_rcv()
534 		if (qp->ibqp.qp_num == 0)  in ipath_ud_rcv()
541 	if (wc.byte_len > qp->r_len) {  in ipath_ud_rcv()
542 		qp->r_flags |= IPATH_R_REUSE_SGE;  in ipath_ud_rcv()
547 		ipath_copy_sge(&qp->r_sge, &hdr->u.l.grh,  in ipath_ud_rcv()
551 		ipath_skip_sge(&qp->r_sge, sizeof(struct ib_grh));  in ipath_ud_rcv()
552 	ipath_copy_sge(&qp->r_sge, data,  in ipath_ud_rcv()
554 	if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))  in ipath_ud_rcv()
556 	wc.wr_id = qp->r_wr_id;  in ipath_ud_rcv()
560 	wc.qp = &qp->ibqp;  in ipath_ud_rcv()
574 	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,  in ipath_ud_rcv()