Lines Matching refs:qp
58 struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev, in find_prev_entry() argument
60 __must_hold(&qp->s_lock) in find_prev_entry()
66 for (i = qp->r_head_ack_queue; ; i = p) { in find_prev_entry()
67 if (i == qp->s_tail_ack_queue) in find_prev_entry()
72 p = rvt_size_atomic(ib_to_rvt(qp->ibqp.device)); in find_prev_entry()
73 if (p == qp->r_head_ack_queue) { in find_prev_entry()
77 e = &qp->s_ack_queue[p]; in find_prev_entry()
83 if (p == qp->s_tail_ack_queue && in find_prev_entry()
109 static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, in make_rc_ack() argument
117 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); in make_rc_ack()
119 u32 pmtu = qp->pmtu; in make_rc_ack()
120 struct hfi1_qp_priv *qpriv = qp->priv; in make_rc_ack()
123 u8 next = qp->s_tail_ack_queue; in make_rc_ack()
126 trace_hfi1_rsp_make_rc_ack(qp, 0); in make_rc_ack()
127 lockdep_assert_held(&qp->s_lock); in make_rc_ack()
129 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in make_rc_ack()
139 switch (qp->s_ack_state) { in make_rc_ack()
142 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
157 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
159 qp->s_acked_ack_queue == qp->s_tail_ack_queue) in make_rc_ack()
160 qp->s_acked_ack_queue = next; in make_rc_ack()
161 qp->s_tail_ack_queue = next; in make_rc_ack()
162 trace_hfi1_rsp_make_rc_ack(qp, e->psn); in make_rc_ack()
167 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { in make_rc_ack()
168 if (qp->s_flags & RVT_S_ACK_PENDING) in make_rc_ack()
173 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
176 hfi1_tid_rdma_ack_interlock(qp, e)) { in make_rc_ack()
189 if (qp->s_acked_ack_queue == in make_rc_ack()
190 qp->s_tail_ack_queue) in make_rc_ack()
191 qp->s_acked_ack_queue = in make_rc_ack()
192 qp->r_head_ack_queue; in make_rc_ack()
193 qp->s_tail_ack_queue = qp->r_head_ack_queue; in make_rc_ack()
200 qp->s_ack_rdma_sge.sge = e->rdma_sge; in make_rc_ack()
201 qp->s_ack_rdma_sge.num_sge = 1; in make_rc_ack()
202 ps->s_txreq->ss = &qp->s_ack_rdma_sge; in make_rc_ack()
205 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); in make_rc_ack()
207 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); in make_rc_ack()
210 ohdr->u.aeth = rvt_compute_aeth(qp); in make_rc_ack()
212 qp->s_ack_rdma_psn = e->psn; in make_rc_ack()
213 bth2 = mask_psn(qp->s_ack_rdma_psn++); in make_rc_ack()
226 qp->s_ack_state = TID_OP(WRITE_RESP); in make_rc_ack()
227 qp->s_ack_rdma_psn = mask_psn(e->psn + req->cur_seg); in make_rc_ack()
238 if (qp->s_acked_ack_queue == in make_rc_ack()
239 qp->s_tail_ack_queue) in make_rc_ack()
240 qp->s_acked_ack_queue = in make_rc_ack()
241 qp->r_head_ack_queue; in make_rc_ack()
242 qp->s_tail_ack_queue = qp->r_head_ack_queue; in make_rc_ack()
249 qp->s_ack_rdma_sge.sge = e->rdma_sge; in make_rc_ack()
250 qp->s_ack_rdma_sge.num_sge = 1; in make_rc_ack()
251 qp->s_ack_state = TID_OP(READ_RESP); in make_rc_ack()
257 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in make_rc_ack()
258 ohdr->u.at.aeth = rvt_compute_aeth(qp); in make_rc_ack()
264 trace_hfi1_tid_write_rsp_make_rc_ack(qp); in make_rc_ack()
265 bth0 = qp->s_ack_state << 24; in make_rc_ack()
269 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); in make_rc_ack()
272 ps->s_txreq->ss = &qp->s_ack_rdma_sge; in make_rc_ack()
273 ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr; in make_rc_ack()
276 len = qp->s_ack_rdma_sge.sge.sge_length; in make_rc_ack()
281 ohdr->u.aeth = rvt_compute_aeth(qp); in make_rc_ack()
283 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in make_rc_ack()
284 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
287 bth0 = qp->s_ack_state << 24; in make_rc_ack()
288 bth2 = mask_psn(qp->s_ack_rdma_psn++); in make_rc_ack()
312 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
321 qp->s_tail_ack_queue == qpriv->r_tid_alloc && in make_rc_ack()
327 bth2 = mask_psn(qp->s_ack_rdma_psn); in make_rc_ack()
328 hdrlen = hfi1_build_tid_rdma_write_resp(qp, e, ohdr, &bth1, in make_rc_ack()
335 bth0 = qp->s_ack_state << 24; in make_rc_ack()
336 qp->s_ack_rdma_psn++; in make_rc_ack()
337 trace_hfi1_tid_req_make_rc_ack_write(qp, 0, e->opcode, e->psn, in make_rc_ack()
344 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in make_rc_ack()
349 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
350 ps->s_txreq->ss = &qp->s_ack_rdma_sge; in make_rc_ack()
351 delta = hfi1_build_tid_rdma_read_resp(qp, e, ohdr, &bth0, in make_rc_ack()
363 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in make_rc_ack()
377 qp->s_ack_state = OP(SEND_ONLY); in make_rc_ack()
379 if (qp->s_nak_state) in make_rc_ack()
381 cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in make_rc_ack()
382 (qp->s_nak_state << in make_rc_ack()
385 ohdr->u.aeth = rvt_compute_aeth(qp); in make_rc_ack()
389 bth2 = mask_psn(qp->s_ack_psn); in make_rc_ack()
390 qp->s_flags &= ~RVT_S_ACK_PENDING; in make_rc_ack()
394 qp->s_rdma_ack_cnt++; in make_rc_ack()
398 hfi1_make_ruc_header(qp, ohdr, bth0, bth1, bth2, middle, ps); in make_rc_ack()
401 spin_unlock_irqrestore(&qp->s_lock, ps->flags); in make_rc_ack()
402 spin_lock_irqsave(&qp->r_lock, ps->flags); in make_rc_ack()
403 spin_lock(&qp->s_lock); in make_rc_ack()
404 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in make_rc_ack()
405 spin_unlock(&qp->s_lock); in make_rc_ack()
406 spin_unlock_irqrestore(&qp->r_lock, ps->flags); in make_rc_ack()
407 spin_lock_irqsave(&qp->s_lock, ps->flags); in make_rc_ack()
409 qp->s_ack_state = OP(ACKNOWLEDGE); in make_rc_ack()
415 qp->s_flags &= ~(RVT_S_RESP_PENDING in make_rc_ack()
429 int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) in hfi1_make_rc_req() argument
431 struct hfi1_qp_priv *priv = qp->priv; in hfi1_make_rc_req()
432 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); in hfi1_make_rc_req()
442 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); in hfi1_make_rc_req()
443 u32 pmtu = qp->pmtu; in hfi1_make_rc_req()
450 trace_hfi1_sender_make_rc_req(qp); in hfi1_make_rc_req()
451 lockdep_assert_held(&qp->s_lock); in hfi1_make_rc_req()
452 ps->s_txreq = get_txreq(ps->dev, qp); in hfi1_make_rc_req()
459 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) in hfi1_make_rc_req()
466 if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && in hfi1_make_rc_req()
467 (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr)))) in hfi1_make_rc_req()
474 if ((qp->s_flags & RVT_S_RESP_PENDING) && in hfi1_make_rc_req()
475 make_rc_ack(dev, qp, ohdr, ps)) in hfi1_make_rc_req()
478 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { in hfi1_make_rc_req()
479 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) in hfi1_make_rc_req()
482 if (qp->s_last == READ_ONCE(qp->s_head)) in hfi1_make_rc_req()
486 qp->s_flags |= RVT_S_WAIT_DMA; in hfi1_make_rc_req()
489 clear_ahg(qp); in hfi1_make_rc_req()
490 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_make_rc_req()
491 hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in hfi1_make_rc_req()
497 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK | HFI1_S_WAIT_HALT)) in hfi1_make_rc_req()
500 if (cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) { in hfi1_make_rc_req()
501 if (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { in hfi1_make_rc_req()
502 qp->s_flags |= RVT_S_WAIT_PSN; in hfi1_make_rc_req()
505 qp->s_sending_psn = qp->s_psn; in hfi1_make_rc_req()
506 qp->s_sending_hpsn = qp->s_psn - 1; in hfi1_make_rc_req()
510 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in hfi1_make_rc_req()
512 switch (qp->s_state) { in hfi1_make_rc_req()
514 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) in hfi1_make_rc_req()
524 if (qp->s_cur == qp->s_tail) { in hfi1_make_rc_req()
526 if (qp->s_tail == READ_ONCE(qp->s_head)) { in hfi1_make_rc_req()
527 clear_ahg(qp); in hfi1_make_rc_req()
537 qp->s_num_rd_atomic && in hfi1_make_rc_req()
539 priv->pending_tid_r_segs < qp->s_num_rd_atomic)) { in hfi1_make_rc_req()
540 qp->s_flags |= RVT_S_WAIT_FENCE; in hfi1_make_rc_req()
552 if (qp->s_last != qp->s_cur) in hfi1_make_rc_req()
554 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
555 qp->s_cur = 0; in hfi1_make_rc_req()
556 if (++qp->s_tail == qp->s_size) in hfi1_make_rc_req()
557 qp->s_tail = 0; in hfi1_make_rc_req()
561 qp, in hfi1_make_rc_req()
565 rvt_send_complete(qp, wqe, in hfi1_make_rc_req()
569 atomic_dec(&qp->local_ops_pending); in hfi1_make_rc_req()
574 qp->s_psn = wqe->psn; in hfi1_make_rc_req()
582 ss = &qp->s_sge; in hfi1_make_rc_req()
583 bth2 = mask_psn(qp->s_psn); in hfi1_make_rc_req()
590 hfi1_tid_rdma_wqe_interlock(qp, wqe)) in hfi1_make_rc_req()
598 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
601 qp->s_state = OP(SEND_FIRST); in hfi1_make_rc_req()
606 qp->s_state = OP(SEND_ONLY); in hfi1_make_rc_req()
608 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); in hfi1_make_rc_req()
613 qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE); in hfi1_make_rc_req()
622 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
623 qp->s_cur = 0; in hfi1_make_rc_req()
627 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
628 qp->s_lsn++; in hfi1_make_rc_req()
632 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
643 qp->s_state = OP(RDMA_WRITE_FIRST); in hfi1_make_rc_req()
648 qp->s_state = OP(RDMA_WRITE_ONLY); in hfi1_make_rc_req()
650 qp->s_state = in hfi1_make_rc_req()
659 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
660 qp->s_cur = 0; in hfi1_make_rc_req()
672 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
673 qp->s_lsn++; in hfi1_make_rc_req()
676 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, in hfi1_make_rc_req()
681 priv->s_tid_cur = qp->s_cur; in hfi1_make_rc_req()
683 priv->s_tid_tail = qp->s_cur; in hfi1_make_rc_req()
690 __w = rvt_get_swqe_ptr(qp, priv->s_tid_cur); in hfi1_make_rc_req()
723 priv->s_tid_tail = qp->s_cur; in hfi1_make_rc_req()
727 priv->s_tid_cur = qp->s_cur; in hfi1_make_rc_req()
739 if (priv->s_tid_tail == qp->s_cur && in hfi1_make_rc_req()
745 priv->s_tid_head = qp->s_cur; in hfi1_make_rc_req()
761 trace_hfi1_tid_write_sender_make_req(qp, newreq); in hfi1_make_rc_req()
762 trace_hfi1_tid_req_make_req_write(qp, newreq, in hfi1_make_rc_req()
766 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
767 qp->s_cur = 0; in hfi1_make_rc_req()
775 if (qp->s_num_rd_atomic >= in hfi1_make_rc_req()
776 qp->s_max_rd_atomic) { in hfi1_make_rc_req()
777 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
780 qp->s_num_rd_atomic++; in hfi1_make_rc_req()
781 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
782 qp->s_lsn++; in hfi1_make_rc_req()
789 qp->s_state = OP(RDMA_READ_REQUEST); in hfi1_make_rc_req()
794 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
795 qp->s_cur = 0; in hfi1_make_rc_req()
799 trace_hfi1_tid_read_sender_make_req(qp, newreq); in hfi1_make_rc_req()
802 trace_hfi1_tid_req_make_req_read(qp, newreq, in hfi1_make_rc_req()
806 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
818 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { in hfi1_make_rc_req()
819 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
833 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
834 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
835 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
836 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
837 qp->s_len = wqe->length; in hfi1_make_rc_req()
851 req->s_next_psn = qp->s_psn; in hfi1_make_rc_req()
855 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, in hfi1_make_rc_req()
862 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
863 qp->s_lsn++; in hfi1_make_rc_req()
868 ++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
869 qp->s_cur = 0; in hfi1_make_rc_req()
878 if (qp->s_num_rd_atomic >= in hfi1_make_rc_req()
879 qp->s_max_rd_atomic) { in hfi1_make_rc_req()
880 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
883 qp->s_num_rd_atomic++; in hfi1_make_rc_req()
886 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
887 qp->s_lsn++; in hfi1_make_rc_req()
890 qp->s_state = OP(COMPARE_SWAP); in hfi1_make_rc_req()
896 qp->s_state = OP(FETCH_ADD); in hfi1_make_rc_req()
909 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
910 qp->s_cur = 0; in hfi1_make_rc_req()
917 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
918 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
919 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
920 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
921 qp->s_len = wqe->length; in hfi1_make_rc_req()
924 qp->s_tail++; in hfi1_make_rc_req()
925 if (qp->s_tail >= qp->s_size) in hfi1_make_rc_req()
926 qp->s_tail = 0; in hfi1_make_rc_req()
930 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
932 qp->s_psn = req->s_next_psn; in hfi1_make_rc_req()
934 qp->s_psn++; in hfi1_make_rc_req()
947 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
950 qp->s_state = OP(SEND_MIDDLE); in hfi1_make_rc_req()
953 bth2 = mask_psn(qp->s_psn++); in hfi1_make_rc_req()
954 ss = &qp->s_sge; in hfi1_make_rc_req()
955 len = qp->s_len; in hfi1_make_rc_req()
962 qp->s_state = OP(SEND_LAST); in hfi1_make_rc_req()
964 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); in hfi1_make_rc_req()
969 qp->s_state = OP(SEND_LAST_WITH_INVALIDATE); in hfi1_make_rc_req()
977 qp->s_cur++; in hfi1_make_rc_req()
978 if (qp->s_cur >= qp->s_size) in hfi1_make_rc_req()
979 qp->s_cur = 0; in hfi1_make_rc_req()
992 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
995 qp->s_state = OP(RDMA_WRITE_MIDDLE); in hfi1_make_rc_req()
998 bth2 = mask_psn(qp->s_psn++); in hfi1_make_rc_req()
999 ss = &qp->s_sge; in hfi1_make_rc_req()
1000 len = qp->s_len; in hfi1_make_rc_req()
1007 qp->s_state = OP(RDMA_WRITE_LAST); in hfi1_make_rc_req()
1009 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); in hfi1_make_rc_req()
1017 qp->s_cur++; in hfi1_make_rc_req()
1018 if (qp->s_cur >= qp->s_size) in hfi1_make_rc_req()
1019 qp->s_cur = 0; in hfi1_make_rc_req()
1032 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu; in hfi1_make_rc_req()
1039 qp->s_state = OP(RDMA_READ_REQUEST); in hfi1_make_rc_req()
1041 bth2 = mask_psn(qp->s_psn) | IB_BTH_REQ_ACK; in hfi1_make_rc_req()
1042 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1045 qp->s_cur++; in hfi1_make_rc_req()
1046 if (qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1047 qp->s_cur = 0; in hfi1_make_rc_req()
1060 req->comp_seg = delta_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1064 bth2 = mask_psn(qp->s_psn); in hfi1_make_rc_req()
1065 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1067 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1069 qp->s_state = TID_OP(WRITE_REQ); in hfi1_make_rc_req()
1071 priv->s_tid_cur = qp->s_cur; in hfi1_make_rc_req()
1072 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1073 qp->s_cur = 0; in hfi1_make_rc_req()
1074 trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1089 req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps; in hfi1_make_rc_req()
1098 hfi1_tid_rdma_restart_req(qp, wqe, &bth2); in hfi1_make_rc_req()
1105 hfi1_kern_clear_hw_flow(priv->rcd, qp); in hfi1_make_rc_req()
1107 hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR); in hfi1_make_rc_req()
1126 ++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1127 qp->s_cur = 0; in hfi1_make_rc_req()
1128 qp->s_psn = req->s_next_psn; in hfi1_make_rc_req()
1129 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1134 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1141 qp->s_cur == qp->s_tail) { in hfi1_make_rc_req()
1142 qp->s_state = OP(RDMA_READ_REQUEST); in hfi1_make_rc_req()
1143 if (delta == 0 || qp->s_cur == qp->s_tail) in hfi1_make_rc_req()
1150 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { in hfi1_make_rc_req()
1151 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
1159 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1169 ++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1170 qp->s_cur = 0; in hfi1_make_rc_req()
1171 qp->s_psn = req->s_next_psn; in hfi1_make_rc_req()
1172 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1176 qp->s_sending_hpsn = bth2; in hfi1_make_rc_req()
1181 if (qp->s_flags & RVT_S_SEND_ONE) { in hfi1_make_rc_req()
1182 qp->s_flags &= ~RVT_S_SEND_ONE; in hfi1_make_rc_req()
1183 qp->s_flags |= RVT_S_WAIT_ACK; in hfi1_make_rc_req()
1186 qp->s_len -= len; in hfi1_make_rc_req()
1192 qp, in hfi1_make_rc_req()
1194 bth0 | (qp->s_state << 24), in hfi1_make_rc_req()
1211 qp->s_flags &= ~RVT_S_BUSY; in hfi1_make_rc_req()
1221 static inline void hfi1_make_bth_aeth(struct rvt_qp *qp, in hfi1_make_bth_aeth() argument
1225 if (qp->r_nak_state) in hfi1_make_bth_aeth()
1226 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in hfi1_make_bth_aeth()
1227 (qp->r_nak_state << in hfi1_make_bth_aeth()
1230 ohdr->u.aeth = rvt_compute_aeth(qp); in hfi1_make_bth_aeth()
1233 ohdr->bth[1] = cpu_to_be32(bth1 | qp->remote_qpn); in hfi1_make_bth_aeth()
1234 ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn)); in hfi1_make_bth_aeth()
1239 struct rvt_qp *qp = packet->qp; in hfi1_queue_rc_ack() local
1243 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_queue_rc_ack()
1244 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in hfi1_queue_rc_ack()
1248 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; in hfi1_queue_rc_ack()
1249 qp->s_nak_state = qp->r_nak_state; in hfi1_queue_rc_ack()
1250 qp->s_ack_psn = qp->r_ack_psn; in hfi1_queue_rc_ack()
1252 qp->s_flags |= RVT_S_ECN; in hfi1_queue_rc_ack()
1255 hfi1_schedule_send(qp); in hfi1_queue_rc_ack()
1257 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_queue_rc_ack()
1266 struct rvt_qp *qp = packet->qp; in hfi1_make_rc_ack_9B() local
1280 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { in hfi1_make_rc_ack_9B()
1282 rdma_ah_read_grh(&qp->remote_ah_attr), in hfi1_make_rc_ack_9B()
1291 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); in hfi1_make_rc_ack_9B()
1294 (rdma_ah_get_sl(&qp->remote_ah_attr) & IB_SL_MASK) << in hfi1_make_rc_ack_9B()
1298 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), in hfi1_make_rc_ack_9B()
1299 ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr)); in hfi1_make_rc_ack_9B()
1302 if (qp->s_mig_state == IB_MIG_MIGRATED) in hfi1_make_rc_ack_9B()
1310 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); in hfi1_make_rc_ack_9B()
1319 struct rvt_qp *qp = packet->qp; in hfi1_make_rc_ack_16B() local
1337 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && in hfi1_make_rc_ack_16B()
1338 hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { in hfi1_make_rc_ack_16B()
1340 rdma_ah_read_grh(&qp->remote_ah_attr), in hfi1_make_rc_ack_16B()
1348 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); in hfi1_make_rc_ack_16B()
1354 (rdma_ah_get_path_bits(&qp->remote_ah_attr) & in hfi1_make_rc_ack_16B()
1356 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), in hfi1_make_rc_ack_16B()
1361 if (qp->s_mig_state == IB_MIG_MIGRATED) in hfi1_make_rc_ack_16B()
1363 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); in hfi1_make_rc_ack_16B()
1389 struct rvt_qp *qp = packet->qp; in hfi1_send_rc_ack() local
1391 struct hfi1_qp_priv *priv = qp->priv; in hfi1_send_rc_ack()
1393 u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; in hfi1_send_rc_ack()
1402 qp->r_adefered = 0; in hfi1_send_rc_ack()
1405 if (qp->s_flags & RVT_S_RESP_PENDING) { in hfi1_send_rc_ack()
1411 if (qp->s_rdma_ack_cnt) { in hfi1_send_rc_ack()
1425 pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, in hfi1_send_rc_ack()
1438 trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), in hfi1_send_rc_ack()
1459 static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn, in update_num_rd_atomic() argument
1467 qp->s_num_rd_atomic++; in update_num_rd_atomic()
1470 struct hfi1_qp_priv *priv = qp->priv; in update_num_rd_atomic()
1478 qp->s_num_rd_atomic += req->ack_pending; in update_num_rd_atomic()
1479 trace_hfi1_tid_req_update_num_rd_atomic(qp, 0, in update_num_rd_atomic()
1486 qp->s_num_rd_atomic += req->total_segs; in update_num_rd_atomic()
1500 static void reset_psn(struct rvt_qp *qp, u32 psn) in reset_psn() argument
1502 u32 n = qp->s_acked; in reset_psn()
1503 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
1505 struct hfi1_qp_priv *priv = qp->priv; in reset_psn()
1507 lockdep_assert_held(&qp->s_lock); in reset_psn()
1508 qp->s_cur = n; in reset_psn()
1511 qp->s_num_rd_atomic = 0; in reset_psn()
1518 qp->s_state = OP(SEND_LAST); in reset_psn()
1521 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1527 if (++n == qp->s_size) in reset_psn()
1529 if (n == qp->s_tail) in reset_psn()
1531 wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
1535 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in reset_psn()
1538 qp->s_cur = n; in reset_psn()
1544 qp->s_state = OP(SEND_LAST); in reset_psn()
1548 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1560 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); in reset_psn()
1565 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); in reset_psn()
1569 qp->s_state = TID_OP(WRITE_RESP); in reset_psn()
1573 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); in reset_psn()
1577 qp->s_state = TID_OP(READ_RESP); in reset_psn()
1585 qp->s_state = OP(SEND_LAST); in reset_psn()
1589 qp->s_psn = psn; in reset_psn()
1595 if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) && in reset_psn()
1596 (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) in reset_psn()
1597 qp->s_flags |= RVT_S_WAIT_PSN; in reset_psn()
1598 qp->s_flags &= ~HFI1_S_AHG_VALID; in reset_psn()
1599 trace_hfi1_sender_reset_psn(qp); in reset_psn()
1606 void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait) in hfi1_restart_rc() argument
1608 struct hfi1_qp_priv *priv = qp->priv; in hfi1_restart_rc()
1609 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_restart_rc()
1612 lockdep_assert_held(&qp->r_lock); in hfi1_restart_rc()
1613 lockdep_assert_held(&qp->s_lock); in hfi1_restart_rc()
1614 trace_hfi1_sender_restart_rc(qp); in hfi1_restart_rc()
1615 if (qp->s_retry == 0) { in hfi1_restart_rc()
1616 if (qp->s_mig_state == IB_MIG_ARMED) { in hfi1_restart_rc()
1617 hfi1_migrate_qp(qp); in hfi1_restart_rc()
1618 qp->s_retry = qp->s_retry_cnt; in hfi1_restart_rc()
1619 } else if (qp->s_last == qp->s_acked) { in hfi1_restart_rc()
1626 to_iport(qp->ibqp.device, qp->port_num); in hfi1_restart_rc()
1632 opfn_conn_reply(qp, priv->opfn.curr); in hfi1_restart_rc()
1633 wqe = do_rc_completion(qp, wqe, ibp); in hfi1_restart_rc()
1634 qp->s_flags &= ~RVT_S_WAIT_ACK; in hfi1_restart_rc()
1636 trace_hfi1_tid_write_sender_restart_rc(qp, 0); in hfi1_restart_rc()
1642 hfi1_kern_clear_hw_flow(priv->rcd, qp); in hfi1_restart_rc()
1645 hfi1_trdma_send_complete(qp, wqe, in hfi1_restart_rc()
1647 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in hfi1_restart_rc()
1654 qp->s_retry--; in hfi1_restart_rc()
1657 ibp = to_iport(qp->ibqp.device, qp->port_num); in hfi1_restart_rc()
1662 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); in hfi1_restart_rc()
1664 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | in hfi1_restart_rc()
1668 qp->s_flags |= RVT_S_SEND_ONE; in hfi1_restart_rc()
1669 reset_psn(qp, psn); in hfi1_restart_rc()
1677 static void reset_sending_psn(struct rvt_qp *qp, u32 psn) in reset_sending_psn() argument
1680 u32 n = qp->s_last; in reset_sending_psn()
1682 lockdep_assert_held(&qp->s_lock); in reset_sending_psn()
1685 wqe = rvt_get_swqe_ptr(qp, n); in reset_sending_psn()
1690 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
1692 qp->s_sending_psn = psn + 1; in reset_sending_psn()
1695 if (++n == qp->s_size) in reset_sending_psn()
1697 if (n == qp->s_tail) in reset_sending_psn()
1714 void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah) in hfi1_rc_verbs_aborted() argument
1729 qp->s_flags |= RVT_S_SEND_ONE; in hfi1_rc_verbs_aborted()
1735 void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) in hfi1_rc_send_complete() argument
1738 struct hfi1_qp_priv *priv = qp->priv; in hfi1_rc_send_complete()
1744 lockdep_assert_held(&qp->s_lock); in hfi1_rc_send_complete()
1745 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) in hfi1_rc_send_complete()
1754 WARN_ON(!qp->s_rdma_ack_cnt); in hfi1_rc_send_complete()
1755 qp->s_rdma_ack_cnt--; in hfi1_rc_send_complete()
1767 reset_sending_psn(qp, psn); in hfi1_rc_send_complete()
1782 wqe = rvt_get_swqe_ptr(qp, tail); in hfi1_rc_send_complete()
1786 tail = qp->s_size - 1; in hfi1_rc_send_complete()
1791 head = qp->s_tail; in hfi1_rc_send_complete()
1792 tail = qp->s_acked; in hfi1_rc_send_complete()
1802 !(qp->s_flags & in hfi1_rc_send_complete()
1804 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { in hfi1_rc_send_complete()
1806 rvt_add_retry_timer_ext(qp, priv->timeout_shift); in hfi1_rc_send_complete()
1808 rvt_add_retry_timer(qp); in hfi1_rc_send_complete()
1817 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { in hfi1_rc_send_complete()
1823 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_send_complete()
1827 hfi1_add_tid_retry_timer(qp); in hfi1_rc_send_complete()
1830 while (qp->s_last != qp->s_acked) { in hfi1_rc_send_complete()
1831 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_rc_send_complete()
1832 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && in hfi1_rc_send_complete()
1833 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) in hfi1_rc_send_complete()
1835 trdma_clean_swqe(qp, wqe); in hfi1_rc_send_complete()
1836 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in hfi1_rc_send_complete()
1837 rvt_qp_complete_swqe(qp, in hfi1_rc_send_complete()
1846 trace_hfi1_sendcomplete(qp, psn); in hfi1_rc_send_complete()
1847 if (qp->s_flags & RVT_S_WAIT_PSN && in hfi1_rc_send_complete()
1848 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in hfi1_rc_send_complete()
1849 qp->s_flags &= ~RVT_S_WAIT_PSN; in hfi1_rc_send_complete()
1850 qp->s_sending_psn = qp->s_psn; in hfi1_rc_send_complete()
1851 qp->s_sending_hpsn = qp->s_psn - 1; in hfi1_rc_send_complete()
1852 hfi1_schedule_send(qp); in hfi1_rc_send_complete()
1856 static inline void update_last_psn(struct rvt_qp *qp, u32 psn) in update_last_psn() argument
1858 qp->s_last_psn = psn; in update_last_psn()
1866 struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, in do_rc_completion() argument
1870 struct hfi1_qp_priv *priv = qp->priv; in do_rc_completion()
1872 lockdep_assert_held(&qp->s_lock); in do_rc_completion()
1878 trace_hfi1_rc_completion(qp, wqe->lpsn); in do_rc_completion()
1879 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
1880 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in do_rc_completion()
1881 trdma_clean_swqe(qp, wqe); in do_rc_completion()
1882 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in do_rc_completion()
1883 rvt_qp_complete_swqe(qp, in do_rc_completion()
1897 u8 sl = rdma_ah_get_sl(&qp->remote_ah_attr); in do_rc_completion()
1902 engine = qp_to_sdma_engine(qp, sc5); in do_rc_completion()
1907 qp->s_retry = qp->s_retry_cnt; in do_rc_completion()
1917 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
1924 if (qp->s_acked == qp->s_cur) { in do_rc_completion()
1925 if (++qp->s_cur >= qp->s_size) in do_rc_completion()
1926 qp->s_cur = 0; in do_rc_completion()
1927 qp->s_acked = qp->s_cur; in do_rc_completion()
1928 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
1929 if (qp->s_acked != qp->s_tail) { in do_rc_completion()
1930 qp->s_state = OP(SEND_LAST); in do_rc_completion()
1931 qp->s_psn = wqe->psn; in do_rc_completion()
1934 if (++qp->s_acked >= qp->s_size) in do_rc_completion()
1935 qp->s_acked = 0; in do_rc_completion()
1936 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) in do_rc_completion()
1937 qp->s_draining = 0; in do_rc_completion()
1938 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
1942 hfi1_schedule_send(qp); in do_rc_completion()
1947 static void set_restart_qp(struct rvt_qp *qp, struct hfi1_ctxtdata *rcd) in set_restart_qp() argument
1950 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { in set_restart_qp()
1951 qp->r_flags |= RVT_R_RDMAR_SEQ; in set_restart_qp()
1952 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); in set_restart_qp()
1953 if (list_empty(&qp->rspwait)) { in set_restart_qp()
1954 qp->r_flags |= RVT_R_RSP_SEND; in set_restart_qp()
1955 rvt_get_qp(qp); in set_restart_qp()
1956 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in set_restart_qp()
1972 static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn, in update_qp_retry_state() argument
1975 struct hfi1_qp_priv *qpriv = qp->priv; in update_qp_retry_state()
1977 qp->s_psn = psn + 1; in update_qp_retry_state()
1985 qp->s_cur = qpriv->s_tid_cur + 1; in update_qp_retry_state()
1986 if (qp->s_cur >= qp->s_size) in update_qp_retry_state()
1987 qp->s_cur = 0; in update_qp_retry_state()
1988 qp->s_state = TID_OP(WRITE_REQ); in update_qp_retry_state()
1990 qp->s_cur = qpriv->s_tid_cur; in update_qp_retry_state()
1991 qp->s_state = TID_OP(WRITE_RESP); in update_qp_retry_state()
2006 int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, in do_rc_ack() argument
2011 struct hfi1_qp_priv *qpriv = qp->priv; in do_rc_ack()
2018 lockdep_assert_held(&qp->s_lock); in do_rc_ack()
2028 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
2065 (delta_psn(psn, qp->s_last_psn) != 1))) { in do_rc_ack()
2066 set_restart_qp(qp, rcd); in do_rc_ack()
2079 opfn_conn_reply(qp, val); in do_rc_ack()
2081 if (qp->s_num_rd_atomic && in do_rc_ack()
2085 qp->s_num_rd_atomic--; in do_rc_ack()
2087 if ((qp->s_flags & RVT_S_WAIT_FENCE) && in do_rc_ack()
2088 !qp->s_num_rd_atomic) { in do_rc_ack()
2089 qp->s_flags &= ~(RVT_S_WAIT_FENCE | in do_rc_ack()
2091 hfi1_schedule_send(qp); in do_rc_ack()
2092 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { in do_rc_ack()
2093 qp->s_flags &= ~(RVT_S_WAIT_RDMAR | in do_rc_ack()
2095 hfi1_schedule_send(qp); in do_rc_ack()
2106 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
2107 if (qp->s_acked == qp->s_tail) in do_rc_ack()
2111 trace_hfi1_rc_ack_do(qp, aeth, psn, wqe); in do_rc_ack()
2112 trace_hfi1_sender_do_rc_ack(qp); in do_rc_ack()
2118 rvt_mod_retry_timer_ext(qp, in do_rc_ack()
2121 rvt_stop_rc_timers(qp); in do_rc_ack()
2122 } else if (qp->s_acked != qp->s_tail) { in do_rc_ack()
2126 __w = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); in do_rc_ack()
2146 if (cmp_psn(psn, qp->s_last_psn + 1)) { in do_rc_ack()
2147 set_restart_qp(qp, rcd); in do_rc_ack()
2154 if (qp->s_cur != qp->s_tail && in do_rc_ack()
2155 cmp_psn(qp->s_psn, psn) <= 0) in do_rc_ack()
2156 update_qp_retry_state(qp, psn, in do_rc_ack()
2160 rvt_mod_retry_timer(qp); in do_rc_ack()
2162 rvt_stop_rc_timers(qp); in do_rc_ack()
2168 rvt_mod_retry_timer(qp); in do_rc_ack()
2174 if (cmp_psn(qp->s_psn, psn) <= 0) in do_rc_ack()
2175 reset_psn(qp, psn + 1); in do_rc_ack()
2179 rvt_stop_rc_timers(qp); in do_rc_ack()
2180 if (cmp_psn(qp->s_psn, psn) <= 0) { in do_rc_ack()
2181 qp->s_state = OP(SEND_LAST); in do_rc_ack()
2182 qp->s_psn = psn + 1; in do_rc_ack()
2185 if (qp->s_flags & RVT_S_WAIT_ACK) { in do_rc_ack()
2186 qp->s_flags &= ~RVT_S_WAIT_ACK; in do_rc_ack()
2187 hfi1_schedule_send(qp); in do_rc_ack()
2189 rvt_get_credit(qp, aeth); in do_rc_ack()
2190 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
2191 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
2201 update_last_psn(qp, psn); in do_rc_ack()
2206 if (qp->s_acked == qp->s_tail) in do_rc_ack()
2208 if (qp->s_flags & RVT_S_WAIT_RNR) in do_rc_ack()
2210 rdi = ib_to_rvt(qp->ibqp.device); in do_rc_ack()
2213 if (qp->s_rnr_retry == 0) { in do_rc_ack()
2217 if (qp->s_rnr_retry_cnt < 7 && qp->s_rnr_retry_cnt > 0) in do_rc_ack()
2218 qp->s_rnr_retry--; in do_rc_ack()
2228 reset_psn(qp, qp->s_last_psn + 1); in do_rc_ack()
2230 update_last_psn(qp, psn - 1); in do_rc_ack()
2231 reset_psn(qp, psn); in do_rc_ack()
2234 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); in do_rc_ack()
2235 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); in do_rc_ack()
2236 rvt_stop_rc_timers(qp); in do_rc_ack()
2237 rvt_add_rnr_timer(qp, aeth); in do_rc_ack()
2241 if (qp->s_acked == qp->s_tail) in do_rc_ack()
2244 update_last_psn(qp, psn - 1); in do_rc_ack()
2255 hfi1_restart_rc(qp, psn, 0); in do_rc_ack()
2256 hfi1_schedule_send(qp); in do_rc_ack()
2273 if (qp->s_last == qp->s_acked) { in do_rc_ack()
2275 hfi1_kern_read_tid_flow_free(qp); in do_rc_ack()
2277 hfi1_trdma_send_complete(qp, wqe, status); in do_rc_ack()
2278 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in do_rc_ack()
2286 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
2287 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
2297 rvt_stop_rc_timers(qp); in do_rc_ack()
2305 static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn, in rdma_seq_err() argument
2310 lockdep_assert_held(&qp->s_lock); in rdma_seq_err()
2312 rvt_stop_rc_timers(qp); in rdma_seq_err()
2314 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
2323 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
2327 qp->r_flags |= RVT_R_RDMAR_SEQ; in rdma_seq_err()
2328 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); in rdma_seq_err()
2329 if (list_empty(&qp->rspwait)) { in rdma_seq_err()
2330 qp->r_flags |= RVT_R_RSP_SEND; in rdma_seq_err()
2331 rvt_get_qp(qp); in rdma_seq_err()
2332 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in rdma_seq_err()
2349 struct rvt_qp *qp = packet->qp; in rc_rcv_resp() local
2359 u32 pmtu = qp->pmtu; in rc_rcv_resp()
2365 spin_lock_irqsave(&qp->s_lock, flags); in rc_rcv_resp()
2366 trace_hfi1_ack(qp, psn); in rc_rcv_resp()
2369 if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0) in rc_rcv_resp()
2373 diff = cmp_psn(psn, qp->s_last_psn); in rc_rcv_resp()
2379 rvt_get_credit(qp, aeth); in rc_rcv_resp()
2388 if (qp->r_flags & RVT_R_RDMAR_SEQ) { in rc_rcv_resp()
2389 if (cmp_psn(psn, qp->s_last_psn + 1) != 0) in rc_rcv_resp()
2391 qp->r_flags &= ~RVT_R_RDMAR_SEQ; in rc_rcv_resp()
2394 if (unlikely(qp->s_acked == qp->s_tail)) in rc_rcv_resp()
2396 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2408 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || in rc_rcv_resp()
2411 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2419 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in rc_rcv_resp()
2425 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) in rc_rcv_resp()
2432 if (unlikely(pmtu >= qp->s_rdma_read_len)) in rc_rcv_resp()
2439 rvt_mod_retry_timer(qp); in rc_rcv_resp()
2440 if (qp->s_flags & RVT_S_WAIT_ACK) { in rc_rcv_resp()
2441 qp->s_flags &= ~RVT_S_WAIT_ACK; in rc_rcv_resp()
2442 hfi1_schedule_send(qp); in rc_rcv_resp()
2446 qp->s_retry = qp->s_retry_cnt; in rc_rcv_resp()
2452 qp->s_rdma_read_len -= pmtu; in rc_rcv_resp()
2453 update_last_psn(qp, psn); in rc_rcv_resp()
2454 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_resp()
2455 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in rc_rcv_resp()
2461 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) in rc_rcv_resp()
2474 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2475 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in rc_rcv_resp()
2481 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) in rc_rcv_resp()
2493 if (unlikely(tlen != qp->s_rdma_read_len)) in rc_rcv_resp()
2496 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in rc_rcv_resp()
2498 WARN_ON(qp->s_rdma_read_sge.num_sge); in rc_rcv_resp()
2499 (void)do_rc_ack(qp, aeth, psn, in rc_rcv_resp()
2510 rdma_seq_err(qp, ibp, psn, rcd); in rc_rcv_resp()
2516 if (qp->s_last == qp->s_acked) { in rc_rcv_resp()
2517 rvt_send_complete(qp, wqe, status); in rc_rcv_resp()
2518 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in rc_rcv_resp()
2521 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_resp()
2526 static inline void rc_cancel_ack(struct rvt_qp *qp) in rc_cancel_ack() argument
2528 qp->r_adefered = 0; in rc_cancel_ack()
2529 if (list_empty(&qp->rspwait)) in rc_cancel_ack()
2531 list_del_init(&qp->rspwait); in rc_cancel_ack()
2532 qp->r_flags &= ~RVT_R_RSP_NAK; in rc_cancel_ack()
2533 rvt_put_qp(qp); in rc_cancel_ack()
2552 struct rvt_qp *qp, u32 opcode, u32 psn, in rc_rcv_error() argument
2562 trace_hfi1_rcv_error(qp, psn); in rc_rcv_error()
2569 if (!qp->r_nak_state) { in rc_rcv_error()
2571 qp->r_nak_state = IB_NAK_PSN_ERROR; in rc_rcv_error()
2573 qp->r_ack_psn = qp->r_psn; in rc_rcv_error()
2579 rc_defered_ack(rcd, qp); in rc_rcv_error()
2604 spin_lock_irqsave(&qp->s_lock, flags); in rc_rcv_error()
2606 e = find_prev_entry(qp, psn, &prev, &mra, &old_req); in rc_rcv_error()
2629 offset = delta_psn(psn, e->psn) * qp->pmtu; in rc_rcv_error()
2639 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, in rc_rcv_error()
2651 if (qp->s_acked_ack_queue == qp->s_tail_ack_queue) in rc_rcv_error()
2652 qp->s_acked_ack_queue = prev; in rc_rcv_error()
2653 qp->s_tail_ack_queue = prev; in rc_rcv_error()
2666 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) in rc_rcv_error()
2667 qp->s_acked_ack_queue = prev; in rc_rcv_error()
2668 qp->s_tail_ack_queue = prev; in rc_rcv_error()
2683 if (mra == qp->r_head_ack_queue) { in rc_rcv_error()
2684 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_error()
2685 qp->r_nak_state = 0; in rc_rcv_error()
2686 qp->r_ack_psn = qp->r_psn - 1; in rc_rcv_error()
2694 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) in rc_rcv_error()
2695 qp->s_acked_ack_queue = mra; in rc_rcv_error()
2696 qp->s_tail_ack_queue = mra; in rc_rcv_error()
2699 qp->s_ack_state = OP(ACKNOWLEDGE); in rc_rcv_error()
2700 qp->s_flags |= RVT_S_RESP_PENDING; in rc_rcv_error()
2701 qp->r_nak_state = 0; in rc_rcv_error()
2702 hfi1_schedule_send(qp); in rc_rcv_error()
2705 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_error()
2809 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv() local
2810 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_rc_rcv()
2818 u32 pmtu = qp->pmtu; in hfi1_rc_rcv()
2827 lockdep_assert_held(&qp->r_lock); in hfi1_rc_rcv()
2832 fecn = process_ecn(qp, packet); in hfi1_rc_rcv()
2833 opfn_trigger_conn_request(qp, be32_to_cpu(ohdr->bth[1])); in hfi1_rc_rcv()
2848 diff = delta_psn(psn, qp->r_psn); in hfi1_rc_rcv()
2850 if (rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) in hfi1_rc_rcv()
2856 switch (qp->r_state) { in hfi1_rc_rcv()
2891 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) in hfi1_rc_rcv()
2892 rvt_comm_est(qp); in hfi1_rc_rcv()
2897 ret = rvt_get_rwqe(qp, false); in hfi1_rc_rcv()
2902 qp->r_rcv_len = 0; in hfi1_rc_rcv()
2915 qp->r_rcv_len += pmtu; in hfi1_rc_rcv()
2916 if (unlikely(qp->r_rcv_len > qp->r_len)) in hfi1_rc_rcv()
2918 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); in hfi1_rc_rcv()
2923 ret = rvt_get_rwqe(qp, true); in hfi1_rc_rcv()
2933 ret = rvt_get_rwqe(qp, false); in hfi1_rc_rcv()
2938 qp->r_rcv_len = 0; in hfi1_rc_rcv()
2952 if (rvt_invalidate_rkey(qp, rkey)) in hfi1_rc_rcv()
2958 copy_last = rvt_is_user_qp(qp); in hfi1_rc_rcv()
2971 wc.byte_len = tlen + qp->r_rcv_len; in hfi1_rc_rcv()
2972 if (unlikely(wc.byte_len > qp->r_len)) in hfi1_rc_rcv()
2974 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last); in hfi1_rc_rcv()
2975 rvt_put_ss(&qp->r_sge); in hfi1_rc_rcv()
2976 qp->r_msn++; in hfi1_rc_rcv()
2977 if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) in hfi1_rc_rcv()
2979 wc.wr_id = qp->r_wr_id; in hfi1_rc_rcv()
2986 wc.qp = &qp->ibqp; in hfi1_rc_rcv()
2987 wc.src_qp = qp->remote_qpn; in hfi1_rc_rcv()
2988 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; in hfi1_rc_rcv()
3000 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); in hfi1_rc_rcv()
3007 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); in hfi1_rc_rcv()
3011 copy_last = rvt_is_user_qp(qp); in hfi1_rc_rcv()
3015 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) in hfi1_rc_rcv()
3019 qp->r_len = be32_to_cpu(reth->length); in hfi1_rc_rcv()
3020 qp->r_rcv_len = 0; in hfi1_rc_rcv()
3021 qp->r_sge.sg_list = NULL; in hfi1_rc_rcv()
3022 if (qp->r_len != 0) { in hfi1_rc_rcv()
3028 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, in hfi1_rc_rcv()
3032 qp->r_sge.num_sge = 1; in hfi1_rc_rcv()
3034 qp->r_sge.num_sge = 0; in hfi1_rc_rcv()
3035 qp->r_sge.sge.mr = NULL; in hfi1_rc_rcv()
3036 qp->r_sge.sge.vaddr = NULL; in hfi1_rc_rcv()
3037 qp->r_sge.sge.length = 0; in hfi1_rc_rcv()
3038 qp->r_sge.sge.sge_length = 0; in hfi1_rc_rcv()
3044 ret = rvt_get_rwqe(qp, true); in hfi1_rc_rcv()
3049 rvt_put_ss(&qp->r_sge); in hfi1_rc_rcv()
3061 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) in hfi1_rc_rcv()
3063 next = qp->r_head_ack_queue + 1; in hfi1_rc_rcv()
3065 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_rc_rcv()
3067 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv()
3068 if (unlikely(next == qp->s_acked_ack_queue)) { in hfi1_rc_rcv()
3069 if (!qp->s_ack_queue[next].sent) in hfi1_rc_rcv()
3071 update_ack_queue(qp, next); in hfi1_rc_rcv()
3073 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in hfi1_rc_rcv()
3083 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, in hfi1_rc_rcv()
3091 qp->r_psn += rvt_div_mtu(qp, len - 1); in hfi1_rc_rcv()
3101 e->lpsn = qp->r_psn; in hfi1_rc_rcv()
3107 qp->r_msn++; in hfi1_rc_rcv()
3108 qp->r_psn++; in hfi1_rc_rcv()
3109 qp->r_state = opcode; in hfi1_rc_rcv()
3110 qp->r_nak_state = 0; in hfi1_rc_rcv()
3111 qp->r_head_ack_queue = next; in hfi1_rc_rcv()
3112 qpriv->r_tid_alloc = qp->r_head_ack_queue; in hfi1_rc_rcv()
3115 qp->s_flags |= RVT_S_RESP_PENDING; in hfi1_rc_rcv()
3117 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv()
3118 hfi1_schedule_send(qp); in hfi1_rc_rcv()
3120 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3136 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) && in hfi1_rc_rcv()
3139 next = qp->r_head_ack_queue + 1; in hfi1_rc_rcv()
3140 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_rc_rcv()
3142 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv()
3143 if (unlikely(next == qp->s_acked_ack_queue)) { in hfi1_rc_rcv()
3144 if (!qp->s_ack_queue[next].sent) in hfi1_rc_rcv()
3146 update_ack_queue(qp, next); in hfi1_rc_rcv()
3148 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in hfi1_rc_rcv()
3152 opfn_conn_response(qp, e, ateth); in hfi1_rc_rcv()
3159 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), in hfi1_rc_rcv()
3164 maddr = (atomic64_t *)qp->r_sge.sge.vaddr; in hfi1_rc_rcv()
3168 (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr, in hfi1_rc_rcv()
3171 rvt_put_mr(qp->r_sge.sge.mr); in hfi1_rc_rcv()
3172 qp->r_sge.num_sge = 0; in hfi1_rc_rcv()
3178 qp->r_msn++; in hfi1_rc_rcv()
3179 qp->r_psn++; in hfi1_rc_rcv()
3180 qp->r_state = opcode; in hfi1_rc_rcv()
3181 qp->r_nak_state = 0; in hfi1_rc_rcv()
3182 qp->r_head_ack_queue = next; in hfi1_rc_rcv()
3183 qpriv->r_tid_alloc = qp->r_head_ack_queue; in hfi1_rc_rcv()
3186 qp->s_flags |= RVT_S_RESP_PENDING; in hfi1_rc_rcv()
3188 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv()
3189 hfi1_schedule_send(qp); in hfi1_rc_rcv()
3191 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3199 qp->r_psn++; in hfi1_rc_rcv()
3200 qp->r_state = opcode; in hfi1_rc_rcv()
3201 qp->r_ack_psn = psn; in hfi1_rc_rcv()
3202 qp->r_nak_state = 0; in hfi1_rc_rcv()
3206 qp->r_adefered >= HFI1_PSN_CREDIT) { in hfi1_rc_rcv()
3207 rc_cancel_ack(qp); in hfi1_rc_rcv()
3210 qp->r_adefered++; in hfi1_rc_rcv()
3211 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3216 qp->r_nak_state = qp->r_min_rnr_timer | IB_RNR_NAK; in hfi1_rc_rcv()
3217 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3219 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3223 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in hfi1_rc_rcv()
3224 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; in hfi1_rc_rcv()
3225 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3227 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3231 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3233 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in hfi1_rc_rcv()
3234 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in hfi1_rc_rcv()
3235 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3237 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3241 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3243 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); in hfi1_rc_rcv()
3244 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in hfi1_rc_rcv()
3245 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3253 struct rvt_qp *qp) in hfi1_rc_hdrerr() argument
3268 diff = delta_psn(psn, qp->r_psn); in hfi1_rc_hdrerr()
3269 if (!qp->r_nak_state && diff >= 0) { in hfi1_rc_hdrerr()
3271 qp->r_nak_state = IB_NAK_PSN_ERROR; in hfi1_rc_hdrerr()
3273 qp->r_ack_psn = qp->r_psn; in hfi1_rc_hdrerr()
3282 rc_defered_ack(rcd, qp); in hfi1_rc_hdrerr()