Lines Matching full:qp
53 * @dev: the device for this QP
54 * @qp: a pointer to the QP
59 * Note that we are in the responder's side of the QP context.
60 * Note the QP s_lock must be held.
62 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp, in qib_make_rc_ack() argument
72 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_make_rc_ack()
78 switch (qp->s_ack_state) { in qib_make_rc_ack()
81 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
93 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) in qib_make_rc_ack()
94 qp->s_tail_ack_queue = 0; in qib_make_rc_ack()
99 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { in qib_make_rc_ack()
100 if (qp->s_flags & RVT_S_ACK_PENDING) in qib_make_rc_ack()
105 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
115 qp->s_tail_ack_queue = qp->r_head_ack_queue; in qib_make_rc_ack()
119 qp->s_rdma_mr = e->rdma_sge.mr; in qib_make_rc_ack()
120 if (qp->s_rdma_mr) in qib_make_rc_ack()
121 rvt_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
122 qp->s_ack_rdma_sge.sge = e->rdma_sge; in qib_make_rc_ack()
123 qp->s_ack_rdma_sge.num_sge = 1; in qib_make_rc_ack()
124 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
127 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); in qib_make_rc_ack()
129 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); in qib_make_rc_ack()
132 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
134 qp->s_ack_rdma_psn = e->psn; in qib_make_rc_ack()
135 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
138 qp->s_cur_sge = NULL; in qib_make_rc_ack()
140 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in qib_make_rc_ack()
141 ohdr->u.at.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
147 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
151 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); in qib_make_rc_ack()
154 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
155 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; in qib_make_rc_ack()
156 if (qp->s_rdma_mr) in qib_make_rc_ack()
157 rvt_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
158 len = qp->s_ack_rdma_sge.sge.sge_length; in qib_make_rc_ack()
162 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
164 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in qib_make_rc_ack()
165 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
168 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
169 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
180 qp->s_ack_state = OP(SEND_ONLY); in qib_make_rc_ack()
181 qp->s_flags &= ~RVT_S_ACK_PENDING; in qib_make_rc_ack()
182 qp->s_cur_sge = NULL; in qib_make_rc_ack()
183 if (qp->s_nak_state) in qib_make_rc_ack()
185 cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in qib_make_rc_ack()
186 (qp->s_nak_state << in qib_make_rc_ack()
189 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
193 bth2 = qp->s_ack_psn & QIB_PSN_MASK; in qib_make_rc_ack()
195 qp->s_rdma_ack_cnt++; in qib_make_rc_ack()
196 qp->s_hdrwords = hwords; in qib_make_rc_ack()
197 qp->s_cur_size = len; in qib_make_rc_ack()
198 qib_make_ruc_header(qp, ohdr, bth0, bth2); in qib_make_rc_ack()
202 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_make_rc_ack()
203 qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING); in qib_make_rc_ack()
209 * @qp: a pointer to the QP
215 int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags) in qib_make_rc_req() argument
217 struct qib_qp_priv *priv = qp->priv; in qib_make_rc_req()
218 struct qib_ibdev *dev = to_idev(qp->ibqp.device); in qib_make_rc_req()
226 u32 pmtu = qp->pmtu; in qib_make_rc_req()
232 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) in qib_make_rc_req()
236 if ((qp->s_flags & RVT_S_RESP_PENDING) && in qib_make_rc_req()
237 qib_make_rc_ack(dev, qp, ohdr, pmtu)) in qib_make_rc_req()
240 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { in qib_make_rc_req()
241 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) in qib_make_rc_req()
244 if (qp->s_last == READ_ONCE(qp->s_head)) in qib_make_rc_req()
248 qp->s_flags |= RVT_S_WAIT_DMA; in qib_make_rc_req()
251 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_make_rc_req()
252 rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in qib_make_rc_req()
258 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK)) in qib_make_rc_req()
261 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
262 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
263 qp->s_flags |= RVT_S_WAIT_PSN; in qib_make_rc_req()
266 qp->s_sending_psn = qp->s_psn; in qib_make_rc_req()
267 qp->s_sending_hpsn = qp->s_psn - 1; in qib_make_rc_req()
275 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in qib_make_rc_req()
276 switch (qp->s_state) { in qib_make_rc_req()
278 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) in qib_make_rc_req()
288 if (qp->s_cur == qp->s_tail) { in qib_make_rc_req()
290 if (qp->s_tail == READ_ONCE(qp->s_head)) in qib_make_rc_req()
297 qp->s_num_rd_atomic) { in qib_make_rc_req()
298 qp->s_flags |= RVT_S_WAIT_FENCE; in qib_make_rc_req()
302 qp->s_psn = wqe->psn; in qib_make_rc_req()
310 ss = &qp->s_sge; in qib_make_rc_req()
311 bth2 = qp->s_psn & QIB_PSN_MASK; in qib_make_rc_req()
316 if (!rvt_rc_credit_avail(qp, wqe)) in qib_make_rc_req()
319 qp->s_state = OP(SEND_FIRST); in qib_make_rc_req()
324 qp->s_state = OP(SEND_ONLY); in qib_make_rc_req()
326 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
334 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
335 qp->s_cur = 0; in qib_make_rc_req()
339 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
340 qp->s_lsn++; in qib_make_rc_req()
344 if (!rvt_rc_credit_avail(qp, wqe)) in qib_make_rc_req()
354 qp->s_state = OP(RDMA_WRITE_FIRST); in qib_make_rc_req()
359 qp->s_state = OP(RDMA_WRITE_ONLY); in qib_make_rc_req()
361 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
370 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
371 qp->s_cur = 0; in qib_make_rc_req()
377 * than the QP limits allow. in qib_make_rc_req()
380 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
381 qp->s_max_rd_atomic) { in qib_make_rc_req()
382 qp->s_flags |= RVT_S_WAIT_RDMAR; in qib_make_rc_req()
385 qp->s_num_rd_atomic++; in qib_make_rc_req()
386 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
387 qp->s_lsn++; in qib_make_rc_req()
395 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
400 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
401 qp->s_cur = 0; in qib_make_rc_req()
408 * than the QP limits allow. in qib_make_rc_req()
411 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
412 qp->s_max_rd_atomic) { in qib_make_rc_req()
413 qp->s_flags |= RVT_S_WAIT_RDMAR; in qib_make_rc_req()
416 qp->s_num_rd_atomic++; in qib_make_rc_req()
417 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
418 qp->s_lsn++; in qib_make_rc_req()
421 qp->s_state = OP(COMPARE_SWAP); in qib_make_rc_req()
427 qp->s_state = OP(FETCH_ADD); in qib_make_rc_req()
440 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
441 qp->s_cur = 0; in qib_make_rc_req()
447 qp->s_sge.sge = wqe->sg_list[0]; in qib_make_rc_req()
448 qp->s_sge.sg_list = wqe->sg_list + 1; in qib_make_rc_req()
449 qp->s_sge.num_sge = wqe->wr.num_sge; in qib_make_rc_req()
450 qp->s_sge.total_len = wqe->length; in qib_make_rc_req()
451 qp->s_len = wqe->length; in qib_make_rc_req()
453 qp->s_tail++; in qib_make_rc_req()
454 if (qp->s_tail >= qp->s_size) in qib_make_rc_req()
455 qp->s_tail = 0; in qib_make_rc_req()
458 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
460 qp->s_psn++; in qib_make_rc_req()
465 * qp->s_state is normally set to the opcode of the in qib_make_rc_req()
473 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
476 qp->s_state = OP(SEND_MIDDLE); in qib_make_rc_req()
479 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
480 ss = &qp->s_sge; in qib_make_rc_req()
481 len = qp->s_len; in qib_make_rc_req()
487 qp->s_state = OP(SEND_LAST); in qib_make_rc_req()
489 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
497 qp->s_cur++; in qib_make_rc_req()
498 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
499 qp->s_cur = 0; in qib_make_rc_req()
504 * qp->s_state is normally set to the opcode of the in qib_make_rc_req()
512 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
515 qp->s_state = OP(RDMA_WRITE_MIDDLE); in qib_make_rc_req()
518 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
519 ss = &qp->s_sge; in qib_make_rc_req()
520 len = qp->s_len; in qib_make_rc_req()
526 qp->s_state = OP(RDMA_WRITE_LAST); in qib_make_rc_req()
528 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
536 qp->s_cur++; in qib_make_rc_req()
537 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
538 qp->s_cur = 0; in qib_make_rc_req()
543 * qp->s_state is normally set to the opcode of the in qib_make_rc_req()
551 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; in qib_make_rc_req()
557 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
559 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; in qib_make_rc_req()
560 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
563 qp->s_cur++; in qib_make_rc_req()
564 if (qp->s_cur == qp->s_size) in qib_make_rc_req()
565 qp->s_cur = 0; in qib_make_rc_req()
568 qp->s_sending_hpsn = bth2; in qib_make_rc_req()
572 if (qp->s_flags & RVT_S_SEND_ONE) { in qib_make_rc_req()
573 qp->s_flags &= ~RVT_S_SEND_ONE; in qib_make_rc_req()
574 qp->s_flags |= RVT_S_WAIT_ACK; in qib_make_rc_req()
577 qp->s_len -= len; in qib_make_rc_req()
578 qp->s_hdrwords = hwords; in qib_make_rc_req()
579 qp->s_cur_sge = ss; in qib_make_rc_req()
580 qp->s_cur_size = len; in qib_make_rc_req()
581 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); in qib_make_rc_req()
585 qp->s_flags &= ~RVT_S_BUSY; in qib_make_rc_req()
591 * @qp: a pointer to the QP
595 * send side QP state and tasklet.
597 void qib_send_rc_ack(struct rvt_qp *qp) in qib_send_rc_ack() argument
599 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); in qib_send_rc_ack()
600 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_send_rc_ack()
613 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
615 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_send_rc_ack()
619 if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt) in qib_send_rc_ack()
627 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & in qib_send_rc_ack()
630 rdma_ah_read_grh(&qp->remote_ah_attr), in qib_send_rc_ack()
636 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); in qib_send_rc_ack()
637 if (qp->s_mig_state == IB_MIG_MIGRATED) in qib_send_rc_ack()
639 if (qp->r_nak_state) in qib_send_rc_ack()
640 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in qib_send_rc_ack()
641 (qp->r_nak_state << in qib_send_rc_ack()
644 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_send_rc_ack()
645 lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 | in qib_send_rc_ack()
646 rdma_ah_get_sl(&qp->remote_ah_attr) << 4; in qib_send_rc_ack()
648 hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); in qib_send_rc_ack()
651 rdma_ah_get_path_bits(&qp->remote_ah_attr)); in qib_send_rc_ack()
653 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); in qib_send_rc_ack()
654 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); in qib_send_rc_ack()
656 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
663 qp->s_srate, lrh0 >> 12); in qib_send_rc_ack()
676 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
711 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { in qib_send_rc_ack()
713 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; in qib_send_rc_ack()
714 qp->s_nak_state = qp->r_nak_state; in qib_send_rc_ack()
715 qp->s_ack_psn = qp->r_ack_psn; in qib_send_rc_ack()
718 qib_schedule_send(qp); in qib_send_rc_ack()
721 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
727 * reset_psn - reset the QP state to send starting from PSN
728 * @qp: the QP
732 * for the given QP.
733 * Called at interrupt level with the QP s_lock held.
735 static void reset_psn(struct rvt_qp *qp, u32 psn) in reset_psn() argument
737 u32 n = qp->s_acked; in reset_psn()
738 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
741 qp->s_cur = n; in reset_psn()
748 qp->s_state = OP(SEND_LAST); in reset_psn()
757 if (++n == qp->s_size) in reset_psn()
759 if (n == qp->s_tail) in reset_psn()
761 wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
765 qp->s_cur = n; in reset_psn()
771 qp->s_state = OP(SEND_LAST); in reset_psn()
785 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); in reset_psn()
790 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); in reset_psn()
794 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); in reset_psn()
802 qp->s_state = OP(SEND_LAST); in reset_psn()
805 qp->s_psn = psn; in reset_psn()
811 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && in reset_psn()
812 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) in reset_psn()
813 qp->s_flags |= RVT_S_WAIT_PSN; in reset_psn()
818 * The QP r_lock and s_lock should be held and interrupts disabled.
820 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait) in qib_restart_rc() argument
822 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_restart_rc()
825 if (qp->s_retry == 0) { in qib_restart_rc()
826 if (qp->s_mig_state == IB_MIG_ARMED) { in qib_restart_rc()
827 qib_migrate_qp(qp); in qib_restart_rc()
828 qp->s_retry = qp->s_retry_cnt; in qib_restart_rc()
829 } else if (qp->s_last == qp->s_acked) { in qib_restart_rc()
830 rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in qib_restart_rc()
831 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in qib_restart_rc()
836 qp->s_retry--; in qib_restart_rc()
838 ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_restart_rc()
842 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in qib_restart_rc()
844 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | in qib_restart_rc()
848 qp->s_flags |= RVT_S_SEND_ONE; in qib_restart_rc()
849 reset_psn(qp, psn); in qib_restart_rc()
853 * Set qp->s_sending_psn to the next PSN after the given one.
856 static void reset_sending_psn(struct rvt_qp *qp, u32 psn) in reset_sending_psn() argument
859 u32 n = qp->s_last; in reset_sending_psn()
863 wqe = rvt_get_swqe_ptr(qp, n); in reset_sending_psn()
866 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
868 qp->s_sending_psn = psn + 1; in reset_sending_psn()
871 if (++n == qp->s_size) in reset_sending_psn()
873 if (n == qp->s_tail) in reset_sending_psn()
879 * This should be called with the QP s_lock held and interrupts disabled.
881 void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr) in qib_rc_send_complete() argument
888 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) in qib_rc_send_complete()
900 WARN_ON(!qp->s_rdma_ack_cnt); in qib_rc_send_complete()
901 qp->s_rdma_ack_cnt--; in qib_rc_send_complete()
906 reset_sending_psn(qp, psn); in qib_rc_send_complete()
912 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && in qib_rc_send_complete()
913 !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) && in qib_rc_send_complete()
914 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_rc_send_complete()
915 rvt_add_retry_timer(qp); in qib_rc_send_complete()
917 while (qp->s_last != qp->s_acked) { in qib_rc_send_complete()
918 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_rc_send_complete()
919 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && in qib_rc_send_complete()
920 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) in qib_rc_send_complete()
922 rvt_qp_complete_swqe(qp, in qib_rc_send_complete()
931 if (qp->s_flags & RVT_S_WAIT_PSN && in qib_rc_send_complete()
932 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in qib_rc_send_complete()
933 qp->s_flags &= ~RVT_S_WAIT_PSN; in qib_rc_send_complete()
934 qp->s_sending_psn = qp->s_psn; in qib_rc_send_complete()
935 qp->s_sending_hpsn = qp->s_psn - 1; in qib_rc_send_complete()
936 qib_schedule_send(qp); in qib_rc_send_complete()
940 static inline void update_last_psn(struct rvt_qp *qp, u32 psn) in update_last_psn() argument
942 qp->s_last_psn = psn; in update_last_psn()
950 static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, in do_rc_completion() argument
959 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
960 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) in do_rc_completion()
961 rvt_qp_complete_swqe(qp, in do_rc_completion()
968 qp->s_retry = qp->s_retry_cnt; in do_rc_completion()
969 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
976 if (qp->s_acked == qp->s_cur) { in do_rc_completion()
977 if (++qp->s_cur >= qp->s_size) in do_rc_completion()
978 qp->s_cur = 0; in do_rc_completion()
979 qp->s_acked = qp->s_cur; in do_rc_completion()
980 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
981 if (qp->s_acked != qp->s_tail) { in do_rc_completion()
982 qp->s_state = OP(SEND_LAST); in do_rc_completion()
983 qp->s_psn = wqe->psn; in do_rc_completion()
986 if (++qp->s_acked >= qp->s_size) in do_rc_completion()
987 qp->s_acked = 0; in do_rc_completion()
988 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) in do_rc_completion()
989 qp->s_draining = 0; in do_rc_completion()
990 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
997 * @qp: the QP the ACK came in on
1002 * for the given QP.
1003 * Called at interrupt level with the QP s_lock held.
1006 static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, in do_rc_ack() argument
1025 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
1026 ibp = to_iport(qp->ibqp.device, qp->port_num); in do_rc_ack()
1060 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { in do_rc_ack()
1061 qp->r_flags |= RVT_R_RDMAR_SEQ; in do_rc_ack()
1062 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in do_rc_ack()
1063 if (list_empty(&qp->rspwait)) { in do_rc_ack()
1064 qp->r_flags |= RVT_R_RSP_SEND; in do_rc_ack()
1065 rvt_get_qp(qp); in do_rc_ack()
1066 list_add_tail(&qp->rspwait, in do_rc_ack()
1081 if (qp->s_num_rd_atomic && in do_rc_ack()
1085 qp->s_num_rd_atomic--; in do_rc_ack()
1087 if ((qp->s_flags & RVT_S_WAIT_FENCE) && in do_rc_ack()
1088 !qp->s_num_rd_atomic) { in do_rc_ack()
1089 qp->s_flags &= ~(RVT_S_WAIT_FENCE | in do_rc_ack()
1091 qib_schedule_send(qp); in do_rc_ack()
1092 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { in do_rc_ack()
1093 qp->s_flags &= ~(RVT_S_WAIT_RDMAR | in do_rc_ack()
1095 qib_schedule_send(qp); in do_rc_ack()
1098 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
1099 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1106 if (qp->s_acked != qp->s_tail) { in do_rc_ack()
1111 rvt_mod_retry_timer(qp); in do_rc_ack()
1116 if (qib_cmp24(qp->s_psn, psn) <= 0) in do_rc_ack()
1117 reset_psn(qp, psn + 1); in do_rc_ack()
1120 rvt_stop_rc_timers(qp); in do_rc_ack()
1121 if (qib_cmp24(qp->s_psn, psn) <= 0) { in do_rc_ack()
1122 qp->s_state = OP(SEND_LAST); in do_rc_ack()
1123 qp->s_psn = psn + 1; in do_rc_ack()
1126 if (qp->s_flags & RVT_S_WAIT_ACK) { in do_rc_ack()
1127 qp->s_flags &= ~RVT_S_WAIT_ACK; in do_rc_ack()
1128 qib_schedule_send(qp); in do_rc_ack()
1130 rvt_get_credit(qp, aeth); in do_rc_ack()
1131 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1132 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1133 update_last_psn(qp, psn); in do_rc_ack()
1138 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1140 if (qp->s_flags & RVT_S_WAIT_RNR) in do_rc_ack()
1142 if (qp->s_rnr_retry == 0) { in do_rc_ack()
1146 if (qp->s_rnr_retry_cnt < 7) in do_rc_ack()
1147 qp->s_rnr_retry--; in do_rc_ack()
1150 update_last_psn(qp, psn - 1); in do_rc_ack()
1152 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in do_rc_ack()
1154 reset_psn(qp, psn); in do_rc_ack()
1156 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); in do_rc_ack()
1157 rvt_stop_rc_timers(qp); in do_rc_ack()
1158 rvt_add_rnr_timer(qp, aeth); in do_rc_ack()
1162 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1165 update_last_psn(qp, psn - 1); in do_rc_ack()
1176 qib_restart_rc(qp, psn, 0); in do_rc_ack()
1177 qib_schedule_send(qp); in do_rc_ack()
1194 if (qp->s_last == qp->s_acked) { in do_rc_ack()
1195 rvt_send_complete(qp, wqe, status); in do_rc_ack()
1196 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in do_rc_ack()
1204 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1205 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1215 rvt_stop_rc_timers(qp); in do_rc_ack()
1223 static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn, in rdma_seq_err() argument
1228 /* Remove QP from retry timer */ in rdma_seq_err()
1229 rvt_stop_rc_timers(qp); in rdma_seq_err()
1231 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
1238 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
1242 qp->r_flags |= RVT_R_RDMAR_SEQ; in rdma_seq_err()
1243 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in rdma_seq_err()
1244 if (list_empty(&qp->rspwait)) { in rdma_seq_err()
1245 qp->r_flags |= RVT_R_RSP_SEND; in rdma_seq_err()
1246 rvt_get_qp(qp); in rdma_seq_err()
1247 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in rdma_seq_err()
1257 * @qp: the QP for this packet
1264 * packet for the given QP.
1270 struct rvt_qp *qp, in qib_rc_rcv_resp() argument
1289 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && in qib_rc_rcv_resp()
1290 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { in qib_rc_rcv_resp()
1296 if (!(qp->s_flags & RVT_S_BUSY)) { in qib_rc_rcv_resp()
1307 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_resp()
1308 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_rc_rcv_resp()
1312 if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0) in qib_rc_rcv_resp()
1316 diff = qib_cmp24(psn, qp->s_last_psn); in qib_rc_rcv_resp()
1322 rvt_get_credit(qp, aeth); in qib_rc_rcv_resp()
1331 if (qp->r_flags & RVT_R_RDMAR_SEQ) { in qib_rc_rcv_resp()
1332 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) in qib_rc_rcv_resp()
1334 qp->r_flags &= ~RVT_R_RDMAR_SEQ; in qib_rc_rcv_resp()
1337 if (unlikely(qp->s_acked == qp->s_tail)) in qib_rc_rcv_resp()
1339 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1351 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || in qib_rc_rcv_resp()
1355 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1363 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1369 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1376 if (unlikely(pmtu >= qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1381 * 4.096 usec. * (1 << qp->timeout) in qib_rc_rcv_resp()
1383 rvt_mod_retry_timer(qp); in qib_rc_rcv_resp()
1384 if (qp->s_flags & RVT_S_WAIT_ACK) { in qib_rc_rcv_resp()
1385 qp->s_flags &= ~RVT_S_WAIT_ACK; in qib_rc_rcv_resp()
1386 qib_schedule_send(qp); in qib_rc_rcv_resp()
1390 qp->s_retry = qp->s_retry_cnt; in qib_rc_rcv_resp()
1396 qp->s_rdma_read_len -= pmtu; in qib_rc_rcv_resp()
1397 update_last_psn(qp, psn); in qib_rc_rcv_resp()
1398 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1399 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1405 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) in qib_rc_rcv_resp()
1421 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1422 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1428 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1443 if (unlikely(tlen != qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1446 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1448 WARN_ON(qp->s_rdma_read_sge.num_sge); in qib_rc_rcv_resp()
1449 (void) do_rc_ack(qp, aeth, psn, in qib_rc_rcv_resp()
1459 rdma_seq_err(qp, ibp, psn, rcd); in qib_rc_rcv_resp()
1465 if (qp->s_last == qp->s_acked) { in qib_rc_rcv_resp()
1466 rvt_send_complete(qp, wqe, status); in qib_rc_rcv_resp()
1467 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in qib_rc_rcv_resp()
1470 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1479 * @qp: the QP for this packet
1485 * incoming RC packet for the given QP.
1492 struct rvt_qp *qp, in qib_rc_rcv_error() argument
1498 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_rc_rcv_error()
1510 if (!qp->r_nak_state) { in qib_rc_rcv_error()
1512 qp->r_nak_state = IB_NAK_PSN_ERROR; in qib_rc_rcv_error()
1514 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv_error()
1520 if (list_empty(&qp->rspwait)) { in qib_rc_rcv_error()
1521 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv_error()
1522 rvt_get_qp(qp); in qib_rc_rcv_error()
1523 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv_error()
1549 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_error()
1551 for (i = qp->r_head_ack_queue; ; i = prev) { in qib_rc_rcv_error()
1552 if (i == qp->s_tail_ack_queue) in qib_rc_rcv_error()
1558 if (prev == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1562 e = &qp->s_ack_queue[prev]; in qib_rc_rcv_error()
1568 if (prev == qp->s_tail_ack_queue && in qib_rc_rcv_error()
1596 qp->pmtu; in qib_rc_rcv_error()
1609 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, in qib_rc_rcv_error()
1621 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1634 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1649 if (i == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1650 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1651 qp->r_nak_state = 0; in qib_rc_rcv_error()
1652 qp->r_ack_psn = qp->r_psn - 1; in qib_rc_rcv_error()
1660 if (!(qp->s_flags & RVT_S_RESP_PENDING)) { in qib_rc_rcv_error()
1661 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1662 qp->r_nak_state = 0; in qib_rc_rcv_error()
1663 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; in qib_rc_rcv_error()
1670 qp->s_tail_ack_queue = i; in qib_rc_rcv_error()
1673 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_rc_rcv_error()
1674 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv_error()
1675 qp->r_nak_state = 0; in qib_rc_rcv_error()
1676 qib_schedule_send(qp); in qib_rc_rcv_error()
1679 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1687 static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n) in qib_update_ack_queue() argument
1694 qp->s_tail_ack_queue = next; in qib_update_ack_queue()
1695 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_update_ack_queue()
1705 * @qp: the QP for this packet
1708 * for the given QP.
1712 int has_grh, void *data, u32 tlen, struct rvt_qp *qp) in qib_rc_rcv() argument
1721 u32 pmtu = qp->pmtu; in qib_rc_rcv()
1737 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) in qib_rc_rcv()
1747 * In other words, this QP is the requester. in qib_rc_rcv()
1751 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, in qib_rc_rcv()
1757 diff = qib_cmp24(psn, qp->r_psn); in qib_rc_rcv()
1759 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) in qib_rc_rcv()
1765 switch (qp->r_state) { in qib_rc_rcv()
1798 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) in qib_rc_rcv()
1799 rvt_comm_est(qp); in qib_rc_rcv()
1804 ret = rvt_get_rwqe(qp, false); in qib_rc_rcv()
1809 qp->r_rcv_len = 0; in qib_rc_rcv()
1817 qp->r_rcv_len += pmtu; in qib_rc_rcv()
1818 if (unlikely(qp->r_rcv_len > qp->r_len)) in qib_rc_rcv()
1820 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); in qib_rc_rcv()
1825 ret = rvt_get_rwqe(qp, true); in qib_rc_rcv()
1834 ret = rvt_get_rwqe(qp, false); in qib_rc_rcv()
1839 qp->r_rcv_len = 0; in qib_rc_rcv()
1863 wc.byte_len = tlen + qp->r_rcv_len; in qib_rc_rcv()
1864 if (unlikely(wc.byte_len > qp->r_len)) in qib_rc_rcv()
1866 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); in qib_rc_rcv()
1867 rvt_put_ss(&qp->r_sge); in qib_rc_rcv()
1868 qp->r_msn++; in qib_rc_rcv()
1869 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) in qib_rc_rcv()
1871 wc.wr_id = qp->r_wr_id; in qib_rc_rcv()
1878 wc.qp = &qp->ibqp; in qib_rc_rcv()
1879 wc.src_qp = qp->remote_qpn; in qib_rc_rcv()
1880 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr); in qib_rc_rcv()
1881 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); in qib_rc_rcv()
1888 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); in qib_rc_rcv()
1894 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) in qib_rc_rcv()
1899 qp->r_len = be32_to_cpu(reth->length); in qib_rc_rcv()
1900 qp->r_rcv_len = 0; in qib_rc_rcv()
1901 qp->r_sge.sg_list = NULL; in qib_rc_rcv()
1902 if (qp->r_len != 0) { in qib_rc_rcv()
1908 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, in qib_rc_rcv()
1912 qp->r_sge.num_sge = 1; in qib_rc_rcv()
1914 qp->r_sge.num_sge = 0; in qib_rc_rcv()
1915 qp->r_sge.sge.mr = NULL; in qib_rc_rcv()
1916 qp->r_sge.sge.vaddr = NULL; in qib_rc_rcv()
1917 qp->r_sge.sge.length = 0; in qib_rc_rcv()
1918 qp->r_sge.sge.sge_length = 0; in qib_rc_rcv()
1924 ret = rvt_get_rwqe(qp, true); in qib_rc_rcv()
1928 rvt_put_ss(&qp->r_sge); in qib_rc_rcv()
1941 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) in qib_rc_rcv()
1943 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
1947 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
1948 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
1949 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
1951 qib_update_ack_queue(qp, next); in qib_rc_rcv()
1953 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
1966 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, in qib_rc_rcv()
1974 qp->r_psn += rvt_div_mtu(qp, len - 1); in qib_rc_rcv()
1984 e->lpsn = qp->r_psn; in qib_rc_rcv()
1990 qp->r_msn++; in qib_rc_rcv()
1991 qp->r_psn++; in qib_rc_rcv()
1992 qp->r_state = opcode; in qib_rc_rcv()
1993 qp->r_nak_state = 0; in qib_rc_rcv()
1994 qp->r_head_ack_queue = next; in qib_rc_rcv()
1997 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv()
1998 qib_schedule_send(qp); in qib_rc_rcv()
2013 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) in qib_rc_rcv()
2015 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
2018 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
2019 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
2020 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
2022 qib_update_ack_queue(qp, next); in qib_rc_rcv()
2024 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
2035 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), in qib_rc_rcv()
2040 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; in qib_rc_rcv()
2044 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, in qib_rc_rcv()
2047 rvt_put_mr(qp->r_sge.sge.mr); in qib_rc_rcv()
2048 qp->r_sge.num_sge = 0; in qib_rc_rcv()
2053 qp->r_msn++; in qib_rc_rcv()
2054 qp->r_psn++; in qib_rc_rcv()
2055 qp->r_state = opcode; in qib_rc_rcv()
2056 qp->r_nak_state = 0; in qib_rc_rcv()
2057 qp->r_head_ack_queue = next; in qib_rc_rcv()
2060 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv()
2061 qib_schedule_send(qp); in qib_rc_rcv()
2070 qp->r_psn++; in qib_rc_rcv()
2071 qp->r_state = opcode; in qib_rc_rcv()
2072 qp->r_ack_psn = psn; in qib_rc_rcv()
2073 qp->r_nak_state = 0; in qib_rc_rcv()
2080 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; in qib_rc_rcv()
2081 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2083 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2084 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2085 rvt_get_qp(qp); in qib_rc_rcv()
2086 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2091 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in qib_rc_rcv()
2092 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; in qib_rc_rcv()
2093 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2095 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2096 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2097 rvt_get_qp(qp); in qib_rc_rcv()
2098 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2103 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2105 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in qib_rc_rcv()
2106 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in qib_rc_rcv()
2107 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2109 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2110 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2111 rvt_get_qp(qp); in qib_rc_rcv()
2112 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2117 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2119 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); in qib_rc_rcv()
2120 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in qib_rc_rcv()
2121 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2123 qib_send_rc_ack(qp); in qib_rc_rcv()
2127 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()