Lines Matching refs:qp
62 static void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe) in ipath_init_restart() argument
66 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, in ipath_init_restart()
67 ib_mtu_enum_to_int(qp->path_mtu)); in ipath_init_restart()
68 dev = to_idev(qp->ibqp.device); in ipath_init_restart()
70 if (list_empty(&qp->timerwait)) in ipath_init_restart()
71 list_add_tail(&qp->timerwait, in ipath_init_restart()
86 static int ipath_make_rc_ack(struct ipath_ibdev *dev, struct ipath_qp *qp, in ipath_make_rc_ack() argument
96 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) in ipath_make_rc_ack()
102 switch (qp->s_ack_state) { in ipath_make_rc_ack()
111 if (++qp->s_tail_ack_queue > IPATH_MAX_RDMA_ATOMIC) in ipath_make_rc_ack()
112 qp->s_tail_ack_queue = 0; in ipath_make_rc_ack()
117 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { in ipath_make_rc_ack()
118 if (qp->s_flags & IPATH_S_ACK_PENDING) in ipath_make_rc_ack()
120 qp->s_ack_state = OP(ACKNOWLEDGE); in ipath_make_rc_ack()
124 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in ipath_make_rc_ack()
127 qp->s_ack_rdma_sge = e->rdma_sge; in ipath_make_rc_ack()
128 qp->s_cur_sge = &qp->s_ack_rdma_sge; in ipath_make_rc_ack()
132 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); in ipath_make_rc_ack()
134 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); in ipath_make_rc_ack()
137 ohdr->u.aeth = ipath_compute_aeth(qp); in ipath_make_rc_ack()
139 qp->s_ack_rdma_psn = e->psn; in ipath_make_rc_ack()
140 bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; in ipath_make_rc_ack()
143 qp->s_cur_sge = NULL; in ipath_make_rc_ack()
145 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in ipath_make_rc_ack()
146 ohdr->u.at.aeth = ipath_compute_aeth(qp); in ipath_make_rc_ack()
155 bth0 = qp->s_ack_state << 24; in ipath_make_rc_ack()
159 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); in ipath_make_rc_ack()
162 len = qp->s_ack_rdma_sge.sge.sge_length; in ipath_make_rc_ack()
166 ohdr->u.aeth = ipath_compute_aeth(qp); in ipath_make_rc_ack()
168 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in ipath_make_rc_ack()
169 qp->s_ack_queue[qp->s_tail_ack_queue].sent = 1; in ipath_make_rc_ack()
171 bth0 = qp->s_ack_state << 24; in ipath_make_rc_ack()
172 bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; in ipath_make_rc_ack()
183 qp->s_ack_state = OP(SEND_ONLY); in ipath_make_rc_ack()
184 qp->s_flags &= ~IPATH_S_ACK_PENDING; in ipath_make_rc_ack()
185 qp->s_cur_sge = NULL; in ipath_make_rc_ack()
186 if (qp->s_nak_state) in ipath_make_rc_ack()
188 cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | in ipath_make_rc_ack()
189 (qp->s_nak_state << in ipath_make_rc_ack()
192 ohdr->u.aeth = ipath_compute_aeth(qp); in ipath_make_rc_ack()
196 bth2 = qp->s_ack_psn & IPATH_PSN_MASK; in ipath_make_rc_ack()
198 qp->s_hdrwords = hwords; in ipath_make_rc_ack()
199 qp->s_cur_size = len; in ipath_make_rc_ack()
200 ipath_make_ruc_header(dev, qp, ohdr, bth0, bth2); in ipath_make_rc_ack()
213 int ipath_make_rc_req(struct ipath_qp *qp) in ipath_make_rc_req() argument
215 struct ipath_ibdev *dev = to_idev(qp->ibqp.device); in ipath_make_rc_req()
223 u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); in ipath_make_rc_req()
228 ohdr = &qp->s_hdr.u.oth; in ipath_make_rc_req()
229 if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) in ipath_make_rc_req()
230 ohdr = &qp->s_hdr.u.l.oth; in ipath_make_rc_req()
236 spin_lock_irqsave(&qp->s_lock, flags); in ipath_make_rc_req()
239 if ((qp->r_head_ack_queue != qp->s_tail_ack_queue || in ipath_make_rc_req()
240 (qp->s_flags & IPATH_S_ACK_PENDING) || in ipath_make_rc_req()
241 qp->s_ack_state != OP(ACKNOWLEDGE)) && in ipath_make_rc_req()
242 ipath_make_rc_ack(dev, qp, ohdr, pmtu)) in ipath_make_rc_req()
245 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) { in ipath_make_rc_req()
246 if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND)) in ipath_make_rc_req()
249 if (qp->s_last == qp->s_head) in ipath_make_rc_req()
252 if (atomic_read(&qp->s_dma_busy)) { in ipath_make_rc_req()
253 qp->s_flags |= IPATH_S_WAIT_DMA; in ipath_make_rc_req()
256 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_make_rc_req()
257 ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); in ipath_make_rc_req()
262 if (qp->s_rnr_timeout) { in ipath_make_rc_req()
263 qp->s_flags |= IPATH_S_WAITING; in ipath_make_rc_req()
272 wqe = get_swqe_ptr(qp, qp->s_cur); in ipath_make_rc_req()
273 switch (qp->s_state) { in ipath_make_rc_req()
275 if (!(ib_ipath_state_ops[qp->state] & in ipath_make_rc_req()
286 if (qp->s_cur == qp->s_tail) { in ipath_make_rc_req()
288 if (qp->s_tail == qp->s_head) in ipath_make_rc_req()
295 qp->s_num_rd_atomic) { in ipath_make_rc_req()
296 qp->s_flags |= IPATH_S_FENCE_PENDING; in ipath_make_rc_req()
299 wqe->psn = qp->s_next_psn; in ipath_make_rc_req()
308 ss = &qp->s_sge; in ipath_make_rc_req()
314 if (qp->s_lsn != (u32) -1 && in ipath_make_rc_req()
315 ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { in ipath_make_rc_req()
316 qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT; in ipath_make_rc_req()
322 qp->s_state = OP(SEND_FIRST); in ipath_make_rc_req()
327 qp->s_state = OP(SEND_ONLY); in ipath_make_rc_req()
329 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); in ipath_make_rc_req()
337 if (++qp->s_cur == qp->s_size) in ipath_make_rc_req()
338 qp->s_cur = 0; in ipath_make_rc_req()
342 if (newreq && qp->s_lsn != (u32) -1) in ipath_make_rc_req()
343 qp->s_lsn++; in ipath_make_rc_req()
347 if (qp->s_lsn != (u32) -1 && in ipath_make_rc_req()
348 ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { in ipath_make_rc_req()
349 qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT; in ipath_make_rc_req()
361 qp->s_state = OP(RDMA_WRITE_FIRST); in ipath_make_rc_req()
366 qp->s_state = OP(RDMA_WRITE_ONLY); in ipath_make_rc_req()
368 qp->s_state = in ipath_make_rc_req()
377 if (++qp->s_cur == qp->s_size) in ipath_make_rc_req()
378 qp->s_cur = 0; in ipath_make_rc_req()
387 if (qp->s_num_rd_atomic >= in ipath_make_rc_req()
388 qp->s_max_rd_atomic) { in ipath_make_rc_req()
389 qp->s_flags |= IPATH_S_RDMAR_PENDING; in ipath_make_rc_req()
392 qp->s_num_rd_atomic++; in ipath_make_rc_req()
393 if (qp->s_lsn != (u32) -1) in ipath_make_rc_req()
394 qp->s_lsn++; in ipath_make_rc_req()
400 qp->s_next_psn += (len - 1) / pmtu; in ipath_make_rc_req()
401 wqe->lpsn = qp->s_next_psn++; in ipath_make_rc_req()
408 qp->s_state = OP(RDMA_READ_REQUEST); in ipath_make_rc_req()
412 if (++qp->s_cur == qp->s_size) in ipath_make_rc_req()
413 qp->s_cur = 0; in ipath_make_rc_req()
423 if (qp->s_num_rd_atomic >= in ipath_make_rc_req()
424 qp->s_max_rd_atomic) { in ipath_make_rc_req()
425 qp->s_flags |= IPATH_S_RDMAR_PENDING; in ipath_make_rc_req()
428 qp->s_num_rd_atomic++; in ipath_make_rc_req()
429 if (qp->s_lsn != (u32) -1) in ipath_make_rc_req()
430 qp->s_lsn++; in ipath_make_rc_req()
434 qp->s_state = OP(COMPARE_SWAP); in ipath_make_rc_req()
440 qp->s_state = OP(FETCH_ADD); in ipath_make_rc_req()
454 if (++qp->s_cur == qp->s_size) in ipath_make_rc_req()
455 qp->s_cur = 0; in ipath_make_rc_req()
461 qp->s_sge.sge = wqe->sg_list[0]; in ipath_make_rc_req()
462 qp->s_sge.sg_list = wqe->sg_list + 1; in ipath_make_rc_req()
463 qp->s_sge.num_sge = wqe->wr.num_sge; in ipath_make_rc_req()
464 qp->s_len = wqe->length; in ipath_make_rc_req()
466 qp->s_tail++; in ipath_make_rc_req()
467 if (qp->s_tail >= qp->s_size) in ipath_make_rc_req()
468 qp->s_tail = 0; in ipath_make_rc_req()
470 bth2 |= qp->s_psn & IPATH_PSN_MASK; in ipath_make_rc_req()
472 qp->s_psn = wqe->lpsn + 1; in ipath_make_rc_req()
474 qp->s_psn++; in ipath_make_rc_req()
475 if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) in ipath_make_rc_req()
476 qp->s_next_psn = qp->s_psn; in ipath_make_rc_req()
484 if (list_empty(&qp->timerwait)) in ipath_make_rc_req()
485 list_add_tail(&qp->timerwait, in ipath_make_rc_req()
495 ipath_init_restart(qp, wqe); in ipath_make_rc_req()
498 qp->s_state = OP(SEND_MIDDLE); in ipath_make_rc_req()
501 bth2 = qp->s_psn++ & IPATH_PSN_MASK; in ipath_make_rc_req()
502 if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) in ipath_make_rc_req()
503 qp->s_next_psn = qp->s_psn; in ipath_make_rc_req()
504 ss = &qp->s_sge; in ipath_make_rc_req()
505 len = qp->s_len; in ipath_make_rc_req()
511 qp->s_state = OP(SEND_LAST); in ipath_make_rc_req()
513 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); in ipath_make_rc_req()
521 qp->s_cur++; in ipath_make_rc_req()
522 if (qp->s_cur >= qp->s_size) in ipath_make_rc_req()
523 qp->s_cur = 0; in ipath_make_rc_req()
531 ipath_init_restart(qp, wqe); in ipath_make_rc_req()
534 qp->s_state = OP(RDMA_WRITE_MIDDLE); in ipath_make_rc_req()
537 bth2 = qp->s_psn++ & IPATH_PSN_MASK; in ipath_make_rc_req()
538 if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) in ipath_make_rc_req()
539 qp->s_next_psn = qp->s_psn; in ipath_make_rc_req()
540 ss = &qp->s_sge; in ipath_make_rc_req()
541 len = qp->s_len; in ipath_make_rc_req()
547 qp->s_state = OP(RDMA_WRITE_LAST); in ipath_make_rc_req()
549 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); in ipath_make_rc_req()
557 qp->s_cur++; in ipath_make_rc_req()
558 if (qp->s_cur >= qp->s_size) in ipath_make_rc_req()
559 qp->s_cur = 0; in ipath_make_rc_req()
567 ipath_init_restart(qp, wqe); in ipath_make_rc_req()
568 len = ((qp->s_psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; in ipath_make_rc_req()
573 ohdr->u.rc.reth.length = cpu_to_be32(qp->s_len); in ipath_make_rc_req()
574 qp->s_state = OP(RDMA_READ_REQUEST); in ipath_make_rc_req()
576 bth2 = qp->s_psn & IPATH_PSN_MASK; in ipath_make_rc_req()
577 qp->s_psn = wqe->lpsn + 1; in ipath_make_rc_req()
580 qp->s_cur++; in ipath_make_rc_req()
581 if (qp->s_cur == qp->s_size) in ipath_make_rc_req()
582 qp->s_cur = 0; in ipath_make_rc_req()
585 if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT - 1) >= 0) in ipath_make_rc_req()
587 qp->s_len -= len; in ipath_make_rc_req()
588 qp->s_hdrwords = hwords; in ipath_make_rc_req()
589 qp->s_cur_sge = ss; in ipath_make_rc_req()
590 qp->s_cur_size = len; in ipath_make_rc_req()
591 ipath_make_ruc_header(dev, qp, ohdr, bth0 | (qp->s_state << 24), bth2); in ipath_make_rc_req()
597 qp->s_flags &= ~IPATH_S_BUSY; in ipath_make_rc_req()
599 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_make_rc_req()
612 static void send_rc_ack(struct ipath_qp *qp) in send_rc_ack() argument
614 struct ipath_ibdev *dev = to_idev(qp->ibqp.device); in send_rc_ack()
624 spin_lock_irqsave(&qp->s_lock, flags); in send_rc_ack()
627 if (qp->r_head_ack_queue != qp->s_tail_ack_queue || in send_rc_ack()
628 (qp->s_flags & IPATH_S_ACK_PENDING) || in send_rc_ack()
629 qp->s_ack_state != OP(ACKNOWLEDGE)) in send_rc_ack()
632 spin_unlock_irqrestore(&qp->s_lock, flags); in send_rc_ack()
648 spin_lock_irqsave(&qp->s_lock, flags); in send_rc_ack()
657 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { in send_rc_ack()
659 &qp->remote_ah_attr.grh, in send_rc_ack()
665 bth0 = ipath_get_pkey(dd, qp->s_pkey_index) | in send_rc_ack()
667 if (qp->r_nak_state) in send_rc_ack()
668 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | in send_rc_ack()
669 (qp->r_nak_state << in send_rc_ack()
672 ohdr->u.aeth = ipath_compute_aeth(qp); in send_rc_ack()
673 lrh0 |= qp->remote_ah_attr.sl << 4; in send_rc_ack()
675 hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); in send_rc_ack()
678 qp->remote_ah_attr.src_path_bits); in send_rc_ack()
680 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); in send_rc_ack()
681 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & IPATH_PSN_MASK); in send_rc_ack()
701 if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK) { in send_rc_ack()
703 qp->s_flags |= IPATH_S_ACK_PENDING; in send_rc_ack()
704 qp->s_nak_state = qp->r_nak_state; in send_rc_ack()
705 qp->s_ack_psn = qp->r_ack_psn; in send_rc_ack()
708 ipath_schedule_send(qp); in send_rc_ack()
710 spin_unlock_irqrestore(&qp->s_lock, flags); in send_rc_ack()
724 static void reset_psn(struct ipath_qp *qp, u32 psn) in reset_psn() argument
726 u32 n = qp->s_last; in reset_psn()
727 struct ipath_swqe *wqe = get_swqe_ptr(qp, n); in reset_psn()
730 qp->s_cur = n; in reset_psn()
737 qp->s_state = OP(SEND_LAST); in reset_psn()
746 if (++n == qp->s_size) in reset_psn()
748 if (n == qp->s_tail) in reset_psn()
750 wqe = get_swqe_ptr(qp, n); in reset_psn()
754 qp->s_cur = n; in reset_psn()
760 qp->s_state = OP(SEND_LAST); in reset_psn()
774 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); in reset_psn()
779 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); in reset_psn()
783 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); in reset_psn()
791 qp->s_state = OP(SEND_LAST); in reset_psn()
794 qp->s_psn = psn; in reset_psn()
805 void ipath_restart_rc(struct ipath_qp *qp, u32 psn) in ipath_restart_rc() argument
807 struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last); in ipath_restart_rc()
810 if (qp->s_retry == 0) { in ipath_restart_rc()
811 ipath_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in ipath_restart_rc()
812 ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR); in ipath_restart_rc()
815 qp->s_retry--; in ipath_restart_rc()
821 dev = to_idev(qp->ibqp.device); in ipath_restart_rc()
823 if (!list_empty(&qp->timerwait)) in ipath_restart_rc()
824 list_del_init(&qp->timerwait); in ipath_restart_rc()
825 if (!list_empty(&qp->piowait)) in ipath_restart_rc()
826 list_del_init(&qp->piowait); in ipath_restart_rc()
832 dev->n_rc_resends += (qp->s_psn - psn) & IPATH_PSN_MASK; in ipath_restart_rc()
834 reset_psn(qp, psn); in ipath_restart_rc()
835 ipath_schedule_send(qp); in ipath_restart_rc()
841 static inline void update_last_psn(struct ipath_qp *qp, u32 psn) in update_last_psn() argument
843 qp->s_last_psn = psn; in update_last_psn()
857 static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode, in do_rc_ack() argument
860 struct ipath_ibdev *dev = to_idev(qp->ibqp.device); in do_rc_ack()
875 if (!list_empty(&qp->timerwait)) in do_rc_ack()
876 list_del_init(&qp->timerwait); in do_rc_ack()
888 wqe = get_swqe_ptr(qp, qp->s_last); in do_rc_ack()
925 update_last_psn(qp, wqe->psn - 1); in do_rc_ack()
927 ipath_restart_rc(qp, wqe->psn); in do_rc_ack()
937 if (qp->s_num_rd_atomic && in do_rc_ack()
941 qp->s_num_rd_atomic--; in do_rc_ack()
943 if (((qp->s_flags & IPATH_S_FENCE_PENDING) && in do_rc_ack()
944 !qp->s_num_rd_atomic) || in do_rc_ack()
945 qp->s_flags & IPATH_S_RDMAR_PENDING) in do_rc_ack()
946 ipath_schedule_send(qp); in do_rc_ack()
949 if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) || in do_rc_ack()
956 wc.qp = &qp->ibqp; in do_rc_ack()
957 wc.src_qp = qp->remote_qpn; in do_rc_ack()
958 wc.slid = qp->remote_ah_attr.dlid; in do_rc_ack()
959 wc.sl = qp->remote_ah_attr.sl; in do_rc_ack()
960 ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); in do_rc_ack()
962 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
968 if (qp->s_last == qp->s_cur) { in do_rc_ack()
969 if (++qp->s_cur >= qp->s_size) in do_rc_ack()
970 qp->s_cur = 0; in do_rc_ack()
971 qp->s_last = qp->s_cur; in do_rc_ack()
972 if (qp->s_last == qp->s_tail) in do_rc_ack()
974 wqe = get_swqe_ptr(qp, qp->s_cur); in do_rc_ack()
975 qp->s_state = OP(SEND_LAST); in do_rc_ack()
976 qp->s_psn = wqe->psn; in do_rc_ack()
978 if (++qp->s_last >= qp->s_size) in do_rc_ack()
979 qp->s_last = 0; in do_rc_ack()
980 if (qp->state == IB_QPS_SQD && qp->s_last == qp->s_cur) in do_rc_ack()
981 qp->s_draining = 0; in do_rc_ack()
982 if (qp->s_last == qp->s_tail) in do_rc_ack()
984 wqe = get_swqe_ptr(qp, qp->s_last); in do_rc_ack()
992 if (qp->s_last != qp->s_tail) { in do_rc_ack()
994 if (list_empty(&qp->timerwait)) in do_rc_ack()
995 list_add_tail(&qp->timerwait, in do_rc_ack()
1003 if (ipath_cmp24(qp->s_psn, psn) <= 0) { in do_rc_ack()
1004 reset_psn(qp, psn + 1); in do_rc_ack()
1005 ipath_schedule_send(qp); in do_rc_ack()
1007 } else if (ipath_cmp24(qp->s_psn, psn) <= 0) { in do_rc_ack()
1008 qp->s_state = OP(SEND_LAST); in do_rc_ack()
1009 qp->s_psn = psn + 1; in do_rc_ack()
1011 ipath_get_credit(qp, aeth); in do_rc_ack()
1012 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1013 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1014 update_last_psn(qp, psn); in do_rc_ack()
1020 if (qp->s_last == qp->s_tail) in do_rc_ack()
1022 if (qp->s_rnr_retry == 0) { in do_rc_ack()
1026 if (qp->s_rnr_retry_cnt < 7) in do_rc_ack()
1027 qp->s_rnr_retry--; in do_rc_ack()
1030 update_last_psn(qp, psn - 1); in do_rc_ack()
1036 (qp->s_psn - psn) & IPATH_PSN_MASK; in do_rc_ack()
1038 reset_psn(qp, psn); in do_rc_ack()
1040 qp->s_rnr_timeout = in do_rc_ack()
1043 ipath_insert_rnr_queue(qp); in do_rc_ack()
1044 ipath_schedule_send(qp); in do_rc_ack()
1048 if (qp->s_last == qp->s_tail) in do_rc_ack()
1051 update_last_psn(qp, psn - 1); in do_rc_ack()
1062 ipath_restart_rc(qp, psn); in do_rc_ack()
1079 ipath_send_complete(qp, wqe, status); in do_rc_ack()
1080 ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR); in do_rc_ack()
1087 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1120 struct ipath_qp *qp, in ipath_rc_rcv_resp() argument
1133 spin_lock_irqsave(&qp->s_lock, flags); in ipath_rc_rcv_resp()
1136 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) in ipath_rc_rcv_resp()
1140 if (ipath_cmp24(psn, qp->s_next_psn) >= 0) in ipath_rc_rcv_resp()
1144 diff = ipath_cmp24(psn, qp->s_last_psn); in ipath_rc_rcv_resp()
1155 ipath_get_credit(qp, aeth); in ipath_rc_rcv_resp()
1160 if (unlikely(qp->s_last == qp->s_tail)) in ipath_rc_rcv_resp()
1162 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_rc_rcv_resp()
1185 if (!do_rc_ack(qp, aeth, psn, opcode, val) || in ipath_rc_rcv_resp()
1189 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_rc_rcv_resp()
1192 qp->r_flags &= ~IPATH_R_RDMAR_SEQ; in ipath_rc_rcv_resp()
1198 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in ipath_rc_rcv_resp()
1204 if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { in ipath_rc_rcv_resp()
1206 if (qp->r_flags & IPATH_R_RDMAR_SEQ) in ipath_rc_rcv_resp()
1208 qp->r_flags |= IPATH_R_RDMAR_SEQ; in ipath_rc_rcv_resp()
1209 ipath_restart_rc(qp, qp->s_last_psn + 1); in ipath_rc_rcv_resp()
1217 if (unlikely(pmtu >= qp->s_rdma_read_len)) in ipath_rc_rcv_resp()
1222 if (qp->s_rnr_timeout == 0 && !list_empty(&qp->timerwait)) in ipath_rc_rcv_resp()
1223 list_move_tail(&qp->timerwait, in ipath_rc_rcv_resp()
1228 qp->s_retry = qp->s_retry_cnt; in ipath_rc_rcv_resp()
1234 qp->s_rdma_read_len -= pmtu; in ipath_rc_rcv_resp()
1235 update_last_psn(qp, psn); in ipath_rc_rcv_resp()
1236 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv_resp()
1237 ipath_copy_sge(&qp->s_rdma_read_sge, data, pmtu); in ipath_rc_rcv_resp()
1245 if (!do_rc_ack(qp, aeth, psn, opcode, 0)) in ipath_rc_rcv_resp()
1261 wqe = get_swqe_ptr(qp, qp->s_last); in ipath_rc_rcv_resp()
1262 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in ipath_rc_rcv_resp()
1268 if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { in ipath_rc_rcv_resp()
1270 if (qp->r_flags & IPATH_R_RDMAR_SEQ) in ipath_rc_rcv_resp()
1272 qp->r_flags |= IPATH_R_RDMAR_SEQ; in ipath_rc_rcv_resp()
1273 ipath_restart_rc(qp, qp->s_last_psn + 1); in ipath_rc_rcv_resp()
1289 if (unlikely(tlen != qp->s_rdma_read_len)) in ipath_rc_rcv_resp()
1297 ipath_copy_sge(&qp->s_rdma_read_sge, data, tlen); in ipath_rc_rcv_resp()
1298 (void) do_rc_ack(qp, aeth, psn, in ipath_rc_rcv_resp()
1310 ipath_send_complete(qp, wqe, status); in ipath_rc_rcv_resp()
1311 ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR); in ipath_rc_rcv_resp()
1313 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv_resp()
1338 struct ipath_qp *qp, in ipath_rc_rcv_error() argument
1355 if (!qp->r_nak_state) { in ipath_rc_rcv_error()
1356 qp->r_nak_state = IB_NAK_PSN_ERROR; in ipath_rc_rcv_error()
1358 qp->r_ack_psn = qp->r_psn; in ipath_rc_rcv_error()
1382 spin_lock_irqsave(&qp->s_lock, flags); in ipath_rc_rcv_error()
1384 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) in ipath_rc_rcv_error()
1387 for (i = qp->r_head_ack_queue; ; i = prev) { in ipath_rc_rcv_error()
1388 if (i == qp->s_tail_ack_queue) in ipath_rc_rcv_error()
1394 if (prev == qp->r_head_ack_queue) { in ipath_rc_rcv_error()
1398 e = &qp->s_ack_queue[prev]; in ipath_rc_rcv_error()
1404 if (prev == qp->s_tail_ack_queue) in ipath_rc_rcv_error()
1437 ib_mtu_enum_to_int(qp->path_mtu); in ipath_rc_rcv_error()
1446 ok = ipath_rkey_ok(qp, &e->rdma_sge, in ipath_rc_rcv_error()
1460 qp->s_ack_state = OP(ACKNOWLEDGE); in ipath_rc_rcv_error()
1461 qp->s_tail_ack_queue = prev; in ipath_rc_rcv_error()
1474 qp->s_ack_state = OP(ACKNOWLEDGE); in ipath_rc_rcv_error()
1475 qp->s_tail_ack_queue = prev; in ipath_rc_rcv_error()
1486 if (i == qp->r_head_ack_queue) { in ipath_rc_rcv_error()
1487 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv_error()
1488 qp->r_nak_state = 0; in ipath_rc_rcv_error()
1489 qp->r_ack_psn = qp->r_psn - 1; in ipath_rc_rcv_error()
1497 if (qp->r_head_ack_queue == qp->s_tail_ack_queue && in ipath_rc_rcv_error()
1498 !(qp->s_flags & IPATH_S_ACK_PENDING) && in ipath_rc_rcv_error()
1499 qp->s_ack_state == OP(ACKNOWLEDGE)) { in ipath_rc_rcv_error()
1500 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv_error()
1501 qp->r_nak_state = 0; in ipath_rc_rcv_error()
1502 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; in ipath_rc_rcv_error()
1509 qp->s_ack_state = OP(ACKNOWLEDGE); in ipath_rc_rcv_error()
1510 qp->s_tail_ack_queue = i; in ipath_rc_rcv_error()
1513 qp->r_nak_state = 0; in ipath_rc_rcv_error()
1514 ipath_schedule_send(qp); in ipath_rc_rcv_error()
1517 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv_error()
1525 void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err) in ipath_rc_error() argument
1530 spin_lock_irqsave(&qp->s_lock, flags); in ipath_rc_error()
1531 lastwqe = ipath_error_qp(qp, err); in ipath_rc_error()
1532 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_error()
1537 ev.device = qp->ibqp.device; in ipath_rc_error()
1538 ev.element.qp = &qp->ibqp; in ipath_rc_error()
1540 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); in ipath_rc_error()
1544 static inline void ipath_update_ack_queue(struct ipath_qp *qp, unsigned n) in ipath_update_ack_queue() argument
1551 if (n == qp->s_tail_ack_queue) { in ipath_update_ack_queue()
1552 qp->s_tail_ack_queue = next; in ipath_update_ack_queue()
1553 qp->s_ack_state = OP(ACKNOWLEDGE); in ipath_update_ack_queue()
1571 int has_grh, void *data, u32 tlen, struct ipath_qp *qp) in ipath_rc_rcv() argument
1579 u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); in ipath_rc_rcv()
1586 if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid)) in ipath_rc_rcv()
1620 ipath_rc_rcv_resp(dev, ohdr, data, tlen, qp, opcode, psn, in ipath_rc_rcv()
1626 diff = ipath_cmp24(psn, qp->r_psn); in ipath_rc_rcv()
1628 if (ipath_rc_rcv_error(dev, ohdr, data, qp, opcode, in ipath_rc_rcv()
1635 switch (qp->r_state) { in ipath_rc_rcv()
1673 if (!ipath_get_rwqe(qp, 0)) in ipath_rc_rcv()
1675 qp->r_rcv_len = 0; in ipath_rc_rcv()
1683 qp->r_rcv_len += pmtu; in ipath_rc_rcv()
1684 if (unlikely(qp->r_rcv_len > qp->r_len)) in ipath_rc_rcv()
1686 ipath_copy_sge(&qp->r_sge, data, pmtu); in ipath_rc_rcv()
1691 if (!ipath_get_rwqe(qp, 1)) in ipath_rc_rcv()
1697 if (!ipath_get_rwqe(qp, 0)) in ipath_rc_rcv()
1699 qp->r_rcv_len = 0; in ipath_rc_rcv()
1726 wc.byte_len = tlen + qp->r_rcv_len; in ipath_rc_rcv()
1727 if (unlikely(wc.byte_len > qp->r_len)) in ipath_rc_rcv()
1729 ipath_copy_sge(&qp->r_sge, data, tlen); in ipath_rc_rcv()
1730 qp->r_msn++; in ipath_rc_rcv()
1731 if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) in ipath_rc_rcv()
1733 wc.wr_id = qp->r_wr_id; in ipath_rc_rcv()
1740 wc.qp = &qp->ibqp; in ipath_rc_rcv()
1741 wc.src_qp = qp->remote_qpn; in ipath_rc_rcv()
1742 wc.slid = qp->remote_ah_attr.dlid; in ipath_rc_rcv()
1743 wc.sl = qp->remote_ah_attr.sl; in ipath_rc_rcv()
1745 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, in ipath_rc_rcv()
1753 if (unlikely(!(qp->qp_access_flags & in ipath_rc_rcv()
1765 qp->r_len = be32_to_cpu(reth->length); in ipath_rc_rcv()
1766 qp->r_rcv_len = 0; in ipath_rc_rcv()
1767 if (qp->r_len != 0) { in ipath_rc_rcv()
1773 ok = ipath_rkey_ok(qp, &qp->r_sge, in ipath_rc_rcv()
1774 qp->r_len, vaddr, rkey, in ipath_rc_rcv()
1779 qp->r_sge.sg_list = NULL; in ipath_rc_rcv()
1780 qp->r_sge.sge.mr = NULL; in ipath_rc_rcv()
1781 qp->r_sge.sge.vaddr = NULL; in ipath_rc_rcv()
1782 qp->r_sge.sge.length = 0; in ipath_rc_rcv()
1783 qp->r_sge.sge.sge_length = 0; in ipath_rc_rcv()
1789 if (!ipath_get_rwqe(qp, 1)) in ipath_rc_rcv()
1798 if (unlikely(!(qp->qp_access_flags & in ipath_rc_rcv()
1801 next = qp->r_head_ack_queue + 1; in ipath_rc_rcv()
1804 spin_lock_irqsave(&qp->s_lock, flags); in ipath_rc_rcv()
1806 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) in ipath_rc_rcv()
1808 if (unlikely(next == qp->s_tail_ack_queue)) { in ipath_rc_rcv()
1809 if (!qp->s_ack_queue[next].sent) in ipath_rc_rcv()
1811 ipath_update_ack_queue(qp, next); in ipath_rc_rcv()
1813 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in ipath_rc_rcv()
1828 ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr, in ipath_rc_rcv()
1837 qp->r_psn += (len - 1) / pmtu; in ipath_rc_rcv()
1854 qp->r_msn++; in ipath_rc_rcv()
1855 qp->r_psn++; in ipath_rc_rcv()
1856 qp->r_state = opcode; in ipath_rc_rcv()
1857 qp->r_nak_state = 0; in ipath_rc_rcv()
1858 qp->r_head_ack_queue = next; in ipath_rc_rcv()
1861 ipath_schedule_send(qp); in ipath_rc_rcv()
1876 if (unlikely(!(qp->qp_access_flags & in ipath_rc_rcv()
1879 next = qp->r_head_ack_queue + 1; in ipath_rc_rcv()
1882 spin_lock_irqsave(&qp->s_lock, flags); in ipath_rc_rcv()
1884 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) in ipath_rc_rcv()
1886 if (unlikely(next == qp->s_tail_ack_queue)) { in ipath_rc_rcv()
1887 if (!qp->s_ack_queue[next].sent) in ipath_rc_rcv()
1889 ipath_update_ack_queue(qp, next); in ipath_rc_rcv()
1901 if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, in ipath_rc_rcv()
1906 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; in ipath_rc_rcv()
1908 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in ipath_rc_rcv()
1911 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, in ipath_rc_rcv()
1917 qp->r_msn++; in ipath_rc_rcv()
1918 qp->r_psn++; in ipath_rc_rcv()
1919 qp->r_state = opcode; in ipath_rc_rcv()
1920 qp->r_nak_state = 0; in ipath_rc_rcv()
1921 qp->r_head_ack_queue = next; in ipath_rc_rcv()
1924 ipath_schedule_send(qp); in ipath_rc_rcv()
1933 qp->r_psn++; in ipath_rc_rcv()
1934 qp->r_state = opcode; in ipath_rc_rcv()
1935 qp->r_ack_psn = psn; in ipath_rc_rcv()
1936 qp->r_nak_state = 0; in ipath_rc_rcv()
1943 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; in ipath_rc_rcv()
1944 qp->r_ack_psn = qp->r_psn; in ipath_rc_rcv()
1948 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv()
1950 ipath_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in ipath_rc_rcv()
1951 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in ipath_rc_rcv()
1952 qp->r_ack_psn = qp->r_psn; in ipath_rc_rcv()
1956 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv()
1958 ipath_rc_error(qp, IB_WC_LOC_PROT_ERR); in ipath_rc_rcv()
1959 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in ipath_rc_rcv()
1960 qp->r_ack_psn = qp->r_psn; in ipath_rc_rcv()
1962 send_rc_ack(qp); in ipath_rc_rcv()
1966 spin_unlock_irqrestore(&qp->s_lock, flags); in ipath_rc_rcv()