Lines Matching refs:wqe
435 struct rvt_swqe *wqe; in hfi1_make_rc_req() local
490 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_make_rc_req()
491 hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in hfi1_make_rc_req()
510 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in hfi1_make_rc_req()
536 if ((wqe->wr.send_flags & IB_SEND_FENCE) && in hfi1_make_rc_req()
538 (wqe->wr.opcode != IB_WR_TID_RDMA_READ || in hfi1_make_rc_req()
547 if (wqe->wr.opcode == IB_WR_REG_MR || in hfi1_make_rc_req()
548 wqe->wr.opcode == IB_WR_LOCAL_INV) { in hfi1_make_rc_req()
558 if (!(wqe->wr.send_flags & in hfi1_make_rc_req()
562 wqe->wr.ex.invalidate_rkey); in hfi1_make_rc_req()
565 rvt_send_complete(qp, wqe, in hfi1_make_rc_req()
574 qp->s_psn = wqe->psn; in hfi1_make_rc_req()
581 len = wqe->length; in hfi1_make_rc_req()
590 hfi1_tid_rdma_wqe_interlock(qp, wqe)) in hfi1_make_rc_req()
593 switch (wqe->wr.opcode) { in hfi1_make_rc_req()
598 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
605 if (wqe->wr.opcode == IB_WR_SEND) { in hfi1_make_rc_req()
607 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { in hfi1_make_rc_req()
610 ohdr->u.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
616 wqe->wr.ex.invalidate_rkey); in hfi1_make_rc_req()
619 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
632 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
636 wqe->rdma_wr.remote_addr, in hfi1_make_rc_req()
639 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_make_rc_req()
647 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { in hfi1_make_rc_req()
653 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
655 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
676 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, in hfi1_make_rc_req()
743 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
751 req->comp_seg = delta_psn(bth2, wqe->psn); in hfi1_make_rc_req()
758 delta_psn(wqe->lpsn, bth2) + 1; in hfi1_make_rc_req()
763 wqe->wr.opcode, in hfi1_make_rc_req()
764 wqe->psn, wqe->lpsn, in hfi1_make_rc_req()
784 wqe->rdma_wr.remote_addr, in hfi1_make_rc_req()
787 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_make_rc_req()
800 wpriv = wqe->priv; in hfi1_make_rc_req()
801 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
803 wqe->wr.opcode, in hfi1_make_rc_req()
804 wqe->psn, wqe->lpsn, in hfi1_make_rc_req()
806 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
833 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
834 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
835 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
836 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
837 qp->s_len = wqe->length; in hfi1_make_rc_req()
854 wqe->length - req->seg_len * req->cur_seg); in hfi1_make_rc_req()
855 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, in hfi1_make_rc_req()
888 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in hfi1_make_rc_req()
889 wqe->wr.opcode == IB_WR_OPFN) { in hfi1_make_rc_req()
891 put_ib_ateth_swap(wqe->atomic_wr.swap, in hfi1_make_rc_req()
893 put_ib_ateth_compare(wqe->atomic_wr.compare_add, in hfi1_make_rc_req()
897 put_ib_ateth_swap(wqe->atomic_wr.compare_add, in hfi1_make_rc_req()
901 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, in hfi1_make_rc_req()
904 wqe->atomic_wr.rkey); in hfi1_make_rc_req()
916 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) { in hfi1_make_rc_req()
917 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
918 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
919 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
920 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
921 qp->s_len = wqe->length; in hfi1_make_rc_req()
928 if (wqe->wr.opcode == IB_WR_RDMA_READ || in hfi1_make_rc_req()
929 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in hfi1_make_rc_req()
930 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
931 else if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_make_rc_req()
947 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
961 if (wqe->wr.opcode == IB_WR_SEND) { in hfi1_make_rc_req()
963 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { in hfi1_make_rc_req()
966 ohdr->u.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
971 ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey); in hfi1_make_rc_req()
974 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
992 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
1006 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { in hfi1_make_rc_req()
1011 ohdr->u.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
1013 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
1032 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu; in hfi1_make_rc_req()
1034 wqe->rdma_wr.remote_addr + len, in hfi1_make_rc_req()
1037 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_make_rc_req()
1038 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); in hfi1_make_rc_req()
1042 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1056 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
1060 req->comp_seg = delta_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1061 len = wqe->length - (req->comp_seg * remote->max_len); in hfi1_make_rc_req()
1065 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1067 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1070 priv->pending_tid_w_resp += delta_psn(wqe->lpsn, bth2) + 1; in hfi1_make_rc_req()
1074 trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1075 wqe->psn, wqe->lpsn, req); in hfi1_make_rc_req()
1079 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) in hfi1_make_rc_req()
1082 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
1083 wpriv = wqe->priv; in hfi1_make_rc_req()
1089 req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps; in hfi1_make_rc_req()
1098 hfi1_tid_rdma_restart_req(qp, wqe, &bth2); in hfi1_make_rc_req()
1107 hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR); in hfi1_make_rc_req()
1112 wqe->length - req->seg_len * req->cur_seg); in hfi1_make_rc_req()
1116 delta = hfi1_build_tid_rdma_read_packet(wqe, ohdr, &bth1, in hfi1_make_rc_req()
1129 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1130 wqe->psn, wqe->lpsn, req); in hfi1_make_rc_req()
1133 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
1134 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1140 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ || delta == 0 || in hfi1_make_rc_req()
1155 wpriv = wqe->priv; in hfi1_make_rc_req()
1158 wqe->length - req->seg_len * req->cur_seg); in hfi1_make_rc_req()
1159 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1172 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1173 wqe->psn, wqe->lpsn, req); in hfi1_make_rc_req()
1177 delta = delta_psn(bth2, wqe->psn); in hfi1_make_rc_req()
1179 wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in hfi1_make_rc_req()
1460 struct rvt_swqe *wqe) in update_num_rd_atomic() argument
1462 u32 opcode = wqe->wr.opcode; in update_num_rd_atomic()
1469 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in update_num_rd_atomic()
1472 if (cmp_psn(psn, wqe->lpsn) <= 0) { in update_num_rd_atomic()
1475 cur_seg = (psn - wqe->psn) / priv->pkts_ps; in update_num_rd_atomic()
1480 wqe->wr.opcode, in update_num_rd_atomic()
1481 wqe->psn, in update_num_rd_atomic()
1482 wqe->lpsn, in update_num_rd_atomic()
1503 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); in reset_psn() local
1517 if (cmp_psn(psn, wqe->psn) <= 0) { in reset_psn()
1521 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1531 wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
1532 diff = cmp_psn(psn, wqe->psn); in reset_psn()
1535 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in reset_psn()
1548 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1550 opcode = wqe->wr.opcode; in reset_psn()
1609 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_restart_rc() local
1624 if (wqe->wr.opcode == IB_WR_OPFN) { in hfi1_restart_rc()
1633 wqe = do_rc_completion(qp, wqe, ibp); in hfi1_restart_rc()
1637 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in hfi1_restart_rc()
1640 req = wqe_to_tid_req(wqe); in hfi1_restart_rc()
1645 hfi1_trdma_send_complete(qp, wqe, in hfi1_restart_rc()
1658 if (wqe->wr.opcode == IB_WR_RDMA_READ || in hfi1_restart_rc()
1659 wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_restart_rc()
1679 struct rvt_swqe *wqe; in reset_sending_psn() local
1685 wqe = rvt_get_swqe_ptr(qp, n); in reset_sending_psn()
1686 if (cmp_psn(psn, wqe->lpsn) <= 0) { in reset_sending_psn()
1687 if (wqe->wr.opcode == IB_WR_RDMA_READ || in reset_sending_psn()
1688 wqe->wr.opcode == IB_WR_TID_RDMA_READ || in reset_sending_psn()
1689 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in reset_sending_psn()
1690 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
1739 struct rvt_swqe *wqe; in hfi1_rc_send_complete() local
1782 wqe = rvt_get_swqe_ptr(qp, tail); in hfi1_rc_send_complete()
1783 req = wqe_to_tid_req(wqe); in hfi1_rc_send_complete()
1823 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_send_complete()
1824 req = wqe_to_tid_req(wqe); in hfi1_rc_send_complete()
1825 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in hfi1_rc_send_complete()
1831 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_rc_send_complete()
1832 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && in hfi1_rc_send_complete()
1835 trdma_clean_swqe(qp, wqe); in hfi1_rc_send_complete()
1836 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in hfi1_rc_send_complete()
1838 wqe, in hfi1_rc_send_complete()
1839 ib_hfi1_wc_opcode[wqe->wr.opcode], in hfi1_rc_send_complete()
1867 struct rvt_swqe *wqe, in do_rc_completion() argument
1878 trace_hfi1_rc_completion(qp, wqe->lpsn); in do_rc_completion()
1879 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
1881 trdma_clean_swqe(qp, wqe); in do_rc_completion()
1882 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in do_rc_completion()
1884 wqe, in do_rc_completion()
1885 ib_hfi1_wc_opcode[wqe->wr.opcode], in do_rc_completion()
1916 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in do_rc_completion()
1917 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
1928 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
1931 qp->s_psn = wqe->psn; in do_rc_completion()
1938 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
1944 return wqe; in do_rc_completion()
2012 struct rvt_swqe *wqe; in do_rc_ack() local
2028 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
2035 while ((diff = delta_psn(ack_psn, wqe->lpsn)) >= 0) { in do_rc_ack()
2042 if (wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
2057 if ((wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
2059 (wqe->wr.opcode == IB_WR_TID_RDMA_READ && in do_rc_ack()
2061 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
2062 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && in do_rc_ack()
2064 (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in do_rc_ack()
2073 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
2074 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { in do_rc_ack()
2075 u64 *vaddr = wqe->sg_list[0].vaddr; in do_rc_ack()
2078 if (wqe->wr.opcode == IB_WR_OPFN) in do_rc_ack()
2082 (wqe->wr.opcode == IB_WR_RDMA_READ || in do_rc_ack()
2083 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
2084 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { in do_rc_ack()
2103 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in do_rc_ack()
2106 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
2111 trace_hfi1_rc_ack_do(qp, aeth, psn, wqe); in do_rc_ack()
2116 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in do_rc_ack()
2117 if (wqe_to_tid_req(wqe)->ack_pending) in do_rc_ack()
2197 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in do_rc_ack()
2199 cmp_psn(psn, wqe->psn) >= 0) in do_rc_ack()
2211 if (!(rdi->post_parms[wqe->wr.opcode].flags & in do_rc_ack()
2227 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { in do_rc_ack()
2274 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in do_rc_ack()
2277 hfi1_trdma_send_complete(qp, wqe, status); in do_rc_ack()
2308 struct rvt_swqe *wqe; in rdma_seq_err() local
2314 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
2316 while (cmp_psn(psn, wqe->lpsn) > 0) { in rdma_seq_err()
2317 if (wqe->wr.opcode == IB_WR_RDMA_READ || in rdma_seq_err()
2318 wqe->wr.opcode == IB_WR_TID_RDMA_READ || in rdma_seq_err()
2319 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE || in rdma_seq_err()
2320 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in rdma_seq_err()
2321 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) in rdma_seq_err()
2323 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
2352 struct rvt_swqe *wqe; in rc_rcv_resp() local
2396 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2411 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2412 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in rc_rcv_resp()
2420 wqe, psn, pmtu); in rc_rcv_resp()
2427 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in rc_rcv_resp()
2474 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2476 wqe, psn, pmtu); in rc_rcv_resp()
2483 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in rc_rcv_resp()
2517 rvt_send_complete(qp, wqe, status); in rc_rcv_resp()