Lines Matching refs:wqe
394 struct rvt_swqe *wqe; in hfi1_make_rc_req() local
449 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_make_rc_req()
450 hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in hfi1_make_rc_req()
469 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in hfi1_make_rc_req()
495 if ((wqe->wr.send_flags & IB_SEND_FENCE) && in hfi1_make_rc_req()
497 (wqe->wr.opcode != IB_WR_TID_RDMA_READ || in hfi1_make_rc_req()
506 if (wqe->wr.opcode == IB_WR_REG_MR || in hfi1_make_rc_req()
507 wqe->wr.opcode == IB_WR_LOCAL_INV) { in hfi1_make_rc_req()
517 if (!(wqe->wr.send_flags & in hfi1_make_rc_req()
521 wqe->wr.ex.invalidate_rkey); in hfi1_make_rc_req()
524 rvt_send_complete(qp, wqe, in hfi1_make_rc_req()
533 qp->s_psn = wqe->psn; in hfi1_make_rc_req()
540 len = wqe->length; in hfi1_make_rc_req()
549 hfi1_tid_rdma_wqe_interlock(qp, wqe)) in hfi1_make_rc_req()
552 switch (wqe->wr.opcode) { in hfi1_make_rc_req()
557 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
564 if (wqe->wr.opcode == IB_WR_SEND) { in hfi1_make_rc_req()
566 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { in hfi1_make_rc_req()
569 ohdr->u.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
575 wqe->wr.ex.invalidate_rkey); in hfi1_make_rc_req()
578 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
591 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
595 wqe->rdma_wr.remote_addr, in hfi1_make_rc_req()
598 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_make_rc_req()
606 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { in hfi1_make_rc_req()
612 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
614 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
635 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, in hfi1_make_rc_req()
702 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
710 req->comp_seg = delta_psn(bth2, wqe->psn); in hfi1_make_rc_req()
717 delta_psn(wqe->lpsn, bth2) + 1; in hfi1_make_rc_req()
722 wqe->wr.opcode, in hfi1_make_rc_req()
723 wqe->psn, wqe->lpsn, in hfi1_make_rc_req()
743 wqe->rdma_wr.remote_addr, in hfi1_make_rc_req()
746 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_make_rc_req()
759 wpriv = wqe->priv; in hfi1_make_rc_req()
760 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
762 wqe->wr.opcode, in hfi1_make_rc_req()
763 wqe->psn, wqe->lpsn, in hfi1_make_rc_req()
765 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
792 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
793 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
794 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
795 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
796 qp->s_len = wqe->length; in hfi1_make_rc_req()
813 wqe->length - req->seg_len * req->cur_seg); in hfi1_make_rc_req()
814 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, in hfi1_make_rc_req()
847 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in hfi1_make_rc_req()
848 wqe->wr.opcode == IB_WR_OPFN) { in hfi1_make_rc_req()
850 put_ib_ateth_swap(wqe->atomic_wr.swap, in hfi1_make_rc_req()
852 put_ib_ateth_compare(wqe->atomic_wr.compare_add, in hfi1_make_rc_req()
856 put_ib_ateth_swap(wqe->atomic_wr.compare_add, in hfi1_make_rc_req()
860 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, in hfi1_make_rc_req()
863 wqe->atomic_wr.rkey); in hfi1_make_rc_req()
875 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) { in hfi1_make_rc_req()
876 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
877 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
878 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
879 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
880 qp->s_len = wqe->length; in hfi1_make_rc_req()
887 if (wqe->wr.opcode == IB_WR_RDMA_READ || in hfi1_make_rc_req()
888 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in hfi1_make_rc_req()
889 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
890 else if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_make_rc_req()
906 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
920 if (wqe->wr.opcode == IB_WR_SEND) { in hfi1_make_rc_req()
922 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { in hfi1_make_rc_req()
925 ohdr->u.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
930 ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey); in hfi1_make_rc_req()
933 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
951 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
965 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { in hfi1_make_rc_req()
970 ohdr->u.imm_data = wqe->wr.ex.imm_data; in hfi1_make_rc_req()
972 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in hfi1_make_rc_req()
991 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu; in hfi1_make_rc_req()
993 wqe->rdma_wr.remote_addr + len, in hfi1_make_rc_req()
996 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_make_rc_req()
997 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); in hfi1_make_rc_req()
1001 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1015 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
1019 req->comp_seg = delta_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1020 len = wqe->length - (req->comp_seg * remote->max_len); in hfi1_make_rc_req()
1024 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1026 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1029 priv->pending_tid_w_resp += delta_psn(wqe->lpsn, bth2) + 1; in hfi1_make_rc_req()
1033 trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1034 wqe->psn, wqe->lpsn, req); in hfi1_make_rc_req()
1038 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) in hfi1_make_rc_req()
1041 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
1042 wpriv = wqe->priv; in hfi1_make_rc_req()
1048 req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps; in hfi1_make_rc_req()
1057 hfi1_tid_rdma_restart_req(qp, wqe, &bth2); in hfi1_make_rc_req()
1066 hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR); in hfi1_make_rc_req()
1071 wqe->length - req->seg_len * req->cur_seg); in hfi1_make_rc_req()
1075 delta = hfi1_build_tid_rdma_read_packet(wqe, ohdr, &bth1, in hfi1_make_rc_req()
1088 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1089 wqe->psn, wqe->lpsn, req); in hfi1_make_rc_req()
1092 req = wqe_to_tid_req(wqe); in hfi1_make_rc_req()
1093 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1099 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ || delta == 0 || in hfi1_make_rc_req()
1114 wpriv = wqe->priv; in hfi1_make_rc_req()
1117 wqe->length - req->seg_len * req->cur_seg); in hfi1_make_rc_req()
1118 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1131 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1132 wqe->psn, wqe->lpsn, req); in hfi1_make_rc_req()
1136 delta = delta_psn(bth2, wqe->psn); in hfi1_make_rc_req()
1138 wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in hfi1_make_rc_req()
1418 struct rvt_swqe *wqe) in update_num_rd_atomic() argument
1420 u32 opcode = wqe->wr.opcode; in update_num_rd_atomic()
1427 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in update_num_rd_atomic()
1430 if (cmp_psn(psn, wqe->lpsn) <= 0) { in update_num_rd_atomic()
1433 cur_seg = (psn - wqe->psn) / priv->pkts_ps; in update_num_rd_atomic()
1438 wqe->wr.opcode, in update_num_rd_atomic()
1439 wqe->psn, in update_num_rd_atomic()
1440 wqe->lpsn, in update_num_rd_atomic()
1461 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); in reset_psn() local
1475 if (cmp_psn(psn, wqe->psn) <= 0) { in reset_psn()
1479 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1489 wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
1490 diff = cmp_psn(psn, wqe->psn); in reset_psn()
1493 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in reset_psn()
1506 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1508 opcode = wqe->wr.opcode; in reset_psn()
1567 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_restart_rc() local
1582 if (wqe->wr.opcode == IB_WR_OPFN) { in hfi1_restart_rc()
1591 wqe = do_rc_completion(qp, wqe, ibp); in hfi1_restart_rc()
1595 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in hfi1_restart_rc()
1598 req = wqe_to_tid_req(wqe); in hfi1_restart_rc()
1603 hfi1_trdma_send_complete(qp, wqe, in hfi1_restart_rc()
1616 if (wqe->wr.opcode == IB_WR_RDMA_READ || in hfi1_restart_rc()
1617 wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_restart_rc()
1637 struct rvt_swqe *wqe; in reset_sending_psn() local
1643 wqe = rvt_get_swqe_ptr(qp, n); in reset_sending_psn()
1644 if (cmp_psn(psn, wqe->lpsn) <= 0) { in reset_sending_psn()
1645 if (wqe->wr.opcode == IB_WR_RDMA_READ || in reset_sending_psn()
1646 wqe->wr.opcode == IB_WR_TID_RDMA_READ || in reset_sending_psn()
1647 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in reset_sending_psn()
1648 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
1697 struct rvt_swqe *wqe; in hfi1_rc_send_complete() local
1740 wqe = rvt_get_swqe_ptr(qp, tail); in hfi1_rc_send_complete()
1741 req = wqe_to_tid_req(wqe); in hfi1_rc_send_complete()
1781 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_send_complete()
1782 req = wqe_to_tid_req(wqe); in hfi1_rc_send_complete()
1783 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in hfi1_rc_send_complete()
1789 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_rc_send_complete()
1790 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && in hfi1_rc_send_complete()
1793 trdma_clean_swqe(qp, wqe); in hfi1_rc_send_complete()
1794 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in hfi1_rc_send_complete()
1796 wqe, in hfi1_rc_send_complete()
1797 ib_hfi1_wc_opcode[wqe->wr.opcode], in hfi1_rc_send_complete()
1825 struct rvt_swqe *wqe, in do_rc_completion() argument
1836 trace_hfi1_rc_completion(qp, wqe->lpsn); in do_rc_completion()
1837 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
1839 trdma_clean_swqe(qp, wqe); in do_rc_completion()
1840 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in do_rc_completion()
1842 wqe, in do_rc_completion()
1843 ib_hfi1_wc_opcode[wqe->wr.opcode], in do_rc_completion()
1874 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in do_rc_completion()
1875 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
1886 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
1889 qp->s_psn = wqe->psn; in do_rc_completion()
1896 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
1902 return wqe; in do_rc_completion()
1970 struct rvt_swqe *wqe; in do_rc_ack() local
1986 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
1993 while ((diff = delta_psn(ack_psn, wqe->lpsn)) >= 0) { in do_rc_ack()
2000 if (wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
2015 if ((wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
2017 (wqe->wr.opcode == IB_WR_TID_RDMA_READ && in do_rc_ack()
2019 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
2020 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && in do_rc_ack()
2022 (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in do_rc_ack()
2031 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
2032 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { in do_rc_ack()
2033 u64 *vaddr = wqe->sg_list[0].vaddr; in do_rc_ack()
2036 if (wqe->wr.opcode == IB_WR_OPFN) in do_rc_ack()
2040 (wqe->wr.opcode == IB_WR_RDMA_READ || in do_rc_ack()
2041 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
2042 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { in do_rc_ack()
2061 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in do_rc_ack()
2064 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
2069 trace_hfi1_rc_ack_do(qp, aeth, psn, wqe); in do_rc_ack()
2074 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in do_rc_ack()
2075 if (wqe_to_tid_req(wqe)->ack_pending) in do_rc_ack()
2155 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in do_rc_ack()
2157 cmp_psn(psn, wqe->psn) >= 0) in do_rc_ack()
2169 if (!(rdi->post_parms[wqe->wr.opcode].flags & in do_rc_ack()
2185 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { in do_rc_ack()
2232 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in do_rc_ack()
2235 hfi1_trdma_send_complete(qp, wqe, status); in do_rc_ack()
2266 struct rvt_swqe *wqe; in rdma_seq_err() local
2272 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
2274 while (cmp_psn(psn, wqe->lpsn) > 0) { in rdma_seq_err()
2275 if (wqe->wr.opcode == IB_WR_RDMA_READ || in rdma_seq_err()
2276 wqe->wr.opcode == IB_WR_TID_RDMA_READ || in rdma_seq_err()
2277 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE || in rdma_seq_err()
2278 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in rdma_seq_err()
2279 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) in rdma_seq_err()
2281 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
2310 struct rvt_swqe *wqe; in rc_rcv_resp() local
2354 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2369 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2370 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in rc_rcv_resp()
2378 wqe, psn, pmtu); in rc_rcv_resp()
2385 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in rc_rcv_resp()
2432 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2434 wqe, psn, pmtu); in rc_rcv_resp()
2441 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in rc_rcv_resp()
2475 rvt_send_complete(qp, wqe, status); in rc_rcv_resp()