Lines Matching refs:wqe
378 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_priv_init() local
386 priv->tid_req.e.swqe = wqe; in hfi1_qp_priv_init()
387 wqe->priv = priv; in hfi1_qp_priv_init()
416 struct rvt_swqe *wqe; in hfi1_qp_priv_tid_free() local
421 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_priv_tid_free()
422 kfree(wqe->priv); in hfi1_qp_priv_tid_free()
423 wqe->priv = NULL; in hfi1_qp_priv_tid_free()
1621 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) in __trdma_clean_swqe() argument
1623 struct hfi1_swqe_priv *p = wqe->priv; in __trdma_clean_swqe()
1703 u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, in hfi1_build_tid_rdma_read_packet() argument
1707 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_read_packet()
1711 struct hfi1_swqe_priv *wpriv = wqe->priv; in hfi1_build_tid_rdma_read_packet()
1748 rreq->reth.vaddr = cpu_to_be64(wqe->rdma_wr.remote_addr + in hfi1_build_tid_rdma_read_packet()
1750 rreq->reth.rkey = cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_build_tid_rdma_read_packet()
1787 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_read_req() argument
1793 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_read_req()
1800 trace_hfi1_tid_req_build_read_req(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_build_tid_rdma_read_req()
1801 wqe->lpsn, req); in hfi1_build_tid_rdma_read_req()
1829 restart_sge(&qp->s_sge, wqe, req->s_next_psn, in hfi1_build_tid_rdma_read_req()
1879 hdwords = hfi1_build_tid_rdma_read_packet(wqe, ohdr, bth1, bth2, len); in hfi1_build_tid_rdma_read_req()
2418 struct rvt_swqe *wqe; in find_tid_request() local
2426 wqe = rvt_get_swqe_ptr(qp, i); in find_tid_request()
2427 if (cmp_psn(psn, wqe->psn) >= 0 && in find_tid_request()
2428 cmp_psn(psn, wqe->lpsn) <= 0) { in find_tid_request()
2429 if (wqe->wr.opcode == opcode) in find_tid_request()
2430 req = wqe_to_tid_req(wqe); in find_tid_request()
2582 struct rvt_swqe *wqe; in hfi1_kern_read_tid_flow_free() local
2589 wqe = rvt_get_swqe_ptr(qp, n); in hfi1_kern_read_tid_flow_free()
2590 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in hfi1_kern_read_tid_flow_free()
2591 req = wqe_to_tid_req(wqe); in hfi1_kern_read_tid_flow_free()
2630 struct rvt_qp *qp, struct rvt_swqe *wqe) in restart_tid_rdma_read_req() argument
2637 req = wqe_to_tid_req(wqe); in restart_tid_rdma_read_req()
2663 struct rvt_swqe *wqe; in handle_read_kdeth_eflags() local
2689 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in handle_read_kdeth_eflags()
2693 while ((int)delta_psn(ack_psn, wqe->lpsn) >= 0) { in handle_read_kdeth_eflags()
2699 if (wqe->wr.opcode == IB_WR_RDMA_READ || in handle_read_kdeth_eflags()
2700 wqe->wr.opcode == IB_WR_TID_RDMA_READ || in handle_read_kdeth_eflags()
2701 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in handle_read_kdeth_eflags()
2702 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { in handle_read_kdeth_eflags()
2706 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in handle_read_kdeth_eflags()
2708 wqe); in handle_read_kdeth_eflags()
2728 wqe = do_rc_completion(qp, wqe, ibp); in handle_read_kdeth_eflags()
2737 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) in handle_read_kdeth_eflags()
2740 req = wqe_to_tid_req(wqe); in handle_read_kdeth_eflags()
2741 trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe->wr.opcode, wqe->psn, in handle_read_kdeth_eflags()
2742 wqe->lpsn, req); in handle_read_kdeth_eflags()
2808 wqe); in handle_read_kdeth_eflags()
3046 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_tid_rdma_restart_req() argument
3049 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_tid_rdma_restart_req()
3056 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in hfi1_tid_rdma_restart_req()
3063 trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, in hfi1_tid_rdma_restart_req()
3064 wqe->psn, wqe->lpsn, in hfi1_tid_rdma_restart_req()
3074 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_tid_rdma_restart_req()
3106 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { in hfi1_tid_rdma_restart_req()
3125 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_tid_rdma_restart_req()
3132 trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_tid_rdma_restart_req()
3133 wqe->lpsn, req); in hfi1_tid_rdma_restart_req()
3135 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { in hfi1_tid_rdma_restart_req()
3152 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_tid_rdma_restart_req()
3153 } while (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE); in hfi1_tid_rdma_restart_req()
3154 req = wqe_to_tid_req(wqe); in hfi1_tid_rdma_restart_req()
3181 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_kern_exp_rcv_clear_all() local
3186 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) in hfi1_qp_kern_exp_rcv_clear_all()
3189 struct hfi1_swqe_priv *priv = wqe->priv; in hfi1_qp_kern_exp_rcv_clear_all()
3210 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) in hfi1_tid_rdma_wqe_interlock() argument
3220 switch (wqe->wr.opcode) { in hfi1_tid_rdma_wqe_interlock()
3282 void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe) in setup_tid_rdma_wqe() argument
3285 struct hfi1_swqe_priv *priv = wqe->priv; in setup_tid_rdma_wqe()
3306 if (wqe->wr.opcode == IB_WR_RDMA_READ) { in setup_tid_rdma_wqe()
3307 if (hfi1_check_sge_align(qp, &wqe->sg_list[0], in setup_tid_rdma_wqe()
3308 wqe->wr.num_sge)) { in setup_tid_rdma_wqe()
3312 } else if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { in setup_tid_rdma_wqe()
3319 if (!(wqe->rdma_wr.remote_addr & ~PAGE_MASK) && in setup_tid_rdma_wqe()
3320 !(wqe->length & ~PAGE_MASK)) { in setup_tid_rdma_wqe()
3329 wqe->wr.opcode = new_opcode; in setup_tid_rdma_wqe()
3331 min_t(u32, remote->max_len, wqe->length); in setup_tid_rdma_wqe()
3333 DIV_ROUND_UP(wqe->length, priv->tid_req.seg_len); in setup_tid_rdma_wqe()
3335 wqe->lpsn = wqe->psn; in setup_tid_rdma_wqe()
3336 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in setup_tid_rdma_wqe()
3339 wqe->lpsn += rvt_div_round_up_mtu(qp, wqe->length) - 1; in setup_tid_rdma_wqe()
3341 wqe->lpsn += priv->tid_req.total_segs - 1; in setup_tid_rdma_wqe()
3356 trace_hfi1_tid_req_setup_tid_wqe(qp, 1, wqe->wr.opcode, in setup_tid_rdma_wqe()
3357 wqe->psn, wqe->lpsn, in setup_tid_rdma_wqe()
3366 u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_write_req() argument
3371 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_write_req()
3386 cpu_to_be64(wqe->rdma_wr.remote_addr + (wqe->length - *len)); in hfi1_build_tid_rdma_write_req()
3388 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_build_tid_rdma_write_req()
4044 struct rvt_swqe *wqe; in hfi1_rc_rcv_tid_rdma_write_resp() local
4081 wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); in hfi1_rc_rcv_tid_rdma_write_resp()
4082 if (unlikely(wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)) in hfi1_rc_rcv_tid_rdma_write_resp()
4085 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_write_resp()
4120 (wqe->length - (req->comp_seg * req->seg_len))); in hfi1_rc_rcv_tid_rdma_write_resp()
4155 trace_hfi1_tid_req_rcv_write_resp(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_write_resp()
4156 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_write_resp()
4161 if (!cmp_psn(psn, wqe->psn)) { in hfi1_rc_rcv_tid_rdma_write_resp()
4162 req->r_last_acked = mask_psn(wqe->psn - 1); in hfi1_rc_rcv_tid_rdma_write_resp()
4183 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_rc_rcv_tid_rdma_write_resp()
4186 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in hfi1_rc_rcv_tid_rdma_write_resp()
4205 bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe, in hfi1_build_tid_rdma_packet() argument
4209 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_packet()
4221 hfi1_trdma_send_complete(qp, wqe, IB_WC_REM_INV_RD_REQ_ERR); in hfi1_build_tid_rdma_packet()
4502 struct rvt_swqe *wqe; in hfi1_rc_rcv_tid_rdma_ack() local
4537 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_rcv_tid_rdma_ack()
4539 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in hfi1_rc_rcv_tid_rdma_ack()
4542 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4543 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4544 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_ack()
4560 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4561 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_ack()
4564 wqe = do_rc_completion(qp, wqe, in hfi1_rc_rcv_tid_rdma_ack()
4571 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in hfi1_rc_rcv_tid_rdma_ack()
4573 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4579 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4580 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_ack()
4587 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in hfi1_rc_rcv_tid_rdma_ack()
4634 if (delta_psn(ack_psn, wqe->lpsn)) in hfi1_rc_rcv_tid_rdma_ack()
4635 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_rcv_tid_rdma_ack()
4636 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4704 wqe = rvt_get_swqe_ptr(qp, last_acked); in hfi1_rc_rcv_tid_rdma_ack()
4705 rptr = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4802 struct rvt_swqe *wqe; in hfi1_tid_retry_timeout() local
4817 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_tid_retry_timeout()
4818 hfi1_trdma_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in hfi1_tid_retry_timeout()
4821 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_tid_retry_timeout()
4822 req = wqe_to_tid_req(wqe); in hfi1_tid_retry_timeout()
4824 qp, 0, wqe->wr.opcode, wqe->psn, wqe->lpsn, req); in hfi1_tid_retry_timeout()
4843 u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_resync() argument
4849 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_resync()
4988 struct rvt_swqe *wqe; in update_tid_tail() local
5000 wqe = rvt_get_swqe_ptr(qp, i); in update_tid_tail()
5001 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in update_tid_tail()
5012 struct rvt_swqe *wqe; in hfi1_make_tid_rdma_pkt() local
5068 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); in hfi1_make_tid_rdma_pkt()
5069 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5070 trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_make_tid_rdma_pkt()
5071 wqe->lpsn, req); in hfi1_make_tid_rdma_pkt()
5075 priv->tid_ss.sge = wqe->sg_list[0]; in hfi1_make_tid_rdma_pkt()
5076 priv->tid_ss.sg_list = wqe->sg_list + 1; in hfi1_make_tid_rdma_pkt()
5077 priv->tid_ss.num_sge = wqe->wr.num_sge; in hfi1_make_tid_rdma_pkt()
5078 priv->tid_ss.total_len = wqe->length; in hfi1_make_tid_rdma_pkt()
5081 hfi1_tid_rdma_restart_req(qp, wqe, &bth2); in hfi1_make_tid_rdma_pkt()
5101 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); in hfi1_make_tid_rdma_pkt()
5102 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5103 len = wqe->length; in hfi1_make_tid_rdma_pkt()
5108 trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, in hfi1_make_tid_rdma_pkt()
5109 wqe->psn, wqe->lpsn, req); in hfi1_make_tid_rdma_pkt()
5110 last = hfi1_build_tid_rdma_packet(wqe, ohdr, &bth1, &bth2, in hfi1_make_tid_rdma_pkt()
5136 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_cur); in hfi1_make_tid_rdma_pkt()
5137 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5140 wqe = rvt_get_swqe_ptr(qp, in hfi1_make_tid_rdma_pkt()
5143 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5145 hwords += hfi1_build_tid_rdma_resync(qp, wqe, ohdr, &bth1, in hfi1_make_tid_rdma_pkt()