Lines Matching refs:wqe
633 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last); in rvt_clear_mr_refs() local
635 rvt_put_qp_swqe(qp, wqe); in rvt_clear_mr_refs()
663 static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey) in rvt_swqe_has_lkey() argument
667 for (i = 0; i < wqe->wr.num_sge; i++) { in rvt_swqe_has_lkey()
668 struct rvt_sge *sge = &wqe->sg_list[i]; in rvt_swqe_has_lkey()
686 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last); in rvt_qp_sends_has_lkey() local
688 if (rvt_swqe_has_lkey(wqe, lkey)) in rvt_qp_sends_has_lkey()
1018 struct rvt_swqe *wqe; in free_ud_wq_attr() local
1022 wqe = rvt_get_swqe_ptr(qp, i); in free_ud_wq_attr()
1023 kfree(wqe->ud_wr.attr); in free_ud_wq_attr()
1024 wqe->ud_wr.attr = NULL; in free_ud_wq_attr()
1038 struct rvt_swqe *wqe; in alloc_ud_wq_attr() local
1042 wqe = rvt_get_swqe_ptr(qp, i); in alloc_ud_wq_attr()
1043 wqe->ud_wr.attr = kzalloc_node(sizeof(*wqe->ud_wr.attr), in alloc_ud_wq_attr()
1045 if (!wqe->ud_wr.attr) { in alloc_ud_wq_attr()
1851 struct rvt_rwqe *wqe; in rvt_post_recv() local
1879 wqe = rvt_get_rwqe_ptr(&qp->r_rq, wq->head); in rvt_post_recv()
1880 wqe->wr_id = wr->wr_id; in rvt_post_recv()
1881 wqe->num_sge = wr->num_sge; in rvt_post_recv()
1883 wqe->sg_list[i].addr = wr->sg_list[i].addr; in rvt_post_recv()
1884 wqe->sg_list[i].length = wr->sg_list[i].length; in rvt_post_recv()
1885 wqe->sg_list[i].lkey = wr->sg_list[i].lkey; in rvt_post_recv()
2017 struct rvt_swqe *wqe; in rvt_post_one_wr() local
2091 wqe = rvt_get_swqe_ptr(qp, qp->s_head); in rvt_post_one_wr()
2094 memcpy(&wqe->wr, wr, cplen); in rvt_post_one_wr()
2096 wqe->length = 0; in rvt_post_one_wr()
2108 ret = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], last_sge, in rvt_post_one_wr()
2112 wqe->length += length; in rvt_post_one_wr()
2114 last_sge = &wqe->sg_list[j]; in rvt_post_one_wr()
2117 wqe->wr.num_sge = j; in rvt_post_one_wr()
2127 struct rvt_ah *ah = rvt_get_swqe_ah(wqe); in rvt_post_one_wr()
2130 rdma_copy_ah_attr(wqe->ud_wr.attr, &ah->attr); in rvt_post_one_wr()
2137 wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY; in rvt_post_one_wr()
2138 wqe->ssn = 0; in rvt_post_one_wr()
2139 wqe->psn = 0; in rvt_post_one_wr()
2140 wqe->lpsn = 0; in rvt_post_one_wr()
2142 wqe->ssn = qp->s_ssn++; in rvt_post_one_wr()
2143 wqe->psn = qp->s_next_psn; in rvt_post_one_wr()
2144 wqe->lpsn = wqe->psn + in rvt_post_one_wr()
2145 (wqe->length ? in rvt_post_one_wr()
2146 ((wqe->length - 1) >> log_pmtu) : in rvt_post_one_wr()
2152 ret = rdi->driver_f.setup_wqe(qp, wqe, call_send); in rvt_post_one_wr()
2158 qp->s_next_psn = wqe->lpsn + 1; in rvt_post_one_wr()
2161 wqe->wr.send_flags |= RVT_SEND_RESERVE_USED; in rvt_post_one_wr()
2162 rvt_qp_wqe_reserve(qp, wqe); in rvt_post_one_wr()
2164 wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED; in rvt_post_one_wr()
2167 trace_rvt_post_one_wr(qp, wqe, wr->num_sge); in rvt_post_one_wr()
2175 rdma_destroy_ah_attr(wqe->ud_wr.attr); in rvt_post_one_wr()
2179 struct rvt_sge *sge = &wqe->sg_list[--j]; in rvt_post_one_wr()
2265 struct rvt_rwqe *wqe; in rvt_post_srq_recv() local
2285 wqe = rvt_get_rwqe_ptr(&srq->rq, wq->head); in rvt_post_srq_recv()
2286 wqe->wr_id = wr->wr_id; in rvt_post_srq_recv()
2287 wqe->num_sge = wr->num_sge; in rvt_post_srq_recv()
2289 wqe->sg_list[i].addr = wr->sg_list[i].addr; in rvt_post_srq_recv()
2290 wqe->sg_list[i].length = wr->sg_list[i].length; in rvt_post_srq_recv()
2291 wqe->sg_list[i].lkey = wr->sg_list[i].lkey; in rvt_post_srq_recv()
2320 static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe) in init_sge() argument
2334 for (i = j = 0; i < wqe->num_sge; i++) { in init_sge()
2335 if (wqe->sg_list[i].length == 0) in init_sge()
2339 NULL, rvt_cast_sge(&wqe->sg_list[i]), in init_sge()
2343 qp->r_len += wqe->sg_list[i].length; in init_sge()
2358 wc.wr_id = wqe->wr_id; in init_sge()
2403 struct rvt_rwqe *wqe; in rvt_get_rwqe() local
2449 wqe = rvt_get_rwqe_ptr(rq, tail); in rvt_get_rwqe()
2461 if (!wr_id_only && !init_sge(qp, wqe)) { in rvt_get_rwqe()
2465 qp->r_wr_id = wqe->wr_id; in rvt_get_rwqe()
2831 void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, in rvt_send_complete() argument
2842 trace_rvt_qp_send_completion(qp, wqe, old_last); in rvt_send_complete()
2843 last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode], in rvt_send_complete()
2960 struct rvt_swqe *wqe; in rvt_ruc_loopback() local
2995 wqe = rvt_get_swqe_ptr(sqp, sqp->s_last); in rvt_ruc_loopback()
3032 sqp->s_sge.sge = wqe->sg_list[0]; in rvt_ruc_loopback()
3033 sqp->s_sge.sg_list = wqe->sg_list + 1; in rvt_ruc_loopback()
3034 sqp->s_sge.num_sge = wqe->wr.num_sge; in rvt_ruc_loopback()
3035 sqp->s_len = wqe->length; in rvt_ruc_loopback()
3036 switch (wqe->wr.opcode) { in rvt_ruc_loopback()
3041 if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) { in rvt_ruc_loopback()
3043 wqe->wr.ex.invalidate_rkey)) in rvt_ruc_loopback()
3057 if (wqe->length > qp->r_len) in rvt_ruc_loopback()
3059 switch (wqe->wr.opcode) { in rvt_ruc_loopback()
3062 wqe->wr.ex.invalidate_rkey)) { in rvt_ruc_loopback()
3065 wqe->wr.ex.invalidate_rkey; in rvt_ruc_loopback()
3070 wc.ex.imm_data = wqe->wr.ex.imm_data; in rvt_ruc_loopback()
3081 wc.ex.imm_data = wqe->wr.ex.imm_data; in rvt_ruc_loopback()
3094 if (wqe->length == 0) in rvt_ruc_loopback()
3096 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length, in rvt_ruc_loopback()
3097 wqe->rdma_wr.remote_addr, in rvt_ruc_loopback()
3098 wqe->rdma_wr.rkey, in rvt_ruc_loopback()
3103 qp->r_sge.total_len = wqe->length; in rvt_ruc_loopback()
3109 if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length, in rvt_ruc_loopback()
3110 wqe->rdma_wr.remote_addr, in rvt_ruc_loopback()
3111 wqe->rdma_wr.rkey, in rvt_ruc_loopback()
3117 qp->r_sge.sge = wqe->sg_list[0]; in rvt_ruc_loopback()
3118 qp->r_sge.sg_list = wqe->sg_list + 1; in rvt_ruc_loopback()
3119 qp->r_sge.num_sge = wqe->wr.num_sge; in rvt_ruc_loopback()
3120 qp->r_sge.total_len = wqe->length; in rvt_ruc_loopback()
3127 if (unlikely(wqe->atomic_wr.remote_addr & (sizeof(u64) - 1))) in rvt_ruc_loopback()
3130 wqe->atomic_wr.remote_addr, in rvt_ruc_loopback()
3131 wqe->atomic_wr.rkey, in rvt_ruc_loopback()
3136 sdata = wqe->atomic_wr.compare_add; in rvt_ruc_loopback()
3138 (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ? in rvt_ruc_loopback()
3141 sdata, wqe->atomic_wr.swap); in rvt_ruc_loopback()
3167 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) in rvt_ruc_loopback()
3173 wc.byte_len = wqe->length; in rvt_ruc_loopback()
3180 rvt_recv_cq(qp, &wc, wqe->wr.send_flags & IB_SEND_SOLICITED); in rvt_ruc_loopback()
3189 rvt_send_complete(sqp, wqe, send_status); in rvt_ruc_loopback()
3245 rvt_send_complete(sqp, wqe, send_status); in rvt_ruc_loopback()