Lines Matching refs:qp
114 static void hfi1_init_trdma_req(struct rvt_qp *qp,
116 static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx);
118 static void hfi1_add_tid_reap_timer(struct rvt_qp *qp);
119 static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp);
120 static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp);
121 static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp);
123 static int make_tid_rdma_ack(struct rvt_qp *qp,
126 static void hfi1_do_tid_send(struct rvt_qp *qp);
130 struct rvt_qp *qp, u32 psn, int diff, bool fecn);
143 static void tid_rdma_schedule_ack(struct rvt_qp *qp) in tid_rdma_schedule_ack() argument
145 struct hfi1_qp_priv *priv = qp->priv; in tid_rdma_schedule_ack()
148 hfi1_schedule_tid_send(qp); in tid_rdma_schedule_ack()
151 static void tid_rdma_trigger_ack(struct rvt_qp *qp) in tid_rdma_trigger_ack() argument
153 validate_r_tid_ack(qp->priv); in tid_rdma_trigger_ack()
154 tid_rdma_schedule_ack(qp); in tid_rdma_trigger_ack()
160 (((u64)p->qp & TID_OPFN_QP_CTXT_MASK) << in tid_rdma_opfn_encode()
162 ((((u64)p->qp >> 16) & TID_OPFN_QP_KDETH_MASK) << in tid_rdma_opfn_encode()
185 p->qp = in tid_rdma_opfn_decode()
193 void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p) in tid_rdma_opfn_init() argument
195 struct hfi1_qp_priv *priv = qp->priv; in tid_rdma_opfn_init()
197 p->qp = (RVT_KDETH_QP_PREFIX << 16) | priv->rcd->ctxt; in tid_rdma_opfn_init()
202 p->timeout = qp->timeout; in tid_rdma_opfn_init()
206 bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data) in tid_rdma_conn_req() argument
208 struct hfi1_qp_priv *priv = qp->priv; in tid_rdma_conn_req()
214 bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data) in tid_rdma_conn_reply() argument
216 struct hfi1_qp_priv *priv = qp->priv; in tid_rdma_conn_reply()
246 trace_hfi1_opfn_param(qp, 0, &priv->tid_rdma.local); in tid_rdma_conn_reply()
247 trace_hfi1_opfn_param(qp, 1, remote); in tid_rdma_conn_reply()
257 priv->pkts_ps = (u16)rvt_div_mtu(qp, remote->max_len); in tid_rdma_conn_reply()
269 bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data) in tid_rdma_conn_resp() argument
273 ret = tid_rdma_conn_reply(qp, *data); in tid_rdma_conn_resp()
281 (void)tid_rdma_conn_req(qp, data); in tid_rdma_conn_resp()
285 void tid_rdma_conn_error(struct rvt_qp *qp) in tid_rdma_conn_error() argument
287 struct hfi1_qp_priv *priv = qp->priv; in tid_rdma_conn_error()
320 struct rvt_qp *qp) in qp_to_rcd() argument
330 if (qp->ibqp.qp_num == 0) in qp_to_rcd()
333 ctxt = hfi1_get_qp_map(dd, qp->ibqp.qp_num >> dd->qos_shift); in qp_to_rcd()
337 int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, in hfi1_qp_priv_init() argument
340 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_qp_priv_init()
343 qpriv->rcd = qp_to_rcd(rdi, qp); in hfi1_qp_priv_init()
375 for (i = 0; i < qp->s_size; i++) { in hfi1_qp_priv_init()
377 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_priv_init()
384 hfi1_init_trdma_req(qp, &priv->tid_req); in hfi1_qp_priv_init()
396 hfi1_init_trdma_req(qp, &priv->tid_req); in hfi1_qp_priv_init()
397 priv->tid_req.e.ack = &qp->s_ack_queue[i]; in hfi1_qp_priv_init()
405 qp->s_ack_queue[i].priv = priv; in hfi1_qp_priv_init()
412 void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) in hfi1_qp_priv_tid_free() argument
414 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_qp_priv_tid_free()
418 if (qp->ibqp.qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) { in hfi1_qp_priv_tid_free()
419 for (i = 0; i < qp->s_size; i++) { in hfi1_qp_priv_tid_free()
420 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_priv_tid_free()
425 struct hfi1_ack_priv *priv = qp->s_ack_queue[i].priv; in hfi1_qp_priv_tid_free()
430 qp->s_ack_queue[i].priv = NULL; in hfi1_qp_priv_tid_free()
503 struct tid_queue *queue, struct rvt_qp *qp) in kernel_tid_waiters() argument
504 __must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) in kernel_tid_waiters()
509 lockdep_assert_held(&qp->s_lock); in kernel_tid_waiters()
512 if (!fqp || (fqp == qp && (qp->s_flags & HFI1_S_WAIT_TID_SPACE))) in kernel_tid_waiters()
534 struct tid_queue *queue, struct rvt_qp *qp) in dequeue_tid_waiter() argument
535 __must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) in dequeue_tid_waiter()
537 struct hfi1_qp_priv *priv = qp->priv; in dequeue_tid_waiter()
539 lockdep_assert_held(&qp->s_lock); in dequeue_tid_waiter()
544 qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; in dequeue_tid_waiter()
546 rvt_put_qp(qp); in dequeue_tid_waiter()
560 struct tid_queue *queue, struct rvt_qp *qp) in queue_qp_for_tid_wait() argument
561 __must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) in queue_qp_for_tid_wait()
563 struct hfi1_qp_priv *priv = qp->priv; in queue_qp_for_tid_wait()
565 lockdep_assert_held(&qp->s_lock); in queue_qp_for_tid_wait()
568 qp->s_flags |= HFI1_S_WAIT_TID_SPACE; in queue_qp_for_tid_wait()
572 trace_hfi1_qpsleep(qp, HFI1_S_WAIT_TID_SPACE); in queue_qp_for_tid_wait()
573 rvt_get_qp(qp); in queue_qp_for_tid_wait()
584 static void __trigger_tid_waiter(struct rvt_qp *qp) in __trigger_tid_waiter() argument
585 __must_hold(&qp->s_lock) in __trigger_tid_waiter()
587 lockdep_assert_held(&qp->s_lock); in __trigger_tid_waiter()
588 if (!(qp->s_flags & HFI1_S_WAIT_TID_SPACE)) in __trigger_tid_waiter()
590 trace_hfi1_qpwakeup(qp, HFI1_S_WAIT_TID_SPACE); in __trigger_tid_waiter()
591 hfi1_schedule_send(qp); in __trigger_tid_waiter()
606 static void tid_rdma_schedule_tid_wakeup(struct rvt_qp *qp) in tid_rdma_schedule_tid_wakeup() argument
614 if (!qp) in tid_rdma_schedule_tid_wakeup()
617 priv = qp->priv; in tid_rdma_schedule_tid_wakeup()
618 ibp = to_iport(qp->ibqp.device, qp->port_num); in tid_rdma_schedule_tid_wakeup()
620 dd = dd_from_ibdev(qp->ibqp.device); in tid_rdma_schedule_tid_wakeup()
628 rvt_put_qp(qp); in tid_rdma_schedule_tid_wakeup()
642 struct rvt_qp *qp; in tid_rdma_trigger_resume() local
646 qp = priv->owner; in tid_rdma_trigger_resume()
647 spin_lock_irq(&qp->s_lock); in tid_rdma_trigger_resume()
648 if (qp->s_flags & HFI1_S_WAIT_TID_SPACE) { in tid_rdma_trigger_resume()
649 spin_unlock_irq(&qp->s_lock); in tid_rdma_trigger_resume()
652 spin_unlock_irq(&qp->s_lock); in tid_rdma_trigger_resume()
654 rvt_put_qp(qp); in tid_rdma_trigger_resume()
664 static void _tid_rdma_flush_wait(struct rvt_qp *qp, struct tid_queue *queue) in _tid_rdma_flush_wait() argument
665 __must_hold(&qp->s_lock) in _tid_rdma_flush_wait()
669 if (!qp) in _tid_rdma_flush_wait()
671 lockdep_assert_held(&qp->s_lock); in _tid_rdma_flush_wait()
672 priv = qp->priv; in _tid_rdma_flush_wait()
673 qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; in _tid_rdma_flush_wait()
677 qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; in _tid_rdma_flush_wait()
679 rvt_put_qp(qp); in _tid_rdma_flush_wait()
684 void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp) in hfi1_tid_rdma_flush_wait() argument
685 __must_hold(&qp->s_lock) in hfi1_tid_rdma_flush_wait()
687 struct hfi1_qp_priv *priv = qp->priv; in hfi1_tid_rdma_flush_wait()
689 _tid_rdma_flush_wait(qp, &priv->rcd->flow_queue); in hfi1_tid_rdma_flush_wait()
690 _tid_rdma_flush_wait(qp, &priv->rcd->rarr_queue); in hfi1_tid_rdma_flush_wait()
775 int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp) in hfi1_kern_setup_hw_flow() argument
777 struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; in hfi1_kern_setup_hw_flow()
788 if (kernel_tid_waiters(rcd, &rcd->flow_queue, qp)) in hfi1_kern_setup_hw_flow()
802 dequeue_tid_waiter(rcd, &rcd->flow_queue, qp); in hfi1_kern_setup_hw_flow()
810 queue_qp_for_tid_wait(rcd, &rcd->flow_queue, qp); in hfi1_kern_setup_hw_flow()
815 void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp) in hfi1_kern_clear_hw_flow() argument
817 struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; in hfi1_kern_clear_hw_flow()
835 if (fqp == qp) { in hfi1_kern_clear_hw_flow()
892 trace_hfi1_tid_flow_page(flow->req->qp, flow, 0, 0, 0, vaddr); in tid_rdma_find_phys_blocks_4k()
895 trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 0, 0, in tid_rdma_find_phys_blocks_4k()
929 trace_hfi1_tid_pageset(flow->req->qp, setcount, in tid_rdma_find_phys_blocks_4k()
1027 trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 0, v0); in tid_rdma_find_phys_blocks_8k()
1030 trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 1, v1); in tid_rdma_find_phys_blocks_8k()
1176 trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head, in kern_get_phys_blocks()
1185 if (flow->req->qp->pmtu == enum_to_mtu(OPA_MTU_4096)) in kern_get_phys_blocks()
1213 trace_hfi1_tid_node_add(flow->req->qp, s, flow->tnode_cnt - 1, in kern_add_tid_node()
1286 trace_hfi1_msg_alloc_tids(flow->req->qp, " insufficient tids: needed ", in kern_alloc_tids()
1301 u32 pmtu_pg = flow->req->qp->pmtu >> PAGE_SHIFT; in kern_program_rcv_group()
1340 flow->req->qp, flow->tidcnt - 1, in kern_program_rcv_group()
1409 trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head, flow); in kern_program_rcvarray()
1456 __must_hold(&req->qp->s_lock) in hfi1_kern_exp_rcv_setup()
1460 struct hfi1_qp_priv *qpriv = req->qp->priv; in hfi1_kern_exp_rcv_setup()
1465 lockdep_assert_held(&req->qp->s_lock); in hfi1_kern_exp_rcv_setup()
1483 hfi1_wait_kmem(flow->req->qp); in hfi1_kern_exp_rcv_setup()
1488 if (kernel_tid_waiters(rcd, &rcd->rarr_queue, flow->req->qp)) in hfi1_kern_exp_rcv_setup()
1521 dequeue_tid_waiter(rcd, &rcd->rarr_queue, flow->req->qp); in hfi1_kern_exp_rcv_setup()
1530 queue_qp_for_tid_wait(rcd, &rcd->rarr_queue, flow->req->qp); in hfi1_kern_exp_rcv_setup()
1547 __must_hold(&req->qp->s_lock) in hfi1_kern_exp_rcv_clear()
1555 lockdep_assert_held(&req->qp->s_lock); in hfi1_kern_exp_rcv_clear()
1575 if (fqp == req->qp) { in hfi1_kern_exp_rcv_clear()
1590 __must_hold(&req->qp->s_lock) in hfi1_kern_exp_rcv_clear_all()
1614 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) in __trdma_clean_swqe() argument
1647 static void hfi1_init_trdma_req(struct rvt_qp *qp, in hfi1_init_trdma_req() argument
1650 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_init_trdma_req()
1662 req->qp = qp; in hfi1_init_trdma_req()
1702 struct rvt_qp *qp = req->qp; in hfi1_build_tid_rdma_read_packet() local
1703 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_read_packet()
1712 trace_hfi1_tid_flow_build_read_pkt(qp, req->flow_idx, flow); in hfi1_build_tid_rdma_read_packet()
1751 cpu_to_be32(qpriv->tid_rdma.local.qp | in hfi1_build_tid_rdma_read_packet()
1755 rreq->verbs_qp = cpu_to_be32(qp->remote_qpn); in hfi1_build_tid_rdma_read_packet()
1757 *bth1 |= remote->qp; in hfi1_build_tid_rdma_read_packet()
1764 qp->s_state = TID_OP(READ_REQ); in hfi1_build_tid_rdma_read_packet()
1768 qp->s_num_rd_atomic++; in hfi1_build_tid_rdma_read_packet()
1780 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_read_req() argument
1783 __must_hold(&qp->s_lock) in hfi1_build_tid_rdma_read_req()
1785 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_read_req()
1791 u32 npkts = rvt_div_round_up_mtu(qp, *len); in hfi1_build_tid_rdma_read_req()
1793 trace_hfi1_tid_req_build_read_req(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_build_tid_rdma_read_req()
1804 hfi1_kern_clear_hw_flow(req->rcd, qp); in hfi1_build_tid_rdma_read_req()
1822 restart_sge(&qp->s_sge, wqe, req->s_next_psn, in hfi1_build_tid_rdma_read_req()
1823 qp->pmtu); in hfi1_build_tid_rdma_read_req()
1838 if (hfi1_kern_setup_hw_flow(qpriv->rcd, qp)) in hfi1_build_tid_rdma_read_req()
1845 if (hfi1_kern_exp_rcv_setup(req, &qp->s_sge, &last)) { in hfi1_build_tid_rdma_read_req()
1882 static int tid_rdma_rcv_read_request(struct rvt_qp *qp, in tid_rdma_rcv_read_request() argument
1888 struct hfi1_qp_priv *qpriv = qp->priv; in tid_rdma_rcv_read_request()
1909 flow->npkts = rvt_div_round_up_mtu(qp, len); in tid_rdma_rcv_read_request()
1911 trace_hfi1_tid_entry_rcv_read_req(qp, i, in tid_rdma_rcv_read_request()
1947 trace_hfi1_tid_flow_rcv_read_req(qp, req->setup_head, flow); in tid_rdma_rcv_read_request()
1973 trace_hfi1_tid_req_rcv_read_req(qp, 0, e->opcode, e->psn, e->lpsn, in tid_rdma_rcv_read_request()
1980 struct rvt_qp *qp, u32 psn, int diff) in tid_rdma_rcv_error() argument
1982 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in tid_rdma_rcv_error()
1983 struct hfi1_ctxtdata *rcd = ((struct hfi1_qp_priv *)qp->priv)->rcd; in tid_rdma_rcv_error()
1984 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); in tid_rdma_rcv_error()
1985 struct hfi1_qp_priv *qpriv = qp->priv; in tid_rdma_rcv_error()
1992 trace_hfi1_rsp_tid_rcv_error(qp, psn); in tid_rdma_rcv_error()
1993 trace_hfi1_tid_rdma_rcv_err(qp, 0, psn, diff); in tid_rdma_rcv_error()
1996 if (!qp->r_nak_state) { in tid_rdma_rcv_error()
1998 qp->r_nak_state = IB_NAK_PSN_ERROR; in tid_rdma_rcv_error()
1999 qp->r_ack_psn = qp->r_psn; in tid_rdma_rcv_error()
2000 rc_defered_ack(rcd, qp); in tid_rdma_rcv_error()
2007 spin_lock_irqsave(&qp->s_lock, flags); in tid_rdma_rcv_error()
2008 e = find_prev_entry(qp, psn, &prev, NULL, &old_req); in tid_rdma_rcv_error()
2015 trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, e->lpsn, req); in tid_rdma_rcv_error()
2038 qp->r_len = len; in tid_rdma_rcv_error()
2039 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, in tid_rdma_rcv_error()
2055 if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn, in tid_rdma_rcv_error()
2089 if (i == qp->r_head_ack_queue) in tid_rdma_rcv_error()
2091 e = &qp->s_ack_queue[i]; in tid_rdma_rcv_error()
2151 if (i == qp->r_head_ack_queue) in tid_rdma_rcv_error()
2153 e = &qp->s_ack_queue[i]; in tid_rdma_rcv_error()
2155 trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, in tid_rdma_rcv_error()
2176 if (qp->s_acked_ack_queue == qp->s_tail_ack_queue) in tid_rdma_rcv_error()
2177 qp->s_acked_ack_queue = prev; in tid_rdma_rcv_error()
2178 qp->s_tail_ack_queue = prev; in tid_rdma_rcv_error()
2185 qp->s_ack_state = OP(ACKNOWLEDGE); in tid_rdma_rcv_error()
2192 qp->s_nak_state = 0; in tid_rdma_rcv_error()
2194 qp->r_psn = e->lpsn + 1; in tid_rdma_rcv_error()
2195 hfi1_tid_write_alloc_resources(qp, true); in tid_rdma_rcv_error()
2198 qp->r_state = e->opcode; in tid_rdma_rcv_error()
2199 qp->r_nak_state = 0; in tid_rdma_rcv_error()
2200 qp->s_flags |= RVT_S_RESP_PENDING; in tid_rdma_rcv_error()
2201 hfi1_schedule_send(qp); in tid_rdma_rcv_error()
2203 spin_unlock_irqrestore(&qp->s_lock, flags); in tid_rdma_rcv_error()
2224 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv_tid_rdma_read_req() local
2225 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in hfi1_rc_rcv_tid_rdma_read_req()
2230 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_rc_rcv_tid_rdma_read_req()
2242 fecn = process_ecn(qp, packet); in hfi1_rc_rcv_tid_rdma_read_req()
2244 trace_hfi1_rsp_rcv_tid_read_req(qp, psn); in hfi1_rc_rcv_tid_rdma_read_req()
2246 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) in hfi1_rc_rcv_tid_rdma_read_req()
2247 rvt_comm_est(qp); in hfi1_rc_rcv_tid_rdma_read_req()
2249 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) in hfi1_rc_rcv_tid_rdma_read_req()
2259 diff = delta_psn(psn, qp->r_psn); in hfi1_rc_rcv_tid_rdma_read_req()
2261 tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn); in hfi1_rc_rcv_tid_rdma_read_req()
2266 next = qp->r_head_ack_queue + 1; in hfi1_rc_rcv_tid_rdma_read_req()
2267 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_rc_rcv_tid_rdma_read_req()
2269 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_read_req()
2270 if (unlikely(next == qp->s_tail_ack_queue)) { in hfi1_rc_rcv_tid_rdma_read_req()
2271 if (!qp->s_ack_queue[next].sent) { in hfi1_rc_rcv_tid_rdma_read_req()
2275 update_ack_queue(qp, next); in hfi1_rc_rcv_tid_rdma_read_req()
2277 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in hfi1_rc_rcv_tid_rdma_read_req()
2281 qp->r_len = len; in hfi1_rc_rcv_tid_rdma_read_req()
2283 if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr, in hfi1_rc_rcv_tid_rdma_read_req()
2288 if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn, vaddr, in hfi1_rc_rcv_tid_rdma_read_req()
2292 qp->r_state = e->opcode; in hfi1_rc_rcv_tid_rdma_read_req()
2293 qp->r_nak_state = 0; in hfi1_rc_rcv_tid_rdma_read_req()
2299 qp->r_msn++; in hfi1_rc_rcv_tid_rdma_read_req()
2300 qp->r_psn += e->lpsn - e->psn + 1; in hfi1_rc_rcv_tid_rdma_read_req()
2302 qp->r_head_ack_queue = next; in hfi1_rc_rcv_tid_rdma_read_req()
2310 qpriv->r_tid_alloc = qp->r_head_ack_queue; in hfi1_rc_rcv_tid_rdma_read_req()
2313 qp->s_flags |= RVT_S_RESP_PENDING; in hfi1_rc_rcv_tid_rdma_read_req()
2315 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv_tid_rdma_read_req()
2316 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_read_req()
2318 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_read_req()
2322 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_read_req()
2324 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in hfi1_rc_rcv_tid_rdma_read_req()
2325 qp->r_nak_state = nack_state; in hfi1_rc_rcv_tid_rdma_read_req()
2326 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv_tid_rdma_read_req()
2328 rc_defered_ack(rcd, qp); in hfi1_rc_rcv_tid_rdma_read_req()
2331 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_read_req()
2332 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); in hfi1_rc_rcv_tid_rdma_read_req()
2333 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in hfi1_rc_rcv_tid_rdma_read_req()
2334 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv_tid_rdma_read_req()
2337 u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, in hfi1_build_tid_rdma_read_resp() argument
2343 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_read_resp()
2353 *len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset); in hfi1_build_tid_rdma_read_resp()
2358 trace_hfi1_tid_entry_build_read_resp(qp, flow->tid_idx, tidentry); in hfi1_build_tid_rdma_read_resp()
2359 trace_hfi1_tid_flow_build_read_resp(qp, req->clear_tail, flow); in hfi1_build_tid_rdma_read_resp()
2375 resp->verbs_qp = cpu_to_be32(qp->remote_qpn); in hfi1_build_tid_rdma_read_resp()
2378 resp->aeth = rvt_compute_aeth(qp); in hfi1_build_tid_rdma_read_resp()
2408 find_tid_request(struct rvt_qp *qp, u32 psn, enum ib_wr_opcode opcode) in find_tid_request() argument
2409 __must_hold(&qp->s_lock) in find_tid_request()
2415 end = qp->s_cur + 1; in find_tid_request()
2416 if (end == qp->s_size) in find_tid_request()
2418 for (i = qp->s_acked; i != end;) { in find_tid_request()
2419 wqe = rvt_get_swqe_ptr(qp, i); in find_tid_request()
2426 if (++i == qp->s_size) in find_tid_request()
2445 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv_tid_rdma_read_resp() local
2446 struct hfi1_qp_priv *priv = qp->priv; in hfi1_rc_rcv_tid_rdma_read_resp()
2455 trace_hfi1_sender_rcv_tid_read_resp(qp); in hfi1_rc_rcv_tid_rdma_read_resp()
2456 fecn = process_ecn(qp, packet); in hfi1_rc_rcv_tid_rdma_read_resp()
2461 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_read_resp()
2463 req = find_tid_request(qp, ipsn, IB_WR_TID_RDMA_READ); in hfi1_rc_rcv_tid_rdma_read_resp()
2490 u32 pmtu = qp->pmtu; in hfi1_rc_rcv_tid_rdma_read_resp()
2497 rvt_copy_sge(qp, &ss, packet->payload, pmtu, false, in hfi1_rc_rcv_tid_rdma_read_resp()
2508 qp->s_num_rd_atomic--; in hfi1_rc_rcv_tid_rdma_read_resp()
2509 if ((qp->s_flags & RVT_S_WAIT_FENCE) && in hfi1_rc_rcv_tid_rdma_read_resp()
2510 !qp->s_num_rd_atomic) { in hfi1_rc_rcv_tid_rdma_read_resp()
2511 qp->s_flags &= ~(RVT_S_WAIT_FENCE | in hfi1_rc_rcv_tid_rdma_read_resp()
2513 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_read_resp()
2515 if (qp->s_flags & RVT_S_WAIT_RDMAR) { in hfi1_rc_rcv_tid_rdma_read_resp()
2516 qp->s_flags &= ~(RVT_S_WAIT_RDMAR | RVT_S_WAIT_ACK); in hfi1_rc_rcv_tid_rdma_read_resp()
2517 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_read_resp()
2520 trace_hfi1_ack(qp, ipsn); in hfi1_rc_rcv_tid_rdma_read_resp()
2521 trace_hfi1_tid_req_rcv_read_resp(qp, 0, req->e.swqe->wr.opcode, in hfi1_rc_rcv_tid_rdma_read_resp()
2524 trace_hfi1_tid_flow_rcv_read_resp(qp, req->clear_tail, flow); in hfi1_rc_rcv_tid_rdma_read_resp()
2529 if (!do_rc_ack(qp, aeth, ipsn, opcode, 0, rcd)) in hfi1_rc_rcv_tid_rdma_read_resp()
2546 hfi1_kern_clear_hw_flow(priv->rcd, qp); in hfi1_rc_rcv_tid_rdma_read_resp()
2552 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_read_resp()
2564 if (qp->s_last == qp->s_acked) in hfi1_rc_rcv_tid_rdma_read_resp()
2565 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in hfi1_rc_rcv_tid_rdma_read_resp()
2568 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_read_resp()
2571 void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp) in hfi1_kern_read_tid_flow_free() argument
2572 __must_hold(&qp->s_lock) in hfi1_kern_read_tid_flow_free()
2574 u32 n = qp->s_acked; in hfi1_kern_read_tid_flow_free()
2577 struct hfi1_qp_priv *priv = qp->priv; in hfi1_kern_read_tid_flow_free()
2579 lockdep_assert_held(&qp->s_lock); in hfi1_kern_read_tid_flow_free()
2581 while (n != qp->s_tail) { in hfi1_kern_read_tid_flow_free()
2582 wqe = rvt_get_swqe_ptr(qp, n); in hfi1_kern_read_tid_flow_free()
2588 if (++n == qp->s_size) in hfi1_kern_read_tid_flow_free()
2592 hfi1_kern_clear_hw_flow(priv->rcd, qp); in hfi1_kern_read_tid_flow_free()
2597 struct rvt_qp *qp = packet->qp; in tid_rdma_tid_err() local
2602 spin_lock(&qp->s_lock); in tid_rdma_tid_err()
2612 hfi1_restart_rc(qp, qp->s_last_psn + 1, 1); in tid_rdma_tid_err()
2613 hfi1_schedule_send(qp); in tid_rdma_tid_err()
2617 spin_unlock(&qp->s_lock); in tid_rdma_tid_err()
2623 struct rvt_qp *qp, struct rvt_swqe *wqe) in restart_tid_rdma_read_req() argument
2629 qp->r_flags |= RVT_R_RDMAR_SEQ; in restart_tid_rdma_read_req()
2632 hfi1_restart_rc(qp, flow->flow_state.ib_spsn, 0); in restart_tid_rdma_read_req()
2633 if (list_empty(&qp->rspwait)) { in restart_tid_rdma_read_req()
2634 qp->r_flags |= RVT_R_RSP_SEND; in restart_tid_rdma_read_req()
2635 rvt_get_qp(qp); in restart_tid_rdma_read_req()
2636 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in restart_tid_rdma_read_req()
2651 __must_hold(&packet->qp->r_lock) __must_hold(RCU) in handle_read_kdeth_eflags()
2660 struct rvt_qp *qp = packet->qp; in handle_read_kdeth_eflags() local
2661 struct hfi1_qp_priv *priv = qp->priv; in handle_read_kdeth_eflags()
2666 lockdep_assert_held(&qp->r_lock); in handle_read_kdeth_eflags()
2667 trace_hfi1_rsp_read_kdeth_eflags(qp, ibpsn); in handle_read_kdeth_eflags()
2668 trace_hfi1_sender_read_kdeth_eflags(qp); in handle_read_kdeth_eflags()
2669 trace_hfi1_tid_read_sender_kdeth_eflags(qp, 0); in handle_read_kdeth_eflags()
2670 spin_lock(&qp->s_lock); in handle_read_kdeth_eflags()
2672 if (cmp_psn(ibpsn, qp->s_last_psn) < 0 || in handle_read_kdeth_eflags()
2673 cmp_psn(ibpsn, qp->s_psn) > 0) in handle_read_kdeth_eflags()
2682 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in handle_read_kdeth_eflags()
2683 ibp = to_iport(qp->ibqp.device, qp->port_num); in handle_read_kdeth_eflags()
2697 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { in handle_read_kdeth_eflags()
2698 qp->r_flags |= RVT_R_RDMAR_SEQ; in handle_read_kdeth_eflags()
2700 restart_tid_rdma_read_req(rcd, qp, in handle_read_kdeth_eflags()
2703 hfi1_restart_rc(qp, qp->s_last_psn + 1, in handle_read_kdeth_eflags()
2705 if (list_empty(&qp->rspwait)) { in handle_read_kdeth_eflags()
2706 qp->r_flags |= RVT_R_RSP_SEND; in handle_read_kdeth_eflags()
2707 rvt_get_qp(qp); in handle_read_kdeth_eflags()
2709 &qp->rspwait, in handle_read_kdeth_eflags()
2721 wqe = do_rc_completion(qp, wqe, ibp); in handle_read_kdeth_eflags()
2722 if (qp->s_acked == qp->s_tail) in handle_read_kdeth_eflags()
2726 if (qp->s_acked == qp->s_tail) in handle_read_kdeth_eflags()
2734 trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe->wr.opcode, wqe->psn, in handle_read_kdeth_eflags()
2750 trace_hfi1_tid_flow_read_kdeth_eflags(qp, in handle_read_kdeth_eflags()
2765 if (qp->r_flags & RVT_R_RDMAR_SEQ) in handle_read_kdeth_eflags()
2766 qp->r_flags &= in handle_read_kdeth_eflags()
2782 if (qp->r_flags & RVT_R_RDMAR_SEQ) in handle_read_kdeth_eflags()
2783 qp->r_flags &= in handle_read_kdeth_eflags()
2799 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) in handle_read_kdeth_eflags()
2800 restart_tid_rdma_read_req(rcd, qp, in handle_read_kdeth_eflags()
2833 spin_unlock(&qp->s_lock); in handle_read_kdeth_eflags()
2852 struct rvt_qp *qp; in hfi1_handle_kdeth_eflags() local
2880 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); in hfi1_handle_kdeth_eflags()
2881 if (!qp) in hfi1_handle_kdeth_eflags()
2884 packet->qp = qp; in hfi1_handle_kdeth_eflags()
2887 spin_lock_irqsave(&qp->r_lock, flags); in hfi1_handle_kdeth_eflags()
2888 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { in hfi1_handle_kdeth_eflags()
2926 spin_lock(&qp->s_lock); in hfi1_handle_kdeth_eflags()
2927 qpriv = qp->priv; in hfi1_handle_kdeth_eflags()
2931 e = &qp->s_ack_queue[qpriv->r_tid_tail]; in hfi1_handle_kdeth_eflags()
2938 trace_hfi1_eflags_err_write(qp, rcv_type, rte, psn); in hfi1_handle_kdeth_eflags()
2939 trace_hfi1_rsp_handle_kdeth_eflags(qp, psn); in hfi1_handle_kdeth_eflags()
2940 trace_hfi1_tid_write_rsp_handle_kdeth_eflags(qp); in hfi1_handle_kdeth_eflags()
2941 trace_hfi1_tid_req_handle_kdeth_eflags(qp, 0, e->opcode, e->psn, in hfi1_handle_kdeth_eflags()
2943 trace_hfi1_tid_flow_handle_kdeth_eflags(qp, req->clear_tail, flow); in hfi1_handle_kdeth_eflags()
3013 spin_unlock(&qp->s_lock); in hfi1_handle_kdeth_eflags()
3015 spin_unlock_irqrestore(&qp->r_lock, flags); in hfi1_handle_kdeth_eflags()
3026 tid_rdma_trigger_ack(qp); in hfi1_handle_kdeth_eflags()
3037 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_tid_rdma_restart_req() argument
3042 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_tid_rdma_restart_req()
3048 *bth2 = mask_psn(qp->s_psn); in hfi1_tid_rdma_restart_req()
3052 qp, "!!!!!! Could not find flow to restart: bth2 ", in hfi1_tid_rdma_restart_req()
3054 trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, in hfi1_tid_rdma_restart_req()
3072 trace_hfi1_tid_flow_restart_req(qp, fidx, flow); in hfi1_tid_rdma_restart_req()
3086 tidnpkts = rvt_div_round_up_mtu(qp, tidlen); in hfi1_tid_rdma_restart_req()
3090 npkts * qp->pmtu); in hfi1_tid_rdma_restart_req()
3091 flow->tid_offset += npkts * qp->pmtu; in hfi1_tid_rdma_restart_req()
3122 trace_hfi1_tid_flow_restart_req(qp, fidx, flow); in hfi1_tid_rdma_restart_req()
3123 trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_tid_rdma_restart_req()
3142 i = (++i == qp->s_size ? 0 : i); in hfi1_tid_rdma_restart_req()
3143 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_tid_rdma_restart_req()
3154 void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp) in hfi1_qp_kern_exp_rcv_clear_all() argument
3157 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_qp_kern_exp_rcv_clear_all()
3160 if (qp->ibqp.qp_type != IB_QPT_RC || !HFI1_CAP_IS_KSET(TID_RDMA)) in hfi1_qp_kern_exp_rcv_clear_all()
3169 hfi1_kern_clear_hw_flow(qpriv->rcd, qp); in hfi1_qp_kern_exp_rcv_clear_all()
3171 for (i = qp->s_acked; i != qp->s_head;) { in hfi1_qp_kern_exp_rcv_clear_all()
3172 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_kern_exp_rcv_clear_all()
3174 if (++i == qp->s_size) in hfi1_qp_kern_exp_rcv_clear_all()
3185 for (i = qp->s_acked_ack_queue; i != qp->r_head_ack_queue;) { in hfi1_qp_kern_exp_rcv_clear_all()
3186 struct rvt_ack_entry *e = &qp->s_ack_queue[i]; in hfi1_qp_kern_exp_rcv_clear_all()
3188 if (++i == rvt_max_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_qp_kern_exp_rcv_clear_all()
3201 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) in hfi1_tid_rdma_wqe_interlock() argument
3204 struct hfi1_qp_priv *priv = qp->priv; in hfi1_tid_rdma_wqe_interlock()
3208 s_prev = (qp->s_cur == 0 ? qp->s_size : qp->s_cur) - 1; in hfi1_tid_rdma_wqe_interlock()
3209 prev = rvt_get_swqe_ptr(qp, s_prev); in hfi1_tid_rdma_wqe_interlock()
3235 if (qp->s_acked != qp->s_cur) in hfi1_tid_rdma_wqe_interlock()
3256 static inline bool hfi1_check_sge_align(struct rvt_qp *qp, in hfi1_check_sge_align() argument
3262 trace_hfi1_sge_check_align(qp, i, sge); in hfi1_check_sge_align()
3270 void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe) in setup_tid_rdma_wqe() argument
3272 struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; in setup_tid_rdma_wqe()
3279 if ((rdma_ah_get_dlid(&qp->remote_ah_attr) & ~((1 << ppd->lmc) - 1)) == in setup_tid_rdma_wqe()
3295 if (hfi1_check_sge_align(qp, &wqe->sg_list[0], in setup_tid_rdma_wqe()
3327 wqe->lpsn += rvt_div_round_up_mtu(qp, wqe->length) - 1; in setup_tid_rdma_wqe()
3344 trace_hfi1_tid_req_setup_tid_wqe(qp, 1, wqe->wr.opcode, in setup_tid_rdma_wqe()
3354 u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_write_req() argument
3358 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_write_req()
3378 ohdr->u.tid_rdma.w_req.verbs_qp = cpu_to_be32(qp->remote_qpn); in hfi1_build_tid_rdma_write_req()
3380 *bth1 |= remote->qp; in hfi1_build_tid_rdma_write_req()
3381 qp->s_state = TID_OP(WRITE_REQ); in hfi1_build_tid_rdma_write_req()
3382 qp->s_flags |= HFI1_S_WAIT_TID_RESP; in hfi1_build_tid_rdma_write_req()
3390 static u32 hfi1_compute_tid_rdma_flow_wt(struct rvt_qp *qp) in hfi1_compute_tid_rdma_flow_wt() argument
3400 return (MAX_TID_FLOW_PSN * qp->pmtu) >> TID_RDMA_SEGMENT_SHIFT; in hfi1_compute_tid_rdma_flow_wt()
3414 static u32 hfi1_compute_tid_rnr_timeout(struct rvt_qp *qp, u32 to_seg) in hfi1_compute_tid_rnr_timeout() argument
3416 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_compute_tid_rnr_timeout()
3452 static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx) in hfi1_tid_write_alloc_resources() argument
3455 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_tid_write_alloc_resources()
3463 lockdep_assert_held(&qp->s_lock); in hfi1_tid_write_alloc_resources()
3466 trace_hfi1_rsp_tid_write_alloc_res(qp, 0); in hfi1_tid_write_alloc_resources()
3467 trace_hfi1_tid_write_rsp_alloc_res(qp); in hfi1_tid_write_alloc_resources()
3488 hfi1_kern_clear_hw_flow(rcd, qp); in hfi1_tid_write_alloc_resources()
3494 e = &qp->s_ack_queue[qpriv->r_tid_alloc]; in hfi1_tid_write_alloc_resources()
3498 trace_hfi1_tid_req_write_alloc_res(qp, 0, e->opcode, e->psn, in hfi1_tid_write_alloc_resources()
3514 hfi1_kern_clear_hw_flow(rcd, qp); in hfi1_tid_write_alloc_resources()
3521 ret = hfi1_kern_setup_hw_flow(qpriv->rcd, qp); in hfi1_tid_write_alloc_resources()
3523 to_seg = hfi1_compute_tid_rdma_flow_wt(qp) * in hfi1_tid_write_alloc_resources()
3530 npkts = rvt_div_round_up_mtu(qp, req->seg_len); in hfi1_tid_write_alloc_resources()
3552 tid_rdma_trigger_ack(qp); in hfi1_tid_write_alloc_resources()
3569 rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_tid_write_alloc_resources()
3578 if (ret == -EAGAIN && intr_ctx && !qp->r_nak_state) in hfi1_tid_write_alloc_resources()
3584 lockdep_assert_held(&qp->r_lock); in hfi1_tid_write_alloc_resources()
3587 qp->r_nak_state = hfi1_compute_tid_rnr_timeout(qp, to_seg) | IB_RNR_NAK; in hfi1_tid_write_alloc_resources()
3590 qp->r_psn = e->psn + req->alloc_seg; in hfi1_tid_write_alloc_resources()
3591 qp->r_ack_psn = qp->r_psn; in hfi1_tid_write_alloc_resources()
3597 qp->r_head_ack_queue = qpriv->r_tid_alloc + 1; in hfi1_tid_write_alloc_resources()
3598 if (qp->r_head_ack_queue > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_tid_write_alloc_resources()
3599 qp->r_head_ack_queue = 0; in hfi1_tid_write_alloc_resources()
3600 qpriv->r_tid_head = qp->r_head_ack_queue; in hfi1_tid_write_alloc_resources()
3606 qp->s_nak_state = qp->r_nak_state; in hfi1_tid_write_alloc_resources()
3607 qp->s_ack_psn = qp->r_ack_psn; in hfi1_tid_write_alloc_resources()
3612 qp->s_flags &= ~(RVT_S_ACK_PENDING); in hfi1_tid_write_alloc_resources()
3614 trace_hfi1_rsp_tid_write_alloc_res(qp, qp->r_psn); in hfi1_tid_write_alloc_resources()
3629 rc_defered_ack(rcd, qp); in hfi1_tid_write_alloc_resources()
3648 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv_tid_rdma_write_req() local
3649 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in hfi1_rc_rcv_tid_rdma_write_req()
3654 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_rc_rcv_tid_rdma_write_req()
3666 fecn = process_ecn(qp, packet); in hfi1_rc_rcv_tid_rdma_write_req()
3668 trace_hfi1_rsp_rcv_tid_write_req(qp, psn); in hfi1_rc_rcv_tid_rdma_write_req()
3670 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) in hfi1_rc_rcv_tid_rdma_write_req()
3671 rvt_comm_est(qp); in hfi1_rc_rcv_tid_rdma_write_req()
3673 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) in hfi1_rc_rcv_tid_rdma_write_req()
3681 diff = delta_psn(psn, qp->r_psn); in hfi1_rc_rcv_tid_rdma_write_req()
3683 tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn); in hfi1_rc_rcv_tid_rdma_write_req()
3693 qp->r_head_ack_queue = qp->r_head_ack_queue ? in hfi1_rc_rcv_tid_rdma_write_req()
3694 qp->r_head_ack_queue - 1 : in hfi1_rc_rcv_tid_rdma_write_req()
3695 rvt_size_atomic(ib_to_rvt(qp->ibqp.device)); in hfi1_rc_rcv_tid_rdma_write_req()
3698 next = qp->r_head_ack_queue + 1; in hfi1_rc_rcv_tid_rdma_write_req()
3699 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_rc_rcv_tid_rdma_write_req()
3701 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_req()
3702 if (unlikely(next == qp->s_acked_ack_queue)) { in hfi1_rc_rcv_tid_rdma_write_req()
3703 if (!qp->s_ack_queue[next].sent) in hfi1_rc_rcv_tid_rdma_write_req()
3705 update_ack_queue(qp, next); in hfi1_rc_rcv_tid_rdma_write_req()
3707 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in hfi1_rc_rcv_tid_rdma_write_req()
3712 qp->r_nak_state = 0; in hfi1_rc_rcv_tid_rdma_write_req()
3713 qp->s_nak_state = 0; in hfi1_rc_rcv_tid_rdma_write_req()
3715 qp->r_psn = e->lpsn + 1; in hfi1_rc_rcv_tid_rdma_write_req()
3727 qp->r_len = len; in hfi1_rc_rcv_tid_rdma_write_req()
3734 if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr, in hfi1_rc_rcv_tid_rdma_write_req()
3738 qp->r_psn += num_segs - 1; in hfi1_rc_rcv_tid_rdma_write_req()
3742 e->lpsn = qp->r_psn; in hfi1_rc_rcv_tid_rdma_write_req()
3763 qp->r_state = e->opcode; in hfi1_rc_rcv_tid_rdma_write_req()
3764 qp->r_nak_state = 0; in hfi1_rc_rcv_tid_rdma_write_req()
3770 qp->r_msn++; in hfi1_rc_rcv_tid_rdma_write_req()
3771 qp->r_psn++; in hfi1_rc_rcv_tid_rdma_write_req()
3773 trace_hfi1_tid_req_rcv_write_req(qp, 0, e->opcode, e->psn, e->lpsn, in hfi1_rc_rcv_tid_rdma_write_req()
3777 qpriv->r_tid_tail = qp->r_head_ack_queue; in hfi1_rc_rcv_tid_rdma_write_req()
3781 e = &qp->s_ack_queue[qpriv->r_tid_tail]; in hfi1_rc_rcv_tid_rdma_write_req()
3787 qpriv->r_tid_ack = qp->r_head_ack_queue; in hfi1_rc_rcv_tid_rdma_write_req()
3788 qpriv->r_tid_tail = qp->r_head_ack_queue; in hfi1_rc_rcv_tid_rdma_write_req()
3792 qp->r_head_ack_queue = next; in hfi1_rc_rcv_tid_rdma_write_req()
3793 qpriv->r_tid_head = qp->r_head_ack_queue; in hfi1_rc_rcv_tid_rdma_write_req()
3795 hfi1_tid_write_alloc_resources(qp, true); in hfi1_rc_rcv_tid_rdma_write_req()
3796 trace_hfi1_tid_write_rsp_rcv_req(qp); in hfi1_rc_rcv_tid_rdma_write_req()
3799 qp->s_flags |= RVT_S_RESP_PENDING; in hfi1_rc_rcv_tid_rdma_write_req()
3801 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv_tid_rdma_write_req()
3802 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_write_req()
3804 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_req()
3808 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_req()
3810 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in hfi1_rc_rcv_tid_rdma_write_req()
3811 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in hfi1_rc_rcv_tid_rdma_write_req()
3812 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv_tid_rdma_write_req()
3814 rc_defered_ack(rcd, qp); in hfi1_rc_rcv_tid_rdma_write_req()
3817 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_req()
3818 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); in hfi1_rc_rcv_tid_rdma_write_req()
3819 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in hfi1_rc_rcv_tid_rdma_write_req()
3820 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv_tid_rdma_write_req()
3823 u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, in hfi1_build_tid_rdma_write_resp() argument
3830 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_write_resp()
3836 trace_hfi1_tid_req_build_write_resp(qp, 0, e->opcode, e->psn, e->lpsn, in hfi1_build_tid_rdma_write_resp()
3838 trace_hfi1_tid_write_rsp_build_resp(qp); in hfi1_build_tid_rdma_write_resp()
3839 trace_hfi1_rsp_build_tid_write_resp(qp, bth2); in hfi1_build_tid_rdma_write_resp()
3847 hfi1_tid_write_alloc_resources(qp, false); in hfi1_build_tid_rdma_write_resp()
3861 trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow); in hfi1_build_tid_rdma_write_resp()
3863 hfi1_add_tid_reap_timer(qp); in hfi1_build_tid_rdma_write_resp()
3868 trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow); in hfi1_build_tid_rdma_write_resp()
3873 hfi1_mod_tid_reap_timer(qp); in hfi1_build_tid_rdma_write_resp()
3906 ohdr->u.tid_rdma.w_rsp.aeth = rvt_compute_aeth(qp); in hfi1_build_tid_rdma_write_resp()
3913 cpu_to_be32(qpriv->tid_rdma.local.qp | in hfi1_build_tid_rdma_write_resp()
3917 ohdr->u.tid_rdma.w_rsp.verbs_qp = cpu_to_be32(qp->remote_qpn); in hfi1_build_tid_rdma_write_resp()
3918 *bth1 = remote->qp; in hfi1_build_tid_rdma_write_resp()
3926 static void hfi1_add_tid_reap_timer(struct rvt_qp *qp) in hfi1_add_tid_reap_timer() argument
3928 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_add_tid_reap_timer()
3930 lockdep_assert_held(&qp->s_lock); in hfi1_add_tid_reap_timer()
3939 static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp) in hfi1_mod_tid_reap_timer() argument
3941 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_mod_tid_reap_timer()
3943 lockdep_assert_held(&qp->s_lock); in hfi1_mod_tid_reap_timer()
3949 static int hfi1_stop_tid_reap_timer(struct rvt_qp *qp) in hfi1_stop_tid_reap_timer() argument
3951 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_stop_tid_reap_timer()
3954 lockdep_assert_held(&qp->s_lock); in hfi1_stop_tid_reap_timer()
3962 void hfi1_del_tid_reap_timer(struct rvt_qp *qp) in hfi1_del_tid_reap_timer() argument
3964 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_del_tid_reap_timer()
3973 struct rvt_qp *qp = qpriv->owner; in hfi1_tid_timeout() local
3974 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in hfi1_tid_timeout()
3978 spin_lock_irqsave(&qp->r_lock, flags); in hfi1_tid_timeout()
3979 spin_lock(&qp->s_lock); in hfi1_tid_timeout()
3981 dd_dev_warn(dd_from_ibdev(qp->ibqp.device), "[QP%u] %s %d\n", in hfi1_tid_timeout()
3982 qp->ibqp.qp_num, __func__, __LINE__); in hfi1_tid_timeout()
3984 qp, "resource timeout = ", in hfi1_tid_timeout()
3986 hfi1_stop_tid_reap_timer(qp); in hfi1_tid_timeout()
3991 hfi1_kern_clear_hw_flow(qpriv->rcd, qp); in hfi1_tid_timeout()
3994 ack_to_tid_req(&qp->s_ack_queue[i]); in hfi1_tid_timeout()
3998 spin_unlock(&qp->s_lock); in hfi1_tid_timeout()
3999 if (qp->ibqp.event_handler) { in hfi1_tid_timeout()
4002 ev.device = qp->ibqp.device; in hfi1_tid_timeout()
4003 ev.element.qp = &qp->ibqp; in hfi1_tid_timeout()
4005 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); in hfi1_tid_timeout()
4007 rvt_rc_error(qp, IB_WC_RESP_TIMEOUT_ERR); in hfi1_tid_timeout()
4010 spin_unlock(&qp->s_lock); in hfi1_tid_timeout()
4012 spin_unlock_irqrestore(&qp->r_lock, flags); in hfi1_tid_timeout()
4029 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv_tid_rdma_write_resp() local
4030 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_rc_rcv_tid_rdma_write_resp()
4040 fecn = process_ecn(qp, packet); in hfi1_rc_rcv_tid_rdma_write_resp()
4045 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_resp()
4048 if (cmp_psn(psn, qp->s_next_psn) >= 0) in hfi1_rc_rcv_tid_rdma_write_resp()
4052 if (unlikely(cmp_psn(psn, qp->s_last_psn) <= 0)) in hfi1_rc_rcv_tid_rdma_write_resp()
4055 if (unlikely(qp->s_acked == qp->s_tail)) in hfi1_rc_rcv_tid_rdma_write_resp()
4063 if (qp->r_flags & RVT_R_RDMAR_SEQ) { in hfi1_rc_rcv_tid_rdma_write_resp()
4064 if (cmp_psn(psn, qp->s_last_psn + 1) != 0) in hfi1_rc_rcv_tid_rdma_write_resp()
4066 qp->r_flags &= ~RVT_R_RDMAR_SEQ; in hfi1_rc_rcv_tid_rdma_write_resp()
4069 wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); in hfi1_rc_rcv_tid_rdma_write_resp()
4089 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) in hfi1_rc_rcv_tid_rdma_write_resp()
4092 trace_hfi1_ack(qp, psn); in hfi1_rc_rcv_tid_rdma_write_resp()
4110 flow->npkts = rvt_div_round_up_mtu(qp, flow->length); in hfi1_rc_rcv_tid_rdma_write_resp()
4121 trace_hfi1_tid_flow_rcv_write_resp(qp, req->setup_head, flow); in hfi1_rc_rcv_tid_rdma_write_resp()
4124 trace_hfi1_tid_write_sender_rcv_resp(qp, 0); in hfi1_rc_rcv_tid_rdma_write_resp()
4131 qp, i, flow->tid_entry[i]); in hfi1_rc_rcv_tid_rdma_write_resp()
4143 trace_hfi1_tid_req_rcv_write_resp(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_write_resp()
4169 if (i == qp->s_size) in hfi1_rc_rcv_tid_rdma_write_resp()
4171 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_rc_rcv_tid_rdma_write_resp()
4179 qp->s_flags &= ~HFI1_S_WAIT_TID_RESP; in hfi1_rc_rcv_tid_rdma_write_resp()
4180 hfi1_schedule_tid_send(qp); in hfi1_rc_rcv_tid_rdma_write_resp()
4186 rvt_error_qp(qp, status); in hfi1_rc_rcv_tid_rdma_write_resp()
4189 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv_tid_rdma_write_resp()
4190 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_resp()
4200 struct rvt_qp *qp = req->qp; in hfi1_build_tid_rdma_packet() local
4201 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_packet()
4209 hfi1_trdma_send_complete(qp, wqe, IB_WC_REM_INV_RD_REQ_ERR); in hfi1_build_tid_rdma_packet()
4210 rvt_error_qp(qp, IB_WC_REM_INV_RD_REQ_ERR); in hfi1_build_tid_rdma_packet()
4213 *len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset); in hfi1_build_tid_rdma_packet()
4218 trace_hfi1_tid_entry_build_write_data(qp, flow->tid_idx, tidentry); in hfi1_build_tid_rdma_packet()
4219 trace_hfi1_tid_flow_build_write_data(qp, req->clear_tail, flow); in hfi1_build_tid_rdma_packet()
4231 wd->verbs_qp = cpu_to_be32(qp->remote_qpn); in hfi1_build_tid_rdma_packet()
4242 rvt_div_round_up_mtu(qp, req->seg_len) > in hfi1_build_tid_rdma_packet()
4259 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv_tid_rdma_write_data() local
4260 struct hfi1_qp_priv *priv = qp->priv; in hfi1_rc_rcv_tid_rdma_write_data()
4266 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); in hfi1_rc_rcv_tid_rdma_write_data()
4272 fecn = process_ecn(qp, packet); in hfi1_rc_rcv_tid_rdma_write_data()
4280 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_data()
4281 e = &qp->s_ack_queue[priv->r_tid_tail]; in hfi1_rc_rcv_tid_rdma_write_data()
4306 u32 pmtu = qp->pmtu; in hfi1_rc_rcv_tid_rdma_write_data()
4326 rvt_copy_sge(qp, &ss, packet->payload, pmtu, false, in hfi1_rc_rcv_tid_rdma_write_data()
4348 trace_hfi1_rsp_rcv_tid_write_data(qp, psn); in hfi1_rc_rcv_tid_rdma_write_data()
4349 trace_hfi1_tid_req_rcv_write_data(qp, 0, e->opcode, e->psn, e->lpsn, in hfi1_rc_rcv_tid_rdma_write_data()
4351 trace_hfi1_tid_write_rsp_rcv_data(qp); in hfi1_rc_rcv_tid_rdma_write_data()
4361 e = &qp->s_ack_queue[next]; in hfi1_rc_rcv_tid_rdma_write_data()
4366 if (++qp->s_acked_ack_queue > rvt_size_atomic(&dev->rdi)) in hfi1_rc_rcv_tid_rdma_write_data()
4367 qp->s_acked_ack_queue = 0; in hfi1_rc_rcv_tid_rdma_write_data()
4370 hfi1_tid_write_alloc_resources(qp, true); in hfi1_rc_rcv_tid_rdma_write_data()
4377 qp->s_tail_ack_queue != qp->r_head_ack_queue) { in hfi1_rc_rcv_tid_rdma_write_data()
4378 qp->s_flags |= RVT_S_RESP_PENDING; in hfi1_rc_rcv_tid_rdma_write_data()
4379 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_write_data()
4385 hfi1_mod_tid_reap_timer(req->qp); in hfi1_rc_rcv_tid_rdma_write_data()
4387 hfi1_stop_tid_reap_timer(req->qp); in hfi1_rc_rcv_tid_rdma_write_data()
4391 tid_rdma_schedule_ack(qp); in hfi1_rc_rcv_tid_rdma_write_data()
4395 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv_tid_rdma_write_data()
4396 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_write_data()
4403 tid_rdma_trigger_ack(qp); in hfi1_rc_rcv_tid_rdma_write_data()
4414 u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e, in hfi1_build_tid_rdma_write_ack() argument
4418 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_write_ack()
4427 ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn); in hfi1_build_tid_rdma_write_ack()
4428 *bth1 = remote->qp; in hfi1_build_tid_rdma_write_ack()
4434 ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp); in hfi1_build_tid_rdma_write_ack()
4438 cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in hfi1_build_tid_rdma_write_ack()
4443 ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp); in hfi1_build_tid_rdma_write_ack()
4447 cpu_to_be32(qpriv->tid_rdma.local.qp | in hfi1_build_tid_rdma_write_ack()
4488 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv_tid_rdma_ack() local
4489 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_rc_rcv_tid_rdma_ack()
4497 trace_hfi1_tid_write_sender_rcv_tid_ack(qp, 0); in hfi1_rc_rcv_tid_rdma_ack()
4498 process_ecn(qp, packet); in hfi1_rc_rcv_tid_rdma_ack()
4504 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_ack()
4505 trace_hfi1_rcv_tid_ack(qp, aeth, psn, req_psn, resync_psn); in hfi1_rc_rcv_tid_rdma_ack()
4508 if ((qp->s_flags & HFI1_S_WAIT_HALT) && in hfi1_rc_rcv_tid_rdma_ack()
4522 if (unlikely(qp->s_acked == qp->s_tail)) in hfi1_rc_rcv_tid_rdma_ack()
4525 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_rcv_tid_rdma_ack()
4531 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4534 trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); in hfi1_rc_rcv_tid_rdma_ack()
4548 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4552 wqe = do_rc_completion(qp, wqe, in hfi1_rc_rcv_tid_rdma_ack()
4553 to_iport(qp->ibqp.device, in hfi1_rc_rcv_tid_rdma_ack()
4554 qp->port_num)); in hfi1_rc_rcv_tid_rdma_ack()
4555 trace_hfi1_sender_rcv_tid_ack(qp); in hfi1_rc_rcv_tid_rdma_ack()
4557 if (qp->s_acked == qp->s_tail) in hfi1_rc_rcv_tid_rdma_ack()
4564 trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); in hfi1_rc_rcv_tid_rdma_ack()
4567 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4577 hfi1_mod_tid_retry_timer(qp); in hfi1_rc_rcv_tid_rdma_ack()
4579 hfi1_stop_tid_retry_timer(qp); in hfi1_rc_rcv_tid_rdma_ack()
4580 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_ack()
4586 hfi1_stop_tid_retry_timer(qp); in hfi1_rc_rcv_tid_rdma_ack()
4588 qp->s_flags &= ~HFI1_S_WAIT_HALT; in hfi1_rc_rcv_tid_rdma_ack()
4596 hfi1_schedule_send(qp); in hfi1_rc_rcv_tid_rdma_ack()
4598 if ((qp->s_acked == qpriv->s_tid_tail && in hfi1_rc_rcv_tid_rdma_ack()
4600 qp->s_acked == qp->s_tail) { in hfi1_rc_rcv_tid_rdma_ack()
4623 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_rcv_tid_rdma_ack()
4652 last_acked = qp->s_acked; in hfi1_rc_rcv_tid_rdma_ack()
4684 trace_hfi1_tid_flow_rcv_tid_ack(qp, in hfi1_rc_rcv_tid_rdma_ack()
4690 if (last_acked == qp->s_size) in hfi1_rc_rcv_tid_rdma_ack()
4692 wqe = rvt_get_swqe_ptr(qp, last_acked); in hfi1_rc_rcv_tid_rdma_ack()
4696 qpriv->s_tid_tail = qp->s_acked; in hfi1_rc_rcv_tid_rdma_ack()
4698 hfi1_schedule_tid_send(qp); in hfi1_rc_rcv_tid_rdma_ack()
4701 qpriv->s_retry = qp->s_retry_cnt; in hfi1_rc_rcv_tid_rdma_ack()
4705 hfi1_stop_tid_retry_timer(qp); in hfi1_rc_rcv_tid_rdma_ack()
4715 trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, in hfi1_rc_rcv_tid_rdma_ack()
4719 qpriv->s_tid_tail = qp->s_acked; in hfi1_rc_rcv_tid_rdma_ack()
4721 qpriv->s_retry = qp->s_retry_cnt; in hfi1_rc_rcv_tid_rdma_ack()
4722 hfi1_schedule_tid_send(qp); in hfi1_rc_rcv_tid_rdma_ack()
4735 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_ack()
4738 void hfi1_add_tid_retry_timer(struct rvt_qp *qp) in hfi1_add_tid_retry_timer() argument
4740 struct hfi1_qp_priv *priv = qp->priv; in hfi1_add_tid_retry_timer()
4741 struct ib_qp *ibqp = &qp->ibqp; in hfi1_add_tid_retry_timer()
4744 lockdep_assert_held(&qp->s_lock); in hfi1_add_tid_retry_timer()
4753 static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp) in hfi1_mod_tid_retry_timer() argument
4755 struct hfi1_qp_priv *priv = qp->priv; in hfi1_mod_tid_retry_timer()
4756 struct ib_qp *ibqp = &qp->ibqp; in hfi1_mod_tid_retry_timer()
4759 lockdep_assert_held(&qp->s_lock); in hfi1_mod_tid_retry_timer()
4765 static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp) in hfi1_stop_tid_retry_timer() argument
4767 struct hfi1_qp_priv *priv = qp->priv; in hfi1_stop_tid_retry_timer()
4770 lockdep_assert_held(&qp->s_lock); in hfi1_stop_tid_retry_timer()
4778 void hfi1_del_tid_retry_timer(struct rvt_qp *qp) in hfi1_del_tid_retry_timer() argument
4780 struct hfi1_qp_priv *priv = qp->priv; in hfi1_del_tid_retry_timer()
4789 struct rvt_qp *qp = priv->owner; in hfi1_tid_retry_timeout() local
4794 spin_lock_irqsave(&qp->r_lock, flags); in hfi1_tid_retry_timeout()
4795 spin_lock(&qp->s_lock); in hfi1_tid_retry_timeout()
4796 trace_hfi1_tid_write_sender_retry_timeout(qp, 0); in hfi1_tid_retry_timeout()
4798 hfi1_stop_tid_retry_timer(qp); in hfi1_tid_retry_timeout()
4801 qp, in hfi1_tid_retry_timeout()
4805 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_tid_retry_timeout()
4806 hfi1_trdma_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in hfi1_tid_retry_timeout()
4807 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in hfi1_tid_retry_timeout()
4809 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_tid_retry_timeout()
4812 qp, 0, wqe->wr.opcode, wqe->psn, wqe->lpsn, req); in hfi1_tid_retry_timeout()
4821 qp->s_flags |= HFI1_S_WAIT_HALT; in hfi1_tid_retry_timeout()
4824 hfi1_schedule_tid_send(qp); in hfi1_tid_retry_timeout()
4827 spin_unlock(&qp->s_lock); in hfi1_tid_retry_timeout()
4828 spin_unlock_irqrestore(&qp->r_lock, flags); in hfi1_tid_retry_timeout()
4831 u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_resync() argument
4835 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_build_tid_rdma_resync()
4844 ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn); in hfi1_build_tid_rdma_resync()
4845 *bth1 = remote->qp; in hfi1_build_tid_rdma_resync()
4860 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv_tid_rdma_resync() local
4861 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_rc_rcv_tid_rdma_resync()
4863 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); in hfi1_rc_rcv_tid_rdma_resync()
4872 fecn = process_ecn(qp, packet); in hfi1_rc_rcv_tid_rdma_resync()
4876 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_resync()
4909 trace_hfi1_tid_write_rsp_rcv_resync(qp); in hfi1_rc_rcv_tid_rdma_resync()
4921 e = &qp->s_ack_queue[idx]; in hfi1_rc_rcv_tid_rdma_resync()
4924 trace_hfi1_tid_req_rcv_resync(qp, 0, e->opcode, e->psn, in hfi1_rc_rcv_tid_rdma_resync()
4948 trace_hfi1_tid_flow_rcv_resync(qp, flow_idx, in hfi1_rc_rcv_tid_rdma_resync()
4952 if (idx == qp->s_tail_ack_queue) in hfi1_rc_rcv_tid_rdma_resync()
4960 tid_rdma_trigger_ack(qp); in hfi1_rc_rcv_tid_rdma_resync()
4963 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv_tid_rdma_resync()
4964 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv_tid_rdma_resync()
4971 static void update_tid_tail(struct rvt_qp *qp) in update_tid_tail() argument
4972 __must_hold(&qp->s_lock) in update_tid_tail()
4974 struct hfi1_qp_priv *priv = qp->priv; in update_tid_tail()
4978 lockdep_assert_held(&qp->s_lock); in update_tid_tail()
4983 if (i == qp->s_size) in update_tid_tail()
4988 wqe = rvt_get_swqe_ptr(qp, i); in update_tid_tail()
4996 int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps) in hfi1_make_tid_rdma_pkt() argument
4997 __must_hold(&qp->s_lock) in hfi1_make_tid_rdma_pkt()
4999 struct hfi1_qp_priv *priv = qp->priv; in hfi1_make_tid_rdma_pkt()
5003 struct rvt_sge_state *ss = &qp->s_sge; in hfi1_make_tid_rdma_pkt()
5004 struct rvt_ack_entry *e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in hfi1_make_tid_rdma_pkt()
5009 lockdep_assert_held(&qp->s_lock); in hfi1_make_tid_rdma_pkt()
5010 trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0); in hfi1_make_tid_rdma_pkt()
5017 !(qp->s_flags & (RVT_S_BUSY | RVT_S_WAIT_ACK | in hfi1_make_tid_rdma_pkt()
5020 !(qp->s_flags & (RVT_S_BUSY | HFI1_S_ANY_WAIT_IO)))) { in hfi1_make_tid_rdma_pkt()
5025 if (ps->s_txreq || hfi1_make_rc_req(qp, ps)) { in hfi1_make_tid_rdma_pkt()
5031 ps->s_txreq = get_txreq(ps->dev, qp); in hfi1_make_tid_rdma_pkt()
5038 make_tid_rdma_ack(qp, ohdr, ps)) in hfi1_make_tid_rdma_pkt()
5047 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) in hfi1_make_tid_rdma_pkt()
5056 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); in hfi1_make_tid_rdma_pkt()
5058 trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_make_tid_rdma_pkt()
5069 hfi1_tid_rdma_restart_req(qp, wqe, &bth2); in hfi1_make_tid_rdma_pkt()
5087 trace_hfi1_sender_make_tid_pkt(qp); in hfi1_make_tid_rdma_pkt()
5088 trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0); in hfi1_make_tid_rdma_pkt()
5089 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); in hfi1_make_tid_rdma_pkt()
5096 trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, in hfi1_make_tid_rdma_pkt()
5108 qp->s_flags |= HFI1_S_WAIT_TID_RESP; in hfi1_make_tid_rdma_pkt()
5114 update_tid_tail(qp); in hfi1_make_tid_rdma_pkt()
5122 trace_hfi1_sender_make_tid_pkt(qp); in hfi1_make_tid_rdma_pkt()
5124 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_cur); in hfi1_make_tid_rdma_pkt()
5128 wqe = rvt_get_swqe_ptr(qp, in hfi1_make_tid_rdma_pkt()
5129 (!priv->s_tid_cur ? qp->s_size : in hfi1_make_tid_rdma_pkt()
5133 hwords += hfi1_build_tid_rdma_resync(qp, wqe, ohdr, &bth1, in hfi1_make_tid_rdma_pkt()
5150 qp->s_len -= len; in hfi1_make_tid_rdma_pkt()
5155 hfi1_make_ruc_header(qp, ohdr, (opcode << 24), bth1, bth2, in hfi1_make_tid_rdma_pkt()
5174 static int make_tid_rdma_ack(struct rvt_qp *qp, in make_tid_rdma_ack() argument
5179 struct hfi1_qp_priv *qpriv = qp->priv; in make_tid_rdma_ack()
5180 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); in make_tid_rdma_ack()
5188 trace_hfi1_tid_write_rsp_make_tid_ack(qp); in make_tid_rdma_ack()
5190 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in make_tid_rdma_ack()
5196 e = &qp->s_ack_queue[qpriv->r_tid_ack]; in make_tid_rdma_ack()
5215 e = &qp->s_ack_queue[qpriv->r_tid_ack]; in make_tid_rdma_ack()
5219 trace_hfi1_rsp_make_tid_ack(qp, e->psn); in make_tid_rdma_ack()
5220 trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn, in make_tid_rdma_ack()
5258 if (qp->s_ack_queue[next].opcode != TID_OP(WRITE_REQ)) in make_tid_rdma_ack()
5260 nreq = ack_to_tid_req(&qp->s_ack_queue[next]); in make_tid_rdma_ack()
5265 e = &qp->s_ack_queue[qpriv->r_tid_ack]; in make_tid_rdma_ack()
5286 e = &qp->s_ack_queue[qpriv->r_tid_ack]; in make_tid_rdma_ack()
5293 trace_hfi1_tid_write_rsp_make_tid_ack(qp); in make_tid_rdma_ack()
5294 trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn, in make_tid_rdma_ack()
5296 hwords += hfi1_build_tid_rdma_write_ack(qp, e, ohdr, flow, &bth1, in make_tid_rdma_ack()
5304 hfi1_make_ruc_header(qp, ohdr, (TID_OP(ACK) << 24), bth1, bth2, middle, in make_tid_rdma_ack()
5318 static int hfi1_send_tid_ok(struct rvt_qp *qp) in hfi1_send_tid_ok() argument
5320 struct hfi1_qp_priv *priv = qp->priv; in hfi1_send_tid_ok()
5323 qp->s_flags & HFI1_S_ANY_WAIT_IO) && in hfi1_send_tid_ok()
5326 !(qp->s_flags & HFI1_S_ANY_TID_WAIT_SEND)); in hfi1_send_tid_ok()
5332 struct rvt_qp *qp = iowait_to_qp(w->iow); in _hfi1_do_tid_send() local
5334 hfi1_do_tid_send(qp); in _hfi1_do_tid_send()
5337 static void hfi1_do_tid_send(struct rvt_qp *qp) in hfi1_do_tid_send() argument
5340 struct hfi1_qp_priv *priv = qp->priv; in hfi1_do_tid_send()
5342 ps.dev = to_idev(qp->ibqp.device); in hfi1_do_tid_send()
5343 ps.ibp = to_iport(qp->ibqp.device, qp->port_num); in hfi1_do_tid_send()
5347 ps.timeout_int = qp->timeout_jiffies / 8; in hfi1_do_tid_send()
5349 trace_hfi1_rc_do_tid_send(qp, false); in hfi1_do_tid_send()
5350 spin_lock_irqsave(&qp->s_lock, ps.flags); in hfi1_do_tid_send()
5353 if (!hfi1_send_tid_ok(qp)) { in hfi1_do_tid_send()
5354 if (qp->s_flags & HFI1_S_ANY_WAIT_IO) in hfi1_do_tid_send()
5356 spin_unlock_irqrestore(&qp->s_lock, ps.flags); in hfi1_do_tid_send()
5373 qp->s_flags |= RVT_S_BUSY; in hfi1_do_tid_send()
5376 spin_unlock_irqrestore(&qp->s_lock, ps.flags); in hfi1_do_tid_send()
5382 if (hfi1_verbs_send(qp, &ps)) in hfi1_do_tid_send()
5386 if (hfi1_schedule_send_yield(qp, &ps, true)) in hfi1_do_tid_send()
5389 spin_lock_irqsave(&qp->s_lock, ps.flags); in hfi1_do_tid_send()
5391 qp->s_flags &= ~RVT_S_BUSY; in hfi1_do_tid_send()
5396 hfi1_schedule_send(qp); in hfi1_do_tid_send()
5399 } while (hfi1_make_tid_rdma_pkt(qp, &ps)); in hfi1_do_tid_send()
5401 spin_unlock_irqrestore(&qp->s_lock, ps.flags); in hfi1_do_tid_send()
5404 static bool _hfi1_schedule_tid_send(struct rvt_qp *qp) in _hfi1_schedule_tid_send() argument
5406 struct hfi1_qp_priv *priv = qp->priv; in _hfi1_schedule_tid_send()
5408 to_iport(qp->ibqp.device, qp->port_num); in _hfi1_schedule_tid_send()
5434 bool hfi1_schedule_tid_send(struct rvt_qp *qp) in hfi1_schedule_tid_send() argument
5436 lockdep_assert_held(&qp->s_lock); in hfi1_schedule_tid_send()
5437 if (hfi1_send_tid_ok(qp)) { in hfi1_schedule_tid_send()
5444 _hfi1_schedule_tid_send(qp); in hfi1_schedule_tid_send()
5447 if (qp->s_flags & HFI1_S_ANY_WAIT_IO) in hfi1_schedule_tid_send()
5448 iowait_set_flag(&((struct hfi1_qp_priv *)qp->priv)->s_iowait, in hfi1_schedule_tid_send()
5453 bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e) in hfi1_tid_rdma_ack_interlock() argument
5457 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); in hfi1_tid_rdma_ack_interlock()
5458 struct hfi1_qp_priv *priv = qp->priv; in hfi1_tid_rdma_ack_interlock()
5461 s_prev = qp->s_tail_ack_queue == 0 ? rvt_size_atomic(&dev->rdi) : in hfi1_tid_rdma_ack_interlock()
5462 (qp->s_tail_ack_queue - 1); in hfi1_tid_rdma_ack_interlock()
5463 prev = &qp->s_ack_queue[s_prev]; in hfi1_tid_rdma_ack_interlock()
5491 struct rvt_qp *qp, u32 psn, int diff, bool fecn) in tid_rdma_rcv_err() argument
5495 tid_rdma_rcv_error(packet, ohdr, qp, psn, diff); in tid_rdma_rcv_err()
5497 spin_lock_irqsave(&qp->s_lock, flags); in tid_rdma_rcv_err()
5498 qp->s_flags |= RVT_S_ECN; in tid_rdma_rcv_err()
5499 spin_unlock_irqrestore(&qp->s_lock, flags); in tid_rdma_rcv_err()