Lines Matching full:qp
14 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
17 static inline void retry_first_write_send(struct rxe_qp *qp, in retry_first_write_send() argument
24 int to_send = (wqe->dma.resid > qp->mtu) ? in retry_first_write_send()
25 qp->mtu : wqe->dma.resid; in retry_first_write_send()
27 qp->req.opcode = next_opcode(qp, wqe, in retry_first_write_send()
37 wqe->iova += qp->mtu; in retry_first_write_send()
41 static void req_retry(struct rxe_qp *qp) in req_retry() argument
49 qp->req.wqe_index = consumer_index(qp->sq.queue); in req_retry()
50 qp->req.psn = qp->comp.psn; in req_retry()
51 qp->req.opcode = -1; in req_retry()
53 for (wqe_index = consumer_index(qp->sq.queue); in req_retry()
54 wqe_index != producer_index(qp->sq.queue); in req_retry()
55 wqe_index = next_index(qp->sq.queue, wqe_index)) { in req_retry()
56 wqe = addr_from_index(qp->sq.queue, wqe_index); in req_retry()
57 mask = wr_opcode_mask(wqe->wr.opcode, qp); in req_retry()
81 npsn = (qp->comp.psn - wqe->first_psn) & in req_retry()
83 retry_first_write_send(qp, wqe, mask, npsn); in req_retry()
88 qp->mtu; in req_retry()
89 wqe->iova += npsn * qp->mtu; in req_retry()
99 struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer); in rnr_nak_timer() local
101 pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp)); in rnr_nak_timer()
102 rxe_run_task(&qp->req.task, 1); in rnr_nak_timer()
105 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) in req_next_wqe() argument
107 struct rxe_send_wqe *wqe = queue_head(qp->sq.queue); in req_next_wqe()
110 if (unlikely(qp->req.state == QP_STATE_DRAIN)) { in req_next_wqe()
114 spin_lock_irqsave(&qp->state_lock, flags); in req_next_wqe()
116 if (qp->req.state != QP_STATE_DRAIN) { in req_next_wqe()
118 spin_unlock_irqrestore(&qp->state_lock, in req_next_wqe()
123 if (wqe && ((qp->req.wqe_index != in req_next_wqe()
124 consumer_index(qp->sq.queue)) || in req_next_wqe()
127 spin_unlock_irqrestore(&qp->state_lock, in req_next_wqe()
132 qp->req.state = QP_STATE_DRAINED; in req_next_wqe()
133 spin_unlock_irqrestore(&qp->state_lock, flags); in req_next_wqe()
135 if (qp->ibqp.event_handler) { in req_next_wqe()
138 ev.device = qp->ibqp.device; in req_next_wqe()
139 ev.element.qp = &qp->ibqp; in req_next_wqe()
141 qp->ibqp.event_handler(&ev, in req_next_wqe()
142 qp->ibqp.qp_context); in req_next_wqe()
147 if (qp->req.wqe_index == producer_index(qp->sq.queue)) in req_next_wqe()
150 wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index); in req_next_wqe()
152 if (unlikely((qp->req.state == QP_STATE_DRAIN || in req_next_wqe()
153 qp->req.state == QP_STATE_DRAINED) && in req_next_wqe()
158 (qp->req.wqe_index != consumer_index(qp->sq.queue)))) { in req_next_wqe()
159 qp->req.wait_fence = 1; in req_next_wqe()
163 wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp); in req_next_wqe()
167 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits) in next_opcode_rc() argument
171 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || in next_opcode_rc()
172 qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) in next_opcode_rc()
182 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || in next_opcode_rc()
183 qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) in next_opcode_rc()
193 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
194 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
204 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
205 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
224 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
225 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
239 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits) in next_opcode_uc() argument
243 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || in next_opcode_uc()
244 qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) in next_opcode_uc()
254 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || in next_opcode_uc()
255 qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) in next_opcode_uc()
265 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || in next_opcode_uc()
266 qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) in next_opcode_uc()
276 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || in next_opcode_uc()
277 qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) in next_opcode_uc()
290 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe, in next_opcode() argument
293 int fits = (wqe->dma.resid <= qp->mtu); in next_opcode()
295 switch (qp_type(qp)) { in next_opcode()
297 return next_opcode_rc(qp, opcode, fits); in next_opcode()
300 return next_opcode_uc(qp, opcode, fits); in next_opcode()
321 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe) in check_init_depth() argument
328 qp->req.need_rd_atomic = 1; in check_init_depth()
329 depth = atomic_dec_return(&qp->req.rd_atomic); in check_init_depth()
332 qp->req.need_rd_atomic = 0; in check_init_depth()
337 atomic_inc(&qp->req.rd_atomic); in check_init_depth()
341 static inline int get_mtu(struct rxe_qp *qp) in get_mtu() argument
343 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in get_mtu()
345 if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC)) in get_mtu()
346 return qp->mtu; in get_mtu()
351 static struct sk_buff *init_req_packet(struct rxe_qp *qp, in init_req_packet() argument
356 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in init_req_packet()
374 pkt->qp = qp; in init_req_packet()
375 pkt->psn = qp->req.psn; in init_req_packet()
397 qp->attr.dest_qp_num; in init_req_packet()
400 (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK)); in init_req_packet()
402 qp->req.noack_pkts = 0; in init_req_packet()
433 if (qp->ibqp.qp_num == 1) in init_req_packet()
437 deth_set_sqp(pkt, qp->ibqp.qp_num); in init_req_packet()
443 static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe, in fill_packet() argument
447 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in fill_packet()
466 err = copy_data(qp->pd, 0, &wqe->dma, in fill_packet()
487 static void update_wqe_state(struct rxe_qp *qp, in update_wqe_state() argument
492 if (qp_type(qp) == IB_QPT_RC) in update_wqe_state()
499 static void update_wqe_psn(struct rxe_qp *qp, in update_wqe_psn() argument
505 int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu; in update_wqe_psn()
512 wqe->first_psn = qp->req.psn; in update_wqe_psn()
513 wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK; in update_wqe_psn()
517 qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK; in update_wqe_psn()
519 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; in update_wqe_psn()
523 struct rxe_qp *qp, in save_state() argument
530 *rollback_psn = qp->req.psn; in save_state()
534 struct rxe_qp *qp, in rollback_state() argument
541 qp->req.psn = rollback_psn; in rollback_state()
544 static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe, in update_state() argument
547 qp->req.opcode = pkt->opcode; in update_state()
550 qp->req.wqe_index = next_index(qp->sq.queue, qp->req.wqe_index); in update_state()
552 qp->need_req_skb = 0; in update_state()
554 if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer)) in update_state()
555 mod_timer(&qp->retrans_timer, in update_state()
556 jiffies + qp->qp_timeout_jiffies); in update_state()
561 struct rxe_qp *qp = (struct rxe_qp *)arg; in rxe_requester() local
573 rxe_add_ref(qp); in rxe_requester()
576 if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR)) in rxe_requester()
579 if (unlikely(qp->req.state == QP_STATE_RESET)) { in rxe_requester()
580 qp->req.wqe_index = consumer_index(qp->sq.queue); in rxe_requester()
581 qp->req.opcode = -1; in rxe_requester()
582 qp->req.need_rd_atomic = 0; in rxe_requester()
583 qp->req.wait_psn = 0; in rxe_requester()
584 qp->req.need_retry = 0; in rxe_requester()
588 if (unlikely(qp->req.need_retry)) { in rxe_requester()
589 req_retry(qp); in rxe_requester()
590 qp->req.need_retry = 0; in rxe_requester()
593 wqe = req_next_wqe(qp); in rxe_requester()
599 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in rxe_requester()
629 qp->sq_sig_type == IB_SIGNAL_ALL_WR) in rxe_requester()
630 rxe_run_task(&qp->comp.task, 1); in rxe_requester()
631 qp->req.wqe_index = next_index(qp->sq.queue, in rxe_requester()
632 qp->req.wqe_index); in rxe_requester()
636 if (unlikely(qp_type(qp) == IB_QPT_RC && in rxe_requester()
637 psn_compare(qp->req.psn, (qp->comp.psn + in rxe_requester()
639 qp->req.wait_psn = 1; in rxe_requester()
643 /* Limit the number of inflight SKBs per QP */ in rxe_requester()
644 if (unlikely(atomic_read(&qp->skb_out) > in rxe_requester()
646 qp->need_req_skb = 1; in rxe_requester()
650 opcode = next_opcode(qp, wqe, wqe->wr.opcode); in rxe_requester()
658 if (check_init_depth(qp, wqe)) in rxe_requester()
662 mtu = get_mtu(qp); in rxe_requester()
665 if (qp_type(qp) == IB_QPT_UD) { in rxe_requester()
673 wqe->first_psn = qp->req.psn; in rxe_requester()
674 wqe->last_psn = qp->req.psn; in rxe_requester()
675 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; in rxe_requester()
676 qp->req.opcode = IB_OPCODE_UD_SEND_ONLY; in rxe_requester()
677 qp->req.wqe_index = next_index(qp->sq.queue, in rxe_requester()
678 qp->req.wqe_index); in rxe_requester()
681 __rxe_do_task(&qp->comp.task); in rxe_requester()
682 rxe_drop_ref(qp); in rxe_requester()
688 skb = init_req_packet(qp, wqe, opcode, payload, &pkt); in rxe_requester()
690 pr_err("qp#%d Failed allocating skb\n", qp_num(qp)); in rxe_requester()
694 if (fill_packet(qp, wqe, &pkt, skb, payload)) { in rxe_requester()
695 pr_debug("qp#%d Error during fill packet\n", qp_num(qp)); in rxe_requester()
706 save_state(wqe, qp, &rollback_wqe, &rollback_psn); in rxe_requester()
707 update_wqe_state(qp, wqe, &pkt); in rxe_requester()
708 update_wqe_psn(qp, wqe, &pkt, payload); in rxe_requester()
709 ret = rxe_xmit_packet(qp, &pkt, skb); in rxe_requester()
711 qp->need_req_skb = 1; in rxe_requester()
713 rollback_state(wqe, qp, &rollback_wqe, rollback_psn); in rxe_requester()
716 rxe_run_task(&qp->req.task, 1); in rxe_requester()
723 update_state(qp, wqe, &pkt, payload); in rxe_requester()
730 __rxe_do_task(&qp->comp.task); in rxe_requester()
733 rxe_drop_ref(qp); in rxe_requester()