Lines Matching refs:qp
66 static void flush_tx_list(struct rvt_qp *qp);
75 static void qp_pio_drain(struct rvt_qp *qp);
164 static void flush_tx_list(struct rvt_qp *qp) in flush_tx_list() argument
166 struct hfi1_qp_priv *priv = qp->priv; in flush_tx_list()
172 static void flush_iowait(struct rvt_qp *qp) in flush_iowait() argument
174 struct hfi1_qp_priv *priv = qp->priv; in flush_iowait()
184 rvt_put_qp(qp); in flush_iowait()
202 int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, in hfi1_check_modify_qp() argument
205 struct ib_qp *ibqp = &qp->ibqp; in hfi1_check_modify_qp()
215 if (!qp_to_sdma_engine(qp, sc) && in hfi1_check_modify_qp()
219 if (!qp_to_send_context(qp, sc)) in hfi1_check_modify_qp()
228 if (!qp_to_sdma_engine(qp, sc) && in hfi1_check_modify_qp()
232 if (!qp_to_send_context(qp, sc)) in hfi1_check_modify_qp()
244 static inline void qp_set_16b(struct rvt_qp *qp) in qp_set_16b() argument
248 struct hfi1_qp_priv *priv = qp->priv; in qp_set_16b()
251 hfi1_update_ah_attr(qp->ibqp.device, &qp->remote_ah_attr); in qp_set_16b()
254 hfi1_make_opa_lid(&qp->remote_ah_attr); in qp_set_16b()
256 if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) in qp_set_16b()
259 ibp = to_iport(qp->ibqp.device, qp->port_num); in qp_set_16b()
261 priv->hdr_type = hfi1_get_hdr_type(ppd->lid, &qp->remote_ah_attr); in qp_set_16b()
264 void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, in hfi1_modify_qp() argument
267 struct ib_qp *ibqp = &qp->ibqp; in hfi1_modify_qp()
268 struct hfi1_qp_priv *priv = qp->priv; in hfi1_modify_qp()
271 priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); in hfi1_modify_qp()
272 priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); in hfi1_modify_qp()
273 priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); in hfi1_modify_qp()
274 qp_set_16b(qp); in hfi1_modify_qp()
279 qp->s_mig_state == IB_MIG_ARMED) { in hfi1_modify_qp()
280 qp->s_flags |= HFI1_S_AHG_CLEAR; in hfi1_modify_qp()
281 priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); in hfi1_modify_qp()
282 priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); in hfi1_modify_qp()
283 priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); in hfi1_modify_qp()
284 qp_set_16b(qp); in hfi1_modify_qp()
287 opfn_qp_init(qp, attr, attr_mask); in hfi1_modify_qp()
305 int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) in hfi1_setup_wqe() argument
307 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in hfi1_setup_wqe()
312 switch (qp->ibqp.qp_type) { in hfi1_setup_wqe()
314 hfi1_setup_tid_rdma_wqe(qp, wqe); in hfi1_setup_wqe()
319 if (wqe->length > qp->pmtu) in hfi1_setup_wqe()
364 bool _hfi1_schedule_send(struct rvt_qp *qp) in _hfi1_schedule_send() argument
366 struct hfi1_qp_priv *priv = qp->priv; in _hfi1_schedule_send()
368 to_iport(qp->ibqp.device, qp->port_num); in _hfi1_schedule_send()
381 static void qp_pio_drain(struct rvt_qp *qp) in qp_pio_drain() argument
383 struct hfi1_qp_priv *priv = qp->priv; in qp_pio_drain()
407 bool hfi1_schedule_send(struct rvt_qp *qp) in hfi1_schedule_send() argument
409 lockdep_assert_held(&qp->s_lock); in hfi1_schedule_send()
410 if (hfi1_send_ok(qp)) { in hfi1_schedule_send()
411 _hfi1_schedule_send(qp); in hfi1_schedule_send()
414 if (qp->s_flags & HFI1_S_ANY_WAIT_IO) in hfi1_schedule_send()
415 iowait_set_flag(&((struct hfi1_qp_priv *)qp->priv)->s_iowait, in hfi1_schedule_send()
420 static void hfi1_qp_schedule(struct rvt_qp *qp) in hfi1_qp_schedule() argument
422 struct hfi1_qp_priv *priv = qp->priv; in hfi1_qp_schedule()
426 ret = hfi1_schedule_send(qp); in hfi1_qp_schedule()
431 ret = hfi1_schedule_tid_send(qp); in hfi1_qp_schedule()
437 void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag) in hfi1_qp_wakeup() argument
441 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_qp_wakeup()
442 if (qp->s_flags & flag) { in hfi1_qp_wakeup()
443 qp->s_flags &= ~flag; in hfi1_qp_wakeup()
444 trace_hfi1_qpwakeup(qp, flag); in hfi1_qp_wakeup()
445 hfi1_qp_schedule(qp); in hfi1_qp_wakeup()
447 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_qp_wakeup()
449 rvt_put_qp(qp); in hfi1_qp_wakeup()
452 void hfi1_qp_unbusy(struct rvt_qp *qp, struct iowait_work *wait) in hfi1_qp_unbusy() argument
454 struct hfi1_qp_priv *priv = qp->priv; in hfi1_qp_unbusy()
457 qp->s_flags &= ~RVT_S_BUSY; in hfi1_qp_unbusy()
485 struct rvt_qp *qp; in iowait_sleep() local
490 qp = tx->qp; in iowait_sleep()
491 priv = qp->priv; in iowait_sleep()
493 spin_lock_irqsave(&qp->s_lock, flags); in iowait_sleep()
494 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { in iowait_sleep()
507 to_iport(qp->ibqp.device, qp->port_num); in iowait_sleep()
510 qp->s_flags |= RVT_S_WAIT_DMA_DESC; in iowait_sleep()
515 trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC); in iowait_sleep()
516 rvt_get_qp(qp); in iowait_sleep()
519 hfi1_qp_unbusy(qp, wait); in iowait_sleep()
520 spin_unlock_irqrestore(&qp->s_lock, flags); in iowait_sleep()
523 spin_unlock_irqrestore(&qp->s_lock, flags); in iowait_sleep()
529 spin_unlock_irqrestore(&qp->s_lock, flags); in iowait_sleep()
536 struct rvt_qp *qp = iowait_to_qp(wait); in iowait_wakeup() local
539 hfi1_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC); in iowait_wakeup()
544 struct rvt_qp *qp = iowait_to_qp(wait); in iowait_sdma_drained() local
553 spin_lock_irqsave(&qp->s_lock, flags); in iowait_sdma_drained()
554 if (qp->s_flags & RVT_S_WAIT_DMA) { in iowait_sdma_drained()
555 qp->s_flags &= ~RVT_S_WAIT_DMA; in iowait_sdma_drained()
556 hfi1_schedule_send(qp); in iowait_sdma_drained()
558 spin_unlock_irqrestore(&qp->s_lock, flags); in iowait_sdma_drained()
563 struct rvt_qp *qp = iowait_to_qp(w); in hfi1_init_priority() local
564 struct hfi1_qp_priv *priv = qp->priv; in hfi1_init_priority()
566 if (qp->s_flags & RVT_S_ACK_PENDING) in hfi1_init_priority()
580 struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5) in qp_to_sdma_engine() argument
582 struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); in qp_to_sdma_engine()
587 switch (qp->ibqp.qp_type) { in qp_to_sdma_engine()
593 sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5); in qp_to_sdma_engine()
605 struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5) in qp_to_send_context() argument
607 struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); in qp_to_send_context()
609 switch (qp->ibqp.qp_type) { in qp_to_send_context()
617 return pio_select_send_context_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, in qp_to_send_context()
625 static int qp_idle(struct rvt_qp *qp) in qp_idle() argument
628 qp->s_last == qp->s_acked && in qp_idle()
629 qp->s_acked == qp->s_cur && in qp_idle()
630 qp->s_cur == qp->s_tail && in qp_idle()
631 qp->s_tail == qp->s_head; in qp_idle()
642 struct rvt_qp *qp = iter->qp; in qp_iter_print() local
643 struct hfi1_qp_priv *priv = qp->priv; in qp_iter_print()
647 struct rvt_srq *srq = qp->ibqp.srq ? in qp_iter_print()
648 ibsrq_to_rvtsrq(qp->ibqp.srq) : NULL; in qp_iter_print()
650 sde = qp_to_sdma_engine(qp, priv->s_sc); in qp_iter_print()
651 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qp_iter_print()
652 send_context = qp_to_send_context(qp, priv->s_sc); in qp_iter_print()
653 if (qp->s_ack_queue) in qp_iter_print()
654 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qp_iter_print()
658 qp_idle(qp) ? "I" : "B", in qp_iter_print()
659 qp->ibqp.qp_num, in qp_iter_print()
660 atomic_read(&qp->refcount), in qp_iter_print()
661 qp_type_str[qp->ibqp.qp_type], in qp_iter_print()
662 qp->state, in qp_iter_print()
664 qp->s_flags, in qp_iter_print()
668 qp->timeout, in qp_iter_print()
670 qp->s_lsn, in qp_iter_print()
671 qp->s_last_psn, in qp_iter_print()
672 qp->s_psn, qp->s_next_psn, in qp_iter_print()
673 qp->s_sending_psn, qp->s_sending_hpsn, in qp_iter_print()
674 qp->r_psn, in qp_iter_print()
675 qp->s_last, qp->s_acked, qp->s_cur, in qp_iter_print()
676 qp->s_tail, qp->s_head, qp->s_size, in qp_iter_print()
677 qp->s_avail, in qp_iter_print()
679 qp->s_tail_ack_queue, qp->r_head_ack_queue, in qp_iter_print()
680 rvt_max_atomic(&to_idev(qp->ibqp.device)->rdi), in qp_iter_print()
682 qp->remote_qpn, in qp_iter_print()
683 rdma_ah_get_dlid(&qp->remote_ah_attr), in qp_iter_print()
684 rdma_ah_get_sl(&qp->remote_ah_attr), in qp_iter_print()
685 qp->pmtu, in qp_iter_print()
686 qp->s_retry, in qp_iter_print()
687 qp->s_retry_cnt, in qp_iter_print()
688 qp->s_rnr_retry_cnt, in qp_iter_print()
689 qp->s_rnr_retry, in qp_iter_print()
694 ib_cq_head(qp->ibqp.send_cq), in qp_iter_print()
695 ib_cq_tail(qp->ibqp.send_cq), in qp_iter_print()
696 qp->pid, in qp_iter_print()
697 qp->s_state, in qp_iter_print()
698 qp->s_ack_state, in qp_iter_print()
703 qp->r_min_rnr_timer, in qp_iter_print()
705 srq ? srq->rq.size : qp->r_rq.size in qp_iter_print()
709 void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp) in qp_priv_alloc() argument
717 priv->owner = qp; in qp_priv_alloc()
739 void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) in qp_priv_free() argument
741 struct hfi1_qp_priv *priv = qp->priv; in qp_priv_free()
743 hfi1_qp_priv_tid_free(rdi, qp); in qp_priv_free()
763 if (rcu_dereference(ibp->rvp.qp[0])) in free_all_qps()
765 if (rcu_dereference(ibp->rvp.qp[1])) in free_all_qps()
773 void flush_qp_waiters(struct rvt_qp *qp) in flush_qp_waiters() argument
775 lockdep_assert_held(&qp->s_lock); in flush_qp_waiters()
776 flush_iowait(qp); in flush_qp_waiters()
777 hfi1_tid_rdma_flush_wait(qp); in flush_qp_waiters()
780 void stop_send_queue(struct rvt_qp *qp) in stop_send_queue() argument
782 struct hfi1_qp_priv *priv = qp->priv; in stop_send_queue()
786 rvt_put_qp(qp); in stop_send_queue()
789 void quiesce_qp(struct rvt_qp *qp) in quiesce_qp() argument
791 struct hfi1_qp_priv *priv = qp->priv; in quiesce_qp()
793 hfi1_del_tid_reap_timer(qp); in quiesce_qp()
794 hfi1_del_tid_retry_timer(qp); in quiesce_qp()
796 qp_pio_drain(qp); in quiesce_qp()
797 flush_tx_list(qp); in quiesce_qp()
800 void notify_qp_reset(struct rvt_qp *qp) in notify_qp_reset() argument
802 hfi1_qp_kern_exp_rcv_clear_all(qp); in notify_qp_reset()
803 qp->r_adefered = 0; in notify_qp_reset()
804 clear_ahg(qp); in notify_qp_reset()
807 if (qp->ibqp.qp_type == IB_QPT_RC) in notify_qp_reset()
808 opfn_conn_error(qp); in notify_qp_reset()
815 void hfi1_migrate_qp(struct rvt_qp *qp) in hfi1_migrate_qp() argument
817 struct hfi1_qp_priv *priv = qp->priv; in hfi1_migrate_qp()
820 qp->s_mig_state = IB_MIG_MIGRATED; in hfi1_migrate_qp()
821 qp->remote_ah_attr = qp->alt_ah_attr; in hfi1_migrate_qp()
822 qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); in hfi1_migrate_qp()
823 qp->s_pkey_index = qp->s_alt_pkey_index; in hfi1_migrate_qp()
824 qp->s_flags |= HFI1_S_AHG_CLEAR; in hfi1_migrate_qp()
825 priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr); in hfi1_migrate_qp()
826 priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); in hfi1_migrate_qp()
827 qp_set_16b(qp); in hfi1_migrate_qp()
829 ev.device = qp->ibqp.device; in hfi1_migrate_qp()
830 ev.element.qp = &qp->ibqp; in hfi1_migrate_qp()
832 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); in hfi1_migrate_qp()
840 u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu) in mtu_from_qp() argument
852 ibp = &dd->pport[qp->port_num - 1].ibport_data; in mtu_from_qp()
853 sc = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; in mtu_from_qp()
856 mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu); in mtu_from_qp()
862 int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, in get_pmtu_from_attr() argument
865 int mtu, pidx = qp->port_num - 1; in get_pmtu_from_attr()
872 mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu); in get_pmtu_from_attr()
882 void notify_error_qp(struct rvt_qp *qp) in notify_error_qp() argument
884 struct hfi1_qp_priv *priv = qp->priv; in notify_error_qp()
890 !(qp->s_flags & RVT_S_BUSY) && in notify_error_qp()
892 qp->s_flags &= ~HFI1_S_ANY_WAIT_IO; in notify_error_qp()
897 rvt_put_qp(qp); in notify_error_qp()
902 if (!(qp->s_flags & RVT_S_BUSY) && !(priv->s_flags & RVT_S_BUSY)) { in notify_error_qp()
903 qp->s_hdrwords = 0; in notify_error_qp()
904 if (qp->s_rdma_mr) { in notify_error_qp()
905 rvt_put_mr(qp->s_rdma_mr); in notify_error_qp()
906 qp->s_rdma_mr = NULL; in notify_error_qp()
908 flush_tx_list(qp); in notify_error_qp()
920 static void hfi1_qp_iter_cb(struct rvt_qp *qp, u64 v) in hfi1_qp_iter_cb() argument
925 to_iport(qp->ibqp.device, qp->port_num); in hfi1_qp_iter_cb()
929 if (qp->port_num != ppd->port || in hfi1_qp_iter_cb()
930 (qp->ibqp.qp_type != IB_QPT_UC && in hfi1_qp_iter_cb()
931 qp->ibqp.qp_type != IB_QPT_RC) || in hfi1_qp_iter_cb()
932 rdma_ah_get_sl(&qp->remote_ah_attr) != sl || in hfi1_qp_iter_cb()
933 !(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK)) in hfi1_qp_iter_cb()
936 spin_lock_irq(&qp->r_lock); in hfi1_qp_iter_cb()
937 spin_lock(&qp->s_hlock); in hfi1_qp_iter_cb()
938 spin_lock(&qp->s_lock); in hfi1_qp_iter_cb()
939 lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in hfi1_qp_iter_cb()
940 spin_unlock(&qp->s_lock); in hfi1_qp_iter_cb()
941 spin_unlock(&qp->s_hlock); in hfi1_qp_iter_cb()
942 spin_unlock_irq(&qp->r_lock); in hfi1_qp_iter_cb()
944 ev.device = qp->ibqp.device; in hfi1_qp_iter_cb()
945 ev.element.qp = &qp->ibqp; in hfi1_qp_iter_cb()
947 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); in hfi1_qp_iter_cb()