Lines Matching refs:rdi
64 static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
145 void rvt_wss_exit(struct rvt_dev_info *rdi) in rvt_wss_exit() argument
147 struct rvt_wss *wss = rdi->wss; in rvt_wss_exit()
155 kfree(rdi->wss); in rvt_wss_exit()
156 rdi->wss = NULL; in rvt_wss_exit()
164 int rvt_wss_init(struct rvt_dev_info *rdi) in rvt_wss_init() argument
166 unsigned int sge_copy_mode = rdi->dparms.sge_copy_mode; in rvt_wss_init()
167 unsigned int wss_threshold = rdi->dparms.wss_threshold; in rvt_wss_init()
168 unsigned int wss_clean_period = rdi->dparms.wss_clean_period; in rvt_wss_init()
174 int node = rdi->dparms.node; in rvt_wss_init()
177 rdi->wss = NULL; in rvt_wss_init()
181 rdi->wss = kzalloc_node(sizeof(*rdi->wss), GFP_KERNEL, node); in rvt_wss_init()
182 if (!rdi->wss) in rvt_wss_init()
184 wss = rdi->wss; in rvt_wss_init()
221 rvt_wss_exit(rdi); in rvt_wss_init()
328 static int init_qpn_table(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt) in init_qpn_table() argument
334 if (!(rdi->dparms.qpn_res_end >= rdi->dparms.qpn_res_start)) in init_qpn_table()
339 qpt->last = rdi->dparms.qpn_start; in init_qpn_table()
340 qpt->incr = rdi->dparms.qpn_inc << rdi->dparms.qos_shift; in init_qpn_table()
350 qpt->nmaps = rdi->dparms.qpn_res_start / RVT_BITS_PER_PAGE; in init_qpn_table()
353 offset = rdi->dparms.qpn_res_start & RVT_BITS_PER_PAGE_MASK; in init_qpn_table()
358 rvt_pr_info(rdi, "Reserving QPNs from 0x%x to 0x%x for non-verbs use\n", in init_qpn_table()
359 rdi->dparms.qpn_res_start, rdi->dparms.qpn_res_end); in init_qpn_table()
360 for (i = rdi->dparms.qpn_res_start; i <= rdi->dparms.qpn_res_end; i++) { in init_qpn_table()
398 int rvt_driver_qp_init(struct rvt_dev_info *rdi) in rvt_driver_qp_init() argument
403 if (!rdi->dparms.qp_table_size) in rvt_driver_qp_init()
410 if (!rdi->driver_f.free_all_qps || in rvt_driver_qp_init()
411 !rdi->driver_f.qp_priv_alloc || in rvt_driver_qp_init()
412 !rdi->driver_f.qp_priv_free || in rvt_driver_qp_init()
413 !rdi->driver_f.notify_qp_reset || in rvt_driver_qp_init()
414 !rdi->driver_f.notify_restart_rc) in rvt_driver_qp_init()
418 rdi->qp_dev = kzalloc_node(sizeof(*rdi->qp_dev), GFP_KERNEL, in rvt_driver_qp_init()
419 rdi->dparms.node); in rvt_driver_qp_init()
420 if (!rdi->qp_dev) in rvt_driver_qp_init()
424 rdi->qp_dev->qp_table_size = rdi->dparms.qp_table_size; in rvt_driver_qp_init()
425 rdi->qp_dev->qp_table_bits = ilog2(rdi->dparms.qp_table_size); in rvt_driver_qp_init()
426 rdi->qp_dev->qp_table = in rvt_driver_qp_init()
427 kmalloc_array_node(rdi->qp_dev->qp_table_size, in rvt_driver_qp_init()
428 sizeof(*rdi->qp_dev->qp_table), in rvt_driver_qp_init()
429 GFP_KERNEL, rdi->dparms.node); in rvt_driver_qp_init()
430 if (!rdi->qp_dev->qp_table) in rvt_driver_qp_init()
433 for (i = 0; i < rdi->qp_dev->qp_table_size; i++) in rvt_driver_qp_init()
434 RCU_INIT_POINTER(rdi->qp_dev->qp_table[i], NULL); in rvt_driver_qp_init()
436 spin_lock_init(&rdi->qp_dev->qpt_lock); in rvt_driver_qp_init()
439 if (init_qpn_table(rdi, &rdi->qp_dev->qpn_table)) in rvt_driver_qp_init()
442 spin_lock_init(&rdi->n_qps_lock); in rvt_driver_qp_init()
447 kfree(rdi->qp_dev->qp_table); in rvt_driver_qp_init()
448 free_qpn_table(&rdi->qp_dev->qpn_table); in rvt_driver_qp_init()
451 kfree(rdi->qp_dev); in rvt_driver_qp_init()
467 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_free_qp_cb() local
470 rvt_reset_qp(rdi, qp, qp->ibqp.qp_type); in rvt_free_qp_cb()
484 static unsigned rvt_free_all_qps(struct rvt_dev_info *rdi) in rvt_free_all_qps() argument
488 qp_inuse += rvt_mcast_tree_empty(rdi); in rvt_free_all_qps()
490 rvt_qp_iter(rdi, (u64)&qp_inuse, rvt_free_qp_cb); in rvt_free_all_qps()
501 void rvt_qp_exit(struct rvt_dev_info *rdi) in rvt_qp_exit() argument
503 u32 qps_inuse = rvt_free_all_qps(rdi); in rvt_qp_exit()
506 rvt_pr_err(rdi, "QP memory leak! %u still in use\n", in rvt_qp_exit()
508 if (!rdi->qp_dev) in rvt_qp_exit()
511 kfree(rdi->qp_dev->qp_table); in rvt_qp_exit()
512 free_qpn_table(&rdi->qp_dev->qpn_table); in rvt_qp_exit()
513 kfree(rdi->qp_dev); in rvt_qp_exit()
532 static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, in alloc_qpn() argument
541 if (rdi->driver_f.alloc_qpn) in alloc_qpn()
542 return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num); in alloc_qpn()
605 WARN_ON(rdi->dparms.qos_shift > 1 && in alloc_qpn()
606 offset & ((BIT(rdi->dparms.qos_shift - 1) - 1) << 1)); in alloc_qpn()
624 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_clear_mr_refs() local
646 for (n = 0; qp->s_ack_queue && n < rvt_max_atomic(rdi); n++) { in rvt_clear_mr_refs()
708 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_qp_acks_has_lkey() local
710 for (i = 0; qp->s_ack_queue && i < rvt_max_atomic(rdi); i++) { in rvt_qp_acks_has_lkey()
771 static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) in rvt_remove_qp() argument
773 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; in rvt_remove_qp()
774 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); in rvt_remove_qp()
778 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags); in rvt_remove_qp()
781 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { in rvt_remove_qp()
784 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { in rvt_remove_qp()
791 qpp = &rdi->qp_dev->qp_table[n]; in rvt_remove_qp()
793 lockdep_is_held(&rdi->qp_dev->qpt_lock))) != NULL; in rvt_remove_qp()
798 lockdep_is_held(&rdi->qp_dev->qpt_lock))); in rvt_remove_qp()
806 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags); in rvt_remove_qp()
864 static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, in rvt_init_qp() argument
915 static void _rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, in _rvt_reset_qp() argument
928 rdi->driver_f.flush_qp_waiters(qp); in _rvt_reset_qp()
936 rdi->driver_f.stop_send_queue(qp); in _rvt_reset_qp()
939 rdi->driver_f.quiesce_qp(qp); in _rvt_reset_qp()
942 rvt_remove_qp(rdi, qp); in _rvt_reset_qp()
954 rdi->driver_f.notify_qp_reset(qp); in _rvt_reset_qp()
956 rvt_init_qp(rdi, qp, type); in _rvt_reset_qp()
971 static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, in rvt_reset_qp() argument
977 _rvt_reset_qp(rdi, qp, type); in rvt_reset_qp()
1078 struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device); in rvt_create_qp() local
1083 if (!rdi) in rvt_create_qp()
1086 if (init_attr->cap.max_send_sge > rdi->dparms.props.max_send_sge || in rvt_create_qp()
1087 init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr || in rvt_create_qp()
1095 rdi->dparms.props.max_recv_sge || in rvt_create_qp()
1096 init_attr->cap.max_recv_wr > rdi->dparms.props.max_qp_wr) in rvt_create_qp()
1107 rdi->dparms.reserved_operations; in rvt_create_qp()
1119 swq = vzalloc_node(array_size(sz, sqsize), rdi->dparms.node); in rvt_create_qp()
1135 rdi->dparms.node); in rvt_create_qp()
1143 kcalloc_node(rvt_max_atomic(rdi), in rvt_create_qp()
1146 rdi->dparms.node); in rvt_create_qp()
1160 priv = rdi->driver_f.qp_priv_alloc(rdi, qp); in rvt_create_qp()
1177 rdi->dparms.node, udata); in rvt_create_qp()
1202 err = alloc_ud_wq_attr(qp, rdi->dparms.node); in rvt_create_qp()
1211 err = alloc_qpn(rdi, &rdi->qp_dev->qpn_table, in rvt_create_qp()
1223 rvt_init_qp(rdi, qp, init_attr->qp_type); in rvt_create_qp()
1224 if (rdi->driver_f.qp_priv_init) { in rvt_create_qp()
1225 err = rdi->driver_f.qp_priv_init(rdi, qp, init_attr); in rvt_create_qp()
1257 qp->ip = rvt_create_mmap_info(rdi, s, udata, in rvt_create_qp()
1274 spin_lock(&rdi->n_qps_lock); in rvt_create_qp()
1275 if (rdi->n_qps_allocated == rdi->dparms.props.max_qp) { in rvt_create_qp()
1276 spin_unlock(&rdi->n_qps_lock); in rvt_create_qp()
1281 rdi->n_qps_allocated++; in rvt_create_qp()
1292 rdi->n_rc_qps++; in rvt_create_qp()
1293 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; in rvt_create_qp()
1295 spin_unlock(&rdi->n_qps_lock); in rvt_create_qp()
1298 spin_lock_irq(&rdi->pending_lock); in rvt_create_qp()
1299 list_add(&qp->ip->pending_mmaps, &rdi->pending_mmaps); in rvt_create_qp()
1300 spin_unlock_irq(&rdi->pending_lock); in rvt_create_qp()
1312 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); in rvt_create_qp()
1321 rdi->driver_f.qp_priv_free(rdi, qp); in rvt_create_qp()
1348 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_error_qp() local
1365 rdi->driver_f.notify_error_qp(qp); in rvt_error_qp()
1369 rdi->driver_f.schedule_send(qp); in rvt_error_qp()
1430 static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) in rvt_insert_qp() argument
1432 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; in rvt_insert_qp()
1436 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags); in rvt_insert_qp()
1441 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); in rvt_insert_qp()
1443 qp->next = rdi->qp_dev->qp_table[n]; in rvt_insert_qp()
1444 rcu_assign_pointer(rdi->qp_dev->qp_table[n], qp); in rvt_insert_qp()
1448 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags); in rvt_insert_qp()
1463 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); in rvt_modify_qp() local
1485 if (rdi->driver_f.check_modify_qp && in rvt_modify_qp()
1486 rdi->driver_f.check_modify_qp(qp, attr, attr_mask, udata)) in rvt_modify_qp()
1517 if (attr->alt_pkey_index >= rvt_get_npkeys(rdi)) in rvt_modify_qp()
1522 if (attr->pkey_index >= rvt_get_npkeys(rdi)) in rvt_modify_qp()
1557 pmtu = rdi->driver_f.get_pmtu_from_attr(rdi, qp, attr); in rvt_modify_qp()
1581 if (attr->max_dest_rd_atomic > rdi->dparms.max_rdma_atomic) in rvt_modify_qp()
1587 _rvt_reset_qp(rdi, qp, ibqp->qp_type); in rvt_modify_qp()
1626 qp->s_next_psn = attr->sq_psn & rdi->dparms.psn_modify_mask; in rvt_modify_qp()
1634 qp->r_psn = attr->rq_psn & rdi->dparms.psn_modify_mask; in rvt_modify_qp()
1660 qp->pmtu = rdi->driver_f.mtu_from_qp(rdi, qp, pmtu); in rvt_modify_qp()
1691 if (rdi->driver_f.modify_qp) in rvt_modify_qp()
1692 rdi->driver_f.modify_qp(qp, attr, attr_mask, udata); in rvt_modify_qp()
1699 rvt_insert_qp(rdi, qp); in rvt_modify_qp()
1734 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); in rvt_destroy_qp() local
1736 rvt_reset_qp(rdi, qp, ibqp->qp_type); in rvt_destroy_qp()
1740 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); in rvt_destroy_qp()
1742 spin_lock(&rdi->n_qps_lock); in rvt_destroy_qp()
1743 rdi->n_qps_allocated--; in rvt_destroy_qp()
1745 rdi->n_rc_qps--; in rvt_destroy_qp()
1746 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; in rvt_destroy_qp()
1748 spin_unlock(&rdi->n_qps_lock); in rvt_destroy_qp()
1753 rdi->driver_f.qp_priv_free(rdi, qp); in rvt_destroy_qp()
1776 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); in rvt_query_qp() local
1780 attr->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu); in rvt_query_qp()
1783 attr->rq_psn = qp->r_psn & rdi->dparms.psn_mask; in rvt_query_qp()
1784 attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask; in rvt_query_qp()
1788 rdi->dparms.reserved_operations; in rvt_query_qp()
1965 struct rvt_dev_info *rdi, in rvt_qp_is_avail() argument
1977 if (reserved_used >= rdi->dparms.reserved_operations) in rvt_qp_is_avail()
1993 (rdi->dparms.reserved_operations - reserved_used); in rvt_qp_is_avail()
1999 (qp->s_size - 1 - rdi->dparms.reserved_operations))) in rvt_qp_is_avail()
2000 rvt_pr_err(rdi, in rvt_qp_is_avail()
2024 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_post_one_wr() local
2037 ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr); in rvt_post_one_wr()
2053 if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) { in rvt_post_one_wr()
2079 reserved_op = rdi->post_parms[wr->opcode].flags & in rvt_post_one_wr()
2082 ret = rvt_qp_is_avail(qp, rdi, reserved_op); in rvt_post_one_wr()
2089 rkt = &rdi->lkey_table; in rvt_post_one_wr()
2133 if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) { in rvt_post_one_wr()
2151 if (rdi->driver_f.setup_wqe) { in rvt_post_one_wr()
2152 ret = rdi->driver_f.setup_wqe(qp, wqe, call_send); in rvt_post_one_wr()
2157 if (!(rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) in rvt_post_one_wr()
2200 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); in rvt_post_send() local
2240 rdi->driver_f.do_send(qp); in rvt_post_send()
2242 rdi->driver_f.schedule_send_no_lock(qp); in rvt_post_send()
2327 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in init_sge() local
2329 rkt = &rdi->lkey_table; in init_sge()
2563 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); in rvt_add_retry_timer_ext() local
2568 qp->s_timer.expires = jiffies + rdi->busy_jiffies + in rvt_add_retry_timer_ext()
2643 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_rc_timeout() local
2649 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; in rvt_rc_timeout()
2655 if (rdi->driver_f.notify_restart_rc) in rvt_rc_timeout()
2656 rdi->driver_f.notify_restart_rc(qp, in rvt_rc_timeout()
2659 rdi->driver_f.schedule_send(qp); in rvt_rc_timeout()
2671 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_rc_rnr_retry() local
2677 rdi->driver_f.schedule_send(qp); in rvt_rc_rnr_retry()
2702 struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi, in rvt_qp_iter_init() argument
2712 i->rdi = rdi; in rvt_qp_iter_init()
2714 i->specials = rdi->ibdev.phys_port_cnt * 2; in rvt_qp_iter_init()
2741 struct rvt_dev_info *rdi = iter->rdi; in rvt_qp_iter_next() local
2757 for (; n < rdi->qp_dev->qp_table_size + iter->specials; n++) { in rvt_qp_iter_next()
2765 pidx = n % rdi->ibdev.phys_port_cnt; in rvt_qp_iter_next()
2766 rvp = rdi->ports[pidx]; in rvt_qp_iter_next()
2770 rdi->qp_dev->qp_table[ in rvt_qp_iter_next()
2801 void rvt_qp_iter(struct rvt_dev_info *rdi, in rvt_qp_iter() argument
2807 .rdi = rdi, in rvt_qp_iter()
2808 .specials = rdi->ibdev.phys_port_cnt * 2, in rvt_qp_iter()
2835 struct rvt_dev_info *rdi; in rvt_send_complete() local
2839 rdi = ib_to_rvt(qp->ibqp.device); in rvt_send_complete()
2843 last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode], in rvt_send_complete()
2873 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); in rvt_copy_sge() local
2874 struct rvt_wss *wss = rdi->wss; in rvt_copy_sge()
2875 unsigned int sge_copy_mode = rdi->dparms.sge_copy_mode; in rvt_copy_sge()
2958 struct rvt_dev_info *rdi = ib_to_rvt(sqp->ibqp.device); in rvt_ruc_loopback() local
2973 rvp = rdi->ports[sqp->port_num - 1]; in rvt_ruc_loopback()