Lines Matching refs:ic
59 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_set_flow_control() local
63 ic->i_flowctl = 1; in rds_ib_set_flow_control()
66 ic->i_flowctl = 0; in rds_ib_set_flow_control()
85 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr) in rds_ib_tune_rnr() argument
90 ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER); in rds_ib_tune_rnr()
101 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_connect_complete() local
150 ic->i_active_side ? "Active" : "Passive", in rds_ib_cm_connect_complete()
154 ic->i_flowctl ? ", flow control" : ""); in rds_ib_cm_connect_complete()
157 ic->i_sl = ic->i_cm_id->route.path_rec->sl; in rds_ib_cm_connect_complete()
159 atomic_set(&ic->i_cq_quiesce, 0); in rds_ib_cm_connect_complete()
165 rds_ib_send_init_ring(ic); in rds_ib_cm_connect_complete()
166 rds_ib_recv_init_ring(ic); in rds_ib_cm_connect_complete()
172 rds_ib_tune_rnr(ic, &qp_attr); in rds_ib_cm_connect_complete()
175 err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE); in rds_ib_cm_connect_complete()
180 err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr); in rds_ib_cm_connect_complete()
205 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_fill_conn_param() local
206 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_cm_fill_conn_param()
229 cpu_to_be64(rds_ib_piggyb_ack(ic)); in rds_ib_cm_fill_conn_param()
244 cpu_to_be64(rds_ib_piggyb_ack(ic)); in rds_ib_cm_fill_conn_param()
252 if (ic->i_flowctl) { in rds_ib_cm_fill_conn_param()
256 (atomic_read(&ic->i_credits)); in rds_ib_cm_fill_conn_param()
262 &ic->i_credits); in rds_ib_cm_fill_conn_param()
284 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_recv() local
290 tasklet_schedule(&ic->i_recv_tasklet); in rds_ib_cq_comp_handler_recv()
293 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq, in poll_scq() argument
306 if (wc->wr_id <= ic->i_send_ring.w_nr || in poll_scq()
308 rds_ib_send_cqe_handler(ic, wc); in poll_scq()
310 rds_ib_mr_cqe_handler(ic, wc); in poll_scq()
318 struct rds_ib_connection *ic = (struct rds_ib_connection *)data; in rds_ib_tasklet_fn_send() local
319 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_send()
324 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_send()
327 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
328 ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_tasklet_fn_send()
329 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
334 rds_send_xmit(&ic->conn->c_path[0]); in rds_ib_tasklet_fn_send()
337 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq, in poll_rcq() argument
351 rds_ib_recv_cqe_handler(ic, wc, ack_state); in poll_rcq()
358 struct rds_ib_connection *ic = (struct rds_ib_connection *)data; in rds_ib_tasklet_fn_recv() local
359 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_recv()
360 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_tasklet_fn_recv()
369 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_recv()
373 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
374 ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_tasklet_fn_recv()
375 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
378 rds_ib_set_ack(ic, state.ack_next, state.ack_required); in rds_ib_tasklet_fn_recv()
379 if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) { in rds_ib_tasklet_fn_recv()
381 ic->i_ack_recv = state.ack_recv; in rds_ib_tasklet_fn_recv()
385 rds_ib_attempt_ack(ic); in rds_ib_tasklet_fn_recv()
391 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_qp_event_handler() local
393 rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event, in rds_ib_qp_event_handler()
398 rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST); in rds_ib_qp_event_handler()
412 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_send() local
418 tasklet_schedule(&ic->i_send_tasklet); in rds_ib_cq_comp_handler_send()
538 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_setup_qp() local
539 struct ib_device *dev = ic->i_cm_id->device; in rds_ib_setup_qp()
565 if (ic->i_send_ring.w_nr != max_wrs) in rds_ib_setup_qp()
566 rds_ib_ring_resize(&ic->i_send_ring, max_wrs); in rds_ib_setup_qp()
570 if (ic->i_recv_ring.w_nr != max_wrs) in rds_ib_setup_qp()
571 rds_ib_ring_resize(&ic->i_recv_ring, max_wrs); in rds_ib_setup_qp()
574 ic->i_pd = rds_ibdev->pd; in rds_ib_setup_qp()
576 ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
577 cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
578 cq_attr.comp_vector = ic->i_scq_vector; in rds_ib_setup_qp()
579 ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send, in rds_ib_setup_qp()
582 if (IS_ERR(ic->i_send_cq)) { in rds_ib_setup_qp()
583 ret = PTR_ERR(ic->i_send_cq); in rds_ib_setup_qp()
584 ic->i_send_cq = NULL; in rds_ib_setup_qp()
585 ibdev_put_vector(rds_ibdev, ic->i_scq_vector); in rds_ib_setup_qp()
590 ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
591 cq_attr.cqe = ic->i_recv_ring.w_nr; in rds_ib_setup_qp()
592 cq_attr.comp_vector = ic->i_rcq_vector; in rds_ib_setup_qp()
593 ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv, in rds_ib_setup_qp()
596 if (IS_ERR(ic->i_recv_cq)) { in rds_ib_setup_qp()
597 ret = PTR_ERR(ic->i_recv_cq); in rds_ib_setup_qp()
598 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
599 ibdev_put_vector(rds_ibdev, ic->i_rcq_vector); in rds_ib_setup_qp()
604 ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_setup_qp()
610 ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_setup_qp()
621 attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
622 attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1; in rds_ib_setup_qp()
627 attr.send_cq = ic->i_send_cq; in rds_ib_setup_qp()
628 attr.recv_cq = ic->i_recv_cq; in rds_ib_setup_qp()
634 ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr); in rds_ib_setup_qp()
640 ic->i_send_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_send_hdrs_dma, in rds_ib_setup_qp()
641 ic->i_send_ring.w_nr, in rds_ib_setup_qp()
643 if (!ic->i_send_hdrs) { in rds_ib_setup_qp()
649 ic->i_recv_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
650 ic->i_recv_ring.w_nr, in rds_ib_setup_qp()
652 if (!ic->i_recv_hdrs) { in rds_ib_setup_qp()
658 ic->i_ack = rds_dma_hdr_alloc(rds_ibdev->dev, &ic->i_ack_dma, in rds_ib_setup_qp()
660 if (!ic->i_ack) { in rds_ib_setup_qp()
666 ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work), in rds_ib_setup_qp()
667 ic->i_send_ring.w_nr), in rds_ib_setup_qp()
669 if (!ic->i_sends) { in rds_ib_setup_qp()
675 ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work), in rds_ib_setup_qp()
676 ic->i_recv_ring.w_nr), in rds_ib_setup_qp()
678 if (!ic->i_recvs) { in rds_ib_setup_qp()
684 rds_ib_recv_init_ack(ic); in rds_ib_setup_qp()
686 rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd, in rds_ib_setup_qp()
687 ic->i_send_cq, ic->i_recv_cq); in rds_ib_setup_qp()
692 vfree(ic->i_sends); in rds_ib_setup_qp()
695 rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma, in rds_ib_setup_qp()
697 ic->i_ack = NULL; in rds_ib_setup_qp()
700 rds_dma_hdrs_free(rds_ibdev, ic->i_recv_hdrs, ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
701 ic->i_recv_ring.w_nr, DMA_FROM_DEVICE); in rds_ib_setup_qp()
702 ic->i_recv_hdrs = NULL; in rds_ib_setup_qp()
703 ic->i_recv_hdrs_dma = NULL; in rds_ib_setup_qp()
706 rds_dma_hdrs_free(rds_ibdev, ic->i_send_hdrs, ic->i_send_hdrs_dma, in rds_ib_setup_qp()
707 ic->i_send_ring.w_nr, DMA_TO_DEVICE); in rds_ib_setup_qp()
708 ic->i_send_hdrs = NULL; in rds_ib_setup_qp()
709 ic->i_send_hdrs_dma = NULL; in rds_ib_setup_qp()
712 rdma_destroy_qp(ic->i_cm_id); in rds_ib_setup_qp()
714 ib_destroy_cq(ic->i_recv_cq); in rds_ib_setup_qp()
715 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
717 ib_destroy_cq(ic->i_send_cq); in rds_ib_setup_qp()
718 ic->i_send_cq = NULL; in rds_ib_setup_qp()
823 struct rds_ib_connection *ic = NULL; in rds_ib_cm_handle_connect() local
919 ic = conn->c_transport_data; in rds_ib_cm_handle_connect()
931 BUG_ON(ic->i_cm_id); in rds_ib_cm_handle_connect()
933 ic->i_cm_id = cm_id; in rds_ib_cm_handle_connect()
967 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_initiate_connect() local
975 ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */ in rds_ib_cm_initiate_connect()
996 if (ic->i_cm_id == cm_id) in rds_ib_cm_initiate_connect()
999 ic->i_active_side = true; in rds_ib_cm_initiate_connect()
1008 struct rds_ib_connection *ic; in rds_ib_conn_path_connect() local
1011 ic = conn->c_transport_data; in rds_ib_conn_path_connect()
1021 ic->i_cm_id = rdma_create_id(&init_net, handler, conn, in rds_ib_conn_path_connect()
1023 if (IS_ERR(ic->i_cm_id)) { in rds_ib_conn_path_connect()
1024 ret = PTR_ERR(ic->i_cm_id); in rds_ib_conn_path_connect()
1025 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
1030 rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn); in rds_ib_conn_path_connect()
1060 ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src, in rds_ib_conn_path_connect()
1064 rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id, in rds_ib_conn_path_connect()
1066 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_connect()
1067 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
1082 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_conn_path_shutdown() local
1085 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id, in rds_ib_conn_path_shutdown()
1086 ic->i_pd, ic->i_send_cq, ic->i_recv_cq, in rds_ib_conn_path_shutdown()
1087 ic->i_cm_id ? ic->i_cm_id->qp : NULL); in rds_ib_conn_path_shutdown()
1089 if (ic->i_cm_id) { in rds_ib_conn_path_shutdown()
1090 rdsdebug("disconnecting cm %p\n", ic->i_cm_id); in rds_ib_conn_path_shutdown()
1091 err = rdma_disconnect(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1097 ic->i_cm_id, err); in rds_ib_conn_path_shutdown()
1115 rds_ib_ring_empty(&ic->i_recv_ring) && in rds_ib_conn_path_shutdown()
1116 (atomic_read(&ic->i_signaled_sends) == 0) && in rds_ib_conn_path_shutdown()
1117 (atomic_read(&ic->i_fastreg_inuse_count) == 0) && in rds_ib_conn_path_shutdown()
1118 (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR)); in rds_ib_conn_path_shutdown()
1119 tasklet_kill(&ic->i_send_tasklet); in rds_ib_conn_path_shutdown()
1120 tasklet_kill(&ic->i_recv_tasklet); in rds_ib_conn_path_shutdown()
1122 atomic_set(&ic->i_cq_quiesce, 1); in rds_ib_conn_path_shutdown()
1125 if (ic->i_cm_id->qp) in rds_ib_conn_path_shutdown()
1126 rdma_destroy_qp(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1127 if (ic->i_send_cq) { in rds_ib_conn_path_shutdown()
1128 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1129 ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector); in rds_ib_conn_path_shutdown()
1130 ib_destroy_cq(ic->i_send_cq); in rds_ib_conn_path_shutdown()
1133 if (ic->i_recv_cq) { in rds_ib_conn_path_shutdown()
1134 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1135 ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector); in rds_ib_conn_path_shutdown()
1136 ib_destroy_cq(ic->i_recv_cq); in rds_ib_conn_path_shutdown()
1139 if (ic->rds_ibdev) { in rds_ib_conn_path_shutdown()
1141 if (ic->i_send_hdrs) { in rds_ib_conn_path_shutdown()
1142 rds_dma_hdrs_free(ic->rds_ibdev, in rds_ib_conn_path_shutdown()
1143 ic->i_send_hdrs, in rds_ib_conn_path_shutdown()
1144 ic->i_send_hdrs_dma, in rds_ib_conn_path_shutdown()
1145 ic->i_send_ring.w_nr, in rds_ib_conn_path_shutdown()
1147 ic->i_send_hdrs = NULL; in rds_ib_conn_path_shutdown()
1148 ic->i_send_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1151 if (ic->i_recv_hdrs) { in rds_ib_conn_path_shutdown()
1152 rds_dma_hdrs_free(ic->rds_ibdev, in rds_ib_conn_path_shutdown()
1153 ic->i_recv_hdrs, in rds_ib_conn_path_shutdown()
1154 ic->i_recv_hdrs_dma, in rds_ib_conn_path_shutdown()
1155 ic->i_recv_ring.w_nr, in rds_ib_conn_path_shutdown()
1157 ic->i_recv_hdrs = NULL; in rds_ib_conn_path_shutdown()
1158 ic->i_recv_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1161 if (ic->i_ack) { in rds_ib_conn_path_shutdown()
1162 rds_dma_hdr_free(ic->rds_ibdev->dev, ic->i_ack, in rds_ib_conn_path_shutdown()
1163 ic->i_ack_dma, DMA_TO_DEVICE); in rds_ib_conn_path_shutdown()
1164 ic->i_ack = NULL; in rds_ib_conn_path_shutdown()
1167 WARN_ON(ic->i_send_hdrs); in rds_ib_conn_path_shutdown()
1168 WARN_ON(ic->i_send_hdrs_dma); in rds_ib_conn_path_shutdown()
1169 WARN_ON(ic->i_recv_hdrs); in rds_ib_conn_path_shutdown()
1170 WARN_ON(ic->i_recv_hdrs_dma); in rds_ib_conn_path_shutdown()
1171 WARN_ON(ic->i_ack); in rds_ib_conn_path_shutdown()
1174 if (ic->i_sends) in rds_ib_conn_path_shutdown()
1175 rds_ib_send_clear_ring(ic); in rds_ib_conn_path_shutdown()
1176 if (ic->i_recvs) in rds_ib_conn_path_shutdown()
1177 rds_ib_recv_clear_ring(ic); in rds_ib_conn_path_shutdown()
1179 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1184 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1185 rds_ib_remove_conn(ic->rds_ibdev, conn); in rds_ib_conn_path_shutdown()
1187 ic->i_cm_id = NULL; in rds_ib_conn_path_shutdown()
1188 ic->i_pd = NULL; in rds_ib_conn_path_shutdown()
1189 ic->i_send_cq = NULL; in rds_ib_conn_path_shutdown()
1190 ic->i_recv_cq = NULL; in rds_ib_conn_path_shutdown()
1192 BUG_ON(ic->rds_ibdev); in rds_ib_conn_path_shutdown()
1195 if (ic->i_data_op) { in rds_ib_conn_path_shutdown()
1198 rm = container_of(ic->i_data_op, struct rds_message, data); in rds_ib_conn_path_shutdown()
1200 ic->i_data_op = NULL; in rds_ib_conn_path_shutdown()
1204 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_conn_path_shutdown()
1206 atomic64_set(&ic->i_ack_next, 0); in rds_ib_conn_path_shutdown()
1208 ic->i_ack_next = 0; in rds_ib_conn_path_shutdown()
1210 ic->i_ack_recv = 0; in rds_ib_conn_path_shutdown()
1213 ic->i_flowctl = 0; in rds_ib_conn_path_shutdown()
1214 atomic_set(&ic->i_credits, 0); in rds_ib_conn_path_shutdown()
1217 rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr); in rds_ib_conn_path_shutdown()
1218 rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr); in rds_ib_conn_path_shutdown()
1220 if (ic->i_ibinc) { in rds_ib_conn_path_shutdown()
1221 rds_inc_put(&ic->i_ibinc->ii_inc); in rds_ib_conn_path_shutdown()
1222 ic->i_ibinc = NULL; in rds_ib_conn_path_shutdown()
1225 vfree(ic->i_sends); in rds_ib_conn_path_shutdown()
1226 ic->i_sends = NULL; in rds_ib_conn_path_shutdown()
1227 vfree(ic->i_recvs); in rds_ib_conn_path_shutdown()
1228 ic->i_recvs = NULL; in rds_ib_conn_path_shutdown()
1229 ic->i_active_side = false; in rds_ib_conn_path_shutdown()
1234 struct rds_ib_connection *ic; in rds_ib_conn_alloc() local
1239 ic = kzalloc(sizeof(struct rds_ib_connection), gfp); in rds_ib_conn_alloc()
1240 if (!ic) in rds_ib_conn_alloc()
1243 ret = rds_ib_recv_alloc_caches(ic, gfp); in rds_ib_conn_alloc()
1245 kfree(ic); in rds_ib_conn_alloc()
1249 INIT_LIST_HEAD(&ic->ib_node); in rds_ib_conn_alloc()
1250 tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send, in rds_ib_conn_alloc()
1251 (unsigned long)ic); in rds_ib_conn_alloc()
1252 tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv, in rds_ib_conn_alloc()
1253 (unsigned long)ic); in rds_ib_conn_alloc()
1254 mutex_init(&ic->i_recv_mutex); in rds_ib_conn_alloc()
1256 spin_lock_init(&ic->i_ack_lock); in rds_ib_conn_alloc()
1258 atomic_set(&ic->i_signaled_sends, 0); in rds_ib_conn_alloc()
1259 atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR); in rds_ib_conn_alloc()
1265 rds_ib_ring_init(&ic->i_send_ring, 0); in rds_ib_conn_alloc()
1266 rds_ib_ring_init(&ic->i_recv_ring, 0); in rds_ib_conn_alloc()
1268 ic->conn = conn; in rds_ib_conn_alloc()
1269 conn->c_transport_data = ic; in rds_ib_conn_alloc()
1272 list_add_tail(&ic->ib_node, &ib_nodev_conns); in rds_ib_conn_alloc()
1285 struct rds_ib_connection *ic = arg; in rds_ib_conn_free() local
1288 rdsdebug("ic %p\n", ic); in rds_ib_conn_free()
1295 lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock; in rds_ib_conn_free()
1298 list_del(&ic->ib_node); in rds_ib_conn_free()
1301 rds_ib_recv_free_caches(ic); in rds_ib_conn_free()
1303 kfree(ic); in rds_ib_conn_free()