57 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_set_flow_control()  local
61 		ic->i_flowctl = 1;  in rds_ib_set_flow_control()
64 		ic->i_flowctl = 0;  in rds_ib_set_flow_control()
83 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)  in rds_ib_tune_rnr()  argument
88 	ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);  in rds_ib_tune_rnr()
99 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_connect_complete()  local
146 			  ic->i_active_side ? "Active" : "Passive",  in rds_ib_cm_connect_complete()
150 			  ic->i_flowctl ? ", flow control" : "");  in rds_ib_cm_connect_complete()
153 	atomic_set(&ic->i_cq_quiesce, 0);  in rds_ib_cm_connect_complete()
159 	rds_ib_send_init_ring(ic);  in rds_ib_cm_connect_complete()
160 	rds_ib_recv_init_ring(ic);  in rds_ib_cm_connect_complete()
166 	rds_ib_tune_rnr(ic, &qp_attr);  in rds_ib_cm_connect_complete()
169 	err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);  in rds_ib_cm_connect_complete()
174 	err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr);  in rds_ib_cm_connect_complete()
198 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_fill_conn_param()  local
199 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;  in rds_ib_cm_fill_conn_param()
222 			    cpu_to_be64(rds_ib_piggyb_ack(ic));  in rds_ib_cm_fill_conn_param()
236 			    cpu_to_be64(rds_ib_piggyb_ack(ic));  in rds_ib_cm_fill_conn_param()
243 		if (ic->i_flowctl) {  in rds_ib_cm_fill_conn_param()
247 				(atomic_read(&ic->i_credits));  in rds_ib_cm_fill_conn_param()
253 				   &ic->i_credits);  in rds_ib_cm_fill_conn_param()
275 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cq_comp_handler_recv()  local
281 	tasklet_schedule(&ic->i_recv_tasklet);  in rds_ib_cq_comp_handler_recv()
284 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,  in poll_scq()  argument
297 			if (wc->wr_id <= ic->i_send_ring.w_nr ||  in poll_scq()
299 				rds_ib_send_cqe_handler(ic, wc);  in poll_scq()
301 				rds_ib_mr_cqe_handler(ic, wc);  in poll_scq()
309 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;  in rds_ib_tasklet_fn_send()  local
310 	struct rds_connection *conn = ic->conn;  in rds_ib_tasklet_fn_send()
315 	if (atomic_read(&ic->i_cq_quiesce))  in rds_ib_tasklet_fn_send()
318 	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);  in rds_ib_tasklet_fn_send()
319 	ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);  in rds_ib_tasklet_fn_send()
320 	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);  in rds_ib_tasklet_fn_send()
325 		rds_send_xmit(&ic->conn->c_path[0]);  in rds_ib_tasklet_fn_send()
328 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,  in poll_rcq()  argument
342 			rds_ib_recv_cqe_handler(ic, wc, ack_state);  in poll_rcq()
349 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;  in rds_ib_tasklet_fn_recv()  local
350 	struct rds_connection *conn = ic->conn;  in rds_ib_tasklet_fn_recv()
351 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;  in rds_ib_tasklet_fn_recv()
360 	if (atomic_read(&ic->i_cq_quiesce))  in rds_ib_tasklet_fn_recv()
364 	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);  in rds_ib_tasklet_fn_recv()
365 	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_ib_tasklet_fn_recv()
366 	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);  in rds_ib_tasklet_fn_recv()
369 		rds_ib_set_ack(ic, state.ack_next, state.ack_required);  in rds_ib_tasklet_fn_recv()
370 	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {  in rds_ib_tasklet_fn_recv()
372 		ic->i_ack_recv = state.ack_recv;  in rds_ib_tasklet_fn_recv()
376 		rds_ib_attempt_ack(ic);  in rds_ib_tasklet_fn_recv()
382 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_qp_event_handler()  local
384 	rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,  in rds_ib_qp_event_handler()
389 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);  in rds_ib_qp_event_handler()
404 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cq_comp_handler_send()  local
410 	tasklet_schedule(&ic->i_send_tasklet);  in rds_ib_cq_comp_handler_send()
441 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_setup_qp()  local
442 	struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_setup_qp()
468 	if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1)  in rds_ib_setup_qp()
469 		rds_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1);  in rds_ib_setup_qp()
470 	if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1)  in rds_ib_setup_qp()
471 		rds_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1);  in rds_ib_setup_qp()
474 	ic->i_pd = rds_ibdev->pd;  in rds_ib_setup_qp()
476 	ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);  in rds_ib_setup_qp()
477 	cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;  in rds_ib_setup_qp()
478 	cq_attr.comp_vector = ic->i_scq_vector;  in rds_ib_setup_qp()
479 	ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,  in rds_ib_setup_qp()
482 	if (IS_ERR(ic->i_send_cq)) {  in rds_ib_setup_qp()
483 		ret = PTR_ERR(ic->i_send_cq);  in rds_ib_setup_qp()
484 		ic->i_send_cq = NULL;  in rds_ib_setup_qp()
485 		ibdev_put_vector(rds_ibdev, ic->i_scq_vector);  in rds_ib_setup_qp()
490 	ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);  in rds_ib_setup_qp()
491 	cq_attr.cqe = ic->i_recv_ring.w_nr;  in rds_ib_setup_qp()
492 	cq_attr.comp_vector = ic->i_rcq_vector;  in rds_ib_setup_qp()
493 	ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,  in rds_ib_setup_qp()
496 	if (IS_ERR(ic->i_recv_cq)) {  in rds_ib_setup_qp()
497 		ret = PTR_ERR(ic->i_recv_cq);  in rds_ib_setup_qp()
498 		ic->i_recv_cq = NULL;  in rds_ib_setup_qp()
499 		ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);  in rds_ib_setup_qp()
504 	ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);  in rds_ib_setup_qp()
510 	ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_ib_setup_qp()
521 	attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1;  in rds_ib_setup_qp()
522 	attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;  in rds_ib_setup_qp()
527 	attr.send_cq = ic->i_send_cq;  in rds_ib_setup_qp()
528 	attr.recv_cq = ic->i_recv_cq;  in rds_ib_setup_qp()
529 	atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);  in rds_ib_setup_qp()
530 	atomic_set(&ic->i_fastunreg_wrs, RDS_IB_DEFAULT_FR_INV_WR);  in rds_ib_setup_qp()
536 	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);  in rds_ib_setup_qp()
542 	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
543 					   ic->i_send_ring.w_nr *  in rds_ib_setup_qp()
545 					   &ic->i_send_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
546 	if (!ic->i_send_hdrs) {  in rds_ib_setup_qp()
552 	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
553 					   ic->i_recv_ring.w_nr *  in rds_ib_setup_qp()
555 					   &ic->i_recv_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
556 	if (!ic->i_recv_hdrs) {  in rds_ib_setup_qp()
562 	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),  in rds_ib_setup_qp()
563 				       &ic->i_ack_dma, GFP_KERNEL);  in rds_ib_setup_qp()
564 	if (!ic->i_ack) {  in rds_ib_setup_qp()
570 	ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work),  in rds_ib_setup_qp()
571 					      ic->i_send_ring.w_nr),  in rds_ib_setup_qp()
573 	if (!ic->i_sends) {  in rds_ib_setup_qp()
579 	ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work),  in rds_ib_setup_qp()
580 					      ic->i_recv_ring.w_nr),  in rds_ib_setup_qp()
582 	if (!ic->i_recvs) {  in rds_ib_setup_qp()
588 	rds_ib_recv_init_ack(ic);  in rds_ib_setup_qp()
590 	rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,  in rds_ib_setup_qp()
591 		 ic->i_send_cq, ic->i_recv_cq);  in rds_ib_setup_qp()
596 	vfree(ic->i_sends);  in rds_ib_setup_qp()
599 			     ic->i_ack, ic->i_ack_dma);  in rds_ib_setup_qp()
601 	ib_dma_free_coherent(dev, ic->i_recv_ring.w_nr *  in rds_ib_setup_qp()
603 					ic->i_recv_hdrs, ic->i_recv_hdrs_dma);  in rds_ib_setup_qp()
605 	ib_dma_free_coherent(dev, ic->i_send_ring.w_nr *  in rds_ib_setup_qp()
607 					ic->i_send_hdrs, ic->i_send_hdrs_dma);  in rds_ib_setup_qp()
609 	rdma_destroy_qp(ic->i_cm_id);  in rds_ib_setup_qp()
611 	if (!ib_destroy_cq(ic->i_recv_cq))  in rds_ib_setup_qp()
612 		ic->i_recv_cq = NULL;  in rds_ib_setup_qp()
614 	if (!ib_destroy_cq(ic->i_send_cq))  in rds_ib_setup_qp()
615 		ic->i_send_cq = NULL;  in rds_ib_setup_qp()
717 	struct rds_ib_connection *ic = NULL;  in rds_ib_cm_handle_connect()  local
810 	ic = conn->c_transport_data;  in rds_ib_cm_handle_connect()
822 	BUG_ON(ic->i_cm_id);  in rds_ib_cm_handle_connect()
824 	ic->i_cm_id = cm_id;  in rds_ib_cm_handle_connect()
857 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_initiate_connect()  local
865 	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */  in rds_ib_cm_initiate_connect()
884 		if (ic->i_cm_id == cm_id)  in rds_ib_cm_initiate_connect()
887 	ic->i_active_side = true;  in rds_ib_cm_initiate_connect()
896 	struct rds_ib_connection *ic;  in rds_ib_conn_path_connect()  local
899 	ic = conn->c_transport_data;  in rds_ib_conn_path_connect()
909 	ic->i_cm_id = rdma_create_id(&init_net, handler, conn,  in rds_ib_conn_path_connect()
911 	if (IS_ERR(ic->i_cm_id)) {  in rds_ib_conn_path_connect()
912 		ret = PTR_ERR(ic->i_cm_id);  in rds_ib_conn_path_connect()
913 		ic->i_cm_id = NULL;  in rds_ib_conn_path_connect()
918 	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);  in rds_ib_conn_path_connect()
948 	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,  in rds_ib_conn_path_connect()
952 		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,  in rds_ib_conn_path_connect()
954 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_path_connect()
955 		ic->i_cm_id = NULL;  in rds_ib_conn_path_connect()
970 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_conn_path_shutdown()  local
973 	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,  in rds_ib_conn_path_shutdown()
974 		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,  in rds_ib_conn_path_shutdown()
975 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);  in rds_ib_conn_path_shutdown()
977 	if (ic->i_cm_id) {  in rds_ib_conn_path_shutdown()
978 		struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_conn_path_shutdown()
980 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);  in rds_ib_conn_path_shutdown()
981 		err = rdma_disconnect(ic->i_cm_id);  in rds_ib_conn_path_shutdown()
987 				ic->i_cm_id, err);  in rds_ib_conn_path_shutdown()
1000 			   rds_ib_ring_empty(&ic->i_recv_ring) &&  in rds_ib_conn_path_shutdown()
1001 			   (atomic_read(&ic->i_signaled_sends) == 0) &&  in rds_ib_conn_path_shutdown()
1002 			   (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR) &&  in rds_ib_conn_path_shutdown()
1003 			   (atomic_read(&ic->i_fastunreg_wrs) == RDS_IB_DEFAULT_FR_INV_WR));  in rds_ib_conn_path_shutdown()
1004 		tasklet_kill(&ic->i_send_tasklet);  in rds_ib_conn_path_shutdown()
1005 		tasklet_kill(&ic->i_recv_tasklet);  in rds_ib_conn_path_shutdown()
1007 		atomic_set(&ic->i_cq_quiesce, 1);  in rds_ib_conn_path_shutdown()
1010 		if (ic->i_cm_id->qp)  in rds_ib_conn_path_shutdown()
1011 			rdma_destroy_qp(ic->i_cm_id);  in rds_ib_conn_path_shutdown()
1012 		if (ic->i_send_cq) {  in rds_ib_conn_path_shutdown()
1013 			if (ic->rds_ibdev)  in rds_ib_conn_path_shutdown()
1014 				ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector);  in rds_ib_conn_path_shutdown()
1015 			ib_destroy_cq(ic->i_send_cq);  in rds_ib_conn_path_shutdown()
1018 		if (ic->i_recv_cq) {  in rds_ib_conn_path_shutdown()
1019 			if (ic->rds_ibdev)  in rds_ib_conn_path_shutdown()
1020 				ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector);  in rds_ib_conn_path_shutdown()
1021 			ib_destroy_cq(ic->i_recv_cq);  in rds_ib_conn_path_shutdown()
1025 		if (ic->i_send_hdrs)  in rds_ib_conn_path_shutdown()
1027 					   ic->i_send_ring.w_nr *  in rds_ib_conn_path_shutdown()
1029 					   ic->i_send_hdrs,  in rds_ib_conn_path_shutdown()
1030 					   ic->i_send_hdrs_dma);  in rds_ib_conn_path_shutdown()
1032 		if (ic->i_recv_hdrs)  in rds_ib_conn_path_shutdown()
1034 					   ic->i_recv_ring.w_nr *  in rds_ib_conn_path_shutdown()
1036 					   ic->i_recv_hdrs,  in rds_ib_conn_path_shutdown()
1037 					   ic->i_recv_hdrs_dma);  in rds_ib_conn_path_shutdown()
1039 		if (ic->i_ack)  in rds_ib_conn_path_shutdown()
1041 					     ic->i_ack, ic->i_ack_dma);  in rds_ib_conn_path_shutdown()
1043 		if (ic->i_sends)  in rds_ib_conn_path_shutdown()
1044 			rds_ib_send_clear_ring(ic);  in rds_ib_conn_path_shutdown()
1045 		if (ic->i_recvs)  in rds_ib_conn_path_shutdown()
1046 			rds_ib_recv_clear_ring(ic);  in rds_ib_conn_path_shutdown()
1048 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_path_shutdown()
1053 		if (ic->rds_ibdev)  in rds_ib_conn_path_shutdown()
1054 			rds_ib_remove_conn(ic->rds_ibdev, conn);  in rds_ib_conn_path_shutdown()
1056 		ic->i_cm_id = NULL;  in rds_ib_conn_path_shutdown()
1057 		ic->i_pd = NULL;  in rds_ib_conn_path_shutdown()
1058 		ic->i_send_cq = NULL;  in rds_ib_conn_path_shutdown()
1059 		ic->i_recv_cq = NULL;  in rds_ib_conn_path_shutdown()
1060 		ic->i_send_hdrs = NULL;  in rds_ib_conn_path_shutdown()
1061 		ic->i_recv_hdrs = NULL;  in rds_ib_conn_path_shutdown()
1062 		ic->i_ack = NULL;  in rds_ib_conn_path_shutdown()
1064 	BUG_ON(ic->rds_ibdev);  in rds_ib_conn_path_shutdown()
1067 	if (ic->i_data_op) {  in rds_ib_conn_path_shutdown()
1070 		rm = container_of(ic->i_data_op, struct rds_message, data);  in rds_ib_conn_path_shutdown()
1072 		ic->i_data_op = NULL;  in rds_ib_conn_path_shutdown()
1076 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_ib_conn_path_shutdown()
1078 	atomic64_set(&ic->i_ack_next, 0);  in rds_ib_conn_path_shutdown()
1080 	ic->i_ack_next = 0;  in rds_ib_conn_path_shutdown()
1082 	ic->i_ack_recv = 0;  in rds_ib_conn_path_shutdown()
1085 	ic->i_flowctl = 0;  in rds_ib_conn_path_shutdown()
1086 	atomic_set(&ic->i_credits, 0);  in rds_ib_conn_path_shutdown()
1088 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);  in rds_ib_conn_path_shutdown()
1089 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);  in rds_ib_conn_path_shutdown()
1091 	if (ic->i_ibinc) {  in rds_ib_conn_path_shutdown()
1092 		rds_inc_put(&ic->i_ibinc->ii_inc);  in rds_ib_conn_path_shutdown()
1093 		ic->i_ibinc = NULL;  in rds_ib_conn_path_shutdown()
1096 	vfree(ic->i_sends);  in rds_ib_conn_path_shutdown()
1097 	ic->i_sends = NULL;  in rds_ib_conn_path_shutdown()
1098 	vfree(ic->i_recvs);  in rds_ib_conn_path_shutdown()
1099 	ic->i_recvs = NULL;  in rds_ib_conn_path_shutdown()
1100 	ic->i_active_side = false;  in rds_ib_conn_path_shutdown()
1105 	struct rds_ib_connection *ic;  in rds_ib_conn_alloc()  local
1110 	ic = kzalloc(sizeof(struct rds_ib_connection), gfp);  in rds_ib_conn_alloc()
1111 	if (!ic)  in rds_ib_conn_alloc()
1114 	ret = rds_ib_recv_alloc_caches(ic, gfp);  in rds_ib_conn_alloc()
1116 		kfree(ic);  in rds_ib_conn_alloc()
1120 	INIT_LIST_HEAD(&ic->ib_node);  in rds_ib_conn_alloc()
1121 	tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,  in rds_ib_conn_alloc()
1122 		     (unsigned long)ic);  in rds_ib_conn_alloc()
1123 	tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,  in rds_ib_conn_alloc()
1124 		     (unsigned long)ic);  in rds_ib_conn_alloc()
1125 	mutex_init(&ic->i_recv_mutex);  in rds_ib_conn_alloc()
1127 	spin_lock_init(&ic->i_ack_lock);  in rds_ib_conn_alloc()
1129 	atomic_set(&ic->i_signaled_sends, 0);  in rds_ib_conn_alloc()
1135 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);  in rds_ib_conn_alloc()
1136 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);  in rds_ib_conn_alloc()
1138 	ic->conn = conn;  in rds_ib_conn_alloc()
1139 	conn->c_transport_data = ic;  in rds_ib_conn_alloc()
1142 	list_add_tail(&ic->ib_node, &ib_nodev_conns);  in rds_ib_conn_alloc()
1146 	rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data);  in rds_ib_conn_alloc()
1155 	struct rds_ib_connection *ic = arg;  in rds_ib_conn_free()  local
1158 	rdsdebug("ic %p\n", ic);  in rds_ib_conn_free()
1165 	lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;  in rds_ib_conn_free()
1168 	list_del(&ic->ib_node);  in rds_ib_conn_free()
1171 	rds_ib_recv_free_caches(ic);  in rds_ib_conn_free()
1173 	kfree(ic);  in rds_ib_conn_free()