• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17  */
18 
19 FILE_LICENCE ( GPL2_OR_LATER );
20 
21 #include <stdint.h>
22 #include <string.h>
23 #include <byteswap.h>
24 #include <errno.h>
25 #include <gpxe/list.h>
26 #include <gpxe/infiniband.h>
27 #include <gpxe/ib_mi.h>
28 #include <gpxe/ib_mcast.h>
29 
30 /** @file
31  *
32  * Infiniband multicast groups
33  *
34  */
35 
36 /**
37  * Generate multicast membership MAD
38  *
39  * @v ibdev		Infiniband device
40  * @v gid		Multicast GID
41  * @v join		Join (rather than leave) group
42  * @v mad		MAD to fill in
43  */
ib_mcast_mad(struct ib_device * ibdev,struct ib_gid * gid,int join,union ib_mad * mad)44 static void ib_mcast_mad ( struct ib_device *ibdev, struct ib_gid *gid,
45 			   int join, union ib_mad *mad ) {
46 	struct ib_mad_sa *sa = &mad->sa;
47 
48 	/* Construct multicast membership record request */
49 	memset ( sa, 0, sizeof ( *sa ) );
50 	sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
51 	sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
52 	sa->mad_hdr.method =
53 		( join ? IB_MGMT_METHOD_SET : IB_MGMT_METHOD_DELETE );
54 	sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC );
55 	sa->sa_hdr.comp_mask[1] =
56 		htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
57 			IB_SA_MCMEMBER_REC_JOIN_STATE );
58 	sa->sa_data.mc_member_record.scope__join_state = 1;
59 	memcpy ( &sa->sa_data.mc_member_record.mgid, gid,
60 		 sizeof ( sa->sa_data.mc_member_record.mgid ) );
61 	memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
62 		 sizeof ( sa->sa_data.mc_member_record.port_gid ) );
63 }
64 
65 /**
66  * Handle multicast membership record join response
67  *
68  * @v ibdev		Infiniband device
69  * @v mi		Management interface
70  * @v madx		Management transaction
71  * @v rc		Status code
72  * @v mad		Received MAD (or NULL on error)
73  * @v av		Source address vector (or NULL on error)
74  */
ib_mcast_complete(struct ib_device * ibdev,struct ib_mad_interface * mi __unused,struct ib_mad_transaction * madx,int rc,union ib_mad * mad,struct ib_address_vector * av __unused)75 static void ib_mcast_complete ( struct ib_device *ibdev,
76 				struct ib_mad_interface *mi __unused,
77 				struct ib_mad_transaction *madx,
78 				int rc, union ib_mad *mad,
79 				struct ib_address_vector *av __unused ) {
80 	struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
81 	struct ib_queue_pair *qp = membership->qp;
82 	struct ib_gid *gid = &membership->gid;
83 	struct ib_mc_member_record *mc_member_record =
84 		&mad->sa.sa_data.mc_member_record;
85 	int joined;
86 	unsigned long qkey;
87 
88 	/* Report failures */
89 	if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
90 		rc = -ENOTCONN;
91 	if ( rc != 0 ) {
92 		DBGC ( ibdev, "IBDEV %p QPN %lx join failed: %s\n",
93 		       ibdev, qp->qpn, strerror ( rc ) );
94 		goto out;
95 	}
96 
97 	/* Extract values from MAD */
98 	joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
99 	qkey = ntohl ( mc_member_record->qkey );
100 	DBGC ( ibdev, "IBDEV %p QPN %lx %s %08x:%08x:%08x:%08x qkey %lx\n",
101 	       ibdev, qp->qpn, ( joined ? "joined" : "left" ),
102 	       ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ),
103 	       ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ),
104 	       qkey );
105 
106 	/* Set queue key */
107 	qp->qkey = qkey;
108 	if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
109 		DBGC ( ibdev, "IBDEV %p QPN %lx could not modify qkey: %s\n",
110 		       ibdev, qp->qpn, strerror ( rc ) );
111 		goto out;
112 	}
113 
114  out:
115 	/* Destroy the completed transaction */
116 	ib_destroy_madx ( ibdev, mi, madx );
117 	membership->madx = NULL;
118 
119 	/* Hand off to upper completion handler */
120 	membership->complete ( ibdev, qp, membership, rc, mad );
121 }
122 
123 /** Multicast membership management transaction completion operations */
124 static struct ib_mad_transaction_operations ib_mcast_op = {
125 	.complete = ib_mcast_complete,
126 };
127 
128 /**
129  * Join multicast group
130  *
131  * @v ibdev		Infiniband device
132  * @v qp		Queue pair
133  * @v membership	Multicast group membership
134  * @v gid		Multicast GID to join
135  * @v joined		Join completion handler
136  * @ret rc		Return status code
137  */
ib_mcast_join(struct ib_device * ibdev,struct ib_queue_pair * qp,struct ib_mc_membership * membership,struct ib_gid * gid,void (* complete)(struct ib_device * ibdev,struct ib_queue_pair * qp,struct ib_mc_membership * membership,int rc,union ib_mad * mad))138 int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
139 		    struct ib_mc_membership *membership, struct ib_gid *gid,
140 		    void ( * complete ) ( struct ib_device *ibdev,
141 					  struct ib_queue_pair *qp,
142 					  struct ib_mc_membership *membership,
143 					  int rc, union ib_mad *mad ) ) {
144 	union ib_mad mad;
145 	int rc;
146 
147 	DBGC ( ibdev, "IBDEV %p QPN %lx joining %08x:%08x:%08x:%08x\n",
148 	       ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ),
149 	       ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
150 	       ntohl ( gid->u.dwords[3] ) );
151 
152 	/* Initialise structure */
153 	membership->qp = qp;
154 	memcpy ( &membership->gid, gid, sizeof ( membership->gid ) );
155 	membership->complete = complete;
156 
157 	/* Attach queue pair to multicast GID */
158 	if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) {
159 		DBGC ( ibdev, "IBDEV %p QPN %lx could not attach: %s\n",
160 		       ibdev, qp->qpn, strerror ( rc ) );
161 		goto err_mcast_attach;
162 	}
163 
164 	/* Initiate multicast membership join */
165 	ib_mcast_mad ( ibdev, gid, 1, &mad );
166 	membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
167 					    &ib_mcast_op );
168 	if ( ! membership->madx ) {
169 		DBGC ( ibdev, "IBDEV %p QPN %lx could not create join "
170 		       "transaction\n", ibdev, qp->qpn );
171 		rc = -ENOMEM;
172 		goto err_create_madx;
173 	}
174 	ib_madx_set_ownerdata ( membership->madx, membership );
175 
176 	return 0;
177 
178 	ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
179  err_create_madx:
180 	ib_mcast_detach ( ibdev, qp, gid );
181  err_mcast_attach:
182 	return rc;
183 }
184 
185 /**
186  * Leave multicast group
187  *
188  * @v ibdev		Infiniband device
189  * @v qp		Queue pair
190  * @v membership	Multicast group membership
191  */
ib_mcast_leave(struct ib_device * ibdev,struct ib_queue_pair * qp,struct ib_mc_membership * membership)192 void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
193 		      struct ib_mc_membership *membership ) {
194 	struct ib_gid *gid = &membership->gid;
195 	union ib_mad mad;
196 	int rc;
197 
198 	DBGC ( ibdev, "IBDEV %p QPN %lx leaving %08x:%08x:%08x:%08x\n",
199 	       ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ),
200 	       ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
201 	       ntohl ( gid->u.dwords[3] ) );
202 
203 	/* Detach from multicast GID */
204 	ib_mcast_detach ( ibdev, qp, &membership->gid );
205 
206 	/* Cancel multicast membership join, if applicable */
207 	if ( membership->madx ) {
208 		ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
209 		membership->madx = NULL;
210 	}
211 
212 	/* Send a single group leave MAD */
213 	ib_mcast_mad ( ibdev, &membership->gid, 0, &mad );
214 	if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
215 		DBGC ( ibdev, "IBDEV %p QPN %lx could not send leave request: "
216 		       "%s\n", ibdev, qp->qpn, strerror ( rc ) );
217 	}
218 }
219