1 /*
2 * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License as
6 * published by the Free Software Foundation; either version 2 of the
7 * License, or any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17 */
18
19 FILE_LICENCE ( GPL2_OR_LATER );
20
21 #include <stdint.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <byteswap.h>
25 #include <errno.h>
26 #include <gpxe/infiniband.h>
27 #include <gpxe/ib_mi.h>
28 #include <gpxe/ib_pathrec.h>
29
30 /** @file
31 *
32 * Infiniband path lookups
33 *
34 */
35
36 /**
37 * Handle path transaction completion
38 *
39 * @v ibdev Infiniband device
40 * @v mi Management interface
41 * @v madx Management transaction
42 * @v rc Status code
43 * @v mad Received MAD (or NULL on error)
44 * @v av Source address vector (or NULL on error)
45 */
ib_path_complete(struct ib_device * ibdev,struct ib_mad_interface * mi,struct ib_mad_transaction * madx,int rc,union ib_mad * mad,struct ib_address_vector * av __unused)46 static void ib_path_complete ( struct ib_device *ibdev,
47 struct ib_mad_interface *mi,
48 struct ib_mad_transaction *madx,
49 int rc, union ib_mad *mad,
50 struct ib_address_vector *av __unused ) {
51 struct ib_path *path = ib_madx_get_ownerdata ( madx );
52 struct ib_gid *dgid = &path->av.gid;
53 struct ib_path_record *pathrec = &mad->sa.sa_data.path_record;
54
55 /* Report failures */
56 if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
57 rc = -ENETUNREACH;
58 if ( rc != 0 ) {
59 DBGC ( ibdev, "IBDEV %p path lookup for %08x:%08x:%08x:%08x "
60 "failed: %s\n", ibdev, htonl ( dgid->u.dwords[0] ),
61 htonl ( dgid->u.dwords[1] ),
62 htonl ( dgid->u.dwords[2] ),
63 htonl ( dgid->u.dwords[3] ), strerror ( rc ) );
64 goto out;
65 }
66
67 /* Extract values from MAD */
68 path->av.lid = ntohs ( pathrec->dlid );
69 path->av.sl = ( pathrec->reserved__sl & 0x0f );
70 path->av.rate = ( pathrec->rate_selector__rate & 0x3f );
71 DBGC ( ibdev, "IBDEV %p path to %08x:%08x:%08x:%08x is %04x sl %d "
72 "rate %d\n", ibdev, htonl ( dgid->u.dwords[0] ),
73 htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ),
74 htonl ( dgid->u.dwords[3] ), path->av.lid, path->av.sl,
75 path->av.rate );
76
77 out:
78 /* Destroy the completed transaction */
79 ib_destroy_madx ( ibdev, mi, madx );
80 path->madx = NULL;
81
82 /* Hand off to upper completion handler */
83 path->op->complete ( ibdev, path, rc, &path->av );
84 }
85
86 /** Path transaction completion operations */
87 static struct ib_mad_transaction_operations ib_path_op = {
88 .complete = ib_path_complete,
89 };
90
91 /**
92 * Create path
93 *
94 * @v ibdev Infiniband device
95 * @v av Address vector to complete
96 * @v op Path operations
97 * @ret path Path
98 */
99 struct ib_path *
ib_create_path(struct ib_device * ibdev,struct ib_address_vector * av,struct ib_path_operations * op)100 ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av,
101 struct ib_path_operations *op ) {
102 struct ib_path *path;
103 union ib_mad mad;
104 struct ib_mad_sa *sa = &mad.sa;
105
106 /* Allocate and initialise structure */
107 path = zalloc ( sizeof ( *path ) );
108 if ( ! path )
109 goto err_alloc_path;
110 path->ibdev = ibdev;
111 memcpy ( &path->av, av, sizeof ( path->av ) );
112 path->op = op;
113
114 /* Construct path request */
115 memset ( sa, 0, sizeof ( *sa ) );
116 sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
117 sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
118 sa->mad_hdr.method = IB_MGMT_METHOD_GET;
119 sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
120 sa->sa_hdr.comp_mask[1] =
121 htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
122 memcpy ( &sa->sa_data.path_record.dgid, &path->av.gid,
123 sizeof ( sa->sa_data.path_record.dgid ) );
124 memcpy ( &sa->sa_data.path_record.sgid, &ibdev->gid,
125 sizeof ( sa->sa_data.path_record.sgid ) );
126
127 /* Create management transaction */
128 path->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
129 &ib_path_op );
130 if ( ! path->madx )
131 goto err_create_madx;
132 ib_madx_set_ownerdata ( path->madx, path );
133
134 return path;
135
136 ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
137 err_create_madx:
138 free ( path );
139 err_alloc_path:
140 return NULL;
141 }
142
143 /**
144 * Destroy path
145 *
146 * @v ibdev Infiniband device
147 * @v path Path
148 */
ib_destroy_path(struct ib_device * ibdev,struct ib_path * path)149 void ib_destroy_path ( struct ib_device *ibdev, struct ib_path *path ) {
150
151 if ( path->madx )
152 ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
153 free ( path );
154 }
155
156 /** Number of path cache entries
157 *
158 * Must be a power of two.
159 */
160 #define IB_NUM_CACHED_PATHS 4
161
162 /** A cached path */
163 struct ib_cached_path {
164 /** Path */
165 struct ib_path *path;
166 };
167
168 /** Path cache */
169 static struct ib_cached_path ib_path_cache[IB_NUM_CACHED_PATHS];
170
171 /** Oldest path cache entry index */
172 static unsigned int ib_path_cache_idx;
173
174 /**
175 * Find path cache entry
176 *
177 * @v ibdev Infiniband device
178 * @v dgid Destination GID
179 * @ret path Path cache entry, or NULL
180 */
181 static struct ib_cached_path *
ib_find_path_cache_entry(struct ib_device * ibdev,struct ib_gid * dgid)182 ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) {
183 struct ib_cached_path *cached;
184 unsigned int i;
185
186 for ( i = 0 ; i < IB_NUM_CACHED_PATHS ; i++ ) {
187 cached = &ib_path_cache[i];
188 if ( ! cached->path )
189 continue;
190 if ( cached->path->ibdev != ibdev )
191 continue;
192 if ( memcmp ( &cached->path->av.gid, dgid,
193 sizeof ( cached->path->av.gid ) ) != 0 )
194 continue;
195 return cached;
196 }
197
198 return NULL;
199 }
200
201 /**
202 * Handle cached path transaction completion
203 *
204 * @v ibdev Infiniband device
205 * @v path Path
206 * @v rc Status code
207 * @v av Address vector, or NULL on error
208 */
ib_cached_path_complete(struct ib_device * ibdev,struct ib_path * path,int rc,struct ib_address_vector * av __unused)209 static void ib_cached_path_complete ( struct ib_device *ibdev,
210 struct ib_path *path, int rc,
211 struct ib_address_vector *av __unused ) {
212 struct ib_cached_path *cached = ib_path_get_ownerdata ( path );
213
214 /* If the transaction failed, erase the cache entry */
215 if ( rc != 0 ) {
216 /* Destroy the old cache entry */
217 ib_destroy_path ( ibdev, path );
218 memset ( cached, 0, sizeof ( *cached ) );
219 return;
220 }
221
222 /* Do not destroy the completed transaction; we still need to
223 * refer to the resolved path.
224 */
225 }
226
227 /** Cached path transaction completion operations */
228 static struct ib_path_operations ib_cached_path_op = {
229 .complete = ib_cached_path_complete,
230 };
231
232 /**
233 * Resolve path
234 *
235 * @v ibdev Infiniband device
236 * @v av Address vector to complete
237 * @ret rc Return status code
238 *
239 * This provides a non-transactional way to resolve a path, via a
240 * cache similar to ARP.
241 */
ib_resolve_path(struct ib_device * ibdev,struct ib_address_vector * av)242 int ib_resolve_path ( struct ib_device *ibdev, struct ib_address_vector *av ) {
243 struct ib_gid *gid = &av->gid;
244 struct ib_cached_path *cached;
245 unsigned int cache_idx;
246
247 /* Sanity check */
248 if ( ! av->gid_present ) {
249 DBGC ( ibdev, "IBDEV %p attempt to look up path "
250 "without GID\n", ibdev );
251 return -EINVAL;
252 }
253
254 /* Look in cache for a matching entry */
255 cached = ib_find_path_cache_entry ( ibdev, gid );
256 if ( cached && cached->path->av.lid ) {
257 /* Populated entry found */
258 av->lid = cached->path->av.lid;
259 av->rate = cached->path->av.rate;
260 av->sl = cached->path->av.sl;
261 DBGC2 ( ibdev, "IBDEV %p cache hit for %08x:%08x:%08x:%08x\n",
262 ibdev, htonl ( gid->u.dwords[0] ),
263 htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ),
264 htonl ( gid->u.dwords[3] ) );
265 return 0;
266 }
267 DBGC ( ibdev, "IBDEV %p cache miss for %08x:%08x:%08x:%08x%s\n",
268 ibdev, htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ),
269 htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ),
270 ( cached ? " (in progress)" : "" ) );
271
272 /* If lookup is already in progress, do nothing */
273 if ( cached )
274 return -ENOENT;
275
276 /* Locate a new cache entry to use */
277 cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS );
278 cached = &ib_path_cache[cache_idx];
279
280 /* Destroy the old cache entry */
281 if ( cached->path )
282 ib_destroy_path ( ibdev, cached->path );
283 memset ( cached, 0, sizeof ( *cached ) );
284
285 /* Create new path */
286 cached->path = ib_create_path ( ibdev, av, &ib_cached_path_op );
287 if ( ! cached->path ) {
288 DBGC ( ibdev, "IBDEV %p could not create path\n",
289 ibdev );
290 return -ENOMEM;
291 }
292 ib_path_set_ownerdata ( cached->path, cached );
293
294 /* Not found yet */
295 return -ENOENT;
296 }
297