1 /*
2 * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License as
6 * published by the Free Software Foundation; either version 2 of the
7 * License, or any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17 */
18
19 FILE_LICENCE ( GPL2_OR_LATER );
20
21 #include <stdint.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <errno.h>
25 #include <byteswap.h>
26 #include <gpxe/iobuf.h>
27 #include <gpxe/infiniband.h>
28 #include <gpxe/ib_packet.h>
29
30 /**
31 * @file
32 *
33 * Infiniband Packet Formats
34 *
35 */
36
37 /**
38 * Add IB headers
39 *
40 * @v ibdev Infiniband device
41 * @v iobuf I/O buffer to contain headers
42 * @v qp Queue pair
43 * @v payload_len Payload length
44 * @v av Address vector
45 */
ib_push(struct ib_device * ibdev,struct io_buffer * iobuf,struct ib_queue_pair * qp,size_t payload_len,const struct ib_address_vector * av)46 int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
47 struct ib_queue_pair *qp, size_t payload_len,
48 const struct ib_address_vector *av ) {
49 struct ib_local_route_header *lrh;
50 struct ib_global_route_header *grh;
51 struct ib_base_transport_header *bth;
52 struct ib_datagram_extended_transport_header *deth;
53 size_t orig_iob_len = iob_len ( iobuf );
54 size_t pad_len;
55 size_t lrh_len;
56 size_t grh_len;
57 unsigned int vl;
58 unsigned int lnh;
59
60 DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
61 ibdev, ibdev->lid, qp->ext_qpn, av->lid, av->qpn, av->qkey );
62
63 /* Calculate packet length */
64 pad_len = ( (-payload_len) & 0x3 );
65 payload_len += pad_len;
66 payload_len += 4; /* ICRC */
67
68 /* Reserve space for headers */
69 orig_iob_len = iob_len ( iobuf );
70 deth = iob_push ( iobuf, sizeof ( *deth ) );
71 bth = iob_push ( iobuf, sizeof ( *bth ) );
72 grh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
73 grh = ( av->gid_present ?
74 iob_push ( iobuf, sizeof ( *grh ) ) : NULL );
75 lrh = iob_push ( iobuf, sizeof ( *lrh ) );
76 lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
77
78 /* Construct LRH */
79 vl = ( ( qp->ext_qpn == IB_QPN_SMI ) ? IB_VL_SMP : IB_VL_DEFAULT );
80 lrh->vl__lver = ( vl << 4 );
81 lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
82 lrh->sl__lnh = ( ( av->sl << 4 ) | lnh );
83 lrh->dlid = htons ( av->lid );
84 lrh->length = htons ( lrh_len >> 2 );
85 lrh->slid = htons ( ibdev->lid );
86
87 /* Construct GRH, if required */
88 if ( grh ) {
89 grh->ipver__tclass__flowlabel =
90 htonl ( IB_GRH_IPVER_IPv6 << 28 );
91 grh->paylen = htons ( grh_len );
92 grh->nxthdr = IB_GRH_NXTHDR_IBA;
93 grh->hoplmt = 0;
94 memcpy ( &grh->sgid, &ibdev->gid, sizeof ( grh->sgid ) );
95 memcpy ( &grh->dgid, &av->gid, sizeof ( grh->dgid ) );
96 }
97
98 /* Construct BTH */
99 bth->opcode = BTH_OPCODE_UD_SEND;
100 bth->se__m__padcnt__tver = ( pad_len << 4 );
101 bth->pkey = htons ( ibdev->pkey );
102 bth->dest_qp = htonl ( av->qpn );
103 bth->ack__psn = htonl ( ( qp->send.psn++ ) & 0xffffffUL );
104
105 /* Construct DETH */
106 deth->qkey = htonl ( av->qkey );
107 deth->src_qp = htonl ( qp->ext_qpn );
108
109 DBGCP_HDA ( ibdev, 0, iobuf->data,
110 ( iob_len ( iobuf ) - orig_iob_len ) );
111
112 return 0;
113 }
114
115 /**
116 * Remove IB headers
117 *
118 * @v ibdev Infiniband device
119 * @v iobuf I/O buffer containing headers
120 * @v qp Queue pair to fill in, or NULL
121 * @v payload_len Payload length to fill in, or NULL
122 * @v av Address vector to fill in
123 */
ib_pull(struct ib_device * ibdev,struct io_buffer * iobuf,struct ib_queue_pair ** qp,size_t * payload_len,struct ib_address_vector * av)124 int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
125 struct ib_queue_pair **qp, size_t *payload_len,
126 struct ib_address_vector *av ) {
127 struct ib_local_route_header *lrh;
128 struct ib_global_route_header *grh;
129 struct ib_base_transport_header *bth;
130 struct ib_datagram_extended_transport_header *deth;
131 size_t orig_iob_len = iob_len ( iobuf );
132 unsigned int lnh;
133 size_t pad_len;
134 unsigned long qpn;
135 unsigned int lid;
136
137 /* Clear return values */
138 if ( qp )
139 *qp = NULL;
140 if ( payload_len )
141 *payload_len = 0;
142 memset ( av, 0, sizeof ( *av ) );
143
144 /* Extract LRH */
145 if ( iob_len ( iobuf ) < sizeof ( *lrh ) ) {
146 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for LRH\n",
147 ibdev, iob_len ( iobuf ) );
148 return -EINVAL;
149 }
150 lrh = iobuf->data;
151 iob_pull ( iobuf, sizeof ( *lrh ) );
152 av->lid = ntohs ( lrh->slid );
153 av->sl = ( lrh->sl__lnh >> 4 );
154 lnh = ( lrh->sl__lnh & 0x3 );
155 lid = ntohs ( lrh->dlid );
156
157 /* Reject unsupported packets */
158 if ( ! ( ( lnh == IB_LNH_BTH ) || ( lnh == IB_LNH_GRH ) ) ) {
159 DBGC ( ibdev, "IBDEV %p RX unsupported LNH %x\n",
160 ibdev, lnh );
161 return -ENOTSUP;
162 }
163
164 /* Extract GRH, if present */
165 if ( lnh == IB_LNH_GRH ) {
166 if ( iob_len ( iobuf ) < sizeof ( *grh ) ) {
167 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) "
168 "for GRH\n", ibdev, iob_len ( iobuf ) );
169 return -EINVAL;
170 }
171 grh = iobuf->data;
172 iob_pull ( iobuf, sizeof ( *grh ) );
173 av->gid_present = 1;
174 memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) );
175 } else {
176 grh = NULL;
177 }
178
179 /* Extract BTH */
180 if ( iob_len ( iobuf ) < sizeof ( *bth ) ) {
181 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for BTH\n",
182 ibdev, iob_len ( iobuf ) );
183 return -EINVAL;
184 }
185 bth = iobuf->data;
186 iob_pull ( iobuf, sizeof ( *bth ) );
187 if ( bth->opcode != BTH_OPCODE_UD_SEND ) {
188 DBGC ( ibdev, "IBDEV %p unsupported BTH opcode %x\n",
189 ibdev, bth->opcode );
190 return -ENOTSUP;
191 }
192 qpn = ntohl ( bth->dest_qp );
193
194 /* Extract DETH */
195 if ( iob_len ( iobuf ) < sizeof ( *deth ) ) {
196 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for DETH\n",
197 ibdev, iob_len ( iobuf ) );
198 return -EINVAL;
199 }
200 deth = iobuf->data;
201 iob_pull ( iobuf, sizeof ( *deth ) );
202 av->qpn = ntohl ( deth->src_qp );
203 av->qkey = ntohl ( deth->qkey );
204
205 /* Calculate payload length, if applicable */
206 if ( payload_len ) {
207 pad_len = ( ( bth->se__m__padcnt__tver >> 4 ) & 0x3 );
208 *payload_len = ( ( ntohs ( lrh->length ) << 2 )
209 - ( orig_iob_len - iob_len ( iobuf ) )
210 - pad_len - 4 /* ICRC */ );
211 }
212
213 /* Determine destination QP, if applicable */
214 if ( qp ) {
215 if ( IB_LID_MULTICAST ( lid ) && grh ) {
216 if ( ! ( *qp = ib_find_qp_mgid ( ibdev, &grh->dgid ))){
217 DBGC ( ibdev, "IBDEV %p RX for unknown MGID "
218 "%08x:%08x:%08x:%08x\n", ibdev,
219 ntohl ( grh->dgid.u.dwords[0] ),
220 ntohl ( grh->dgid.u.dwords[1] ),
221 ntohl ( grh->dgid.u.dwords[2] ),
222 ntohl ( grh->dgid.u.dwords[3] ) );
223 return -ENODEV;
224 }
225 } else {
226 if ( ! ( *qp = ib_find_qp_qpn ( ibdev, qpn ) ) ) {
227 DBGC ( ibdev, "IBDEV %p RX for nonexistent "
228 "QPN %lx\n", ibdev, qpn );
229 return -ENODEV;
230 }
231 }
232 assert ( *qp );
233 }
234
235 DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08x)\n",
236 ibdev, lid, ( IB_LID_MULTICAST( lid ) ?
237 ( qp ? (*qp)->ext_qpn : -1UL ) : qpn ),
238 av->lid, av->qpn, ntohl ( deth->qkey ) );
239 DBGCP_HDA ( ibdev, 0,
240 ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),
241 ( orig_iob_len - iob_len ( iobuf ) ) );
242
243 return 0;
244 }
245