1 /*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 *
36 */
37
38 #include <linux/delay.h>
39
40 #include "c2.h"
41 #include "c2_vq.h"
42 #include "c2_status.h"
43
44 #define C2_MAX_ORD_PER_QP 128
45 #define C2_MAX_IRD_PER_QP 128
46
47 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
48 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
49 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
50
51 #define NO_SUPPORT -1
52 static const u8 c2_opcode[] = {
53 [IB_WR_SEND] = C2_WR_TYPE_SEND,
54 [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
55 [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
56 [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
57 [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
58 [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
59 [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
60 };
61
to_c2_state(enum ib_qp_state ib_state)62 static int to_c2_state(enum ib_qp_state ib_state)
63 {
64 switch (ib_state) {
65 case IB_QPS_RESET:
66 return C2_QP_STATE_IDLE;
67 case IB_QPS_RTS:
68 return C2_QP_STATE_RTS;
69 case IB_QPS_SQD:
70 return C2_QP_STATE_CLOSING;
71 case IB_QPS_SQE:
72 return C2_QP_STATE_CLOSING;
73 case IB_QPS_ERR:
74 return C2_QP_STATE_ERROR;
75 default:
76 return -1;
77 }
78 }
79
to_ib_state(enum c2_qp_state c2_state)80 static int to_ib_state(enum c2_qp_state c2_state)
81 {
82 switch (c2_state) {
83 case C2_QP_STATE_IDLE:
84 return IB_QPS_RESET;
85 case C2_QP_STATE_CONNECTING:
86 return IB_QPS_RTR;
87 case C2_QP_STATE_RTS:
88 return IB_QPS_RTS;
89 case C2_QP_STATE_CLOSING:
90 return IB_QPS_SQD;
91 case C2_QP_STATE_ERROR:
92 return IB_QPS_ERR;
93 case C2_QP_STATE_TERMINATE:
94 return IB_QPS_SQE;
95 default:
96 return -1;
97 }
98 }
99
to_ib_state_str(int ib_state)100 static const char *to_ib_state_str(int ib_state)
101 {
102 static const char *state_str[] = {
103 "IB_QPS_RESET",
104 "IB_QPS_INIT",
105 "IB_QPS_RTR",
106 "IB_QPS_RTS",
107 "IB_QPS_SQD",
108 "IB_QPS_SQE",
109 "IB_QPS_ERR"
110 };
111 if (ib_state < IB_QPS_RESET ||
112 ib_state > IB_QPS_ERR)
113 return "<invalid IB QP state>";
114
115 ib_state -= IB_QPS_RESET;
116 return state_str[ib_state];
117 }
118
c2_set_qp_state(struct c2_qp * qp,int c2_state)119 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
120 {
121 int new_state = to_ib_state(c2_state);
122
123 pr_debug("%s: qp[%p] state modify %s --> %s\n",
124 __func__,
125 qp,
126 to_ib_state_str(qp->state),
127 to_ib_state_str(new_state));
128 qp->state = new_state;
129 }
130
131 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
132
c2_qp_modify(struct c2_dev * c2dev,struct c2_qp * qp,struct ib_qp_attr * attr,int attr_mask)133 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
134 struct ib_qp_attr *attr, int attr_mask)
135 {
136 struct c2wr_qp_modify_req wr;
137 struct c2wr_qp_modify_rep *reply;
138 struct c2_vq_req *vq_req;
139 unsigned long flags;
140 u8 next_state;
141 int err;
142
143 pr_debug("%s:%d qp=%p, %s --> %s\n",
144 __func__, __LINE__,
145 qp,
146 to_ib_state_str(qp->state),
147 to_ib_state_str(attr->qp_state));
148
149 vq_req = vq_req_alloc(c2dev);
150 if (!vq_req)
151 return -ENOMEM;
152
153 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
154 wr.hdr.context = (unsigned long) vq_req;
155 wr.rnic_handle = c2dev->adapter_handle;
156 wr.qp_handle = qp->adapter_handle;
157 wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
158 wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
159 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
160 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
161
162 if (attr_mask & IB_QP_STATE) {
163 /* Ensure the state is valid */
164 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) {
165 err = -EINVAL;
166 goto bail0;
167 }
168
169 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
170
171 if (attr->qp_state == IB_QPS_ERR) {
172 spin_lock_irqsave(&qp->lock, flags);
173 if (qp->cm_id && qp->state == IB_QPS_RTS) {
174 pr_debug("Generating CLOSE event for QP-->ERR, "
175 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
176 /* Generate an CLOSE event */
177 vq_req->cm_id = qp->cm_id;
178 vq_req->event = IW_CM_EVENT_CLOSE;
179 }
180 spin_unlock_irqrestore(&qp->lock, flags);
181 }
182 next_state = attr->qp_state;
183
184 } else if (attr_mask & IB_QP_CUR_STATE) {
185
186 if (attr->cur_qp_state != IB_QPS_RTR &&
187 attr->cur_qp_state != IB_QPS_RTS &&
188 attr->cur_qp_state != IB_QPS_SQD &&
189 attr->cur_qp_state != IB_QPS_SQE) {
190 err = -EINVAL;
191 goto bail0;
192 } else
193 wr.next_qp_state =
194 cpu_to_be32(to_c2_state(attr->cur_qp_state));
195
196 next_state = attr->cur_qp_state;
197
198 } else {
199 err = 0;
200 goto bail0;
201 }
202
203 /* reference the request struct */
204 vq_req_get(c2dev, vq_req);
205
206 err = vq_send_wr(c2dev, (union c2wr *) & wr);
207 if (err) {
208 vq_req_put(c2dev, vq_req);
209 goto bail0;
210 }
211
212 err = vq_wait_for_reply(c2dev, vq_req);
213 if (err)
214 goto bail0;
215
216 reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
217 if (!reply) {
218 err = -ENOMEM;
219 goto bail0;
220 }
221
222 err = c2_errno(reply);
223 if (!err)
224 qp->state = next_state;
225 #ifdef DEBUG
226 else
227 pr_debug("%s: c2_errno=%d\n", __func__, err);
228 #endif
229 /*
230 * If we're going to error and generating the event here, then
231 * we need to remove the reference because there will be no
232 * close event generated by the adapter
233 */
234 spin_lock_irqsave(&qp->lock, flags);
235 if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
236 qp->cm_id->rem_ref(qp->cm_id);
237 qp->cm_id = NULL;
238 }
239 spin_unlock_irqrestore(&qp->lock, flags);
240
241 vq_repbuf_free(c2dev, reply);
242 bail0:
243 vq_req_free(c2dev, vq_req);
244
245 pr_debug("%s:%d qp=%p, cur_state=%s\n",
246 __func__, __LINE__,
247 qp,
248 to_ib_state_str(qp->state));
249 return err;
250 }
251
c2_qp_set_read_limits(struct c2_dev * c2dev,struct c2_qp * qp,int ord,int ird)252 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
253 int ord, int ird)
254 {
255 struct c2wr_qp_modify_req wr;
256 struct c2wr_qp_modify_rep *reply;
257 struct c2_vq_req *vq_req;
258 int err;
259
260 vq_req = vq_req_alloc(c2dev);
261 if (!vq_req)
262 return -ENOMEM;
263
264 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
265 wr.hdr.context = (unsigned long) vq_req;
266 wr.rnic_handle = c2dev->adapter_handle;
267 wr.qp_handle = qp->adapter_handle;
268 wr.ord = cpu_to_be32(ord);
269 wr.ird = cpu_to_be32(ird);
270 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
271 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
272 wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
273
274 /* reference the request struct */
275 vq_req_get(c2dev, vq_req);
276
277 err = vq_send_wr(c2dev, (union c2wr *) & wr);
278 if (err) {
279 vq_req_put(c2dev, vq_req);
280 goto bail0;
281 }
282
283 err = vq_wait_for_reply(c2dev, vq_req);
284 if (err)
285 goto bail0;
286
287 reply = (struct c2wr_qp_modify_rep *) (unsigned long)
288 vq_req->reply_msg;
289 if (!reply) {
290 err = -ENOMEM;
291 goto bail0;
292 }
293
294 err = c2_errno(reply);
295 vq_repbuf_free(c2dev, reply);
296 bail0:
297 vq_req_free(c2dev, vq_req);
298 return err;
299 }
300
destroy_qp(struct c2_dev * c2dev,struct c2_qp * qp)301 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
302 {
303 struct c2_vq_req *vq_req;
304 struct c2wr_qp_destroy_req wr;
305 struct c2wr_qp_destroy_rep *reply;
306 unsigned long flags;
307 int err;
308
309 /*
310 * Allocate a verb request message
311 */
312 vq_req = vq_req_alloc(c2dev);
313 if (!vq_req) {
314 return -ENOMEM;
315 }
316
317 /*
318 * Initialize the WR
319 */
320 c2_wr_set_id(&wr, CCWR_QP_DESTROY);
321 wr.hdr.context = (unsigned long) vq_req;
322 wr.rnic_handle = c2dev->adapter_handle;
323 wr.qp_handle = qp->adapter_handle;
324
325 /*
326 * reference the request struct. dereferenced in the int handler.
327 */
328 vq_req_get(c2dev, vq_req);
329
330 spin_lock_irqsave(&qp->lock, flags);
331 if (qp->cm_id && qp->state == IB_QPS_RTS) {
332 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
333 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
334 /* Generate an CLOSE event */
335 vq_req->qp = qp;
336 vq_req->cm_id = qp->cm_id;
337 vq_req->event = IW_CM_EVENT_CLOSE;
338 }
339 spin_unlock_irqrestore(&qp->lock, flags);
340
341 /*
342 * Send WR to adapter
343 */
344 err = vq_send_wr(c2dev, (union c2wr *) & wr);
345 if (err) {
346 vq_req_put(c2dev, vq_req);
347 goto bail0;
348 }
349
350 /*
351 * Wait for reply from adapter
352 */
353 err = vq_wait_for_reply(c2dev, vq_req);
354 if (err) {
355 goto bail0;
356 }
357
358 /*
359 * Process reply
360 */
361 reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
362 if (!reply) {
363 err = -ENOMEM;
364 goto bail0;
365 }
366
367 spin_lock_irqsave(&qp->lock, flags);
368 if (qp->cm_id) {
369 qp->cm_id->rem_ref(qp->cm_id);
370 qp->cm_id = NULL;
371 }
372 spin_unlock_irqrestore(&qp->lock, flags);
373
374 vq_repbuf_free(c2dev, reply);
375 bail0:
376 vq_req_free(c2dev, vq_req);
377 return err;
378 }
379
c2_alloc_qpn(struct c2_dev * c2dev,struct c2_qp * qp)380 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
381 {
382 int ret;
383
384 do {
385 spin_lock_irq(&c2dev->qp_table.lock);
386 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
387 c2dev->qp_table.last++, &qp->qpn);
388 spin_unlock_irq(&c2dev->qp_table.lock);
389 } while ((ret == -EAGAIN) &&
390 idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
391 return ret;
392 }
393
c2_free_qpn(struct c2_dev * c2dev,int qpn)394 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
395 {
396 spin_lock_irq(&c2dev->qp_table.lock);
397 idr_remove(&c2dev->qp_table.idr, qpn);
398 spin_unlock_irq(&c2dev->qp_table.lock);
399 }
400
c2_find_qpn(struct c2_dev * c2dev,int qpn)401 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
402 {
403 unsigned long flags;
404 struct c2_qp *qp;
405
406 spin_lock_irqsave(&c2dev->qp_table.lock, flags);
407 qp = idr_find(&c2dev->qp_table.idr, qpn);
408 spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
409 return qp;
410 }
411
c2_alloc_qp(struct c2_dev * c2dev,struct c2_pd * pd,struct ib_qp_init_attr * qp_attrs,struct c2_qp * qp)412 int c2_alloc_qp(struct c2_dev *c2dev,
413 struct c2_pd *pd,
414 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
415 {
416 struct c2wr_qp_create_req wr;
417 struct c2wr_qp_create_rep *reply;
418 struct c2_vq_req *vq_req;
419 struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
420 struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
421 unsigned long peer_pa;
422 u32 q_size, msg_size, mmap_size;
423 void __iomem *mmap;
424 int err;
425
426 err = c2_alloc_qpn(c2dev, qp);
427 if (err)
428 return err;
429 qp->ibqp.qp_num = qp->qpn;
430 qp->ibqp.qp_type = IB_QPT_RC;
431
432 /* Allocate the SQ and RQ shared pointers */
433 qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
434 &qp->sq_mq.shared_dma, GFP_KERNEL);
435 if (!qp->sq_mq.shared) {
436 err = -ENOMEM;
437 goto bail0;
438 }
439
440 qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
441 &qp->rq_mq.shared_dma, GFP_KERNEL);
442 if (!qp->rq_mq.shared) {
443 err = -ENOMEM;
444 goto bail1;
445 }
446
447 /* Allocate the verbs request */
448 vq_req = vq_req_alloc(c2dev);
449 if (vq_req == NULL) {
450 err = -ENOMEM;
451 goto bail2;
452 }
453
454 /* Initialize the work request */
455 memset(&wr, 0, sizeof(wr));
456 c2_wr_set_id(&wr, CCWR_QP_CREATE);
457 wr.hdr.context = (unsigned long) vq_req;
458 wr.rnic_handle = c2dev->adapter_handle;
459 wr.sq_cq_handle = send_cq->adapter_handle;
460 wr.rq_cq_handle = recv_cq->adapter_handle;
461 wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
462 wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
463 wr.srq_handle = 0;
464 wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
465 QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
466 wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
467 wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
468 wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
469 wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
470 wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
471 wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
472 wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
473 wr.pd_id = pd->pd_id;
474 wr.user_context = (unsigned long) qp;
475
476 vq_req_get(c2dev, vq_req);
477
478 /* Send the WR to the adapter */
479 err = vq_send_wr(c2dev, (union c2wr *) & wr);
480 if (err) {
481 vq_req_put(c2dev, vq_req);
482 goto bail3;
483 }
484
485 /* Wait for the verb reply */
486 err = vq_wait_for_reply(c2dev, vq_req);
487 if (err) {
488 goto bail3;
489 }
490
491 /* Process the reply */
492 reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
493 if (!reply) {
494 err = -ENOMEM;
495 goto bail3;
496 }
497
498 if ((err = c2_wr_get_result(reply)) != 0) {
499 goto bail4;
500 }
501
502 /* Fill in the kernel QP struct */
503 atomic_set(&qp->refcount, 1);
504 qp->adapter_handle = reply->qp_handle;
505 qp->state = IB_QPS_RESET;
506 qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
507 qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
508 qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
509 init_waitqueue_head(&qp->wait);
510
511 /* Initialize the SQ MQ */
512 q_size = be32_to_cpu(reply->sq_depth);
513 msg_size = be32_to_cpu(reply->sq_msg_size);
514 peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
515 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
516 mmap = ioremap_nocache(peer_pa, mmap_size);
517 if (!mmap) {
518 err = -ENOMEM;
519 goto bail5;
520 }
521
522 c2_mq_req_init(&qp->sq_mq,
523 be32_to_cpu(reply->sq_mq_index),
524 q_size,
525 msg_size,
526 mmap + sizeof(struct c2_mq_shared), /* pool start */
527 mmap, /* peer */
528 C2_MQ_ADAPTER_TARGET);
529
530 /* Initialize the RQ mq */
531 q_size = be32_to_cpu(reply->rq_depth);
532 msg_size = be32_to_cpu(reply->rq_msg_size);
533 peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
534 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
535 mmap = ioremap_nocache(peer_pa, mmap_size);
536 if (!mmap) {
537 err = -ENOMEM;
538 goto bail6;
539 }
540
541 c2_mq_req_init(&qp->rq_mq,
542 be32_to_cpu(reply->rq_mq_index),
543 q_size,
544 msg_size,
545 mmap + sizeof(struct c2_mq_shared), /* pool start */
546 mmap, /* peer */
547 C2_MQ_ADAPTER_TARGET);
548
549 vq_repbuf_free(c2dev, reply);
550 vq_req_free(c2dev, vq_req);
551
552 return 0;
553
554 bail6:
555 iounmap(qp->sq_mq.peer);
556 bail5:
557 destroy_qp(c2dev, qp);
558 bail4:
559 vq_repbuf_free(c2dev, reply);
560 bail3:
561 vq_req_free(c2dev, vq_req);
562 bail2:
563 c2_free_mqsp(qp->rq_mq.shared);
564 bail1:
565 c2_free_mqsp(qp->sq_mq.shared);
566 bail0:
567 c2_free_qpn(c2dev, qp->qpn);
568 return err;
569 }
570
c2_lock_cqs(struct c2_cq * send_cq,struct c2_cq * recv_cq)571 static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
572 {
573 if (send_cq == recv_cq)
574 spin_lock_irq(&send_cq->lock);
575 else if (send_cq > recv_cq) {
576 spin_lock_irq(&send_cq->lock);
577 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
578 } else {
579 spin_lock_irq(&recv_cq->lock);
580 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
581 }
582 }
583
c2_unlock_cqs(struct c2_cq * send_cq,struct c2_cq * recv_cq)584 static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
585 {
586 if (send_cq == recv_cq)
587 spin_unlock_irq(&send_cq->lock);
588 else if (send_cq > recv_cq) {
589 spin_unlock(&recv_cq->lock);
590 spin_unlock_irq(&send_cq->lock);
591 } else {
592 spin_unlock(&send_cq->lock);
593 spin_unlock_irq(&recv_cq->lock);
594 }
595 }
596
c2_free_qp(struct c2_dev * c2dev,struct c2_qp * qp)597 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
598 {
599 struct c2_cq *send_cq;
600 struct c2_cq *recv_cq;
601
602 send_cq = to_c2cq(qp->ibqp.send_cq);
603 recv_cq = to_c2cq(qp->ibqp.recv_cq);
604
605 /*
606 * Lock CQs here, so that CQ polling code can do QP lookup
607 * without taking a lock.
608 */
609 c2_lock_cqs(send_cq, recv_cq);
610 c2_free_qpn(c2dev, qp->qpn);
611 c2_unlock_cqs(send_cq, recv_cq);
612
613 /*
614 * Destory qp in the rnic...
615 */
616 destroy_qp(c2dev, qp);
617
618 /*
619 * Mark any unreaped CQEs as null and void.
620 */
621 c2_cq_clean(c2dev, qp, send_cq->cqn);
622 if (send_cq != recv_cq)
623 c2_cq_clean(c2dev, qp, recv_cq->cqn);
624 /*
625 * Unmap the MQs and return the shared pointers
626 * to the message pool.
627 */
628 iounmap(qp->sq_mq.peer);
629 iounmap(qp->rq_mq.peer);
630 c2_free_mqsp(qp->sq_mq.shared);
631 c2_free_mqsp(qp->rq_mq.shared);
632
633 atomic_dec(&qp->refcount);
634 wait_event(qp->wait, !atomic_read(&qp->refcount));
635 }
636
637 /*
638 * Function: move_sgl
639 *
640 * Description:
641 * Move an SGL from the user's work request struct into a CCIL Work Request
642 * message, swapping to WR byte order and ensure the total length doesn't
643 * overflow.
644 *
645 * IN:
646 * dst - ptr to CCIL Work Request message SGL memory.
647 * src - ptr to the consumers SGL memory.
648 *
649 * OUT: none
650 *
651 * Return:
652 * CCIL status codes.
653 */
654 static int
move_sgl(struct c2_data_addr * dst,struct ib_sge * src,int count,u32 * p_len,u8 * actual_count)655 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
656 u8 * actual_count)
657 {
658 u32 tot = 0; /* running total */
659 u8 acount = 0; /* running total non-0 len sge's */
660
661 while (count > 0) {
662 /*
663 * If the addition of this SGE causes the
664 * total SGL length to exceed 2^32-1, then
665 * fail-n-bail.
666 *
667 * If the current total plus the next element length
668 * wraps, then it will go negative and be less than the
669 * current total...
670 */
671 if ((tot + src->length) < tot) {
672 return -EINVAL;
673 }
674 /*
675 * Bug: 1456 (as well as 1498 & 1643)
676 * Skip over any sge's supplied with len=0
677 */
678 if (src->length) {
679 tot += src->length;
680 dst->stag = cpu_to_be32(src->lkey);
681 dst->to = cpu_to_be64(src->addr);
682 dst->length = cpu_to_be32(src->length);
683 dst++;
684 acount++;
685 }
686 src++;
687 count--;
688 }
689
690 if (acount == 0) {
691 /*
692 * Bug: 1476 (as well as 1498, 1456 and 1643)
693 * Setup the SGL in the WR to make it easier for the RNIC.
694 * This way, the FW doesn't have to deal with special cases.
695 * Setting length=0 should be sufficient.
696 */
697 dst->stag = 0;
698 dst->to = 0;
699 dst->length = 0;
700 }
701
702 *p_len = tot;
703 *actual_count = acount;
704 return 0;
705 }
706
707 /*
708 * Function: c2_activity (private function)
709 *
710 * Description:
711 * Post an mq index to the host->adapter activity fifo.
712 *
713 * IN:
714 * c2dev - ptr to c2dev structure
715 * mq_index - mq index to post
716 * shared - value most recently written to shared
717 *
718 * OUT:
719 *
720 * Return:
721 * none
722 */
c2_activity(struct c2_dev * c2dev,u32 mq_index,u16 shared)723 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
724 {
725 /*
726 * First read the register to see if the FIFO is full, and if so,
727 * spin until it's not. This isn't perfect -- there is no
728 * synchronization among the clients of the register, but in
729 * practice it prevents multiple CPU from hammering the bus
730 * with PCI RETRY. Note that when this does happen, the card
731 * cannot get on the bus and the card and system hang in a
732 * deadlock -- thus the need for this code. [TOT]
733 */
734 while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
735 udelay(10);
736
737 __raw_writel(C2_HINT_MAKE(mq_index, shared),
738 c2dev->regs + PCI_BAR0_ADAPTER_HINT);
739 }
740
741 /*
742 * Function: qp_wr_post
743 *
744 * Description:
745 * This in-line function allocates a MQ msg, then moves the host-copy of
746 * the completed WR into msg. Then it posts the message.
747 *
748 * IN:
749 * q - ptr to user MQ.
750 * wr - ptr to host-copy of the WR.
751 * qp - ptr to user qp
752 * size - Number of bytes to post. Assumed to be divisible by 4.
753 *
754 * OUT: none
755 *
756 * Return:
757 * CCIL status codes.
758 */
qp_wr_post(struct c2_mq * q,union c2wr * wr,struct c2_qp * qp,u32 size)759 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
760 {
761 union c2wr *msg;
762
763 msg = c2_mq_alloc(q);
764 if (msg == NULL) {
765 return -EINVAL;
766 }
767 #ifdef CCMSGMAGIC
768 ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
769 #endif
770
771 /*
772 * Since all header fields in the WR are the same as the
773 * CQE, set the following so the adapter need not.
774 */
775 c2_wr_set_result(wr, CCERR_PENDING);
776
777 /*
778 * Copy the wr down to the adapter
779 */
780 memcpy((void *) msg, (void *) wr, size);
781
782 c2_mq_produce(q);
783 return 0;
784 }
785
786
c2_post_send(struct ib_qp * ibqp,struct ib_send_wr * ib_wr,struct ib_send_wr ** bad_wr)787 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
788 struct ib_send_wr **bad_wr)
789 {
790 struct c2_dev *c2dev = to_c2dev(ibqp->device);
791 struct c2_qp *qp = to_c2qp(ibqp);
792 union c2wr wr;
793 unsigned long lock_flags;
794 int err = 0;
795
796 u32 flags;
797 u32 tot_len;
798 u8 actual_sge_count;
799 u32 msg_size;
800
801 if (qp->state > IB_QPS_RTS)
802 return -EINVAL;
803
804 while (ib_wr) {
805
806 flags = 0;
807 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
808 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
809 flags |= SQ_SIGNALED;
810 }
811
812 switch (ib_wr->opcode) {
813 case IB_WR_SEND:
814 case IB_WR_SEND_WITH_INV:
815 if (ib_wr->opcode == IB_WR_SEND) {
816 if (ib_wr->send_flags & IB_SEND_SOLICITED)
817 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
818 else
819 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
820 wr.sqwr.send.remote_stag = 0;
821 } else {
822 if (ib_wr->send_flags & IB_SEND_SOLICITED)
823 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV);
824 else
825 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV);
826 wr.sqwr.send.remote_stag =
827 cpu_to_be32(ib_wr->ex.invalidate_rkey);
828 }
829
830 msg_size = sizeof(struct c2wr_send_req) +
831 sizeof(struct c2_data_addr) * ib_wr->num_sge;
832 if (ib_wr->num_sge > qp->send_sgl_depth) {
833 err = -EINVAL;
834 break;
835 }
836 if (ib_wr->send_flags & IB_SEND_FENCE) {
837 flags |= SQ_READ_FENCE;
838 }
839 err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
840 ib_wr->sg_list,
841 ib_wr->num_sge,
842 &tot_len, &actual_sge_count);
843 wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
844 c2_wr_set_sge_count(&wr, actual_sge_count);
845 break;
846 case IB_WR_RDMA_WRITE:
847 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
848 msg_size = sizeof(struct c2wr_rdma_write_req) +
849 (sizeof(struct c2_data_addr) * ib_wr->num_sge);
850 if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
851 err = -EINVAL;
852 break;
853 }
854 if (ib_wr->send_flags & IB_SEND_FENCE) {
855 flags |= SQ_READ_FENCE;
856 }
857 wr.sqwr.rdma_write.remote_stag =
858 cpu_to_be32(ib_wr->wr.rdma.rkey);
859 wr.sqwr.rdma_write.remote_to =
860 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
861 err = move_sgl((struct c2_data_addr *)
862 & (wr.sqwr.rdma_write.data),
863 ib_wr->sg_list,
864 ib_wr->num_sge,
865 &tot_len, &actual_sge_count);
866 wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
867 c2_wr_set_sge_count(&wr, actual_sge_count);
868 break;
869 case IB_WR_RDMA_READ:
870 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
871 msg_size = sizeof(struct c2wr_rdma_read_req);
872
873 /* IWarp only suppots 1 sge for RDMA reads */
874 if (ib_wr->num_sge > 1) {
875 err = -EINVAL;
876 break;
877 }
878
879 /*
880 * Move the local and remote stag/to/len into the WR.
881 */
882 wr.sqwr.rdma_read.local_stag =
883 cpu_to_be32(ib_wr->sg_list->lkey);
884 wr.sqwr.rdma_read.local_to =
885 cpu_to_be64(ib_wr->sg_list->addr);
886 wr.sqwr.rdma_read.remote_stag =
887 cpu_to_be32(ib_wr->wr.rdma.rkey);
888 wr.sqwr.rdma_read.remote_to =
889 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
890 wr.sqwr.rdma_read.length =
891 cpu_to_be32(ib_wr->sg_list->length);
892 break;
893 default:
894 /* error */
895 msg_size = 0;
896 err = -EINVAL;
897 break;
898 }
899
900 /*
901 * If we had an error on the last wr build, then
902 * break out. Possible errors include bogus WR
903 * type, and a bogus SGL length...
904 */
905 if (err) {
906 break;
907 }
908
909 /*
910 * Store flags
911 */
912 c2_wr_set_flags(&wr, flags);
913
914 /*
915 * Post the puppy!
916 */
917 spin_lock_irqsave(&qp->lock, lock_flags);
918 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
919 if (err) {
920 spin_unlock_irqrestore(&qp->lock, lock_flags);
921 break;
922 }
923
924 /*
925 * Enqueue mq index to activity FIFO.
926 */
927 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
928 spin_unlock_irqrestore(&qp->lock, lock_flags);
929
930 ib_wr = ib_wr->next;
931 }
932
933 if (err)
934 *bad_wr = ib_wr;
935 return err;
936 }
937
c2_post_receive(struct ib_qp * ibqp,struct ib_recv_wr * ib_wr,struct ib_recv_wr ** bad_wr)938 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
939 struct ib_recv_wr **bad_wr)
940 {
941 struct c2_dev *c2dev = to_c2dev(ibqp->device);
942 struct c2_qp *qp = to_c2qp(ibqp);
943 union c2wr wr;
944 unsigned long lock_flags;
945 int err = 0;
946
947 if (qp->state > IB_QPS_RTS)
948 return -EINVAL;
949
950 /*
951 * Try and post each work request
952 */
953 while (ib_wr) {
954 u32 tot_len;
955 u8 actual_sge_count;
956
957 if (ib_wr->num_sge > qp->recv_sgl_depth) {
958 err = -EINVAL;
959 break;
960 }
961
962 /*
963 * Create local host-copy of the WR
964 */
965 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
966 c2_wr_set_id(&wr, CCWR_RECV);
967 c2_wr_set_flags(&wr, 0);
968
969 /* sge_count is limited to eight bits. */
970 BUG_ON(ib_wr->num_sge >= 256);
971 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
972 ib_wr->sg_list,
973 ib_wr->num_sge, &tot_len, &actual_sge_count);
974 c2_wr_set_sge_count(&wr, actual_sge_count);
975
976 /*
977 * If we had an error on the last wr build, then
978 * break out. Possible errors include bogus WR
979 * type, and a bogus SGL length...
980 */
981 if (err) {
982 break;
983 }
984
985 spin_lock_irqsave(&qp->lock, lock_flags);
986 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
987 if (err) {
988 spin_unlock_irqrestore(&qp->lock, lock_flags);
989 break;
990 }
991
992 /*
993 * Enqueue mq index to activity FIFO
994 */
995 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
996 spin_unlock_irqrestore(&qp->lock, lock_flags);
997
998 ib_wr = ib_wr->next;
999 }
1000
1001 if (err)
1002 *bad_wr = ib_wr;
1003 return err;
1004 }
1005
c2_init_qp_table(struct c2_dev * c2dev)1006 void __devinit c2_init_qp_table(struct c2_dev *c2dev)
1007 {
1008 spin_lock_init(&c2dev->qp_table.lock);
1009 idr_init(&c2dev->qp_table.idr);
1010 }
1011
c2_cleanup_qp_table(struct c2_dev * c2dev)1012 void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
1013 {
1014 idr_destroy(&c2dev->qp_table.idr);
1015 }
1016