1 /*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 *
36 */
37
38 #include <linux/delay.h>
39 #include <linux/gfp.h>
40
41 #include "c2.h"
42 #include "c2_vq.h"
43 #include "c2_status.h"
44
45 #define C2_MAX_ORD_PER_QP 128
46 #define C2_MAX_IRD_PER_QP 128
47
48 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
49 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
50 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
51
52 #define NO_SUPPORT -1
53 static const u8 c2_opcode[] = {
54 [IB_WR_SEND] = C2_WR_TYPE_SEND,
55 [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
56 [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
57 [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
58 [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
59 [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
60 [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
61 };
62
to_c2_state(enum ib_qp_state ib_state)63 static int to_c2_state(enum ib_qp_state ib_state)
64 {
65 switch (ib_state) {
66 case IB_QPS_RESET:
67 return C2_QP_STATE_IDLE;
68 case IB_QPS_RTS:
69 return C2_QP_STATE_RTS;
70 case IB_QPS_SQD:
71 return C2_QP_STATE_CLOSING;
72 case IB_QPS_SQE:
73 return C2_QP_STATE_CLOSING;
74 case IB_QPS_ERR:
75 return C2_QP_STATE_ERROR;
76 default:
77 return -1;
78 }
79 }
80
to_ib_state(enum c2_qp_state c2_state)81 static int to_ib_state(enum c2_qp_state c2_state)
82 {
83 switch (c2_state) {
84 case C2_QP_STATE_IDLE:
85 return IB_QPS_RESET;
86 case C2_QP_STATE_CONNECTING:
87 return IB_QPS_RTR;
88 case C2_QP_STATE_RTS:
89 return IB_QPS_RTS;
90 case C2_QP_STATE_CLOSING:
91 return IB_QPS_SQD;
92 case C2_QP_STATE_ERROR:
93 return IB_QPS_ERR;
94 case C2_QP_STATE_TERMINATE:
95 return IB_QPS_SQE;
96 default:
97 return -1;
98 }
99 }
100
to_ib_state_str(int ib_state)101 static const char *to_ib_state_str(int ib_state)
102 {
103 static const char *state_str[] = {
104 "IB_QPS_RESET",
105 "IB_QPS_INIT",
106 "IB_QPS_RTR",
107 "IB_QPS_RTS",
108 "IB_QPS_SQD",
109 "IB_QPS_SQE",
110 "IB_QPS_ERR"
111 };
112 if (ib_state < IB_QPS_RESET ||
113 ib_state > IB_QPS_ERR)
114 return "<invalid IB QP state>";
115
116 ib_state -= IB_QPS_RESET;
117 return state_str[ib_state];
118 }
119
c2_set_qp_state(struct c2_qp * qp,int c2_state)120 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
121 {
122 int new_state = to_ib_state(c2_state);
123
124 pr_debug("%s: qp[%p] state modify %s --> %s\n",
125 __func__,
126 qp,
127 to_ib_state_str(qp->state),
128 to_ib_state_str(new_state));
129 qp->state = new_state;
130 }
131
132 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
133
c2_qp_modify(struct c2_dev * c2dev,struct c2_qp * qp,struct ib_qp_attr * attr,int attr_mask)134 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
135 struct ib_qp_attr *attr, int attr_mask)
136 {
137 struct c2wr_qp_modify_req wr;
138 struct c2wr_qp_modify_rep *reply;
139 struct c2_vq_req *vq_req;
140 unsigned long flags;
141 u8 next_state;
142 int err;
143
144 pr_debug("%s:%d qp=%p, %s --> %s\n",
145 __func__, __LINE__,
146 qp,
147 to_ib_state_str(qp->state),
148 to_ib_state_str(attr->qp_state));
149
150 vq_req = vq_req_alloc(c2dev);
151 if (!vq_req)
152 return -ENOMEM;
153
154 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
155 wr.hdr.context = (unsigned long) vq_req;
156 wr.rnic_handle = c2dev->adapter_handle;
157 wr.qp_handle = qp->adapter_handle;
158 wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
159 wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
160 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
161 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
162
163 if (attr_mask & IB_QP_STATE) {
164 /* Ensure the state is valid */
165 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) {
166 err = -EINVAL;
167 goto bail0;
168 }
169
170 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
171
172 if (attr->qp_state == IB_QPS_ERR) {
173 spin_lock_irqsave(&qp->lock, flags);
174 if (qp->cm_id && qp->state == IB_QPS_RTS) {
175 pr_debug("Generating CLOSE event for QP-->ERR, "
176 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
177 /* Generate an CLOSE event */
178 vq_req->cm_id = qp->cm_id;
179 vq_req->event = IW_CM_EVENT_CLOSE;
180 }
181 spin_unlock_irqrestore(&qp->lock, flags);
182 }
183 next_state = attr->qp_state;
184
185 } else if (attr_mask & IB_QP_CUR_STATE) {
186
187 if (attr->cur_qp_state != IB_QPS_RTR &&
188 attr->cur_qp_state != IB_QPS_RTS &&
189 attr->cur_qp_state != IB_QPS_SQD &&
190 attr->cur_qp_state != IB_QPS_SQE) {
191 err = -EINVAL;
192 goto bail0;
193 } else
194 wr.next_qp_state =
195 cpu_to_be32(to_c2_state(attr->cur_qp_state));
196
197 next_state = attr->cur_qp_state;
198
199 } else {
200 err = 0;
201 goto bail0;
202 }
203
204 /* reference the request struct */
205 vq_req_get(c2dev, vq_req);
206
207 err = vq_send_wr(c2dev, (union c2wr *) & wr);
208 if (err) {
209 vq_req_put(c2dev, vq_req);
210 goto bail0;
211 }
212
213 err = vq_wait_for_reply(c2dev, vq_req);
214 if (err)
215 goto bail0;
216
217 reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
218 if (!reply) {
219 err = -ENOMEM;
220 goto bail0;
221 }
222
223 err = c2_errno(reply);
224 if (!err)
225 qp->state = next_state;
226 #ifdef DEBUG
227 else
228 pr_debug("%s: c2_errno=%d\n", __func__, err);
229 #endif
230 /*
231 * If we're going to error and generating the event here, then
232 * we need to remove the reference because there will be no
233 * close event generated by the adapter
234 */
235 spin_lock_irqsave(&qp->lock, flags);
236 if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
237 qp->cm_id->rem_ref(qp->cm_id);
238 qp->cm_id = NULL;
239 }
240 spin_unlock_irqrestore(&qp->lock, flags);
241
242 vq_repbuf_free(c2dev, reply);
243 bail0:
244 vq_req_free(c2dev, vq_req);
245
246 pr_debug("%s:%d qp=%p, cur_state=%s\n",
247 __func__, __LINE__,
248 qp,
249 to_ib_state_str(qp->state));
250 return err;
251 }
252
c2_qp_set_read_limits(struct c2_dev * c2dev,struct c2_qp * qp,int ord,int ird)253 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
254 int ord, int ird)
255 {
256 struct c2wr_qp_modify_req wr;
257 struct c2wr_qp_modify_rep *reply;
258 struct c2_vq_req *vq_req;
259 int err;
260
261 vq_req = vq_req_alloc(c2dev);
262 if (!vq_req)
263 return -ENOMEM;
264
265 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
266 wr.hdr.context = (unsigned long) vq_req;
267 wr.rnic_handle = c2dev->adapter_handle;
268 wr.qp_handle = qp->adapter_handle;
269 wr.ord = cpu_to_be32(ord);
270 wr.ird = cpu_to_be32(ird);
271 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
272 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
273 wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
274
275 /* reference the request struct */
276 vq_req_get(c2dev, vq_req);
277
278 err = vq_send_wr(c2dev, (union c2wr *) & wr);
279 if (err) {
280 vq_req_put(c2dev, vq_req);
281 goto bail0;
282 }
283
284 err = vq_wait_for_reply(c2dev, vq_req);
285 if (err)
286 goto bail0;
287
288 reply = (struct c2wr_qp_modify_rep *) (unsigned long)
289 vq_req->reply_msg;
290 if (!reply) {
291 err = -ENOMEM;
292 goto bail0;
293 }
294
295 err = c2_errno(reply);
296 vq_repbuf_free(c2dev, reply);
297 bail0:
298 vq_req_free(c2dev, vq_req);
299 return err;
300 }
301
destroy_qp(struct c2_dev * c2dev,struct c2_qp * qp)302 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
303 {
304 struct c2_vq_req *vq_req;
305 struct c2wr_qp_destroy_req wr;
306 struct c2wr_qp_destroy_rep *reply;
307 unsigned long flags;
308 int err;
309
310 /*
311 * Allocate a verb request message
312 */
313 vq_req = vq_req_alloc(c2dev);
314 if (!vq_req) {
315 return -ENOMEM;
316 }
317
318 /*
319 * Initialize the WR
320 */
321 c2_wr_set_id(&wr, CCWR_QP_DESTROY);
322 wr.hdr.context = (unsigned long) vq_req;
323 wr.rnic_handle = c2dev->adapter_handle;
324 wr.qp_handle = qp->adapter_handle;
325
326 /*
327 * reference the request struct. dereferenced in the int handler.
328 */
329 vq_req_get(c2dev, vq_req);
330
331 spin_lock_irqsave(&qp->lock, flags);
332 if (qp->cm_id && qp->state == IB_QPS_RTS) {
333 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
334 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
335 /* Generate an CLOSE event */
336 vq_req->qp = qp;
337 vq_req->cm_id = qp->cm_id;
338 vq_req->event = IW_CM_EVENT_CLOSE;
339 }
340 spin_unlock_irqrestore(&qp->lock, flags);
341
342 /*
343 * Send WR to adapter
344 */
345 err = vq_send_wr(c2dev, (union c2wr *) & wr);
346 if (err) {
347 vq_req_put(c2dev, vq_req);
348 goto bail0;
349 }
350
351 /*
352 * Wait for reply from adapter
353 */
354 err = vq_wait_for_reply(c2dev, vq_req);
355 if (err) {
356 goto bail0;
357 }
358
359 /*
360 * Process reply
361 */
362 reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
363 if (!reply) {
364 err = -ENOMEM;
365 goto bail0;
366 }
367
368 spin_lock_irqsave(&qp->lock, flags);
369 if (qp->cm_id) {
370 qp->cm_id->rem_ref(qp->cm_id);
371 qp->cm_id = NULL;
372 }
373 spin_unlock_irqrestore(&qp->lock, flags);
374
375 vq_repbuf_free(c2dev, reply);
376 bail0:
377 vq_req_free(c2dev, vq_req);
378 return err;
379 }
380
c2_alloc_qpn(struct c2_dev * c2dev,struct c2_qp * qp)381 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
382 {
383 int ret;
384
385 idr_preload(GFP_KERNEL);
386 spin_lock_irq(&c2dev->qp_table.lock);
387
388 ret = idr_alloc_cyclic(&c2dev->qp_table.idr, qp, 0, 0, GFP_NOWAIT);
389 if (ret >= 0)
390 qp->qpn = ret;
391
392 spin_unlock_irq(&c2dev->qp_table.lock);
393 idr_preload_end();
394 return ret < 0 ? ret : 0;
395 }
396
c2_free_qpn(struct c2_dev * c2dev,int qpn)397 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
398 {
399 spin_lock_irq(&c2dev->qp_table.lock);
400 idr_remove(&c2dev->qp_table.idr, qpn);
401 spin_unlock_irq(&c2dev->qp_table.lock);
402 }
403
c2_find_qpn(struct c2_dev * c2dev,int qpn)404 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
405 {
406 unsigned long flags;
407 struct c2_qp *qp;
408
409 spin_lock_irqsave(&c2dev->qp_table.lock, flags);
410 qp = idr_find(&c2dev->qp_table.idr, qpn);
411 spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
412 return qp;
413 }
414
c2_alloc_qp(struct c2_dev * c2dev,struct c2_pd * pd,struct ib_qp_init_attr * qp_attrs,struct c2_qp * qp)415 int c2_alloc_qp(struct c2_dev *c2dev,
416 struct c2_pd *pd,
417 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
418 {
419 struct c2wr_qp_create_req wr;
420 struct c2wr_qp_create_rep *reply;
421 struct c2_vq_req *vq_req;
422 struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
423 struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
424 unsigned long peer_pa;
425 u32 q_size, msg_size, mmap_size;
426 void __iomem *mmap;
427 int err;
428
429 err = c2_alloc_qpn(c2dev, qp);
430 if (err)
431 return err;
432 qp->ibqp.qp_num = qp->qpn;
433 qp->ibqp.qp_type = IB_QPT_RC;
434
435 /* Allocate the SQ and RQ shared pointers */
436 qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
437 &qp->sq_mq.shared_dma, GFP_KERNEL);
438 if (!qp->sq_mq.shared) {
439 err = -ENOMEM;
440 goto bail0;
441 }
442
443 qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
444 &qp->rq_mq.shared_dma, GFP_KERNEL);
445 if (!qp->rq_mq.shared) {
446 err = -ENOMEM;
447 goto bail1;
448 }
449
450 /* Allocate the verbs request */
451 vq_req = vq_req_alloc(c2dev);
452 if (vq_req == NULL) {
453 err = -ENOMEM;
454 goto bail2;
455 }
456
457 /* Initialize the work request */
458 memset(&wr, 0, sizeof(wr));
459 c2_wr_set_id(&wr, CCWR_QP_CREATE);
460 wr.hdr.context = (unsigned long) vq_req;
461 wr.rnic_handle = c2dev->adapter_handle;
462 wr.sq_cq_handle = send_cq->adapter_handle;
463 wr.rq_cq_handle = recv_cq->adapter_handle;
464 wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
465 wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
466 wr.srq_handle = 0;
467 wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
468 QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
469 wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
470 wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
471 wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
472 wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
473 wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
474 wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
475 wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
476 wr.pd_id = pd->pd_id;
477 wr.user_context = (unsigned long) qp;
478
479 vq_req_get(c2dev, vq_req);
480
481 /* Send the WR to the adapter */
482 err = vq_send_wr(c2dev, (union c2wr *) & wr);
483 if (err) {
484 vq_req_put(c2dev, vq_req);
485 goto bail3;
486 }
487
488 /* Wait for the verb reply */
489 err = vq_wait_for_reply(c2dev, vq_req);
490 if (err) {
491 goto bail3;
492 }
493
494 /* Process the reply */
495 reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
496 if (!reply) {
497 err = -ENOMEM;
498 goto bail3;
499 }
500
501 if ((err = c2_wr_get_result(reply)) != 0) {
502 goto bail4;
503 }
504
505 /* Fill in the kernel QP struct */
506 atomic_set(&qp->refcount, 1);
507 qp->adapter_handle = reply->qp_handle;
508 qp->state = IB_QPS_RESET;
509 qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
510 qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
511 qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
512 init_waitqueue_head(&qp->wait);
513
514 /* Initialize the SQ MQ */
515 q_size = be32_to_cpu(reply->sq_depth);
516 msg_size = be32_to_cpu(reply->sq_msg_size);
517 peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
518 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
519 mmap = ioremap_nocache(peer_pa, mmap_size);
520 if (!mmap) {
521 err = -ENOMEM;
522 goto bail5;
523 }
524
525 c2_mq_req_init(&qp->sq_mq,
526 be32_to_cpu(reply->sq_mq_index),
527 q_size,
528 msg_size,
529 mmap + sizeof(struct c2_mq_shared), /* pool start */
530 mmap, /* peer */
531 C2_MQ_ADAPTER_TARGET);
532
533 /* Initialize the RQ mq */
534 q_size = be32_to_cpu(reply->rq_depth);
535 msg_size = be32_to_cpu(reply->rq_msg_size);
536 peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
537 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
538 mmap = ioremap_nocache(peer_pa, mmap_size);
539 if (!mmap) {
540 err = -ENOMEM;
541 goto bail6;
542 }
543
544 c2_mq_req_init(&qp->rq_mq,
545 be32_to_cpu(reply->rq_mq_index),
546 q_size,
547 msg_size,
548 mmap + sizeof(struct c2_mq_shared), /* pool start */
549 mmap, /* peer */
550 C2_MQ_ADAPTER_TARGET);
551
552 vq_repbuf_free(c2dev, reply);
553 vq_req_free(c2dev, vq_req);
554
555 return 0;
556
557 bail6:
558 iounmap(qp->sq_mq.peer);
559 bail5:
560 destroy_qp(c2dev, qp);
561 bail4:
562 vq_repbuf_free(c2dev, reply);
563 bail3:
564 vq_req_free(c2dev, vq_req);
565 bail2:
566 c2_free_mqsp(qp->rq_mq.shared);
567 bail1:
568 c2_free_mqsp(qp->sq_mq.shared);
569 bail0:
570 c2_free_qpn(c2dev, qp->qpn);
571 return err;
572 }
573
c2_lock_cqs(struct c2_cq * send_cq,struct c2_cq * recv_cq)574 static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
575 {
576 if (send_cq == recv_cq)
577 spin_lock_irq(&send_cq->lock);
578 else if (send_cq > recv_cq) {
579 spin_lock_irq(&send_cq->lock);
580 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
581 } else {
582 spin_lock_irq(&recv_cq->lock);
583 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
584 }
585 }
586
c2_unlock_cqs(struct c2_cq * send_cq,struct c2_cq * recv_cq)587 static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
588 {
589 if (send_cq == recv_cq)
590 spin_unlock_irq(&send_cq->lock);
591 else if (send_cq > recv_cq) {
592 spin_unlock(&recv_cq->lock);
593 spin_unlock_irq(&send_cq->lock);
594 } else {
595 spin_unlock(&send_cq->lock);
596 spin_unlock_irq(&recv_cq->lock);
597 }
598 }
599
c2_free_qp(struct c2_dev * c2dev,struct c2_qp * qp)600 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
601 {
602 struct c2_cq *send_cq;
603 struct c2_cq *recv_cq;
604
605 send_cq = to_c2cq(qp->ibqp.send_cq);
606 recv_cq = to_c2cq(qp->ibqp.recv_cq);
607
608 /*
609 * Lock CQs here, so that CQ polling code can do QP lookup
610 * without taking a lock.
611 */
612 c2_lock_cqs(send_cq, recv_cq);
613 c2_free_qpn(c2dev, qp->qpn);
614 c2_unlock_cqs(send_cq, recv_cq);
615
616 /*
617 * Destroy qp in the rnic...
618 */
619 destroy_qp(c2dev, qp);
620
621 /*
622 * Mark any unreaped CQEs as null and void.
623 */
624 c2_cq_clean(c2dev, qp, send_cq->cqn);
625 if (send_cq != recv_cq)
626 c2_cq_clean(c2dev, qp, recv_cq->cqn);
627 /*
628 * Unmap the MQs and return the shared pointers
629 * to the message pool.
630 */
631 iounmap(qp->sq_mq.peer);
632 iounmap(qp->rq_mq.peer);
633 c2_free_mqsp(qp->sq_mq.shared);
634 c2_free_mqsp(qp->rq_mq.shared);
635
636 atomic_dec(&qp->refcount);
637 wait_event(qp->wait, !atomic_read(&qp->refcount));
638 }
639
640 /*
641 * Function: move_sgl
642 *
643 * Description:
644 * Move an SGL from the user's work request struct into a CCIL Work Request
645 * message, swapping to WR byte order and ensure the total length doesn't
646 * overflow.
647 *
648 * IN:
649 * dst - ptr to CCIL Work Request message SGL memory.
650 * src - ptr to the consumers SGL memory.
651 *
652 * OUT: none
653 *
654 * Return:
655 * CCIL status codes.
656 */
657 static int
move_sgl(struct c2_data_addr * dst,struct ib_sge * src,int count,u32 * p_len,u8 * actual_count)658 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
659 u8 * actual_count)
660 {
661 u32 tot = 0; /* running total */
662 u8 acount = 0; /* running total non-0 len sge's */
663
664 while (count > 0) {
665 /*
666 * If the addition of this SGE causes the
667 * total SGL length to exceed 2^32-1, then
668 * fail-n-bail.
669 *
670 * If the current total plus the next element length
671 * wraps, then it will go negative and be less than the
672 * current total...
673 */
674 if ((tot + src->length) < tot) {
675 return -EINVAL;
676 }
677 /*
678 * Bug: 1456 (as well as 1498 & 1643)
679 * Skip over any sge's supplied with len=0
680 */
681 if (src->length) {
682 tot += src->length;
683 dst->stag = cpu_to_be32(src->lkey);
684 dst->to = cpu_to_be64(src->addr);
685 dst->length = cpu_to_be32(src->length);
686 dst++;
687 acount++;
688 }
689 src++;
690 count--;
691 }
692
693 if (acount == 0) {
694 /*
695 * Bug: 1476 (as well as 1498, 1456 and 1643)
696 * Setup the SGL in the WR to make it easier for the RNIC.
697 * This way, the FW doesn't have to deal with special cases.
698 * Setting length=0 should be sufficient.
699 */
700 dst->stag = 0;
701 dst->to = 0;
702 dst->length = 0;
703 }
704
705 *p_len = tot;
706 *actual_count = acount;
707 return 0;
708 }
709
710 /*
711 * Function: c2_activity (private function)
712 *
713 * Description:
714 * Post an mq index to the host->adapter activity fifo.
715 *
716 * IN:
717 * c2dev - ptr to c2dev structure
718 * mq_index - mq index to post
719 * shared - value most recently written to shared
720 *
721 * OUT:
722 *
723 * Return:
724 * none
725 */
c2_activity(struct c2_dev * c2dev,u32 mq_index,u16 shared)726 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
727 {
728 /*
729 * First read the register to see if the FIFO is full, and if so,
730 * spin until it's not. This isn't perfect -- there is no
731 * synchronization among the clients of the register, but in
732 * practice it prevents multiple CPU from hammering the bus
733 * with PCI RETRY. Note that when this does happen, the card
734 * cannot get on the bus and the card and system hang in a
735 * deadlock -- thus the need for this code. [TOT]
736 */
737 while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
738 udelay(10);
739
740 __raw_writel(C2_HINT_MAKE(mq_index, shared),
741 c2dev->regs + PCI_BAR0_ADAPTER_HINT);
742 }
743
744 /*
745 * Function: qp_wr_post
746 *
747 * Description:
748 * This in-line function allocates a MQ msg, then moves the host-copy of
749 * the completed WR into msg. Then it posts the message.
750 *
751 * IN:
752 * q - ptr to user MQ.
753 * wr - ptr to host-copy of the WR.
754 * qp - ptr to user qp
755 * size - Number of bytes to post. Assumed to be divisible by 4.
756 *
757 * OUT: none
758 *
759 * Return:
760 * CCIL status codes.
761 */
qp_wr_post(struct c2_mq * q,union c2wr * wr,struct c2_qp * qp,u32 size)762 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
763 {
764 union c2wr *msg;
765
766 msg = c2_mq_alloc(q);
767 if (msg == NULL) {
768 return -EINVAL;
769 }
770 #ifdef CCMSGMAGIC
771 ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
772 #endif
773
774 /*
775 * Since all header fields in the WR are the same as the
776 * CQE, set the following so the adapter need not.
777 */
778 c2_wr_set_result(wr, CCERR_PENDING);
779
780 /*
781 * Copy the wr down to the adapter
782 */
783 memcpy((void *) msg, (void *) wr, size);
784
785 c2_mq_produce(q);
786 return 0;
787 }
788
789
c2_post_send(struct ib_qp * ibqp,struct ib_send_wr * ib_wr,struct ib_send_wr ** bad_wr)790 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
791 struct ib_send_wr **bad_wr)
792 {
793 struct c2_dev *c2dev = to_c2dev(ibqp->device);
794 struct c2_qp *qp = to_c2qp(ibqp);
795 union c2wr wr;
796 unsigned long lock_flags;
797 int err = 0;
798
799 u32 flags;
800 u32 tot_len;
801 u8 actual_sge_count;
802 u32 msg_size;
803
804 if (qp->state > IB_QPS_RTS) {
805 err = -EINVAL;
806 goto out;
807 }
808
809 while (ib_wr) {
810
811 flags = 0;
812 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
813 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
814 flags |= SQ_SIGNALED;
815 }
816
817 switch (ib_wr->opcode) {
818 case IB_WR_SEND:
819 case IB_WR_SEND_WITH_INV:
820 if (ib_wr->opcode == IB_WR_SEND) {
821 if (ib_wr->send_flags & IB_SEND_SOLICITED)
822 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
823 else
824 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
825 wr.sqwr.send.remote_stag = 0;
826 } else {
827 if (ib_wr->send_flags & IB_SEND_SOLICITED)
828 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV);
829 else
830 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV);
831 wr.sqwr.send.remote_stag =
832 cpu_to_be32(ib_wr->ex.invalidate_rkey);
833 }
834
835 msg_size = sizeof(struct c2wr_send_req) +
836 sizeof(struct c2_data_addr) * ib_wr->num_sge;
837 if (ib_wr->num_sge > qp->send_sgl_depth) {
838 err = -EINVAL;
839 break;
840 }
841 if (ib_wr->send_flags & IB_SEND_FENCE) {
842 flags |= SQ_READ_FENCE;
843 }
844 err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
845 ib_wr->sg_list,
846 ib_wr->num_sge,
847 &tot_len, &actual_sge_count);
848 wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
849 c2_wr_set_sge_count(&wr, actual_sge_count);
850 break;
851 case IB_WR_RDMA_WRITE:
852 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
853 msg_size = sizeof(struct c2wr_rdma_write_req) +
854 (sizeof(struct c2_data_addr) * ib_wr->num_sge);
855 if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
856 err = -EINVAL;
857 break;
858 }
859 if (ib_wr->send_flags & IB_SEND_FENCE) {
860 flags |= SQ_READ_FENCE;
861 }
862 wr.sqwr.rdma_write.remote_stag =
863 cpu_to_be32(ib_wr->wr.rdma.rkey);
864 wr.sqwr.rdma_write.remote_to =
865 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
866 err = move_sgl((struct c2_data_addr *)
867 & (wr.sqwr.rdma_write.data),
868 ib_wr->sg_list,
869 ib_wr->num_sge,
870 &tot_len, &actual_sge_count);
871 wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
872 c2_wr_set_sge_count(&wr, actual_sge_count);
873 break;
874 case IB_WR_RDMA_READ:
875 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
876 msg_size = sizeof(struct c2wr_rdma_read_req);
877
878 /* IWarp only suppots 1 sge for RDMA reads */
879 if (ib_wr->num_sge > 1) {
880 err = -EINVAL;
881 break;
882 }
883
884 /*
885 * Move the local and remote stag/to/len into the WR.
886 */
887 wr.sqwr.rdma_read.local_stag =
888 cpu_to_be32(ib_wr->sg_list->lkey);
889 wr.sqwr.rdma_read.local_to =
890 cpu_to_be64(ib_wr->sg_list->addr);
891 wr.sqwr.rdma_read.remote_stag =
892 cpu_to_be32(ib_wr->wr.rdma.rkey);
893 wr.sqwr.rdma_read.remote_to =
894 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
895 wr.sqwr.rdma_read.length =
896 cpu_to_be32(ib_wr->sg_list->length);
897 break;
898 default:
899 /* error */
900 msg_size = 0;
901 err = -EINVAL;
902 break;
903 }
904
905 /*
906 * If we had an error on the last wr build, then
907 * break out. Possible errors include bogus WR
908 * type, and a bogus SGL length...
909 */
910 if (err) {
911 break;
912 }
913
914 /*
915 * Store flags
916 */
917 c2_wr_set_flags(&wr, flags);
918
919 /*
920 * Post the puppy!
921 */
922 spin_lock_irqsave(&qp->lock, lock_flags);
923 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
924 if (err) {
925 spin_unlock_irqrestore(&qp->lock, lock_flags);
926 break;
927 }
928
929 /*
930 * Enqueue mq index to activity FIFO.
931 */
932 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
933 spin_unlock_irqrestore(&qp->lock, lock_flags);
934
935 ib_wr = ib_wr->next;
936 }
937
938 out:
939 if (err)
940 *bad_wr = ib_wr;
941 return err;
942 }
943
c2_post_receive(struct ib_qp * ibqp,struct ib_recv_wr * ib_wr,struct ib_recv_wr ** bad_wr)944 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
945 struct ib_recv_wr **bad_wr)
946 {
947 struct c2_dev *c2dev = to_c2dev(ibqp->device);
948 struct c2_qp *qp = to_c2qp(ibqp);
949 union c2wr wr;
950 unsigned long lock_flags;
951 int err = 0;
952
953 if (qp->state > IB_QPS_RTS) {
954 err = -EINVAL;
955 goto out;
956 }
957
958 /*
959 * Try and post each work request
960 */
961 while (ib_wr) {
962 u32 tot_len;
963 u8 actual_sge_count;
964
965 if (ib_wr->num_sge > qp->recv_sgl_depth) {
966 err = -EINVAL;
967 break;
968 }
969
970 /*
971 * Create local host-copy of the WR
972 */
973 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
974 c2_wr_set_id(&wr, CCWR_RECV);
975 c2_wr_set_flags(&wr, 0);
976
977 /* sge_count is limited to eight bits. */
978 BUG_ON(ib_wr->num_sge >= 256);
979 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
980 ib_wr->sg_list,
981 ib_wr->num_sge, &tot_len, &actual_sge_count);
982 c2_wr_set_sge_count(&wr, actual_sge_count);
983
984 /*
985 * If we had an error on the last wr build, then
986 * break out. Possible errors include bogus WR
987 * type, and a bogus SGL length...
988 */
989 if (err) {
990 break;
991 }
992
993 spin_lock_irqsave(&qp->lock, lock_flags);
994 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
995 if (err) {
996 spin_unlock_irqrestore(&qp->lock, lock_flags);
997 break;
998 }
999
1000 /*
1001 * Enqueue mq index to activity FIFO
1002 */
1003 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
1004 spin_unlock_irqrestore(&qp->lock, lock_flags);
1005
1006 ib_wr = ib_wr->next;
1007 }
1008
1009 out:
1010 if (err)
1011 *bad_wr = ib_wr;
1012 return err;
1013 }
1014
c2_init_qp_table(struct c2_dev * c2dev)1015 void c2_init_qp_table(struct c2_dev *c2dev)
1016 {
1017 spin_lock_init(&c2dev->qp_table.lock);
1018 idr_init(&c2dev->qp_table.idr);
1019 }
1020
c2_cleanup_qp_table(struct c2_dev * c2dev)1021 void c2_cleanup_qp_table(struct c2_dev *c2dev)
1022 {
1023 idr_destroy(&c2dev->qp_table.idr);
1024 }
1025