1 /*
2 * IBM eServer eHCA Infiniband device driver for Linux on POWER
3 *
4 * post_send/recv, poll_cq, req_notify
5 *
6 * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
7 * Waleri Fomin <fomin@de.ibm.com>
8 * Joachim Fenkes <fenkes@de.ibm.com>
9 * Reinhard Ernst <rernst@de.ibm.com>
10 *
11 * Copyright (c) 2005 IBM Corporation
12 *
13 * All rights reserved.
14 *
15 * This source code is distributed under a dual license of GPL v2.0 and OpenIB
16 * BSD.
17 *
18 * OpenIB BSD License
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions are met:
22 *
23 * Redistributions of source code must retain the above copyright notice, this
24 * list of conditions and the following disclaimer.
25 *
26 * Redistributions in binary form must reproduce the above copyright notice,
27 * this list of conditions and the following disclaimer in the documentation
28 * and/or other materials
29 * provided with the distribution.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
32 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
35 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
36 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
37 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
38 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
39 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
41 * POSSIBILITY OF SUCH DAMAGE.
42 */
43
44
45 #include <asm/system.h>
46 #include "ehca_classes.h"
47 #include "ehca_tools.h"
48 #include "ehca_qes.h"
49 #include "ehca_iverbs.h"
50 #include "hcp_if.h"
51 #include "hipz_fns.h"
52
53 /* in RC traffic, insert an empty RDMA READ every this many packets */
54 #define ACK_CIRC_THRESHOLD 2000000
55
replace_wr_id(u64 wr_id,u16 idx)56 static u64 replace_wr_id(u64 wr_id, u16 idx)
57 {
58 u64 ret;
59
60 ret = wr_id & ~QMAP_IDX_MASK;
61 ret |= idx & QMAP_IDX_MASK;
62
63 return ret;
64 }
65
get_app_wr_id(u64 wr_id)66 static u16 get_app_wr_id(u64 wr_id)
67 {
68 return wr_id & QMAP_IDX_MASK;
69 }
70
ehca_write_rwqe(struct ipz_queue * ipz_rqueue,struct ehca_wqe * wqe_p,struct ib_recv_wr * recv_wr,u32 rq_map_idx)71 static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
72 struct ehca_wqe *wqe_p,
73 struct ib_recv_wr *recv_wr,
74 u32 rq_map_idx)
75 {
76 u8 cnt_ds;
77 if (unlikely((recv_wr->num_sge < 0) ||
78 (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
79 ehca_gen_err("Invalid number of WQE SGE. "
80 "num_sqe=%x max_nr_of_sg=%x",
81 recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
82 return -EINVAL; /* invalid SG list length */
83 }
84
85 /* clear wqe header until sglist */
86 memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
87
88 wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
89 wqe_p->nr_of_data_seg = recv_wr->num_sge;
90
91 for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
92 wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
93 recv_wr->sg_list[cnt_ds].addr;
94 wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
95 recv_wr->sg_list[cnt_ds].lkey;
96 wqe_p->u.all_rcv.sg_list[cnt_ds].length =
97 recv_wr->sg_list[cnt_ds].length;
98 }
99
100 if (ehca_debug_level >= 3) {
101 ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
102 ipz_rqueue);
103 ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
104 }
105
106 return 0;
107 }
108
109 #if defined(DEBUG_GSI_SEND_WR)
110
111 /* need ib_mad struct */
112 #include <rdma/ib_mad.h>
113
trace_send_wr_ud(const struct ib_send_wr * send_wr)114 static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
115 {
116 int idx;
117 int j;
118 while (send_wr) {
119 struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
120 struct ib_sge *sge = send_wr->sg_list;
121 ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
122 "send_flags=%x opcode=%x", idx, send_wr->wr_id,
123 send_wr->num_sge, send_wr->send_flags,
124 send_wr->opcode);
125 if (mad_hdr) {
126 ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
127 "mgmt_class=%x class_version=%x method=%x "
128 "status=%x class_specific=%x tid=%lx "
129 "attr_id=%x resv=%x attr_mod=%x",
130 idx, mad_hdr->base_version,
131 mad_hdr->mgmt_class,
132 mad_hdr->class_version, mad_hdr->method,
133 mad_hdr->status, mad_hdr->class_specific,
134 mad_hdr->tid, mad_hdr->attr_id,
135 mad_hdr->resv,
136 mad_hdr->attr_mod);
137 }
138 for (j = 0; j < send_wr->num_sge; j++) {
139 u8 *data = (u8 *)abs_to_virt(sge->addr);
140 ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
141 "lkey=%x",
142 idx, j, data, sge->length, sge->lkey);
143 /* assume length is n*16 */
144 ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
145 idx, j);
146 sge++;
147 } /* eof for j */
148 idx++;
149 send_wr = send_wr->next;
150 } /* eof while send_wr */
151 }
152
153 #endif /* DEBUG_GSI_SEND_WR */
154
ehca_write_swqe(struct ehca_qp * qp,struct ehca_wqe * wqe_p,const struct ib_send_wr * send_wr,u32 sq_map_idx,int hidden)155 static inline int ehca_write_swqe(struct ehca_qp *qp,
156 struct ehca_wqe *wqe_p,
157 const struct ib_send_wr *send_wr,
158 u32 sq_map_idx,
159 int hidden)
160 {
161 u32 idx;
162 u64 dma_length;
163 struct ehca_av *my_av;
164 u32 remote_qkey = send_wr->wr.ud.remote_qkey;
165 struct ehca_qmap_entry *qmap_entry = &qp->sq_map.map[sq_map_idx];
166
167 if (unlikely((send_wr->num_sge < 0) ||
168 (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
169 ehca_gen_err("Invalid number of WQE SGE. "
170 "num_sqe=%x max_nr_of_sg=%x",
171 send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
172 return -EINVAL; /* invalid SG list length */
173 }
174
175 /* clear wqe header until sglist */
176 memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
177
178 wqe_p->work_request_id = replace_wr_id(send_wr->wr_id, sq_map_idx);
179
180 qmap_entry->app_wr_id = get_app_wr_id(send_wr->wr_id);
181 qmap_entry->reported = 0;
182 qmap_entry->cqe_req = 0;
183
184 switch (send_wr->opcode) {
185 case IB_WR_SEND:
186 case IB_WR_SEND_WITH_IMM:
187 wqe_p->optype = WQE_OPTYPE_SEND;
188 break;
189 case IB_WR_RDMA_WRITE:
190 case IB_WR_RDMA_WRITE_WITH_IMM:
191 wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
192 break;
193 case IB_WR_RDMA_READ:
194 wqe_p->optype = WQE_OPTYPE_RDMAREAD;
195 break;
196 default:
197 ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
198 return -EINVAL; /* invalid opcode */
199 }
200
201 wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
202
203 wqe_p->wr_flag = 0;
204
205 if ((send_wr->send_flags & IB_SEND_SIGNALED ||
206 qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
207 && !hidden) {
208 wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
209 qmap_entry->cqe_req = 1;
210 }
211
212 if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
213 send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
214 /* this might not work as long as HW does not support it */
215 wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
216 wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
217 }
218
219 wqe_p->nr_of_data_seg = send_wr->num_sge;
220
221 switch (qp->qp_type) {
222 case IB_QPT_SMI:
223 case IB_QPT_GSI:
224 /* no break is intential here */
225 case IB_QPT_UD:
226 /* IB 1.2 spec C10-15 compliance */
227 if (send_wr->wr.ud.remote_qkey & 0x80000000)
228 remote_qkey = qp->qkey;
229
230 wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
231 wqe_p->local_ee_context_qkey = remote_qkey;
232 if (unlikely(!send_wr->wr.ud.ah)) {
233 ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
234 return -EINVAL;
235 }
236 if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
237 ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
238 return -EINVAL;
239 }
240 my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
241 wqe_p->u.ud_av.ud_av = my_av->av;
242
243 /*
244 * omitted check of IB_SEND_INLINE
245 * since HW does not support it
246 */
247 for (idx = 0; idx < send_wr->num_sge; idx++) {
248 wqe_p->u.ud_av.sg_list[idx].vaddr =
249 send_wr->sg_list[idx].addr;
250 wqe_p->u.ud_av.sg_list[idx].lkey =
251 send_wr->sg_list[idx].lkey;
252 wqe_p->u.ud_av.sg_list[idx].length =
253 send_wr->sg_list[idx].length;
254 } /* eof for idx */
255 if (qp->qp_type == IB_QPT_SMI ||
256 qp->qp_type == IB_QPT_GSI)
257 wqe_p->u.ud_av.ud_av.pmtu = 1;
258 if (qp->qp_type == IB_QPT_GSI) {
259 wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
260 #ifdef DEBUG_GSI_SEND_WR
261 trace_send_wr_ud(send_wr);
262 #endif /* DEBUG_GSI_SEND_WR */
263 }
264 break;
265
266 case IB_QPT_UC:
267 if (send_wr->send_flags & IB_SEND_FENCE)
268 wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
269 /* no break is intentional here */
270 case IB_QPT_RC:
271 /* TODO: atomic not implemented */
272 wqe_p->u.nud.remote_virtual_adress =
273 send_wr->wr.rdma.remote_addr;
274 wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
275
276 /*
277 * omitted checking of IB_SEND_INLINE
278 * since HW does not support it
279 */
280 dma_length = 0;
281 for (idx = 0; idx < send_wr->num_sge; idx++) {
282 wqe_p->u.nud.sg_list[idx].vaddr =
283 send_wr->sg_list[idx].addr;
284 wqe_p->u.nud.sg_list[idx].lkey =
285 send_wr->sg_list[idx].lkey;
286 wqe_p->u.nud.sg_list[idx].length =
287 send_wr->sg_list[idx].length;
288 dma_length += send_wr->sg_list[idx].length;
289 } /* eof idx */
290 wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
291
292 /* unsolicited ack circumvention */
293 if (send_wr->opcode == IB_WR_RDMA_READ) {
294 /* on RDMA read, switch on and reset counters */
295 qp->message_count = qp->packet_count = 0;
296 qp->unsol_ack_circ = 1;
297 } else
298 /* else estimate #packets */
299 qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
300
301 break;
302
303 default:
304 ehca_gen_err("Invalid qptype=%x", qp->qp_type);
305 return -EINVAL;
306 }
307
308 if (ehca_debug_level >= 3) {
309 ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
310 ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
311 }
312 return 0;
313 }
314
315 /* map_ib_wc_status converts raw cqe_status to ib_wc_status */
map_ib_wc_status(u32 cqe_status,enum ib_wc_status * wc_status)316 static inline void map_ib_wc_status(u32 cqe_status,
317 enum ib_wc_status *wc_status)
318 {
319 if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
320 switch (cqe_status & 0x3F) {
321 case 0x01:
322 case 0x21:
323 *wc_status = IB_WC_LOC_LEN_ERR;
324 break;
325 case 0x02:
326 case 0x22:
327 *wc_status = IB_WC_LOC_QP_OP_ERR;
328 break;
329 case 0x03:
330 case 0x23:
331 *wc_status = IB_WC_LOC_EEC_OP_ERR;
332 break;
333 case 0x04:
334 case 0x24:
335 *wc_status = IB_WC_LOC_PROT_ERR;
336 break;
337 case 0x05:
338 case 0x25:
339 *wc_status = IB_WC_WR_FLUSH_ERR;
340 break;
341 case 0x06:
342 *wc_status = IB_WC_MW_BIND_ERR;
343 break;
344 case 0x07: /* remote error - look into bits 20:24 */
345 switch ((cqe_status
346 & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
347 case 0x0:
348 /*
349 * PSN Sequence Error!
350 * couldn't find a matching status!
351 */
352 *wc_status = IB_WC_GENERAL_ERR;
353 break;
354 case 0x1:
355 *wc_status = IB_WC_REM_INV_REQ_ERR;
356 break;
357 case 0x2:
358 *wc_status = IB_WC_REM_ACCESS_ERR;
359 break;
360 case 0x3:
361 *wc_status = IB_WC_REM_OP_ERR;
362 break;
363 case 0x4:
364 *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
365 break;
366 }
367 break;
368 case 0x08:
369 *wc_status = IB_WC_RETRY_EXC_ERR;
370 break;
371 case 0x09:
372 *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
373 break;
374 case 0x0A:
375 case 0x2D:
376 *wc_status = IB_WC_REM_ABORT_ERR;
377 break;
378 case 0x0B:
379 case 0x2E:
380 *wc_status = IB_WC_INV_EECN_ERR;
381 break;
382 case 0x0C:
383 case 0x2F:
384 *wc_status = IB_WC_INV_EEC_STATE_ERR;
385 break;
386 case 0x0D:
387 *wc_status = IB_WC_BAD_RESP_ERR;
388 break;
389 case 0x10:
390 /* WQE purged */
391 *wc_status = IB_WC_WR_FLUSH_ERR;
392 break;
393 default:
394 *wc_status = IB_WC_FATAL_ERR;
395
396 }
397 } else
398 *wc_status = IB_WC_SUCCESS;
399 }
400
post_one_send(struct ehca_qp * my_qp,struct ib_send_wr * cur_send_wr,struct ib_send_wr ** bad_send_wr,int hidden)401 static inline int post_one_send(struct ehca_qp *my_qp,
402 struct ib_send_wr *cur_send_wr,
403 struct ib_send_wr **bad_send_wr,
404 int hidden)
405 {
406 struct ehca_wqe *wqe_p;
407 int ret;
408 u32 sq_map_idx;
409 u64 start_offset = my_qp->ipz_squeue.current_q_offset;
410
411 /* get pointer next to free WQE */
412 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
413 if (unlikely(!wqe_p)) {
414 /* too many posted work requests: queue overflow */
415 if (bad_send_wr)
416 *bad_send_wr = cur_send_wr;
417 ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
418 "qp_num=%x", my_qp->ib_qp.qp_num);
419 return -ENOMEM;
420 }
421
422 /*
423 * Get the index of the WQE in the send queue. The same index is used
424 * for writing into the sq_map.
425 */
426 sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
427
428 /* write a SEND WQE into the QUEUE */
429 ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
430 /*
431 * if something failed,
432 * reset the free entry pointer to the start value
433 */
434 if (unlikely(ret)) {
435 my_qp->ipz_squeue.current_q_offset = start_offset;
436 if (bad_send_wr)
437 *bad_send_wr = cur_send_wr;
438 ehca_err(my_qp->ib_qp.device, "Could not write WQE "
439 "qp_num=%x", my_qp->ib_qp.qp_num);
440 return -EINVAL;
441 }
442
443 return 0;
444 }
445
ehca_post_send(struct ib_qp * qp,struct ib_send_wr * send_wr,struct ib_send_wr ** bad_send_wr)446 int ehca_post_send(struct ib_qp *qp,
447 struct ib_send_wr *send_wr,
448 struct ib_send_wr **bad_send_wr)
449 {
450 struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
451 struct ib_send_wr *cur_send_wr;
452 int wqe_cnt = 0;
453 int ret = 0;
454 unsigned long flags;
455
456 /* Reject WR if QP is in RESET, INIT or RTR state */
457 if (unlikely(my_qp->state < IB_QPS_RTS)) {
458 ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
459 my_qp->state, qp->qp_num);
460 return -EINVAL;
461 }
462
463 /* LOCK the QUEUE */
464 spin_lock_irqsave(&my_qp->spinlock_s, flags);
465
466 /* Send an empty extra RDMA read if:
467 * 1) there has been an RDMA read on this connection before
468 * 2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
469 * 3) we can be sure that any previous extra RDMA read has been
470 * processed so we don't overflow the SQ
471 */
472 if (unlikely(my_qp->unsol_ack_circ &&
473 my_qp->packet_count > ACK_CIRC_THRESHOLD &&
474 my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
475 /* insert an empty RDMA READ to fix up the remote QP state */
476 struct ib_send_wr circ_wr;
477 memset(&circ_wr, 0, sizeof(circ_wr));
478 circ_wr.opcode = IB_WR_RDMA_READ;
479 post_one_send(my_qp, &circ_wr, NULL, 1); /* ignore retcode */
480 wqe_cnt++;
481 ehca_dbg(qp->device, "posted circ wr qp_num=%x", qp->qp_num);
482 my_qp->message_count = my_qp->packet_count = 0;
483 }
484
485 /* loop processes list of send reqs */
486 for (cur_send_wr = send_wr; cur_send_wr != NULL;
487 cur_send_wr = cur_send_wr->next) {
488 ret = post_one_send(my_qp, cur_send_wr, bad_send_wr, 0);
489 if (unlikely(ret)) {
490 /* if one or more WQEs were successful, don't fail */
491 if (wqe_cnt)
492 ret = 0;
493 goto post_send_exit0;
494 }
495 wqe_cnt++;
496 } /* eof for cur_send_wr */
497
498 post_send_exit0:
499 iosync(); /* serialize GAL register access */
500 hipz_update_sqa(my_qp, wqe_cnt);
501 if (unlikely(ret || ehca_debug_level >= 2))
502 ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
503 my_qp, qp->qp_num, wqe_cnt, ret);
504 my_qp->message_count += wqe_cnt;
505 spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
506 return ret;
507 }
508
internal_post_recv(struct ehca_qp * my_qp,struct ib_device * dev,struct ib_recv_wr * recv_wr,struct ib_recv_wr ** bad_recv_wr)509 static int internal_post_recv(struct ehca_qp *my_qp,
510 struct ib_device *dev,
511 struct ib_recv_wr *recv_wr,
512 struct ib_recv_wr **bad_recv_wr)
513 {
514 struct ib_recv_wr *cur_recv_wr;
515 struct ehca_wqe *wqe_p;
516 int wqe_cnt = 0;
517 int ret = 0;
518 u32 rq_map_idx;
519 unsigned long flags;
520 struct ehca_qmap_entry *qmap_entry;
521
522 if (unlikely(!HAS_RQ(my_qp))) {
523 ehca_err(dev, "QP has no RQ ehca_qp=%p qp_num=%x ext_type=%d",
524 my_qp, my_qp->real_qp_num, my_qp->ext_type);
525 return -ENODEV;
526 }
527
528 /* LOCK the QUEUE */
529 spin_lock_irqsave(&my_qp->spinlock_r, flags);
530
531 /* loop processes list of send reqs */
532 for (cur_recv_wr = recv_wr; cur_recv_wr != NULL;
533 cur_recv_wr = cur_recv_wr->next) {
534 u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
535 /* get pointer next to free WQE */
536 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
537 if (unlikely(!wqe_p)) {
538 /* too many posted work requests: queue overflow */
539 if (bad_recv_wr)
540 *bad_recv_wr = cur_recv_wr;
541 if (wqe_cnt == 0) {
542 ret = -ENOMEM;
543 ehca_err(dev, "Too many posted WQEs "
544 "qp_num=%x", my_qp->real_qp_num);
545 }
546 goto post_recv_exit0;
547 }
548 /*
549 * Get the index of the WQE in the recv queue. The same index
550 * is used for writing into the rq_map.
551 */
552 rq_map_idx = start_offset / my_qp->ipz_rqueue.qe_size;
553
554 /* write a RECV WQE into the QUEUE */
555 ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr,
556 rq_map_idx);
557 /*
558 * if something failed,
559 * reset the free entry pointer to the start value
560 */
561 if (unlikely(ret)) {
562 my_qp->ipz_rqueue.current_q_offset = start_offset;
563 *bad_recv_wr = cur_recv_wr;
564 if (wqe_cnt == 0) {
565 ret = -EINVAL;
566 ehca_err(dev, "Could not write WQE "
567 "qp_num=%x", my_qp->real_qp_num);
568 }
569 goto post_recv_exit0;
570 }
571
572 qmap_entry = &my_qp->rq_map.map[rq_map_idx];
573 qmap_entry->app_wr_id = get_app_wr_id(cur_recv_wr->wr_id);
574 qmap_entry->reported = 0;
575 qmap_entry->cqe_req = 1;
576
577 wqe_cnt++;
578 } /* eof for cur_recv_wr */
579
580 post_recv_exit0:
581 iosync(); /* serialize GAL register access */
582 hipz_update_rqa(my_qp, wqe_cnt);
583 if (unlikely(ret || ehca_debug_level >= 2))
584 ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
585 my_qp, my_qp->real_qp_num, wqe_cnt, ret);
586 spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
587 return ret;
588 }
589
ehca_post_recv(struct ib_qp * qp,struct ib_recv_wr * recv_wr,struct ib_recv_wr ** bad_recv_wr)590 int ehca_post_recv(struct ib_qp *qp,
591 struct ib_recv_wr *recv_wr,
592 struct ib_recv_wr **bad_recv_wr)
593 {
594 struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
595
596 /* Reject WR if QP is in RESET state */
597 if (unlikely(my_qp->state == IB_QPS_RESET)) {
598 ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
599 my_qp->state, qp->qp_num);
600 return -EINVAL;
601 }
602
603 return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
604 }
605
ehca_post_srq_recv(struct ib_srq * srq,struct ib_recv_wr * recv_wr,struct ib_recv_wr ** bad_recv_wr)606 int ehca_post_srq_recv(struct ib_srq *srq,
607 struct ib_recv_wr *recv_wr,
608 struct ib_recv_wr **bad_recv_wr)
609 {
610 return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
611 srq->device, recv_wr, bad_recv_wr);
612 }
613
614 /*
615 * ib_wc_opcode table converts ehca wc opcode to ib
616 * Since we use zero to indicate invalid opcode, the actual ib opcode must
617 * be decremented!!!
618 */
619 static const u8 ib_wc_opcode[255] = {
620 [0x01] = IB_WC_RECV+1,
621 [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
622 [0x04] = IB_WC_BIND_MW+1,
623 [0x08] = IB_WC_FETCH_ADD+1,
624 [0x10] = IB_WC_COMP_SWAP+1,
625 [0x20] = IB_WC_RDMA_WRITE+1,
626 [0x40] = IB_WC_RDMA_READ+1,
627 [0x80] = IB_WC_SEND+1
628 };
629
630 /* internal function to poll one entry of cq */
ehca_poll_cq_one(struct ib_cq * cq,struct ib_wc * wc)631 static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
632 {
633 int ret = 0, qmap_tail_idx;
634 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
635 struct ehca_cqe *cqe;
636 struct ehca_qp *my_qp;
637 struct ehca_qmap_entry *qmap_entry;
638 struct ehca_queue_map *qmap;
639 int cqe_count = 0, is_error;
640
641 repoll:
642 cqe = (struct ehca_cqe *)
643 ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
644 if (!cqe) {
645 ret = -EAGAIN;
646 if (ehca_debug_level >= 3)
647 ehca_dbg(cq->device, "Completion queue is empty "
648 "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
649 goto poll_cq_one_exit0;
650 }
651
652 /* prevents loads being reordered across this point */
653 rmb();
654
655 cqe_count++;
656 if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
657 struct ehca_qp *qp;
658 int purgeflag;
659 unsigned long flags;
660
661 qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
662 if (!qp) {
663 ehca_err(cq->device, "cq_num=%x qp_num=%x "
664 "could not find qp -> ignore cqe",
665 my_cq->cq_number, cqe->local_qp_number);
666 ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
667 my_cq->cq_number, cqe->local_qp_number);
668 /* ignore this purged cqe */
669 goto repoll;
670 }
671 spin_lock_irqsave(&qp->spinlock_s, flags);
672 purgeflag = qp->sqerr_purgeflag;
673 spin_unlock_irqrestore(&qp->spinlock_s, flags);
674
675 if (purgeflag) {
676 ehca_dbg(cq->device,
677 "Got CQE with purged bit qp_num=%x src_qp=%x",
678 cqe->local_qp_number, cqe->remote_qp_number);
679 if (ehca_debug_level >= 2)
680 ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
681 cqe->local_qp_number,
682 cqe->remote_qp_number);
683 /*
684 * ignore this to avoid double cqes of bad wqe
685 * that caused sqe and turn off purge flag
686 */
687 qp->sqerr_purgeflag = 0;
688 goto repoll;
689 }
690 }
691
692 is_error = cqe->status & WC_STATUS_ERROR_BIT;
693
694 /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
695 if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
696 ehca_dbg(cq->device,
697 "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
698 is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
699 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
700 my_cq, my_cq->cq_number);
701 ehca_dbg(cq->device,
702 "ehca_cq=%p cq_num=%x -------------------------",
703 my_cq, my_cq->cq_number);
704 }
705
706 read_lock(&ehca_qp_idr_lock);
707 my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
708 read_unlock(&ehca_qp_idr_lock);
709 if (!my_qp)
710 goto repoll;
711 wc->qp = &my_qp->ib_qp;
712
713 qmap_tail_idx = get_app_wr_id(cqe->work_request_id);
714 if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT))
715 /* We got a send completion. */
716 qmap = &my_qp->sq_map;
717 else
718 /* We got a receive completion. */
719 qmap = &my_qp->rq_map;
720
721 /* advance the tail pointer */
722 qmap->tail = qmap_tail_idx;
723
724 if (is_error) {
725 /*
726 * set left_to_poll to 0 because in error state, we will not
727 * get any additional CQEs
728 */
729 my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
730 my_qp->sq_map.entries);
731 my_qp->sq_map.left_to_poll = 0;
732 ehca_add_to_err_list(my_qp, 1);
733
734 my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
735 my_qp->rq_map.entries);
736 my_qp->rq_map.left_to_poll = 0;
737 if (HAS_RQ(my_qp))
738 ehca_add_to_err_list(my_qp, 0);
739 }
740
741 qmap_entry = &qmap->map[qmap_tail_idx];
742 if (qmap_entry->reported) {
743 ehca_warn(cq->device, "Double cqe on qp_num=%#x",
744 my_qp->real_qp_num);
745 /* found a double cqe, discard it and read next one */
746 goto repoll;
747 }
748
749 wc->wr_id = replace_wr_id(cqe->work_request_id, qmap_entry->app_wr_id);
750 qmap_entry->reported = 1;
751
752 /* if left_to_poll is decremented to 0, add the QP to the error list */
753 if (qmap->left_to_poll > 0) {
754 qmap->left_to_poll--;
755 if ((my_qp->sq_map.left_to_poll == 0) &&
756 (my_qp->rq_map.left_to_poll == 0)) {
757 ehca_add_to_err_list(my_qp, 1);
758 if (HAS_RQ(my_qp))
759 ehca_add_to_err_list(my_qp, 0);
760 }
761 }
762
763 /* eval ib_wc_opcode */
764 wc->opcode = ib_wc_opcode[cqe->optype]-1;
765 if (unlikely(wc->opcode == -1)) {
766 ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
767 "ehca_cq=%p cq_num=%x",
768 cqe->optype, cqe->status, my_cq, my_cq->cq_number);
769 /* dump cqe for other infos */
770 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
771 my_cq, my_cq->cq_number);
772 /* update also queue adder to throw away this entry!!! */
773 goto repoll;
774 }
775
776 /* eval ib_wc_status */
777 if (unlikely(is_error)) {
778 /* complete with errors */
779 map_ib_wc_status(cqe->status, &wc->status);
780 wc->vendor_err = wc->status;
781 } else
782 wc->status = IB_WC_SUCCESS;
783
784 wc->byte_len = cqe->nr_bytes_transferred;
785 wc->pkey_index = cqe->pkey_index;
786 wc->slid = cqe->rlid;
787 wc->dlid_path_bits = cqe->dlid;
788 wc->src_qp = cqe->remote_qp_number;
789 wc->wc_flags = cqe->w_completion_flags;
790 wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
791 wc->sl = cqe->service_level;
792
793 poll_cq_one_exit0:
794 if (cqe_count > 0)
795 hipz_update_feca(my_cq, cqe_count);
796
797 return ret;
798 }
799
generate_flush_cqes(struct ehca_qp * my_qp,struct ib_cq * cq,struct ib_wc * wc,int num_entries,struct ipz_queue * ipz_queue,int on_sq)800 static int generate_flush_cqes(struct ehca_qp *my_qp, struct ib_cq *cq,
801 struct ib_wc *wc, int num_entries,
802 struct ipz_queue *ipz_queue, int on_sq)
803 {
804 int nr = 0;
805 struct ehca_wqe *wqe;
806 u64 offset;
807 struct ehca_queue_map *qmap;
808 struct ehca_qmap_entry *qmap_entry;
809
810 if (on_sq)
811 qmap = &my_qp->sq_map;
812 else
813 qmap = &my_qp->rq_map;
814
815 qmap_entry = &qmap->map[qmap->next_wqe_idx];
816
817 while ((nr < num_entries) && (qmap_entry->reported == 0)) {
818 /* generate flush CQE */
819
820 memset(wc, 0, sizeof(*wc));
821
822 offset = qmap->next_wqe_idx * ipz_queue->qe_size;
823 wqe = (struct ehca_wqe *)ipz_qeit_calc(ipz_queue, offset);
824 if (!wqe) {
825 ehca_err(cq->device, "Invalid wqe offset=%#llx on "
826 "qp_num=%#x", offset, my_qp->real_qp_num);
827 return nr;
828 }
829
830 wc->wr_id = replace_wr_id(wqe->work_request_id,
831 qmap_entry->app_wr_id);
832
833 if (on_sq) {
834 switch (wqe->optype) {
835 case WQE_OPTYPE_SEND:
836 wc->opcode = IB_WC_SEND;
837 break;
838 case WQE_OPTYPE_RDMAWRITE:
839 wc->opcode = IB_WC_RDMA_WRITE;
840 break;
841 case WQE_OPTYPE_RDMAREAD:
842 wc->opcode = IB_WC_RDMA_READ;
843 break;
844 default:
845 ehca_err(cq->device, "Invalid optype=%x",
846 wqe->optype);
847 return nr;
848 }
849 } else
850 wc->opcode = IB_WC_RECV;
851
852 if (wqe->wr_flag & WQE_WRFLAG_IMM_DATA_PRESENT) {
853 wc->ex.imm_data = wqe->immediate_data;
854 wc->wc_flags |= IB_WC_WITH_IMM;
855 }
856
857 wc->status = IB_WC_WR_FLUSH_ERR;
858
859 wc->qp = &my_qp->ib_qp;
860
861 /* mark as reported and advance next_wqe pointer */
862 qmap_entry->reported = 1;
863 qmap->next_wqe_idx = next_index(qmap->next_wqe_idx,
864 qmap->entries);
865 qmap_entry = &qmap->map[qmap->next_wqe_idx];
866
867 wc++; nr++;
868 }
869
870 return nr;
871
872 }
873
ehca_poll_cq(struct ib_cq * cq,int num_entries,struct ib_wc * wc)874 int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
875 {
876 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
877 int nr;
878 struct ehca_qp *err_qp;
879 struct ib_wc *current_wc = wc;
880 int ret = 0;
881 unsigned long flags;
882 int entries_left = num_entries;
883
884 if (num_entries < 1) {
885 ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
886 "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
887 ret = -EINVAL;
888 goto poll_cq_exit0;
889 }
890
891 spin_lock_irqsave(&my_cq->spinlock, flags);
892
893 /* generate flush cqes for send queues */
894 list_for_each_entry(err_qp, &my_cq->sqp_err_list, sq_err_node) {
895 nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
896 &err_qp->ipz_squeue, 1);
897 entries_left -= nr;
898 current_wc += nr;
899
900 if (entries_left == 0)
901 break;
902 }
903
904 /* generate flush cqes for receive queues */
905 list_for_each_entry(err_qp, &my_cq->rqp_err_list, rq_err_node) {
906 nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
907 &err_qp->ipz_rqueue, 0);
908 entries_left -= nr;
909 current_wc += nr;
910
911 if (entries_left == 0)
912 break;
913 }
914
915 for (nr = 0; nr < entries_left; nr++) {
916 ret = ehca_poll_cq_one(cq, current_wc);
917 if (ret)
918 break;
919 current_wc++;
920 } /* eof for nr */
921 entries_left -= nr;
922
923 spin_unlock_irqrestore(&my_cq->spinlock, flags);
924 if (ret == -EAGAIN || !ret)
925 ret = num_entries - entries_left;
926
927 poll_cq_exit0:
928 return ret;
929 }
930
ehca_req_notify_cq(struct ib_cq * cq,enum ib_cq_notify_flags notify_flags)931 int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
932 {
933 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
934 int ret = 0;
935
936 switch (notify_flags & IB_CQ_SOLICITED_MASK) {
937 case IB_CQ_SOLICITED:
938 hipz_set_cqx_n0(my_cq, 1);
939 break;
940 case IB_CQ_NEXT_COMP:
941 hipz_set_cqx_n1(my_cq, 1);
942 break;
943 default:
944 return -EINVAL;
945 }
946
947 if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
948 unsigned long spl_flags;
949 spin_lock_irqsave(&my_cq->spinlock, spl_flags);
950 ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
951 spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
952 }
953
954 return ret;
955 }
956