1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2
3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4 /* Copyright (c) 2008-2019, IBM Corporation */
5
6 #include <linux/errno.h>
7 #include <linux/types.h>
8 #include <linux/uaccess.h>
9 #include <linux/vmalloc.h>
10 #include <linux/xarray.h>
11
12 #include <rdma/iw_cm.h>
13 #include <rdma/ib_verbs.h>
14 #include <rdma/ib_user_verbs.h>
15 #include <rdma/uverbs_ioctl.h>
16
17 #include "siw.h"
18 #include "siw_verbs.h"
19 #include "siw_mem.h"
20
21 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
22 [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
23 [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
24 [IB_QPS_RTR] = SIW_QP_STATE_RTR,
25 [IB_QPS_RTS] = SIW_QP_STATE_RTS,
26 [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
27 [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
28 [IB_QPS_ERR] = SIW_QP_STATE_ERROR
29 };
30
31 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
32 [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
33 [IB_QPS_RTS] = "RTS", [IB_QPS_SQD] = "SQD", [IB_QPS_SQE] = "SQE",
34 [IB_QPS_ERR] = "ERR"
35 };
36
siw_create_uobj(struct siw_ucontext * uctx,void * vaddr,u32 size)37 static u32 siw_create_uobj(struct siw_ucontext *uctx, void *vaddr, u32 size)
38 {
39 struct siw_uobj *uobj;
40 struct xa_limit limit = XA_LIMIT(0, SIW_UOBJ_MAX_KEY);
41 u32 key;
42
43 uobj = kzalloc(sizeof(*uobj), GFP_KERNEL);
44 if (!uobj)
45 return SIW_INVAL_UOBJ_KEY;
46
47 if (xa_alloc_cyclic(&uctx->xa, &key, uobj, limit, &uctx->uobj_nextkey,
48 GFP_KERNEL) < 0) {
49 kfree(uobj);
50 return SIW_INVAL_UOBJ_KEY;
51 }
52 uobj->size = PAGE_ALIGN(size);
53 uobj->addr = vaddr;
54
55 return key;
56 }
57
siw_get_uobj(struct siw_ucontext * uctx,unsigned long off,u32 size)58 static struct siw_uobj *siw_get_uobj(struct siw_ucontext *uctx,
59 unsigned long off, u32 size)
60 {
61 struct siw_uobj *uobj = xa_load(&uctx->xa, off);
62
63 if (uobj && uobj->size == size)
64 return uobj;
65
66 return NULL;
67 }
68
siw_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)69 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
70 {
71 struct siw_ucontext *uctx = to_siw_ctx(ctx);
72 struct siw_uobj *uobj;
73 unsigned long off = vma->vm_pgoff;
74 int size = vma->vm_end - vma->vm_start;
75 int rv = -EINVAL;
76
77 /*
78 * Must be page aligned
79 */
80 if (vma->vm_start & (PAGE_SIZE - 1)) {
81 pr_warn("siw: mmap not page aligned\n");
82 goto out;
83 }
84 uobj = siw_get_uobj(uctx, off, size);
85 if (!uobj) {
86 siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %u\n",
87 off, size);
88 goto out;
89 }
90 rv = remap_vmalloc_range(vma, uobj->addr, 0);
91 if (rv)
92 pr_warn("remap_vmalloc_range failed: %lu, %u\n", off, size);
93 out:
94 return rv;
95 }
96
siw_alloc_ucontext(struct ib_ucontext * base_ctx,struct ib_udata * udata)97 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
98 {
99 struct siw_device *sdev = to_siw_dev(base_ctx->device);
100 struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
101 struct siw_uresp_alloc_ctx uresp = {};
102 int rv;
103
104 if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
105 rv = -ENOMEM;
106 goto err_out;
107 }
108 xa_init_flags(&ctx->xa, XA_FLAGS_ALLOC);
109 ctx->uobj_nextkey = 0;
110 ctx->sdev = sdev;
111
112 uresp.dev_id = sdev->vendor_part_id;
113
114 if (udata->outlen < sizeof(uresp)) {
115 rv = -EINVAL;
116 goto err_out;
117 }
118 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
119 if (rv)
120 goto err_out;
121
122 siw_dbg(base_ctx->device, "success. now %d context(s)\n",
123 atomic_read(&sdev->num_ctx));
124
125 return 0;
126
127 err_out:
128 atomic_dec(&sdev->num_ctx);
129 siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
130 atomic_read(&sdev->num_ctx));
131
132 return rv;
133 }
134
siw_dealloc_ucontext(struct ib_ucontext * base_ctx)135 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
136 {
137 struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
138 void *entry;
139 unsigned long index;
140
141 /*
142 * Make sure all user mmap objects are gone. Since QP, CQ
143 * and SRQ destroy routines destroy related objects, nothing
144 * should be found here.
145 */
146 xa_for_each(&uctx->xa, index, entry) {
147 kfree(xa_erase(&uctx->xa, index));
148 pr_warn("siw: dropping orphaned uobj at %lu\n", index);
149 }
150 xa_destroy(&uctx->xa);
151 atomic_dec(&uctx->sdev->num_ctx);
152 }
153
siw_query_device(struct ib_device * base_dev,struct ib_device_attr * attr,struct ib_udata * udata)154 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
155 struct ib_udata *udata)
156 {
157 struct siw_device *sdev = to_siw_dev(base_dev);
158
159 if (udata->inlen || udata->outlen)
160 return -EINVAL;
161
162 memset(attr, 0, sizeof(*attr));
163
164 /* Revisit atomic caps if RFC 7306 gets supported */
165 attr->atomic_cap = 0;
166 attr->device_cap_flags =
167 IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_ALLOW_USER_UNREG;
168 attr->max_cq = sdev->attrs.max_cq;
169 attr->max_cqe = sdev->attrs.max_cqe;
170 attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
171 attr->max_fmr = sdev->attrs.max_fmr;
172 attr->max_mr = sdev->attrs.max_mr;
173 attr->max_mw = sdev->attrs.max_mw;
174 attr->max_mr_size = ~0ull;
175 attr->max_pd = sdev->attrs.max_pd;
176 attr->max_qp = sdev->attrs.max_qp;
177 attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
178 attr->max_qp_rd_atom = sdev->attrs.max_ord;
179 attr->max_qp_wr = sdev->attrs.max_qp_wr;
180 attr->max_recv_sge = sdev->attrs.max_sge;
181 attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
182 attr->max_send_sge = sdev->attrs.max_sge;
183 attr->max_sge_rd = sdev->attrs.max_sge_rd;
184 attr->max_srq = sdev->attrs.max_srq;
185 attr->max_srq_sge = sdev->attrs.max_srq_sge;
186 attr->max_srq_wr = sdev->attrs.max_srq_wr;
187 attr->page_size_cap = PAGE_SIZE;
188 attr->vendor_id = SIW_VENDOR_ID;
189 attr->vendor_part_id = sdev->vendor_part_id;
190
191 memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);
192
193 return 0;
194 }
195
siw_query_port(struct ib_device * base_dev,u8 port,struct ib_port_attr * attr)196 int siw_query_port(struct ib_device *base_dev, u8 port,
197 struct ib_port_attr *attr)
198 {
199 struct siw_device *sdev = to_siw_dev(base_dev);
200
201 memset(attr, 0, sizeof(*attr));
202
203 attr->active_mtu = attr->max_mtu;
204 attr->active_speed = 2;
205 attr->active_width = 2;
206 attr->gid_tbl_len = 1;
207 attr->max_msg_sz = -1;
208 attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
209 attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
210 IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
211 attr->pkey_tbl_len = 1;
212 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
213 attr->state = sdev->state;
214 /*
215 * All zero
216 *
217 * attr->lid = 0;
218 * attr->bad_pkey_cntr = 0;
219 * attr->qkey_viol_cntr = 0;
220 * attr->sm_lid = 0;
221 * attr->lmc = 0;
222 * attr->max_vl_num = 0;
223 * attr->sm_sl = 0;
224 * attr->subnet_timeout = 0;
225 * attr->init_type_repy = 0;
226 */
227 return 0;
228 }
229
siw_get_port_immutable(struct ib_device * base_dev,u8 port,struct ib_port_immutable * port_immutable)230 int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
231 struct ib_port_immutable *port_immutable)
232 {
233 struct ib_port_attr attr;
234 int rv = siw_query_port(base_dev, port, &attr);
235
236 if (rv)
237 return rv;
238
239 port_immutable->pkey_tbl_len = attr.pkey_tbl_len;
240 port_immutable->gid_tbl_len = attr.gid_tbl_len;
241 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
242
243 return 0;
244 }
245
siw_query_pkey(struct ib_device * base_dev,u8 port,u16 idx,u16 * pkey)246 int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey)
247 {
248 /* Report the default pkey */
249 *pkey = 0xffff;
250 return 0;
251 }
252
siw_query_gid(struct ib_device * base_dev,u8 port,int idx,union ib_gid * gid)253 int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
254 union ib_gid *gid)
255 {
256 struct siw_device *sdev = to_siw_dev(base_dev);
257
258 /* subnet_prefix == interface_id == 0; */
259 memset(gid, 0, sizeof(*gid));
260 memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
261
262 return 0;
263 }
264
siw_alloc_pd(struct ib_pd * pd,struct ib_udata * udata)265 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
266 {
267 struct siw_device *sdev = to_siw_dev(pd->device);
268
269 if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
270 atomic_dec(&sdev->num_pd);
271 return -ENOMEM;
272 }
273 siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
274
275 return 0;
276 }
277
siw_dealloc_pd(struct ib_pd * pd,struct ib_udata * udata)278 void siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
279 {
280 struct siw_device *sdev = to_siw_dev(pd->device);
281
282 siw_dbg_pd(pd, "free PD\n");
283 atomic_dec(&sdev->num_pd);
284 }
285
siw_qp_get_ref(struct ib_qp * base_qp)286 void siw_qp_get_ref(struct ib_qp *base_qp)
287 {
288 siw_qp_get(to_siw_qp(base_qp));
289 }
290
siw_qp_put_ref(struct ib_qp * base_qp)291 void siw_qp_put_ref(struct ib_qp *base_qp)
292 {
293 siw_qp_put(to_siw_qp(base_qp));
294 }
295
296 /*
297 * siw_create_qp()
298 *
299 * Create QP of requested size on given device.
300 *
301 * @pd: Protection Domain
302 * @attrs: Initial QP attributes.
303 * @udata: used to provide QP ID, SQ and RQ size back to user.
304 */
305
siw_create_qp(struct ib_pd * pd,struct ib_qp_init_attr * attrs,struct ib_udata * udata)306 struct ib_qp *siw_create_qp(struct ib_pd *pd,
307 struct ib_qp_init_attr *attrs,
308 struct ib_udata *udata)
309 {
310 struct siw_qp *qp = NULL;
311 struct siw_base_qp *siw_base_qp = NULL;
312 struct ib_device *base_dev = pd->device;
313 struct siw_device *sdev = to_siw_dev(base_dev);
314 struct siw_ucontext *uctx =
315 rdma_udata_to_drv_context(udata, struct siw_ucontext,
316 base_ucontext);
317 struct siw_cq *scq = NULL, *rcq = NULL;
318 unsigned long flags;
319 int num_sqe, num_rqe, rv = 0;
320
321 siw_dbg(base_dev, "create new QP\n");
322
323 if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
324 siw_dbg(base_dev, "too many QP's\n");
325 rv = -ENOMEM;
326 goto err_out;
327 }
328 if (attrs->qp_type != IB_QPT_RC) {
329 siw_dbg(base_dev, "only RC QP's supported\n");
330 rv = -EINVAL;
331 goto err_out;
332 }
333 if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
334 (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
335 (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
336 (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
337 siw_dbg(base_dev, "QP size error\n");
338 rv = -EINVAL;
339 goto err_out;
340 }
341 if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
342 siw_dbg(base_dev, "max inline send: %d > %d\n",
343 attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
344 rv = -EINVAL;
345 goto err_out;
346 }
347 /*
348 * NOTE: we allow for zero element SQ and RQ WQE's SGL's
349 * but not for a QP unable to hold any WQE (SQ + RQ)
350 */
351 if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
352 siw_dbg(base_dev, "QP must have send or receive queue\n");
353 rv = -EINVAL;
354 goto err_out;
355 }
356 scq = to_siw_cq(attrs->send_cq);
357 rcq = to_siw_cq(attrs->recv_cq);
358
359 if (!scq || (!rcq && !attrs->srq)) {
360 siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
361 rv = -EINVAL;
362 goto err_out;
363 }
364 siw_base_qp = kzalloc(sizeof(*siw_base_qp), GFP_KERNEL);
365 if (!siw_base_qp) {
366 rv = -ENOMEM;
367 goto err_out;
368 }
369 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
370 if (!qp) {
371 rv = -ENOMEM;
372 goto err_out;
373 }
374 siw_base_qp->qp = qp;
375 qp->ib_qp = &siw_base_qp->base_qp;
376
377 init_rwsem(&qp->state_lock);
378 spin_lock_init(&qp->sq_lock);
379 spin_lock_init(&qp->rq_lock);
380 spin_lock_init(&qp->orq_lock);
381
382 qp->kernel_verbs = !udata;
383 qp->xa_sq_index = SIW_INVAL_UOBJ_KEY;
384 qp->xa_rq_index = SIW_INVAL_UOBJ_KEY;
385
386 rv = siw_qp_add(sdev, qp);
387 if (rv)
388 goto err_out;
389
390 /* All queue indices are derived from modulo operations
391 * on a free running 'get' (consumer) and 'put' (producer)
392 * unsigned counter. Having queue sizes at power of two
393 * avoids handling counter wrap around.
394 */
395 num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr);
396 num_rqe = roundup_pow_of_two(attrs->cap.max_recv_wr);
397
398 if (qp->kernel_verbs)
399 qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
400 else
401 qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
402
403 if (qp->sendq == NULL) {
404 siw_dbg(base_dev, "SQ size %d alloc failed\n", num_sqe);
405 rv = -ENOMEM;
406 goto err_out_xa;
407 }
408 if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
409 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
410 qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
411 else {
412 rv = -EINVAL;
413 goto err_out_xa;
414 }
415 }
416 qp->pd = pd;
417 qp->scq = scq;
418 qp->rcq = rcq;
419
420 if (attrs->srq) {
421 /*
422 * SRQ support.
423 * Verbs 6.3.7: ignore RQ size, if SRQ present
424 * Verbs 6.3.5: do not check PD of SRQ against PD of QP
425 */
426 qp->srq = to_siw_srq(attrs->srq);
427 qp->attrs.rq_size = 0;
428 siw_dbg(base_dev, "QP [%u]: SRQ attached\n", qp->qp_num);
429 } else if (num_rqe) {
430 if (qp->kernel_verbs)
431 qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
432 else
433 qp->recvq =
434 vmalloc_user(num_rqe * sizeof(struct siw_rqe));
435
436 if (qp->recvq == NULL) {
437 siw_dbg(base_dev, "RQ size %d alloc failed\n", num_rqe);
438 rv = -ENOMEM;
439 goto err_out_xa;
440 }
441 qp->attrs.rq_size = num_rqe;
442 }
443 qp->attrs.sq_size = num_sqe;
444 qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
445 qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
446
447 /* Make those two tunables fixed for now. */
448 qp->tx_ctx.gso_seg_limit = 1;
449 qp->tx_ctx.zcopy_tx = zcopy_tx;
450
451 qp->attrs.state = SIW_QP_STATE_IDLE;
452
453 if (udata) {
454 struct siw_uresp_create_qp uresp = {};
455
456 uresp.num_sqe = num_sqe;
457 uresp.num_rqe = num_rqe;
458 uresp.qp_id = qp_id(qp);
459
460 if (qp->sendq) {
461 qp->xa_sq_index =
462 siw_create_uobj(uctx, qp->sendq,
463 num_sqe * sizeof(struct siw_sqe));
464 }
465 if (qp->recvq) {
466 qp->xa_rq_index =
467 siw_create_uobj(uctx, qp->recvq,
468 num_rqe * sizeof(struct siw_rqe));
469 }
470 if (qp->xa_sq_index == SIW_INVAL_UOBJ_KEY ||
471 qp->xa_rq_index == SIW_INVAL_UOBJ_KEY) {
472 rv = -ENOMEM;
473 goto err_out_xa;
474 }
475 uresp.sq_key = qp->xa_sq_index << PAGE_SHIFT;
476 uresp.rq_key = qp->xa_rq_index << PAGE_SHIFT;
477
478 if (udata->outlen < sizeof(uresp)) {
479 rv = -EINVAL;
480 goto err_out_xa;
481 }
482 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
483 if (rv)
484 goto err_out_xa;
485 }
486 qp->tx_cpu = siw_get_tx_cpu(sdev);
487 if (qp->tx_cpu < 0) {
488 rv = -EINVAL;
489 goto err_out_xa;
490 }
491 INIT_LIST_HEAD(&qp->devq);
492 spin_lock_irqsave(&sdev->lock, flags);
493 list_add_tail(&qp->devq, &sdev->qp_list);
494 spin_unlock_irqrestore(&sdev->lock, flags);
495
496 return qp->ib_qp;
497
498 err_out_xa:
499 xa_erase(&sdev->qp_xa, qp_id(qp));
500 err_out:
501 kfree(siw_base_qp);
502
503 if (qp) {
504 if (qp->xa_sq_index != SIW_INVAL_UOBJ_KEY)
505 kfree(xa_erase(&uctx->xa, qp->xa_sq_index));
506 if (qp->xa_rq_index != SIW_INVAL_UOBJ_KEY)
507 kfree(xa_erase(&uctx->xa, qp->xa_rq_index));
508
509 vfree(qp->sendq);
510 vfree(qp->recvq);
511 kfree(qp);
512 }
513 atomic_dec(&sdev->num_qp);
514
515 return ERR_PTR(rv);
516 }
517
518 /*
519 * Minimum siw_query_qp() verb interface.
520 *
521 * @qp_attr_mask is not used but all available information is provided
522 */
siw_query_qp(struct ib_qp * base_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)523 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
524 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
525 {
526 struct siw_qp *qp;
527 struct siw_device *sdev;
528
529 if (base_qp && qp_attr && qp_init_attr) {
530 qp = to_siw_qp(base_qp);
531 sdev = to_siw_dev(base_qp->device);
532 } else {
533 return -EINVAL;
534 }
535 qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
536 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
537 qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
538 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
539 qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
540 qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
541 qp_attr->max_rd_atomic = qp->attrs.irq_size;
542 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
543
544 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
545 IB_ACCESS_REMOTE_WRITE |
546 IB_ACCESS_REMOTE_READ;
547
548 qp_init_attr->qp_type = base_qp->qp_type;
549 qp_init_attr->send_cq = base_qp->send_cq;
550 qp_init_attr->recv_cq = base_qp->recv_cq;
551 qp_init_attr->srq = base_qp->srq;
552
553 qp_init_attr->cap = qp_attr->cap;
554
555 return 0;
556 }
557
siw_verbs_modify_qp(struct ib_qp * base_qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)558 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
559 int attr_mask, struct ib_udata *udata)
560 {
561 struct siw_qp_attrs new_attrs;
562 enum siw_qp_attr_mask siw_attr_mask = 0;
563 struct siw_qp *qp = to_siw_qp(base_qp);
564 int rv = 0;
565
566 if (!attr_mask)
567 return 0;
568
569 memset(&new_attrs, 0, sizeof(new_attrs));
570
571 if (attr_mask & IB_QP_ACCESS_FLAGS) {
572 siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
573
574 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
575 new_attrs.flags |= SIW_RDMA_READ_ENABLED;
576 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
577 new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
578 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
579 new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
580 }
581 if (attr_mask & IB_QP_STATE) {
582 siw_dbg_qp(qp, "desired IB QP state: %s\n",
583 ib_qp_state_to_string[attr->qp_state]);
584
585 new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
586
587 if (new_attrs.state > SIW_QP_STATE_RTS)
588 qp->tx_ctx.tx_suspend = 1;
589
590 siw_attr_mask |= SIW_QP_ATTR_STATE;
591 }
592 if (!siw_attr_mask)
593 goto out;
594
595 down_write(&qp->state_lock);
596
597 rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
598
599 up_write(&qp->state_lock);
600 out:
601 return rv;
602 }
603
siw_destroy_qp(struct ib_qp * base_qp,struct ib_udata * udata)604 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
605 {
606 struct siw_qp *qp = to_siw_qp(base_qp);
607 struct siw_ucontext *uctx =
608 rdma_udata_to_drv_context(udata, struct siw_ucontext,
609 base_ucontext);
610 struct siw_qp_attrs qp_attrs;
611
612 siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
613
614 /*
615 * Mark QP as in process of destruction to prevent from
616 * any async callbacks to RDMA core
617 */
618 qp->attrs.flags |= SIW_QP_IN_DESTROY;
619 qp->rx_stream.rx_suspend = 1;
620
621 if (uctx && qp->xa_sq_index != SIW_INVAL_UOBJ_KEY)
622 kfree(xa_erase(&uctx->xa, qp->xa_sq_index));
623 if (uctx && qp->xa_rq_index != SIW_INVAL_UOBJ_KEY)
624 kfree(xa_erase(&uctx->xa, qp->xa_rq_index));
625
626 down_write(&qp->state_lock);
627
628 qp_attrs.state = SIW_QP_STATE_ERROR;
629 siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
630
631 if (qp->cep) {
632 siw_cep_put(qp->cep);
633 qp->cep = NULL;
634 }
635 up_write(&qp->state_lock);
636
637 kfree(qp->tx_ctx.mpa_crc_hd);
638 kfree(qp->rx_stream.mpa_crc_hd);
639
640 qp->scq = qp->rcq = NULL;
641
642 siw_qp_put(qp);
643
644 return 0;
645 }
646
647 /*
648 * siw_copy_inline_sgl()
649 *
650 * Prepare sgl of inlined data for sending. For userland callers
651 * function checks if given buffer addresses and len's are within
652 * process context bounds.
653 * Data from all provided sge's are copied together into the wqe,
654 * referenced by a single sge.
655 */
siw_copy_inline_sgl(const struct ib_send_wr * core_wr,struct siw_sqe * sqe)656 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
657 struct siw_sqe *sqe)
658 {
659 struct ib_sge *core_sge = core_wr->sg_list;
660 void *kbuf = &sqe->sge[1];
661 int num_sge = core_wr->num_sge, bytes = 0;
662
663 sqe->sge[0].laddr = (uintptr_t)kbuf;
664 sqe->sge[0].lkey = 0;
665
666 while (num_sge--) {
667 if (!core_sge->length) {
668 core_sge++;
669 continue;
670 }
671 bytes += core_sge->length;
672 if (bytes > SIW_MAX_INLINE) {
673 bytes = -EINVAL;
674 break;
675 }
676 memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
677 core_sge->length);
678
679 kbuf += core_sge->length;
680 core_sge++;
681 }
682 sqe->sge[0].length = bytes > 0 ? bytes : 0;
683 sqe->num_sge = bytes > 0 ? 1 : 0;
684
685 return bytes;
686 }
687
688 /* Complete SQ WR's without processing */
siw_sq_flush_wr(struct siw_qp * qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)689 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
690 const struct ib_send_wr **bad_wr)
691 {
692 struct siw_sqe sqe = {};
693 int rv = 0;
694
695 while (wr) {
696 sqe.id = wr->wr_id;
697 sqe.opcode = wr->opcode;
698 rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR);
699 if (rv) {
700 if (bad_wr)
701 *bad_wr = wr;
702 break;
703 }
704 wr = wr->next;
705 }
706 return rv;
707 }
708
709 /* Complete RQ WR's without processing */
siw_rq_flush_wr(struct siw_qp * qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)710 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
711 const struct ib_recv_wr **bad_wr)
712 {
713 struct siw_rqe rqe = {};
714 int rv = 0;
715
716 while (wr) {
717 rqe.id = wr->wr_id;
718 rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
719 if (rv) {
720 if (bad_wr)
721 *bad_wr = wr;
722 break;
723 }
724 wr = wr->next;
725 }
726 return rv;
727 }
728
729 /*
730 * siw_post_send()
731 *
732 * Post a list of S-WR's to a SQ.
733 *
734 * @base_qp: Base QP contained in siw QP
735 * @wr: Null terminated list of user WR's
736 * @bad_wr: Points to failing WR in case of synchronous failure.
737 */
siw_post_send(struct ib_qp * base_qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)738 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
739 const struct ib_send_wr **bad_wr)
740 {
741 struct siw_qp *qp = to_siw_qp(base_qp);
742 struct siw_wqe *wqe = tx_wqe(qp);
743
744 unsigned long flags;
745 int rv = 0;
746
747 if (wr && !qp->kernel_verbs) {
748 siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
749 *bad_wr = wr;
750 return -EINVAL;
751 }
752
753 /*
754 * Try to acquire QP state lock. Must be non-blocking
755 * to accommodate kernel clients needs.
756 */
757 if (!down_read_trylock(&qp->state_lock)) {
758 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
759 /*
760 * ERROR state is final, so we can be sure
761 * this state will not change as long as the QP
762 * exists.
763 *
764 * This handles an ib_drain_sq() call with
765 * a concurrent request to set the QP state
766 * to ERROR.
767 */
768 rv = siw_sq_flush_wr(qp, wr, bad_wr);
769 } else {
770 siw_dbg_qp(qp, "QP locked, state %d\n",
771 qp->attrs.state);
772 *bad_wr = wr;
773 rv = -ENOTCONN;
774 }
775 return rv;
776 }
777 if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
778 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
779 /*
780 * Immediately flush this WR to CQ, if QP
781 * is in ERROR state. SQ is guaranteed to
782 * be empty, so WR complets in-order.
783 *
784 * Typically triggered by ib_drain_sq().
785 */
786 rv = siw_sq_flush_wr(qp, wr, bad_wr);
787 } else {
788 siw_dbg_qp(qp, "QP out of state %d\n",
789 qp->attrs.state);
790 *bad_wr = wr;
791 rv = -ENOTCONN;
792 }
793 up_read(&qp->state_lock);
794 return rv;
795 }
796 spin_lock_irqsave(&qp->sq_lock, flags);
797
798 while (wr) {
799 u32 idx = qp->sq_put % qp->attrs.sq_size;
800 struct siw_sqe *sqe = &qp->sendq[idx];
801
802 if (sqe->flags) {
803 siw_dbg_qp(qp, "sq full\n");
804 rv = -ENOMEM;
805 break;
806 }
807 if (wr->num_sge > qp->attrs.sq_max_sges) {
808 siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
809 rv = -EINVAL;
810 break;
811 }
812 sqe->id = wr->wr_id;
813
814 if ((wr->send_flags & IB_SEND_SIGNALED) ||
815 (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
816 sqe->flags |= SIW_WQE_SIGNALLED;
817
818 if (wr->send_flags & IB_SEND_FENCE)
819 sqe->flags |= SIW_WQE_READ_FENCE;
820
821 switch (wr->opcode) {
822 case IB_WR_SEND:
823 case IB_WR_SEND_WITH_INV:
824 if (wr->send_flags & IB_SEND_SOLICITED)
825 sqe->flags |= SIW_WQE_SOLICITED;
826
827 if (!(wr->send_flags & IB_SEND_INLINE)) {
828 siw_copy_sgl(wr->sg_list, sqe->sge,
829 wr->num_sge);
830 sqe->num_sge = wr->num_sge;
831 } else {
832 rv = siw_copy_inline_sgl(wr, sqe);
833 if (rv <= 0) {
834 rv = -EINVAL;
835 break;
836 }
837 sqe->flags |= SIW_WQE_INLINE;
838 sqe->num_sge = 1;
839 }
840 if (wr->opcode == IB_WR_SEND)
841 sqe->opcode = SIW_OP_SEND;
842 else {
843 sqe->opcode = SIW_OP_SEND_REMOTE_INV;
844 sqe->rkey = wr->ex.invalidate_rkey;
845 }
846 break;
847
848 case IB_WR_RDMA_READ_WITH_INV:
849 case IB_WR_RDMA_READ:
850 /*
851 * iWarp restricts RREAD sink to SGL containing
852 * 1 SGE only. we could relax to SGL with multiple
853 * elements referring the SAME ltag or even sending
854 * a private per-rreq tag referring to a checked
855 * local sgl with MULTIPLE ltag's.
856 */
857 if (unlikely(wr->num_sge != 1)) {
858 rv = -EINVAL;
859 break;
860 }
861 siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
862 /*
863 * NOTE: zero length RREAD is allowed!
864 */
865 sqe->raddr = rdma_wr(wr)->remote_addr;
866 sqe->rkey = rdma_wr(wr)->rkey;
867 sqe->num_sge = 1;
868
869 if (wr->opcode == IB_WR_RDMA_READ)
870 sqe->opcode = SIW_OP_READ;
871 else
872 sqe->opcode = SIW_OP_READ_LOCAL_INV;
873 break;
874
875 case IB_WR_RDMA_WRITE:
876 if (!(wr->send_flags & IB_SEND_INLINE)) {
877 siw_copy_sgl(wr->sg_list, &sqe->sge[0],
878 wr->num_sge);
879 sqe->num_sge = wr->num_sge;
880 } else {
881 rv = siw_copy_inline_sgl(wr, sqe);
882 if (unlikely(rv < 0)) {
883 rv = -EINVAL;
884 break;
885 }
886 sqe->flags |= SIW_WQE_INLINE;
887 sqe->num_sge = 1;
888 }
889 sqe->raddr = rdma_wr(wr)->remote_addr;
890 sqe->rkey = rdma_wr(wr)->rkey;
891 sqe->opcode = SIW_OP_WRITE;
892 break;
893
894 case IB_WR_REG_MR:
895 sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
896 sqe->rkey = reg_wr(wr)->key;
897 sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
898 sqe->opcode = SIW_OP_REG_MR;
899 break;
900
901 case IB_WR_LOCAL_INV:
902 sqe->rkey = wr->ex.invalidate_rkey;
903 sqe->opcode = SIW_OP_INVAL_STAG;
904 break;
905
906 default:
907 siw_dbg_qp(qp, "ib wr type %d unsupported\n",
908 wr->opcode);
909 rv = -EINVAL;
910 break;
911 }
912 siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
913 sqe->opcode, sqe->flags,
914 (void *)(uintptr_t)sqe->id);
915
916 if (unlikely(rv < 0))
917 break;
918
919 /* make SQE only valid after completely written */
920 smp_wmb();
921 sqe->flags |= SIW_WQE_VALID;
922
923 qp->sq_put++;
924 wr = wr->next;
925 }
926
927 /*
928 * Send directly if SQ processing is not in progress.
929 * Eventual immediate errors (rv < 0) do not affect the involved
930 * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
931 * processing, if new work is already pending. But rv must be passed
932 * to caller.
933 */
934 if (wqe->wr_status != SIW_WR_IDLE) {
935 spin_unlock_irqrestore(&qp->sq_lock, flags);
936 goto skip_direct_sending;
937 }
938 rv = siw_activate_tx(qp);
939 spin_unlock_irqrestore(&qp->sq_lock, flags);
940
941 if (rv <= 0)
942 goto skip_direct_sending;
943
944 if (qp->kernel_verbs) {
945 rv = siw_sq_start(qp);
946 } else {
947 qp->tx_ctx.in_syscall = 1;
948
949 if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
950 siw_qp_cm_drop(qp, 0);
951
952 qp->tx_ctx.in_syscall = 0;
953 }
954 skip_direct_sending:
955
956 up_read(&qp->state_lock);
957
958 if (rv >= 0)
959 return 0;
960 /*
961 * Immediate error
962 */
963 siw_dbg_qp(qp, "error %d\n", rv);
964
965 *bad_wr = wr;
966 return rv;
967 }
968
969 /*
970 * siw_post_receive()
971 *
972 * Post a list of R-WR's to a RQ.
973 *
974 * @base_qp: Base QP contained in siw QP
975 * @wr: Null terminated list of user WR's
976 * @bad_wr: Points to failing WR in case of synchronous failure.
977 */
siw_post_receive(struct ib_qp * base_qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)978 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
979 const struct ib_recv_wr **bad_wr)
980 {
981 struct siw_qp *qp = to_siw_qp(base_qp);
982 unsigned long flags;
983 int rv = 0;
984
985 if (qp->srq) {
986 *bad_wr = wr;
987 return -EOPNOTSUPP; /* what else from errno.h? */
988 }
989 if (!qp->kernel_verbs) {
990 siw_dbg_qp(qp, "no kernel post_recv for user mapped sq\n");
991 *bad_wr = wr;
992 return -EINVAL;
993 }
994
995 /*
996 * Try to acquire QP state lock. Must be non-blocking
997 * to accommodate kernel clients needs.
998 */
999 if (!down_read_trylock(&qp->state_lock)) {
1000 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1001 /*
1002 * ERROR state is final, so we can be sure
1003 * this state will not change as long as the QP
1004 * exists.
1005 *
1006 * This handles an ib_drain_rq() call with
1007 * a concurrent request to set the QP state
1008 * to ERROR.
1009 */
1010 rv = siw_rq_flush_wr(qp, wr, bad_wr);
1011 } else {
1012 siw_dbg_qp(qp, "QP locked, state %d\n",
1013 qp->attrs.state);
1014 *bad_wr = wr;
1015 rv = -ENOTCONN;
1016 }
1017 return rv;
1018 }
1019 if (qp->attrs.state > SIW_QP_STATE_RTS) {
1020 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1021 /*
1022 * Immediately flush this WR to CQ, if QP
1023 * is in ERROR state. RQ is guaranteed to
1024 * be empty, so WR complets in-order.
1025 *
1026 * Typically triggered by ib_drain_rq().
1027 */
1028 rv = siw_rq_flush_wr(qp, wr, bad_wr);
1029 } else {
1030 siw_dbg_qp(qp, "QP out of state %d\n",
1031 qp->attrs.state);
1032 *bad_wr = wr;
1033 rv = -ENOTCONN;
1034 }
1035 up_read(&qp->state_lock);
1036 return rv;
1037 }
1038 /*
1039 * Serialize potentially multiple producers.
1040 * Not needed for single threaded consumer side.
1041 */
1042 spin_lock_irqsave(&qp->rq_lock, flags);
1043
1044 while (wr) {
1045 u32 idx = qp->rq_put % qp->attrs.rq_size;
1046 struct siw_rqe *rqe = &qp->recvq[idx];
1047
1048 if (rqe->flags) {
1049 siw_dbg_qp(qp, "RQ full\n");
1050 rv = -ENOMEM;
1051 break;
1052 }
1053 if (wr->num_sge > qp->attrs.rq_max_sges) {
1054 siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1055 rv = -EINVAL;
1056 break;
1057 }
1058 rqe->id = wr->wr_id;
1059 rqe->num_sge = wr->num_sge;
1060 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1061
1062 /* make sure RQE is completely written before valid */
1063 smp_wmb();
1064
1065 rqe->flags = SIW_WQE_VALID;
1066
1067 qp->rq_put++;
1068 wr = wr->next;
1069 }
1070 spin_unlock_irqrestore(&qp->rq_lock, flags);
1071
1072 up_read(&qp->state_lock);
1073
1074 if (rv < 0) {
1075 siw_dbg_qp(qp, "error %d\n", rv);
1076 *bad_wr = wr;
1077 }
1078 return rv > 0 ? 0 : rv;
1079 }
1080
siw_destroy_cq(struct ib_cq * base_cq,struct ib_udata * udata)1081 void siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1082 {
1083 struct siw_cq *cq = to_siw_cq(base_cq);
1084 struct siw_device *sdev = to_siw_dev(base_cq->device);
1085 struct siw_ucontext *ctx =
1086 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1087 base_ucontext);
1088
1089 siw_dbg_cq(cq, "free CQ resources\n");
1090
1091 siw_cq_flush(cq);
1092
1093 if (ctx && cq->xa_cq_index != SIW_INVAL_UOBJ_KEY)
1094 kfree(xa_erase(&ctx->xa, cq->xa_cq_index));
1095
1096 atomic_dec(&sdev->num_cq);
1097
1098 vfree(cq->queue);
1099 }
1100
1101 /*
1102 * siw_create_cq()
1103 *
1104 * Populate CQ of requested size
1105 *
1106 * @base_cq: CQ as allocated by RDMA midlayer
1107 * @attr: Initial CQ attributes
1108 * @udata: relates to user context
1109 */
1110
siw_create_cq(struct ib_cq * base_cq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)1111 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1112 struct ib_udata *udata)
1113 {
1114 struct siw_device *sdev = to_siw_dev(base_cq->device);
1115 struct siw_cq *cq = to_siw_cq(base_cq);
1116 int rv, size = attr->cqe;
1117
1118 if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1119 siw_dbg(base_cq->device, "too many CQ's\n");
1120 rv = -ENOMEM;
1121 goto err_out;
1122 }
1123 if (size < 1 || size > sdev->attrs.max_cqe) {
1124 siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1125 rv = -EINVAL;
1126 goto err_out;
1127 }
1128 size = roundup_pow_of_two(size);
1129 cq->base_cq.cqe = size;
1130 cq->num_cqe = size;
1131 cq->xa_cq_index = SIW_INVAL_UOBJ_KEY;
1132
1133 if (!udata) {
1134 cq->kernel_verbs = 1;
1135 cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1136 sizeof(struct siw_cq_ctrl));
1137 } else {
1138 cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1139 sizeof(struct siw_cq_ctrl));
1140 }
1141 if (cq->queue == NULL) {
1142 rv = -ENOMEM;
1143 goto err_out;
1144 }
1145 get_random_bytes(&cq->id, 4);
1146 siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1147
1148 spin_lock_init(&cq->lock);
1149
1150 cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1151
1152 if (udata) {
1153 struct siw_uresp_create_cq uresp = {};
1154 struct siw_ucontext *ctx =
1155 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1156 base_ucontext);
1157
1158 cq->xa_cq_index =
1159 siw_create_uobj(ctx, cq->queue,
1160 size * sizeof(struct siw_cqe) +
1161 sizeof(struct siw_cq_ctrl));
1162 if (cq->xa_cq_index == SIW_INVAL_UOBJ_KEY) {
1163 rv = -ENOMEM;
1164 goto err_out;
1165 }
1166 uresp.cq_key = cq->xa_cq_index << PAGE_SHIFT;
1167 uresp.cq_id = cq->id;
1168 uresp.num_cqe = size;
1169
1170 if (udata->outlen < sizeof(uresp)) {
1171 rv = -EINVAL;
1172 goto err_out;
1173 }
1174 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1175 if (rv)
1176 goto err_out;
1177 }
1178 return 0;
1179
1180 err_out:
1181 siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1182
1183 if (cq && cq->queue) {
1184 struct siw_ucontext *ctx =
1185 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1186 base_ucontext);
1187 if (cq->xa_cq_index != SIW_INVAL_UOBJ_KEY)
1188 kfree(xa_erase(&ctx->xa, cq->xa_cq_index));
1189 vfree(cq->queue);
1190 }
1191 atomic_dec(&sdev->num_cq);
1192
1193 return rv;
1194 }
1195
1196 /*
1197 * siw_poll_cq()
1198 *
1199 * Reap CQ entries if available and copy work completion status into
1200 * array of WC's provided by caller. Returns number of reaped CQE's.
1201 *
1202 * @base_cq: Base CQ contained in siw CQ.
1203 * @num_cqe: Maximum number of CQE's to reap.
1204 * @wc: Array of work completions to be filled by siw.
1205 */
siw_poll_cq(struct ib_cq * base_cq,int num_cqe,struct ib_wc * wc)1206 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1207 {
1208 struct siw_cq *cq = to_siw_cq(base_cq);
1209 int i;
1210
1211 for (i = 0; i < num_cqe; i++) {
1212 if (!siw_reap_cqe(cq, wc))
1213 break;
1214 wc++;
1215 }
1216 return i;
1217 }
1218
1219 /*
1220 * siw_req_notify_cq()
1221 *
1222 * Request notification for new CQE's added to that CQ.
1223 * Defined flags:
1224 * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1225 * event if a WQE with notification flag set enters the CQ
1226 * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1227 * event if a WQE enters the CQ.
1228 * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1229 * number of not reaped CQE's regardless of its notification
1230 * type and current or new CQ notification settings.
1231 *
1232 * @base_cq: Base CQ contained in siw CQ.
1233 * @flags: Requested notification flags.
1234 */
siw_req_notify_cq(struct ib_cq * base_cq,enum ib_cq_notify_flags flags)1235 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1236 {
1237 struct siw_cq *cq = to_siw_cq(base_cq);
1238
1239 siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1240
1241 if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1242 /*
1243 * Enable CQ event for next solicited completion.
1244 * and make it visible to all associated producers.
1245 */
1246 smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1247 else
1248 /*
1249 * Enable CQ event for any signalled completion.
1250 * and make it visible to all associated producers.
1251 */
1252 smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1253
1254 if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1255 return cq->cq_put - cq->cq_get;
1256
1257 return 0;
1258 }
1259
1260 /*
1261 * siw_dereg_mr()
1262 *
1263 * Release Memory Region.
1264 *
1265 * @base_mr: Base MR contained in siw MR.
1266 * @udata: points to user context, unused.
1267 */
siw_dereg_mr(struct ib_mr * base_mr,struct ib_udata * udata)1268 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1269 {
1270 struct siw_mr *mr = to_siw_mr(base_mr);
1271 struct siw_device *sdev = to_siw_dev(base_mr->device);
1272
1273 siw_dbg_mem(mr->mem, "deregister MR\n");
1274
1275 atomic_dec(&sdev->num_mr);
1276
1277 siw_mr_drop_mem(mr);
1278 kfree_rcu(mr, rcu);
1279
1280 return 0;
1281 }
1282
1283 /*
1284 * siw_reg_user_mr()
1285 *
1286 * Register Memory Region.
1287 *
1288 * @pd: Protection Domain
1289 * @start: starting address of MR (virtual address)
1290 * @len: len of MR
1291 * @rnic_va: not used by siw
1292 * @rights: MR access rights
1293 * @udata: user buffer to communicate STag and Key.
1294 */
siw_reg_user_mr(struct ib_pd * pd,u64 start,u64 len,u64 rnic_va,int rights,struct ib_udata * udata)1295 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1296 u64 rnic_va, int rights, struct ib_udata *udata)
1297 {
1298 struct siw_mr *mr = NULL;
1299 struct siw_umem *umem = NULL;
1300 struct siw_ureq_reg_mr ureq;
1301 struct siw_device *sdev = to_siw_dev(pd->device);
1302
1303 unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1304 int rv;
1305
1306 siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1307 (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1308 (unsigned long long)len);
1309
1310 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1311 siw_dbg_pd(pd, "too many mr's\n");
1312 rv = -ENOMEM;
1313 goto err_out;
1314 }
1315 if (!len) {
1316 rv = -EINVAL;
1317 goto err_out;
1318 }
1319 if (mem_limit != RLIM_INFINITY) {
1320 unsigned long num_pages =
1321 (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1322 mem_limit >>= PAGE_SHIFT;
1323
1324 if (num_pages > mem_limit - current->mm->locked_vm) {
1325 siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1326 num_pages, mem_limit,
1327 current->mm->locked_vm);
1328 rv = -ENOMEM;
1329 goto err_out;
1330 }
1331 }
1332 umem = siw_umem_get(start, len, ib_access_writable(rights));
1333 if (IS_ERR(umem)) {
1334 rv = PTR_ERR(umem);
1335 siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1336 umem = NULL;
1337 goto err_out;
1338 }
1339 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1340 if (!mr) {
1341 rv = -ENOMEM;
1342 goto err_out;
1343 }
1344 rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1345 if (rv)
1346 goto err_out;
1347
1348 if (udata) {
1349 struct siw_uresp_reg_mr uresp = {};
1350 struct siw_mem *mem = mr->mem;
1351
1352 if (udata->inlen < sizeof(ureq)) {
1353 rv = -EINVAL;
1354 goto err_out;
1355 }
1356 rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1357 if (rv)
1358 goto err_out;
1359
1360 mr->base_mr.lkey |= ureq.stag_key;
1361 mr->base_mr.rkey |= ureq.stag_key;
1362 mem->stag |= ureq.stag_key;
1363 uresp.stag = mem->stag;
1364
1365 if (udata->outlen < sizeof(uresp)) {
1366 rv = -EINVAL;
1367 goto err_out;
1368 }
1369 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1370 if (rv)
1371 goto err_out;
1372 }
1373 mr->mem->stag_valid = 1;
1374
1375 return &mr->base_mr;
1376
1377 err_out:
1378 atomic_dec(&sdev->num_mr);
1379 if (mr) {
1380 if (mr->mem)
1381 siw_mr_drop_mem(mr);
1382 kfree_rcu(mr, rcu);
1383 } else {
1384 if (umem)
1385 siw_umem_release(umem, false);
1386 }
1387 return ERR_PTR(rv);
1388 }
1389
siw_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_sge,struct ib_udata * udata)1390 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1391 u32 max_sge, struct ib_udata *udata)
1392 {
1393 struct siw_device *sdev = to_siw_dev(pd->device);
1394 struct siw_mr *mr = NULL;
1395 struct siw_pbl *pbl = NULL;
1396 int rv;
1397
1398 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1399 siw_dbg_pd(pd, "too many mr's\n");
1400 rv = -ENOMEM;
1401 goto err_out;
1402 }
1403 if (mr_type != IB_MR_TYPE_MEM_REG) {
1404 siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1405 rv = -EOPNOTSUPP;
1406 goto err_out;
1407 }
1408 if (max_sge > SIW_MAX_SGE_PBL) {
1409 siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1410 rv = -ENOMEM;
1411 goto err_out;
1412 }
1413 pbl = siw_pbl_alloc(max_sge);
1414 if (IS_ERR(pbl)) {
1415 rv = PTR_ERR(pbl);
1416 siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1417 pbl = NULL;
1418 goto err_out;
1419 }
1420 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1421 if (!mr) {
1422 rv = -ENOMEM;
1423 goto err_out;
1424 }
1425 rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1426 if (rv)
1427 goto err_out;
1428
1429 mr->mem->is_pbl = 1;
1430
1431 siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1432
1433 return &mr->base_mr;
1434
1435 err_out:
1436 atomic_dec(&sdev->num_mr);
1437
1438 if (!mr) {
1439 kfree(pbl);
1440 } else {
1441 if (mr->mem)
1442 siw_mr_drop_mem(mr);
1443 kfree_rcu(mr, rcu);
1444 }
1445 siw_dbg_pd(pd, "failed: %d\n", rv);
1446
1447 return ERR_PTR(rv);
1448 }
1449
1450 /* Just used to count number of pages being mapped */
siw_set_pbl_page(struct ib_mr * base_mr,u64 buf_addr)1451 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1452 {
1453 return 0;
1454 }
1455
siw_map_mr_sg(struct ib_mr * base_mr,struct scatterlist * sl,int num_sle,unsigned int * sg_off)1456 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1457 unsigned int *sg_off)
1458 {
1459 struct scatterlist *slp;
1460 struct siw_mr *mr = to_siw_mr(base_mr);
1461 struct siw_mem *mem = mr->mem;
1462 struct siw_pbl *pbl = mem->pbl;
1463 struct siw_pble *pble;
1464 unsigned long pbl_size;
1465 int i, rv;
1466
1467 if (!pbl) {
1468 siw_dbg_mem(mem, "no PBL allocated\n");
1469 return -EINVAL;
1470 }
1471 pble = pbl->pbe;
1472
1473 if (pbl->max_buf < num_sle) {
1474 siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1475 mem->pbl->max_buf, num_sle);
1476 return -ENOMEM;
1477 }
1478 for_each_sg(sl, slp, num_sle, i) {
1479 if (sg_dma_len(slp) == 0) {
1480 siw_dbg_mem(mem, "empty SGE\n");
1481 return -EINVAL;
1482 }
1483 if (i == 0) {
1484 pble->addr = sg_dma_address(slp);
1485 pble->size = sg_dma_len(slp);
1486 pble->pbl_off = 0;
1487 pbl_size = pble->size;
1488 pbl->num_buf = 1;
1489 } else {
1490 /* Merge PBL entries if adjacent */
1491 if (pble->addr + pble->size == sg_dma_address(slp)) {
1492 pble->size += sg_dma_len(slp);
1493 } else {
1494 pble++;
1495 pbl->num_buf++;
1496 pble->addr = sg_dma_address(slp);
1497 pble->size = sg_dma_len(slp);
1498 pble->pbl_off = pbl_size;
1499 }
1500 pbl_size += sg_dma_len(slp);
1501 }
1502 siw_dbg_mem(mem,
1503 "sge[%d], size %u, addr 0x%p, total %lu\n",
1504 i, pble->size, (void *)(uintptr_t)pble->addr,
1505 pbl_size);
1506 }
1507 rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1508 if (rv > 0) {
1509 mem->len = base_mr->length;
1510 mem->va = base_mr->iova;
1511 siw_dbg_mem(mem,
1512 "%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1513 mem->len, (void *)(uintptr_t)mem->va, num_sle,
1514 pbl->num_buf);
1515 }
1516 return rv;
1517 }
1518
1519 /*
1520 * siw_get_dma_mr()
1521 *
1522 * Create a (empty) DMA memory region, where no umem is attached.
1523 */
siw_get_dma_mr(struct ib_pd * pd,int rights)1524 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1525 {
1526 struct siw_device *sdev = to_siw_dev(pd->device);
1527 struct siw_mr *mr = NULL;
1528 int rv;
1529
1530 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1531 siw_dbg_pd(pd, "too many mr's\n");
1532 rv = -ENOMEM;
1533 goto err_out;
1534 }
1535 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1536 if (!mr) {
1537 rv = -ENOMEM;
1538 goto err_out;
1539 }
1540 rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1541 if (rv)
1542 goto err_out;
1543
1544 mr->mem->stag_valid = 1;
1545
1546 siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1547
1548 return &mr->base_mr;
1549
1550 err_out:
1551 if (rv)
1552 kfree(mr);
1553
1554 atomic_dec(&sdev->num_mr);
1555
1556 return ERR_PTR(rv);
1557 }
1558
1559 /*
1560 * siw_create_srq()
1561 *
1562 * Create Shared Receive Queue of attributes @init_attrs
1563 * within protection domain given by @pd.
1564 *
1565 * @base_srq: Base SRQ contained in siw SRQ.
1566 * @init_attrs: SRQ init attributes.
1567 * @udata: points to user context
1568 */
siw_create_srq(struct ib_srq * base_srq,struct ib_srq_init_attr * init_attrs,struct ib_udata * udata)1569 int siw_create_srq(struct ib_srq *base_srq,
1570 struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1571 {
1572 struct siw_srq *srq = to_siw_srq(base_srq);
1573 struct ib_srq_attr *attrs = &init_attrs->attr;
1574 struct siw_device *sdev = to_siw_dev(base_srq->device);
1575 struct siw_ucontext *ctx =
1576 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1577 base_ucontext);
1578 int rv;
1579
1580 if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1581 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1582 rv = -ENOMEM;
1583 goto err_out;
1584 }
1585 if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1586 attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1587 rv = -EINVAL;
1588 goto err_out;
1589 }
1590 srq->max_sge = attrs->max_sge;
1591 srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1592 srq->xa_srq_index = SIW_INVAL_UOBJ_KEY;
1593 srq->limit = attrs->srq_limit;
1594 if (srq->limit)
1595 srq->armed = 1;
1596
1597 srq->kernel_verbs = !udata;
1598
1599 if (udata)
1600 srq->recvq =
1601 vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1602 else
1603 srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1604
1605 if (srq->recvq == NULL) {
1606 rv = -ENOMEM;
1607 goto err_out;
1608 }
1609 if (udata) {
1610 struct siw_uresp_create_srq uresp = {};
1611
1612 srq->xa_srq_index = siw_create_uobj(
1613 ctx, srq->recvq, srq->num_rqe * sizeof(struct siw_rqe));
1614
1615 if (srq->xa_srq_index == SIW_INVAL_UOBJ_KEY) {
1616 rv = -ENOMEM;
1617 goto err_out;
1618 }
1619 uresp.srq_key = srq->xa_srq_index;
1620 uresp.num_rqe = srq->num_rqe;
1621
1622 if (udata->outlen < sizeof(uresp)) {
1623 rv = -EINVAL;
1624 goto err_out;
1625 }
1626 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1627 if (rv)
1628 goto err_out;
1629 }
1630 spin_lock_init(&srq->lock);
1631
1632 siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1633
1634 return 0;
1635
1636 err_out:
1637 if (srq->recvq) {
1638 if (ctx && srq->xa_srq_index != SIW_INVAL_UOBJ_KEY)
1639 kfree(xa_erase(&ctx->xa, srq->xa_srq_index));
1640 vfree(srq->recvq);
1641 }
1642 atomic_dec(&sdev->num_srq);
1643
1644 return rv;
1645 }
1646
1647 /*
1648 * siw_modify_srq()
1649 *
1650 * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1651 * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1652 *
1653 * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1654 * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1655 */
siw_modify_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)1656 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1657 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1658 {
1659 struct siw_srq *srq = to_siw_srq(base_srq);
1660 unsigned long flags;
1661 int rv = 0;
1662
1663 spin_lock_irqsave(&srq->lock, flags);
1664
1665 if (attr_mask & IB_SRQ_MAX_WR) {
1666 /* resize request not yet supported */
1667 rv = -EOPNOTSUPP;
1668 goto out;
1669 }
1670 if (attr_mask & IB_SRQ_LIMIT) {
1671 if (attrs->srq_limit) {
1672 if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1673 rv = -EINVAL;
1674 goto out;
1675 }
1676 srq->armed = 1;
1677 } else {
1678 srq->armed = 0;
1679 }
1680 srq->limit = attrs->srq_limit;
1681 }
1682 out:
1683 spin_unlock_irqrestore(&srq->lock, flags);
1684
1685 return rv;
1686 }
1687
1688 /*
1689 * siw_query_srq()
1690 *
1691 * Query SRQ attributes.
1692 */
siw_query_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs)1693 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1694 {
1695 struct siw_srq *srq = to_siw_srq(base_srq);
1696 unsigned long flags;
1697
1698 spin_lock_irqsave(&srq->lock, flags);
1699
1700 attrs->max_wr = srq->num_rqe;
1701 attrs->max_sge = srq->max_sge;
1702 attrs->srq_limit = srq->limit;
1703
1704 spin_unlock_irqrestore(&srq->lock, flags);
1705
1706 return 0;
1707 }
1708
1709 /*
1710 * siw_destroy_srq()
1711 *
1712 * Destroy SRQ.
1713 * It is assumed that the SRQ is not referenced by any
1714 * QP anymore - the code trusts the RDMA core environment to keep track
1715 * of QP references.
1716 */
siw_destroy_srq(struct ib_srq * base_srq,struct ib_udata * udata)1717 void siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1718 {
1719 struct siw_srq *srq = to_siw_srq(base_srq);
1720 struct siw_device *sdev = to_siw_dev(base_srq->device);
1721 struct siw_ucontext *ctx =
1722 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1723 base_ucontext);
1724
1725 if (ctx && srq->xa_srq_index != SIW_INVAL_UOBJ_KEY)
1726 kfree(xa_erase(&ctx->xa, srq->xa_srq_index));
1727
1728 vfree(srq->recvq);
1729 atomic_dec(&sdev->num_srq);
1730 }
1731
1732 /*
1733 * siw_post_srq_recv()
1734 *
1735 * Post a list of receive queue elements to SRQ.
1736 * NOTE: The function does not check or lock a certain SRQ state
1737 * during the post operation. The code simply trusts the
1738 * RDMA core environment.
1739 *
1740 * @base_srq: Base SRQ contained in siw SRQ
1741 * @wr: List of R-WR's
1742 * @bad_wr: Updated to failing WR if posting fails.
1743 */
siw_post_srq_recv(struct ib_srq * base_srq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)1744 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1745 const struct ib_recv_wr **bad_wr)
1746 {
1747 struct siw_srq *srq = to_siw_srq(base_srq);
1748 unsigned long flags;
1749 int rv = 0;
1750
1751 if (unlikely(!srq->kernel_verbs)) {
1752 siw_dbg_pd(base_srq->pd,
1753 "[SRQ]: no kernel post_recv for mapped srq\n");
1754 rv = -EINVAL;
1755 goto out;
1756 }
1757 /*
1758 * Serialize potentially multiple producers.
1759 * Also needed to serialize potentially multiple
1760 * consumers.
1761 */
1762 spin_lock_irqsave(&srq->lock, flags);
1763
1764 while (wr) {
1765 u32 idx = srq->rq_put % srq->num_rqe;
1766 struct siw_rqe *rqe = &srq->recvq[idx];
1767
1768 if (rqe->flags) {
1769 siw_dbg_pd(base_srq->pd, "SRQ full\n");
1770 rv = -ENOMEM;
1771 break;
1772 }
1773 if (unlikely(wr->num_sge > srq->max_sge)) {
1774 siw_dbg_pd(base_srq->pd,
1775 "[SRQ]: too many sge's: %d\n", wr->num_sge);
1776 rv = -EINVAL;
1777 break;
1778 }
1779 rqe->id = wr->wr_id;
1780 rqe->num_sge = wr->num_sge;
1781 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1782
1783 /* Make sure S-RQE is completely written before valid */
1784 smp_wmb();
1785
1786 rqe->flags = SIW_WQE_VALID;
1787
1788 srq->rq_put++;
1789 wr = wr->next;
1790 }
1791 spin_unlock_irqrestore(&srq->lock, flags);
1792 out:
1793 if (unlikely(rv < 0)) {
1794 siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1795 *bad_wr = wr;
1796 }
1797 return rv;
1798 }
1799
siw_qp_event(struct siw_qp * qp,enum ib_event_type etype)1800 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1801 {
1802 struct ib_event event;
1803 struct ib_qp *base_qp = qp->ib_qp;
1804
1805 /*
1806 * Do not report asynchronous errors on QP which gets
1807 * destroyed via verbs interface (siw_destroy_qp())
1808 */
1809 if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1810 return;
1811
1812 event.event = etype;
1813 event.device = base_qp->device;
1814 event.element.qp = base_qp;
1815
1816 if (base_qp->event_handler) {
1817 siw_dbg_qp(qp, "reporting event %d\n", etype);
1818 base_qp->event_handler(&event, base_qp->qp_context);
1819 }
1820 }
1821
siw_cq_event(struct siw_cq * cq,enum ib_event_type etype)1822 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1823 {
1824 struct ib_event event;
1825 struct ib_cq *base_cq = &cq->base_cq;
1826
1827 event.event = etype;
1828 event.device = base_cq->device;
1829 event.element.cq = base_cq;
1830
1831 if (base_cq->event_handler) {
1832 siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1833 base_cq->event_handler(&event, base_cq->cq_context);
1834 }
1835 }
1836
siw_srq_event(struct siw_srq * srq,enum ib_event_type etype)1837 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1838 {
1839 struct ib_event event;
1840 struct ib_srq *base_srq = &srq->base_srq;
1841
1842 event.event = etype;
1843 event.device = base_srq->device;
1844 event.element.srq = base_srq;
1845
1846 if (base_srq->event_handler) {
1847 siw_dbg_pd(srq->base_srq.pd,
1848 "reporting SRQ event %d\n", etype);
1849 base_srq->event_handler(&event, base_srq->srq_context);
1850 }
1851 }
1852
siw_port_event(struct siw_device * sdev,u8 port,enum ib_event_type etype)1853 void siw_port_event(struct siw_device *sdev, u8 port, enum ib_event_type etype)
1854 {
1855 struct ib_event event;
1856
1857 event.event = etype;
1858 event.device = &sdev->base_dev;
1859 event.element.port_num = port;
1860
1861 siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1862
1863 ib_dispatch_event(&event);
1864 }
1865