1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2
3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4 /* Copyright (c) 2008-2019, IBM Corporation */
5
6 #include <linux/errno.h>
7 #include <linux/types.h>
8 #include <linux/uaccess.h>
9 #include <linux/vmalloc.h>
10 #include <linux/xarray.h>
11
12 #include <rdma/iw_cm.h>
13 #include <rdma/ib_verbs.h>
14 #include <rdma/ib_user_verbs.h>
15 #include <rdma/uverbs_ioctl.h>
16
17 #include "siw.h"
18 #include "siw_verbs.h"
19 #include "siw_mem.h"
20
21 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
22 [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
23 [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
24 [IB_QPS_RTR] = SIW_QP_STATE_RTR,
25 [IB_QPS_RTS] = SIW_QP_STATE_RTS,
26 [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
27 [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
28 [IB_QPS_ERR] = SIW_QP_STATE_ERROR
29 };
30
31 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
32 [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
33 [IB_QPS_RTS] = "RTS", [IB_QPS_SQD] = "SQD", [IB_QPS_SQE] = "SQE",
34 [IB_QPS_ERR] = "ERR"
35 };
36
siw_mmap_free(struct rdma_user_mmap_entry * rdma_entry)37 void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
38 {
39 struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
40
41 kfree(entry);
42 }
43
siw_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)44 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
45 {
46 struct siw_ucontext *uctx = to_siw_ctx(ctx);
47 size_t size = vma->vm_end - vma->vm_start;
48 struct rdma_user_mmap_entry *rdma_entry;
49 struct siw_user_mmap_entry *entry;
50 int rv = -EINVAL;
51
52 /*
53 * Must be page aligned
54 */
55 if (vma->vm_start & (PAGE_SIZE - 1)) {
56 pr_warn("siw: mmap not page aligned\n");
57 return -EINVAL;
58 }
59 rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
60 if (!rdma_entry) {
61 siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
62 vma->vm_pgoff, size);
63 return -EINVAL;
64 }
65 entry = to_siw_mmap_entry(rdma_entry);
66
67 rv = remap_vmalloc_range(vma, entry->address, 0);
68 if (rv) {
69 pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
70 size);
71 goto out;
72 }
73 out:
74 rdma_user_mmap_entry_put(rdma_entry);
75
76 return rv;
77 }
78
siw_alloc_ucontext(struct ib_ucontext * base_ctx,struct ib_udata * udata)79 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
80 {
81 struct siw_device *sdev = to_siw_dev(base_ctx->device);
82 struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
83 struct siw_uresp_alloc_ctx uresp = {};
84 int rv;
85
86 if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
87 rv = -ENOMEM;
88 goto err_out;
89 }
90 ctx->sdev = sdev;
91
92 uresp.dev_id = sdev->vendor_part_id;
93
94 if (udata->outlen < sizeof(uresp)) {
95 rv = -EINVAL;
96 goto err_out;
97 }
98 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
99 if (rv)
100 goto err_out;
101
102 siw_dbg(base_ctx->device, "success. now %d context(s)\n",
103 atomic_read(&sdev->num_ctx));
104
105 return 0;
106
107 err_out:
108 atomic_dec(&sdev->num_ctx);
109 siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
110 atomic_read(&sdev->num_ctx));
111
112 return rv;
113 }
114
siw_dealloc_ucontext(struct ib_ucontext * base_ctx)115 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
116 {
117 struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
118
119 atomic_dec(&uctx->sdev->num_ctx);
120 }
121
siw_query_device(struct ib_device * base_dev,struct ib_device_attr * attr,struct ib_udata * udata)122 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
123 struct ib_udata *udata)
124 {
125 struct siw_device *sdev = to_siw_dev(base_dev);
126
127 if (udata->inlen || udata->outlen)
128 return -EINVAL;
129
130 memset(attr, 0, sizeof(*attr));
131
132 /* Revisit atomic caps if RFC 7306 gets supported */
133 attr->atomic_cap = 0;
134 attr->device_cap_flags =
135 IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_ALLOW_USER_UNREG;
136 attr->max_cq = sdev->attrs.max_cq;
137 attr->max_cqe = sdev->attrs.max_cqe;
138 attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
139 attr->max_mr = sdev->attrs.max_mr;
140 attr->max_mw = sdev->attrs.max_mw;
141 attr->max_mr_size = ~0ull;
142 attr->max_pd = sdev->attrs.max_pd;
143 attr->max_qp = sdev->attrs.max_qp;
144 attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
145 attr->max_qp_rd_atom = sdev->attrs.max_ord;
146 attr->max_qp_wr = sdev->attrs.max_qp_wr;
147 attr->max_recv_sge = sdev->attrs.max_sge;
148 attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
149 attr->max_send_sge = sdev->attrs.max_sge;
150 attr->max_sge_rd = sdev->attrs.max_sge_rd;
151 attr->max_srq = sdev->attrs.max_srq;
152 attr->max_srq_sge = sdev->attrs.max_srq_sge;
153 attr->max_srq_wr = sdev->attrs.max_srq_wr;
154 attr->page_size_cap = PAGE_SIZE;
155 attr->vendor_id = SIW_VENDOR_ID;
156 attr->vendor_part_id = sdev->vendor_part_id;
157
158 memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);
159
160 return 0;
161 }
162
siw_query_port(struct ib_device * base_dev,u8 port,struct ib_port_attr * attr)163 int siw_query_port(struct ib_device *base_dev, u8 port,
164 struct ib_port_attr *attr)
165 {
166 struct siw_device *sdev = to_siw_dev(base_dev);
167 int rv;
168
169 memset(attr, 0, sizeof(*attr));
170
171 rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
172 &attr->active_width);
173 attr->gid_tbl_len = 1;
174 attr->max_msg_sz = -1;
175 attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
176 attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
177 attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
178 IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
179 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
180 attr->state = sdev->state;
181 /*
182 * All zero
183 *
184 * attr->lid = 0;
185 * attr->bad_pkey_cntr = 0;
186 * attr->qkey_viol_cntr = 0;
187 * attr->sm_lid = 0;
188 * attr->lmc = 0;
189 * attr->max_vl_num = 0;
190 * attr->sm_sl = 0;
191 * attr->subnet_timeout = 0;
192 * attr->init_type_repy = 0;
193 */
194 return rv;
195 }
196
siw_get_port_immutable(struct ib_device * base_dev,u8 port,struct ib_port_immutable * port_immutable)197 int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
198 struct ib_port_immutable *port_immutable)
199 {
200 struct ib_port_attr attr;
201 int rv = siw_query_port(base_dev, port, &attr);
202
203 if (rv)
204 return rv;
205
206 port_immutable->gid_tbl_len = attr.gid_tbl_len;
207 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
208
209 return 0;
210 }
211
siw_query_gid(struct ib_device * base_dev,u8 port,int idx,union ib_gid * gid)212 int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
213 union ib_gid *gid)
214 {
215 struct siw_device *sdev = to_siw_dev(base_dev);
216
217 /* subnet_prefix == interface_id == 0; */
218 memset(gid, 0, sizeof(*gid));
219 memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
220
221 return 0;
222 }
223
siw_alloc_pd(struct ib_pd * pd,struct ib_udata * udata)224 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
225 {
226 struct siw_device *sdev = to_siw_dev(pd->device);
227
228 if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
229 atomic_dec(&sdev->num_pd);
230 return -ENOMEM;
231 }
232 siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
233
234 return 0;
235 }
236
siw_dealloc_pd(struct ib_pd * pd,struct ib_udata * udata)237 int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
238 {
239 struct siw_device *sdev = to_siw_dev(pd->device);
240
241 siw_dbg_pd(pd, "free PD\n");
242 atomic_dec(&sdev->num_pd);
243 return 0;
244 }
245
siw_qp_get_ref(struct ib_qp * base_qp)246 void siw_qp_get_ref(struct ib_qp *base_qp)
247 {
248 siw_qp_get(to_siw_qp(base_qp));
249 }
250
siw_qp_put_ref(struct ib_qp * base_qp)251 void siw_qp_put_ref(struct ib_qp *base_qp)
252 {
253 siw_qp_put(to_siw_qp(base_qp));
254 }
255
256 static struct rdma_user_mmap_entry *
siw_mmap_entry_insert(struct siw_ucontext * uctx,void * address,size_t length,u64 * offset)257 siw_mmap_entry_insert(struct siw_ucontext *uctx,
258 void *address, size_t length,
259 u64 *offset)
260 {
261 struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
262 int rv;
263
264 *offset = SIW_INVAL_UOBJ_KEY;
265 if (!entry)
266 return NULL;
267
268 entry->address = address;
269
270 rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
271 &entry->rdma_entry,
272 length);
273 if (rv) {
274 kfree(entry);
275 return NULL;
276 }
277
278 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
279
280 return &entry->rdma_entry;
281 }
282
283 /*
284 * siw_create_qp()
285 *
286 * Create QP of requested size on given device.
287 *
288 * @pd: Protection Domain
289 * @attrs: Initial QP attributes.
290 * @udata: used to provide QP ID, SQ and RQ size back to user.
291 */
292
siw_create_qp(struct ib_pd * pd,struct ib_qp_init_attr * attrs,struct ib_udata * udata)293 struct ib_qp *siw_create_qp(struct ib_pd *pd,
294 struct ib_qp_init_attr *attrs,
295 struct ib_udata *udata)
296 {
297 struct siw_qp *qp = NULL;
298 struct ib_device *base_dev = pd->device;
299 struct siw_device *sdev = to_siw_dev(base_dev);
300 struct siw_ucontext *uctx =
301 rdma_udata_to_drv_context(udata, struct siw_ucontext,
302 base_ucontext);
303 unsigned long flags;
304 int num_sqe, num_rqe, rv = 0;
305 size_t length;
306
307 siw_dbg(base_dev, "create new QP\n");
308
309 if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
310 siw_dbg(base_dev, "too many QP's\n");
311 rv = -ENOMEM;
312 goto err_out;
313 }
314 if (attrs->qp_type != IB_QPT_RC) {
315 siw_dbg(base_dev, "only RC QP's supported\n");
316 rv = -EOPNOTSUPP;
317 goto err_out;
318 }
319 if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
320 (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
321 (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
322 (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
323 siw_dbg(base_dev, "QP size error\n");
324 rv = -EINVAL;
325 goto err_out;
326 }
327 if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
328 siw_dbg(base_dev, "max inline send: %d > %d\n",
329 attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
330 rv = -EINVAL;
331 goto err_out;
332 }
333 /*
334 * NOTE: we allow for zero element SQ and RQ WQE's SGL's
335 * but not for a QP unable to hold any WQE (SQ + RQ)
336 */
337 if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
338 siw_dbg(base_dev, "QP must have send or receive queue\n");
339 rv = -EINVAL;
340 goto err_out;
341 }
342
343 if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) {
344 siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
345 rv = -EINVAL;
346 goto err_out;
347 }
348 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
349 if (!qp) {
350 rv = -ENOMEM;
351 goto err_out;
352 }
353 init_rwsem(&qp->state_lock);
354 spin_lock_init(&qp->sq_lock);
355 spin_lock_init(&qp->rq_lock);
356 spin_lock_init(&qp->orq_lock);
357
358 rv = siw_qp_add(sdev, qp);
359 if (rv)
360 goto err_out;
361
362 num_sqe = attrs->cap.max_send_wr;
363 num_rqe = attrs->cap.max_recv_wr;
364
365 /* All queue indices are derived from modulo operations
366 * on a free running 'get' (consumer) and 'put' (producer)
367 * unsigned counter. Having queue sizes at power of two
368 * avoids handling counter wrap around.
369 */
370 if (num_sqe)
371 num_sqe = roundup_pow_of_two(num_sqe);
372 else {
373 /* Zero sized SQ is not supported */
374 rv = -EINVAL;
375 goto err_out_xa;
376 }
377 if (num_rqe)
378 num_rqe = roundup_pow_of_two(num_rqe);
379
380 if (udata)
381 qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
382 else
383 qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
384
385 if (qp->sendq == NULL) {
386 rv = -ENOMEM;
387 goto err_out_xa;
388 }
389 if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
390 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
391 qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
392 else {
393 rv = -EINVAL;
394 goto err_out_xa;
395 }
396 }
397 qp->pd = pd;
398 qp->scq = to_siw_cq(attrs->send_cq);
399 qp->rcq = to_siw_cq(attrs->recv_cq);
400
401 if (attrs->srq) {
402 /*
403 * SRQ support.
404 * Verbs 6.3.7: ignore RQ size, if SRQ present
405 * Verbs 6.3.5: do not check PD of SRQ against PD of QP
406 */
407 qp->srq = to_siw_srq(attrs->srq);
408 qp->attrs.rq_size = 0;
409 siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
410 qp->base_qp.qp_num);
411 } else if (num_rqe) {
412 if (udata)
413 qp->recvq =
414 vmalloc_user(num_rqe * sizeof(struct siw_rqe));
415 else
416 qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
417
418 if (qp->recvq == NULL) {
419 rv = -ENOMEM;
420 goto err_out_xa;
421 }
422 qp->attrs.rq_size = num_rqe;
423 }
424 qp->attrs.sq_size = num_sqe;
425 qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
426 qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
427
428 /* Make those two tunables fixed for now. */
429 qp->tx_ctx.gso_seg_limit = 1;
430 qp->tx_ctx.zcopy_tx = zcopy_tx;
431
432 qp->attrs.state = SIW_QP_STATE_IDLE;
433
434 if (udata) {
435 struct siw_uresp_create_qp uresp = {};
436
437 uresp.num_sqe = num_sqe;
438 uresp.num_rqe = num_rqe;
439 uresp.qp_id = qp_id(qp);
440
441 if (qp->sendq) {
442 length = num_sqe * sizeof(struct siw_sqe);
443 qp->sq_entry =
444 siw_mmap_entry_insert(uctx, qp->sendq,
445 length, &uresp.sq_key);
446 if (!qp->sq_entry) {
447 rv = -ENOMEM;
448 goto err_out_xa;
449 }
450 }
451
452 if (qp->recvq) {
453 length = num_rqe * sizeof(struct siw_rqe);
454 qp->rq_entry =
455 siw_mmap_entry_insert(uctx, qp->recvq,
456 length, &uresp.rq_key);
457 if (!qp->rq_entry) {
458 uresp.sq_key = SIW_INVAL_UOBJ_KEY;
459 rv = -ENOMEM;
460 goto err_out_xa;
461 }
462 }
463
464 if (udata->outlen < sizeof(uresp)) {
465 rv = -EINVAL;
466 goto err_out_xa;
467 }
468 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
469 if (rv)
470 goto err_out_xa;
471 }
472 qp->tx_cpu = siw_get_tx_cpu(sdev);
473 if (qp->tx_cpu < 0) {
474 rv = -EINVAL;
475 goto err_out_xa;
476 }
477 INIT_LIST_HEAD(&qp->devq);
478 spin_lock_irqsave(&sdev->lock, flags);
479 list_add_tail(&qp->devq, &sdev->qp_list);
480 spin_unlock_irqrestore(&sdev->lock, flags);
481
482 return &qp->base_qp;
483
484 err_out_xa:
485 xa_erase(&sdev->qp_xa, qp_id(qp));
486 err_out:
487 if (qp) {
488 if (uctx) {
489 rdma_user_mmap_entry_remove(qp->sq_entry);
490 rdma_user_mmap_entry_remove(qp->rq_entry);
491 }
492 vfree(qp->sendq);
493 vfree(qp->recvq);
494 kfree(qp);
495 }
496 atomic_dec(&sdev->num_qp);
497
498 return ERR_PTR(rv);
499 }
500
501 /*
502 * Minimum siw_query_qp() verb interface.
503 *
504 * @qp_attr_mask is not used but all available information is provided
505 */
siw_query_qp(struct ib_qp * base_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)506 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
507 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
508 {
509 struct siw_qp *qp;
510 struct siw_device *sdev;
511
512 if (base_qp && qp_attr && qp_init_attr) {
513 qp = to_siw_qp(base_qp);
514 sdev = to_siw_dev(base_qp->device);
515 } else {
516 return -EINVAL;
517 }
518 qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
519 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
520 qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
521 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
522 qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
523 qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
524 qp_attr->max_rd_atomic = qp->attrs.irq_size;
525 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
526
527 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
528 IB_ACCESS_REMOTE_WRITE |
529 IB_ACCESS_REMOTE_READ;
530
531 qp_init_attr->qp_type = base_qp->qp_type;
532 qp_init_attr->send_cq = base_qp->send_cq;
533 qp_init_attr->recv_cq = base_qp->recv_cq;
534 qp_init_attr->srq = base_qp->srq;
535
536 qp_init_attr->cap = qp_attr->cap;
537
538 return 0;
539 }
540
siw_verbs_modify_qp(struct ib_qp * base_qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)541 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
542 int attr_mask, struct ib_udata *udata)
543 {
544 struct siw_qp_attrs new_attrs;
545 enum siw_qp_attr_mask siw_attr_mask = 0;
546 struct siw_qp *qp = to_siw_qp(base_qp);
547 int rv = 0;
548
549 if (!attr_mask)
550 return 0;
551
552 memset(&new_attrs, 0, sizeof(new_attrs));
553
554 if (attr_mask & IB_QP_ACCESS_FLAGS) {
555 siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
556
557 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
558 new_attrs.flags |= SIW_RDMA_READ_ENABLED;
559 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
560 new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
561 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
562 new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
563 }
564 if (attr_mask & IB_QP_STATE) {
565 siw_dbg_qp(qp, "desired IB QP state: %s\n",
566 ib_qp_state_to_string[attr->qp_state]);
567
568 new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
569
570 if (new_attrs.state > SIW_QP_STATE_RTS)
571 qp->tx_ctx.tx_suspend = 1;
572
573 siw_attr_mask |= SIW_QP_ATTR_STATE;
574 }
575 if (!siw_attr_mask)
576 goto out;
577
578 down_write(&qp->state_lock);
579
580 rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
581
582 up_write(&qp->state_lock);
583 out:
584 return rv;
585 }
586
siw_destroy_qp(struct ib_qp * base_qp,struct ib_udata * udata)587 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
588 {
589 struct siw_qp *qp = to_siw_qp(base_qp);
590 struct siw_ucontext *uctx =
591 rdma_udata_to_drv_context(udata, struct siw_ucontext,
592 base_ucontext);
593 struct siw_qp_attrs qp_attrs;
594
595 siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
596
597 /*
598 * Mark QP as in process of destruction to prevent from
599 * any async callbacks to RDMA core
600 */
601 qp->attrs.flags |= SIW_QP_IN_DESTROY;
602 qp->rx_stream.rx_suspend = 1;
603
604 if (uctx) {
605 rdma_user_mmap_entry_remove(qp->sq_entry);
606 rdma_user_mmap_entry_remove(qp->rq_entry);
607 }
608
609 down_write(&qp->state_lock);
610
611 qp_attrs.state = SIW_QP_STATE_ERROR;
612 siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
613
614 if (qp->cep) {
615 siw_cep_put(qp->cep);
616 qp->cep = NULL;
617 }
618 up_write(&qp->state_lock);
619
620 kfree(qp->tx_ctx.mpa_crc_hd);
621 kfree(qp->rx_stream.mpa_crc_hd);
622
623 qp->scq = qp->rcq = NULL;
624
625 siw_qp_put(qp);
626
627 return 0;
628 }
629
630 /*
631 * siw_copy_inline_sgl()
632 *
633 * Prepare sgl of inlined data for sending. For userland callers
634 * function checks if given buffer addresses and len's are within
635 * process context bounds.
636 * Data from all provided sge's are copied together into the wqe,
637 * referenced by a single sge.
638 */
siw_copy_inline_sgl(const struct ib_send_wr * core_wr,struct siw_sqe * sqe)639 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
640 struct siw_sqe *sqe)
641 {
642 struct ib_sge *core_sge = core_wr->sg_list;
643 void *kbuf = &sqe->sge[1];
644 int num_sge = core_wr->num_sge, bytes = 0;
645
646 sqe->sge[0].laddr = (uintptr_t)kbuf;
647 sqe->sge[0].lkey = 0;
648
649 while (num_sge--) {
650 if (!core_sge->length) {
651 core_sge++;
652 continue;
653 }
654 bytes += core_sge->length;
655 if (bytes > SIW_MAX_INLINE) {
656 bytes = -EINVAL;
657 break;
658 }
659 memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
660 core_sge->length);
661
662 kbuf += core_sge->length;
663 core_sge++;
664 }
665 sqe->sge[0].length = bytes > 0 ? bytes : 0;
666 sqe->num_sge = bytes > 0 ? 1 : 0;
667
668 return bytes;
669 }
670
671 /* Complete SQ WR's without processing */
siw_sq_flush_wr(struct siw_qp * qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)672 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
673 const struct ib_send_wr **bad_wr)
674 {
675 struct siw_sqe sqe = {};
676 int rv = 0;
677
678 while (wr) {
679 sqe.id = wr->wr_id;
680 sqe.opcode = wr->opcode;
681 rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR);
682 if (rv) {
683 if (bad_wr)
684 *bad_wr = wr;
685 break;
686 }
687 wr = wr->next;
688 }
689 return rv;
690 }
691
692 /* Complete RQ WR's without processing */
siw_rq_flush_wr(struct siw_qp * qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)693 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
694 const struct ib_recv_wr **bad_wr)
695 {
696 struct siw_rqe rqe = {};
697 int rv = 0;
698
699 while (wr) {
700 rqe.id = wr->wr_id;
701 rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
702 if (rv) {
703 if (bad_wr)
704 *bad_wr = wr;
705 break;
706 }
707 wr = wr->next;
708 }
709 return rv;
710 }
711
712 /*
713 * siw_post_send()
714 *
715 * Post a list of S-WR's to a SQ.
716 *
717 * @base_qp: Base QP contained in siw QP
718 * @wr: Null terminated list of user WR's
719 * @bad_wr: Points to failing WR in case of synchronous failure.
720 */
siw_post_send(struct ib_qp * base_qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)721 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
722 const struct ib_send_wr **bad_wr)
723 {
724 struct siw_qp *qp = to_siw_qp(base_qp);
725 struct siw_wqe *wqe = tx_wqe(qp);
726
727 unsigned long flags;
728 int rv = 0;
729
730 if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
731 siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
732 *bad_wr = wr;
733 return -EINVAL;
734 }
735
736 /*
737 * Try to acquire QP state lock. Must be non-blocking
738 * to accommodate kernel clients needs.
739 */
740 if (!down_read_trylock(&qp->state_lock)) {
741 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
742 /*
743 * ERROR state is final, so we can be sure
744 * this state will not change as long as the QP
745 * exists.
746 *
747 * This handles an ib_drain_sq() call with
748 * a concurrent request to set the QP state
749 * to ERROR.
750 */
751 rv = siw_sq_flush_wr(qp, wr, bad_wr);
752 } else {
753 siw_dbg_qp(qp, "QP locked, state %d\n",
754 qp->attrs.state);
755 *bad_wr = wr;
756 rv = -ENOTCONN;
757 }
758 return rv;
759 }
760 if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
761 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
762 /*
763 * Immediately flush this WR to CQ, if QP
764 * is in ERROR state. SQ is guaranteed to
765 * be empty, so WR complets in-order.
766 *
767 * Typically triggered by ib_drain_sq().
768 */
769 rv = siw_sq_flush_wr(qp, wr, bad_wr);
770 } else {
771 siw_dbg_qp(qp, "QP out of state %d\n",
772 qp->attrs.state);
773 *bad_wr = wr;
774 rv = -ENOTCONN;
775 }
776 up_read(&qp->state_lock);
777 return rv;
778 }
779 spin_lock_irqsave(&qp->sq_lock, flags);
780
781 while (wr) {
782 u32 idx = qp->sq_put % qp->attrs.sq_size;
783 struct siw_sqe *sqe = &qp->sendq[idx];
784
785 if (sqe->flags) {
786 siw_dbg_qp(qp, "sq full\n");
787 rv = -ENOMEM;
788 break;
789 }
790 if (wr->num_sge > qp->attrs.sq_max_sges) {
791 siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
792 rv = -EINVAL;
793 break;
794 }
795 sqe->id = wr->wr_id;
796
797 if ((wr->send_flags & IB_SEND_SIGNALED) ||
798 (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
799 sqe->flags |= SIW_WQE_SIGNALLED;
800
801 if (wr->send_flags & IB_SEND_FENCE)
802 sqe->flags |= SIW_WQE_READ_FENCE;
803
804 switch (wr->opcode) {
805 case IB_WR_SEND:
806 case IB_WR_SEND_WITH_INV:
807 if (wr->send_flags & IB_SEND_SOLICITED)
808 sqe->flags |= SIW_WQE_SOLICITED;
809
810 if (!(wr->send_flags & IB_SEND_INLINE)) {
811 siw_copy_sgl(wr->sg_list, sqe->sge,
812 wr->num_sge);
813 sqe->num_sge = wr->num_sge;
814 } else {
815 rv = siw_copy_inline_sgl(wr, sqe);
816 if (rv <= 0) {
817 rv = -EINVAL;
818 break;
819 }
820 sqe->flags |= SIW_WQE_INLINE;
821 sqe->num_sge = 1;
822 }
823 if (wr->opcode == IB_WR_SEND)
824 sqe->opcode = SIW_OP_SEND;
825 else {
826 sqe->opcode = SIW_OP_SEND_REMOTE_INV;
827 sqe->rkey = wr->ex.invalidate_rkey;
828 }
829 break;
830
831 case IB_WR_RDMA_READ_WITH_INV:
832 case IB_WR_RDMA_READ:
833 /*
834 * iWarp restricts RREAD sink to SGL containing
835 * 1 SGE only. we could relax to SGL with multiple
836 * elements referring the SAME ltag or even sending
837 * a private per-rreq tag referring to a checked
838 * local sgl with MULTIPLE ltag's.
839 */
840 if (unlikely(wr->num_sge != 1)) {
841 rv = -EINVAL;
842 break;
843 }
844 siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
845 /*
846 * NOTE: zero length RREAD is allowed!
847 */
848 sqe->raddr = rdma_wr(wr)->remote_addr;
849 sqe->rkey = rdma_wr(wr)->rkey;
850 sqe->num_sge = 1;
851
852 if (wr->opcode == IB_WR_RDMA_READ)
853 sqe->opcode = SIW_OP_READ;
854 else
855 sqe->opcode = SIW_OP_READ_LOCAL_INV;
856 break;
857
858 case IB_WR_RDMA_WRITE:
859 if (!(wr->send_flags & IB_SEND_INLINE)) {
860 siw_copy_sgl(wr->sg_list, &sqe->sge[0],
861 wr->num_sge);
862 sqe->num_sge = wr->num_sge;
863 } else {
864 rv = siw_copy_inline_sgl(wr, sqe);
865 if (unlikely(rv < 0)) {
866 rv = -EINVAL;
867 break;
868 }
869 sqe->flags |= SIW_WQE_INLINE;
870 sqe->num_sge = 1;
871 }
872 sqe->raddr = rdma_wr(wr)->remote_addr;
873 sqe->rkey = rdma_wr(wr)->rkey;
874 sqe->opcode = SIW_OP_WRITE;
875 break;
876
877 case IB_WR_REG_MR:
878 sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
879 sqe->rkey = reg_wr(wr)->key;
880 sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
881 sqe->opcode = SIW_OP_REG_MR;
882 break;
883
884 case IB_WR_LOCAL_INV:
885 sqe->rkey = wr->ex.invalidate_rkey;
886 sqe->opcode = SIW_OP_INVAL_STAG;
887 break;
888
889 default:
890 siw_dbg_qp(qp, "ib wr type %d unsupported\n",
891 wr->opcode);
892 rv = -EINVAL;
893 break;
894 }
895 siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
896 sqe->opcode, sqe->flags,
897 (void *)(uintptr_t)sqe->id);
898
899 if (unlikely(rv < 0))
900 break;
901
902 /* make SQE only valid after completely written */
903 smp_wmb();
904 sqe->flags |= SIW_WQE_VALID;
905
906 qp->sq_put++;
907 wr = wr->next;
908 }
909
910 /*
911 * Send directly if SQ processing is not in progress.
912 * Eventual immediate errors (rv < 0) do not affect the involved
913 * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
914 * processing, if new work is already pending. But rv must be passed
915 * to caller.
916 */
917 if (wqe->wr_status != SIW_WR_IDLE) {
918 spin_unlock_irqrestore(&qp->sq_lock, flags);
919 goto skip_direct_sending;
920 }
921 rv = siw_activate_tx(qp);
922 spin_unlock_irqrestore(&qp->sq_lock, flags);
923
924 if (rv <= 0)
925 goto skip_direct_sending;
926
927 if (rdma_is_kernel_res(&qp->base_qp.res)) {
928 rv = siw_sq_start(qp);
929 } else {
930 qp->tx_ctx.in_syscall = 1;
931
932 if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
933 siw_qp_cm_drop(qp, 0);
934
935 qp->tx_ctx.in_syscall = 0;
936 }
937 skip_direct_sending:
938
939 up_read(&qp->state_lock);
940
941 if (rv >= 0)
942 return 0;
943 /*
944 * Immediate error
945 */
946 siw_dbg_qp(qp, "error %d\n", rv);
947
948 *bad_wr = wr;
949 return rv;
950 }
951
952 /*
953 * siw_post_receive()
954 *
955 * Post a list of R-WR's to a RQ.
956 *
957 * @base_qp: Base QP contained in siw QP
958 * @wr: Null terminated list of user WR's
959 * @bad_wr: Points to failing WR in case of synchronous failure.
960 */
siw_post_receive(struct ib_qp * base_qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)961 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
962 const struct ib_recv_wr **bad_wr)
963 {
964 struct siw_qp *qp = to_siw_qp(base_qp);
965 unsigned long flags;
966 int rv = 0;
967
968 if (qp->srq || qp->attrs.rq_size == 0) {
969 *bad_wr = wr;
970 return -EINVAL;
971 }
972 if (!rdma_is_kernel_res(&qp->base_qp.res)) {
973 siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
974 *bad_wr = wr;
975 return -EINVAL;
976 }
977
978 /*
979 * Try to acquire QP state lock. Must be non-blocking
980 * to accommodate kernel clients needs.
981 */
982 if (!down_read_trylock(&qp->state_lock)) {
983 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
984 /*
985 * ERROR state is final, so we can be sure
986 * this state will not change as long as the QP
987 * exists.
988 *
989 * This handles an ib_drain_rq() call with
990 * a concurrent request to set the QP state
991 * to ERROR.
992 */
993 rv = siw_rq_flush_wr(qp, wr, bad_wr);
994 } else {
995 siw_dbg_qp(qp, "QP locked, state %d\n",
996 qp->attrs.state);
997 *bad_wr = wr;
998 rv = -ENOTCONN;
999 }
1000 return rv;
1001 }
1002 if (qp->attrs.state > SIW_QP_STATE_RTS) {
1003 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1004 /*
1005 * Immediately flush this WR to CQ, if QP
1006 * is in ERROR state. RQ is guaranteed to
1007 * be empty, so WR complets in-order.
1008 *
1009 * Typically triggered by ib_drain_rq().
1010 */
1011 rv = siw_rq_flush_wr(qp, wr, bad_wr);
1012 } else {
1013 siw_dbg_qp(qp, "QP out of state %d\n",
1014 qp->attrs.state);
1015 *bad_wr = wr;
1016 rv = -ENOTCONN;
1017 }
1018 up_read(&qp->state_lock);
1019 return rv;
1020 }
1021 /*
1022 * Serialize potentially multiple producers.
1023 * Not needed for single threaded consumer side.
1024 */
1025 spin_lock_irqsave(&qp->rq_lock, flags);
1026
1027 while (wr) {
1028 u32 idx = qp->rq_put % qp->attrs.rq_size;
1029 struct siw_rqe *rqe = &qp->recvq[idx];
1030
1031 if (rqe->flags) {
1032 siw_dbg_qp(qp, "RQ full\n");
1033 rv = -ENOMEM;
1034 break;
1035 }
1036 if (wr->num_sge > qp->attrs.rq_max_sges) {
1037 siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1038 rv = -EINVAL;
1039 break;
1040 }
1041 rqe->id = wr->wr_id;
1042 rqe->num_sge = wr->num_sge;
1043 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1044
1045 /* make sure RQE is completely written before valid */
1046 smp_wmb();
1047
1048 rqe->flags = SIW_WQE_VALID;
1049
1050 qp->rq_put++;
1051 wr = wr->next;
1052 }
1053 spin_unlock_irqrestore(&qp->rq_lock, flags);
1054
1055 up_read(&qp->state_lock);
1056
1057 if (rv < 0) {
1058 siw_dbg_qp(qp, "error %d\n", rv);
1059 *bad_wr = wr;
1060 }
1061 return rv > 0 ? 0 : rv;
1062 }
1063
siw_destroy_cq(struct ib_cq * base_cq,struct ib_udata * udata)1064 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1065 {
1066 struct siw_cq *cq = to_siw_cq(base_cq);
1067 struct siw_device *sdev = to_siw_dev(base_cq->device);
1068 struct siw_ucontext *ctx =
1069 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1070 base_ucontext);
1071
1072 siw_dbg_cq(cq, "free CQ resources\n");
1073
1074 siw_cq_flush(cq);
1075
1076 if (ctx)
1077 rdma_user_mmap_entry_remove(cq->cq_entry);
1078
1079 atomic_dec(&sdev->num_cq);
1080
1081 vfree(cq->queue);
1082 return 0;
1083 }
1084
1085 /*
1086 * siw_create_cq()
1087 *
1088 * Populate CQ of requested size
1089 *
1090 * @base_cq: CQ as allocated by RDMA midlayer
1091 * @attr: Initial CQ attributes
1092 * @udata: relates to user context
1093 */
1094
siw_create_cq(struct ib_cq * base_cq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)1095 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1096 struct ib_udata *udata)
1097 {
1098 struct siw_device *sdev = to_siw_dev(base_cq->device);
1099 struct siw_cq *cq = to_siw_cq(base_cq);
1100 int rv, size = attr->cqe;
1101
1102 if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1103 siw_dbg(base_cq->device, "too many CQ's\n");
1104 rv = -ENOMEM;
1105 goto err_out;
1106 }
1107 if (size < 1 || size > sdev->attrs.max_cqe) {
1108 siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1109 rv = -EINVAL;
1110 goto err_out;
1111 }
1112 size = roundup_pow_of_two(size);
1113 cq->base_cq.cqe = size;
1114 cq->num_cqe = size;
1115
1116 if (udata)
1117 cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1118 sizeof(struct siw_cq_ctrl));
1119 else
1120 cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1121 sizeof(struct siw_cq_ctrl));
1122
1123 if (cq->queue == NULL) {
1124 rv = -ENOMEM;
1125 goto err_out;
1126 }
1127 get_random_bytes(&cq->id, 4);
1128 siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1129
1130 spin_lock_init(&cq->lock);
1131
1132 cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1133
1134 if (udata) {
1135 struct siw_uresp_create_cq uresp = {};
1136 struct siw_ucontext *ctx =
1137 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1138 base_ucontext);
1139 size_t length = size * sizeof(struct siw_cqe) +
1140 sizeof(struct siw_cq_ctrl);
1141
1142 cq->cq_entry =
1143 siw_mmap_entry_insert(ctx, cq->queue,
1144 length, &uresp.cq_key);
1145 if (!cq->cq_entry) {
1146 rv = -ENOMEM;
1147 goto err_out;
1148 }
1149
1150 uresp.cq_id = cq->id;
1151 uresp.num_cqe = size;
1152
1153 if (udata->outlen < sizeof(uresp)) {
1154 rv = -EINVAL;
1155 goto err_out;
1156 }
1157 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1158 if (rv)
1159 goto err_out;
1160 }
1161 return 0;
1162
1163 err_out:
1164 siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1165
1166 if (cq && cq->queue) {
1167 struct siw_ucontext *ctx =
1168 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1169 base_ucontext);
1170 if (ctx)
1171 rdma_user_mmap_entry_remove(cq->cq_entry);
1172 vfree(cq->queue);
1173 }
1174 atomic_dec(&sdev->num_cq);
1175
1176 return rv;
1177 }
1178
1179 /*
1180 * siw_poll_cq()
1181 *
1182 * Reap CQ entries if available and copy work completion status into
1183 * array of WC's provided by caller. Returns number of reaped CQE's.
1184 *
1185 * @base_cq: Base CQ contained in siw CQ.
1186 * @num_cqe: Maximum number of CQE's to reap.
1187 * @wc: Array of work completions to be filled by siw.
1188 */
siw_poll_cq(struct ib_cq * base_cq,int num_cqe,struct ib_wc * wc)1189 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1190 {
1191 struct siw_cq *cq = to_siw_cq(base_cq);
1192 int i;
1193
1194 for (i = 0; i < num_cqe; i++) {
1195 if (!siw_reap_cqe(cq, wc))
1196 break;
1197 wc++;
1198 }
1199 return i;
1200 }
1201
1202 /*
1203 * siw_req_notify_cq()
1204 *
1205 * Request notification for new CQE's added to that CQ.
1206 * Defined flags:
1207 * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1208 * event if a WQE with notification flag set enters the CQ
1209 * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1210 * event if a WQE enters the CQ.
1211 * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1212 * number of not reaped CQE's regardless of its notification
1213 * type and current or new CQ notification settings.
1214 *
1215 * @base_cq: Base CQ contained in siw CQ.
1216 * @flags: Requested notification flags.
1217 */
siw_req_notify_cq(struct ib_cq * base_cq,enum ib_cq_notify_flags flags)1218 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1219 {
1220 struct siw_cq *cq = to_siw_cq(base_cq);
1221
1222 siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1223
1224 if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1225 /*
1226 * Enable CQ event for next solicited completion.
1227 * and make it visible to all associated producers.
1228 */
1229 smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1230 else
1231 /*
1232 * Enable CQ event for any signalled completion.
1233 * and make it visible to all associated producers.
1234 */
1235 smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1236
1237 if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1238 return cq->cq_put - cq->cq_get;
1239
1240 return 0;
1241 }
1242
1243 /*
1244 * siw_dereg_mr()
1245 *
1246 * Release Memory Region.
1247 *
1248 * @base_mr: Base MR contained in siw MR.
1249 * @udata: points to user context, unused.
1250 */
siw_dereg_mr(struct ib_mr * base_mr,struct ib_udata * udata)1251 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1252 {
1253 struct siw_mr *mr = to_siw_mr(base_mr);
1254 struct siw_device *sdev = to_siw_dev(base_mr->device);
1255
1256 siw_dbg_mem(mr->mem, "deregister MR\n");
1257
1258 atomic_dec(&sdev->num_mr);
1259
1260 siw_mr_drop_mem(mr);
1261 kfree_rcu(mr, rcu);
1262
1263 return 0;
1264 }
1265
1266 /*
1267 * siw_reg_user_mr()
1268 *
1269 * Register Memory Region.
1270 *
1271 * @pd: Protection Domain
1272 * @start: starting address of MR (virtual address)
1273 * @len: len of MR
1274 * @rnic_va: not used by siw
1275 * @rights: MR access rights
1276 * @udata: user buffer to communicate STag and Key.
1277 */
siw_reg_user_mr(struct ib_pd * pd,u64 start,u64 len,u64 rnic_va,int rights,struct ib_udata * udata)1278 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1279 u64 rnic_va, int rights, struct ib_udata *udata)
1280 {
1281 struct siw_mr *mr = NULL;
1282 struct siw_umem *umem = NULL;
1283 struct siw_ureq_reg_mr ureq;
1284 struct siw_device *sdev = to_siw_dev(pd->device);
1285
1286 unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1287 int rv;
1288
1289 siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1290 (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1291 (unsigned long long)len);
1292
1293 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1294 siw_dbg_pd(pd, "too many mr's\n");
1295 rv = -ENOMEM;
1296 goto err_out;
1297 }
1298 if (!len) {
1299 rv = -EINVAL;
1300 goto err_out;
1301 }
1302 if (mem_limit != RLIM_INFINITY) {
1303 unsigned long num_pages =
1304 (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1305 mem_limit >>= PAGE_SHIFT;
1306
1307 if (num_pages > mem_limit - current->mm->locked_vm) {
1308 siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1309 num_pages, mem_limit,
1310 current->mm->locked_vm);
1311 rv = -ENOMEM;
1312 goto err_out;
1313 }
1314 }
1315 umem = siw_umem_get(start, len, ib_access_writable(rights));
1316 if (IS_ERR(umem)) {
1317 rv = PTR_ERR(umem);
1318 siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1319 umem = NULL;
1320 goto err_out;
1321 }
1322 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1323 if (!mr) {
1324 rv = -ENOMEM;
1325 goto err_out;
1326 }
1327 rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1328 if (rv)
1329 goto err_out;
1330
1331 if (udata) {
1332 struct siw_uresp_reg_mr uresp = {};
1333 struct siw_mem *mem = mr->mem;
1334
1335 if (udata->inlen < sizeof(ureq)) {
1336 rv = -EINVAL;
1337 goto err_out;
1338 }
1339 rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1340 if (rv)
1341 goto err_out;
1342
1343 mr->base_mr.lkey |= ureq.stag_key;
1344 mr->base_mr.rkey |= ureq.stag_key;
1345 mem->stag |= ureq.stag_key;
1346 uresp.stag = mem->stag;
1347
1348 if (udata->outlen < sizeof(uresp)) {
1349 rv = -EINVAL;
1350 goto err_out;
1351 }
1352 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1353 if (rv)
1354 goto err_out;
1355 }
1356 mr->mem->stag_valid = 1;
1357
1358 return &mr->base_mr;
1359
1360 err_out:
1361 atomic_dec(&sdev->num_mr);
1362 if (mr) {
1363 if (mr->mem)
1364 siw_mr_drop_mem(mr);
1365 kfree_rcu(mr, rcu);
1366 } else {
1367 if (umem)
1368 siw_umem_release(umem, false);
1369 }
1370 return ERR_PTR(rv);
1371 }
1372
siw_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_sge)1373 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1374 u32 max_sge)
1375 {
1376 struct siw_device *sdev = to_siw_dev(pd->device);
1377 struct siw_mr *mr = NULL;
1378 struct siw_pbl *pbl = NULL;
1379 int rv;
1380
1381 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1382 siw_dbg_pd(pd, "too many mr's\n");
1383 rv = -ENOMEM;
1384 goto err_out;
1385 }
1386 if (mr_type != IB_MR_TYPE_MEM_REG) {
1387 siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1388 rv = -EOPNOTSUPP;
1389 goto err_out;
1390 }
1391 if (max_sge > SIW_MAX_SGE_PBL) {
1392 siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1393 rv = -ENOMEM;
1394 goto err_out;
1395 }
1396 pbl = siw_pbl_alloc(max_sge);
1397 if (IS_ERR(pbl)) {
1398 rv = PTR_ERR(pbl);
1399 siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1400 pbl = NULL;
1401 goto err_out;
1402 }
1403 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1404 if (!mr) {
1405 rv = -ENOMEM;
1406 goto err_out;
1407 }
1408 rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1409 if (rv)
1410 goto err_out;
1411
1412 mr->mem->is_pbl = 1;
1413
1414 siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1415
1416 return &mr->base_mr;
1417
1418 err_out:
1419 atomic_dec(&sdev->num_mr);
1420
1421 if (!mr) {
1422 kfree(pbl);
1423 } else {
1424 if (mr->mem)
1425 siw_mr_drop_mem(mr);
1426 kfree_rcu(mr, rcu);
1427 }
1428 siw_dbg_pd(pd, "failed: %d\n", rv);
1429
1430 return ERR_PTR(rv);
1431 }
1432
1433 /* Just used to count number of pages being mapped */
siw_set_pbl_page(struct ib_mr * base_mr,u64 buf_addr)1434 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1435 {
1436 return 0;
1437 }
1438
siw_map_mr_sg(struct ib_mr * base_mr,struct scatterlist * sl,int num_sle,unsigned int * sg_off)1439 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1440 unsigned int *sg_off)
1441 {
1442 struct scatterlist *slp;
1443 struct siw_mr *mr = to_siw_mr(base_mr);
1444 struct siw_mem *mem = mr->mem;
1445 struct siw_pbl *pbl = mem->pbl;
1446 struct siw_pble *pble;
1447 unsigned long pbl_size;
1448 int i, rv;
1449
1450 if (!pbl) {
1451 siw_dbg_mem(mem, "no PBL allocated\n");
1452 return -EINVAL;
1453 }
1454 pble = pbl->pbe;
1455
1456 if (pbl->max_buf < num_sle) {
1457 siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1458 mem->pbl->max_buf, num_sle);
1459 return -ENOMEM;
1460 }
1461 for_each_sg(sl, slp, num_sle, i) {
1462 if (sg_dma_len(slp) == 0) {
1463 siw_dbg_mem(mem, "empty SGE\n");
1464 return -EINVAL;
1465 }
1466 if (i == 0) {
1467 pble->addr = sg_dma_address(slp);
1468 pble->size = sg_dma_len(slp);
1469 pble->pbl_off = 0;
1470 pbl_size = pble->size;
1471 pbl->num_buf = 1;
1472 } else {
1473 /* Merge PBL entries if adjacent */
1474 if (pble->addr + pble->size == sg_dma_address(slp)) {
1475 pble->size += sg_dma_len(slp);
1476 } else {
1477 pble++;
1478 pbl->num_buf++;
1479 pble->addr = sg_dma_address(slp);
1480 pble->size = sg_dma_len(slp);
1481 pble->pbl_off = pbl_size;
1482 }
1483 pbl_size += sg_dma_len(slp);
1484 }
1485 siw_dbg_mem(mem,
1486 "sge[%d], size %u, addr 0x%p, total %lu\n",
1487 i, pble->size, (void *)(uintptr_t)pble->addr,
1488 pbl_size);
1489 }
1490 rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1491 if (rv > 0) {
1492 mem->len = base_mr->length;
1493 mem->va = base_mr->iova;
1494 siw_dbg_mem(mem,
1495 "%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1496 mem->len, (void *)(uintptr_t)mem->va, num_sle,
1497 pbl->num_buf);
1498 }
1499 return rv;
1500 }
1501
1502 /*
1503 * siw_get_dma_mr()
1504 *
1505 * Create a (empty) DMA memory region, where no umem is attached.
1506 */
siw_get_dma_mr(struct ib_pd * pd,int rights)1507 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1508 {
1509 struct siw_device *sdev = to_siw_dev(pd->device);
1510 struct siw_mr *mr = NULL;
1511 int rv;
1512
1513 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1514 siw_dbg_pd(pd, "too many mr's\n");
1515 rv = -ENOMEM;
1516 goto err_out;
1517 }
1518 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1519 if (!mr) {
1520 rv = -ENOMEM;
1521 goto err_out;
1522 }
1523 rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1524 if (rv)
1525 goto err_out;
1526
1527 mr->mem->stag_valid = 1;
1528
1529 siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1530
1531 return &mr->base_mr;
1532
1533 err_out:
1534 if (rv)
1535 kfree(mr);
1536
1537 atomic_dec(&sdev->num_mr);
1538
1539 return ERR_PTR(rv);
1540 }
1541
1542 /*
1543 * siw_create_srq()
1544 *
1545 * Create Shared Receive Queue of attributes @init_attrs
1546 * within protection domain given by @pd.
1547 *
1548 * @base_srq: Base SRQ contained in siw SRQ.
1549 * @init_attrs: SRQ init attributes.
1550 * @udata: points to user context
1551 */
siw_create_srq(struct ib_srq * base_srq,struct ib_srq_init_attr * init_attrs,struct ib_udata * udata)1552 int siw_create_srq(struct ib_srq *base_srq,
1553 struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1554 {
1555 struct siw_srq *srq = to_siw_srq(base_srq);
1556 struct ib_srq_attr *attrs = &init_attrs->attr;
1557 struct siw_device *sdev = to_siw_dev(base_srq->device);
1558 struct siw_ucontext *ctx =
1559 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1560 base_ucontext);
1561 int rv;
1562
1563 if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1564 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1565 rv = -ENOMEM;
1566 goto err_out;
1567 }
1568 if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1569 attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1570 rv = -EINVAL;
1571 goto err_out;
1572 }
1573 srq->max_sge = attrs->max_sge;
1574 srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1575 srq->limit = attrs->srq_limit;
1576 if (srq->limit)
1577 srq->armed = true;
1578
1579 srq->is_kernel_res = !udata;
1580
1581 if (udata)
1582 srq->recvq =
1583 vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1584 else
1585 srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1586
1587 if (srq->recvq == NULL) {
1588 rv = -ENOMEM;
1589 goto err_out;
1590 }
1591 if (udata) {
1592 struct siw_uresp_create_srq uresp = {};
1593 size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1594
1595 srq->srq_entry =
1596 siw_mmap_entry_insert(ctx, srq->recvq,
1597 length, &uresp.srq_key);
1598 if (!srq->srq_entry) {
1599 rv = -ENOMEM;
1600 goto err_out;
1601 }
1602
1603 uresp.num_rqe = srq->num_rqe;
1604
1605 if (udata->outlen < sizeof(uresp)) {
1606 rv = -EINVAL;
1607 goto err_out;
1608 }
1609 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1610 if (rv)
1611 goto err_out;
1612 }
1613 spin_lock_init(&srq->lock);
1614
1615 siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1616
1617 return 0;
1618
1619 err_out:
1620 if (srq->recvq) {
1621 if (ctx)
1622 rdma_user_mmap_entry_remove(srq->srq_entry);
1623 vfree(srq->recvq);
1624 }
1625 atomic_dec(&sdev->num_srq);
1626
1627 return rv;
1628 }
1629
1630 /*
1631 * siw_modify_srq()
1632 *
1633 * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1634 * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1635 *
1636 * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1637 * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1638 */
siw_modify_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)1639 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1640 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1641 {
1642 struct siw_srq *srq = to_siw_srq(base_srq);
1643 unsigned long flags;
1644 int rv = 0;
1645
1646 spin_lock_irqsave(&srq->lock, flags);
1647
1648 if (attr_mask & IB_SRQ_MAX_WR) {
1649 /* resize request not yet supported */
1650 rv = -EOPNOTSUPP;
1651 goto out;
1652 }
1653 if (attr_mask & IB_SRQ_LIMIT) {
1654 if (attrs->srq_limit) {
1655 if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1656 rv = -EINVAL;
1657 goto out;
1658 }
1659 srq->armed = true;
1660 } else {
1661 srq->armed = false;
1662 }
1663 srq->limit = attrs->srq_limit;
1664 }
1665 out:
1666 spin_unlock_irqrestore(&srq->lock, flags);
1667
1668 return rv;
1669 }
1670
1671 /*
1672 * siw_query_srq()
1673 *
1674 * Query SRQ attributes.
1675 */
siw_query_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs)1676 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1677 {
1678 struct siw_srq *srq = to_siw_srq(base_srq);
1679 unsigned long flags;
1680
1681 spin_lock_irqsave(&srq->lock, flags);
1682
1683 attrs->max_wr = srq->num_rqe;
1684 attrs->max_sge = srq->max_sge;
1685 attrs->srq_limit = srq->limit;
1686
1687 spin_unlock_irqrestore(&srq->lock, flags);
1688
1689 return 0;
1690 }
1691
1692 /*
1693 * siw_destroy_srq()
1694 *
1695 * Destroy SRQ.
1696 * It is assumed that the SRQ is not referenced by any
1697 * QP anymore - the code trusts the RDMA core environment to keep track
1698 * of QP references.
1699 */
siw_destroy_srq(struct ib_srq * base_srq,struct ib_udata * udata)1700 int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1701 {
1702 struct siw_srq *srq = to_siw_srq(base_srq);
1703 struct siw_device *sdev = to_siw_dev(base_srq->device);
1704 struct siw_ucontext *ctx =
1705 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1706 base_ucontext);
1707
1708 if (ctx)
1709 rdma_user_mmap_entry_remove(srq->srq_entry);
1710 vfree(srq->recvq);
1711 atomic_dec(&sdev->num_srq);
1712 return 0;
1713 }
1714
1715 /*
1716 * siw_post_srq_recv()
1717 *
1718 * Post a list of receive queue elements to SRQ.
1719 * NOTE: The function does not check or lock a certain SRQ state
1720 * during the post operation. The code simply trusts the
1721 * RDMA core environment.
1722 *
1723 * @base_srq: Base SRQ contained in siw SRQ
1724 * @wr: List of R-WR's
1725 * @bad_wr: Updated to failing WR if posting fails.
1726 */
siw_post_srq_recv(struct ib_srq * base_srq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)1727 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1728 const struct ib_recv_wr **bad_wr)
1729 {
1730 struct siw_srq *srq = to_siw_srq(base_srq);
1731 unsigned long flags;
1732 int rv = 0;
1733
1734 if (unlikely(!srq->is_kernel_res)) {
1735 siw_dbg_pd(base_srq->pd,
1736 "[SRQ]: no kernel post_recv for mapped srq\n");
1737 rv = -EINVAL;
1738 goto out;
1739 }
1740 /*
1741 * Serialize potentially multiple producers.
1742 * Also needed to serialize potentially multiple
1743 * consumers.
1744 */
1745 spin_lock_irqsave(&srq->lock, flags);
1746
1747 while (wr) {
1748 u32 idx = srq->rq_put % srq->num_rqe;
1749 struct siw_rqe *rqe = &srq->recvq[idx];
1750
1751 if (rqe->flags) {
1752 siw_dbg_pd(base_srq->pd, "SRQ full\n");
1753 rv = -ENOMEM;
1754 break;
1755 }
1756 if (unlikely(wr->num_sge > srq->max_sge)) {
1757 siw_dbg_pd(base_srq->pd,
1758 "[SRQ]: too many sge's: %d\n", wr->num_sge);
1759 rv = -EINVAL;
1760 break;
1761 }
1762 rqe->id = wr->wr_id;
1763 rqe->num_sge = wr->num_sge;
1764 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1765
1766 /* Make sure S-RQE is completely written before valid */
1767 smp_wmb();
1768
1769 rqe->flags = SIW_WQE_VALID;
1770
1771 srq->rq_put++;
1772 wr = wr->next;
1773 }
1774 spin_unlock_irqrestore(&srq->lock, flags);
1775 out:
1776 if (unlikely(rv < 0)) {
1777 siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1778 *bad_wr = wr;
1779 }
1780 return rv;
1781 }
1782
siw_qp_event(struct siw_qp * qp,enum ib_event_type etype)1783 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1784 {
1785 struct ib_event event;
1786 struct ib_qp *base_qp = &qp->base_qp;
1787
1788 /*
1789 * Do not report asynchronous errors on QP which gets
1790 * destroyed via verbs interface (siw_destroy_qp())
1791 */
1792 if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1793 return;
1794
1795 event.event = etype;
1796 event.device = base_qp->device;
1797 event.element.qp = base_qp;
1798
1799 if (base_qp->event_handler) {
1800 siw_dbg_qp(qp, "reporting event %d\n", etype);
1801 base_qp->event_handler(&event, base_qp->qp_context);
1802 }
1803 }
1804
siw_cq_event(struct siw_cq * cq,enum ib_event_type etype)1805 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1806 {
1807 struct ib_event event;
1808 struct ib_cq *base_cq = &cq->base_cq;
1809
1810 event.event = etype;
1811 event.device = base_cq->device;
1812 event.element.cq = base_cq;
1813
1814 if (base_cq->event_handler) {
1815 siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1816 base_cq->event_handler(&event, base_cq->cq_context);
1817 }
1818 }
1819
siw_srq_event(struct siw_srq * srq,enum ib_event_type etype)1820 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1821 {
1822 struct ib_event event;
1823 struct ib_srq *base_srq = &srq->base_srq;
1824
1825 event.event = etype;
1826 event.device = base_srq->device;
1827 event.element.srq = base_srq;
1828
1829 if (base_srq->event_handler) {
1830 siw_dbg_pd(srq->base_srq.pd,
1831 "reporting SRQ event %d\n", etype);
1832 base_srq->event_handler(&event, base_srq->srq_context);
1833 }
1834 }
1835
siw_port_event(struct siw_device * sdev,u8 port,enum ib_event_type etype)1836 void siw_port_event(struct siw_device *sdev, u8 port, enum ib_event_type etype)
1837 {
1838 struct ib_event event;
1839
1840 event.event = etype;
1841 event.device = &sdev->base_dev;
1842 event.element.port_num = port;
1843
1844 siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1845
1846 ib_dispatch_event(&event);
1847 }
1848