1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45
46 #include <linux/qed/common_hsi.h>
47 #include "qedr_hsi_rdma.h"
48 #include <linux/qed/qed_if.h>
49 #include "qedr.h"
50 #include "verbs.h"
51 #include <rdma/qedr-abi.h>
52 #include "qedr_roce_cm.h"
53
54 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
55 #define RDMA_MAX_SGE_PER_SRQ (4)
56 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
57
58 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
59
qedr_ib_copy_to_udata(struct ib_udata * udata,void * src,size_t len)60 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
61 size_t len)
62 {
63 size_t min_len = min_t(size_t, len, udata->outlen);
64
65 return ib_copy_to_udata(udata, src, min_len);
66 }
67
qedr_query_pkey(struct ib_device * ibdev,u8 port,u16 index,u16 * pkey)68 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
69 {
70 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
71 return -EINVAL;
72
73 *pkey = QEDR_ROCE_PKEY_DEFAULT;
74 return 0;
75 }
76
qedr_iw_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * sgid)77 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
78 int index, union ib_gid *sgid)
79 {
80 struct qedr_dev *dev = get_qedr_dev(ibdev);
81
82 memset(sgid->raw, 0, sizeof(sgid->raw));
83 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
84
85 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
86 sgid->global.interface_id, sgid->global.subnet_prefix);
87
88 return 0;
89 }
90
qedr_query_srq(struct ib_srq * ibsrq,struct ib_srq_attr * srq_attr)91 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
92 {
93 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
94 struct qedr_device_attr *qattr = &dev->attr;
95 struct qedr_srq *srq = get_qedr_srq(ibsrq);
96
97 srq_attr->srq_limit = srq->srq_limit;
98 srq_attr->max_wr = qattr->max_srq_wr;
99 srq_attr->max_sge = qattr->max_sge;
100
101 return 0;
102 }
103
qedr_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * udata)104 int qedr_query_device(struct ib_device *ibdev,
105 struct ib_device_attr *attr, struct ib_udata *udata)
106 {
107 struct qedr_dev *dev = get_qedr_dev(ibdev);
108 struct qedr_device_attr *qattr = &dev->attr;
109
110 if (!dev->rdma_ctx) {
111 DP_ERR(dev,
112 "qedr_query_device called with invalid params rdma_ctx=%p\n",
113 dev->rdma_ctx);
114 return -EINVAL;
115 }
116
117 memset(attr, 0, sizeof(*attr));
118
119 attr->fw_ver = qattr->fw_ver;
120 attr->sys_image_guid = qattr->sys_image_guid;
121 attr->max_mr_size = qattr->max_mr_size;
122 attr->page_size_cap = qattr->page_size_caps;
123 attr->vendor_id = qattr->vendor_id;
124 attr->vendor_part_id = qattr->vendor_part_id;
125 attr->hw_ver = qattr->hw_ver;
126 attr->max_qp = qattr->max_qp;
127 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
128 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
129 IB_DEVICE_RC_RNR_NAK_GEN |
130 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
131
132 attr->max_send_sge = qattr->max_sge;
133 attr->max_recv_sge = qattr->max_sge;
134 attr->max_sge_rd = qattr->max_sge;
135 attr->max_cq = qattr->max_cq;
136 attr->max_cqe = qattr->max_cqe;
137 attr->max_mr = qattr->max_mr;
138 attr->max_mw = qattr->max_mw;
139 attr->max_pd = qattr->max_pd;
140 attr->atomic_cap = dev->atomic_cap;
141 attr->max_fmr = qattr->max_fmr;
142 attr->max_map_per_fmr = 16;
143 attr->max_qp_init_rd_atom =
144 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
145 attr->max_qp_rd_atom =
146 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
147 attr->max_qp_init_rd_atom);
148
149 attr->max_srq = qattr->max_srq;
150 attr->max_srq_sge = qattr->max_srq_sge;
151 attr->max_srq_wr = qattr->max_srq_wr;
152
153 attr->local_ca_ack_delay = qattr->dev_ack_delay;
154 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
155 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
156 attr->max_ah = qattr->max_ah;
157
158 return 0;
159 }
160
get_link_speed_and_width(int speed,u8 * ib_speed,u8 * ib_width)161 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
162 u8 *ib_width)
163 {
164 switch (speed) {
165 case 1000:
166 *ib_speed = IB_SPEED_SDR;
167 *ib_width = IB_WIDTH_1X;
168 break;
169 case 10000:
170 *ib_speed = IB_SPEED_QDR;
171 *ib_width = IB_WIDTH_1X;
172 break;
173
174 case 20000:
175 *ib_speed = IB_SPEED_DDR;
176 *ib_width = IB_WIDTH_4X;
177 break;
178
179 case 25000:
180 *ib_speed = IB_SPEED_EDR;
181 *ib_width = IB_WIDTH_1X;
182 break;
183
184 case 40000:
185 *ib_speed = IB_SPEED_QDR;
186 *ib_width = IB_WIDTH_4X;
187 break;
188
189 case 50000:
190 *ib_speed = IB_SPEED_HDR;
191 *ib_width = IB_WIDTH_1X;
192 break;
193
194 case 100000:
195 *ib_speed = IB_SPEED_EDR;
196 *ib_width = IB_WIDTH_4X;
197 break;
198
199 default:
200 /* Unsupported */
201 *ib_speed = IB_SPEED_SDR;
202 *ib_width = IB_WIDTH_1X;
203 }
204 }
205
qedr_query_port(struct ib_device * ibdev,u8 port,struct ib_port_attr * attr)206 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
207 {
208 struct qedr_dev *dev;
209 struct qed_rdma_port *rdma_port;
210
211 dev = get_qedr_dev(ibdev);
212 if (port > 1) {
213 DP_ERR(dev, "invalid_port=0x%x\n", port);
214 return -EINVAL;
215 }
216
217 if (!dev->rdma_ctx) {
218 DP_ERR(dev, "rdma_ctx is NULL\n");
219 return -EINVAL;
220 }
221
222 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
223
224 /* *attr being zeroed by the caller, avoid zeroing it here */
225 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
226 attr->state = IB_PORT_ACTIVE;
227 attr->phys_state = 5;
228 } else {
229 attr->state = IB_PORT_DOWN;
230 attr->phys_state = 3;
231 }
232 attr->max_mtu = IB_MTU_4096;
233 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
234 attr->lid = 0;
235 attr->lmc = 0;
236 attr->sm_lid = 0;
237 attr->sm_sl = 0;
238 attr->ip_gids = true;
239 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
240 attr->gid_tbl_len = 1;
241 attr->pkey_tbl_len = 1;
242 } else {
243 attr->gid_tbl_len = QEDR_MAX_SGID;
244 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
245 }
246 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
247 attr->qkey_viol_cntr = 0;
248 get_link_speed_and_width(rdma_port->link_speed,
249 &attr->active_speed, &attr->active_width);
250 attr->max_msg_sz = rdma_port->max_msg_size;
251 attr->max_vl_num = 4;
252
253 return 0;
254 }
255
qedr_modify_port(struct ib_device * ibdev,u8 port,int mask,struct ib_port_modify * props)256 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
257 struct ib_port_modify *props)
258 {
259 struct qedr_dev *dev;
260
261 dev = get_qedr_dev(ibdev);
262 if (port > 1) {
263 DP_ERR(dev, "invalid_port=0x%x\n", port);
264 return -EINVAL;
265 }
266
267 return 0;
268 }
269
qedr_add_mmap(struct qedr_ucontext * uctx,u64 phy_addr,unsigned long len)270 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
271 unsigned long len)
272 {
273 struct qedr_mm *mm;
274
275 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
276 if (!mm)
277 return -ENOMEM;
278
279 mm->key.phy_addr = phy_addr;
280 /* This function might be called with a length which is not a multiple
281 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
282 * forces this granularity by increasing the requested size if needed.
283 * When qedr_mmap is called, it will search the list with the updated
284 * length as a key. To prevent search failures, the length is rounded up
285 * in advance to PAGE_SIZE.
286 */
287 mm->key.len = roundup(len, PAGE_SIZE);
288 INIT_LIST_HEAD(&mm->entry);
289
290 mutex_lock(&uctx->mm_list_lock);
291 list_add(&mm->entry, &uctx->mm_head);
292 mutex_unlock(&uctx->mm_list_lock);
293
294 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
295 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
296 (unsigned long long)mm->key.phy_addr,
297 (unsigned long)mm->key.len, uctx);
298
299 return 0;
300 }
301
qedr_search_mmap(struct qedr_ucontext * uctx,u64 phy_addr,unsigned long len)302 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
303 unsigned long len)
304 {
305 bool found = false;
306 struct qedr_mm *mm;
307
308 mutex_lock(&uctx->mm_list_lock);
309 list_for_each_entry(mm, &uctx->mm_head, entry) {
310 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
311 continue;
312
313 found = true;
314 break;
315 }
316 mutex_unlock(&uctx->mm_list_lock);
317 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
318 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
319 mm->key.phy_addr, mm->key.len, uctx, found);
320
321 return found;
322 }
323
qedr_alloc_ucontext(struct ib_device * ibdev,struct ib_udata * udata)324 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
325 struct ib_udata *udata)
326 {
327 int rc;
328 struct qedr_ucontext *ctx;
329 struct qedr_alloc_ucontext_resp uresp;
330 struct qedr_dev *dev = get_qedr_dev(ibdev);
331 struct qed_rdma_add_user_out_params oparams;
332
333 if (!udata)
334 return ERR_PTR(-EFAULT);
335
336 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
337 if (!ctx)
338 return ERR_PTR(-ENOMEM);
339
340 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
341 if (rc) {
342 DP_ERR(dev,
343 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
344 rc);
345 goto err;
346 }
347
348 ctx->dpi = oparams.dpi;
349 ctx->dpi_addr = oparams.dpi_addr;
350 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
351 ctx->dpi_size = oparams.dpi_size;
352 INIT_LIST_HEAD(&ctx->mm_head);
353 mutex_init(&ctx->mm_list_lock);
354
355 memset(&uresp, 0, sizeof(uresp));
356
357 uresp.dpm_enabled = dev->user_dpm_enabled;
358 uresp.wids_enabled = 1;
359 uresp.wid_count = oparams.wid_count;
360 uresp.db_pa = ctx->dpi_phys_addr;
361 uresp.db_size = ctx->dpi_size;
362 uresp.max_send_wr = dev->attr.max_sqe;
363 uresp.max_recv_wr = dev->attr.max_rqe;
364 uresp.max_srq_wr = dev->attr.max_srq_wr;
365 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
366 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
367 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
368 uresp.max_cqes = QEDR_MAX_CQES;
369
370 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
371 if (rc)
372 goto err;
373
374 ctx->dev = dev;
375
376 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
377 if (rc)
378 goto err;
379
380 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
381 &ctx->ibucontext);
382 return &ctx->ibucontext;
383
384 err:
385 kfree(ctx);
386 return ERR_PTR(rc);
387 }
388
qedr_dealloc_ucontext(struct ib_ucontext * ibctx)389 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
390 {
391 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
392 struct qedr_mm *mm, *tmp;
393 int status = 0;
394
395 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
396 uctx);
397 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
398
399 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
400 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
401 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
402 mm->key.phy_addr, mm->key.len, uctx);
403 list_del(&mm->entry);
404 kfree(mm);
405 }
406
407 kfree(uctx);
408 return status;
409 }
410
qedr_mmap(struct ib_ucontext * context,struct vm_area_struct * vma)411 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
412 {
413 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
414 struct qedr_dev *dev = get_qedr_dev(context->device);
415 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
416 unsigned long len = (vma->vm_end - vma->vm_start);
417 unsigned long dpi_start;
418
419 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
420
421 DP_DEBUG(dev, QEDR_MSG_INIT,
422 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
423 (void *)vma->vm_start, (void *)vma->vm_end,
424 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
425
426 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
427 DP_ERR(dev,
428 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
429 (void *)vma->vm_start, (void *)vma->vm_end);
430 return -EINVAL;
431 }
432
433 if (!qedr_search_mmap(ucontext, phys_addr, len)) {
434 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
435 vma->vm_pgoff);
436 return -EINVAL;
437 }
438
439 if (phys_addr < dpi_start ||
440 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
441 DP_ERR(dev,
442 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
443 (void *)phys_addr, (void *)dpi_start,
444 ucontext->dpi_size);
445 return -EINVAL;
446 }
447
448 if (vma->vm_flags & VM_READ) {
449 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
450 return -EINVAL;
451 }
452
453 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
454 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
455 vma->vm_page_prot);
456 }
457
qedr_alloc_pd(struct ib_device * ibdev,struct ib_ucontext * context,struct ib_udata * udata)458 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
459 struct ib_ucontext *context, struct ib_udata *udata)
460 {
461 struct qedr_dev *dev = get_qedr_dev(ibdev);
462 struct qedr_pd *pd;
463 u16 pd_id;
464 int rc;
465
466 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
467 (udata && context) ? "User Lib" : "Kernel");
468
469 if (!dev->rdma_ctx) {
470 DP_ERR(dev, "invalid RDMA context\n");
471 return ERR_PTR(-EINVAL);
472 }
473
474 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
475 if (!pd)
476 return ERR_PTR(-ENOMEM);
477
478 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
479 if (rc)
480 goto err;
481
482 pd->pd_id = pd_id;
483
484 if (udata && context) {
485 struct qedr_alloc_pd_uresp uresp = {
486 .pd_id = pd_id,
487 };
488
489 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
490 if (rc) {
491 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
492 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
493 goto err;
494 }
495
496 pd->uctx = get_qedr_ucontext(context);
497 pd->uctx->pd = pd;
498 }
499
500 return &pd->ibpd;
501
502 err:
503 kfree(pd);
504 return ERR_PTR(rc);
505 }
506
qedr_dealloc_pd(struct ib_pd * ibpd)507 int qedr_dealloc_pd(struct ib_pd *ibpd)
508 {
509 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
510 struct qedr_pd *pd = get_qedr_pd(ibpd);
511
512 if (!pd) {
513 pr_err("Invalid PD received in dealloc_pd\n");
514 return -EINVAL;
515 }
516
517 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
518 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
519
520 kfree(pd);
521
522 return 0;
523 }
524
qedr_free_pbl(struct qedr_dev * dev,struct qedr_pbl_info * pbl_info,struct qedr_pbl * pbl)525 static void qedr_free_pbl(struct qedr_dev *dev,
526 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
527 {
528 struct pci_dev *pdev = dev->pdev;
529 int i;
530
531 for (i = 0; i < pbl_info->num_pbls; i++) {
532 if (!pbl[i].va)
533 continue;
534 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
535 pbl[i].va, pbl[i].pa);
536 }
537
538 kfree(pbl);
539 }
540
541 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
542 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
543
544 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
545 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
546 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
547
qedr_alloc_pbl_tbl(struct qedr_dev * dev,struct qedr_pbl_info * pbl_info,gfp_t flags)548 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
549 struct qedr_pbl_info *pbl_info,
550 gfp_t flags)
551 {
552 struct pci_dev *pdev = dev->pdev;
553 struct qedr_pbl *pbl_table;
554 dma_addr_t *pbl_main_tbl;
555 dma_addr_t pa;
556 void *va;
557 int i;
558
559 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
560 if (!pbl_table)
561 return ERR_PTR(-ENOMEM);
562
563 for (i = 0; i < pbl_info->num_pbls; i++) {
564 va = dma_zalloc_coherent(&pdev->dev, pbl_info->pbl_size,
565 &pa, flags);
566 if (!va)
567 goto err;
568
569 pbl_table[i].va = va;
570 pbl_table[i].pa = pa;
571 }
572
573 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
574 * the first one with physical pointers to all of the rest
575 */
576 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
577 for (i = 0; i < pbl_info->num_pbls - 1; i++)
578 pbl_main_tbl[i] = pbl_table[i + 1].pa;
579
580 return pbl_table;
581
582 err:
583 for (i--; i >= 0; i--)
584 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
585 pbl_table[i].va, pbl_table[i].pa);
586
587 qedr_free_pbl(dev, pbl_info, pbl_table);
588
589 return ERR_PTR(-ENOMEM);
590 }
591
qedr_prepare_pbl_tbl(struct qedr_dev * dev,struct qedr_pbl_info * pbl_info,u32 num_pbes,int two_layer_capable)592 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
593 struct qedr_pbl_info *pbl_info,
594 u32 num_pbes, int two_layer_capable)
595 {
596 u32 pbl_capacity;
597 u32 pbl_size;
598 u32 num_pbls;
599
600 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
601 if (num_pbes > MAX_PBES_TWO_LAYER) {
602 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
603 num_pbes);
604 return -EINVAL;
605 }
606
607 /* calculate required pbl page size */
608 pbl_size = MIN_FW_PBL_PAGE_SIZE;
609 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
610 NUM_PBES_ON_PAGE(pbl_size);
611
612 while (pbl_capacity < num_pbes) {
613 pbl_size *= 2;
614 pbl_capacity = pbl_size / sizeof(u64);
615 pbl_capacity = pbl_capacity * pbl_capacity;
616 }
617
618 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
619 num_pbls++; /* One for the layer0 ( points to the pbls) */
620 pbl_info->two_layered = true;
621 } else {
622 /* One layered PBL */
623 num_pbls = 1;
624 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
625 roundup_pow_of_two((num_pbes * sizeof(u64))));
626 pbl_info->two_layered = false;
627 }
628
629 pbl_info->num_pbls = num_pbls;
630 pbl_info->pbl_size = pbl_size;
631 pbl_info->num_pbes = num_pbes;
632
633 DP_DEBUG(dev, QEDR_MSG_MR,
634 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
635 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
636
637 return 0;
638 }
639
qedr_populate_pbls(struct qedr_dev * dev,struct ib_umem * umem,struct qedr_pbl * pbl,struct qedr_pbl_info * pbl_info,u32 pg_shift)640 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
641 struct qedr_pbl *pbl,
642 struct qedr_pbl_info *pbl_info, u32 pg_shift)
643 {
644 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
645 u32 fw_pg_cnt, fw_pg_per_umem_pg;
646 struct qedr_pbl *pbl_tbl;
647 struct scatterlist *sg;
648 struct regpair *pbe;
649 u64 pg_addr;
650 int entry;
651
652 if (!pbl_info->num_pbes)
653 return;
654
655 /* If we have a two layered pbl, the first pbl points to the rest
656 * of the pbls and the first entry lays on the second pbl in the table
657 */
658 if (pbl_info->two_layered)
659 pbl_tbl = &pbl[1];
660 else
661 pbl_tbl = pbl;
662
663 pbe = (struct regpair *)pbl_tbl->va;
664 if (!pbe) {
665 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
666 return;
667 }
668
669 pbe_cnt = 0;
670
671 shift = umem->page_shift;
672
673 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
674
675 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
676 pages = sg_dma_len(sg) >> shift;
677 pg_addr = sg_dma_address(sg);
678 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
679 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
680 pbe->lo = cpu_to_le32(pg_addr);
681 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
682
683 pg_addr += BIT(pg_shift);
684 pbe_cnt++;
685 total_num_pbes++;
686 pbe++;
687
688 if (total_num_pbes == pbl_info->num_pbes)
689 return;
690
691 /* If the given pbl is full storing the pbes,
692 * move to next pbl.
693 */
694 if (pbe_cnt ==
695 (pbl_info->pbl_size / sizeof(u64))) {
696 pbl_tbl++;
697 pbe = (struct regpair *)pbl_tbl->va;
698 pbe_cnt = 0;
699 }
700
701 fw_pg_cnt++;
702 }
703 }
704 }
705 }
706
qedr_copy_cq_uresp(struct qedr_dev * dev,struct qedr_cq * cq,struct ib_udata * udata)707 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
708 struct qedr_cq *cq, struct ib_udata *udata)
709 {
710 struct qedr_create_cq_uresp uresp;
711 int rc;
712
713 memset(&uresp, 0, sizeof(uresp));
714
715 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
716 uresp.icid = cq->icid;
717
718 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
719 if (rc)
720 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
721
722 return rc;
723 }
724
consume_cqe(struct qedr_cq * cq)725 static void consume_cqe(struct qedr_cq *cq)
726 {
727 if (cq->latest_cqe == cq->toggle_cqe)
728 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
729
730 cq->latest_cqe = qed_chain_consume(&cq->pbl);
731 }
732
qedr_align_cq_entries(int entries)733 static inline int qedr_align_cq_entries(int entries)
734 {
735 u64 size, aligned_size;
736
737 /* We allocate an extra entry that we don't report to the FW. */
738 size = (entries + 1) * QEDR_CQE_SIZE;
739 aligned_size = ALIGN(size, PAGE_SIZE);
740
741 return aligned_size / QEDR_CQE_SIZE;
742 }
743
qedr_init_user_queue(struct ib_ucontext * ib_ctx,struct qedr_dev * dev,struct qedr_userq * q,u64 buf_addr,size_t buf_len,int access,int dmasync,int alloc_and_init)744 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
745 struct qedr_dev *dev,
746 struct qedr_userq *q,
747 u64 buf_addr, size_t buf_len,
748 int access, int dmasync,
749 int alloc_and_init)
750 {
751 u32 fw_pages;
752 int rc;
753
754 q->buf_addr = buf_addr;
755 q->buf_len = buf_len;
756 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
757 if (IS_ERR(q->umem)) {
758 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
759 PTR_ERR(q->umem));
760 return PTR_ERR(q->umem);
761 }
762
763 fw_pages = ib_umem_page_count(q->umem) <<
764 (q->umem->page_shift - FW_PAGE_SHIFT);
765
766 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
767 if (rc)
768 goto err0;
769
770 if (alloc_and_init) {
771 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
772 if (IS_ERR(q->pbl_tbl)) {
773 rc = PTR_ERR(q->pbl_tbl);
774 goto err0;
775 }
776 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
777 FW_PAGE_SHIFT);
778 } else {
779 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
780 if (!q->pbl_tbl) {
781 rc = -ENOMEM;
782 goto err0;
783 }
784 }
785
786 return 0;
787
788 err0:
789 ib_umem_release(q->umem);
790 q->umem = NULL;
791
792 return rc;
793 }
794
qedr_init_cq_params(struct qedr_cq * cq,struct qedr_ucontext * ctx,struct qedr_dev * dev,int vector,int chain_entries,int page_cnt,u64 pbl_ptr,struct qed_rdma_create_cq_in_params * params)795 static inline void qedr_init_cq_params(struct qedr_cq *cq,
796 struct qedr_ucontext *ctx,
797 struct qedr_dev *dev, int vector,
798 int chain_entries, int page_cnt,
799 u64 pbl_ptr,
800 struct qed_rdma_create_cq_in_params
801 *params)
802 {
803 memset(params, 0, sizeof(*params));
804 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
805 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
806 params->cnq_id = vector;
807 params->cq_size = chain_entries - 1;
808 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
809 params->pbl_num_pages = page_cnt;
810 params->pbl_ptr = pbl_ptr;
811 params->pbl_two_level = 0;
812 }
813
doorbell_cq(struct qedr_cq * cq,u32 cons,u8 flags)814 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
815 {
816 cq->db.data.agg_flags = flags;
817 cq->db.data.value = cpu_to_le32(cons);
818 writeq(cq->db.raw, cq->db_addr);
819
820 /* Make sure write would stick */
821 mmiowb();
822 }
823
qedr_arm_cq(struct ib_cq * ibcq,enum ib_cq_notify_flags flags)824 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
825 {
826 struct qedr_cq *cq = get_qedr_cq(ibcq);
827 unsigned long sflags;
828 struct qedr_dev *dev;
829
830 dev = get_qedr_dev(ibcq->device);
831
832 if (cq->destroyed) {
833 DP_ERR(dev,
834 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
835 cq, cq->icid);
836 return -EINVAL;
837 }
838
839
840 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
841 return 0;
842
843 spin_lock_irqsave(&cq->cq_lock, sflags);
844
845 cq->arm_flags = 0;
846
847 if (flags & IB_CQ_SOLICITED)
848 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
849
850 if (flags & IB_CQ_NEXT_COMP)
851 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
852
853 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
854
855 spin_unlock_irqrestore(&cq->cq_lock, sflags);
856
857 return 0;
858 }
859
qedr_create_cq(struct ib_device * ibdev,const struct ib_cq_init_attr * attr,struct ib_ucontext * ib_ctx,struct ib_udata * udata)860 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
861 const struct ib_cq_init_attr *attr,
862 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
863 {
864 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
865 struct qed_rdma_destroy_cq_out_params destroy_oparams;
866 struct qed_rdma_destroy_cq_in_params destroy_iparams;
867 struct qedr_dev *dev = get_qedr_dev(ibdev);
868 struct qed_rdma_create_cq_in_params params;
869 struct qedr_create_cq_ureq ureq;
870 int vector = attr->comp_vector;
871 int entries = attr->cqe;
872 struct qedr_cq *cq;
873 int chain_entries;
874 int page_cnt;
875 u64 pbl_ptr;
876 u16 icid;
877 int rc;
878
879 DP_DEBUG(dev, QEDR_MSG_INIT,
880 "create_cq: called from %s. entries=%d, vector=%d\n",
881 udata ? "User Lib" : "Kernel", entries, vector);
882
883 if (entries > QEDR_MAX_CQES) {
884 DP_ERR(dev,
885 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
886 entries, QEDR_MAX_CQES);
887 return ERR_PTR(-EINVAL);
888 }
889
890 chain_entries = qedr_align_cq_entries(entries);
891 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
892
893 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
894 if (!cq)
895 return ERR_PTR(-ENOMEM);
896
897 if (udata) {
898 memset(&ureq, 0, sizeof(ureq));
899 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
900 DP_ERR(dev,
901 "create cq: problem copying data from user space\n");
902 goto err0;
903 }
904
905 if (!ureq.len) {
906 DP_ERR(dev,
907 "create cq: cannot create a cq with 0 entries\n");
908 goto err0;
909 }
910
911 cq->cq_type = QEDR_CQ_TYPE_USER;
912
913 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
914 ureq.len, IB_ACCESS_LOCAL_WRITE,
915 1, 1);
916 if (rc)
917 goto err0;
918
919 pbl_ptr = cq->q.pbl_tbl->pa;
920 page_cnt = cq->q.pbl_info.num_pbes;
921
922 cq->ibcq.cqe = chain_entries;
923 } else {
924 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
925
926 rc = dev->ops->common->chain_alloc(dev->cdev,
927 QED_CHAIN_USE_TO_CONSUME,
928 QED_CHAIN_MODE_PBL,
929 QED_CHAIN_CNT_TYPE_U32,
930 chain_entries,
931 sizeof(union rdma_cqe),
932 &cq->pbl, NULL);
933 if (rc)
934 goto err1;
935
936 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
937 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
938 cq->ibcq.cqe = cq->pbl.capacity;
939 }
940
941 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
942 pbl_ptr, ¶ms);
943
944 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
945 if (rc)
946 goto err2;
947
948 cq->icid = icid;
949 cq->sig = QEDR_CQ_MAGIC_NUMBER;
950 spin_lock_init(&cq->cq_lock);
951
952 if (ib_ctx) {
953 rc = qedr_copy_cq_uresp(dev, cq, udata);
954 if (rc)
955 goto err3;
956 } else {
957 /* Generate doorbell address. */
958 cq->db_addr = dev->db_addr +
959 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
960 cq->db.data.icid = cq->icid;
961 cq->db.data.params = DB_AGG_CMD_SET <<
962 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
963
964 /* point to the very last element, passing it we will toggle */
965 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
966 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
967 cq->latest_cqe = NULL;
968 consume_cqe(cq);
969 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
970 }
971
972 DP_DEBUG(dev, QEDR_MSG_CQ,
973 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
974 cq->icid, cq, params.cq_size);
975
976 return &cq->ibcq;
977
978 err3:
979 destroy_iparams.icid = cq->icid;
980 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
981 &destroy_oparams);
982 err2:
983 if (udata)
984 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
985 else
986 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
987 err1:
988 if (udata)
989 ib_umem_release(cq->q.umem);
990 err0:
991 kfree(cq);
992 return ERR_PTR(-EINVAL);
993 }
994
qedr_resize_cq(struct ib_cq * ibcq,int new_cnt,struct ib_udata * udata)995 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
996 {
997 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
998 struct qedr_cq *cq = get_qedr_cq(ibcq);
999
1000 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1001
1002 return 0;
1003 }
1004
1005 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1006 #define QEDR_DESTROY_CQ_ITER_DURATION (10)
1007
qedr_destroy_cq(struct ib_cq * ibcq)1008 int qedr_destroy_cq(struct ib_cq *ibcq)
1009 {
1010 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1011 struct qed_rdma_destroy_cq_out_params oparams;
1012 struct qed_rdma_destroy_cq_in_params iparams;
1013 struct qedr_cq *cq = get_qedr_cq(ibcq);
1014 int iter;
1015 int rc;
1016
1017 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1018
1019 cq->destroyed = 1;
1020
1021 /* GSIs CQs are handled by driver, so they don't exist in the FW */
1022 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1023 goto done;
1024
1025 iparams.icid = cq->icid;
1026 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1027 if (rc)
1028 return rc;
1029
1030 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1031
1032 if (ibcq->uobject && ibcq->uobject->context) {
1033 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1034 ib_umem_release(cq->q.umem);
1035 }
1036
1037 /* We don't want the IRQ handler to handle a non-existing CQ so we
1038 * wait until all CNQ interrupts, if any, are received. This will always
1039 * happen and will always happen very fast. If not, then a serious error
1040 * has occured. That is why we can use a long delay.
1041 * We spin for a short time so we don’t lose time on context switching
1042 * in case all the completions are handled in that span. Otherwise
1043 * we sleep for a while and check again. Since the CNQ may be
1044 * associated with (only) the current CPU we use msleep to allow the
1045 * current CPU to be freed.
1046 * The CNQ notification is increased in qedr_irq_handler().
1047 */
1048 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1049 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1050 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1051 iter--;
1052 }
1053
1054 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1055 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1056 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1057 iter--;
1058 }
1059
1060 if (oparams.num_cq_notif != cq->cnq_notif)
1061 goto err;
1062
1063 /* Note that we don't need to have explicit code to wait for the
1064 * completion of the event handler because it is invoked from the EQ.
1065 * Since the destroy CQ ramrod has also been received on the EQ we can
1066 * be certain that there's no event handler in process.
1067 */
1068 done:
1069 cq->sig = ~cq->sig;
1070
1071 kfree(cq);
1072
1073 return 0;
1074
1075 err:
1076 DP_ERR(dev,
1077 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1078 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1079
1080 return -EINVAL;
1081 }
1082
get_gid_info_from_table(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct qed_rdma_modify_qp_in_params * qp_params)1083 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1084 struct ib_qp_attr *attr,
1085 int attr_mask,
1086 struct qed_rdma_modify_qp_in_params
1087 *qp_params)
1088 {
1089 const struct ib_gid_attr *gid_attr;
1090 enum rdma_network_type nw_type;
1091 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1092 u32 ipv4_addr;
1093 int i;
1094
1095 gid_attr = grh->sgid_attr;
1096 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev);
1097
1098 nw_type = rdma_gid_attr_network_type(gid_attr);
1099 switch (nw_type) {
1100 case RDMA_NETWORK_IPV6:
1101 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1102 sizeof(qp_params->sgid));
1103 memcpy(&qp_params->dgid.bytes[0],
1104 &grh->dgid,
1105 sizeof(qp_params->dgid));
1106 qp_params->roce_mode = ROCE_V2_IPV6;
1107 SET_FIELD(qp_params->modify_flags,
1108 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1109 break;
1110 case RDMA_NETWORK_IB:
1111 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1112 sizeof(qp_params->sgid));
1113 memcpy(&qp_params->dgid.bytes[0],
1114 &grh->dgid,
1115 sizeof(qp_params->dgid));
1116 qp_params->roce_mode = ROCE_V1;
1117 break;
1118 case RDMA_NETWORK_IPV4:
1119 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1120 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1121 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1122 qp_params->sgid.ipv4_addr = ipv4_addr;
1123 ipv4_addr =
1124 qedr_get_ipv4_from_gid(grh->dgid.raw);
1125 qp_params->dgid.ipv4_addr = ipv4_addr;
1126 SET_FIELD(qp_params->modify_flags,
1127 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1128 qp_params->roce_mode = ROCE_V2_IPV4;
1129 break;
1130 }
1131
1132 for (i = 0; i < 4; i++) {
1133 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1134 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1135 }
1136
1137 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1138 qp_params->vlan_id = 0;
1139
1140 return 0;
1141 }
1142
qedr_check_qp_attrs(struct ib_pd * ibpd,struct qedr_dev * dev,struct ib_qp_init_attr * attrs)1143 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1144 struct ib_qp_init_attr *attrs)
1145 {
1146 struct qedr_device_attr *qattr = &dev->attr;
1147
1148 /* QP0... attrs->qp_type == IB_QPT_GSI */
1149 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1150 DP_DEBUG(dev, QEDR_MSG_QP,
1151 "create qp: unsupported qp type=0x%x requested\n",
1152 attrs->qp_type);
1153 return -EINVAL;
1154 }
1155
1156 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1157 DP_ERR(dev,
1158 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1159 attrs->cap.max_send_wr, qattr->max_sqe);
1160 return -EINVAL;
1161 }
1162
1163 if (attrs->cap.max_inline_data > qattr->max_inline) {
1164 DP_ERR(dev,
1165 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1166 attrs->cap.max_inline_data, qattr->max_inline);
1167 return -EINVAL;
1168 }
1169
1170 if (attrs->cap.max_send_sge > qattr->max_sge) {
1171 DP_ERR(dev,
1172 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1173 attrs->cap.max_send_sge, qattr->max_sge);
1174 return -EINVAL;
1175 }
1176
1177 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1178 DP_ERR(dev,
1179 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1180 attrs->cap.max_recv_sge, qattr->max_sge);
1181 return -EINVAL;
1182 }
1183
1184 /* Unprivileged user space cannot create special QP */
1185 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1186 DP_ERR(dev,
1187 "create qp: userspace can't create special QPs of type=0x%x\n",
1188 attrs->qp_type);
1189 return -EINVAL;
1190 }
1191
1192 return 0;
1193 }
1194
qedr_copy_srq_uresp(struct qedr_dev * dev,struct qedr_srq * srq,struct ib_udata * udata)1195 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1196 struct qedr_srq *srq, struct ib_udata *udata)
1197 {
1198 struct qedr_create_srq_uresp uresp = {};
1199 int rc;
1200
1201 uresp.srq_id = srq->srq_id;
1202
1203 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1204 if (rc)
1205 DP_ERR(dev, "create srq: problem copying data to user space\n");
1206
1207 return rc;
1208 }
1209
qedr_copy_rq_uresp(struct qedr_dev * dev,struct qedr_create_qp_uresp * uresp,struct qedr_qp * qp)1210 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1211 struct qedr_create_qp_uresp *uresp,
1212 struct qedr_qp *qp)
1213 {
1214 /* iWARP requires two doorbells per RQ. */
1215 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1216 uresp->rq_db_offset =
1217 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1218 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1219 } else {
1220 uresp->rq_db_offset =
1221 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1222 }
1223
1224 uresp->rq_icid = qp->icid;
1225 }
1226
qedr_copy_sq_uresp(struct qedr_dev * dev,struct qedr_create_qp_uresp * uresp,struct qedr_qp * qp)1227 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1228 struct qedr_create_qp_uresp *uresp,
1229 struct qedr_qp *qp)
1230 {
1231 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1232
1233 /* iWARP uses the same cid for rq and sq */
1234 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1235 uresp->sq_icid = qp->icid;
1236 else
1237 uresp->sq_icid = qp->icid + 1;
1238 }
1239
qedr_copy_qp_uresp(struct qedr_dev * dev,struct qedr_qp * qp,struct ib_udata * udata)1240 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1241 struct qedr_qp *qp, struct ib_udata *udata)
1242 {
1243 struct qedr_create_qp_uresp uresp;
1244 int rc;
1245
1246 memset(&uresp, 0, sizeof(uresp));
1247 qedr_copy_sq_uresp(dev, &uresp, qp);
1248 qedr_copy_rq_uresp(dev, &uresp, qp);
1249
1250 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1251 uresp.qp_id = qp->qp_id;
1252
1253 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1254 if (rc)
1255 DP_ERR(dev,
1256 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1257 qp->icid);
1258
1259 return rc;
1260 }
1261
qedr_set_common_qp_params(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_pd * pd,struct ib_qp_init_attr * attrs)1262 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1263 struct qedr_qp *qp,
1264 struct qedr_pd *pd,
1265 struct ib_qp_init_attr *attrs)
1266 {
1267 spin_lock_init(&qp->q_lock);
1268 atomic_set(&qp->refcnt, 1);
1269 qp->pd = pd;
1270 qp->qp_type = attrs->qp_type;
1271 qp->max_inline_data = attrs->cap.max_inline_data;
1272 qp->sq.max_sges = attrs->cap.max_send_sge;
1273 qp->state = QED_ROCE_QP_STATE_RESET;
1274 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1275 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1276 qp->dev = dev;
1277
1278 if (attrs->srq) {
1279 qp->srq = get_qedr_srq(attrs->srq);
1280 } else {
1281 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1282 qp->rq.max_sges = attrs->cap.max_recv_sge;
1283 DP_DEBUG(dev, QEDR_MSG_QP,
1284 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1285 qp->rq.max_sges, qp->rq_cq->icid);
1286 }
1287
1288 DP_DEBUG(dev, QEDR_MSG_QP,
1289 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1290 pd->pd_id, qp->qp_type, qp->max_inline_data,
1291 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1292 DP_DEBUG(dev, QEDR_MSG_QP,
1293 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1294 qp->sq.max_sges, qp->sq_cq->icid);
1295 }
1296
qedr_set_roce_db_info(struct qedr_dev * dev,struct qedr_qp * qp)1297 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1298 {
1299 qp->sq.db = dev->db_addr +
1300 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1301 qp->sq.db_data.data.icid = qp->icid + 1;
1302 if (!qp->srq) {
1303 qp->rq.db = dev->db_addr +
1304 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1305 qp->rq.db_data.data.icid = qp->icid;
1306 }
1307 }
1308
qedr_check_srq_params(struct ib_pd * ibpd,struct qedr_dev * dev,struct ib_srq_init_attr * attrs,struct ib_udata * udata)1309 static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev,
1310 struct ib_srq_init_attr *attrs,
1311 struct ib_udata *udata)
1312 {
1313 struct qedr_device_attr *qattr = &dev->attr;
1314
1315 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1316 DP_ERR(dev,
1317 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1318 attrs->attr.max_wr, qattr->max_srq_wr);
1319 return -EINVAL;
1320 }
1321
1322 if (attrs->attr.max_sge > qattr->max_sge) {
1323 DP_ERR(dev,
1324 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1325 attrs->attr.max_sge, qattr->max_sge);
1326 return -EINVAL;
1327 }
1328
1329 return 0;
1330 }
1331
qedr_free_srq_user_params(struct qedr_srq * srq)1332 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1333 {
1334 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1335 ib_umem_release(srq->usrq.umem);
1336 ib_umem_release(srq->prod_umem);
1337 }
1338
qedr_free_srq_kernel_params(struct qedr_srq * srq)1339 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1340 {
1341 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1342 struct qedr_dev *dev = srq->dev;
1343
1344 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1345
1346 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1347 hw_srq->virt_prod_pair_addr,
1348 hw_srq->phy_prod_pair_addr);
1349 }
1350
qedr_init_srq_user_params(struct ib_ucontext * ib_ctx,struct qedr_srq * srq,struct qedr_create_srq_ureq * ureq,int access,int dmasync)1351 static int qedr_init_srq_user_params(struct ib_ucontext *ib_ctx,
1352 struct qedr_srq *srq,
1353 struct qedr_create_srq_ureq *ureq,
1354 int access, int dmasync)
1355 {
1356 struct scatterlist *sg;
1357 int rc;
1358
1359 rc = qedr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
1360 ureq->srq_len, access, dmasync, 1);
1361 if (rc)
1362 return rc;
1363
1364 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
1365 sizeof(struct rdma_srq_producers),
1366 access, dmasync);
1367 if (IS_ERR(srq->prod_umem)) {
1368 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1369 ib_umem_release(srq->usrq.umem);
1370 DP_ERR(srq->dev,
1371 "create srq: failed ib_umem_get for producer, got %ld\n",
1372 PTR_ERR(srq->prod_umem));
1373 return PTR_ERR(srq->prod_umem);
1374 }
1375
1376 sg = srq->prod_umem->sg_head.sgl;
1377 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1378
1379 return 0;
1380 }
1381
qedr_alloc_srq_kernel_params(struct qedr_srq * srq,struct qedr_dev * dev,struct ib_srq_init_attr * init_attr)1382 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1383 struct qedr_dev *dev,
1384 struct ib_srq_init_attr *init_attr)
1385 {
1386 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1387 dma_addr_t phy_prod_pair_addr;
1388 u32 num_elems;
1389 void *va;
1390 int rc;
1391
1392 va = dma_alloc_coherent(&dev->pdev->dev,
1393 sizeof(struct rdma_srq_producers),
1394 &phy_prod_pair_addr, GFP_KERNEL);
1395 if (!va) {
1396 DP_ERR(dev,
1397 "create srq: failed to allocate dma memory for producer\n");
1398 return -ENOMEM;
1399 }
1400
1401 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1402 hw_srq->virt_prod_pair_addr = va;
1403
1404 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1405 rc = dev->ops->common->chain_alloc(dev->cdev,
1406 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1407 QED_CHAIN_MODE_PBL,
1408 QED_CHAIN_CNT_TYPE_U32,
1409 num_elems,
1410 QEDR_SRQ_WQE_ELEM_SIZE,
1411 &hw_srq->pbl, NULL);
1412 if (rc)
1413 goto err0;
1414
1415 hw_srq->num_elems = num_elems;
1416
1417 return 0;
1418
1419 err0:
1420 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1421 va, phy_prod_pair_addr);
1422 return rc;
1423 }
1424
1425 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1426 void *ptr, u32 id);
1427 static void qedr_idr_remove(struct qedr_dev *dev,
1428 struct qedr_idr *qidr, u32 id);
1429
qedr_create_srq(struct ib_pd * ibpd,struct ib_srq_init_attr * init_attr,struct ib_udata * udata)1430 struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
1431 struct ib_srq_init_attr *init_attr,
1432 struct ib_udata *udata)
1433 {
1434 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1435 struct qed_rdma_create_srq_in_params in_params = {};
1436 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1437 struct qed_rdma_create_srq_out_params out_params;
1438 struct qedr_pd *pd = get_qedr_pd(ibpd);
1439 struct qedr_create_srq_ureq ureq = {};
1440 u64 pbl_base_addr, phy_prod_pair_addr;
1441 struct ib_ucontext *ib_ctx = NULL;
1442 struct qedr_srq_hwq_info *hw_srq;
1443 struct qedr_ucontext *ctx = NULL;
1444 u32 page_cnt, page_size;
1445 struct qedr_srq *srq;
1446 int rc = 0;
1447
1448 DP_DEBUG(dev, QEDR_MSG_QP,
1449 "create SRQ called from %s (pd %p)\n",
1450 (udata) ? "User lib" : "kernel", pd);
1451
1452 rc = qedr_check_srq_params(ibpd, dev, init_attr, udata);
1453 if (rc)
1454 return ERR_PTR(-EINVAL);
1455
1456 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1457 if (!srq)
1458 return ERR_PTR(-ENOMEM);
1459
1460 srq->dev = dev;
1461 hw_srq = &srq->hw_srq;
1462 spin_lock_init(&srq->lock);
1463
1464 hw_srq->max_wr = init_attr->attr.max_wr;
1465 hw_srq->max_sges = init_attr->attr.max_sge;
1466
1467 if (udata && ibpd->uobject && ibpd->uobject->context) {
1468 ib_ctx = ibpd->uobject->context;
1469 ctx = get_qedr_ucontext(ib_ctx);
1470
1471 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1472 DP_ERR(dev,
1473 "create srq: problem copying data from user space\n");
1474 goto err0;
1475 }
1476
1477 rc = qedr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
1478 if (rc)
1479 goto err0;
1480
1481 page_cnt = srq->usrq.pbl_info.num_pbes;
1482 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1483 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1484 page_size = BIT(srq->usrq.umem->page_shift);
1485 } else {
1486 struct qed_chain *pbl;
1487
1488 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1489 if (rc)
1490 goto err0;
1491
1492 pbl = &hw_srq->pbl;
1493 page_cnt = qed_chain_get_page_cnt(pbl);
1494 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1495 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1496 page_size = QED_CHAIN_PAGE_SIZE;
1497 }
1498
1499 in_params.pd_id = pd->pd_id;
1500 in_params.pbl_base_addr = pbl_base_addr;
1501 in_params.prod_pair_addr = phy_prod_pair_addr;
1502 in_params.num_pages = page_cnt;
1503 in_params.page_size = page_size;
1504
1505 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1506 if (rc)
1507 goto err1;
1508
1509 srq->srq_id = out_params.srq_id;
1510
1511 if (udata) {
1512 rc = qedr_copy_srq_uresp(dev, srq, udata);
1513 if (rc)
1514 goto err2;
1515 }
1516
1517 rc = qedr_idr_add(dev, &dev->srqidr, srq, srq->srq_id);
1518 if (rc)
1519 goto err2;
1520
1521 DP_DEBUG(dev, QEDR_MSG_SRQ,
1522 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1523 return &srq->ibsrq;
1524
1525 err2:
1526 destroy_in_params.srq_id = srq->srq_id;
1527
1528 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1529 err1:
1530 if (udata)
1531 qedr_free_srq_user_params(srq);
1532 else
1533 qedr_free_srq_kernel_params(srq);
1534 err0:
1535 kfree(srq);
1536
1537 return ERR_PTR(-EFAULT);
1538 }
1539
qedr_destroy_srq(struct ib_srq * ibsrq)1540 int qedr_destroy_srq(struct ib_srq *ibsrq)
1541 {
1542 struct qed_rdma_destroy_srq_in_params in_params = {};
1543 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1544 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1545
1546 qedr_idr_remove(dev, &dev->srqidr, srq->srq_id);
1547 in_params.srq_id = srq->srq_id;
1548 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1549
1550 if (ibsrq->pd->uobject)
1551 qedr_free_srq_user_params(srq);
1552 else
1553 qedr_free_srq_kernel_params(srq);
1554
1555 DP_DEBUG(dev, QEDR_MSG_SRQ,
1556 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1557 srq->srq_id);
1558 kfree(srq);
1559
1560 return 0;
1561 }
1562
qedr_modify_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)1563 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1564 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1565 {
1566 struct qed_rdma_modify_srq_in_params in_params = {};
1567 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1568 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1569 int rc;
1570
1571 if (attr_mask & IB_SRQ_MAX_WR) {
1572 DP_ERR(dev,
1573 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1574 attr_mask, srq);
1575 return -EINVAL;
1576 }
1577
1578 if (attr_mask & IB_SRQ_LIMIT) {
1579 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1580 DP_ERR(dev,
1581 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1582 attr->srq_limit, srq->hw_srq.max_wr);
1583 return -EINVAL;
1584 }
1585
1586 in_params.srq_id = srq->srq_id;
1587 in_params.wqe_limit = attr->srq_limit;
1588 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1589 if (rc)
1590 return rc;
1591 }
1592
1593 srq->srq_limit = attr->srq_limit;
1594
1595 DP_DEBUG(dev, QEDR_MSG_SRQ,
1596 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1597
1598 return 0;
1599 }
1600
1601 static inline void
qedr_init_common_qp_in_params(struct qedr_dev * dev,struct qedr_pd * pd,struct qedr_qp * qp,struct ib_qp_init_attr * attrs,bool fmr_and_reserved_lkey,struct qed_rdma_create_qp_in_params * params)1602 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1603 struct qedr_pd *pd,
1604 struct qedr_qp *qp,
1605 struct ib_qp_init_attr *attrs,
1606 bool fmr_and_reserved_lkey,
1607 struct qed_rdma_create_qp_in_params *params)
1608 {
1609 /* QP handle to be written in an async event */
1610 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1611 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1612
1613 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1614 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1615 params->pd = pd->pd_id;
1616 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1617 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1618 params->stats_queue = 0;
1619 params->srq_id = 0;
1620 params->use_srq = false;
1621
1622 if (!qp->srq) {
1623 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1624
1625 } else {
1626 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1627 params->srq_id = qp->srq->srq_id;
1628 params->use_srq = true;
1629 }
1630 }
1631
qedr_qp_user_print(struct qedr_dev * dev,struct qedr_qp * qp)1632 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1633 {
1634 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1635 "qp=%p. "
1636 "sq_addr=0x%llx, "
1637 "sq_len=%zd, "
1638 "rq_addr=0x%llx, "
1639 "rq_len=%zd"
1640 "\n",
1641 qp,
1642 qp->usq.buf_addr,
1643 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1644 }
1645
qedr_idr_add(struct qedr_dev * dev,struct qedr_idr * qidr,void * ptr,u32 id)1646 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1647 void *ptr, u32 id)
1648 {
1649 int rc;
1650
1651 idr_preload(GFP_KERNEL);
1652 spin_lock_irq(&qidr->idr_lock);
1653
1654 rc = idr_alloc(&qidr->idr, ptr, id, id + 1, GFP_ATOMIC);
1655
1656 spin_unlock_irq(&qidr->idr_lock);
1657 idr_preload_end();
1658
1659 return rc < 0 ? rc : 0;
1660 }
1661
qedr_idr_remove(struct qedr_dev * dev,struct qedr_idr * qidr,u32 id)1662 static void qedr_idr_remove(struct qedr_dev *dev, struct qedr_idr *qidr, u32 id)
1663 {
1664 spin_lock_irq(&qidr->idr_lock);
1665 idr_remove(&qidr->idr, id);
1666 spin_unlock_irq(&qidr->idr_lock);
1667 }
1668
1669 static inline void
qedr_iwarp_populate_user_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct qed_rdma_create_qp_out_params * out_params)1670 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1671 struct qedr_qp *qp,
1672 struct qed_rdma_create_qp_out_params *out_params)
1673 {
1674 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1675 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1676
1677 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1678 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1679 if (!qp->srq) {
1680 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1681 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1682 }
1683
1684 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1685 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1686 }
1687
qedr_cleanup_user(struct qedr_dev * dev,struct qedr_qp * qp)1688 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1689 {
1690 if (qp->usq.umem)
1691 ib_umem_release(qp->usq.umem);
1692 qp->usq.umem = NULL;
1693
1694 if (qp->urq.umem)
1695 ib_umem_release(qp->urq.umem);
1696 qp->urq.umem = NULL;
1697
1698 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1699 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1700 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1701 } else {
1702 kfree(qp->usq.pbl_tbl);
1703 kfree(qp->urq.pbl_tbl);
1704 }
1705 }
1706
qedr_create_user_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct ib_pd * ibpd,struct ib_udata * udata,struct ib_qp_init_attr * attrs)1707 static int qedr_create_user_qp(struct qedr_dev *dev,
1708 struct qedr_qp *qp,
1709 struct ib_pd *ibpd,
1710 struct ib_udata *udata,
1711 struct ib_qp_init_attr *attrs)
1712 {
1713 struct qed_rdma_create_qp_in_params in_params;
1714 struct qed_rdma_create_qp_out_params out_params;
1715 struct qedr_pd *pd = get_qedr_pd(ibpd);
1716 struct ib_ucontext *ib_ctx = NULL;
1717 struct qedr_create_qp_ureq ureq;
1718 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1719 int rc = -EINVAL;
1720
1721 ib_ctx = ibpd->uobject->context;
1722
1723 memset(&ureq, 0, sizeof(ureq));
1724 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1725 if (rc) {
1726 DP_ERR(dev, "Problem copying data from user space\n");
1727 return rc;
1728 }
1729
1730 /* SQ - read access only (0), dma sync not required (0) */
1731 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1732 ureq.sq_len, 0, 0, alloc_and_init);
1733 if (rc)
1734 return rc;
1735
1736 if (!qp->srq) {
1737 /* RQ - read access only (0), dma sync not required (0) */
1738 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1739 ureq.rq_len, 0, 0, alloc_and_init);
1740 if (rc)
1741 return rc;
1742 }
1743
1744 memset(&in_params, 0, sizeof(in_params));
1745 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1746 in_params.qp_handle_lo = ureq.qp_handle_lo;
1747 in_params.qp_handle_hi = ureq.qp_handle_hi;
1748 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1749 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1750 if (!qp->srq) {
1751 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1752 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1753 }
1754
1755 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1756 &in_params, &out_params);
1757
1758 if (!qp->qed_qp) {
1759 rc = -ENOMEM;
1760 goto err1;
1761 }
1762
1763 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1764 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1765
1766 qp->qp_id = out_params.qp_id;
1767 qp->icid = out_params.icid;
1768
1769 rc = qedr_copy_qp_uresp(dev, qp, udata);
1770 if (rc)
1771 goto err;
1772
1773 qedr_qp_user_print(dev, qp);
1774
1775 return 0;
1776 err:
1777 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1778 if (rc)
1779 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1780
1781 err1:
1782 qedr_cleanup_user(dev, qp);
1783 return rc;
1784 }
1785
qedr_set_iwarp_db_info(struct qedr_dev * dev,struct qedr_qp * qp)1786 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1787 {
1788 qp->sq.db = dev->db_addr +
1789 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1790 qp->sq.db_data.data.icid = qp->icid;
1791
1792 qp->rq.db = dev->db_addr +
1793 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1794 qp->rq.db_data.data.icid = qp->icid;
1795 qp->rq.iwarp_db2 = dev->db_addr +
1796 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1797 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1798 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1799 }
1800
1801 static int
qedr_roce_create_kernel_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct qed_rdma_create_qp_in_params * in_params,u32 n_sq_elems,u32 n_rq_elems)1802 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1803 struct qedr_qp *qp,
1804 struct qed_rdma_create_qp_in_params *in_params,
1805 u32 n_sq_elems, u32 n_rq_elems)
1806 {
1807 struct qed_rdma_create_qp_out_params out_params;
1808 int rc;
1809
1810 rc = dev->ops->common->chain_alloc(dev->cdev,
1811 QED_CHAIN_USE_TO_PRODUCE,
1812 QED_CHAIN_MODE_PBL,
1813 QED_CHAIN_CNT_TYPE_U32,
1814 n_sq_elems,
1815 QEDR_SQE_ELEMENT_SIZE,
1816 &qp->sq.pbl, NULL);
1817
1818 if (rc)
1819 return rc;
1820
1821 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1822 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1823
1824 rc = dev->ops->common->chain_alloc(dev->cdev,
1825 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1826 QED_CHAIN_MODE_PBL,
1827 QED_CHAIN_CNT_TYPE_U32,
1828 n_rq_elems,
1829 QEDR_RQE_ELEMENT_SIZE,
1830 &qp->rq.pbl, NULL);
1831 if (rc)
1832 return rc;
1833
1834 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1835 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1836
1837 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1838 in_params, &out_params);
1839
1840 if (!qp->qed_qp)
1841 return -EINVAL;
1842
1843 qp->qp_id = out_params.qp_id;
1844 qp->icid = out_params.icid;
1845
1846 qedr_set_roce_db_info(dev, qp);
1847 return rc;
1848 }
1849
1850 static int
qedr_iwarp_create_kernel_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct qed_rdma_create_qp_in_params * in_params,u32 n_sq_elems,u32 n_rq_elems)1851 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1852 struct qedr_qp *qp,
1853 struct qed_rdma_create_qp_in_params *in_params,
1854 u32 n_sq_elems, u32 n_rq_elems)
1855 {
1856 struct qed_rdma_create_qp_out_params out_params;
1857 struct qed_chain_ext_pbl ext_pbl;
1858 int rc;
1859
1860 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1861 QEDR_SQE_ELEMENT_SIZE,
1862 QED_CHAIN_MODE_PBL);
1863 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1864 QEDR_RQE_ELEMENT_SIZE,
1865 QED_CHAIN_MODE_PBL);
1866
1867 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1868 in_params, &out_params);
1869
1870 if (!qp->qed_qp)
1871 return -EINVAL;
1872
1873 /* Now we allocate the chain */
1874 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1875 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1876
1877 rc = dev->ops->common->chain_alloc(dev->cdev,
1878 QED_CHAIN_USE_TO_PRODUCE,
1879 QED_CHAIN_MODE_PBL,
1880 QED_CHAIN_CNT_TYPE_U32,
1881 n_sq_elems,
1882 QEDR_SQE_ELEMENT_SIZE,
1883 &qp->sq.pbl, &ext_pbl);
1884
1885 if (rc)
1886 goto err;
1887
1888 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1889 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1890
1891 rc = dev->ops->common->chain_alloc(dev->cdev,
1892 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1893 QED_CHAIN_MODE_PBL,
1894 QED_CHAIN_CNT_TYPE_U32,
1895 n_rq_elems,
1896 QEDR_RQE_ELEMENT_SIZE,
1897 &qp->rq.pbl, &ext_pbl);
1898
1899 if (rc)
1900 goto err;
1901
1902 qp->qp_id = out_params.qp_id;
1903 qp->icid = out_params.icid;
1904
1905 qedr_set_iwarp_db_info(dev, qp);
1906 return rc;
1907
1908 err:
1909 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1910
1911 return rc;
1912 }
1913
qedr_cleanup_kernel(struct qedr_dev * dev,struct qedr_qp * qp)1914 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1915 {
1916 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1917 kfree(qp->wqe_wr_id);
1918
1919 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1920 kfree(qp->rqe_wr_id);
1921 }
1922
qedr_create_kernel_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct ib_pd * ibpd,struct ib_qp_init_attr * attrs)1923 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1924 struct qedr_qp *qp,
1925 struct ib_pd *ibpd,
1926 struct ib_qp_init_attr *attrs)
1927 {
1928 struct qed_rdma_create_qp_in_params in_params;
1929 struct qedr_pd *pd = get_qedr_pd(ibpd);
1930 int rc = -EINVAL;
1931 u32 n_rq_elems;
1932 u32 n_sq_elems;
1933 u32 n_sq_entries;
1934
1935 memset(&in_params, 0, sizeof(in_params));
1936
1937 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1938 * the ring. The ring should allow at least a single WR, even if the
1939 * user requested none, due to allocation issues.
1940 * We should add an extra WR since the prod and cons indices of
1941 * wqe_wr_id are managed in such a way that the WQ is considered full
1942 * when (prod+1)%max_wr==cons. We currently don't do that because we
1943 * double the number of entries due an iSER issue that pushes far more
1944 * WRs than indicated. If we decline its ib_post_send() then we get
1945 * error prints in the dmesg we'd like to avoid.
1946 */
1947 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1948 dev->attr.max_sqe);
1949
1950 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1951 GFP_KERNEL);
1952 if (!qp->wqe_wr_id) {
1953 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1954 return -ENOMEM;
1955 }
1956
1957 /* QP handle to be written in CQE */
1958 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1959 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1960
1961 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1962 * the ring. There ring should allow at least a single WR, even if the
1963 * user requested none, due to allocation issues.
1964 */
1965 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1966
1967 /* Allocate driver internal RQ array */
1968 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1969 GFP_KERNEL);
1970 if (!qp->rqe_wr_id) {
1971 DP_ERR(dev,
1972 "create qp: failed RQ shadow memory allocation\n");
1973 kfree(qp->wqe_wr_id);
1974 return -ENOMEM;
1975 }
1976
1977 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1978
1979 n_sq_entries = attrs->cap.max_send_wr;
1980 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1981 n_sq_entries = max_t(u32, n_sq_entries, 1);
1982 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1983
1984 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1985
1986 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1987 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1988 n_sq_elems, n_rq_elems);
1989 else
1990 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1991 n_sq_elems, n_rq_elems);
1992 if (rc)
1993 qedr_cleanup_kernel(dev, qp);
1994
1995 return rc;
1996 }
1997
qedr_create_qp(struct ib_pd * ibpd,struct ib_qp_init_attr * attrs,struct ib_udata * udata)1998 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1999 struct ib_qp_init_attr *attrs,
2000 struct ib_udata *udata)
2001 {
2002 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2003 struct qedr_pd *pd = get_qedr_pd(ibpd);
2004 struct qedr_qp *qp;
2005 struct ib_qp *ibqp;
2006 int rc = 0;
2007
2008 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2009 udata ? "user library" : "kernel", pd);
2010
2011 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
2012 if (rc)
2013 return ERR_PTR(rc);
2014
2015 DP_DEBUG(dev, QEDR_MSG_QP,
2016 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2017 udata ? "user library" : "kernel", attrs->event_handler, pd,
2018 get_qedr_cq(attrs->send_cq),
2019 get_qedr_cq(attrs->send_cq)->icid,
2020 get_qedr_cq(attrs->recv_cq),
2021 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2022
2023 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2024 if (!qp) {
2025 DP_ERR(dev, "create qp: failed allocating memory\n");
2026 return ERR_PTR(-ENOMEM);
2027 }
2028
2029 qedr_set_common_qp_params(dev, qp, pd, attrs);
2030
2031 if (attrs->qp_type == IB_QPT_GSI) {
2032 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2033 if (IS_ERR(ibqp))
2034 kfree(qp);
2035 return ibqp;
2036 }
2037
2038 if (udata)
2039 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2040 else
2041 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2042
2043 if (rc)
2044 goto err;
2045
2046 qp->ibqp.qp_num = qp->qp_id;
2047
2048 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2049 rc = qedr_idr_add(dev, &dev->qpidr, qp, qp->qp_id);
2050 if (rc)
2051 goto err;
2052 }
2053
2054 return &qp->ibqp;
2055
2056 err:
2057 kfree(qp);
2058
2059 return ERR_PTR(-EFAULT);
2060 }
2061
qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)2062 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2063 {
2064 switch (qp_state) {
2065 case QED_ROCE_QP_STATE_RESET:
2066 return IB_QPS_RESET;
2067 case QED_ROCE_QP_STATE_INIT:
2068 return IB_QPS_INIT;
2069 case QED_ROCE_QP_STATE_RTR:
2070 return IB_QPS_RTR;
2071 case QED_ROCE_QP_STATE_RTS:
2072 return IB_QPS_RTS;
2073 case QED_ROCE_QP_STATE_SQD:
2074 return IB_QPS_SQD;
2075 case QED_ROCE_QP_STATE_ERR:
2076 return IB_QPS_ERR;
2077 case QED_ROCE_QP_STATE_SQE:
2078 return IB_QPS_SQE;
2079 }
2080 return IB_QPS_ERR;
2081 }
2082
qedr_get_state_from_ibqp(enum ib_qp_state qp_state)2083 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2084 enum ib_qp_state qp_state)
2085 {
2086 switch (qp_state) {
2087 case IB_QPS_RESET:
2088 return QED_ROCE_QP_STATE_RESET;
2089 case IB_QPS_INIT:
2090 return QED_ROCE_QP_STATE_INIT;
2091 case IB_QPS_RTR:
2092 return QED_ROCE_QP_STATE_RTR;
2093 case IB_QPS_RTS:
2094 return QED_ROCE_QP_STATE_RTS;
2095 case IB_QPS_SQD:
2096 return QED_ROCE_QP_STATE_SQD;
2097 case IB_QPS_ERR:
2098 return QED_ROCE_QP_STATE_ERR;
2099 default:
2100 return QED_ROCE_QP_STATE_ERR;
2101 }
2102 }
2103
qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info * qph)2104 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2105 {
2106 qed_chain_reset(&qph->pbl);
2107 qph->prod = 0;
2108 qph->cons = 0;
2109 qph->wqe_cons = 0;
2110 qph->db_data.data.value = cpu_to_le16(0);
2111 }
2112
qedr_update_qp_state(struct qedr_dev * dev,struct qedr_qp * qp,enum qed_roce_qp_state cur_state,enum qed_roce_qp_state new_state)2113 static int qedr_update_qp_state(struct qedr_dev *dev,
2114 struct qedr_qp *qp,
2115 enum qed_roce_qp_state cur_state,
2116 enum qed_roce_qp_state new_state)
2117 {
2118 int status = 0;
2119
2120 if (new_state == cur_state)
2121 return 0;
2122
2123 switch (cur_state) {
2124 case QED_ROCE_QP_STATE_RESET:
2125 switch (new_state) {
2126 case QED_ROCE_QP_STATE_INIT:
2127 qp->prev_wqe_size = 0;
2128 qedr_reset_qp_hwq_info(&qp->sq);
2129 qedr_reset_qp_hwq_info(&qp->rq);
2130 break;
2131 default:
2132 status = -EINVAL;
2133 break;
2134 };
2135 break;
2136 case QED_ROCE_QP_STATE_INIT:
2137 switch (new_state) {
2138 case QED_ROCE_QP_STATE_RTR:
2139 /* Update doorbell (in case post_recv was
2140 * done before move to RTR)
2141 */
2142
2143 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2144 writel(qp->rq.db_data.raw, qp->rq.db);
2145 /* Make sure write takes effect */
2146 mmiowb();
2147 }
2148 break;
2149 case QED_ROCE_QP_STATE_ERR:
2150 break;
2151 default:
2152 /* Invalid state change. */
2153 status = -EINVAL;
2154 break;
2155 };
2156 break;
2157 case QED_ROCE_QP_STATE_RTR:
2158 /* RTR->XXX */
2159 switch (new_state) {
2160 case QED_ROCE_QP_STATE_RTS:
2161 break;
2162 case QED_ROCE_QP_STATE_ERR:
2163 break;
2164 default:
2165 /* Invalid state change. */
2166 status = -EINVAL;
2167 break;
2168 };
2169 break;
2170 case QED_ROCE_QP_STATE_RTS:
2171 /* RTS->XXX */
2172 switch (new_state) {
2173 case QED_ROCE_QP_STATE_SQD:
2174 break;
2175 case QED_ROCE_QP_STATE_ERR:
2176 break;
2177 default:
2178 /* Invalid state change. */
2179 status = -EINVAL;
2180 break;
2181 };
2182 break;
2183 case QED_ROCE_QP_STATE_SQD:
2184 /* SQD->XXX */
2185 switch (new_state) {
2186 case QED_ROCE_QP_STATE_RTS:
2187 case QED_ROCE_QP_STATE_ERR:
2188 break;
2189 default:
2190 /* Invalid state change. */
2191 status = -EINVAL;
2192 break;
2193 };
2194 break;
2195 case QED_ROCE_QP_STATE_ERR:
2196 /* ERR->XXX */
2197 switch (new_state) {
2198 case QED_ROCE_QP_STATE_RESET:
2199 if ((qp->rq.prod != qp->rq.cons) ||
2200 (qp->sq.prod != qp->sq.cons)) {
2201 DP_NOTICE(dev,
2202 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2203 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2204 qp->sq.cons);
2205 status = -EINVAL;
2206 }
2207 break;
2208 default:
2209 status = -EINVAL;
2210 break;
2211 };
2212 break;
2213 default:
2214 status = -EINVAL;
2215 break;
2216 };
2217
2218 return status;
2219 }
2220
qedr_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)2221 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2222 int attr_mask, struct ib_udata *udata)
2223 {
2224 struct qedr_qp *qp = get_qedr_qp(ibqp);
2225 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2226 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2227 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2228 enum ib_qp_state old_qp_state, new_qp_state;
2229 enum qed_roce_qp_state cur_state;
2230 int rc = 0;
2231
2232 DP_DEBUG(dev, QEDR_MSG_QP,
2233 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2234 attr->qp_state);
2235
2236 old_qp_state = qedr_get_ibqp_state(qp->state);
2237 if (attr_mask & IB_QP_STATE)
2238 new_qp_state = attr->qp_state;
2239 else
2240 new_qp_state = old_qp_state;
2241
2242 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2243 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2244 ibqp->qp_type, attr_mask,
2245 IB_LINK_LAYER_ETHERNET)) {
2246 DP_ERR(dev,
2247 "modify qp: invalid attribute mask=0x%x specified for\n"
2248 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2249 attr_mask, qp->qp_id, ibqp->qp_type,
2250 old_qp_state, new_qp_state);
2251 rc = -EINVAL;
2252 goto err;
2253 }
2254 }
2255
2256 /* Translate the masks... */
2257 if (attr_mask & IB_QP_STATE) {
2258 SET_FIELD(qp_params.modify_flags,
2259 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2260 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2261 }
2262
2263 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2264 qp_params.sqd_async = true;
2265
2266 if (attr_mask & IB_QP_PKEY_INDEX) {
2267 SET_FIELD(qp_params.modify_flags,
2268 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2269 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2270 rc = -EINVAL;
2271 goto err;
2272 }
2273
2274 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2275 }
2276
2277 if (attr_mask & IB_QP_QKEY)
2278 qp->qkey = attr->qkey;
2279
2280 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2281 SET_FIELD(qp_params.modify_flags,
2282 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2283 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2284 IB_ACCESS_REMOTE_READ;
2285 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2286 IB_ACCESS_REMOTE_WRITE;
2287 qp_params.incoming_atomic_en = attr->qp_access_flags &
2288 IB_ACCESS_REMOTE_ATOMIC;
2289 }
2290
2291 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2292 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2293 return -EINVAL;
2294
2295 if (attr_mask & IB_QP_PATH_MTU) {
2296 if (attr->path_mtu < IB_MTU_256 ||
2297 attr->path_mtu > IB_MTU_4096) {
2298 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2299 rc = -EINVAL;
2300 goto err;
2301 }
2302 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2303 ib_mtu_enum_to_int(iboe_get_mtu
2304 (dev->ndev->mtu)));
2305 }
2306
2307 if (!qp->mtu) {
2308 qp->mtu =
2309 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2310 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2311 }
2312
2313 SET_FIELD(qp_params.modify_flags,
2314 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2315
2316 qp_params.traffic_class_tos = grh->traffic_class;
2317 qp_params.flow_label = grh->flow_label;
2318 qp_params.hop_limit_ttl = grh->hop_limit;
2319
2320 qp->sgid_idx = grh->sgid_index;
2321
2322 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2323 if (rc) {
2324 DP_ERR(dev,
2325 "modify qp: problems with GID index %d (rc=%d)\n",
2326 grh->sgid_index, rc);
2327 return rc;
2328 }
2329
2330 rc = qedr_get_dmac(dev, &attr->ah_attr,
2331 qp_params.remote_mac_addr);
2332 if (rc)
2333 return rc;
2334
2335 qp_params.use_local_mac = true;
2336 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2337
2338 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2339 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2340 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2341 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2342 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2343 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2344 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2345 qp_params.remote_mac_addr);
2346
2347 qp_params.mtu = qp->mtu;
2348 qp_params.lb_indication = false;
2349 }
2350
2351 if (!qp_params.mtu) {
2352 /* Stay with current MTU */
2353 if (qp->mtu)
2354 qp_params.mtu = qp->mtu;
2355 else
2356 qp_params.mtu =
2357 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2358 }
2359
2360 if (attr_mask & IB_QP_TIMEOUT) {
2361 SET_FIELD(qp_params.modify_flags,
2362 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2363
2364 /* The received timeout value is an exponent used like this:
2365 * "12.7.34 LOCAL ACK TIMEOUT
2366 * Value representing the transport (ACK) timeout for use by
2367 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2368 * The FW expects timeout in msec so we need to divide the usec
2369 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2370 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2371 * The value of zero means infinite so we use a 'max_t' to make
2372 * sure that sub 1 msec values will be configured as 1 msec.
2373 */
2374 if (attr->timeout)
2375 qp_params.ack_timeout =
2376 1 << max_t(int, attr->timeout - 8, 0);
2377 else
2378 qp_params.ack_timeout = 0;
2379 }
2380
2381 if (attr_mask & IB_QP_RETRY_CNT) {
2382 SET_FIELD(qp_params.modify_flags,
2383 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2384 qp_params.retry_cnt = attr->retry_cnt;
2385 }
2386
2387 if (attr_mask & IB_QP_RNR_RETRY) {
2388 SET_FIELD(qp_params.modify_flags,
2389 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2390 qp_params.rnr_retry_cnt = attr->rnr_retry;
2391 }
2392
2393 if (attr_mask & IB_QP_RQ_PSN) {
2394 SET_FIELD(qp_params.modify_flags,
2395 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2396 qp_params.rq_psn = attr->rq_psn;
2397 qp->rq_psn = attr->rq_psn;
2398 }
2399
2400 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2401 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2402 rc = -EINVAL;
2403 DP_ERR(dev,
2404 "unsupported max_rd_atomic=%d, supported=%d\n",
2405 attr->max_rd_atomic,
2406 dev->attr.max_qp_req_rd_atomic_resc);
2407 goto err;
2408 }
2409
2410 SET_FIELD(qp_params.modify_flags,
2411 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2412 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2413 }
2414
2415 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2416 SET_FIELD(qp_params.modify_flags,
2417 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2418 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2419 }
2420
2421 if (attr_mask & IB_QP_SQ_PSN) {
2422 SET_FIELD(qp_params.modify_flags,
2423 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2424 qp_params.sq_psn = attr->sq_psn;
2425 qp->sq_psn = attr->sq_psn;
2426 }
2427
2428 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2429 if (attr->max_dest_rd_atomic >
2430 dev->attr.max_qp_resp_rd_atomic_resc) {
2431 DP_ERR(dev,
2432 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2433 attr->max_dest_rd_atomic,
2434 dev->attr.max_qp_resp_rd_atomic_resc);
2435
2436 rc = -EINVAL;
2437 goto err;
2438 }
2439
2440 SET_FIELD(qp_params.modify_flags,
2441 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2442 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2443 }
2444
2445 if (attr_mask & IB_QP_DEST_QPN) {
2446 SET_FIELD(qp_params.modify_flags,
2447 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2448
2449 qp_params.dest_qp = attr->dest_qp_num;
2450 qp->dest_qp_num = attr->dest_qp_num;
2451 }
2452
2453 cur_state = qp->state;
2454
2455 /* Update the QP state before the actual ramrod to prevent a race with
2456 * fast path. Modifying the QP state to error will cause the device to
2457 * flush the CQEs and while polling the flushed CQEs will considered as
2458 * a potential issue if the QP isn't in error state.
2459 */
2460 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2461 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2462 qp->state = QED_ROCE_QP_STATE_ERR;
2463
2464 if (qp->qp_type != IB_QPT_GSI)
2465 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2466 qp->qed_qp, &qp_params);
2467
2468 if (attr_mask & IB_QP_STATE) {
2469 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2470 rc = qedr_update_qp_state(dev, qp, cur_state,
2471 qp_params.new_state);
2472 qp->state = qp_params.new_state;
2473 }
2474
2475 err:
2476 return rc;
2477 }
2478
qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params * params)2479 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2480 {
2481 int ib_qp_acc_flags = 0;
2482
2483 if (params->incoming_rdma_write_en)
2484 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2485 if (params->incoming_rdma_read_en)
2486 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2487 if (params->incoming_atomic_en)
2488 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2489 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2490 return ib_qp_acc_flags;
2491 }
2492
qedr_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int attr_mask,struct ib_qp_init_attr * qp_init_attr)2493 int qedr_query_qp(struct ib_qp *ibqp,
2494 struct ib_qp_attr *qp_attr,
2495 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2496 {
2497 struct qed_rdma_query_qp_out_params params;
2498 struct qedr_qp *qp = get_qedr_qp(ibqp);
2499 struct qedr_dev *dev = qp->dev;
2500 int rc = 0;
2501
2502 memset(¶ms, 0, sizeof(params));
2503
2504 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2505 if (rc)
2506 goto err;
2507
2508 memset(qp_attr, 0, sizeof(*qp_attr));
2509 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2510
2511 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2512 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2513 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2514 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2515 qp_attr->rq_psn = params.rq_psn;
2516 qp_attr->sq_psn = params.sq_psn;
2517 qp_attr->dest_qp_num = params.dest_qp;
2518
2519 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2520
2521 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2522 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2523 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2524 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2525 qp_attr->cap.max_inline_data = dev->attr.max_inline;
2526 qp_init_attr->cap = qp_attr->cap;
2527
2528 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2529 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2530 params.flow_label, qp->sgid_idx,
2531 params.hop_limit_ttl, params.traffic_class_tos);
2532 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2533 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2534 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2535 qp_attr->timeout = params.timeout;
2536 qp_attr->rnr_retry = params.rnr_retry;
2537 qp_attr->retry_cnt = params.retry_cnt;
2538 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2539 qp_attr->pkey_index = params.pkey_index;
2540 qp_attr->port_num = 1;
2541 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2542 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2543 qp_attr->alt_pkey_index = 0;
2544 qp_attr->alt_port_num = 0;
2545 qp_attr->alt_timeout = 0;
2546 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2547
2548 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2549 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2550 qp_attr->max_rd_atomic = params.max_rd_atomic;
2551 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2552
2553 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2554 qp_attr->cap.max_inline_data);
2555
2556 err:
2557 return rc;
2558 }
2559
qedr_free_qp_resources(struct qedr_dev * dev,struct qedr_qp * qp)2560 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2561 {
2562 int rc = 0;
2563
2564 if (qp->qp_type != IB_QPT_GSI) {
2565 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2566 if (rc)
2567 return rc;
2568 }
2569
2570 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2571 qedr_cleanup_user(dev, qp);
2572 else
2573 qedr_cleanup_kernel(dev, qp);
2574
2575 return 0;
2576 }
2577
qedr_destroy_qp(struct ib_qp * ibqp)2578 int qedr_destroy_qp(struct ib_qp *ibqp)
2579 {
2580 struct qedr_qp *qp = get_qedr_qp(ibqp);
2581 struct qedr_dev *dev = qp->dev;
2582 struct ib_qp_attr attr;
2583 int attr_mask = 0;
2584 int rc = 0;
2585
2586 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2587 qp, qp->qp_type);
2588
2589 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2590 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2591 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2592 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2593
2594 attr.qp_state = IB_QPS_ERR;
2595 attr_mask |= IB_QP_STATE;
2596
2597 /* Change the QP state to ERROR */
2598 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2599 }
2600 } else {
2601 /* Wait for the connect/accept to complete */
2602 if (qp->ep) {
2603 int wait_count = 1;
2604
2605 while (qp->ep->during_connect) {
2606 DP_DEBUG(dev, QEDR_MSG_QP,
2607 "Still in during connect/accept\n");
2608
2609 msleep(100);
2610 if (wait_count++ > 200) {
2611 DP_NOTICE(dev,
2612 "during connect timeout\n");
2613 break;
2614 }
2615 }
2616 }
2617 }
2618
2619 if (qp->qp_type == IB_QPT_GSI)
2620 qedr_destroy_gsi_qp(dev);
2621
2622 qedr_free_qp_resources(dev, qp);
2623
2624 if (atomic_dec_and_test(&qp->refcnt) &&
2625 rdma_protocol_iwarp(&dev->ibdev, 1)) {
2626 qedr_idr_remove(dev, &dev->qpidr, qp->qp_id);
2627 kfree(qp);
2628 }
2629 return rc;
2630 }
2631
qedr_create_ah(struct ib_pd * ibpd,struct rdma_ah_attr * attr,struct ib_udata * udata)2632 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2633 struct ib_udata *udata)
2634 {
2635 struct qedr_ah *ah;
2636
2637 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2638 if (!ah)
2639 return ERR_PTR(-ENOMEM);
2640
2641 rdma_copy_ah_attr(&ah->attr, attr);
2642
2643 return &ah->ibah;
2644 }
2645
qedr_destroy_ah(struct ib_ah * ibah)2646 int qedr_destroy_ah(struct ib_ah *ibah)
2647 {
2648 struct qedr_ah *ah = get_qedr_ah(ibah);
2649
2650 rdma_destroy_ah_attr(&ah->attr);
2651 kfree(ah);
2652 return 0;
2653 }
2654
free_mr_info(struct qedr_dev * dev,struct mr_info * info)2655 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2656 {
2657 struct qedr_pbl *pbl, *tmp;
2658
2659 if (info->pbl_table)
2660 list_add_tail(&info->pbl_table->list_entry,
2661 &info->free_pbl_list);
2662
2663 if (!list_empty(&info->inuse_pbl_list))
2664 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2665
2666 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2667 list_del(&pbl->list_entry);
2668 qedr_free_pbl(dev, &info->pbl_info, pbl);
2669 }
2670 }
2671
init_mr_info(struct qedr_dev * dev,struct mr_info * info,size_t page_list_len,bool two_layered)2672 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2673 size_t page_list_len, bool two_layered)
2674 {
2675 struct qedr_pbl *tmp;
2676 int rc;
2677
2678 INIT_LIST_HEAD(&info->free_pbl_list);
2679 INIT_LIST_HEAD(&info->inuse_pbl_list);
2680
2681 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2682 page_list_len, two_layered);
2683 if (rc)
2684 goto done;
2685
2686 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2687 if (IS_ERR(info->pbl_table)) {
2688 rc = PTR_ERR(info->pbl_table);
2689 goto done;
2690 }
2691
2692 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2693 &info->pbl_table->pa);
2694
2695 /* in usual case we use 2 PBLs, so we add one to free
2696 * list and allocating another one
2697 */
2698 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2699 if (IS_ERR(tmp)) {
2700 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2701 goto done;
2702 }
2703
2704 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2705
2706 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2707
2708 done:
2709 if (rc)
2710 free_mr_info(dev, info);
2711
2712 return rc;
2713 }
2714
qedr_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 usr_addr,int acc,struct ib_udata * udata)2715 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2716 u64 usr_addr, int acc, struct ib_udata *udata)
2717 {
2718 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2719 struct qedr_mr *mr;
2720 struct qedr_pd *pd;
2721 int rc = -ENOMEM;
2722
2723 pd = get_qedr_pd(ibpd);
2724 DP_DEBUG(dev, QEDR_MSG_MR,
2725 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2726 pd->pd_id, start, len, usr_addr, acc);
2727
2728 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2729 return ERR_PTR(-EINVAL);
2730
2731 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2732 if (!mr)
2733 return ERR_PTR(rc);
2734
2735 mr->type = QEDR_MR_USER;
2736
2737 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2738 if (IS_ERR(mr->umem)) {
2739 rc = -EFAULT;
2740 goto err0;
2741 }
2742
2743 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2744 if (rc)
2745 goto err1;
2746
2747 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2748 &mr->info.pbl_info, mr->umem->page_shift);
2749
2750 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2751 if (rc) {
2752 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2753 goto err1;
2754 }
2755
2756 /* Index only, 18 bit long, lkey = itid << 8 | key */
2757 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2758 mr->hw_mr.key = 0;
2759 mr->hw_mr.pd = pd->pd_id;
2760 mr->hw_mr.local_read = 1;
2761 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2762 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2763 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2764 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2765 mr->hw_mr.mw_bind = false;
2766 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2767 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2768 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2769 mr->hw_mr.page_size_log = mr->umem->page_shift;
2770 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2771 mr->hw_mr.length = len;
2772 mr->hw_mr.vaddr = usr_addr;
2773 mr->hw_mr.zbva = false;
2774 mr->hw_mr.phy_mr = false;
2775 mr->hw_mr.dma_mr = false;
2776
2777 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2778 if (rc) {
2779 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2780 goto err2;
2781 }
2782
2783 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2784 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2785 mr->hw_mr.remote_atomic)
2786 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2787
2788 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2789 mr->ibmr.lkey);
2790 return &mr->ibmr;
2791
2792 err2:
2793 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2794 err1:
2795 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2796 err0:
2797 kfree(mr);
2798 return ERR_PTR(rc);
2799 }
2800
qedr_dereg_mr(struct ib_mr * ib_mr)2801 int qedr_dereg_mr(struct ib_mr *ib_mr)
2802 {
2803 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2804 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2805 int rc = 0;
2806
2807 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2808 if (rc)
2809 return rc;
2810
2811 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2812
2813 if (mr->type != QEDR_MR_DMA)
2814 free_mr_info(dev, &mr->info);
2815
2816 /* it could be user registered memory. */
2817 if (mr->umem)
2818 ib_umem_release(mr->umem);
2819
2820 kfree(mr);
2821
2822 return rc;
2823 }
2824
__qedr_alloc_mr(struct ib_pd * ibpd,int max_page_list_len)2825 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2826 int max_page_list_len)
2827 {
2828 struct qedr_pd *pd = get_qedr_pd(ibpd);
2829 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2830 struct qedr_mr *mr;
2831 int rc = -ENOMEM;
2832
2833 DP_DEBUG(dev, QEDR_MSG_MR,
2834 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2835 max_page_list_len);
2836
2837 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2838 if (!mr)
2839 return ERR_PTR(rc);
2840
2841 mr->dev = dev;
2842 mr->type = QEDR_MR_FRMR;
2843
2844 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2845 if (rc)
2846 goto err0;
2847
2848 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2849 if (rc) {
2850 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2851 goto err0;
2852 }
2853
2854 /* Index only, 18 bit long, lkey = itid << 8 | key */
2855 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2856 mr->hw_mr.key = 0;
2857 mr->hw_mr.pd = pd->pd_id;
2858 mr->hw_mr.local_read = 1;
2859 mr->hw_mr.local_write = 0;
2860 mr->hw_mr.remote_read = 0;
2861 mr->hw_mr.remote_write = 0;
2862 mr->hw_mr.remote_atomic = 0;
2863 mr->hw_mr.mw_bind = false;
2864 mr->hw_mr.pbl_ptr = 0;
2865 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2866 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2867 mr->hw_mr.fbo = 0;
2868 mr->hw_mr.length = 0;
2869 mr->hw_mr.vaddr = 0;
2870 mr->hw_mr.zbva = false;
2871 mr->hw_mr.phy_mr = true;
2872 mr->hw_mr.dma_mr = false;
2873
2874 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2875 if (rc) {
2876 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2877 goto err1;
2878 }
2879
2880 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2881 mr->ibmr.rkey = mr->ibmr.lkey;
2882
2883 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2884 return mr;
2885
2886 err1:
2887 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2888 err0:
2889 kfree(mr);
2890 return ERR_PTR(rc);
2891 }
2892
qedr_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)2893 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2894 enum ib_mr_type mr_type, u32 max_num_sg)
2895 {
2896 struct qedr_mr *mr;
2897
2898 if (mr_type != IB_MR_TYPE_MEM_REG)
2899 return ERR_PTR(-EINVAL);
2900
2901 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2902
2903 if (IS_ERR(mr))
2904 return ERR_PTR(-EINVAL);
2905
2906 return &mr->ibmr;
2907 }
2908
qedr_set_page(struct ib_mr * ibmr,u64 addr)2909 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2910 {
2911 struct qedr_mr *mr = get_qedr_mr(ibmr);
2912 struct qedr_pbl *pbl_table;
2913 struct regpair *pbe;
2914 u32 pbes_in_page;
2915
2916 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2917 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2918 return -ENOMEM;
2919 }
2920
2921 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2922 mr->npages, addr);
2923
2924 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2925 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2926 pbe = (struct regpair *)pbl_table->va;
2927 pbe += mr->npages % pbes_in_page;
2928 pbe->lo = cpu_to_le32((u32)addr);
2929 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2930
2931 mr->npages++;
2932
2933 return 0;
2934 }
2935
handle_completed_mrs(struct qedr_dev * dev,struct mr_info * info)2936 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2937 {
2938 int work = info->completed - info->completed_handled - 1;
2939
2940 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2941 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2942 struct qedr_pbl *pbl;
2943
2944 /* Free all the page list that are possible to be freed
2945 * (all the ones that were invalidated), under the assumption
2946 * that if an FMR was completed successfully that means that
2947 * if there was an invalidate operation before it also ended
2948 */
2949 pbl = list_first_entry(&info->inuse_pbl_list,
2950 struct qedr_pbl, list_entry);
2951 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2952 info->completed_handled++;
2953 }
2954 }
2955
qedr_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)2956 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2957 int sg_nents, unsigned int *sg_offset)
2958 {
2959 struct qedr_mr *mr = get_qedr_mr(ibmr);
2960
2961 mr->npages = 0;
2962
2963 handle_completed_mrs(mr->dev, &mr->info);
2964 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2965 }
2966
qedr_get_dma_mr(struct ib_pd * ibpd,int acc)2967 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2968 {
2969 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2970 struct qedr_pd *pd = get_qedr_pd(ibpd);
2971 struct qedr_mr *mr;
2972 int rc;
2973
2974 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2975 if (!mr)
2976 return ERR_PTR(-ENOMEM);
2977
2978 mr->type = QEDR_MR_DMA;
2979
2980 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2981 if (rc) {
2982 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2983 goto err1;
2984 }
2985
2986 /* index only, 18 bit long, lkey = itid << 8 | key */
2987 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2988 mr->hw_mr.pd = pd->pd_id;
2989 mr->hw_mr.local_read = 1;
2990 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2991 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2992 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2993 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2994 mr->hw_mr.dma_mr = true;
2995
2996 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2997 if (rc) {
2998 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2999 goto err2;
3000 }
3001
3002 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3003 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3004 mr->hw_mr.remote_atomic)
3005 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3006
3007 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3008 return &mr->ibmr;
3009
3010 err2:
3011 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3012 err1:
3013 kfree(mr);
3014 return ERR_PTR(rc);
3015 }
3016
qedr_wq_is_full(struct qedr_qp_hwq_info * wq)3017 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3018 {
3019 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3020 }
3021
sge_data_len(struct ib_sge * sg_list,int num_sge)3022 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3023 {
3024 int i, len = 0;
3025
3026 for (i = 0; i < num_sge; i++)
3027 len += sg_list[i].length;
3028
3029 return len;
3030 }
3031
swap_wqe_data64(u64 * p)3032 static void swap_wqe_data64(u64 *p)
3033 {
3034 int i;
3035
3036 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3037 *p = cpu_to_be64(cpu_to_le64(*p));
3038 }
3039
qedr_prepare_sq_inline_data(struct qedr_dev * dev,struct qedr_qp * qp,u8 * wqe_size,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr,u8 * bits,u8 bit)3040 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3041 struct qedr_qp *qp, u8 *wqe_size,
3042 const struct ib_send_wr *wr,
3043 const struct ib_send_wr **bad_wr,
3044 u8 *bits, u8 bit)
3045 {
3046 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3047 char *seg_prt, *wqe;
3048 int i, seg_siz;
3049
3050 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3051 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3052 *bad_wr = wr;
3053 return 0;
3054 }
3055
3056 if (!data_size)
3057 return data_size;
3058
3059 *bits |= bit;
3060
3061 seg_prt = NULL;
3062 wqe = NULL;
3063 seg_siz = 0;
3064
3065 /* Copy data inline */
3066 for (i = 0; i < wr->num_sge; i++) {
3067 u32 len = wr->sg_list[i].length;
3068 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3069
3070 while (len > 0) {
3071 u32 cur;
3072
3073 /* New segment required */
3074 if (!seg_siz) {
3075 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3076 seg_prt = wqe;
3077 seg_siz = sizeof(struct rdma_sq_common_wqe);
3078 (*wqe_size)++;
3079 }
3080
3081 /* Calculate currently allowed length */
3082 cur = min_t(u32, len, seg_siz);
3083 memcpy(seg_prt, src, cur);
3084
3085 /* Update segment variables */
3086 seg_prt += cur;
3087 seg_siz -= cur;
3088
3089 /* Update sge variables */
3090 src += cur;
3091 len -= cur;
3092
3093 /* Swap fully-completed segments */
3094 if (!seg_siz)
3095 swap_wqe_data64((u64 *)wqe);
3096 }
3097 }
3098
3099 /* swap last not completed segment */
3100 if (seg_siz)
3101 swap_wqe_data64((u64 *)wqe);
3102
3103 return data_size;
3104 }
3105
3106 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3107 do { \
3108 DMA_REGPAIR_LE(sge->addr, vaddr); \
3109 (sge)->length = cpu_to_le32(vlength); \
3110 (sge)->flags = cpu_to_le32(vflags); \
3111 } while (0)
3112
3113 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3114 do { \
3115 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3116 (hdr)->num_sges = num_sge; \
3117 } while (0)
3118
3119 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3120 do { \
3121 DMA_REGPAIR_LE(sge->addr, vaddr); \
3122 (sge)->length = cpu_to_le32(vlength); \
3123 (sge)->l_key = cpu_to_le32(vlkey); \
3124 } while (0)
3125
qedr_prepare_sq_sges(struct qedr_qp * qp,u8 * wqe_size,const struct ib_send_wr * wr)3126 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3127 const struct ib_send_wr *wr)
3128 {
3129 u32 data_size = 0;
3130 int i;
3131
3132 for (i = 0; i < wr->num_sge; i++) {
3133 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3134
3135 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3136 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3137 sge->length = cpu_to_le32(wr->sg_list[i].length);
3138 data_size += wr->sg_list[i].length;
3139 }
3140
3141 if (wqe_size)
3142 *wqe_size += wr->num_sge;
3143
3144 return data_size;
3145 }
3146
qedr_prepare_sq_rdma_data(struct qedr_dev * dev,struct qedr_qp * qp,struct rdma_sq_rdma_wqe_1st * rwqe,struct rdma_sq_rdma_wqe_2nd * rwqe2,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3147 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3148 struct qedr_qp *qp,
3149 struct rdma_sq_rdma_wqe_1st *rwqe,
3150 struct rdma_sq_rdma_wqe_2nd *rwqe2,
3151 const struct ib_send_wr *wr,
3152 const struct ib_send_wr **bad_wr)
3153 {
3154 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3155 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3156
3157 if (wr->send_flags & IB_SEND_INLINE &&
3158 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3159 wr->opcode == IB_WR_RDMA_WRITE)) {
3160 u8 flags = 0;
3161
3162 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3163 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3164 bad_wr, &rwqe->flags, flags);
3165 }
3166
3167 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3168 }
3169
qedr_prepare_sq_send_data(struct qedr_dev * dev,struct qedr_qp * qp,struct rdma_sq_send_wqe_1st * swqe,struct rdma_sq_send_wqe_2st * swqe2,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3170 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3171 struct qedr_qp *qp,
3172 struct rdma_sq_send_wqe_1st *swqe,
3173 struct rdma_sq_send_wqe_2st *swqe2,
3174 const struct ib_send_wr *wr,
3175 const struct ib_send_wr **bad_wr)
3176 {
3177 memset(swqe2, 0, sizeof(*swqe2));
3178 if (wr->send_flags & IB_SEND_INLINE) {
3179 u8 flags = 0;
3180
3181 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3182 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3183 bad_wr, &swqe->flags, flags);
3184 }
3185
3186 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3187 }
3188
qedr_prepare_reg(struct qedr_qp * qp,struct rdma_sq_fmr_wqe_1st * fwqe1,const struct ib_reg_wr * wr)3189 static int qedr_prepare_reg(struct qedr_qp *qp,
3190 struct rdma_sq_fmr_wqe_1st *fwqe1,
3191 const struct ib_reg_wr *wr)
3192 {
3193 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3194 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3195
3196 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3197 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3198 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3199 fwqe1->l_key = wr->key;
3200
3201 fwqe2->access_ctrl = 0;
3202
3203 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3204 !!(wr->access & IB_ACCESS_REMOTE_READ));
3205 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3206 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3207 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3208 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3209 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3210 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3211 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3212 fwqe2->fmr_ctrl = 0;
3213
3214 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3215 ilog2(mr->ibmr.page_size) - 12);
3216
3217 fwqe2->length_hi = 0;
3218 fwqe2->length_lo = mr->ibmr.length;
3219 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3220 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3221
3222 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3223
3224 return 0;
3225 }
3226
qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)3227 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3228 {
3229 switch (opcode) {
3230 case IB_WR_RDMA_WRITE:
3231 case IB_WR_RDMA_WRITE_WITH_IMM:
3232 return IB_WC_RDMA_WRITE;
3233 case IB_WR_SEND_WITH_IMM:
3234 case IB_WR_SEND:
3235 case IB_WR_SEND_WITH_INV:
3236 return IB_WC_SEND;
3237 case IB_WR_RDMA_READ:
3238 case IB_WR_RDMA_READ_WITH_INV:
3239 return IB_WC_RDMA_READ;
3240 case IB_WR_ATOMIC_CMP_AND_SWP:
3241 return IB_WC_COMP_SWAP;
3242 case IB_WR_ATOMIC_FETCH_AND_ADD:
3243 return IB_WC_FETCH_ADD;
3244 case IB_WR_REG_MR:
3245 return IB_WC_REG_MR;
3246 case IB_WR_LOCAL_INV:
3247 return IB_WC_LOCAL_INV;
3248 default:
3249 return IB_WC_SEND;
3250 }
3251 }
3252
qedr_can_post_send(struct qedr_qp * qp,const struct ib_send_wr * wr)3253 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3254 const struct ib_send_wr *wr)
3255 {
3256 int wq_is_full, err_wr, pbl_is_full;
3257 struct qedr_dev *dev = qp->dev;
3258
3259 /* prevent SQ overflow and/or processing of a bad WR */
3260 err_wr = wr->num_sge > qp->sq.max_sges;
3261 wq_is_full = qedr_wq_is_full(&qp->sq);
3262 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3263 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3264 if (wq_is_full || err_wr || pbl_is_full) {
3265 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3266 DP_ERR(dev,
3267 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3268 qp);
3269 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3270 }
3271
3272 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3273 DP_ERR(dev,
3274 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3275 qp);
3276 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3277 }
3278
3279 if (pbl_is_full &&
3280 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3281 DP_ERR(dev,
3282 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3283 qp);
3284 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3285 }
3286 return false;
3287 }
3288 return true;
3289 }
3290
__qedr_post_send(struct ib_qp * ibqp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3291 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3292 const struct ib_send_wr **bad_wr)
3293 {
3294 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3295 struct qedr_qp *qp = get_qedr_qp(ibqp);
3296 struct rdma_sq_atomic_wqe_1st *awqe1;
3297 struct rdma_sq_atomic_wqe_2nd *awqe2;
3298 struct rdma_sq_atomic_wqe_3rd *awqe3;
3299 struct rdma_sq_send_wqe_2st *swqe2;
3300 struct rdma_sq_local_inv_wqe *iwqe;
3301 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3302 struct rdma_sq_send_wqe_1st *swqe;
3303 struct rdma_sq_rdma_wqe_1st *rwqe;
3304 struct rdma_sq_fmr_wqe_1st *fwqe1;
3305 struct rdma_sq_common_wqe *wqe;
3306 u32 length;
3307 int rc = 0;
3308 bool comp;
3309
3310 if (!qedr_can_post_send(qp, wr)) {
3311 *bad_wr = wr;
3312 return -ENOMEM;
3313 }
3314
3315 wqe = qed_chain_produce(&qp->sq.pbl);
3316 qp->wqe_wr_id[qp->sq.prod].signaled =
3317 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3318
3319 wqe->flags = 0;
3320 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3321 !!(wr->send_flags & IB_SEND_SOLICITED));
3322 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3323 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3324 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3325 !!(wr->send_flags & IB_SEND_FENCE));
3326 wqe->prev_wqe_size = qp->prev_wqe_size;
3327
3328 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3329
3330 switch (wr->opcode) {
3331 case IB_WR_SEND_WITH_IMM:
3332 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3333 rc = -EINVAL;
3334 *bad_wr = wr;
3335 break;
3336 }
3337 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3338 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3339 swqe->wqe_size = 2;
3340 swqe2 = qed_chain_produce(&qp->sq.pbl);
3341
3342 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3343 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3344 wr, bad_wr);
3345 swqe->length = cpu_to_le32(length);
3346 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3347 qp->prev_wqe_size = swqe->wqe_size;
3348 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3349 break;
3350 case IB_WR_SEND:
3351 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3352 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3353
3354 swqe->wqe_size = 2;
3355 swqe2 = qed_chain_produce(&qp->sq.pbl);
3356 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3357 wr, bad_wr);
3358 swqe->length = cpu_to_le32(length);
3359 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3360 qp->prev_wqe_size = swqe->wqe_size;
3361 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3362 break;
3363 case IB_WR_SEND_WITH_INV:
3364 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3365 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3366 swqe2 = qed_chain_produce(&qp->sq.pbl);
3367 swqe->wqe_size = 2;
3368 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3369 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3370 wr, bad_wr);
3371 swqe->length = cpu_to_le32(length);
3372 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3373 qp->prev_wqe_size = swqe->wqe_size;
3374 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3375 break;
3376
3377 case IB_WR_RDMA_WRITE_WITH_IMM:
3378 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3379 rc = -EINVAL;
3380 *bad_wr = wr;
3381 break;
3382 }
3383 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3384 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3385
3386 rwqe->wqe_size = 2;
3387 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3388 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3389 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3390 wr, bad_wr);
3391 rwqe->length = cpu_to_le32(length);
3392 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3393 qp->prev_wqe_size = rwqe->wqe_size;
3394 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3395 break;
3396 case IB_WR_RDMA_WRITE:
3397 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3398 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3399
3400 rwqe->wqe_size = 2;
3401 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3402 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3403 wr, bad_wr);
3404 rwqe->length = cpu_to_le32(length);
3405 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3406 qp->prev_wqe_size = rwqe->wqe_size;
3407 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3408 break;
3409 case IB_WR_RDMA_READ_WITH_INV:
3410 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3411 /* fallthrough -- same is identical to RDMA READ */
3412
3413 case IB_WR_RDMA_READ:
3414 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3415 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3416
3417 rwqe->wqe_size = 2;
3418 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3419 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3420 wr, bad_wr);
3421 rwqe->length = cpu_to_le32(length);
3422 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3423 qp->prev_wqe_size = rwqe->wqe_size;
3424 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3425 break;
3426
3427 case IB_WR_ATOMIC_CMP_AND_SWP:
3428 case IB_WR_ATOMIC_FETCH_AND_ADD:
3429 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3430 awqe1->wqe_size = 4;
3431
3432 awqe2 = qed_chain_produce(&qp->sq.pbl);
3433 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3434 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3435
3436 awqe3 = qed_chain_produce(&qp->sq.pbl);
3437
3438 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3439 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3440 DMA_REGPAIR_LE(awqe3->swap_data,
3441 atomic_wr(wr)->compare_add);
3442 } else {
3443 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3444 DMA_REGPAIR_LE(awqe3->swap_data,
3445 atomic_wr(wr)->swap);
3446 DMA_REGPAIR_LE(awqe3->cmp_data,
3447 atomic_wr(wr)->compare_add);
3448 }
3449
3450 qedr_prepare_sq_sges(qp, NULL, wr);
3451
3452 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3453 qp->prev_wqe_size = awqe1->wqe_size;
3454 break;
3455
3456 case IB_WR_LOCAL_INV:
3457 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3458 iwqe->wqe_size = 1;
3459
3460 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3461 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3462 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3463 qp->prev_wqe_size = iwqe->wqe_size;
3464 break;
3465 case IB_WR_REG_MR:
3466 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3467 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3468 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3469 fwqe1->wqe_size = 2;
3470
3471 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3472 if (rc) {
3473 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3474 *bad_wr = wr;
3475 break;
3476 }
3477
3478 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3479 qp->prev_wqe_size = fwqe1->wqe_size;
3480 break;
3481 default:
3482 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3483 rc = -EINVAL;
3484 *bad_wr = wr;
3485 break;
3486 }
3487
3488 if (*bad_wr) {
3489 u16 value;
3490
3491 /* Restore prod to its position before
3492 * this WR was processed
3493 */
3494 value = le16_to_cpu(qp->sq.db_data.data.value);
3495 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3496
3497 /* Restore prev_wqe_size */
3498 qp->prev_wqe_size = wqe->prev_wqe_size;
3499 rc = -EINVAL;
3500 DP_ERR(dev, "POST SEND FAILED\n");
3501 }
3502
3503 return rc;
3504 }
3505
qedr_post_send(struct ib_qp * ibqp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3506 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3507 const struct ib_send_wr **bad_wr)
3508 {
3509 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3510 struct qedr_qp *qp = get_qedr_qp(ibqp);
3511 unsigned long flags;
3512 int rc = 0;
3513
3514 *bad_wr = NULL;
3515
3516 if (qp->qp_type == IB_QPT_GSI)
3517 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3518
3519 spin_lock_irqsave(&qp->q_lock, flags);
3520
3521 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3522 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3523 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3524 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3525 spin_unlock_irqrestore(&qp->q_lock, flags);
3526 *bad_wr = wr;
3527 DP_DEBUG(dev, QEDR_MSG_CQ,
3528 "QP in wrong state! QP icid=0x%x state %d\n",
3529 qp->icid, qp->state);
3530 return -EINVAL;
3531 }
3532 }
3533
3534 while (wr) {
3535 rc = __qedr_post_send(ibqp, wr, bad_wr);
3536 if (rc)
3537 break;
3538
3539 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3540
3541 qedr_inc_sw_prod(&qp->sq);
3542
3543 qp->sq.db_data.data.value++;
3544
3545 wr = wr->next;
3546 }
3547
3548 /* Trigger doorbell
3549 * If there was a failure in the first WR then it will be triggered in
3550 * vane. However this is not harmful (as long as the producer value is
3551 * unchanged). For performance reasons we avoid checking for this
3552 * redundant doorbell.
3553 *
3554 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3555 * soon as we give the doorbell, we could get a completion
3556 * for this wr, therefore we need to make sure that the
3557 * memory is updated before giving the doorbell.
3558 * During qedr_poll_cq, rmb is called before accessing the
3559 * cqe. This covers for the smp_rmb as well.
3560 */
3561 smp_wmb();
3562 writel(qp->sq.db_data.raw, qp->sq.db);
3563
3564 /* Make sure write sticks */
3565 mmiowb();
3566
3567 spin_unlock_irqrestore(&qp->q_lock, flags);
3568
3569 return rc;
3570 }
3571
qedr_srq_elem_left(struct qedr_srq_hwq_info * hw_srq)3572 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3573 {
3574 u32 used;
3575
3576 /* Calculate number of elements used based on producer
3577 * count and consumer count and subtract it from max
3578 * work request supported so that we get elements left.
3579 */
3580 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3581
3582 return hw_srq->max_wr - used;
3583 }
3584
qedr_post_srq_recv(struct ib_srq * ibsrq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)3585 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3586 const struct ib_recv_wr **bad_wr)
3587 {
3588 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3589 struct qedr_srq_hwq_info *hw_srq;
3590 struct qedr_dev *dev = srq->dev;
3591 struct qed_chain *pbl;
3592 unsigned long flags;
3593 int status = 0;
3594 u32 num_sge;
3595
3596 spin_lock_irqsave(&srq->lock, flags);
3597
3598 hw_srq = &srq->hw_srq;
3599 pbl = &srq->hw_srq.pbl;
3600 while (wr) {
3601 struct rdma_srq_wqe_header *hdr;
3602 int i;
3603
3604 if (!qedr_srq_elem_left(hw_srq) ||
3605 wr->num_sge > srq->hw_srq.max_sges) {
3606 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3607 hw_srq->wr_prod_cnt,
3608 atomic_read(&hw_srq->wr_cons_cnt),
3609 wr->num_sge, srq->hw_srq.max_sges);
3610 status = -ENOMEM;
3611 *bad_wr = wr;
3612 break;
3613 }
3614
3615 hdr = qed_chain_produce(pbl);
3616 num_sge = wr->num_sge;
3617 /* Set number of sge and work request id in header */
3618 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3619
3620 srq->hw_srq.wr_prod_cnt++;
3621 hw_srq->wqe_prod++;
3622 hw_srq->sge_prod++;
3623
3624 DP_DEBUG(dev, QEDR_MSG_SRQ,
3625 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3626 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3627
3628 for (i = 0; i < wr->num_sge; i++) {
3629 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3630
3631 /* Set SGE length, lkey and address */
3632 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3633 wr->sg_list[i].length, wr->sg_list[i].lkey);
3634
3635 DP_DEBUG(dev, QEDR_MSG_SRQ,
3636 "[%d]: len %d key %x addr %x:%x\n",
3637 i, srq_sge->length, srq_sge->l_key,
3638 srq_sge->addr.hi, srq_sge->addr.lo);
3639 hw_srq->sge_prod++;
3640 }
3641
3642 /* Update WQE and SGE information before
3643 * updating producer.
3644 */
3645 dma_wmb();
3646
3647 /* SRQ producer is 8 bytes. Need to update SGE producer index
3648 * in first 4 bytes and need to update WQE producer in
3649 * next 4 bytes.
3650 */
3651 srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3652 /* Make sure sge producer is updated first */
3653 dma_wmb();
3654 srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3655
3656 wr = wr->next;
3657 }
3658
3659 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3660 qed_chain_get_elem_left(pbl));
3661 spin_unlock_irqrestore(&srq->lock, flags);
3662
3663 return status;
3664 }
3665
qedr_post_recv(struct ib_qp * ibqp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)3666 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3667 const struct ib_recv_wr **bad_wr)
3668 {
3669 struct qedr_qp *qp = get_qedr_qp(ibqp);
3670 struct qedr_dev *dev = qp->dev;
3671 unsigned long flags;
3672 int status = 0;
3673
3674 if (qp->qp_type == IB_QPT_GSI)
3675 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3676
3677 spin_lock_irqsave(&qp->q_lock, flags);
3678
3679 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3680 spin_unlock_irqrestore(&qp->q_lock, flags);
3681 *bad_wr = wr;
3682 return -EINVAL;
3683 }
3684
3685 while (wr) {
3686 int i;
3687
3688 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3689 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3690 wr->num_sge > qp->rq.max_sges) {
3691 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3692 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3693 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3694 qp->rq.max_sges);
3695 status = -ENOMEM;
3696 *bad_wr = wr;
3697 break;
3698 }
3699 for (i = 0; i < wr->num_sge; i++) {
3700 u32 flags = 0;
3701 struct rdma_rq_sge *rqe =
3702 qed_chain_produce(&qp->rq.pbl);
3703
3704 /* First one must include the number
3705 * of SGE in the list
3706 */
3707 if (!i)
3708 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3709 wr->num_sge);
3710
3711 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3712 wr->sg_list[i].lkey);
3713
3714 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3715 wr->sg_list[i].length, flags);
3716 }
3717
3718 /* Special case of no sges. FW requires between 1-4 sges...
3719 * in this case we need to post 1 sge with length zero. this is
3720 * because rdma write with immediate consumes an RQ.
3721 */
3722 if (!wr->num_sge) {
3723 u32 flags = 0;
3724 struct rdma_rq_sge *rqe =
3725 qed_chain_produce(&qp->rq.pbl);
3726
3727 /* First one must include the number
3728 * of SGE in the list
3729 */
3730 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3731 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3732
3733 RQ_SGE_SET(rqe, 0, 0, flags);
3734 i = 1;
3735 }
3736
3737 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3738 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3739
3740 qedr_inc_sw_prod(&qp->rq);
3741
3742 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3743 * soon as we give the doorbell, we could get a completion
3744 * for this wr, therefore we need to make sure that the
3745 * memory is update before giving the doorbell.
3746 * During qedr_poll_cq, rmb is called before accessing the
3747 * cqe. This covers for the smp_rmb as well.
3748 */
3749 smp_wmb();
3750
3751 qp->rq.db_data.data.value++;
3752
3753 writel(qp->rq.db_data.raw, qp->rq.db);
3754
3755 /* Make sure write sticks */
3756 mmiowb();
3757
3758 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3759 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3760 mmiowb(); /* for second doorbell */
3761 }
3762
3763 wr = wr->next;
3764 }
3765
3766 spin_unlock_irqrestore(&qp->q_lock, flags);
3767
3768 return status;
3769 }
3770
is_valid_cqe(struct qedr_cq * cq,union rdma_cqe * cqe)3771 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3772 {
3773 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3774
3775 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3776 cq->pbl_toggle;
3777 }
3778
cqe_get_qp(union rdma_cqe * cqe)3779 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3780 {
3781 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3782 struct qedr_qp *qp;
3783
3784 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3785 resp_cqe->qp_handle.lo,
3786 u64);
3787 return qp;
3788 }
3789
cqe_get_type(union rdma_cqe * cqe)3790 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3791 {
3792 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3793
3794 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3795 }
3796
3797 /* Return latest CQE (needs processing) */
get_cqe(struct qedr_cq * cq)3798 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3799 {
3800 return cq->latest_cqe;
3801 }
3802
3803 /* In fmr we need to increase the number of fmr completed counter for the fmr
3804 * algorithm determining whether we can free a pbl or not.
3805 * we need to perform this whether the work request was signaled or not. for
3806 * this purpose we call this function from the condition that checks if a wr
3807 * should be skipped, to make sure we don't miss it ( possibly this fmr
3808 * operation was not signalted)
3809 */
qedr_chk_if_fmr(struct qedr_qp * qp)3810 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3811 {
3812 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3813 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3814 }
3815
process_req(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,u16 hw_cons,enum ib_wc_status status,int force)3816 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3817 struct qedr_cq *cq, int num_entries,
3818 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3819 int force)
3820 {
3821 u16 cnt = 0;
3822
3823 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3824 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3825 qedr_chk_if_fmr(qp);
3826 /* skip WC */
3827 goto next_cqe;
3828 }
3829
3830 /* fill WC */
3831 wc->status = status;
3832 wc->vendor_err = 0;
3833 wc->wc_flags = 0;
3834 wc->src_qp = qp->id;
3835 wc->qp = &qp->ibqp;
3836
3837 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3838 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3839
3840 switch (wc->opcode) {
3841 case IB_WC_RDMA_WRITE:
3842 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3843 break;
3844 case IB_WC_COMP_SWAP:
3845 case IB_WC_FETCH_ADD:
3846 wc->byte_len = 8;
3847 break;
3848 case IB_WC_REG_MR:
3849 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3850 break;
3851 case IB_WC_RDMA_READ:
3852 case IB_WC_SEND:
3853 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3854 break;
3855 default:
3856 break;
3857 }
3858
3859 num_entries--;
3860 wc++;
3861 cnt++;
3862 next_cqe:
3863 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3864 qed_chain_consume(&qp->sq.pbl);
3865 qedr_inc_sw_cons(&qp->sq);
3866 }
3867
3868 return cnt;
3869 }
3870
qedr_poll_cq_req(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,struct rdma_cqe_requester * req)3871 static int qedr_poll_cq_req(struct qedr_dev *dev,
3872 struct qedr_qp *qp, struct qedr_cq *cq,
3873 int num_entries, struct ib_wc *wc,
3874 struct rdma_cqe_requester *req)
3875 {
3876 int cnt = 0;
3877
3878 switch (req->status) {
3879 case RDMA_CQE_REQ_STS_OK:
3880 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3881 IB_WC_SUCCESS, 0);
3882 break;
3883 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3884 if (qp->state != QED_ROCE_QP_STATE_ERR)
3885 DP_DEBUG(dev, QEDR_MSG_CQ,
3886 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3887 cq->icid, qp->icid);
3888 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3889 IB_WC_WR_FLUSH_ERR, 1);
3890 break;
3891 default:
3892 /* process all WQE before the cosumer */
3893 qp->state = QED_ROCE_QP_STATE_ERR;
3894 cnt = process_req(dev, qp, cq, num_entries, wc,
3895 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3896 wc += cnt;
3897 /* if we have extra WC fill it with actual error info */
3898 if (cnt < num_entries) {
3899 enum ib_wc_status wc_status;
3900
3901 switch (req->status) {
3902 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3903 DP_ERR(dev,
3904 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3905 cq->icid, qp->icid);
3906 wc_status = IB_WC_BAD_RESP_ERR;
3907 break;
3908 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3909 DP_ERR(dev,
3910 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3911 cq->icid, qp->icid);
3912 wc_status = IB_WC_LOC_LEN_ERR;
3913 break;
3914 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3915 DP_ERR(dev,
3916 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3917 cq->icid, qp->icid);
3918 wc_status = IB_WC_LOC_QP_OP_ERR;
3919 break;
3920 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3921 DP_ERR(dev,
3922 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3923 cq->icid, qp->icid);
3924 wc_status = IB_WC_LOC_PROT_ERR;
3925 break;
3926 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3927 DP_ERR(dev,
3928 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3929 cq->icid, qp->icid);
3930 wc_status = IB_WC_MW_BIND_ERR;
3931 break;
3932 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3933 DP_ERR(dev,
3934 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3935 cq->icid, qp->icid);
3936 wc_status = IB_WC_REM_INV_REQ_ERR;
3937 break;
3938 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3939 DP_ERR(dev,
3940 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3941 cq->icid, qp->icid);
3942 wc_status = IB_WC_REM_ACCESS_ERR;
3943 break;
3944 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3945 DP_ERR(dev,
3946 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3947 cq->icid, qp->icid);
3948 wc_status = IB_WC_REM_OP_ERR;
3949 break;
3950 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3951 DP_ERR(dev,
3952 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3953 cq->icid, qp->icid);
3954 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3955 break;
3956 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3957 DP_ERR(dev,
3958 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3959 cq->icid, qp->icid);
3960 wc_status = IB_WC_RETRY_EXC_ERR;
3961 break;
3962 default:
3963 DP_ERR(dev,
3964 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3965 cq->icid, qp->icid);
3966 wc_status = IB_WC_GENERAL_ERR;
3967 }
3968 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3969 wc_status, 1);
3970 }
3971 }
3972
3973 return cnt;
3974 }
3975
qedr_cqe_resp_status_to_ib(u8 status)3976 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3977 {
3978 switch (status) {
3979 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3980 return IB_WC_LOC_ACCESS_ERR;
3981 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3982 return IB_WC_LOC_LEN_ERR;
3983 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3984 return IB_WC_LOC_QP_OP_ERR;
3985 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3986 return IB_WC_LOC_PROT_ERR;
3987 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3988 return IB_WC_MW_BIND_ERR;
3989 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3990 return IB_WC_REM_INV_RD_REQ_ERR;
3991 case RDMA_CQE_RESP_STS_OK:
3992 return IB_WC_SUCCESS;
3993 default:
3994 return IB_WC_GENERAL_ERR;
3995 }
3996 }
3997
qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder * resp,struct ib_wc * wc)3998 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3999 struct ib_wc *wc)
4000 {
4001 wc->status = IB_WC_SUCCESS;
4002 wc->byte_len = le32_to_cpu(resp->length);
4003
4004 if (resp->flags & QEDR_RESP_IMM) {
4005 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4006 wc->wc_flags |= IB_WC_WITH_IMM;
4007
4008 if (resp->flags & QEDR_RESP_RDMA)
4009 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4010
4011 if (resp->flags & QEDR_RESP_INV)
4012 return -EINVAL;
4013
4014 } else if (resp->flags & QEDR_RESP_INV) {
4015 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4016 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4017
4018 if (resp->flags & QEDR_RESP_RDMA)
4019 return -EINVAL;
4020
4021 } else if (resp->flags & QEDR_RESP_RDMA) {
4022 return -EINVAL;
4023 }
4024
4025 return 0;
4026 }
4027
__process_resp_one(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,struct ib_wc * wc,struct rdma_cqe_responder * resp,u64 wr_id)4028 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4029 struct qedr_cq *cq, struct ib_wc *wc,
4030 struct rdma_cqe_responder *resp, u64 wr_id)
4031 {
4032 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4033 wc->opcode = IB_WC_RECV;
4034 wc->wc_flags = 0;
4035
4036 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4037 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4038 DP_ERR(dev,
4039 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4040 cq, cq->icid, resp->flags);
4041
4042 } else {
4043 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4044 if (wc->status == IB_WC_GENERAL_ERR)
4045 DP_ERR(dev,
4046 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4047 cq, cq->icid, resp->status);
4048 }
4049
4050 /* Fill the rest of the WC */
4051 wc->vendor_err = 0;
4052 wc->src_qp = qp->id;
4053 wc->qp = &qp->ibqp;
4054 wc->wr_id = wr_id;
4055 }
4056
process_resp_one_srq(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,struct ib_wc * wc,struct rdma_cqe_responder * resp)4057 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4058 struct qedr_cq *cq, struct ib_wc *wc,
4059 struct rdma_cqe_responder *resp)
4060 {
4061 struct qedr_srq *srq = qp->srq;
4062 u64 wr_id;
4063
4064 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4065 le32_to_cpu(resp->srq_wr_id.lo), u64);
4066
4067 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4068 wc->status = IB_WC_WR_FLUSH_ERR;
4069 wc->vendor_err = 0;
4070 wc->wr_id = wr_id;
4071 wc->byte_len = 0;
4072 wc->src_qp = qp->id;
4073 wc->qp = &qp->ibqp;
4074 wc->wr_id = wr_id;
4075 } else {
4076 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4077 }
4078 atomic_inc(&srq->hw_srq.wr_cons_cnt);
4079
4080 return 1;
4081 }
process_resp_one(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,struct ib_wc * wc,struct rdma_cqe_responder * resp)4082 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4083 struct qedr_cq *cq, struct ib_wc *wc,
4084 struct rdma_cqe_responder *resp)
4085 {
4086 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4087
4088 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4089
4090 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4091 qed_chain_consume(&qp->rq.pbl);
4092 qedr_inc_sw_cons(&qp->rq);
4093
4094 return 1;
4095 }
4096
process_resp_flush(struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,u16 hw_cons)4097 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4098 int num_entries, struct ib_wc *wc, u16 hw_cons)
4099 {
4100 u16 cnt = 0;
4101
4102 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4103 /* fill WC */
4104 wc->status = IB_WC_WR_FLUSH_ERR;
4105 wc->vendor_err = 0;
4106 wc->wc_flags = 0;
4107 wc->src_qp = qp->id;
4108 wc->byte_len = 0;
4109 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4110 wc->qp = &qp->ibqp;
4111 num_entries--;
4112 wc++;
4113 cnt++;
4114 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4115 qed_chain_consume(&qp->rq.pbl);
4116 qedr_inc_sw_cons(&qp->rq);
4117 }
4118
4119 return cnt;
4120 }
4121
try_consume_resp_cqe(struct qedr_cq * cq,struct qedr_qp * qp,struct rdma_cqe_responder * resp,int * update)4122 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4123 struct rdma_cqe_responder *resp, int *update)
4124 {
4125 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4126 consume_cqe(cq);
4127 *update |= 1;
4128 }
4129 }
4130
qedr_poll_cq_resp_srq(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,struct rdma_cqe_responder * resp)4131 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4132 struct qedr_cq *cq, int num_entries,
4133 struct ib_wc *wc,
4134 struct rdma_cqe_responder *resp)
4135 {
4136 int cnt;
4137
4138 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4139 consume_cqe(cq);
4140
4141 return cnt;
4142 }
4143
qedr_poll_cq_resp(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,struct rdma_cqe_responder * resp,int * update)4144 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4145 struct qedr_cq *cq, int num_entries,
4146 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4147 int *update)
4148 {
4149 int cnt;
4150
4151 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4152 cnt = process_resp_flush(qp, cq, num_entries, wc,
4153 resp->rq_cons_or_srq_id);
4154 try_consume_resp_cqe(cq, qp, resp, update);
4155 } else {
4156 cnt = process_resp_one(dev, qp, cq, wc, resp);
4157 consume_cqe(cq);
4158 *update |= 1;
4159 }
4160
4161 return cnt;
4162 }
4163
try_consume_req_cqe(struct qedr_cq * cq,struct qedr_qp * qp,struct rdma_cqe_requester * req,int * update)4164 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4165 struct rdma_cqe_requester *req, int *update)
4166 {
4167 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4168 consume_cqe(cq);
4169 *update |= 1;
4170 }
4171 }
4172
qedr_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)4173 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4174 {
4175 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4176 struct qedr_cq *cq = get_qedr_cq(ibcq);
4177 union rdma_cqe *cqe;
4178 u32 old_cons, new_cons;
4179 unsigned long flags;
4180 int update = 0;
4181 int done = 0;
4182
4183 if (cq->destroyed) {
4184 DP_ERR(dev,
4185 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4186 cq, cq->icid);
4187 return 0;
4188 }
4189
4190 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4191 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4192
4193 spin_lock_irqsave(&cq->cq_lock, flags);
4194 cqe = cq->latest_cqe;
4195 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4196 while (num_entries && is_valid_cqe(cq, cqe)) {
4197 struct qedr_qp *qp;
4198 int cnt = 0;
4199
4200 /* prevent speculative reads of any field of CQE */
4201 rmb();
4202
4203 qp = cqe_get_qp(cqe);
4204 if (!qp) {
4205 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4206 break;
4207 }
4208
4209 wc->qp = &qp->ibqp;
4210
4211 switch (cqe_get_type(cqe)) {
4212 case RDMA_CQE_TYPE_REQUESTER:
4213 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4214 &cqe->req);
4215 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4216 break;
4217 case RDMA_CQE_TYPE_RESPONDER_RQ:
4218 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4219 &cqe->resp, &update);
4220 break;
4221 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4222 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4223 wc, &cqe->resp);
4224 update = 1;
4225 break;
4226 case RDMA_CQE_TYPE_INVALID:
4227 default:
4228 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4229 cqe_get_type(cqe));
4230 }
4231 num_entries -= cnt;
4232 wc += cnt;
4233 done += cnt;
4234
4235 cqe = get_cqe(cq);
4236 }
4237 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4238
4239 cq->cq_cons += new_cons - old_cons;
4240
4241 if (update)
4242 /* doorbell notifies abount latest VALID entry,
4243 * but chain already point to the next INVALID one
4244 */
4245 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4246
4247 spin_unlock_irqrestore(&cq->cq_lock, flags);
4248 return done;
4249 }
4250
qedr_process_mad(struct ib_device * ibdev,int process_mad_flags,u8 port_num,const struct ib_wc * in_wc,const struct ib_grh * in_grh,const struct ib_mad_hdr * mad_hdr,size_t in_mad_size,struct ib_mad_hdr * out_mad,size_t * out_mad_size,u16 * out_mad_pkey_index)4251 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4252 u8 port_num,
4253 const struct ib_wc *in_wc,
4254 const struct ib_grh *in_grh,
4255 const struct ib_mad_hdr *mad_hdr,
4256 size_t in_mad_size, struct ib_mad_hdr *out_mad,
4257 size_t *out_mad_size, u16 *out_mad_pkey_index)
4258 {
4259 struct qedr_dev *dev = get_qedr_dev(ibdev);
4260
4261 DP_DEBUG(dev, QEDR_MSG_GSI,
4262 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4263 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4264 mad_hdr->class_specific, mad_hdr->class_version,
4265 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4266 return IB_MAD_RESULT_SUCCESS;
4267 }
4268