• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Broadcom NetXtreme-E RoCE driver.
3  *
4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * BSD license below:
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in
21  *    the documentation and/or other materials provided with the
22  *    distribution.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  *
36  * Description: IB Verbs interpreter
37  */
38 
39 #include <linux/interrupt.h>
40 #include <linux/types.h>
41 #include <linux/pci.h>
42 #include <linux/netdevice.h>
43 #include <linux/if_ether.h>
44 
45 #include <rdma/ib_verbs.h>
46 #include <rdma/ib_user_verbs.h>
47 #include <rdma/ib_umem.h>
48 #include <rdma/ib_addr.h>
49 #include <rdma/ib_mad.h>
50 #include <rdma/ib_cache.h>
51 
52 #include "bnxt_ulp.h"
53 
54 #include "roce_hsi.h"
55 #include "qplib_res.h"
56 #include "qplib_sp.h"
57 #include "qplib_fp.h"
58 #include "qplib_rcfw.h"
59 
60 #include "bnxt_re.h"
61 #include "ib_verbs.h"
62 #include <rdma/bnxt_re-abi.h>
63 
__from_ib_access_flags(int iflags)64 static int __from_ib_access_flags(int iflags)
65 {
66 	int qflags = 0;
67 
68 	if (iflags & IB_ACCESS_LOCAL_WRITE)
69 		qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
70 	if (iflags & IB_ACCESS_REMOTE_READ)
71 		qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
72 	if (iflags & IB_ACCESS_REMOTE_WRITE)
73 		qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
74 	if (iflags & IB_ACCESS_REMOTE_ATOMIC)
75 		qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
76 	if (iflags & IB_ACCESS_MW_BIND)
77 		qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
78 	if (iflags & IB_ZERO_BASED)
79 		qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
80 	if (iflags & IB_ACCESS_ON_DEMAND)
81 		qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
82 	return qflags;
83 };
84 
__to_ib_access_flags(int qflags)85 static enum ib_access_flags __to_ib_access_flags(int qflags)
86 {
87 	enum ib_access_flags iflags = 0;
88 
89 	if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
90 		iflags |= IB_ACCESS_LOCAL_WRITE;
91 	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
92 		iflags |= IB_ACCESS_REMOTE_WRITE;
93 	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
94 		iflags |= IB_ACCESS_REMOTE_READ;
95 	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
96 		iflags |= IB_ACCESS_REMOTE_ATOMIC;
97 	if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
98 		iflags |= IB_ACCESS_MW_BIND;
99 	if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
100 		iflags |= IB_ZERO_BASED;
101 	if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
102 		iflags |= IB_ACCESS_ON_DEMAND;
103 	return iflags;
104 };
105 
bnxt_re_build_sgl(struct ib_sge * ib_sg_list,struct bnxt_qplib_sge * sg_list,int num)106 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
107 			     struct bnxt_qplib_sge *sg_list, int num)
108 {
109 	int i, total = 0;
110 
111 	for (i = 0; i < num; i++) {
112 		sg_list[i].addr = ib_sg_list[i].addr;
113 		sg_list[i].lkey = ib_sg_list[i].lkey;
114 		sg_list[i].size = ib_sg_list[i].length;
115 		total += sg_list[i].size;
116 	}
117 	return total;
118 }
119 
120 /* Device */
bnxt_re_get_netdev(struct ib_device * ibdev,u8 port_num)121 struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num)
122 {
123 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
124 	struct net_device *netdev = NULL;
125 
126 	rcu_read_lock();
127 	if (rdev)
128 		netdev = rdev->netdev;
129 	if (netdev)
130 		dev_hold(netdev);
131 
132 	rcu_read_unlock();
133 	return netdev;
134 }
135 
bnxt_re_query_device(struct ib_device * ibdev,struct ib_device_attr * ib_attr,struct ib_udata * udata)136 int bnxt_re_query_device(struct ib_device *ibdev,
137 			 struct ib_device_attr *ib_attr,
138 			 struct ib_udata *udata)
139 {
140 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
141 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
142 
143 	memset(ib_attr, 0, sizeof(*ib_attr));
144 
145 	ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver);
146 	bnxt_qplib_get_guid(rdev->netdev->dev_addr,
147 			    (u8 *)&ib_attr->sys_image_guid);
148 	ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
149 	ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K;
150 
151 	ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
152 	ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
153 	ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
154 	ib_attr->max_qp = dev_attr->max_qp;
155 	ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
156 	ib_attr->device_cap_flags =
157 				    IB_DEVICE_CURR_QP_STATE_MOD
158 				    | IB_DEVICE_RC_RNR_NAK_GEN
159 				    | IB_DEVICE_SHUTDOWN_PORT
160 				    | IB_DEVICE_SYS_IMAGE_GUID
161 				    | IB_DEVICE_LOCAL_DMA_LKEY
162 				    | IB_DEVICE_RESIZE_MAX_WR
163 				    | IB_DEVICE_PORT_ACTIVE_EVENT
164 				    | IB_DEVICE_N_NOTIFY_CQ
165 				    | IB_DEVICE_MEM_WINDOW
166 				    | IB_DEVICE_MEM_WINDOW_TYPE_2B
167 				    | IB_DEVICE_MEM_MGT_EXTENSIONS;
168 	ib_attr->max_sge = dev_attr->max_qp_sges;
169 	ib_attr->max_sge_rd = dev_attr->max_qp_sges;
170 	ib_attr->max_cq = dev_attr->max_cq;
171 	ib_attr->max_cqe = dev_attr->max_cq_wqes;
172 	ib_attr->max_mr = dev_attr->max_mr;
173 	ib_attr->max_pd = dev_attr->max_pd;
174 	ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
175 	ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
176 	if (dev_attr->is_atomic) {
177 		ib_attr->atomic_cap = IB_ATOMIC_HCA;
178 		ib_attr->masked_atomic_cap = IB_ATOMIC_HCA;
179 	}
180 
181 	ib_attr->max_ee_rd_atom = 0;
182 	ib_attr->max_res_rd_atom = 0;
183 	ib_attr->max_ee_init_rd_atom = 0;
184 	ib_attr->max_ee = 0;
185 	ib_attr->max_rdd = 0;
186 	ib_attr->max_mw = dev_attr->max_mw;
187 	ib_attr->max_raw_ipv6_qp = 0;
188 	ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
189 	ib_attr->max_mcast_grp = 0;
190 	ib_attr->max_mcast_qp_attach = 0;
191 	ib_attr->max_total_mcast_qp_attach = 0;
192 	ib_attr->max_ah = dev_attr->max_ah;
193 
194 	ib_attr->max_fmr = 0;
195 	ib_attr->max_map_per_fmr = 0;
196 
197 	ib_attr->max_srq = dev_attr->max_srq;
198 	ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
199 	ib_attr->max_srq_sge = dev_attr->max_srq_sges;
200 
201 	ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
202 
203 	ib_attr->max_pkeys = 1;
204 	ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
205 	return 0;
206 }
207 
bnxt_re_modify_device(struct ib_device * ibdev,int device_modify_mask,struct ib_device_modify * device_modify)208 int bnxt_re_modify_device(struct ib_device *ibdev,
209 			  int device_modify_mask,
210 			  struct ib_device_modify *device_modify)
211 {
212 	switch (device_modify_mask) {
213 	case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:
214 		/* Modify the GUID requires the modification of the GID table */
215 		/* GUID should be made as READ-ONLY */
216 		break;
217 	case IB_DEVICE_MODIFY_NODE_DESC:
218 		/* Node Desc should be made as READ-ONLY */
219 		break;
220 	default:
221 		break;
222 	}
223 	return 0;
224 }
225 
226 /* Port */
bnxt_re_query_port(struct ib_device * ibdev,u8 port_num,struct ib_port_attr * port_attr)227 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
228 		       struct ib_port_attr *port_attr)
229 {
230 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
231 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
232 
233 	memset(port_attr, 0, sizeof(*port_attr));
234 
235 	if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
236 		port_attr->state = IB_PORT_ACTIVE;
237 		port_attr->phys_state = 5;
238 	} else {
239 		port_attr->state = IB_PORT_DOWN;
240 		port_attr->phys_state = 3;
241 	}
242 	port_attr->max_mtu = IB_MTU_4096;
243 	port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
244 	port_attr->gid_tbl_len = dev_attr->max_sgid;
245 	port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
246 				    IB_PORT_DEVICE_MGMT_SUP |
247 				    IB_PORT_VENDOR_CLASS_SUP |
248 				    IB_PORT_IP_BASED_GIDS;
249 
250 	/* Max MSG size set to 2G for now */
251 	port_attr->max_msg_sz = 0x80000000;
252 	port_attr->bad_pkey_cntr = 0;
253 	port_attr->qkey_viol_cntr = 0;
254 	port_attr->pkey_tbl_len = dev_attr->max_pkey;
255 	port_attr->lid = 0;
256 	port_attr->sm_lid = 0;
257 	port_attr->lmc = 0;
258 	port_attr->max_vl_num = 4;
259 	port_attr->sm_sl = 0;
260 	port_attr->subnet_timeout = 0;
261 	port_attr->init_type_reply = 0;
262 	port_attr->active_speed = rdev->active_speed;
263 	port_attr->active_width = rdev->active_width;
264 
265 	return 0;
266 }
267 
bnxt_re_get_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)268 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
269 			       struct ib_port_immutable *immutable)
270 {
271 	struct ib_port_attr port_attr;
272 
273 	if (bnxt_re_query_port(ibdev, port_num, &port_attr))
274 		return -EINVAL;
275 
276 	immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
277 	immutable->gid_tbl_len = port_attr.gid_tbl_len;
278 	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
279 	immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
280 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
281 	return 0;
282 }
283 
bnxt_re_query_pkey(struct ib_device * ibdev,u8 port_num,u16 index,u16 * pkey)284 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
285 		       u16 index, u16 *pkey)
286 {
287 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
288 
289 	/* Ignore port_num */
290 
291 	memset(pkey, 0, sizeof(*pkey));
292 	return bnxt_qplib_get_pkey(&rdev->qplib_res,
293 				   &rdev->qplib_res.pkey_tbl, index, pkey);
294 }
295 
bnxt_re_query_gid(struct ib_device * ibdev,u8 port_num,int index,union ib_gid * gid)296 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
297 		      int index, union ib_gid *gid)
298 {
299 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
300 	int rc = 0;
301 
302 	/* Ignore port_num */
303 	memset(gid, 0, sizeof(*gid));
304 	rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
305 				 &rdev->qplib_res.sgid_tbl, index,
306 				 (struct bnxt_qplib_gid *)gid);
307 	return rc;
308 }
309 
bnxt_re_del_gid(struct ib_device * ibdev,u8 port_num,unsigned int index,void ** context)310 int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
311 		    unsigned int index, void **context)
312 {
313 	int rc = 0;
314 	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
315 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
316 	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
317 	struct bnxt_qplib_gid *gid_to_del;
318 
319 	/* Delete the entry from the hardware */
320 	ctx = *context;
321 	if (!ctx)
322 		return -EINVAL;
323 
324 	if (sgid_tbl && sgid_tbl->active) {
325 		if (ctx->idx >= sgid_tbl->max)
326 			return -EINVAL;
327 		gid_to_del = &sgid_tbl->tbl[ctx->idx];
328 		/* DEL_GID is called in WQ context(netdevice_event_work_handler)
329 		 * or via the ib_unregister_device path. In the former case QP1
330 		 * may not be destroyed yet, in which case just return as FW
331 		 * needs that entry to be present and will fail it's deletion.
332 		 * We could get invoked again after QP1 is destroyed OR get an
333 		 * ADD_GID call with a different GID value for the same index
334 		 * where we issue MODIFY_GID cmd to update the GID entry -- TBD
335 		 */
336 		if (ctx->idx == 0 &&
337 		    rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
338 		    ctx->refcnt == 1 && rdev->qp1_sqp) {
339 			dev_dbg(rdev_to_dev(rdev),
340 				"Trying to delete GID0 while QP1 is alive\n");
341 			return -EFAULT;
342 		}
343 		ctx->refcnt--;
344 		if (!ctx->refcnt) {
345 			rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true);
346 			if (rc) {
347 				dev_err(rdev_to_dev(rdev),
348 					"Failed to remove GID: %#x", rc);
349 			} else {
350 				ctx_tbl = sgid_tbl->ctx;
351 				ctx_tbl[ctx->idx] = NULL;
352 				kfree(ctx);
353 			}
354 		}
355 	} else {
356 		return -EINVAL;
357 	}
358 	return rc;
359 }
360 
bnxt_re_add_gid(struct ib_device * ibdev,u8 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr,void ** context)361 int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
362 		    unsigned int index, const union ib_gid *gid,
363 		    const struct ib_gid_attr *attr, void **context)
364 {
365 	int rc;
366 	u32 tbl_idx = 0;
367 	u16 vlan_id = 0xFFFF;
368 	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
369 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
370 	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
371 
372 	if ((attr->ndev) && is_vlan_dev(attr->ndev))
373 		vlan_id = vlan_dev_vlan_id(attr->ndev);
374 
375 	rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)gid,
376 				 rdev->qplib_res.netdev->dev_addr,
377 				 vlan_id, true, &tbl_idx);
378 	if (rc == -EALREADY) {
379 		ctx_tbl = sgid_tbl->ctx;
380 		ctx_tbl[tbl_idx]->refcnt++;
381 		*context = ctx_tbl[tbl_idx];
382 		return 0;
383 	}
384 
385 	if (rc < 0) {
386 		dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc);
387 		return rc;
388 	}
389 
390 	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
391 	if (!ctx)
392 		return -ENOMEM;
393 	ctx_tbl = sgid_tbl->ctx;
394 	ctx->idx = tbl_idx;
395 	ctx->refcnt = 1;
396 	ctx_tbl[tbl_idx] = ctx;
397 	*context = ctx;
398 
399 	return rc;
400 }
401 
bnxt_re_get_link_layer(struct ib_device * ibdev,u8 port_num)402 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
403 					    u8 port_num)
404 {
405 	return IB_LINK_LAYER_ETHERNET;
406 }
407 
408 #define	BNXT_RE_FENCE_PBL_SIZE	DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
409 
bnxt_re_create_fence_wqe(struct bnxt_re_pd * pd)410 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
411 {
412 	struct bnxt_re_fence_data *fence = &pd->fence;
413 	struct ib_mr *ib_mr = &fence->mr->ib_mr;
414 	struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
415 
416 	memset(wqe, 0, sizeof(*wqe));
417 	wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
418 	wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
419 	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
420 	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
421 	wqe->bind.zero_based = false;
422 	wqe->bind.parent_l_key = ib_mr->lkey;
423 	wqe->bind.va = (u64)(unsigned long)fence->va;
424 	wqe->bind.length = fence->size;
425 	wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
426 	wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
427 
428 	/* Save the initial rkey in fence structure for now;
429 	 * wqe->bind.r_key will be set at (re)bind time.
430 	 */
431 	fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
432 }
433 
bnxt_re_bind_fence_mw(struct bnxt_qplib_qp * qplib_qp)434 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
435 {
436 	struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
437 					     qplib_qp);
438 	struct ib_pd *ib_pd = qp->ib_qp.pd;
439 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
440 	struct bnxt_re_fence_data *fence = &pd->fence;
441 	struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
442 	struct bnxt_qplib_swqe wqe;
443 	int rc;
444 
445 	memcpy(&wqe, fence_wqe, sizeof(wqe));
446 	wqe.bind.r_key = fence->bind_rkey;
447 	fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
448 
449 	dev_dbg(rdev_to_dev(qp->rdev),
450 		"Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
451 		wqe.bind.r_key, qp->qplib_qp.id, pd);
452 	rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
453 	if (rc) {
454 		dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n");
455 		return rc;
456 	}
457 	bnxt_qplib_post_send_db(&qp->qplib_qp);
458 
459 	return rc;
460 }
461 
bnxt_re_destroy_fence_mr(struct bnxt_re_pd * pd)462 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
463 {
464 	struct bnxt_re_fence_data *fence = &pd->fence;
465 	struct bnxt_re_dev *rdev = pd->rdev;
466 	struct device *dev = &rdev->en_dev->pdev->dev;
467 	struct bnxt_re_mr *mr = fence->mr;
468 
469 	if (fence->mw) {
470 		bnxt_re_dealloc_mw(fence->mw);
471 		fence->mw = NULL;
472 	}
473 	if (mr) {
474 		if (mr->ib_mr.rkey)
475 			bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
476 					     true);
477 		if (mr->ib_mr.lkey)
478 			bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
479 		kfree(mr);
480 		fence->mr = NULL;
481 	}
482 	if (fence->dma_addr) {
483 		dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
484 				 DMA_BIDIRECTIONAL);
485 		fence->dma_addr = 0;
486 	}
487 }
488 
bnxt_re_create_fence_mr(struct bnxt_re_pd * pd)489 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
490 {
491 	int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
492 	struct bnxt_re_fence_data *fence = &pd->fence;
493 	struct bnxt_re_dev *rdev = pd->rdev;
494 	struct device *dev = &rdev->en_dev->pdev->dev;
495 	struct bnxt_re_mr *mr = NULL;
496 	dma_addr_t dma_addr = 0;
497 	struct ib_mw *mw;
498 	u64 pbl_tbl;
499 	int rc;
500 
501 	dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
502 				  DMA_BIDIRECTIONAL);
503 	rc = dma_mapping_error(dev, dma_addr);
504 	if (rc) {
505 		dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n");
506 		rc = -EIO;
507 		fence->dma_addr = 0;
508 		goto fail;
509 	}
510 	fence->dma_addr = dma_addr;
511 
512 	/* Allocate a MR */
513 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
514 	if (!mr) {
515 		rc = -ENOMEM;
516 		goto fail;
517 	}
518 	fence->mr = mr;
519 	mr->rdev = rdev;
520 	mr->qplib_mr.pd = &pd->qplib_pd;
521 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
522 	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
523 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
524 	if (rc) {
525 		dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n");
526 		goto fail;
527 	}
528 
529 	/* Register MR */
530 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
531 	mr->qplib_mr.va = (u64)(unsigned long)fence->va;
532 	mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
533 	pbl_tbl = dma_addr;
534 	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl_tbl,
535 			       BNXT_RE_FENCE_PBL_SIZE, false);
536 	if (rc) {
537 		dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n");
538 		goto fail;
539 	}
540 	mr->ib_mr.rkey = mr->qplib_mr.rkey;
541 
542 	/* Create a fence MW only for kernel consumers */
543 	mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
544 	if (IS_ERR(mw)) {
545 		dev_err(rdev_to_dev(rdev),
546 			"Failed to create fence-MW for PD: %p\n", pd);
547 		rc = PTR_ERR(mw);
548 		goto fail;
549 	}
550 	fence->mw = mw;
551 
552 	bnxt_re_create_fence_wqe(pd);
553 	return 0;
554 
555 fail:
556 	bnxt_re_destroy_fence_mr(pd);
557 	return rc;
558 }
559 
560 /* Protection Domains */
bnxt_re_dealloc_pd(struct ib_pd * ib_pd)561 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd)
562 {
563 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
564 	struct bnxt_re_dev *rdev = pd->rdev;
565 	int rc;
566 
567 	bnxt_re_destroy_fence_mr(pd);
568 
569 	if (pd->qplib_pd.id) {
570 		rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res,
571 					   &rdev->qplib_res.pd_tbl,
572 					   &pd->qplib_pd);
573 		if (rc)
574 			dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD");
575 	}
576 
577 	kfree(pd);
578 	return 0;
579 }
580 
bnxt_re_alloc_pd(struct ib_device * ibdev,struct ib_ucontext * ucontext,struct ib_udata * udata)581 struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev,
582 			       struct ib_ucontext *ucontext,
583 			       struct ib_udata *udata)
584 {
585 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
586 	struct bnxt_re_ucontext *ucntx = container_of(ucontext,
587 						      struct bnxt_re_ucontext,
588 						      ib_uctx);
589 	struct bnxt_re_pd *pd;
590 	int rc;
591 
592 	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
593 	if (!pd)
594 		return ERR_PTR(-ENOMEM);
595 
596 	pd->rdev = rdev;
597 	if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
598 		dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD");
599 		rc = -ENOMEM;
600 		goto fail;
601 	}
602 
603 	if (udata) {
604 		struct bnxt_re_pd_resp resp;
605 
606 		if (!ucntx->dpi.dbr) {
607 			/* Allocate DPI in alloc_pd to avoid failing of
608 			 * ibv_devinfo and family of application when DPIs
609 			 * are depleted.
610 			 */
611 			if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
612 						 &ucntx->dpi, ucntx)) {
613 				rc = -ENOMEM;
614 				goto dbfail;
615 			}
616 		}
617 
618 		resp.pdid = pd->qplib_pd.id;
619 		/* Still allow mapping this DBR to the new user PD. */
620 		resp.dpi = ucntx->dpi.dpi;
621 		resp.dbr = (u64)ucntx->dpi.umdbr;
622 
623 		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
624 		if (rc) {
625 			dev_err(rdev_to_dev(rdev),
626 				"Failed to copy user response\n");
627 			goto dbfail;
628 		}
629 	}
630 
631 	if (!udata)
632 		if (bnxt_re_create_fence_mr(pd))
633 			dev_warn(rdev_to_dev(rdev),
634 				 "Failed to create Fence-MR\n");
635 	return &pd->ib_pd;
636 dbfail:
637 	(void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
638 				    &pd->qplib_pd);
639 fail:
640 	kfree(pd);
641 	return ERR_PTR(rc);
642 }
643 
644 /* Address Handles */
bnxt_re_destroy_ah(struct ib_ah * ib_ah)645 int bnxt_re_destroy_ah(struct ib_ah *ib_ah)
646 {
647 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
648 	struct bnxt_re_dev *rdev = ah->rdev;
649 	int rc;
650 
651 	rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah);
652 	if (rc) {
653 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH");
654 		return rc;
655 	}
656 	kfree(ah);
657 	return 0;
658 }
659 
bnxt_re_create_ah(struct ib_pd * ib_pd,struct rdma_ah_attr * ah_attr,struct ib_udata * udata)660 struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
661 				struct rdma_ah_attr *ah_attr,
662 				struct ib_udata *udata)
663 {
664 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
665 	struct bnxt_re_dev *rdev = pd->rdev;
666 	struct bnxt_re_ah *ah;
667 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
668 	int rc;
669 	u16 vlan_tag;
670 	u8 nw_type;
671 
672 	struct ib_gid_attr sgid_attr;
673 
674 	if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
675 		dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set");
676 		return ERR_PTR(-EINVAL);
677 	}
678 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
679 	if (!ah)
680 		return ERR_PTR(-ENOMEM);
681 
682 	ah->rdev = rdev;
683 	ah->qplib_ah.pd = &pd->qplib_pd;
684 
685 	/* Supply the configuration for the HW */
686 	memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
687 	       sizeof(union ib_gid));
688 	/*
689 	 * If RoCE V2 is enabled, stack will have two entries for
690 	 * each GID entry. Avoiding this duplicte entry in HW. Dividing
691 	 * the GID index by 2 for RoCE V2
692 	 */
693 	ah->qplib_ah.sgid_index = grh->sgid_index / 2;
694 	ah->qplib_ah.host_sgid_index = grh->sgid_index;
695 	ah->qplib_ah.traffic_class = grh->traffic_class;
696 	ah->qplib_ah.flow_label = grh->flow_label;
697 	ah->qplib_ah.hop_limit = grh->hop_limit;
698 	ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
699 	if (ib_pd->uobject &&
700 	    !rdma_is_multicast_addr((struct in6_addr *)
701 				    grh->dgid.raw) &&
702 	    !rdma_link_local_addr((struct in6_addr *)
703 				  grh->dgid.raw)) {
704 		union ib_gid sgid;
705 
706 		rc = ib_get_cached_gid(&rdev->ibdev, 1,
707 				       grh->sgid_index, &sgid,
708 				       &sgid_attr);
709 		if (rc) {
710 			dev_err(rdev_to_dev(rdev),
711 				"Failed to query gid at index %d",
712 				grh->sgid_index);
713 			goto fail;
714 		}
715 		if (sgid_attr.ndev) {
716 			if (is_vlan_dev(sgid_attr.ndev))
717 				vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev);
718 			dev_put(sgid_attr.ndev);
719 		}
720 		/* Get network header type for this GID */
721 		nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
722 		switch (nw_type) {
723 		case RDMA_NETWORK_IPV4:
724 			ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
725 			break;
726 		case RDMA_NETWORK_IPV6:
727 			ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
728 			break;
729 		default:
730 			ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1;
731 			break;
732 		}
733 		rc = rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid,
734 						  ah_attr->roce.dmac, &vlan_tag,
735 						  &sgid_attr.ndev->ifindex,
736 						  NULL);
737 		if (rc) {
738 			dev_err(rdev_to_dev(rdev), "Failed to get dmac\n");
739 			goto fail;
740 		}
741 	}
742 
743 	memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
744 	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
745 	if (rc) {
746 		dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH");
747 		goto fail;
748 	}
749 
750 	/* Write AVID to shared page. */
751 	if (ib_pd->uobject) {
752 		struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
753 		struct bnxt_re_ucontext *uctx;
754 		unsigned long flag;
755 		u32 *wrptr;
756 
757 		uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
758 		spin_lock_irqsave(&uctx->sh_lock, flag);
759 		wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
760 		*wrptr = ah->qplib_ah.id;
761 		wmb(); /* make sure cache is updated. */
762 		spin_unlock_irqrestore(&uctx->sh_lock, flag);
763 	}
764 
765 	return &ah->ib_ah;
766 
767 fail:
768 	kfree(ah);
769 	return ERR_PTR(rc);
770 }
771 
bnxt_re_modify_ah(struct ib_ah * ib_ah,struct rdma_ah_attr * ah_attr)772 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
773 {
774 	return 0;
775 }
776 
bnxt_re_query_ah(struct ib_ah * ib_ah,struct rdma_ah_attr * ah_attr)777 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
778 {
779 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
780 
781 	ah_attr->type = ib_ah->type;
782 	rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
783 	memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
784 	rdma_ah_set_grh(ah_attr, NULL, 0,
785 			ah->qplib_ah.host_sgid_index,
786 			0, ah->qplib_ah.traffic_class);
787 	rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
788 	rdma_ah_set_port_num(ah_attr, 1);
789 	rdma_ah_set_static_rate(ah_attr, 0);
790 	return 0;
791 }
792 
793 /* Queue Pairs */
bnxt_re_destroy_qp(struct ib_qp * ib_qp)794 int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
795 {
796 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
797 	struct bnxt_re_dev *rdev = qp->rdev;
798 	int rc;
799 
800 	bnxt_qplib_del_flush_qp(&qp->qplib_qp);
801 	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
802 	if (rc) {
803 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");
804 		return rc;
805 	}
806 	if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) {
807 		rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
808 					   &rdev->sqp_ah->qplib_ah);
809 		if (rc) {
810 			dev_err(rdev_to_dev(rdev),
811 				"Failed to destroy HW AH for shadow QP");
812 			return rc;
813 		}
814 
815 		bnxt_qplib_del_flush_qp(&qp->qplib_qp);
816 		rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
817 					   &rdev->qp1_sqp->qplib_qp);
818 		if (rc) {
819 			dev_err(rdev_to_dev(rdev),
820 				"Failed to destroy Shadow QP");
821 			return rc;
822 		}
823 		mutex_lock(&rdev->qp_lock);
824 		list_del(&rdev->qp1_sqp->list);
825 		atomic_dec(&rdev->qp_count);
826 		mutex_unlock(&rdev->qp_lock);
827 
828 		kfree(rdev->sqp_ah);
829 		kfree(rdev->qp1_sqp);
830 		rdev->qp1_sqp = NULL;
831 		rdev->sqp_ah = NULL;
832 	}
833 
834 	if (!IS_ERR_OR_NULL(qp->rumem))
835 		ib_umem_release(qp->rumem);
836 	if (!IS_ERR_OR_NULL(qp->sumem))
837 		ib_umem_release(qp->sumem);
838 
839 	mutex_lock(&rdev->qp_lock);
840 	list_del(&qp->list);
841 	atomic_dec(&rdev->qp_count);
842 	mutex_unlock(&rdev->qp_lock);
843 	kfree(qp);
844 	return 0;
845 }
846 
__from_ib_qp_type(enum ib_qp_type type)847 static u8 __from_ib_qp_type(enum ib_qp_type type)
848 {
849 	switch (type) {
850 	case IB_QPT_GSI:
851 		return CMDQ_CREATE_QP1_TYPE_GSI;
852 	case IB_QPT_RC:
853 		return CMDQ_CREATE_QP_TYPE_RC;
854 	case IB_QPT_UD:
855 		return CMDQ_CREATE_QP_TYPE_UD;
856 	default:
857 		return IB_QPT_MAX;
858 	}
859 }
860 
bnxt_re_init_user_qp(struct bnxt_re_dev * rdev,struct bnxt_re_pd * pd,struct bnxt_re_qp * qp,struct ib_udata * udata)861 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
862 				struct bnxt_re_qp *qp, struct ib_udata *udata)
863 {
864 	struct bnxt_re_qp_req ureq;
865 	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
866 	struct ib_umem *umem;
867 	int bytes = 0;
868 	struct ib_ucontext *context = pd->ib_pd.uobject->context;
869 	struct bnxt_re_ucontext *cntx = container_of(context,
870 						     struct bnxt_re_ucontext,
871 						     ib_uctx);
872 	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
873 		return -EFAULT;
874 
875 	bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
876 	/* Consider mapping PSN search memory only for RC QPs. */
877 	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC)
878 		bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search));
879 	bytes = PAGE_ALIGN(bytes);
880 	umem = ib_umem_get(context, ureq.qpsva, bytes,
881 			   IB_ACCESS_LOCAL_WRITE, 1);
882 	if (IS_ERR(umem))
883 		return PTR_ERR(umem);
884 
885 	qp->sumem = umem;
886 	qplib_qp->sq.sglist = umem->sg_head.sgl;
887 	qplib_qp->sq.nmap = umem->nmap;
888 	qplib_qp->qp_handle = ureq.qp_handle;
889 
890 	if (!qp->qplib_qp.srq) {
891 		bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
892 		bytes = PAGE_ALIGN(bytes);
893 		umem = ib_umem_get(context, ureq.qprva, bytes,
894 				   IB_ACCESS_LOCAL_WRITE, 1);
895 		if (IS_ERR(umem))
896 			goto rqfail;
897 		qp->rumem = umem;
898 		qplib_qp->rq.sglist = umem->sg_head.sgl;
899 		qplib_qp->rq.nmap = umem->nmap;
900 	}
901 
902 	qplib_qp->dpi = &cntx->dpi;
903 	return 0;
904 rqfail:
905 	ib_umem_release(qp->sumem);
906 	qp->sumem = NULL;
907 	qplib_qp->sq.sglist = NULL;
908 	qplib_qp->sq.nmap = 0;
909 
910 	return PTR_ERR(umem);
911 }
912 
bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd * pd,struct bnxt_qplib_res * qp1_res,struct bnxt_qplib_qp * qp1_qp)913 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
914 				(struct bnxt_re_pd *pd,
915 				 struct bnxt_qplib_res *qp1_res,
916 				 struct bnxt_qplib_qp *qp1_qp)
917 {
918 	struct bnxt_re_dev *rdev = pd->rdev;
919 	struct bnxt_re_ah *ah;
920 	union ib_gid sgid;
921 	int rc;
922 
923 	ah = kzalloc(sizeof(*ah), GFP_KERNEL);
924 	if (!ah)
925 		return NULL;
926 
927 	ah->rdev = rdev;
928 	ah->qplib_ah.pd = &pd->qplib_pd;
929 
930 	rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
931 	if (rc)
932 		goto fail;
933 
934 	/* supply the dgid data same as sgid */
935 	memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
936 	       sizeof(union ib_gid));
937 	ah->qplib_ah.sgid_index = 0;
938 
939 	ah->qplib_ah.traffic_class = 0;
940 	ah->qplib_ah.flow_label = 0;
941 	ah->qplib_ah.hop_limit = 1;
942 	ah->qplib_ah.sl = 0;
943 	/* Have DMAC same as SMAC */
944 	ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
945 
946 	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
947 	if (rc) {
948 		dev_err(rdev_to_dev(rdev),
949 			"Failed to allocate HW AH for Shadow QP");
950 		goto fail;
951 	}
952 
953 	return ah;
954 
955 fail:
956 	kfree(ah);
957 	return NULL;
958 }
959 
bnxt_re_create_shadow_qp(struct bnxt_re_pd * pd,struct bnxt_qplib_res * qp1_res,struct bnxt_qplib_qp * qp1_qp)960 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
961 				(struct bnxt_re_pd *pd,
962 				 struct bnxt_qplib_res *qp1_res,
963 				 struct bnxt_qplib_qp *qp1_qp)
964 {
965 	struct bnxt_re_dev *rdev = pd->rdev;
966 	struct bnxt_re_qp *qp;
967 	int rc;
968 
969 	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
970 	if (!qp)
971 		return NULL;
972 
973 	qp->rdev = rdev;
974 
975 	/* Initialize the shadow QP structure from the QP1 values */
976 	ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
977 
978 	qp->qplib_qp.pd = &pd->qplib_pd;
979 	qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
980 	qp->qplib_qp.type = IB_QPT_UD;
981 
982 	qp->qplib_qp.max_inline_data = 0;
983 	qp->qplib_qp.sig_type = true;
984 
985 	/* Shadow QP SQ depth should be same as QP1 RQ depth */
986 	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
987 	qp->qplib_qp.sq.max_sge = 2;
988 	/* Q full delta can be 1 since it is internal QP */
989 	qp->qplib_qp.sq.q_full_delta = 1;
990 
991 	qp->qplib_qp.scq = qp1_qp->scq;
992 	qp->qplib_qp.rcq = qp1_qp->rcq;
993 
994 	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
995 	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
996 	/* Q full delta can be 1 since it is internal QP */
997 	qp->qplib_qp.rq.q_full_delta = 1;
998 
999 	qp->qplib_qp.mtu = qp1_qp->mtu;
1000 
1001 	qp->qplib_qp.sq_hdr_buf_size = 0;
1002 	qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1003 	qp->qplib_qp.dpi = &rdev->dpi_privileged;
1004 
1005 	rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1006 	if (rc)
1007 		goto fail;
1008 
1009 	rdev->sqp_id = qp->qplib_qp.id;
1010 
1011 	spin_lock_init(&qp->sq_lock);
1012 	INIT_LIST_HEAD(&qp->list);
1013 	mutex_lock(&rdev->qp_lock);
1014 	list_add_tail(&qp->list, &rdev->qp_list);
1015 	atomic_inc(&rdev->qp_count);
1016 	mutex_unlock(&rdev->qp_lock);
1017 	return qp;
1018 fail:
1019 	kfree(qp);
1020 	return NULL;
1021 }
1022 
bnxt_re_create_qp(struct ib_pd * ib_pd,struct ib_qp_init_attr * qp_init_attr,struct ib_udata * udata)1023 struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
1024 				struct ib_qp_init_attr *qp_init_attr,
1025 				struct ib_udata *udata)
1026 {
1027 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1028 	struct bnxt_re_dev *rdev = pd->rdev;
1029 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1030 	struct bnxt_re_qp *qp;
1031 	struct bnxt_re_cq *cq;
1032 	int rc, entries;
1033 
1034 	if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
1035 	    (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) ||
1036 	    (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) ||
1037 	    (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) ||
1038 	    (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data))
1039 		return ERR_PTR(-EINVAL);
1040 
1041 	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1042 	if (!qp)
1043 		return ERR_PTR(-ENOMEM);
1044 
1045 	qp->rdev = rdev;
1046 	ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1047 	qp->qplib_qp.pd = &pd->qplib_pd;
1048 	qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1049 	qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type);
1050 	if (qp->qplib_qp.type == IB_QPT_MAX) {
1051 		dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported",
1052 			qp->qplib_qp.type);
1053 		rc = -EINVAL;
1054 		goto fail;
1055 	}
1056 	qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data;
1057 	qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type ==
1058 				  IB_SIGNAL_ALL_WR) ? true : false);
1059 
1060 	qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge;
1061 	if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
1062 		qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
1063 
1064 	if (qp_init_attr->send_cq) {
1065 		cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq,
1066 				  ib_cq);
1067 		if (!cq) {
1068 			dev_err(rdev_to_dev(rdev), "Send CQ not found");
1069 			rc = -EINVAL;
1070 			goto fail;
1071 		}
1072 		qp->qplib_qp.scq = &cq->qplib_cq;
1073 	}
1074 
1075 	if (qp_init_attr->recv_cq) {
1076 		cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq,
1077 				  ib_cq);
1078 		if (!cq) {
1079 			dev_err(rdev_to_dev(rdev), "Receive CQ not found");
1080 			rc = -EINVAL;
1081 			goto fail;
1082 		}
1083 		qp->qplib_qp.rcq = &cq->qplib_cq;
1084 	}
1085 
1086 	if (qp_init_attr->srq) {
1087 		dev_err(rdev_to_dev(rdev), "SRQ not supported");
1088 		rc = -ENOTSUPP;
1089 		goto fail;
1090 	} else {
1091 		/* Allocate 1 more than what's provided so posting max doesn't
1092 		 * mean empty
1093 		 */
1094 		entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1);
1095 		qp->qplib_qp.rq.max_wqe = min_t(u32, entries,
1096 						dev_attr->max_qp_wqes + 1);
1097 
1098 		qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
1099 						qp_init_attr->cap.max_recv_wr;
1100 
1101 		qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge;
1102 		if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
1103 			qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
1104 	}
1105 
1106 	qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1107 
1108 	if (qp_init_attr->qp_type == IB_QPT_GSI) {
1109 		/* Allocate 1 more than what's provided */
1110 		entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1);
1111 		qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
1112 						dev_attr->max_qp_wqes + 1);
1113 		qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
1114 						qp_init_attr->cap.max_send_wr;
1115 		qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
1116 		if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
1117 			qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
1118 		qp->qplib_qp.sq.max_sge++;
1119 		if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
1120 			qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
1121 
1122 		qp->qplib_qp.rq_hdr_buf_size =
1123 					BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1124 
1125 		qp->qplib_qp.sq_hdr_buf_size =
1126 					BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1127 		qp->qplib_qp.dpi = &rdev->dpi_privileged;
1128 		rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp);
1129 		if (rc) {
1130 			dev_err(rdev_to_dev(rdev), "Failed to create HW QP1");
1131 			goto fail;
1132 		}
1133 		/* Create a shadow QP to handle the QP1 traffic */
1134 		rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res,
1135 							 &qp->qplib_qp);
1136 		if (!rdev->qp1_sqp) {
1137 			rc = -EINVAL;
1138 			dev_err(rdev_to_dev(rdev),
1139 				"Failed to create Shadow QP for QP1");
1140 			goto qp_destroy;
1141 		}
1142 		rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1143 							   &qp->qplib_qp);
1144 		if (!rdev->sqp_ah) {
1145 			bnxt_qplib_destroy_qp(&rdev->qplib_res,
1146 					      &rdev->qp1_sqp->qplib_qp);
1147 			rc = -EINVAL;
1148 			dev_err(rdev_to_dev(rdev),
1149 				"Failed to create AH entry for ShadowQP");
1150 			goto qp_destroy;
1151 		}
1152 
1153 	} else {
1154 		/* Allocate 128 + 1 more than what's provided */
1155 		entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr +
1156 					     BNXT_QPLIB_RESERVED_QP_WRS + 1);
1157 		qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
1158 						dev_attr->max_qp_wqes +
1159 						BNXT_QPLIB_RESERVED_QP_WRS + 1);
1160 		qp->qplib_qp.sq.q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1;
1161 
1162 		/*
1163 		 * Reserving one slot for Phantom WQE. Application can
1164 		 * post one extra entry in this case. But allowing this to avoid
1165 		 * unexpected Queue full condition
1166 		 */
1167 
1168 		qp->qplib_qp.sq.q_full_delta -= 1;
1169 
1170 		qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom;
1171 		qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1172 		if (udata) {
1173 			rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1174 			if (rc)
1175 				goto fail;
1176 		} else {
1177 			qp->qplib_qp.dpi = &rdev->dpi_privileged;
1178 		}
1179 
1180 		rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1181 		if (rc) {
1182 			dev_err(rdev_to_dev(rdev), "Failed to create HW QP");
1183 			goto free_umem;
1184 		}
1185 	}
1186 
1187 	qp->ib_qp.qp_num = qp->qplib_qp.id;
1188 	spin_lock_init(&qp->sq_lock);
1189 	spin_lock_init(&qp->rq_lock);
1190 
1191 	if (udata) {
1192 		struct bnxt_re_qp_resp resp;
1193 
1194 		resp.qpid = qp->ib_qp.qp_num;
1195 		resp.rsvd = 0;
1196 		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1197 		if (rc) {
1198 			dev_err(rdev_to_dev(rdev), "Failed to copy QP udata");
1199 			goto qp_destroy;
1200 		}
1201 	}
1202 	INIT_LIST_HEAD(&qp->list);
1203 	mutex_lock(&rdev->qp_lock);
1204 	list_add_tail(&qp->list, &rdev->qp_list);
1205 	atomic_inc(&rdev->qp_count);
1206 	mutex_unlock(&rdev->qp_lock);
1207 
1208 	return &qp->ib_qp;
1209 qp_destroy:
1210 	bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1211 free_umem:
1212 	if (udata) {
1213 		if (qp->rumem)
1214 			ib_umem_release(qp->rumem);
1215 		if (qp->sumem)
1216 			ib_umem_release(qp->sumem);
1217 	}
1218 fail:
1219 	kfree(qp);
1220 	return ERR_PTR(rc);
1221 }
1222 
__from_ib_qp_state(enum ib_qp_state state)1223 static u8 __from_ib_qp_state(enum ib_qp_state state)
1224 {
1225 	switch (state) {
1226 	case IB_QPS_RESET:
1227 		return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1228 	case IB_QPS_INIT:
1229 		return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1230 	case IB_QPS_RTR:
1231 		return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1232 	case IB_QPS_RTS:
1233 		return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1234 	case IB_QPS_SQD:
1235 		return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1236 	case IB_QPS_SQE:
1237 		return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1238 	case IB_QPS_ERR:
1239 	default:
1240 		return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1241 	}
1242 }
1243 
__to_ib_qp_state(u8 state)1244 static enum ib_qp_state __to_ib_qp_state(u8 state)
1245 {
1246 	switch (state) {
1247 	case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1248 		return IB_QPS_RESET;
1249 	case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1250 		return IB_QPS_INIT;
1251 	case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1252 		return IB_QPS_RTR;
1253 	case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1254 		return IB_QPS_RTS;
1255 	case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1256 		return IB_QPS_SQD;
1257 	case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1258 		return IB_QPS_SQE;
1259 	case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1260 	default:
1261 		return IB_QPS_ERR;
1262 	}
1263 }
1264 
__from_ib_mtu(enum ib_mtu mtu)1265 static u32 __from_ib_mtu(enum ib_mtu mtu)
1266 {
1267 	switch (mtu) {
1268 	case IB_MTU_256:
1269 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1270 	case IB_MTU_512:
1271 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1272 	case IB_MTU_1024:
1273 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1274 	case IB_MTU_2048:
1275 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1276 	case IB_MTU_4096:
1277 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1278 	default:
1279 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1280 	}
1281 }
1282 
__to_ib_mtu(u32 mtu)1283 static enum ib_mtu __to_ib_mtu(u32 mtu)
1284 {
1285 	switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1286 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1287 		return IB_MTU_256;
1288 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1289 		return IB_MTU_512;
1290 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1291 		return IB_MTU_1024;
1292 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1293 		return IB_MTU_2048;
1294 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1295 		return IB_MTU_4096;
1296 	default:
1297 		return IB_MTU_2048;
1298 	}
1299 }
1300 
bnxt_re_modify_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp1_qp,int qp_attr_mask)1301 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1302 				    struct bnxt_re_qp *qp1_qp,
1303 				    int qp_attr_mask)
1304 {
1305 	struct bnxt_re_qp *qp = rdev->qp1_sqp;
1306 	int rc = 0;
1307 
1308 	if (qp_attr_mask & IB_QP_STATE) {
1309 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1310 		qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1311 	}
1312 	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1313 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1314 		qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1315 	}
1316 
1317 	if (qp_attr_mask & IB_QP_QKEY) {
1318 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1319 		/* Using a Random  QKEY */
1320 		qp->qplib_qp.qkey = 0x81818181;
1321 	}
1322 	if (qp_attr_mask & IB_QP_SQ_PSN) {
1323 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1324 		qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1325 	}
1326 
1327 	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1328 	if (rc)
1329 		dev_err(rdev_to_dev(rdev),
1330 			"Failed to modify Shadow QP for QP1");
1331 	return rc;
1332 }
1333 
bnxt_re_modify_qp(struct ib_qp * ib_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_udata * udata)1334 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1335 		      int qp_attr_mask, struct ib_udata *udata)
1336 {
1337 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1338 	struct bnxt_re_dev *rdev = qp->rdev;
1339 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1340 	enum ib_qp_state curr_qp_state, new_qp_state;
1341 	int rc, entries;
1342 	int status;
1343 	union ib_gid sgid;
1344 	struct ib_gid_attr sgid_attr;
1345 	u8 nw_type;
1346 
1347 	qp->qplib_qp.modify_flags = 0;
1348 	if (qp_attr_mask & IB_QP_STATE) {
1349 		curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1350 		new_qp_state = qp_attr->qp_state;
1351 		if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1352 					ib_qp->qp_type, qp_attr_mask,
1353 					IB_LINK_LAYER_ETHERNET)) {
1354 			dev_err(rdev_to_dev(rdev),
1355 				"Invalid attribute mask: %#x specified ",
1356 				qp_attr_mask);
1357 			dev_err(rdev_to_dev(rdev),
1358 				"for qpn: %#x type: %#x",
1359 				ib_qp->qp_num, ib_qp->qp_type);
1360 			dev_err(rdev_to_dev(rdev),
1361 				"curr_qp_state=0x%x, new_qp_state=0x%x\n",
1362 				curr_qp_state, new_qp_state);
1363 			return -EINVAL;
1364 		}
1365 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1366 		qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1367 
1368 		if (!qp->sumem &&
1369 		    qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1370 			dev_dbg(rdev_to_dev(rdev),
1371 				"Move QP = %p to flush list\n",
1372 				qp);
1373 			bnxt_qplib_add_flush_qp(&qp->qplib_qp);
1374 		}
1375 		if (!qp->sumem &&
1376 		    qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
1377 			dev_dbg(rdev_to_dev(rdev),
1378 				"Move QP = %p out of flush list\n",
1379 				qp);
1380 			bnxt_qplib_del_flush_qp(&qp->qplib_qp);
1381 		}
1382 	}
1383 	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1384 		qp->qplib_qp.modify_flags |=
1385 				CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1386 		qp->qplib_qp.en_sqd_async_notify = true;
1387 	}
1388 	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1389 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1390 		qp->qplib_qp.access =
1391 			__from_ib_access_flags(qp_attr->qp_access_flags);
1392 		/* LOCAL_WRITE access must be set to allow RC receive */
1393 		qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1394 	}
1395 	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1396 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1397 		qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1398 	}
1399 	if (qp_attr_mask & IB_QP_QKEY) {
1400 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1401 		qp->qplib_qp.qkey = qp_attr->qkey;
1402 	}
1403 	if (qp_attr_mask & IB_QP_AV) {
1404 		const struct ib_global_route *grh =
1405 			rdma_ah_read_grh(&qp_attr->ah_attr);
1406 
1407 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1408 				     CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1409 				     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1410 				     CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1411 				     CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1412 				     CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1413 				     CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1414 		memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
1415 		       sizeof(qp->qplib_qp.ah.dgid.data));
1416 		qp->qplib_qp.ah.flow_label = grh->flow_label;
1417 		/* If RoCE V2 is enabled, stack will have two entries for
1418 		 * each GID entry. Avoiding this duplicte entry in HW. Dividing
1419 		 * the GID index by 2 for RoCE V2
1420 		 */
1421 		qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2;
1422 		qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
1423 		qp->qplib_qp.ah.hop_limit = grh->hop_limit;
1424 		qp->qplib_qp.ah.traffic_class = grh->traffic_class;
1425 		qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
1426 		ether_addr_copy(qp->qplib_qp.ah.dmac,
1427 				qp_attr->ah_attr.roce.dmac);
1428 
1429 		status = ib_get_cached_gid(&rdev->ibdev, 1,
1430 					   grh->sgid_index,
1431 					   &sgid, &sgid_attr);
1432 		if (!status && sgid_attr.ndev) {
1433 			memcpy(qp->qplib_qp.smac, sgid_attr.ndev->dev_addr,
1434 			       ETH_ALEN);
1435 			dev_put(sgid_attr.ndev);
1436 			nw_type = ib_gid_to_network_type(sgid_attr.gid_type,
1437 							 &sgid);
1438 			switch (nw_type) {
1439 			case RDMA_NETWORK_IPV4:
1440 				qp->qplib_qp.nw_type =
1441 					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1442 				break;
1443 			case RDMA_NETWORK_IPV6:
1444 				qp->qplib_qp.nw_type =
1445 					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1446 				break;
1447 			default:
1448 				qp->qplib_qp.nw_type =
1449 					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1450 				break;
1451 			}
1452 		}
1453 	}
1454 
1455 	if (qp_attr_mask & IB_QP_PATH_MTU) {
1456 		qp->qplib_qp.modify_flags |=
1457 				CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1458 		qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1459 		qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu);
1460 	} else if (qp_attr->qp_state == IB_QPS_RTR) {
1461 		qp->qplib_qp.modify_flags |=
1462 			CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1463 		qp->qplib_qp.path_mtu =
1464 			__from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1465 		qp->qplib_qp.mtu =
1466 			ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1467 	}
1468 
1469 	if (qp_attr_mask & IB_QP_TIMEOUT) {
1470 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1471 		qp->qplib_qp.timeout = qp_attr->timeout;
1472 	}
1473 	if (qp_attr_mask & IB_QP_RETRY_CNT) {
1474 		qp->qplib_qp.modify_flags |=
1475 				CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1476 		qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1477 	}
1478 	if (qp_attr_mask & IB_QP_RNR_RETRY) {
1479 		qp->qplib_qp.modify_flags |=
1480 				CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1481 		qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1482 	}
1483 	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1484 		qp->qplib_qp.modify_flags |=
1485 				CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1486 		qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1487 	}
1488 	if (qp_attr_mask & IB_QP_RQ_PSN) {
1489 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1490 		qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1491 	}
1492 	if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1493 		qp->qplib_qp.modify_flags |=
1494 				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1495 		/* Cap the max_rd_atomic to device max */
1496 		qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
1497 						   dev_attr->max_qp_rd_atom);
1498 	}
1499 	if (qp_attr_mask & IB_QP_SQ_PSN) {
1500 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1501 		qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1502 	}
1503 	if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1504 		if (qp_attr->max_dest_rd_atomic >
1505 		    dev_attr->max_qp_init_rd_atom) {
1506 			dev_err(rdev_to_dev(rdev),
1507 				"max_dest_rd_atomic requested%d is > dev_max%d",
1508 				qp_attr->max_dest_rd_atomic,
1509 				dev_attr->max_qp_init_rd_atom);
1510 			return -EINVAL;
1511 		}
1512 
1513 		qp->qplib_qp.modify_flags |=
1514 				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1515 		qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1516 	}
1517 	if (qp_attr_mask & IB_QP_CAP) {
1518 		qp->qplib_qp.modify_flags |=
1519 				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1520 				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1521 				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1522 				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1523 				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1524 		if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1525 		    (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1526 		    (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1527 		    (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
1528 		    (qp_attr->cap.max_inline_data >=
1529 						dev_attr->max_inline_data)) {
1530 			dev_err(rdev_to_dev(rdev),
1531 				"Create QP failed - max exceeded");
1532 			return -EINVAL;
1533 		}
1534 		entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
1535 		qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
1536 						dev_attr->max_qp_wqes + 1);
1537 		qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
1538 						qp_attr->cap.max_send_wr;
1539 		/*
1540 		 * Reserving one slot for Phantom WQE. Some application can
1541 		 * post one extra entry in this case. Allowing this to avoid
1542 		 * unexpected Queue full condition
1543 		 */
1544 		qp->qplib_qp.sq.q_full_delta -= 1;
1545 		qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
1546 		if (qp->qplib_qp.rq.max_wqe) {
1547 			entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
1548 			qp->qplib_qp.rq.max_wqe =
1549 				min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1550 			qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
1551 						       qp_attr->cap.max_recv_wr;
1552 			qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
1553 		} else {
1554 			/* SRQ was used prior, just ignore the RQ caps */
1555 		}
1556 	}
1557 	if (qp_attr_mask & IB_QP_DEST_QPN) {
1558 		qp->qplib_qp.modify_flags |=
1559 				CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
1560 		qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
1561 	}
1562 	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1563 	if (rc) {
1564 		dev_err(rdev_to_dev(rdev), "Failed to modify HW QP");
1565 		return rc;
1566 	}
1567 	if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp)
1568 		rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
1569 	return rc;
1570 }
1571 
bnxt_re_query_qp(struct ib_qp * ib_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1572 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1573 		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1574 {
1575 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1576 	struct bnxt_re_dev *rdev = qp->rdev;
1577 	struct bnxt_qplib_qp *qplib_qp;
1578 	int rc;
1579 
1580 	qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
1581 	if (!qplib_qp)
1582 		return -ENOMEM;
1583 
1584 	qplib_qp->id = qp->qplib_qp.id;
1585 	qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
1586 
1587 	rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
1588 	if (rc) {
1589 		dev_err(rdev_to_dev(rdev), "Failed to query HW QP");
1590 		goto out;
1591 	}
1592 	qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
1593 	qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
1594 	qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
1595 	qp_attr->pkey_index = qplib_qp->pkey_index;
1596 	qp_attr->qkey = qplib_qp->qkey;
1597 	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
1598 	rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label,
1599 			qplib_qp->ah.host_sgid_index,
1600 			qplib_qp->ah.hop_limit,
1601 			qplib_qp->ah.traffic_class);
1602 	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
1603 	rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
1604 	ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
1605 	qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
1606 	qp_attr->timeout = qplib_qp->timeout;
1607 	qp_attr->retry_cnt = qplib_qp->retry_cnt;
1608 	qp_attr->rnr_retry = qplib_qp->rnr_retry;
1609 	qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
1610 	qp_attr->rq_psn = qplib_qp->rq.psn;
1611 	qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
1612 	qp_attr->sq_psn = qplib_qp->sq.psn;
1613 	qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
1614 	qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
1615 							 IB_SIGNAL_REQ_WR;
1616 	qp_attr->dest_qp_num = qplib_qp->dest_qpn;
1617 
1618 	qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
1619 	qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
1620 	qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
1621 	qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
1622 	qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
1623 	qp_init_attr->cap = qp_attr->cap;
1624 
1625 out:
1626 	kfree(qplib_qp);
1627 	return rc;
1628 }
1629 
1630 /* Routine for sending QP1 packets for RoCE V1 an V2
1631  */
bnxt_re_build_qp1_send_v2(struct bnxt_re_qp * qp,struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe,int payload_size)1632 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
1633 				     struct ib_send_wr *wr,
1634 				     struct bnxt_qplib_swqe *wqe,
1635 				     int payload_size)
1636 {
1637 	struct ib_device *ibdev = &qp->rdev->ibdev;
1638 	struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
1639 					     ib_ah);
1640 	struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
1641 	struct bnxt_qplib_sge sge;
1642 	union ib_gid sgid;
1643 	u8 nw_type;
1644 	u16 ether_type;
1645 	struct ib_gid_attr sgid_attr;
1646 	union ib_gid dgid;
1647 	bool is_eth = false;
1648 	bool is_vlan = false;
1649 	bool is_grh = false;
1650 	bool is_udp = false;
1651 	u8 ip_version = 0;
1652 	u16 vlan_id = 0xFFFF;
1653 	void *buf;
1654 	int i, rc = 0, size;
1655 
1656 	memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
1657 
1658 	rc = ib_get_cached_gid(ibdev, 1,
1659 			       qplib_ah->host_sgid_index, &sgid,
1660 			       &sgid_attr);
1661 	if (rc) {
1662 		dev_err(rdev_to_dev(qp->rdev),
1663 			"Failed to query gid at index %d",
1664 			qplib_ah->host_sgid_index);
1665 		return rc;
1666 	}
1667 	if (sgid_attr.ndev) {
1668 		if (is_vlan_dev(sgid_attr.ndev))
1669 			vlan_id = vlan_dev_vlan_id(sgid_attr.ndev);
1670 		dev_put(sgid_attr.ndev);
1671 	}
1672 	/* Get network header type for this GID */
1673 	nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
1674 	switch (nw_type) {
1675 	case RDMA_NETWORK_IPV4:
1676 		nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
1677 		break;
1678 	case RDMA_NETWORK_IPV6:
1679 		nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
1680 		break;
1681 	default:
1682 		nw_type = BNXT_RE_ROCE_V1_PACKET;
1683 		break;
1684 	}
1685 	memcpy(&dgid.raw, &qplib_ah->dgid, 16);
1686 	is_udp = sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
1687 	if (is_udp) {
1688 		if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) {
1689 			ip_version = 4;
1690 			ether_type = ETH_P_IP;
1691 		} else {
1692 			ip_version = 6;
1693 			ether_type = ETH_P_IPV6;
1694 		}
1695 		is_grh = false;
1696 	} else {
1697 		ether_type = ETH_P_IBOE;
1698 		is_grh = true;
1699 	}
1700 
1701 	is_eth = true;
1702 	is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
1703 
1704 	ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
1705 			  ip_version, is_udp, 0, &qp->qp1_hdr);
1706 
1707 	/* ETH */
1708 	ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
1709 	ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
1710 
1711 	/* For vlan, check the sgid for vlan existence */
1712 
1713 	if (!is_vlan) {
1714 		qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
1715 	} else {
1716 		qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
1717 		qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
1718 	}
1719 
1720 	if (is_grh || (ip_version == 6)) {
1721 		memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid));
1722 		memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
1723 		       sizeof(sgid));
1724 		qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
1725 	}
1726 
1727 	if (ip_version == 4) {
1728 		qp->qp1_hdr.ip4.tos = 0;
1729 		qp->qp1_hdr.ip4.id = 0;
1730 		qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
1731 		qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
1732 
1733 		memcpy(&qp->qp1_hdr.ip4.saddr, sgid.raw + 12, 4);
1734 		memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
1735 		qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
1736 	}
1737 
1738 	if (is_udp) {
1739 		qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
1740 		qp->qp1_hdr.udp.sport = htons(0x8CD1);
1741 		qp->qp1_hdr.udp.csum = 0;
1742 	}
1743 
1744 	/* BTH */
1745 	if (wr->opcode == IB_WR_SEND_WITH_IMM) {
1746 		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1747 		qp->qp1_hdr.immediate_present = 1;
1748 	} else {
1749 		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
1750 	}
1751 	if (wr->send_flags & IB_SEND_SOLICITED)
1752 		qp->qp1_hdr.bth.solicited_event = 1;
1753 	/* pad_count */
1754 	qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
1755 
1756 	/* P_key for QP1 is for all members */
1757 	qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
1758 	qp->qp1_hdr.bth.destination_qpn = IB_QP1;
1759 	qp->qp1_hdr.bth.ack_req = 0;
1760 	qp->send_psn++;
1761 	qp->send_psn &= BTH_PSN_MASK;
1762 	qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
1763 	/* DETH */
1764 	/* Use the priviledged Q_Key for QP1 */
1765 	qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
1766 	qp->qp1_hdr.deth.source_qpn = IB_QP1;
1767 
1768 	/* Pack the QP1 to the transmit buffer */
1769 	buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
1770 	if (buf) {
1771 		size = ib_ud_header_pack(&qp->qp1_hdr, buf);
1772 		for (i = wqe->num_sge; i; i--) {
1773 			wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
1774 			wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
1775 			wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
1776 		}
1777 
1778 		/*
1779 		 * Max Header buf size for IPV6 RoCE V2 is 86,
1780 		 * which is same as the QP1 SQ header buffer.
1781 		 * Header buf size for IPV4 RoCE V2 can be 66.
1782 		 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
1783 		 * Subtract 20 bytes from QP1 SQ header buf size
1784 		 */
1785 		if (is_udp && ip_version == 4)
1786 			sge.size -= 20;
1787 		/*
1788 		 * Max Header buf size for RoCE V1 is 78.
1789 		 * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
1790 		 * Subtract 8 bytes from QP1 SQ header buf size
1791 		 */
1792 		if (!is_udp)
1793 			sge.size -= 8;
1794 
1795 		/* Subtract 4 bytes for non vlan packets */
1796 		if (!is_vlan)
1797 			sge.size -= 4;
1798 
1799 		wqe->sg_list[0].addr = sge.addr;
1800 		wqe->sg_list[0].lkey = sge.lkey;
1801 		wqe->sg_list[0].size = sge.size;
1802 		wqe->num_sge++;
1803 
1804 	} else {
1805 		dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!");
1806 		rc = -ENOMEM;
1807 	}
1808 	return rc;
1809 }
1810 
1811 /* For the MAD layer, it only provides the recv SGE the size of
1812  * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
1813  * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
1814  * receive packet (334 bytes) with no VLAN and then copy the GRH
1815  * and the MAD datagram out to the provided SGE.
1816  */
bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp * qp,struct ib_recv_wr * wr,struct bnxt_qplib_swqe * wqe,int payload_size)1817 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
1818 					    struct ib_recv_wr *wr,
1819 					    struct bnxt_qplib_swqe *wqe,
1820 					    int payload_size)
1821 {
1822 	struct bnxt_qplib_sge ref, sge;
1823 	u32 rq_prod_index;
1824 	struct bnxt_re_sqp_entries *sqp_entry;
1825 
1826 	rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
1827 
1828 	if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
1829 		return -ENOMEM;
1830 
1831 	/* Create 1 SGE to receive the entire
1832 	 * ethernet packet
1833 	 */
1834 	/* Save the reference from ULP */
1835 	ref.addr = wqe->sg_list[0].addr;
1836 	ref.lkey = wqe->sg_list[0].lkey;
1837 	ref.size = wqe->sg_list[0].size;
1838 
1839 	sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index];
1840 
1841 	/* SGE 1 */
1842 	wqe->sg_list[0].addr = sge.addr;
1843 	wqe->sg_list[0].lkey = sge.lkey;
1844 	wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1845 	sge.size -= wqe->sg_list[0].size;
1846 
1847 	sqp_entry->sge.addr = ref.addr;
1848 	sqp_entry->sge.lkey = ref.lkey;
1849 	sqp_entry->sge.size = ref.size;
1850 	/* Store the wrid for reporting completion */
1851 	sqp_entry->wrid = wqe->wr_id;
1852 	/* change the wqe->wrid to table index */
1853 	wqe->wr_id = rq_prod_index;
1854 	return 0;
1855 }
1856 
is_ud_qp(struct bnxt_re_qp * qp)1857 static int is_ud_qp(struct bnxt_re_qp *qp)
1858 {
1859 	return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD;
1860 }
1861 
bnxt_re_build_send_wqe(struct bnxt_re_qp * qp,struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)1862 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
1863 				  struct ib_send_wr *wr,
1864 				  struct bnxt_qplib_swqe *wqe)
1865 {
1866 	struct bnxt_re_ah *ah = NULL;
1867 
1868 	if (is_ud_qp(qp)) {
1869 		ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
1870 		wqe->send.q_key = ud_wr(wr)->remote_qkey;
1871 		wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
1872 		wqe->send.avid = ah->qplib_ah.id;
1873 	}
1874 	switch (wr->opcode) {
1875 	case IB_WR_SEND:
1876 		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
1877 		break;
1878 	case IB_WR_SEND_WITH_IMM:
1879 		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
1880 		wqe->send.imm_data = wr->ex.imm_data;
1881 		break;
1882 	case IB_WR_SEND_WITH_INV:
1883 		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
1884 		wqe->send.inv_key = wr->ex.invalidate_rkey;
1885 		break;
1886 	default:
1887 		return -EINVAL;
1888 	}
1889 	if (wr->send_flags & IB_SEND_SIGNALED)
1890 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1891 	if (wr->send_flags & IB_SEND_FENCE)
1892 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1893 	if (wr->send_flags & IB_SEND_SOLICITED)
1894 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1895 	if (wr->send_flags & IB_SEND_INLINE)
1896 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1897 
1898 	return 0;
1899 }
1900 
bnxt_re_build_rdma_wqe(struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)1901 static int bnxt_re_build_rdma_wqe(struct ib_send_wr *wr,
1902 				  struct bnxt_qplib_swqe *wqe)
1903 {
1904 	switch (wr->opcode) {
1905 	case IB_WR_RDMA_WRITE:
1906 		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
1907 		break;
1908 	case IB_WR_RDMA_WRITE_WITH_IMM:
1909 		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
1910 		wqe->rdma.imm_data = wr->ex.imm_data;
1911 		break;
1912 	case IB_WR_RDMA_READ:
1913 		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
1914 		wqe->rdma.inv_key = wr->ex.invalidate_rkey;
1915 		break;
1916 	default:
1917 		return -EINVAL;
1918 	}
1919 	wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
1920 	wqe->rdma.r_key = rdma_wr(wr)->rkey;
1921 	if (wr->send_flags & IB_SEND_SIGNALED)
1922 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1923 	if (wr->send_flags & IB_SEND_FENCE)
1924 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1925 	if (wr->send_flags & IB_SEND_SOLICITED)
1926 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1927 	if (wr->send_flags & IB_SEND_INLINE)
1928 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1929 
1930 	return 0;
1931 }
1932 
bnxt_re_build_atomic_wqe(struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)1933 static int bnxt_re_build_atomic_wqe(struct ib_send_wr *wr,
1934 				    struct bnxt_qplib_swqe *wqe)
1935 {
1936 	switch (wr->opcode) {
1937 	case IB_WR_ATOMIC_CMP_AND_SWP:
1938 		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
1939 		wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
1940 		wqe->atomic.swap_data = atomic_wr(wr)->swap;
1941 		break;
1942 	case IB_WR_ATOMIC_FETCH_AND_ADD:
1943 		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
1944 		wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
1945 		break;
1946 	default:
1947 		return -EINVAL;
1948 	}
1949 	wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
1950 	wqe->atomic.r_key = atomic_wr(wr)->rkey;
1951 	if (wr->send_flags & IB_SEND_SIGNALED)
1952 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1953 	if (wr->send_flags & IB_SEND_FENCE)
1954 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1955 	if (wr->send_flags & IB_SEND_SOLICITED)
1956 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1957 	return 0;
1958 }
1959 
bnxt_re_build_inv_wqe(struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)1960 static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr,
1961 				 struct bnxt_qplib_swqe *wqe)
1962 {
1963 	wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
1964 	wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
1965 
1966 	/* Need unconditional fence for local invalidate
1967 	 * opcode to work as expected.
1968 	 */
1969 	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1970 
1971 	if (wr->send_flags & IB_SEND_SIGNALED)
1972 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1973 	if (wr->send_flags & IB_SEND_SOLICITED)
1974 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1975 
1976 	return 0;
1977 }
1978 
bnxt_re_build_reg_wqe(struct ib_reg_wr * wr,struct bnxt_qplib_swqe * wqe)1979 static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr,
1980 				 struct bnxt_qplib_swqe *wqe)
1981 {
1982 	struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
1983 	struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
1984 	int access = wr->access;
1985 
1986 	wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
1987 	wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
1988 	wqe->frmr.page_list = mr->pages;
1989 	wqe->frmr.page_list_len = mr->npages;
1990 	wqe->frmr.levels = qplib_frpl->hwq.level + 1;
1991 	wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
1992 
1993 	/* Need unconditional fence for reg_mr
1994 	 * opcode to function as expected.
1995 	 */
1996 
1997 	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1998 
1999 	if (wr->wr.send_flags & IB_SEND_SIGNALED)
2000 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2001 
2002 	if (access & IB_ACCESS_LOCAL_WRITE)
2003 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2004 	if (access & IB_ACCESS_REMOTE_READ)
2005 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2006 	if (access & IB_ACCESS_REMOTE_WRITE)
2007 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2008 	if (access & IB_ACCESS_REMOTE_ATOMIC)
2009 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2010 	if (access & IB_ACCESS_MW_BIND)
2011 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2012 
2013 	wqe->frmr.l_key = wr->key;
2014 	wqe->frmr.length = wr->mr->length;
2015 	wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
2016 	wqe->frmr.va = wr->mr->iova;
2017 	return 0;
2018 }
2019 
bnxt_re_copy_inline_data(struct bnxt_re_dev * rdev,struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2020 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2021 				    struct ib_send_wr *wr,
2022 				    struct bnxt_qplib_swqe *wqe)
2023 {
2024 	/*  Copy the inline data to the data  field */
2025 	u8 *in_data;
2026 	u32 i, sge_len;
2027 	void *sge_addr;
2028 
2029 	in_data = wqe->inline_data;
2030 	for (i = 0; i < wr->num_sge; i++) {
2031 		sge_addr = (void *)(unsigned long)
2032 				wr->sg_list[i].addr;
2033 		sge_len = wr->sg_list[i].length;
2034 
2035 		if ((sge_len + wqe->inline_len) >
2036 		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2037 			dev_err(rdev_to_dev(rdev),
2038 				"Inline data size requested > supported value");
2039 			return -EINVAL;
2040 		}
2041 		sge_len = wr->sg_list[i].length;
2042 
2043 		memcpy(in_data, sge_addr, sge_len);
2044 		in_data += wr->sg_list[i].length;
2045 		wqe->inline_len += wr->sg_list[i].length;
2046 	}
2047 	return wqe->inline_len;
2048 }
2049 
bnxt_re_copy_wr_payload(struct bnxt_re_dev * rdev,struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2050 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2051 				   struct ib_send_wr *wr,
2052 				   struct bnxt_qplib_swqe *wqe)
2053 {
2054 	int payload_sz = 0;
2055 
2056 	if (wr->send_flags & IB_SEND_INLINE)
2057 		payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2058 	else
2059 		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2060 					       wqe->num_sge);
2061 
2062 	return payload_sz;
2063 }
2064 
bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp * qp)2065 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2066 {
2067 	if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2068 	     qp->ib_qp.qp_type == IB_QPT_GSI ||
2069 	     qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2070 	     qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2071 		int qp_attr_mask;
2072 		struct ib_qp_attr qp_attr;
2073 
2074 		qp_attr_mask = IB_QP_STATE;
2075 		qp_attr.qp_state = IB_QPS_RTS;
2076 		bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2077 		qp->qplib_qp.wqe_cnt = 0;
2078 	}
2079 }
2080 
bnxt_re_post_send_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp,struct ib_send_wr * wr)2081 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2082 				       struct bnxt_re_qp *qp,
2083 				struct ib_send_wr *wr)
2084 {
2085 	struct bnxt_qplib_swqe wqe;
2086 	int rc = 0, payload_sz = 0;
2087 	unsigned long flags;
2088 
2089 	spin_lock_irqsave(&qp->sq_lock, flags);
2090 	memset(&wqe, 0, sizeof(wqe));
2091 	while (wr) {
2092 		/* House keeping */
2093 		memset(&wqe, 0, sizeof(wqe));
2094 
2095 		/* Common */
2096 		wqe.num_sge = wr->num_sge;
2097 		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2098 			dev_err(rdev_to_dev(rdev),
2099 				"Limit exceeded for Send SGEs");
2100 			rc = -EINVAL;
2101 			goto bad;
2102 		}
2103 
2104 		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2105 		if (payload_sz < 0) {
2106 			rc = -EINVAL;
2107 			goto bad;
2108 		}
2109 		wqe.wr_id = wr->wr_id;
2110 
2111 		wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2112 
2113 		rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2114 		if (!rc)
2115 			rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2116 bad:
2117 		if (rc) {
2118 			dev_err(rdev_to_dev(rdev),
2119 				"Post send failed opcode = %#x rc = %d",
2120 				wr->opcode, rc);
2121 			break;
2122 		}
2123 		wr = wr->next;
2124 	}
2125 	bnxt_qplib_post_send_db(&qp->qplib_qp);
2126 	bnxt_ud_qp_hw_stall_workaround(qp);
2127 	spin_unlock_irqrestore(&qp->sq_lock, flags);
2128 	return rc;
2129 }
2130 
bnxt_re_post_send(struct ib_qp * ib_qp,struct ib_send_wr * wr,struct ib_send_wr ** bad_wr)2131 int bnxt_re_post_send(struct ib_qp *ib_qp, struct ib_send_wr *wr,
2132 		      struct ib_send_wr **bad_wr)
2133 {
2134 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2135 	struct bnxt_qplib_swqe wqe;
2136 	int rc = 0, payload_sz = 0;
2137 	unsigned long flags;
2138 
2139 	spin_lock_irqsave(&qp->sq_lock, flags);
2140 	while (wr) {
2141 		/* House keeping */
2142 		memset(&wqe, 0, sizeof(wqe));
2143 
2144 		/* Common */
2145 		wqe.num_sge = wr->num_sge;
2146 		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2147 			dev_err(rdev_to_dev(qp->rdev),
2148 				"Limit exceeded for Send SGEs");
2149 			rc = -EINVAL;
2150 			goto bad;
2151 		}
2152 
2153 		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2154 		if (payload_sz < 0) {
2155 			rc = -EINVAL;
2156 			goto bad;
2157 		}
2158 		wqe.wr_id = wr->wr_id;
2159 
2160 		switch (wr->opcode) {
2161 		case IB_WR_SEND:
2162 		case IB_WR_SEND_WITH_IMM:
2163 			if (ib_qp->qp_type == IB_QPT_GSI) {
2164 				rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2165 							       payload_sz);
2166 				if (rc)
2167 					goto bad;
2168 				wqe.rawqp1.lflags |=
2169 					SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2170 			}
2171 			switch (wr->send_flags) {
2172 			case IB_SEND_IP_CSUM:
2173 				wqe.rawqp1.lflags |=
2174 					SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2175 				break;
2176 			default:
2177 				break;
2178 			}
2179 			/* Fall thru to build the wqe */
2180 		case IB_WR_SEND_WITH_INV:
2181 			rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2182 			break;
2183 		case IB_WR_RDMA_WRITE:
2184 		case IB_WR_RDMA_WRITE_WITH_IMM:
2185 		case IB_WR_RDMA_READ:
2186 			rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2187 			break;
2188 		case IB_WR_ATOMIC_CMP_AND_SWP:
2189 		case IB_WR_ATOMIC_FETCH_AND_ADD:
2190 			rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2191 			break;
2192 		case IB_WR_RDMA_READ_WITH_INV:
2193 			dev_err(rdev_to_dev(qp->rdev),
2194 				"RDMA Read with Invalidate is not supported");
2195 			rc = -EINVAL;
2196 			goto bad;
2197 		case IB_WR_LOCAL_INV:
2198 			rc = bnxt_re_build_inv_wqe(wr, &wqe);
2199 			break;
2200 		case IB_WR_REG_MR:
2201 			rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2202 			break;
2203 		default:
2204 			/* Unsupported WRs */
2205 			dev_err(rdev_to_dev(qp->rdev),
2206 				"WR (%#x) is not supported", wr->opcode);
2207 			rc = -EINVAL;
2208 			goto bad;
2209 		}
2210 		if (!rc)
2211 			rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2212 bad:
2213 		if (rc) {
2214 			dev_err(rdev_to_dev(qp->rdev),
2215 				"post_send failed op:%#x qps = %#x rc = %d\n",
2216 				wr->opcode, qp->qplib_qp.state, rc);
2217 			*bad_wr = wr;
2218 			break;
2219 		}
2220 		wr = wr->next;
2221 	}
2222 	bnxt_qplib_post_send_db(&qp->qplib_qp);
2223 	bnxt_ud_qp_hw_stall_workaround(qp);
2224 	spin_unlock_irqrestore(&qp->sq_lock, flags);
2225 
2226 	return rc;
2227 }
2228 
bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp,struct ib_recv_wr * wr)2229 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2230 				       struct bnxt_re_qp *qp,
2231 				       struct ib_recv_wr *wr)
2232 {
2233 	struct bnxt_qplib_swqe wqe;
2234 	int rc = 0, payload_sz = 0;
2235 
2236 	memset(&wqe, 0, sizeof(wqe));
2237 	while (wr) {
2238 		/* House keeping */
2239 		memset(&wqe, 0, sizeof(wqe));
2240 
2241 		/* Common */
2242 		wqe.num_sge = wr->num_sge;
2243 		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2244 			dev_err(rdev_to_dev(rdev),
2245 				"Limit exceeded for Receive SGEs");
2246 			rc = -EINVAL;
2247 			break;
2248 		}
2249 		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2250 					       wr->num_sge);
2251 		wqe.wr_id = wr->wr_id;
2252 		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2253 
2254 		rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2255 		if (rc)
2256 			break;
2257 
2258 		wr = wr->next;
2259 	}
2260 	if (!rc)
2261 		bnxt_qplib_post_recv_db(&qp->qplib_qp);
2262 	return rc;
2263 }
2264 
bnxt_re_post_recv(struct ib_qp * ib_qp,struct ib_recv_wr * wr,struct ib_recv_wr ** bad_wr)2265 int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr,
2266 		      struct ib_recv_wr **bad_wr)
2267 {
2268 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2269 	struct bnxt_qplib_swqe wqe;
2270 	int rc = 0, payload_sz = 0;
2271 	unsigned long flags;
2272 	u32 count = 0;
2273 
2274 	spin_lock_irqsave(&qp->rq_lock, flags);
2275 	while (wr) {
2276 		/* House keeping */
2277 		memset(&wqe, 0, sizeof(wqe));
2278 
2279 		/* Common */
2280 		wqe.num_sge = wr->num_sge;
2281 		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2282 			dev_err(rdev_to_dev(qp->rdev),
2283 				"Limit exceeded for Receive SGEs");
2284 			rc = -EINVAL;
2285 			*bad_wr = wr;
2286 			break;
2287 		}
2288 
2289 		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2290 					       wr->num_sge);
2291 		wqe.wr_id = wr->wr_id;
2292 		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2293 
2294 		if (ib_qp->qp_type == IB_QPT_GSI)
2295 			rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2296 							      payload_sz);
2297 		if (!rc)
2298 			rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2299 		if (rc) {
2300 			*bad_wr = wr;
2301 			break;
2302 		}
2303 
2304 		/* Ring DB if the RQEs posted reaches a threshold value */
2305 		if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
2306 			bnxt_qplib_post_recv_db(&qp->qplib_qp);
2307 			count = 0;
2308 		}
2309 
2310 		wr = wr->next;
2311 	}
2312 
2313 	if (count)
2314 		bnxt_qplib_post_recv_db(&qp->qplib_qp);
2315 
2316 	spin_unlock_irqrestore(&qp->rq_lock, flags);
2317 
2318 	return rc;
2319 }
2320 
2321 /* Completion Queues */
bnxt_re_destroy_cq(struct ib_cq * ib_cq)2322 int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
2323 {
2324 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2325 	struct bnxt_re_dev *rdev = cq->rdev;
2326 	int rc;
2327 	struct bnxt_qplib_nq *nq = cq->qplib_cq.nq;
2328 
2329 	rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2330 	if (rc) {
2331 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ");
2332 		return rc;
2333 	}
2334 	if (!IS_ERR_OR_NULL(cq->umem))
2335 		ib_umem_release(cq->umem);
2336 
2337 	if (cq) {
2338 		kfree(cq->cql);
2339 		kfree(cq);
2340 	}
2341 	atomic_dec(&rdev->cq_count);
2342 	nq->budget--;
2343 	return 0;
2344 }
2345 
bnxt_re_create_cq(struct ib_device * ibdev,const struct ib_cq_init_attr * attr,struct ib_ucontext * context,struct ib_udata * udata)2346 struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
2347 				const struct ib_cq_init_attr *attr,
2348 				struct ib_ucontext *context,
2349 				struct ib_udata *udata)
2350 {
2351 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
2352 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2353 	struct bnxt_re_cq *cq = NULL;
2354 	int rc, entries;
2355 	int cqe = attr->cqe;
2356 	struct bnxt_qplib_nq *nq = NULL;
2357 	unsigned int nq_alloc_cnt;
2358 
2359 	/* Validate CQ fields */
2360 	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2361 		dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded");
2362 		return ERR_PTR(-EINVAL);
2363 	}
2364 	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
2365 	if (!cq)
2366 		return ERR_PTR(-ENOMEM);
2367 
2368 	cq->rdev = rdev;
2369 	cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2370 
2371 	entries = roundup_pow_of_two(cqe + 1);
2372 	if (entries > dev_attr->max_cq_wqes + 1)
2373 		entries = dev_attr->max_cq_wqes + 1;
2374 
2375 	if (context) {
2376 		struct bnxt_re_cq_req req;
2377 		struct bnxt_re_ucontext *uctx = container_of
2378 						(context,
2379 						 struct bnxt_re_ucontext,
2380 						 ib_uctx);
2381 		if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2382 			rc = -EFAULT;
2383 			goto fail;
2384 		}
2385 
2386 		cq->umem = ib_umem_get(context, req.cq_va,
2387 				       entries * sizeof(struct cq_base),
2388 				       IB_ACCESS_LOCAL_WRITE, 1);
2389 		if (IS_ERR(cq->umem)) {
2390 			rc = PTR_ERR(cq->umem);
2391 			goto fail;
2392 		}
2393 		cq->qplib_cq.sghead = cq->umem->sg_head.sgl;
2394 		cq->qplib_cq.nmap = cq->umem->nmap;
2395 		cq->qplib_cq.dpi = &uctx->dpi;
2396 	} else {
2397 		cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2398 		cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2399 				  GFP_KERNEL);
2400 		if (!cq->cql) {
2401 			rc = -ENOMEM;
2402 			goto fail;
2403 		}
2404 
2405 		cq->qplib_cq.dpi = &rdev->dpi_privileged;
2406 		cq->qplib_cq.sghead = NULL;
2407 		cq->qplib_cq.nmap = 0;
2408 	}
2409 	/*
2410 	 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
2411 	 * used for getting the NQ index.
2412 	 */
2413 	nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
2414 	nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
2415 	cq->qplib_cq.max_wqe = entries;
2416 	cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
2417 	cq->qplib_cq.nq	= nq;
2418 
2419 	rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2420 	if (rc) {
2421 		dev_err(rdev_to_dev(rdev), "Failed to create HW CQ");
2422 		goto fail;
2423 	}
2424 
2425 	cq->ib_cq.cqe = entries;
2426 	cq->cq_period = cq->qplib_cq.period;
2427 	nq->budget++;
2428 
2429 	atomic_inc(&rdev->cq_count);
2430 
2431 	if (context) {
2432 		struct bnxt_re_cq_resp resp;
2433 
2434 		resp.cqid = cq->qplib_cq.id;
2435 		resp.tail = cq->qplib_cq.hwq.cons;
2436 		resp.phase = cq->qplib_cq.period;
2437 		resp.rsvd = 0;
2438 		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2439 		if (rc) {
2440 			dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata");
2441 			bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2442 			goto c2fail;
2443 		}
2444 	}
2445 
2446 	return &cq->ib_cq;
2447 
2448 c2fail:
2449 	if (context)
2450 		ib_umem_release(cq->umem);
2451 fail:
2452 	kfree(cq->cql);
2453 	kfree(cq);
2454 	return ERR_PTR(rc);
2455 }
2456 
__req_to_ib_wc_status(u8 qstatus)2457 static u8 __req_to_ib_wc_status(u8 qstatus)
2458 {
2459 	switch (qstatus) {
2460 	case CQ_REQ_STATUS_OK:
2461 		return IB_WC_SUCCESS;
2462 	case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2463 		return IB_WC_BAD_RESP_ERR;
2464 	case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2465 		return IB_WC_LOC_LEN_ERR;
2466 	case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2467 		return IB_WC_LOC_QP_OP_ERR;
2468 	case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2469 		return IB_WC_LOC_PROT_ERR;
2470 	case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2471 		return IB_WC_GENERAL_ERR;
2472 	case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2473 		return IB_WC_REM_INV_REQ_ERR;
2474 	case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2475 		return IB_WC_REM_ACCESS_ERR;
2476 	case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2477 		return IB_WC_REM_OP_ERR;
2478 	case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2479 		return IB_WC_RNR_RETRY_EXC_ERR;
2480 	case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2481 		return IB_WC_RETRY_EXC_ERR;
2482 	case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2483 		return IB_WC_WR_FLUSH_ERR;
2484 	default:
2485 		return IB_WC_GENERAL_ERR;
2486 	}
2487 	return 0;
2488 }
2489 
__rawqp1_to_ib_wc_status(u8 qstatus)2490 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2491 {
2492 	switch (qstatus) {
2493 	case CQ_RES_RAWETH_QP1_STATUS_OK:
2494 		return IB_WC_SUCCESS;
2495 	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2496 		return IB_WC_LOC_ACCESS_ERR;
2497 	case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2498 		return IB_WC_LOC_LEN_ERR;
2499 	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2500 		return IB_WC_LOC_PROT_ERR;
2501 	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2502 		return IB_WC_LOC_QP_OP_ERR;
2503 	case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2504 		return IB_WC_GENERAL_ERR;
2505 	case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2506 		return IB_WC_WR_FLUSH_ERR;
2507 	case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2508 		return IB_WC_WR_FLUSH_ERR;
2509 	default:
2510 		return IB_WC_GENERAL_ERR;
2511 	}
2512 }
2513 
__rc_to_ib_wc_status(u8 qstatus)2514 static u8 __rc_to_ib_wc_status(u8 qstatus)
2515 {
2516 	switch (qstatus) {
2517 	case CQ_RES_RC_STATUS_OK:
2518 		return IB_WC_SUCCESS;
2519 	case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2520 		return IB_WC_LOC_ACCESS_ERR;
2521 	case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2522 		return IB_WC_LOC_LEN_ERR;
2523 	case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2524 		return IB_WC_LOC_PROT_ERR;
2525 	case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2526 		return IB_WC_LOC_QP_OP_ERR;
2527 	case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2528 		return IB_WC_GENERAL_ERR;
2529 	case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2530 		return IB_WC_REM_INV_REQ_ERR;
2531 	case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2532 		return IB_WC_WR_FLUSH_ERR;
2533 	case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2534 		return IB_WC_WR_FLUSH_ERR;
2535 	default:
2536 		return IB_WC_GENERAL_ERR;
2537 	}
2538 }
2539 
bnxt_re_process_req_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)2540 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2541 {
2542 	switch (cqe->type) {
2543 	case BNXT_QPLIB_SWQE_TYPE_SEND:
2544 		wc->opcode = IB_WC_SEND;
2545 		break;
2546 	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2547 		wc->opcode = IB_WC_SEND;
2548 		wc->wc_flags |= IB_WC_WITH_IMM;
2549 		break;
2550 	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
2551 		wc->opcode = IB_WC_SEND;
2552 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2553 		break;
2554 	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
2555 		wc->opcode = IB_WC_RDMA_WRITE;
2556 		break;
2557 	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
2558 		wc->opcode = IB_WC_RDMA_WRITE;
2559 		wc->wc_flags |= IB_WC_WITH_IMM;
2560 		break;
2561 	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
2562 		wc->opcode = IB_WC_RDMA_READ;
2563 		break;
2564 	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
2565 		wc->opcode = IB_WC_COMP_SWAP;
2566 		break;
2567 	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
2568 		wc->opcode = IB_WC_FETCH_ADD;
2569 		break;
2570 	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
2571 		wc->opcode = IB_WC_LOCAL_INV;
2572 		break;
2573 	case BNXT_QPLIB_SWQE_TYPE_REG_MR:
2574 		wc->opcode = IB_WC_REG_MR;
2575 		break;
2576 	default:
2577 		wc->opcode = IB_WC_SEND;
2578 		break;
2579 	}
2580 
2581 	wc->status = __req_to_ib_wc_status(cqe->status);
2582 }
2583 
bnxt_re_check_packet_type(u16 raweth_qp1_flags,u16 raweth_qp1_flags2)2584 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
2585 				     u16 raweth_qp1_flags2)
2586 {
2587 	bool is_udp = false, is_ipv6 = false, is_ipv4 = false;
2588 
2589 	/* raweth_qp1_flags Bit 9-6 indicates itype */
2590 	if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2591 	    != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2592 		return -1;
2593 
2594 	if (raweth_qp1_flags2 &
2595 	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
2596 	    raweth_qp1_flags2 &
2597 	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
2598 		is_udp = true;
2599 		/* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
2600 		(raweth_qp1_flags2 &
2601 		 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
2602 			(is_ipv6 = true) : (is_ipv4 = true);
2603 		return ((is_ipv6) ?
2604 			 BNXT_RE_ROCEV2_IPV6_PACKET :
2605 			 BNXT_RE_ROCEV2_IPV4_PACKET);
2606 	} else {
2607 		return BNXT_RE_ROCE_V1_PACKET;
2608 	}
2609 }
2610 
bnxt_re_to_ib_nw_type(int nw_type)2611 static int bnxt_re_to_ib_nw_type(int nw_type)
2612 {
2613 	u8 nw_hdr_type = 0xFF;
2614 
2615 	switch (nw_type) {
2616 	case BNXT_RE_ROCE_V1_PACKET:
2617 		nw_hdr_type = RDMA_NETWORK_ROCE_V1;
2618 		break;
2619 	case BNXT_RE_ROCEV2_IPV4_PACKET:
2620 		nw_hdr_type = RDMA_NETWORK_IPV4;
2621 		break;
2622 	case BNXT_RE_ROCEV2_IPV6_PACKET:
2623 		nw_hdr_type = RDMA_NETWORK_IPV6;
2624 		break;
2625 	}
2626 	return nw_hdr_type;
2627 }
2628 
bnxt_re_is_loopback_packet(struct bnxt_re_dev * rdev,void * rq_hdr_buf)2629 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
2630 				       void *rq_hdr_buf)
2631 {
2632 	u8 *tmp_buf = NULL;
2633 	struct ethhdr *eth_hdr;
2634 	u16 eth_type;
2635 	bool rc = false;
2636 
2637 	tmp_buf = (u8 *)rq_hdr_buf;
2638 	/*
2639 	 * If dest mac is not same as I/F mac, this could be a
2640 	 * loopback address or multicast address, check whether
2641 	 * it is a loopback packet
2642 	 */
2643 	if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
2644 		tmp_buf += 4;
2645 		/* Check the  ether type */
2646 		eth_hdr = (struct ethhdr *)tmp_buf;
2647 		eth_type = ntohs(eth_hdr->h_proto);
2648 		switch (eth_type) {
2649 		case ETH_P_IBOE:
2650 			rc = true;
2651 			break;
2652 		case ETH_P_IP:
2653 		case ETH_P_IPV6: {
2654 			u32 len;
2655 			struct udphdr *udp_hdr;
2656 
2657 			len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
2658 						      sizeof(struct ipv6hdr));
2659 			tmp_buf += sizeof(struct ethhdr) + len;
2660 			udp_hdr = (struct udphdr *)tmp_buf;
2661 			if (ntohs(udp_hdr->dest) ==
2662 				    ROCE_V2_UDP_DPORT)
2663 				rc = true;
2664 			break;
2665 			}
2666 		default:
2667 			break;
2668 		}
2669 	}
2670 
2671 	return rc;
2672 }
2673 
bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp * qp1_qp,struct bnxt_qplib_cqe * cqe)2674 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp,
2675 					 struct bnxt_qplib_cqe *cqe)
2676 {
2677 	struct bnxt_re_dev *rdev = qp1_qp->rdev;
2678 	struct bnxt_re_sqp_entries *sqp_entry = NULL;
2679 	struct bnxt_re_qp *qp = rdev->qp1_sqp;
2680 	struct ib_send_wr *swr;
2681 	struct ib_ud_wr udwr;
2682 	struct ib_recv_wr rwr;
2683 	int pkt_type = 0;
2684 	u32 tbl_idx;
2685 	void *rq_hdr_buf;
2686 	dma_addr_t rq_hdr_buf_map;
2687 	dma_addr_t shrq_hdr_buf_map;
2688 	u32 offset = 0;
2689 	u32 skip_bytes = 0;
2690 	struct ib_sge s_sge[2];
2691 	struct ib_sge r_sge[2];
2692 	int rc;
2693 
2694 	memset(&udwr, 0, sizeof(udwr));
2695 	memset(&rwr, 0, sizeof(rwr));
2696 	memset(&s_sge, 0, sizeof(s_sge));
2697 	memset(&r_sge, 0, sizeof(r_sge));
2698 
2699 	swr = &udwr.wr;
2700 	tbl_idx = cqe->wr_id;
2701 
2702 	rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf +
2703 			(tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size);
2704 	rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp,
2705 							  tbl_idx);
2706 
2707 	/* Shadow QP header buffer */
2708 	shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp,
2709 							    tbl_idx);
2710 	sqp_entry = &rdev->sqp_tbl[tbl_idx];
2711 
2712 	/* Store this cqe */
2713 	memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
2714 	sqp_entry->qp1_qp = qp1_qp;
2715 
2716 	/* Find packet type from the cqe */
2717 
2718 	pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
2719 					     cqe->raweth_qp1_flags2);
2720 	if (pkt_type < 0) {
2721 		dev_err(rdev_to_dev(rdev), "Invalid packet\n");
2722 		return -EINVAL;
2723 	}
2724 
2725 	/* Adjust the offset for the user buffer and post in the rq */
2726 
2727 	if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
2728 		offset = 20;
2729 
2730 	/*
2731 	 * QP1 loopback packet has 4 bytes of internal header before
2732 	 * ether header. Skip these four bytes.
2733 	 */
2734 	if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
2735 		skip_bytes = 4;
2736 
2737 	/* First send SGE . Skip the ether header*/
2738 	s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
2739 			+ skip_bytes;
2740 	s_sge[0].lkey = 0xFFFFFFFF;
2741 	s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
2742 				BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
2743 
2744 	/* Second Send SGE */
2745 	s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
2746 			BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
2747 	if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
2748 		s_sge[1].addr += 8;
2749 	s_sge[1].lkey = 0xFFFFFFFF;
2750 	s_sge[1].length = 256;
2751 
2752 	/* First recv SGE */
2753 
2754 	r_sge[0].addr = shrq_hdr_buf_map;
2755 	r_sge[0].lkey = 0xFFFFFFFF;
2756 	r_sge[0].length = 40;
2757 
2758 	r_sge[1].addr = sqp_entry->sge.addr + offset;
2759 	r_sge[1].lkey = sqp_entry->sge.lkey;
2760 	r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
2761 
2762 	/* Create receive work request */
2763 	rwr.num_sge = 2;
2764 	rwr.sg_list = r_sge;
2765 	rwr.wr_id = tbl_idx;
2766 	rwr.next = NULL;
2767 
2768 	rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr);
2769 	if (rc) {
2770 		dev_err(rdev_to_dev(rdev),
2771 			"Failed to post Rx buffers to shadow QP");
2772 		return -ENOMEM;
2773 	}
2774 
2775 	swr->num_sge = 2;
2776 	swr->sg_list = s_sge;
2777 	swr->wr_id = tbl_idx;
2778 	swr->opcode = IB_WR_SEND;
2779 	swr->next = NULL;
2780 
2781 	udwr.ah = &rdev->sqp_ah->ib_ah;
2782 	udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id;
2783 	udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey;
2784 
2785 	/* post data received  in the send queue */
2786 	rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr);
2787 
2788 	return 0;
2789 }
2790 
bnxt_re_process_res_rawqp1_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)2791 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
2792 					  struct bnxt_qplib_cqe *cqe)
2793 {
2794 	wc->opcode = IB_WC_RECV;
2795 	wc->status = __rawqp1_to_ib_wc_status(cqe->status);
2796 	wc->wc_flags |= IB_WC_GRH;
2797 }
2798 
bnxt_re_process_res_rc_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)2799 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
2800 				      struct bnxt_qplib_cqe *cqe)
2801 {
2802 	wc->opcode = IB_WC_RECV;
2803 	wc->status = __rc_to_ib_wc_status(cqe->status);
2804 
2805 	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2806 		wc->wc_flags |= IB_WC_WITH_IMM;
2807 	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2808 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2809 	if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2810 	    (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2811 		wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2812 }
2813 
bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp * qp,struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)2814 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp,
2815 					     struct ib_wc *wc,
2816 					     struct bnxt_qplib_cqe *cqe)
2817 {
2818 	u32 tbl_idx;
2819 	struct bnxt_re_dev *rdev = qp->rdev;
2820 	struct bnxt_re_qp *qp1_qp = NULL;
2821 	struct bnxt_qplib_cqe *orig_cqe = NULL;
2822 	struct bnxt_re_sqp_entries *sqp_entry = NULL;
2823 	int nw_type;
2824 
2825 	tbl_idx = cqe->wr_id;
2826 
2827 	sqp_entry = &rdev->sqp_tbl[tbl_idx];
2828 	qp1_qp = sqp_entry->qp1_qp;
2829 	orig_cqe = &sqp_entry->cqe;
2830 
2831 	wc->wr_id = sqp_entry->wrid;
2832 	wc->byte_len = orig_cqe->length;
2833 	wc->qp = &qp1_qp->ib_qp;
2834 
2835 	wc->ex.imm_data = orig_cqe->immdata;
2836 	wc->src_qp = orig_cqe->src_qp;
2837 	memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
2838 	wc->port_num = 1;
2839 	wc->vendor_err = orig_cqe->status;
2840 
2841 	wc->opcode = IB_WC_RECV;
2842 	wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
2843 	wc->wc_flags |= IB_WC_GRH;
2844 
2845 	nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
2846 					    orig_cqe->raweth_qp1_flags2);
2847 	if (nw_type >= 0) {
2848 		wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
2849 		wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2850 	}
2851 }
2852 
bnxt_re_process_res_ud_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)2853 static void bnxt_re_process_res_ud_wc(struct ib_wc *wc,
2854 				      struct bnxt_qplib_cqe *cqe)
2855 {
2856 	wc->opcode = IB_WC_RECV;
2857 	wc->status = __rc_to_ib_wc_status(cqe->status);
2858 
2859 	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2860 		wc->wc_flags |= IB_WC_WITH_IMM;
2861 	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2862 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2863 	if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2864 	    (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2865 		wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2866 }
2867 
send_phantom_wqe(struct bnxt_re_qp * qp)2868 static int send_phantom_wqe(struct bnxt_re_qp *qp)
2869 {
2870 	struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
2871 	unsigned long flags;
2872 	int rc = 0;
2873 
2874 	spin_lock_irqsave(&qp->sq_lock, flags);
2875 
2876 	rc = bnxt_re_bind_fence_mw(lib_qp);
2877 	if (!rc) {
2878 		lib_qp->sq.phantom_wqe_cnt++;
2879 		dev_dbg(&lib_qp->sq.hwq.pdev->dev,
2880 			"qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
2881 			lib_qp->id, lib_qp->sq.hwq.prod,
2882 			HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
2883 			lib_qp->sq.phantom_wqe_cnt);
2884 	}
2885 
2886 	spin_unlock_irqrestore(&qp->sq_lock, flags);
2887 	return rc;
2888 }
2889 
bnxt_re_poll_cq(struct ib_cq * ib_cq,int num_entries,struct ib_wc * wc)2890 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
2891 {
2892 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2893 	struct bnxt_re_qp *qp;
2894 	struct bnxt_qplib_cqe *cqe;
2895 	int i, ncqe, budget;
2896 	struct bnxt_qplib_q *sq;
2897 	struct bnxt_qplib_qp *lib_qp;
2898 	u32 tbl_idx;
2899 	struct bnxt_re_sqp_entries *sqp_entry = NULL;
2900 	unsigned long flags;
2901 
2902 	spin_lock_irqsave(&cq->cq_lock, flags);
2903 	budget = min_t(u32, num_entries, cq->max_cql);
2904 	num_entries = budget;
2905 	if (!cq->cql) {
2906 		dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use");
2907 		goto exit;
2908 	}
2909 	cqe = &cq->cql[0];
2910 	while (budget) {
2911 		lib_qp = NULL;
2912 		ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
2913 		if (lib_qp) {
2914 			sq = &lib_qp->sq;
2915 			if (sq->send_phantom) {
2916 				qp = container_of(lib_qp,
2917 						  struct bnxt_re_qp, qplib_qp);
2918 				if (send_phantom_wqe(qp) == -ENOMEM)
2919 					dev_err(rdev_to_dev(cq->rdev),
2920 						"Phantom failed! Scheduled to send again\n");
2921 				else
2922 					sq->send_phantom = false;
2923 			}
2924 		}
2925 		if (ncqe < budget)
2926 			ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
2927 							      cqe + ncqe,
2928 							      budget - ncqe);
2929 
2930 		if (!ncqe)
2931 			break;
2932 
2933 		for (i = 0; i < ncqe; i++, cqe++) {
2934 			/* Transcribe each qplib_wqe back to ib_wc */
2935 			memset(wc, 0, sizeof(*wc));
2936 
2937 			wc->wr_id = cqe->wr_id;
2938 			wc->byte_len = cqe->length;
2939 			qp = container_of
2940 				((struct bnxt_qplib_qp *)
2941 				 (unsigned long)(cqe->qp_handle),
2942 				 struct bnxt_re_qp, qplib_qp);
2943 			if (!qp) {
2944 				dev_err(rdev_to_dev(cq->rdev),
2945 					"POLL CQ : bad QP handle");
2946 				continue;
2947 			}
2948 			wc->qp = &qp->ib_qp;
2949 			wc->ex.imm_data = cqe->immdata;
2950 			wc->src_qp = cqe->src_qp;
2951 			memcpy(wc->smac, cqe->smac, ETH_ALEN);
2952 			wc->port_num = 1;
2953 			wc->vendor_err = cqe->status;
2954 
2955 			switch (cqe->opcode) {
2956 			case CQ_BASE_CQE_TYPE_REQ:
2957 				if (qp->qplib_qp.id ==
2958 				    qp->rdev->qp1_sqp->qplib_qp.id) {
2959 					/* Handle this completion with
2960 					 * the stored completion
2961 					 */
2962 					memset(wc, 0, sizeof(*wc));
2963 					continue;
2964 				}
2965 				bnxt_re_process_req_wc(wc, cqe);
2966 				break;
2967 			case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2968 				if (!cqe->status) {
2969 					int rc = 0;
2970 
2971 					rc = bnxt_re_process_raw_qp_pkt_rx
2972 								(qp, cqe);
2973 					if (!rc) {
2974 						memset(wc, 0, sizeof(*wc));
2975 						continue;
2976 					}
2977 					cqe->status = -1;
2978 				}
2979 				/* Errors need not be looped back.
2980 				 * But change the wr_id to the one
2981 				 * stored in the table
2982 				 */
2983 				tbl_idx = cqe->wr_id;
2984 				sqp_entry = &cq->rdev->sqp_tbl[tbl_idx];
2985 				wc->wr_id = sqp_entry->wrid;
2986 				bnxt_re_process_res_rawqp1_wc(wc, cqe);
2987 				break;
2988 			case CQ_BASE_CQE_TYPE_RES_RC:
2989 				bnxt_re_process_res_rc_wc(wc, cqe);
2990 				break;
2991 			case CQ_BASE_CQE_TYPE_RES_UD:
2992 				if (qp->qplib_qp.id ==
2993 				    qp->rdev->qp1_sqp->qplib_qp.id) {
2994 					/* Handle this completion with
2995 					 * the stored completion
2996 					 */
2997 					if (cqe->status) {
2998 						continue;
2999 					} else {
3000 						bnxt_re_process_res_shadow_qp_wc
3001 								(qp, wc, cqe);
3002 						break;
3003 					}
3004 				}
3005 				bnxt_re_process_res_ud_wc(wc, cqe);
3006 				break;
3007 			default:
3008 				dev_err(rdev_to_dev(cq->rdev),
3009 					"POLL CQ : type 0x%x not handled",
3010 					cqe->opcode);
3011 				continue;
3012 			}
3013 			wc++;
3014 			budget--;
3015 		}
3016 	}
3017 exit:
3018 	spin_unlock_irqrestore(&cq->cq_lock, flags);
3019 	return num_entries - budget;
3020 }
3021 
bnxt_re_req_notify_cq(struct ib_cq * ib_cq,enum ib_cq_notify_flags ib_cqn_flags)3022 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
3023 			  enum ib_cq_notify_flags ib_cqn_flags)
3024 {
3025 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3026 	int type = 0;
3027 
3028 	/* Trigger on the very next completion */
3029 	if (ib_cqn_flags & IB_CQ_NEXT_COMP)
3030 		type = DBR_DBR_TYPE_CQ_ARMALL;
3031 	/* Trigger on the next solicited completion */
3032 	else if (ib_cqn_flags & IB_CQ_SOLICITED)
3033 		type = DBR_DBR_TYPE_CQ_ARMSE;
3034 
3035 	/* Poll to see if there are missed events */
3036 	if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3037 	    !(bnxt_qplib_is_cq_empty(&cq->qplib_cq)))
3038 		return 1;
3039 
3040 	bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
3041 
3042 	return 0;
3043 }
3044 
3045 /* Memory Regions */
bnxt_re_get_dma_mr(struct ib_pd * ib_pd,int mr_access_flags)3046 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
3047 {
3048 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3049 	struct bnxt_re_dev *rdev = pd->rdev;
3050 	struct bnxt_re_mr *mr;
3051 	u64 pbl = 0;
3052 	int rc;
3053 
3054 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3055 	if (!mr)
3056 		return ERR_PTR(-ENOMEM);
3057 
3058 	mr->rdev = rdev;
3059 	mr->qplib_mr.pd = &pd->qplib_pd;
3060 	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3061 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3062 
3063 	/* Allocate and register 0 as the address */
3064 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3065 	if (rc)
3066 		goto fail;
3067 
3068 	mr->qplib_mr.hwq.level = PBL_LVL_MAX;
3069 	mr->qplib_mr.total_size = -1; /* Infinte length */
3070 	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false);
3071 	if (rc)
3072 		goto fail_mr;
3073 
3074 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
3075 	if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
3076 			       IB_ACCESS_REMOTE_ATOMIC))
3077 		mr->ib_mr.rkey = mr->ib_mr.lkey;
3078 	atomic_inc(&rdev->mr_count);
3079 
3080 	return &mr->ib_mr;
3081 
3082 fail_mr:
3083 	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3084 fail:
3085 	kfree(mr);
3086 	return ERR_PTR(rc);
3087 }
3088 
bnxt_re_dereg_mr(struct ib_mr * ib_mr)3089 int bnxt_re_dereg_mr(struct ib_mr *ib_mr)
3090 {
3091 	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3092 	struct bnxt_re_dev *rdev = mr->rdev;
3093 	int rc;
3094 
3095 	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3096 	if (rc) {
3097 		dev_err(rdev_to_dev(rdev), "Dereg MR failed: %#x\n", rc);
3098 		return rc;
3099 	}
3100 
3101 	if (mr->pages) {
3102 		rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
3103 							&mr->qplib_frpl);
3104 		kfree(mr->pages);
3105 		mr->npages = 0;
3106 		mr->pages = NULL;
3107 	}
3108 	if (!IS_ERR_OR_NULL(mr->ib_umem))
3109 		ib_umem_release(mr->ib_umem);
3110 
3111 	kfree(mr);
3112 	atomic_dec(&rdev->mr_count);
3113 	return rc;
3114 }
3115 
bnxt_re_set_page(struct ib_mr * ib_mr,u64 addr)3116 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
3117 {
3118 	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3119 
3120 	if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
3121 		return -ENOMEM;
3122 
3123 	mr->pages[mr->npages++] = addr;
3124 	return 0;
3125 }
3126 
bnxt_re_map_mr_sg(struct ib_mr * ib_mr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)3127 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
3128 		      unsigned int *sg_offset)
3129 {
3130 	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3131 
3132 	mr->npages = 0;
3133 	return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
3134 }
3135 
bnxt_re_alloc_mr(struct ib_pd * ib_pd,enum ib_mr_type type,u32 max_num_sg)3136 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
3137 			       u32 max_num_sg)
3138 {
3139 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3140 	struct bnxt_re_dev *rdev = pd->rdev;
3141 	struct bnxt_re_mr *mr = NULL;
3142 	int rc;
3143 
3144 	if (type != IB_MR_TYPE_MEM_REG) {
3145 		dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type);
3146 		return ERR_PTR(-EINVAL);
3147 	}
3148 	if (max_num_sg > MAX_PBL_LVL_1_PGS)
3149 		return ERR_PTR(-EINVAL);
3150 
3151 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3152 	if (!mr)
3153 		return ERR_PTR(-ENOMEM);
3154 
3155 	mr->rdev = rdev;
3156 	mr->qplib_mr.pd = &pd->qplib_pd;
3157 	mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
3158 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3159 
3160 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3161 	if (rc)
3162 		goto fail;
3163 
3164 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
3165 	mr->ib_mr.rkey = mr->ib_mr.lkey;
3166 
3167 	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
3168 	if (!mr->pages) {
3169 		rc = -ENOMEM;
3170 		goto fail;
3171 	}
3172 	rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
3173 						 &mr->qplib_frpl, max_num_sg);
3174 	if (rc) {
3175 		dev_err(rdev_to_dev(rdev),
3176 			"Failed to allocate HW FR page list");
3177 		goto fail_mr;
3178 	}
3179 
3180 	atomic_inc(&rdev->mr_count);
3181 	return &mr->ib_mr;
3182 
3183 fail_mr:
3184 	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3185 fail:
3186 	kfree(mr->pages);
3187 	kfree(mr);
3188 	return ERR_PTR(rc);
3189 }
3190 
bnxt_re_alloc_mw(struct ib_pd * ib_pd,enum ib_mw_type type,struct ib_udata * udata)3191 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
3192 			       struct ib_udata *udata)
3193 {
3194 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3195 	struct bnxt_re_dev *rdev = pd->rdev;
3196 	struct bnxt_re_mw *mw;
3197 	int rc;
3198 
3199 	mw = kzalloc(sizeof(*mw), GFP_KERNEL);
3200 	if (!mw)
3201 		return ERR_PTR(-ENOMEM);
3202 	mw->rdev = rdev;
3203 	mw->qplib_mw.pd = &pd->qplib_pd;
3204 
3205 	mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
3206 			       CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
3207 			       CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
3208 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
3209 	if (rc) {
3210 		dev_err(rdev_to_dev(rdev), "Allocate MW failed!");
3211 		goto fail;
3212 	}
3213 	mw->ib_mw.rkey = mw->qplib_mw.rkey;
3214 
3215 	atomic_inc(&rdev->mw_count);
3216 	return &mw->ib_mw;
3217 
3218 fail:
3219 	kfree(mw);
3220 	return ERR_PTR(rc);
3221 }
3222 
bnxt_re_dealloc_mw(struct ib_mw * ib_mw)3223 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
3224 {
3225 	struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
3226 	struct bnxt_re_dev *rdev = mw->rdev;
3227 	int rc;
3228 
3229 	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
3230 	if (rc) {
3231 		dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc);
3232 		return rc;
3233 	}
3234 
3235 	kfree(mw);
3236 	atomic_dec(&rdev->mw_count);
3237 	return rc;
3238 }
3239 
3240 /* uverbs */
bnxt_re_reg_user_mr(struct ib_pd * ib_pd,u64 start,u64 length,u64 virt_addr,int mr_access_flags,struct ib_udata * udata)3241 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
3242 				  u64 virt_addr, int mr_access_flags,
3243 				  struct ib_udata *udata)
3244 {
3245 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3246 	struct bnxt_re_dev *rdev = pd->rdev;
3247 	struct bnxt_re_mr *mr;
3248 	struct ib_umem *umem;
3249 	u64 *pbl_tbl, *pbl_tbl_orig;
3250 	int i, umem_pgs, pages, rc;
3251 	struct scatterlist *sg;
3252 	int entry;
3253 
3254 	if (length > BNXT_RE_MAX_MR_SIZE) {
3255 		dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%ld\n",
3256 			length, BNXT_RE_MAX_MR_SIZE);
3257 		return ERR_PTR(-ENOMEM);
3258 	}
3259 
3260 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3261 	if (!mr)
3262 		return ERR_PTR(-ENOMEM);
3263 
3264 	mr->rdev = rdev;
3265 	mr->qplib_mr.pd = &pd->qplib_pd;
3266 	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3267 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
3268 
3269 	umem = ib_umem_get(ib_pd->uobject->context, start, length,
3270 			   mr_access_flags, 0);
3271 	if (IS_ERR(umem)) {
3272 		dev_err(rdev_to_dev(rdev), "Failed to get umem");
3273 		rc = -EFAULT;
3274 		goto free_mr;
3275 	}
3276 	mr->ib_umem = umem;
3277 
3278 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3279 	if (rc) {
3280 		dev_err(rdev_to_dev(rdev), "Failed to allocate MR");
3281 		goto release_umem;
3282 	}
3283 	/* The fixed portion of the rkey is the same as the lkey */
3284 	mr->ib_mr.rkey = mr->qplib_mr.rkey;
3285 
3286 	mr->qplib_mr.va = virt_addr;
3287 	umem_pgs = ib_umem_page_count(umem);
3288 	if (!umem_pgs) {
3289 		dev_err(rdev_to_dev(rdev), "umem is invalid!");
3290 		rc = -EINVAL;
3291 		goto free_mrw;
3292 	}
3293 	mr->qplib_mr.total_size = length;
3294 
3295 	pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL);
3296 	if (!pbl_tbl) {
3297 		rc = -EINVAL;
3298 		goto free_mrw;
3299 	}
3300 	pbl_tbl_orig = pbl_tbl;
3301 
3302 	if (umem->hugetlb) {
3303 		dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!");
3304 		rc = -EFAULT;
3305 		goto fail;
3306 	}
3307 
3308 	if (umem->page_shift != PAGE_SHIFT) {
3309 		dev_err(rdev_to_dev(rdev), "umem page shift unsupported!");
3310 		rc = -EFAULT;
3311 		goto fail;
3312 	}
3313 	/* Map umem buf ptrs to the PBL */
3314 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
3315 		pages = sg_dma_len(sg) >> umem->page_shift;
3316 		for (i = 0; i < pages; i++, pbl_tbl++)
3317 			*pbl_tbl = sg_dma_address(sg) + (i << umem->page_shift);
3318 	}
3319 	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig,
3320 			       umem_pgs, false);
3321 	if (rc) {
3322 		dev_err(rdev_to_dev(rdev), "Failed to register user MR");
3323 		goto fail;
3324 	}
3325 
3326 	kfree(pbl_tbl_orig);
3327 
3328 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
3329 	mr->ib_mr.rkey = mr->qplib_mr.lkey;
3330 	atomic_inc(&rdev->mr_count);
3331 
3332 	return &mr->ib_mr;
3333 fail:
3334 	kfree(pbl_tbl_orig);
3335 free_mrw:
3336 	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3337 release_umem:
3338 	ib_umem_release(umem);
3339 free_mr:
3340 	kfree(mr);
3341 	return ERR_PTR(rc);
3342 }
3343 
bnxt_re_alloc_ucontext(struct ib_device * ibdev,struct ib_udata * udata)3344 struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev,
3345 					   struct ib_udata *udata)
3346 {
3347 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
3348 	struct bnxt_re_uctx_resp resp;
3349 	struct bnxt_re_ucontext *uctx;
3350 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
3351 	int rc;
3352 
3353 	dev_dbg(rdev_to_dev(rdev), "ABI version requested %d",
3354 		ibdev->uverbs_abi_ver);
3355 
3356 	if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
3357 		dev_dbg(rdev_to_dev(rdev), " is different from the device %d ",
3358 			BNXT_RE_ABI_VERSION);
3359 		return ERR_PTR(-EPERM);
3360 	}
3361 
3362 	uctx = kzalloc(sizeof(*uctx), GFP_KERNEL);
3363 	if (!uctx)
3364 		return ERR_PTR(-ENOMEM);
3365 
3366 	uctx->rdev = rdev;
3367 
3368 	uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
3369 	if (!uctx->shpg) {
3370 		rc = -ENOMEM;
3371 		goto fail;
3372 	}
3373 	spin_lock_init(&uctx->sh_lock);
3374 
3375 	resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/
3376 	resp.max_qp = rdev->qplib_ctx.qpc_count;
3377 	resp.pg_size = PAGE_SIZE;
3378 	resp.cqe_sz = sizeof(struct cq_base);
3379 	resp.max_cqd = dev_attr->max_cq_wqes;
3380 	resp.rsvd    = 0;
3381 
3382 	rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
3383 	if (rc) {
3384 		dev_err(rdev_to_dev(rdev), "Failed to copy user context");
3385 		rc = -EFAULT;
3386 		goto cfail;
3387 	}
3388 
3389 	return &uctx->ib_uctx;
3390 cfail:
3391 	free_page((unsigned long)uctx->shpg);
3392 	uctx->shpg = NULL;
3393 fail:
3394 	kfree(uctx);
3395 	return ERR_PTR(rc);
3396 }
3397 
bnxt_re_dealloc_ucontext(struct ib_ucontext * ib_uctx)3398 int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
3399 {
3400 	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3401 						   struct bnxt_re_ucontext,
3402 						   ib_uctx);
3403 
3404 	struct bnxt_re_dev *rdev = uctx->rdev;
3405 	int rc = 0;
3406 
3407 	if (uctx->shpg)
3408 		free_page((unsigned long)uctx->shpg);
3409 
3410 	if (uctx->dpi.dbr) {
3411 		/* Free DPI only if this is the first PD allocated by the
3412 		 * application and mark the context dpi as NULL
3413 		 */
3414 		rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
3415 					    &rdev->qplib_res.dpi_tbl,
3416 					    &uctx->dpi);
3417 		if (rc)
3418 			dev_err(rdev_to_dev(rdev), "Deallocate HW DPI failed!");
3419 			/* Don't fail, continue*/
3420 		uctx->dpi.dbr = NULL;
3421 	}
3422 
3423 	kfree(uctx);
3424 	return 0;
3425 }
3426 
3427 /* Helper function to mmap the virtual memory from user app */
bnxt_re_mmap(struct ib_ucontext * ib_uctx,struct vm_area_struct * vma)3428 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
3429 {
3430 	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3431 						   struct bnxt_re_ucontext,
3432 						   ib_uctx);
3433 	struct bnxt_re_dev *rdev = uctx->rdev;
3434 	u64 pfn;
3435 
3436 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3437 		return -EINVAL;
3438 
3439 	if (vma->vm_pgoff) {
3440 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3441 		if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
3442 				       PAGE_SIZE, vma->vm_page_prot)) {
3443 			dev_err(rdev_to_dev(rdev), "Failed to map DPI");
3444 			return -EAGAIN;
3445 		}
3446 	} else {
3447 		pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT;
3448 		if (remap_pfn_range(vma, vma->vm_start,
3449 				    pfn, PAGE_SIZE, vma->vm_page_prot)) {
3450 			dev_err(rdev_to_dev(rdev),
3451 				"Failed to map shared page");
3452 			return -EAGAIN;
3453 		}
3454 	}
3455 
3456 	return 0;
3457 }
3458