1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/ib_umem.h>
16 #include <rdma/uverbs_ioctl.h>
17
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21
assemble_qbuf_mtt_for_cmd(struct erdma_mem * mem,u32 * cfg,u64 * addr0,u64 * addr1)22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
23 u64 *addr0, u64 *addr1)
24 {
25 struct erdma_mtt *mtt = mem->mtt;
26
27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) {
28 *addr0 = mtt->buf_dma;
29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
30 ERDMA_MR_MTT_1LEVEL);
31 } else {
32 *addr0 = mtt->buf[0];
33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1));
34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
35 ERDMA_MR_MTT_0LEVEL);
36 }
37 }
38
create_qp_cmd(struct erdma_ucontext * uctx,struct erdma_qp * qp)39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
40 {
41 struct erdma_dev *dev = to_edev(qp->ibqp.device);
42 struct erdma_pd *pd = to_epd(qp->ibqp.pd);
43 struct erdma_cmdq_create_qp_req req;
44 struct erdma_uqp *user_qp;
45 u64 resp0, resp1;
46 int err;
47
48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
49 CMDQ_OPCODE_CREATE_QP);
50
51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
52 ilog2(qp->attrs.sq_size)) |
53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
55 ilog2(qp->attrs.rq_size)) |
56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
57
58 if (rdma_is_kernel_res(&qp->ibqp.res)) {
59 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
60
61 req.sq_cqn_mtt_cfg =
62 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
63 pgsz_range) |
64 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
65 req.rq_cqn_mtt_cfg =
66 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
67 pgsz_range) |
68 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
69
70 req.sq_mtt_cfg =
71 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
72 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
73 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
74 ERDMA_MR_MTT_0LEVEL);
75 req.rq_mtt_cfg = req.sq_mtt_cfg;
76
77 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
78 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
79 req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
80 req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
81 } else {
82 user_qp = &qp->user_qp;
83 req.sq_cqn_mtt_cfg = FIELD_PREP(
84 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
85 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
86 req.sq_cqn_mtt_cfg |=
87 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
88
89 req.rq_cqn_mtt_cfg = FIELD_PREP(
90 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
91 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
92 req.rq_cqn_mtt_cfg |=
93 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
94
95 req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
96 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
97 user_qp->sq_mem.mtt_nents);
98
99 req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
100 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
101 user_qp->rq_mem.mtt_nents);
102
103 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
104 &req.sq_buf_addr, req.sq_mtt_entry);
105 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
106 &req.rq_buf_addr, req.rq_mtt_entry);
107
108 req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
109 req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
110
111 if (uctx->ext_db.enable) {
112 req.sq_cqn_mtt_cfg |=
113 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
114 req.db_cfg =
115 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
116 uctx->ext_db.sdb_off) |
117 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
118 uctx->ext_db.rdb_off);
119 }
120 }
121
122 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
123 &resp1);
124 if (!err)
125 qp->attrs.cookie =
126 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
127
128 return err;
129 }
130
regmr_cmd(struct erdma_dev * dev,struct erdma_mr * mr)131 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
132 {
133 struct erdma_pd *pd = to_epd(mr->ibmr.pd);
134 u32 mtt_level = ERDMA_MR_MTT_0LEVEL;
135 struct erdma_cmdq_reg_mr_req req;
136
137 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
138
139 if (mr->type == ERDMA_MR_TYPE_FRMR ||
140 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) {
141 if (mr->mem.mtt->continuous) {
142 req.phy_addr[0] = mr->mem.mtt->buf_dma;
143 mtt_level = ERDMA_MR_MTT_1LEVEL;
144 } else {
145 req.phy_addr[0] = sg_dma_address(mr->mem.mtt->sglist);
146 mtt_level = mr->mem.mtt->level;
147 }
148 } else if (mr->type != ERDMA_MR_TYPE_DMA) {
149 memcpy(req.phy_addr, mr->mem.mtt->buf,
150 MTT_SIZE(mr->mem.page_cnt));
151 }
152
153 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
154 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
155 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
156 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
157 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
158 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
159 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
160 ilog2(mr->mem.page_size)) |
161 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |
162 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
163
164 if (mr->type == ERDMA_MR_TYPE_DMA)
165 goto post_cmd;
166
167 if (mr->type == ERDMA_MR_TYPE_NORMAL) {
168 req.start_va = mr->mem.va;
169 req.size = mr->mem.len;
170 }
171
172 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) {
173 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);
174 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,
175 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);
176 req.size_h = upper_32_bits(mr->mem.len);
177 req.mtt_cnt_h = mr->mem.page_cnt >> 20;
178 }
179
180 post_cmd:
181 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
182 }
183
create_cq_cmd(struct erdma_ucontext * uctx,struct erdma_cq * cq)184 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
185 {
186 struct erdma_dev *dev = to_edev(cq->ibcq.device);
187 struct erdma_cmdq_create_cq_req req;
188 struct erdma_mem *mem;
189 u32 page_size;
190
191 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
192 CMDQ_OPCODE_CREATE_CQ);
193
194 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
195 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
196 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
197
198 if (rdma_is_kernel_res(&cq->ibcq.res)) {
199 page_size = SZ_32M;
200 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
201 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
202 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
203 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
204
205 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
206 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
207 ERDMA_MR_MTT_0LEVEL);
208
209 req.first_page_offset = 0;
210 req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
211 } else {
212 mem = &cq->user_cq.qbuf_mem;
213 req.cfg0 |=
214 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
215 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
216 if (mem->mtt_nents == 1) {
217 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
218 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
219 req.cfg1 |=
220 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
221 ERDMA_MR_MTT_0LEVEL);
222 } else {
223 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
224 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
225 req.cfg1 |=
226 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
227 ERDMA_MR_MTT_1LEVEL);
228 }
229 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
230 mem->mtt_nents);
231
232 req.first_page_offset = mem->page_offset;
233 req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
234
235 if (uctx->ext_db.enable) {
236 req.cfg1 |= FIELD_PREP(
237 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
238 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
239 uctx->ext_db.cdb_off);
240 }
241 }
242
243 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
244 }
245
erdma_alloc_idx(struct erdma_resource_cb * res_cb)246 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
247 {
248 int idx;
249 unsigned long flags;
250
251 spin_lock_irqsave(&res_cb->lock, flags);
252 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
253 res_cb->next_alloc_idx);
254 if (idx == res_cb->max_cap) {
255 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
256 if (idx == res_cb->max_cap) {
257 res_cb->next_alloc_idx = 1;
258 spin_unlock_irqrestore(&res_cb->lock, flags);
259 return -ENOSPC;
260 }
261 }
262
263 set_bit(idx, res_cb->bitmap);
264 res_cb->next_alloc_idx = idx + 1;
265 spin_unlock_irqrestore(&res_cb->lock, flags);
266
267 return idx;
268 }
269
erdma_free_idx(struct erdma_resource_cb * res_cb,u32 idx)270 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
271 {
272 unsigned long flags;
273 u32 used;
274
275 spin_lock_irqsave(&res_cb->lock, flags);
276 used = __test_and_clear_bit(idx, res_cb->bitmap);
277 spin_unlock_irqrestore(&res_cb->lock, flags);
278 WARN_ON(!used);
279 }
280
281 static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext * uctx,void * address,u32 size,u8 mmap_flag,u64 * mmap_offset)282 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
283 u32 size, u8 mmap_flag, u64 *mmap_offset)
284 {
285 struct erdma_user_mmap_entry *entry =
286 kzalloc(sizeof(*entry), GFP_KERNEL);
287 int ret;
288
289 if (!entry)
290 return NULL;
291
292 entry->address = (u64)address;
293 entry->mmap_flag = mmap_flag;
294
295 size = PAGE_ALIGN(size);
296
297 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
298 size);
299 if (ret) {
300 kfree(entry);
301 return NULL;
302 }
303
304 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
305
306 return &entry->rdma_entry;
307 }
308
erdma_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * unused)309 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
310 struct ib_udata *unused)
311 {
312 struct erdma_dev *dev = to_edev(ibdev);
313
314 memset(attr, 0, sizeof(*attr));
315
316 attr->max_mr_size = dev->attrs.max_mr_size;
317 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
318 attr->vendor_part_id = dev->pdev->device;
319 attr->hw_ver = dev->pdev->revision;
320 attr->max_qp = dev->attrs.max_qp - 1;
321 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
322 attr->max_qp_rd_atom = dev->attrs.max_ord;
323 attr->max_qp_init_rd_atom = dev->attrs.max_ird;
324 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
325 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
326 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
327 ibdev->local_dma_lkey = dev->attrs.local_dma_key;
328 attr->max_send_sge = dev->attrs.max_send_sge;
329 attr->max_recv_sge = dev->attrs.max_recv_sge;
330 attr->max_sge_rd = dev->attrs.max_sge_rd;
331 attr->max_cq = dev->attrs.max_cq - 1;
332 attr->max_cqe = dev->attrs.max_cqe;
333 attr->max_mr = dev->attrs.max_mr;
334 attr->max_pd = dev->attrs.max_pd;
335 attr->max_mw = dev->attrs.max_mw;
336 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
337 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
338
339 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
340 attr->atomic_cap = IB_ATOMIC_GLOB;
341
342 attr->fw_ver = dev->attrs.fw_version;
343
344 if (dev->netdev)
345 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
346 dev->netdev->dev_addr);
347
348 return 0;
349 }
350
erdma_query_gid(struct ib_device * ibdev,u32 port,int idx,union ib_gid * gid)351 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
352 union ib_gid *gid)
353 {
354 struct erdma_dev *dev = to_edev(ibdev);
355
356 memset(gid, 0, sizeof(*gid));
357 ether_addr_copy(gid->raw, dev->attrs.peer_addr);
358
359 return 0;
360 }
361
erdma_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)362 int erdma_query_port(struct ib_device *ibdev, u32 port,
363 struct ib_port_attr *attr)
364 {
365 struct erdma_dev *dev = to_edev(ibdev);
366 struct net_device *ndev = dev->netdev;
367
368 memset(attr, 0, sizeof(*attr));
369
370 attr->gid_tbl_len = 1;
371 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
372 attr->max_msg_sz = -1;
373
374 if (!ndev)
375 goto out;
376
377 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
378 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
379 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
380 if (netif_running(ndev) && netif_carrier_ok(ndev))
381 dev->state = IB_PORT_ACTIVE;
382 else
383 dev->state = IB_PORT_DOWN;
384 attr->state = dev->state;
385
386 out:
387 if (dev->state == IB_PORT_ACTIVE)
388 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
389 else
390 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
391
392 return 0;
393 }
394
erdma_get_port_immutable(struct ib_device * ibdev,u32 port,struct ib_port_immutable * port_immutable)395 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
396 struct ib_port_immutable *port_immutable)
397 {
398 port_immutable->gid_tbl_len = 1;
399 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
400
401 return 0;
402 }
403
erdma_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)404 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
405 {
406 struct erdma_pd *pd = to_epd(ibpd);
407 struct erdma_dev *dev = to_edev(ibpd->device);
408 int pdn;
409
410 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
411 if (pdn < 0)
412 return pdn;
413
414 pd->pdn = pdn;
415
416 return 0;
417 }
418
erdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)419 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
420 {
421 struct erdma_pd *pd = to_epd(ibpd);
422 struct erdma_dev *dev = to_edev(ibpd->device);
423
424 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
425
426 return 0;
427 }
428
erdma_flush_worker(struct work_struct * work)429 static void erdma_flush_worker(struct work_struct *work)
430 {
431 struct delayed_work *dwork = to_delayed_work(work);
432 struct erdma_qp *qp =
433 container_of(dwork, struct erdma_qp, reflush_dwork);
434 struct erdma_cmdq_reflush_req req;
435
436 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
437 CMDQ_OPCODE_REFLUSH);
438 req.qpn = QP_ID(qp);
439 req.sq_pi = qp->kern_qp.sq_pi;
440 req.rq_pi = qp->kern_qp.rq_pi;
441 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL);
442 }
443
erdma_qp_validate_cap(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)444 static int erdma_qp_validate_cap(struct erdma_dev *dev,
445 struct ib_qp_init_attr *attrs)
446 {
447 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
448 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
449 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
450 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
451 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
452 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
453 return -EINVAL;
454 }
455
456 return 0;
457 }
458
erdma_qp_validate_attr(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)459 static int erdma_qp_validate_attr(struct erdma_dev *dev,
460 struct ib_qp_init_attr *attrs)
461 {
462 if (attrs->qp_type != IB_QPT_RC)
463 return -EOPNOTSUPP;
464
465 if (attrs->srq)
466 return -EOPNOTSUPP;
467
468 if (!attrs->send_cq || !attrs->recv_cq)
469 return -EOPNOTSUPP;
470
471 return 0;
472 }
473
free_kernel_qp(struct erdma_qp * qp)474 static void free_kernel_qp(struct erdma_qp *qp)
475 {
476 struct erdma_dev *dev = qp->dev;
477
478 vfree(qp->kern_qp.swr_tbl);
479 vfree(qp->kern_qp.rwr_tbl);
480
481 if (qp->kern_qp.sq_buf)
482 dma_free_coherent(&dev->pdev->dev,
483 qp->attrs.sq_size << SQEBB_SHIFT,
484 qp->kern_qp.sq_buf,
485 qp->kern_qp.sq_buf_dma_addr);
486
487 if (qp->kern_qp.sq_dbrec)
488 dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
489 qp->kern_qp.sq_dbrec_dma);
490
491 if (qp->kern_qp.rq_buf)
492 dma_free_coherent(&dev->pdev->dev,
493 qp->attrs.rq_size << RQE_SHIFT,
494 qp->kern_qp.rq_buf,
495 qp->kern_qp.rq_buf_dma_addr);
496
497 if (qp->kern_qp.rq_dbrec)
498 dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
499 qp->kern_qp.rq_dbrec_dma);
500 }
501
init_kernel_qp(struct erdma_dev * dev,struct erdma_qp * qp,struct ib_qp_init_attr * attrs)502 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
503 struct ib_qp_init_attr *attrs)
504 {
505 struct erdma_kqp *kqp = &qp->kern_qp;
506 int size;
507
508 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
509 kqp->sig_all = 1;
510
511 kqp->sq_pi = 0;
512 kqp->sq_ci = 0;
513 kqp->rq_pi = 0;
514 kqp->rq_ci = 0;
515 kqp->hw_sq_db =
516 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
517 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
518
519 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
520 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
521 if (!kqp->swr_tbl || !kqp->rwr_tbl)
522 goto err_out;
523
524 size = qp->attrs.sq_size << SQEBB_SHIFT;
525 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
526 &kqp->sq_buf_dma_addr, GFP_KERNEL);
527 if (!kqp->sq_buf)
528 goto err_out;
529
530 kqp->sq_dbrec =
531 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
532 if (!kqp->sq_dbrec)
533 goto err_out;
534
535 size = qp->attrs.rq_size << RQE_SHIFT;
536 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
537 &kqp->rq_buf_dma_addr, GFP_KERNEL);
538 if (!kqp->rq_buf)
539 goto err_out;
540
541 kqp->rq_dbrec =
542 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
543 if (!kqp->rq_dbrec)
544 goto err_out;
545
546 return 0;
547
548 err_out:
549 free_kernel_qp(qp);
550 return -ENOMEM;
551 }
552
erdma_fill_bottom_mtt(struct erdma_dev * dev,struct erdma_mem * mem)553 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
554 {
555 struct erdma_mtt *mtt = mem->mtt;
556 struct ib_block_iter biter;
557 u32 idx = 0;
558
559 while (mtt->low_level)
560 mtt = mtt->low_level;
561
562 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
563 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
564 }
565
erdma_create_cont_mtt(struct erdma_dev * dev,size_t size)566 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
567 size_t size)
568 {
569 struct erdma_mtt *mtt;
570
571 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
572 if (!mtt)
573 return ERR_PTR(-ENOMEM);
574
575 mtt->size = size;
576 mtt->buf = kzalloc(mtt->size, GFP_KERNEL);
577 if (!mtt->buf)
578 goto err_free_mtt;
579
580 mtt->continuous = true;
581 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size,
582 DMA_TO_DEVICE);
583 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma))
584 goto err_free_mtt_buf;
585
586 return mtt;
587
588 err_free_mtt_buf:
589 kfree(mtt->buf);
590
591 err_free_mtt:
592 kfree(mtt);
593
594 return ERR_PTR(-ENOMEM);
595 }
596
erdma_destroy_mtt_buf_sg(struct erdma_dev * dev,struct erdma_mtt * mtt)597 static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev,
598 struct erdma_mtt *mtt)
599 {
600 dma_unmap_sg(&dev->pdev->dev, mtt->sglist,
601 DIV_ROUND_UP(mtt->size, PAGE_SIZE), DMA_TO_DEVICE);
602 vfree(mtt->sglist);
603 }
604
erdma_destroy_scatter_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)605 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,
606 struct erdma_mtt *mtt)
607 {
608 erdma_destroy_mtt_buf_sg(dev, mtt);
609 vfree(mtt->buf);
610 kfree(mtt);
611 }
612
erdma_init_middle_mtt(struct erdma_mtt * mtt,struct erdma_mtt * low_mtt)613 static void erdma_init_middle_mtt(struct erdma_mtt *mtt,
614 struct erdma_mtt *low_mtt)
615 {
616 struct scatterlist *sg;
617 u32 idx = 0, i;
618
619 for_each_sg(low_mtt->sglist, sg, low_mtt->nsg, i)
620 mtt->buf[idx++] = sg_dma_address(sg);
621 }
622
erdma_create_mtt_buf_sg(struct erdma_dev * dev,struct erdma_mtt * mtt)623 static int erdma_create_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt)
624 {
625 struct scatterlist *sglist;
626 void *buf = mtt->buf;
627 u32 npages, i, nsg;
628 struct page *pg;
629
630 /* Failed if buf is not page aligned */
631 if ((uintptr_t)buf & ~PAGE_MASK)
632 return -EINVAL;
633
634 npages = DIV_ROUND_UP(mtt->size, PAGE_SIZE);
635 sglist = vzalloc(npages * sizeof(*sglist));
636 if (!sglist)
637 return -ENOMEM;
638
639 sg_init_table(sglist, npages);
640 for (i = 0; i < npages; i++) {
641 pg = vmalloc_to_page(buf);
642 if (!pg)
643 goto err;
644 sg_set_page(&sglist[i], pg, PAGE_SIZE, 0);
645 buf += PAGE_SIZE;
646 }
647
648 nsg = dma_map_sg(&dev->pdev->dev, sglist, npages, DMA_TO_DEVICE);
649 if (!nsg)
650 goto err;
651
652 mtt->sglist = sglist;
653 mtt->nsg = nsg;
654
655 return 0;
656 err:
657 vfree(sglist);
658
659 return -ENOMEM;
660 }
661
erdma_create_scatter_mtt(struct erdma_dev * dev,size_t size)662 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,
663 size_t size)
664 {
665 struct erdma_mtt *mtt;
666 int ret = -ENOMEM;
667
668 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
669 if (!mtt)
670 return ERR_PTR(-ENOMEM);
671
672 mtt->size = ALIGN(size, PAGE_SIZE);
673 mtt->buf = vzalloc(mtt->size);
674 mtt->continuous = false;
675 if (!mtt->buf)
676 goto err_free_mtt;
677
678 ret = erdma_create_mtt_buf_sg(dev, mtt);
679 if (ret)
680 goto err_free_mtt_buf;
681
682 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, nsg:%u\n",
683 mtt->size, mtt->nsg);
684
685 return mtt;
686
687 err_free_mtt_buf:
688 vfree(mtt->buf);
689
690 err_free_mtt:
691 kfree(mtt);
692
693 return ERR_PTR(ret);
694 }
695
erdma_create_mtt(struct erdma_dev * dev,size_t size,bool force_continuous)696 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,
697 bool force_continuous)
698 {
699 struct erdma_mtt *mtt, *tmp_mtt;
700 int ret, level = 0;
701
702 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size,
703 force_continuous);
704
705 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA))
706 force_continuous = true;
707
708 if (force_continuous)
709 return erdma_create_cont_mtt(dev, size);
710
711 mtt = erdma_create_scatter_mtt(dev, size);
712 if (IS_ERR(mtt))
713 return mtt;
714 level = 1;
715
716 /* convergence the mtt table. */
717 while (mtt->nsg != 1 && level <= 3) {
718 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->nsg));
719 if (IS_ERR(tmp_mtt)) {
720 ret = PTR_ERR(tmp_mtt);
721 goto err_free_mtt;
722 }
723 erdma_init_middle_mtt(tmp_mtt, mtt);
724 tmp_mtt->low_level = mtt;
725 mtt = tmp_mtt;
726 level++;
727 }
728
729 if (level > 3) {
730 ret = -ENOMEM;
731 goto err_free_mtt;
732 }
733
734 mtt->level = level;
735 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",
736 mtt->level, mtt->sglist[0].dma_address);
737
738 return mtt;
739 err_free_mtt:
740 while (mtt) {
741 tmp_mtt = mtt->low_level;
742 erdma_destroy_scatter_mtt(dev, mtt);
743 mtt = tmp_mtt;
744 }
745
746 return ERR_PTR(ret);
747 }
748
erdma_destroy_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)749 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
750 {
751 struct erdma_mtt *tmp_mtt;
752
753 if (mtt->continuous) {
754 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size,
755 DMA_TO_DEVICE);
756 kfree(mtt->buf);
757 kfree(mtt);
758 } else {
759 while (mtt) {
760 tmp_mtt = mtt->low_level;
761 erdma_destroy_scatter_mtt(dev, mtt);
762 mtt = tmp_mtt;
763 }
764 }
765 }
766
get_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem,u64 start,u64 len,int access,u64 virt,unsigned long req_page_size,bool force_continuous)767 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
768 u64 start, u64 len, int access, u64 virt,
769 unsigned long req_page_size, bool force_continuous)
770 {
771 int ret = 0;
772
773 mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
774 if (IS_ERR(mem->umem)) {
775 ret = PTR_ERR(mem->umem);
776 mem->umem = NULL;
777 return ret;
778 }
779
780 mem->va = virt;
781 mem->len = len;
782 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
783 mem->page_offset = start & (mem->page_size - 1);
784 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
785 mem->page_cnt = mem->mtt_nents;
786 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
787 force_continuous);
788 if (IS_ERR(mem->mtt)) {
789 ret = PTR_ERR(mem->mtt);
790 goto error_ret;
791 }
792
793 erdma_fill_bottom_mtt(dev, mem);
794
795 return 0;
796
797 error_ret:
798 if (mem->umem) {
799 ib_umem_release(mem->umem);
800 mem->umem = NULL;
801 }
802
803 return ret;
804 }
805
put_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem)806 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
807 {
808 if (mem->mtt)
809 erdma_destroy_mtt(dev, mem->mtt);
810
811 if (mem->umem) {
812 ib_umem_release(mem->umem);
813 mem->umem = NULL;
814 }
815 }
816
erdma_map_user_dbrecords(struct erdma_ucontext * ctx,u64 dbrecords_va,struct erdma_user_dbrecords_page ** dbr_page,dma_addr_t * dma_addr)817 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
818 u64 dbrecords_va,
819 struct erdma_user_dbrecords_page **dbr_page,
820 dma_addr_t *dma_addr)
821 {
822 struct erdma_user_dbrecords_page *page = NULL;
823 int rv = 0;
824
825 mutex_lock(&ctx->dbrecords_page_mutex);
826
827 list_for_each_entry(page, &ctx->dbrecords_page_list, list)
828 if (page->va == (dbrecords_va & PAGE_MASK))
829 goto found;
830
831 page = kmalloc(sizeof(*page), GFP_KERNEL);
832 if (!page) {
833 rv = -ENOMEM;
834 goto out;
835 }
836
837 page->va = (dbrecords_va & PAGE_MASK);
838 page->refcnt = 0;
839
840 page->umem = ib_umem_get(ctx->ibucontext.device,
841 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
842 if (IS_ERR(page->umem)) {
843 rv = PTR_ERR(page->umem);
844 kfree(page);
845 goto out;
846 }
847
848 list_add(&page->list, &ctx->dbrecords_page_list);
849
850 found:
851 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
852 (dbrecords_va & ~PAGE_MASK);
853 *dbr_page = page;
854 page->refcnt++;
855
856 out:
857 mutex_unlock(&ctx->dbrecords_page_mutex);
858 return rv;
859 }
860
861 static void
erdma_unmap_user_dbrecords(struct erdma_ucontext * ctx,struct erdma_user_dbrecords_page ** dbr_page)862 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
863 struct erdma_user_dbrecords_page **dbr_page)
864 {
865 if (!ctx || !(*dbr_page))
866 return;
867
868 mutex_lock(&ctx->dbrecords_page_mutex);
869 if (--(*dbr_page)->refcnt == 0) {
870 list_del(&(*dbr_page)->list);
871 ib_umem_release((*dbr_page)->umem);
872 kfree(*dbr_page);
873 }
874
875 *dbr_page = NULL;
876 mutex_unlock(&ctx->dbrecords_page_mutex);
877 }
878
init_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx,u64 va,u32 len,u64 dbrec_va)879 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
880 u64 va, u32 len, u64 dbrec_va)
881 {
882 dma_addr_t dbrec_dma;
883 u32 rq_offset;
884 int ret;
885
886 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
887 qp->attrs.rq_size * RQE_SIZE))
888 return -EINVAL;
889
890 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
891 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
892 (SZ_1M - SZ_4K), true);
893 if (ret)
894 return ret;
895
896 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
897 qp->user_qp.rq_offset = rq_offset;
898
899 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
900 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
901 (SZ_1M - SZ_4K), true);
902 if (ret)
903 goto put_sq_mtt;
904
905 ret = erdma_map_user_dbrecords(uctx, dbrec_va,
906 &qp->user_qp.user_dbr_page,
907 &dbrec_dma);
908 if (ret)
909 goto put_rq_mtt;
910
911 qp->user_qp.sq_dbrec_dma = dbrec_dma;
912 qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
913
914 return 0;
915
916 put_rq_mtt:
917 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
918
919 put_sq_mtt:
920 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
921
922 return ret;
923 }
924
free_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx)925 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
926 {
927 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
928 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
929 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
930 }
931
erdma_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)932 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
933 struct ib_udata *udata)
934 {
935 struct erdma_qp *qp = to_eqp(ibqp);
936 struct erdma_dev *dev = to_edev(ibqp->device);
937 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
938 udata, struct erdma_ucontext, ibucontext);
939 struct erdma_ureq_create_qp ureq;
940 struct erdma_uresp_create_qp uresp;
941 int ret;
942
943 ret = erdma_qp_validate_cap(dev, attrs);
944 if (ret)
945 goto err_out;
946
947 ret = erdma_qp_validate_attr(dev, attrs);
948 if (ret)
949 goto err_out;
950
951 qp->scq = to_ecq(attrs->send_cq);
952 qp->rcq = to_ecq(attrs->recv_cq);
953 qp->dev = dev;
954 qp->attrs.cc = dev->attrs.cc;
955
956 init_rwsem(&qp->state_lock);
957 kref_init(&qp->ref);
958 init_completion(&qp->safe_free);
959
960 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
961 XA_LIMIT(1, dev->attrs.max_qp - 1),
962 &dev->next_alloc_qpn, GFP_KERNEL);
963 if (ret < 0) {
964 ret = -ENOMEM;
965 goto err_out;
966 }
967
968 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
969 ERDMA_MAX_WQEBB_PER_SQE);
970 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
971
972 if (uctx) {
973 ret = ib_copy_from_udata(&ureq, udata,
974 min(sizeof(ureq), udata->inlen));
975 if (ret)
976 goto err_out_xa;
977
978 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
979 ureq.db_record_va);
980 if (ret)
981 goto err_out_xa;
982
983 memset(&uresp, 0, sizeof(uresp));
984
985 uresp.num_sqe = qp->attrs.sq_size;
986 uresp.num_rqe = qp->attrs.rq_size;
987 uresp.qp_id = QP_ID(qp);
988 uresp.rq_offset = qp->user_qp.rq_offset;
989
990 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
991 if (ret)
992 goto err_out_cmd;
993 } else {
994 ret = init_kernel_qp(dev, qp, attrs);
995 if (ret)
996 goto err_out_xa;
997 }
998
999 qp->attrs.max_send_sge = attrs->cap.max_send_sge;
1000 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
1001 qp->attrs.state = ERDMA_QP_STATE_IDLE;
1002 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
1003
1004 ret = create_qp_cmd(uctx, qp);
1005 if (ret)
1006 goto err_out_cmd;
1007
1008 spin_lock_init(&qp->lock);
1009
1010 return 0;
1011
1012 err_out_cmd:
1013 if (uctx)
1014 free_user_qp(qp, uctx);
1015 else
1016 free_kernel_qp(qp);
1017 err_out_xa:
1018 xa_erase(&dev->qp_xa, QP_ID(qp));
1019 err_out:
1020 return ret;
1021 }
1022
erdma_create_stag(struct erdma_dev * dev,u32 * stag)1023 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
1024 {
1025 int stag_idx;
1026
1027 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
1028 if (stag_idx < 0)
1029 return stag_idx;
1030
1031 /* For now, we always let key field be zero. */
1032 *stag = (stag_idx << 8);
1033
1034 return 0;
1035 }
1036
erdma_get_dma_mr(struct ib_pd * ibpd,int acc)1037 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
1038 {
1039 struct erdma_dev *dev = to_edev(ibpd->device);
1040 struct erdma_mr *mr;
1041 u32 stag;
1042 int ret;
1043
1044 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1045 if (!mr)
1046 return ERR_PTR(-ENOMEM);
1047
1048 ret = erdma_create_stag(dev, &stag);
1049 if (ret)
1050 goto out_free;
1051
1052 mr->type = ERDMA_MR_TYPE_DMA;
1053
1054 mr->ibmr.lkey = stag;
1055 mr->ibmr.rkey = stag;
1056 mr->ibmr.pd = ibpd;
1057 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
1058 ret = regmr_cmd(dev, mr);
1059 if (ret)
1060 goto out_remove_stag;
1061
1062 return &mr->ibmr;
1063
1064 out_remove_stag:
1065 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1066 mr->ibmr.lkey >> 8);
1067
1068 out_free:
1069 kfree(mr);
1070
1071 return ERR_PTR(ret);
1072 }
1073
erdma_ib_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)1074 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
1075 u32 max_num_sg)
1076 {
1077 struct erdma_mr *mr;
1078 struct erdma_dev *dev = to_edev(ibpd->device);
1079 int ret;
1080 u32 stag;
1081
1082 if (mr_type != IB_MR_TYPE_MEM_REG)
1083 return ERR_PTR(-EOPNOTSUPP);
1084
1085 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
1086 return ERR_PTR(-EINVAL);
1087
1088 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1089 if (!mr)
1090 return ERR_PTR(-ENOMEM);
1091
1092 ret = erdma_create_stag(dev, &stag);
1093 if (ret)
1094 goto out_free;
1095
1096 mr->type = ERDMA_MR_TYPE_FRMR;
1097
1098 mr->ibmr.lkey = stag;
1099 mr->ibmr.rkey = stag;
1100 mr->ibmr.pd = ibpd;
1101 /* update it in FRMR. */
1102 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
1103 ERDMA_MR_ACC_RW;
1104
1105 mr->mem.page_size = PAGE_SIZE; /* update it later. */
1106 mr->mem.page_cnt = max_num_sg;
1107 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);
1108 if (IS_ERR(mr->mem.mtt)) {
1109 ret = PTR_ERR(mr->mem.mtt);
1110 goto out_remove_stag;
1111 }
1112
1113 ret = regmr_cmd(dev, mr);
1114 if (ret)
1115 goto out_destroy_mtt;
1116
1117 return &mr->ibmr;
1118
1119 out_destroy_mtt:
1120 erdma_destroy_mtt(dev, mr->mem.mtt);
1121
1122 out_remove_stag:
1123 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1124 mr->ibmr.lkey >> 8);
1125
1126 out_free:
1127 kfree(mr);
1128
1129 return ERR_PTR(ret);
1130 }
1131
erdma_set_page(struct ib_mr * ibmr,u64 addr)1132 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
1133 {
1134 struct erdma_mr *mr = to_emr(ibmr);
1135
1136 if (mr->mem.mtt_nents >= mr->mem.page_cnt)
1137 return -1;
1138
1139 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr;
1140 mr->mem.mtt_nents++;
1141
1142 return 0;
1143 }
1144
erdma_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)1145 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1146 unsigned int *sg_offset)
1147 {
1148 struct erdma_mr *mr = to_emr(ibmr);
1149 int num;
1150
1151 mr->mem.mtt_nents = 0;
1152
1153 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
1154 erdma_set_page);
1155
1156 return num;
1157 }
1158
erdma_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 virt,int access,struct ib_udata * udata)1159 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1160 u64 virt, int access, struct ib_udata *udata)
1161 {
1162 struct erdma_mr *mr = NULL;
1163 struct erdma_dev *dev = to_edev(ibpd->device);
1164 u32 stag;
1165 int ret;
1166
1167 if (!len || len > dev->attrs.max_mr_size)
1168 return ERR_PTR(-EINVAL);
1169
1170 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1171 if (!mr)
1172 return ERR_PTR(-ENOMEM);
1173
1174 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
1175 SZ_2G - SZ_4K, false);
1176 if (ret)
1177 goto err_out_free;
1178
1179 ret = erdma_create_stag(dev, &stag);
1180 if (ret)
1181 goto err_out_put_mtt;
1182
1183 mr->ibmr.lkey = mr->ibmr.rkey = stag;
1184 mr->ibmr.pd = ibpd;
1185 mr->mem.va = virt;
1186 mr->mem.len = len;
1187 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
1188 mr->valid = 1;
1189 mr->type = ERDMA_MR_TYPE_NORMAL;
1190
1191 ret = regmr_cmd(dev, mr);
1192 if (ret)
1193 goto err_out_mr;
1194
1195 return &mr->ibmr;
1196
1197 err_out_mr:
1198 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1199 mr->ibmr.lkey >> 8);
1200
1201 err_out_put_mtt:
1202 put_mtt_entries(dev, &mr->mem);
1203
1204 err_out_free:
1205 kfree(mr);
1206
1207 return ERR_PTR(ret);
1208 }
1209
erdma_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)1210 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1211 {
1212 struct erdma_mr *mr;
1213 struct erdma_dev *dev = to_edev(ibmr->device);
1214 struct erdma_cmdq_dereg_mr_req req;
1215 int ret;
1216
1217 mr = to_emr(ibmr);
1218
1219 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1220 CMDQ_OPCODE_DEREG_MR);
1221
1222 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
1223 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
1224
1225 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1226 if (ret)
1227 return ret;
1228
1229 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
1230
1231 put_mtt_entries(dev, &mr->mem);
1232
1233 kfree(mr);
1234 return 0;
1235 }
1236
erdma_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)1237 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1238 {
1239 struct erdma_cq *cq = to_ecq(ibcq);
1240 struct erdma_dev *dev = to_edev(ibcq->device);
1241 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1242 udata, struct erdma_ucontext, ibucontext);
1243 int err;
1244 struct erdma_cmdq_destroy_cq_req req;
1245
1246 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1247 CMDQ_OPCODE_DESTROY_CQ);
1248 req.cqn = cq->cqn;
1249
1250 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1251 if (err)
1252 return err;
1253
1254 if (rdma_is_kernel_res(&cq->ibcq.res)) {
1255 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1256 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1257 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1258 cq->kern_cq.dbrec_dma);
1259 } else {
1260 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1261 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1262 }
1263
1264 xa_erase(&dev->cq_xa, cq->cqn);
1265
1266 return 0;
1267 }
1268
erdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)1269 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1270 {
1271 struct erdma_qp *qp = to_eqp(ibqp);
1272 struct erdma_dev *dev = to_edev(ibqp->device);
1273 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1274 udata, struct erdma_ucontext, ibucontext);
1275 struct erdma_qp_attrs qp_attrs;
1276 int err;
1277 struct erdma_cmdq_destroy_qp_req req;
1278
1279 down_write(&qp->state_lock);
1280 qp_attrs.state = ERDMA_QP_STATE_ERROR;
1281 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
1282 up_write(&qp->state_lock);
1283
1284 cancel_delayed_work_sync(&qp->reflush_dwork);
1285
1286 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1287 CMDQ_OPCODE_DESTROY_QP);
1288 req.qpn = QP_ID(qp);
1289
1290 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1291 if (err)
1292 return err;
1293
1294 erdma_qp_put(qp);
1295 wait_for_completion(&qp->safe_free);
1296
1297 if (rdma_is_kernel_res(&qp->ibqp.res)) {
1298 free_kernel_qp(qp);
1299 } else {
1300 put_mtt_entries(dev, &qp->user_qp.sq_mem);
1301 put_mtt_entries(dev, &qp->user_qp.rq_mem);
1302 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1303 }
1304
1305 if (qp->cep)
1306 erdma_cep_put(qp->cep);
1307 xa_erase(&dev->qp_xa, QP_ID(qp));
1308
1309 return 0;
1310 }
1311
erdma_qp_get_ref(struct ib_qp * ibqp)1312 void erdma_qp_get_ref(struct ib_qp *ibqp)
1313 {
1314 erdma_qp_get(to_eqp(ibqp));
1315 }
1316
erdma_qp_put_ref(struct ib_qp * ibqp)1317 void erdma_qp_put_ref(struct ib_qp *ibqp)
1318 {
1319 erdma_qp_put(to_eqp(ibqp));
1320 }
1321
erdma_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)1322 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1323 {
1324 struct rdma_user_mmap_entry *rdma_entry;
1325 struct erdma_user_mmap_entry *entry;
1326 pgprot_t prot;
1327 int err;
1328
1329 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1330 if (!rdma_entry)
1331 return -EINVAL;
1332
1333 entry = to_emmap(rdma_entry);
1334
1335 switch (entry->mmap_flag) {
1336 case ERDMA_MMAP_IO_NC:
1337 /* map doorbell. */
1338 prot = pgprot_device(vma->vm_page_prot);
1339 break;
1340 default:
1341 err = -EINVAL;
1342 goto put_entry;
1343 }
1344
1345 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1346 prot, rdma_entry);
1347
1348 put_entry:
1349 rdma_user_mmap_entry_put(rdma_entry);
1350 return err;
1351 }
1352
erdma_mmap_free(struct rdma_user_mmap_entry * rdma_entry)1353 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1354 {
1355 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1356
1357 kfree(entry);
1358 }
1359
alloc_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx,bool ext_db_en)1360 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
1361 bool ext_db_en)
1362 {
1363 struct erdma_cmdq_ext_db_req req = {};
1364 u64 val0, val1;
1365 int ret;
1366
1367 /*
1368 * CAP_SYS_RAWIO is required if hardware does not support extend
1369 * doorbell mechanism.
1370 */
1371 if (!ext_db_en && !capable(CAP_SYS_RAWIO))
1372 return -EPERM;
1373
1374 if (!ext_db_en) {
1375 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;
1376 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1377 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1378 return 0;
1379 }
1380
1381 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1382 CMDQ_OPCODE_ALLOC_DB);
1383
1384 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1385 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1386 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1387
1388 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1);
1389 if (ret)
1390 return ret;
1391
1392 ctx->ext_db.enable = true;
1393 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);
1394 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);
1395 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);
1396
1397 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);
1398 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);
1399 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);
1400
1401 return 0;
1402 }
1403
free_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx)1404 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
1405 {
1406 struct erdma_cmdq_ext_db_req req = {};
1407 int ret;
1408
1409 if (!ctx->ext_db.enable)
1410 return;
1411
1412 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1413 CMDQ_OPCODE_FREE_DB);
1414
1415 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1416 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1417 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1418
1419 req.sdb_off = ctx->ext_db.sdb_off;
1420 req.rdb_off = ctx->ext_db.rdb_off;
1421 req.cdb_off = ctx->ext_db.cdb_off;
1422
1423 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1424 if (ret)
1425 ibdev_err_ratelimited(&dev->ibdev,
1426 "free db resources failed %d", ret);
1427 }
1428
erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext * uctx)1429 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1430 {
1431 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1432 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1433 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1434 }
1435
erdma_alloc_ucontext(struct ib_ucontext * ibctx,struct ib_udata * udata)1436 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1437 {
1438 struct erdma_ucontext *ctx = to_ectx(ibctx);
1439 struct erdma_dev *dev = to_edev(ibctx->device);
1440 int ret;
1441 struct erdma_uresp_alloc_ctx uresp = {};
1442
1443 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1444 ret = -ENOMEM;
1445 goto err_out;
1446 }
1447
1448 if (udata->outlen < sizeof(uresp)) {
1449 ret = -EINVAL;
1450 goto err_out;
1451 }
1452
1453 INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1454 mutex_init(&ctx->dbrecords_page_mutex);
1455
1456 ret = alloc_db_resources(dev, ctx,
1457 !!(dev->attrs.cap_flags &
1458 ERDMA_DEV_CAP_FLAGS_EXTEND_DB));
1459 if (ret)
1460 goto err_out;
1461
1462 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1463 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1464 if (!ctx->sq_db_mmap_entry) {
1465 ret = -ENOMEM;
1466 goto err_free_ext_db;
1467 }
1468
1469 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1470 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1471 if (!ctx->rq_db_mmap_entry) {
1472 ret = -EINVAL;
1473 goto err_put_mmap_entries;
1474 }
1475
1476 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1477 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1478 if (!ctx->cq_db_mmap_entry) {
1479 ret = -EINVAL;
1480 goto err_put_mmap_entries;
1481 }
1482
1483 uresp.dev_id = dev->pdev->device;
1484
1485 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1486 if (ret)
1487 goto err_put_mmap_entries;
1488
1489 return 0;
1490
1491 err_put_mmap_entries:
1492 erdma_uctx_user_mmap_entries_remove(ctx);
1493
1494 err_free_ext_db:
1495 free_db_resources(dev, ctx);
1496
1497 err_out:
1498 atomic_dec(&dev->num_ctx);
1499 return ret;
1500 }
1501
erdma_dealloc_ucontext(struct ib_ucontext * ibctx)1502 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1503 {
1504 struct erdma_dev *dev = to_edev(ibctx->device);
1505 struct erdma_ucontext *ctx = to_ectx(ibctx);
1506
1507 erdma_uctx_user_mmap_entries_remove(ctx);
1508 free_db_resources(dev, ctx);
1509 atomic_dec(&dev->num_ctx);
1510 }
1511
1512 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = {
1513 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE,
1514 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE,
1515 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR,
1516 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS,
1517 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING,
1518 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE,
1519 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR
1520 };
1521
erdma_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1522 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1523 struct ib_udata *udata)
1524 {
1525 struct erdma_qp_attrs new_attrs;
1526 enum erdma_qp_attr_mask erdma_attr_mask = 0;
1527 struct erdma_qp *qp = to_eqp(ibqp);
1528 int ret = 0;
1529
1530 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1531 return -EOPNOTSUPP;
1532
1533 memset(&new_attrs, 0, sizeof(new_attrs));
1534
1535 if (attr_mask & IB_QP_STATE) {
1536 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state];
1537
1538 erdma_attr_mask |= ERDMA_QP_ATTR_STATE;
1539 }
1540
1541 down_write(&qp->state_lock);
1542
1543 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask);
1544
1545 up_write(&qp->state_lock);
1546
1547 return ret;
1548 }
1549
query_qp_state(struct erdma_qp * qp)1550 static enum ib_qp_state query_qp_state(struct erdma_qp *qp)
1551 {
1552 switch (qp->attrs.state) {
1553 case ERDMA_QP_STATE_IDLE:
1554 return IB_QPS_INIT;
1555 case ERDMA_QP_STATE_RTR:
1556 return IB_QPS_RTR;
1557 case ERDMA_QP_STATE_RTS:
1558 return IB_QPS_RTS;
1559 case ERDMA_QP_STATE_CLOSING:
1560 return IB_QPS_ERR;
1561 case ERDMA_QP_STATE_TERMINATE:
1562 return IB_QPS_ERR;
1563 case ERDMA_QP_STATE_ERROR:
1564 return IB_QPS_ERR;
1565 default:
1566 return IB_QPS_ERR;
1567 }
1568 }
1569
erdma_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1570 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1571 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1572 {
1573 struct erdma_dev *dev;
1574 struct erdma_qp *qp;
1575
1576 if (ibqp && qp_attr && qp_init_attr) {
1577 qp = to_eqp(ibqp);
1578 dev = to_edev(ibqp->device);
1579 } else {
1580 return -EINVAL;
1581 }
1582
1583 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1584 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1585
1586 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1587 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1588 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1589 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1590
1591 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1592 qp_attr->max_rd_atomic = qp->attrs.irq_size;
1593 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1594
1595 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1596 IB_ACCESS_REMOTE_WRITE |
1597 IB_ACCESS_REMOTE_READ;
1598
1599 qp_init_attr->cap = qp_attr->cap;
1600
1601 qp_attr->qp_state = query_qp_state(qp);
1602 qp_attr->cur_qp_state = query_qp_state(qp);
1603
1604 return 0;
1605 }
1606
erdma_init_user_cq(struct erdma_ucontext * ctx,struct erdma_cq * cq,struct erdma_ureq_create_cq * ureq)1607 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1608 struct erdma_ureq_create_cq *ureq)
1609 {
1610 int ret;
1611 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1612
1613 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
1614 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1615 true);
1616 if (ret)
1617 return ret;
1618
1619 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1620 &cq->user_cq.user_dbr_page,
1621 &cq->user_cq.dbrec_dma);
1622 if (ret)
1623 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1624
1625 return ret;
1626 }
1627
erdma_init_kernel_cq(struct erdma_cq * cq)1628 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1629 {
1630 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1631
1632 cq->kern_cq.qbuf =
1633 dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1634 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1635 if (!cq->kern_cq.qbuf)
1636 return -ENOMEM;
1637
1638 cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
1639 &cq->kern_cq.dbrec_dma);
1640 if (!cq->kern_cq.dbrec)
1641 goto err_out;
1642
1643 spin_lock_init(&cq->kern_cq.lock);
1644 /* use default cqdb addr */
1645 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1646
1647 return 0;
1648
1649 err_out:
1650 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1651 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1652
1653 return -ENOMEM;
1654 }
1655
erdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)1656 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1657 struct uverbs_attr_bundle *attrs)
1658 {
1659 struct ib_udata *udata = &attrs->driver_udata;
1660 struct erdma_cq *cq = to_ecq(ibcq);
1661 struct erdma_dev *dev = to_edev(ibcq->device);
1662 unsigned int depth = attr->cqe;
1663 int ret;
1664 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1665 udata, struct erdma_ucontext, ibucontext);
1666
1667 if (depth > dev->attrs.max_cqe)
1668 return -EINVAL;
1669
1670 depth = roundup_pow_of_two(depth);
1671 cq->ibcq.cqe = depth;
1672 cq->depth = depth;
1673 cq->assoc_eqn = attr->comp_vector + 1;
1674
1675 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1676 XA_LIMIT(1, dev->attrs.max_cq - 1),
1677 &dev->next_alloc_cqn, GFP_KERNEL);
1678 if (ret < 0)
1679 return ret;
1680
1681 if (!rdma_is_kernel_res(&ibcq->res)) {
1682 struct erdma_ureq_create_cq ureq;
1683 struct erdma_uresp_create_cq uresp;
1684
1685 ret = ib_copy_from_udata(&ureq, udata,
1686 min(udata->inlen, sizeof(ureq)));
1687 if (ret)
1688 goto err_out_xa;
1689
1690 ret = erdma_init_user_cq(ctx, cq, &ureq);
1691 if (ret)
1692 goto err_out_xa;
1693
1694 uresp.cq_id = cq->cqn;
1695 uresp.num_cqe = depth;
1696
1697 ret = ib_copy_to_udata(udata, &uresp,
1698 min(sizeof(uresp), udata->outlen));
1699 if (ret)
1700 goto err_free_res;
1701 } else {
1702 ret = erdma_init_kernel_cq(cq);
1703 if (ret)
1704 goto err_out_xa;
1705 }
1706
1707 ret = create_cq_cmd(ctx, cq);
1708 if (ret)
1709 goto err_free_res;
1710
1711 return 0;
1712
1713 err_free_res:
1714 if (!rdma_is_kernel_res(&ibcq->res)) {
1715 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1716 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1717 } else {
1718 dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
1719 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1720 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1721 cq->kern_cq.dbrec_dma);
1722 }
1723
1724 err_out_xa:
1725 xa_erase(&dev->cq_xa, cq->cqn);
1726
1727 return ret;
1728 }
1729
erdma_disassociate_ucontext(struct ib_ucontext * ibcontext)1730 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)
1731 {
1732 }
1733
erdma_set_mtu(struct erdma_dev * dev,u32 mtu)1734 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
1735 {
1736 struct erdma_cmdq_config_mtu_req req;
1737
1738 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1739 CMDQ_OPCODE_CONF_MTU);
1740 req.mtu = mtu;
1741
1742 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1743 }
1744
erdma_port_event(struct erdma_dev * dev,enum ib_event_type reason)1745 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
1746 {
1747 struct ib_event event;
1748
1749 event.device = &dev->ibdev;
1750 event.element.port_num = 1;
1751 event.event = reason;
1752
1753 ib_dispatch_event(&event);
1754 }
1755
1756 enum counters {
1757 ERDMA_STATS_TX_REQS_CNT,
1758 ERDMA_STATS_TX_PACKETS_CNT,
1759 ERDMA_STATS_TX_BYTES_CNT,
1760 ERDMA_STATS_TX_DISABLE_DROP_CNT,
1761 ERDMA_STATS_TX_BPS_METER_DROP_CNT,
1762 ERDMA_STATS_TX_PPS_METER_DROP_CNT,
1763
1764 ERDMA_STATS_RX_PACKETS_CNT,
1765 ERDMA_STATS_RX_BYTES_CNT,
1766 ERDMA_STATS_RX_DISABLE_DROP_CNT,
1767 ERDMA_STATS_RX_BPS_METER_DROP_CNT,
1768 ERDMA_STATS_RX_PPS_METER_DROP_CNT,
1769
1770 ERDMA_STATS_MAX
1771 };
1772
1773 static const struct rdma_stat_desc erdma_descs[] = {
1774 [ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt",
1775 [ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt",
1776 [ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt",
1777 [ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt",
1778 [ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt",
1779 [ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt",
1780 [ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt",
1781 [ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt",
1782 [ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt",
1783 [ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt",
1784 [ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt",
1785 };
1786
erdma_alloc_hw_port_stats(struct ib_device * device,u32 port_num)1787 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
1788 u32 port_num)
1789 {
1790 return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX,
1791 RDMA_HW_STATS_DEFAULT_LIFESPAN);
1792 }
1793
erdma_query_hw_stats(struct erdma_dev * dev,struct rdma_hw_stats * stats)1794 static int erdma_query_hw_stats(struct erdma_dev *dev,
1795 struct rdma_hw_stats *stats)
1796 {
1797 struct erdma_cmdq_query_stats_resp *resp;
1798 struct erdma_cmdq_query_req req;
1799 dma_addr_t dma_addr;
1800 int err;
1801
1802 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1803 CMDQ_OPCODE_GET_STATS);
1804
1805 resp = dma_pool_zalloc(dev->resp_pool, GFP_KERNEL, &dma_addr);
1806 if (!resp)
1807 return -ENOMEM;
1808
1809 req.target_addr = dma_addr;
1810 req.target_length = ERDMA_HW_RESP_SIZE;
1811
1812 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1813 if (err)
1814 goto out;
1815
1816 if (resp->hdr.magic != ERDMA_HW_RESP_MAGIC) {
1817 err = -EINVAL;
1818 goto out;
1819 }
1820
1821 memcpy(&stats->value[0], &resp->tx_req_cnt,
1822 sizeof(u64) * stats->num_counters);
1823
1824 out:
1825 dma_pool_free(dev->resp_pool, resp, dma_addr);
1826
1827 return err;
1828 }
1829
erdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port,int index)1830 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
1831 u32 port, int index)
1832 {
1833 struct erdma_dev *dev = to_edev(ibdev);
1834 int ret;
1835
1836 if (port == 0)
1837 return 0;
1838
1839 ret = erdma_query_hw_stats(dev, stats);
1840 if (ret)
1841 return ret;
1842
1843 return stats->num_counters;
1844 }
1845