1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/ib_umem.h>
16 #include <rdma/uverbs_ioctl.h>
17
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21
create_qp_cmd(struct erdma_dev * dev,struct erdma_qp * qp)22 static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp)
23 {
24 struct erdma_cmdq_create_qp_req req;
25 struct erdma_pd *pd = to_epd(qp->ibqp.pd);
26 struct erdma_uqp *user_qp;
27 u64 resp0, resp1;
28 int err;
29
30 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
31 CMDQ_OPCODE_CREATE_QP);
32
33 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
34 ilog2(qp->attrs.sq_size)) |
35 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
36 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
37 ilog2(qp->attrs.rq_size)) |
38 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
39
40 if (rdma_is_kernel_res(&qp->ibqp.res)) {
41 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
42
43 req.sq_cqn_mtt_cfg =
44 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
45 pgsz_range) |
46 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
47 req.rq_cqn_mtt_cfg =
48 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
49 pgsz_range) |
50 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
51
52 req.sq_mtt_cfg =
53 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
54 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
55 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
56 ERDMA_MR_INLINE_MTT);
57 req.rq_mtt_cfg = req.sq_mtt_cfg;
58
59 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
60 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
61 req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr +
62 (qp->attrs.sq_size << SQEBB_SHIFT);
63 req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr +
64 (qp->attrs.rq_size << RQE_SHIFT);
65 } else {
66 user_qp = &qp->user_qp;
67 req.sq_cqn_mtt_cfg = FIELD_PREP(
68 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
69 ilog2(user_qp->sq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
70 req.sq_cqn_mtt_cfg |=
71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
72
73 req.rq_cqn_mtt_cfg = FIELD_PREP(
74 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
75 ilog2(user_qp->rq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
76 req.rq_cqn_mtt_cfg |=
77 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
78
79 req.sq_mtt_cfg = user_qp->sq_mtt.page_offset;
80 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
81 user_qp->sq_mtt.mtt_nents) |
82 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
83 user_qp->sq_mtt.mtt_type);
84
85 req.rq_mtt_cfg = user_qp->rq_mtt.page_offset;
86 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
87 user_qp->rq_mtt.mtt_nents) |
88 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
89 user_qp->rq_mtt.mtt_type);
90
91 req.sq_buf_addr = user_qp->sq_mtt.mtt_entry[0];
92 req.rq_buf_addr = user_qp->rq_mtt.mtt_entry[0];
93
94 req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr;
95 req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr;
96 }
97
98 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
99 &resp1);
100 if (!err)
101 qp->attrs.cookie =
102 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
103
104 return err;
105 }
106
regmr_cmd(struct erdma_dev * dev,struct erdma_mr * mr)107 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
108 {
109 struct erdma_cmdq_reg_mr_req req;
110 struct erdma_pd *pd = to_epd(mr->ibmr.pd);
111 u64 *phy_addr;
112 int i;
113
114 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
115
116 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
117 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
118 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
119 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
120 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
121 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access) |
122 FIELD_PREP(ERDMA_CMD_REGMR_ACC_MODE_MASK, 0);
123 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
124 ilog2(mr->mem.page_size)) |
125 FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) |
126 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
127
128 if (mr->type == ERDMA_MR_TYPE_DMA)
129 goto post_cmd;
130
131 if (mr->type == ERDMA_MR_TYPE_NORMAL) {
132 req.start_va = mr->mem.va;
133 req.size = mr->mem.len;
134 }
135
136 if (mr->type == ERDMA_MR_TYPE_FRMR ||
137 mr->mem.mtt_type == ERDMA_MR_INDIRECT_MTT) {
138 phy_addr = req.phy_addr;
139 *phy_addr = mr->mem.mtt_entry[0];
140 } else {
141 phy_addr = req.phy_addr;
142 for (i = 0; i < mr->mem.mtt_nents; i++)
143 *phy_addr++ = mr->mem.mtt_entry[i];
144 }
145
146 post_cmd:
147 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
148 }
149
create_cq_cmd(struct erdma_dev * dev,struct erdma_cq * cq)150 static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq)
151 {
152 struct erdma_cmdq_create_cq_req req;
153 u32 page_size;
154 struct erdma_mem *mtt;
155
156 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
157 CMDQ_OPCODE_CREATE_CQ);
158
159 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
160 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
161 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
162
163 if (rdma_is_kernel_res(&cq->ibcq.res)) {
164 page_size = SZ_32M;
165 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
166 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
167 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
168 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
169
170 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
171 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK,
172 ERDMA_MR_INLINE_MTT);
173
174 req.first_page_offset = 0;
175 req.cq_db_info_addr =
176 cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT);
177 } else {
178 mtt = &cq->user_cq.qbuf_mtt;
179 req.cfg0 |=
180 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
181 ilog2(mtt->page_size) - ERDMA_HW_PAGE_SHIFT);
182 if (mtt->mtt_nents == 1) {
183 req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf);
184 req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf);
185 } else {
186 req.qbuf_addr_l = lower_32_bits(mtt->mtt_entry[0]);
187 req.qbuf_addr_h = upper_32_bits(mtt->mtt_entry[0]);
188 }
189 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
190 mtt->mtt_nents);
191 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK,
192 mtt->mtt_type);
193
194 req.first_page_offset = mtt->page_offset;
195 req.cq_db_info_addr = cq->user_cq.db_info_dma_addr;
196 }
197
198 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
199 }
200
erdma_alloc_idx(struct erdma_resource_cb * res_cb)201 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
202 {
203 int idx;
204 unsigned long flags;
205
206 spin_lock_irqsave(&res_cb->lock, flags);
207 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
208 res_cb->next_alloc_idx);
209 if (idx == res_cb->max_cap) {
210 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
211 if (idx == res_cb->max_cap) {
212 res_cb->next_alloc_idx = 1;
213 spin_unlock_irqrestore(&res_cb->lock, flags);
214 return -ENOSPC;
215 }
216 }
217
218 set_bit(idx, res_cb->bitmap);
219 res_cb->next_alloc_idx = idx + 1;
220 spin_unlock_irqrestore(&res_cb->lock, flags);
221
222 return idx;
223 }
224
erdma_free_idx(struct erdma_resource_cb * res_cb,u32 idx)225 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
226 {
227 unsigned long flags;
228 u32 used;
229
230 spin_lock_irqsave(&res_cb->lock, flags);
231 used = __test_and_clear_bit(idx, res_cb->bitmap);
232 spin_unlock_irqrestore(&res_cb->lock, flags);
233 WARN_ON(!used);
234 }
235
236 static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext * uctx,void * address,u32 size,u8 mmap_flag,u64 * mmap_offset)237 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
238 u32 size, u8 mmap_flag, u64 *mmap_offset)
239 {
240 struct erdma_user_mmap_entry *entry =
241 kzalloc(sizeof(*entry), GFP_KERNEL);
242 int ret;
243
244 if (!entry)
245 return NULL;
246
247 entry->address = (u64)address;
248 entry->mmap_flag = mmap_flag;
249
250 size = PAGE_ALIGN(size);
251
252 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
253 size);
254 if (ret) {
255 kfree(entry);
256 return NULL;
257 }
258
259 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
260
261 return &entry->rdma_entry;
262 }
263
erdma_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * unused)264 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
265 struct ib_udata *unused)
266 {
267 struct erdma_dev *dev = to_edev(ibdev);
268
269 memset(attr, 0, sizeof(*attr));
270
271 attr->max_mr_size = dev->attrs.max_mr_size;
272 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
273 attr->vendor_part_id = dev->pdev->device;
274 attr->hw_ver = dev->pdev->revision;
275 attr->max_qp = dev->attrs.max_qp - 1;
276 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
277 attr->max_qp_rd_atom = dev->attrs.max_ord;
278 attr->max_qp_init_rd_atom = dev->attrs.max_ird;
279 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
280 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
281 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
282 ibdev->local_dma_lkey = dev->attrs.local_dma_key;
283 attr->max_send_sge = dev->attrs.max_send_sge;
284 attr->max_recv_sge = dev->attrs.max_recv_sge;
285 attr->max_sge_rd = dev->attrs.max_sge_rd;
286 attr->max_cq = dev->attrs.max_cq - 1;
287 attr->max_cqe = dev->attrs.max_cqe;
288 attr->max_mr = dev->attrs.max_mr;
289 attr->max_pd = dev->attrs.max_pd;
290 attr->max_mw = dev->attrs.max_mw;
291 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
292 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
293 attr->fw_ver = dev->attrs.fw_version;
294
295 if (dev->netdev)
296 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
297 dev->netdev->dev_addr);
298
299 return 0;
300 }
301
erdma_query_gid(struct ib_device * ibdev,u32 port,int idx,union ib_gid * gid)302 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
303 union ib_gid *gid)
304 {
305 struct erdma_dev *dev = to_edev(ibdev);
306
307 memset(gid, 0, sizeof(*gid));
308 ether_addr_copy(gid->raw, dev->attrs.peer_addr);
309
310 return 0;
311 }
312
erdma_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)313 int erdma_query_port(struct ib_device *ibdev, u32 port,
314 struct ib_port_attr *attr)
315 {
316 struct erdma_dev *dev = to_edev(ibdev);
317 struct net_device *ndev = dev->netdev;
318
319 memset(attr, 0, sizeof(*attr));
320
321 attr->gid_tbl_len = 1;
322 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
323 attr->max_msg_sz = -1;
324
325 if (!ndev)
326 goto out;
327
328 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
329 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
330 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
331 if (netif_running(ndev) && netif_carrier_ok(ndev))
332 dev->state = IB_PORT_ACTIVE;
333 else
334 dev->state = IB_PORT_DOWN;
335 attr->state = dev->state;
336
337 out:
338 if (dev->state == IB_PORT_ACTIVE)
339 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
340 else
341 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
342
343 return 0;
344 }
345
erdma_get_port_immutable(struct ib_device * ibdev,u32 port,struct ib_port_immutable * port_immutable)346 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
347 struct ib_port_immutable *port_immutable)
348 {
349 port_immutable->gid_tbl_len = 1;
350 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
351
352 return 0;
353 }
354
erdma_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)355 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
356 {
357 struct erdma_pd *pd = to_epd(ibpd);
358 struct erdma_dev *dev = to_edev(ibpd->device);
359 int pdn;
360
361 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
362 if (pdn < 0)
363 return pdn;
364
365 pd->pdn = pdn;
366
367 return 0;
368 }
369
erdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)370 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
371 {
372 struct erdma_pd *pd = to_epd(ibpd);
373 struct erdma_dev *dev = to_edev(ibpd->device);
374
375 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
376
377 return 0;
378 }
379
erdma_qp_validate_cap(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)380 static int erdma_qp_validate_cap(struct erdma_dev *dev,
381 struct ib_qp_init_attr *attrs)
382 {
383 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
384 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
385 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
386 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
387 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
388 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
389 return -EINVAL;
390 }
391
392 return 0;
393 }
394
erdma_qp_validate_attr(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)395 static int erdma_qp_validate_attr(struct erdma_dev *dev,
396 struct ib_qp_init_attr *attrs)
397 {
398 if (attrs->qp_type != IB_QPT_RC)
399 return -EOPNOTSUPP;
400
401 if (attrs->srq)
402 return -EOPNOTSUPP;
403
404 if (!attrs->send_cq || !attrs->recv_cq)
405 return -EOPNOTSUPP;
406
407 return 0;
408 }
409
free_kernel_qp(struct erdma_qp * qp)410 static void free_kernel_qp(struct erdma_qp *qp)
411 {
412 struct erdma_dev *dev = qp->dev;
413
414 vfree(qp->kern_qp.swr_tbl);
415 vfree(qp->kern_qp.rwr_tbl);
416
417 if (qp->kern_qp.sq_buf)
418 dma_free_coherent(
419 &dev->pdev->dev,
420 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
421 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
422
423 if (qp->kern_qp.rq_buf)
424 dma_free_coherent(
425 &dev->pdev->dev,
426 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
427 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
428 }
429
init_kernel_qp(struct erdma_dev * dev,struct erdma_qp * qp,struct ib_qp_init_attr * attrs)430 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
431 struct ib_qp_init_attr *attrs)
432 {
433 struct erdma_kqp *kqp = &qp->kern_qp;
434 int size;
435
436 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
437 kqp->sig_all = 1;
438
439 kqp->sq_pi = 0;
440 kqp->sq_ci = 0;
441 kqp->rq_pi = 0;
442 kqp->rq_ci = 0;
443 kqp->hw_sq_db =
444 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
445 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
446
447 kqp->swr_tbl = vmalloc(qp->attrs.sq_size * sizeof(u64));
448 kqp->rwr_tbl = vmalloc(qp->attrs.rq_size * sizeof(u64));
449 if (!kqp->swr_tbl || !kqp->rwr_tbl)
450 goto err_out;
451
452 size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
453 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
454 &kqp->sq_buf_dma_addr, GFP_KERNEL);
455 if (!kqp->sq_buf)
456 goto err_out;
457
458 size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
459 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
460 &kqp->rq_buf_dma_addr, GFP_KERNEL);
461 if (!kqp->rq_buf)
462 goto err_out;
463
464 kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT);
465 kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT);
466
467 return 0;
468
469 err_out:
470 free_kernel_qp(qp);
471 return -ENOMEM;
472 }
473
get_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem,u64 start,u64 len,int access,u64 virt,unsigned long req_page_size,u8 force_indirect_mtt)474 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
475 u64 start, u64 len, int access, u64 virt,
476 unsigned long req_page_size, u8 force_indirect_mtt)
477 {
478 struct ib_block_iter biter;
479 uint64_t *phy_addr = NULL;
480 int ret = 0;
481
482 mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
483 if (IS_ERR(mem->umem)) {
484 ret = PTR_ERR(mem->umem);
485 mem->umem = NULL;
486 return ret;
487 }
488
489 mem->va = virt;
490 mem->len = len;
491 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
492 mem->page_offset = start & (mem->page_size - 1);
493 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
494 mem->page_cnt = mem->mtt_nents;
495
496 if (mem->page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES ||
497 force_indirect_mtt) {
498 mem->mtt_type = ERDMA_MR_INDIRECT_MTT;
499 mem->mtt_buf =
500 alloc_pages_exact(MTT_SIZE(mem->page_cnt), GFP_KERNEL);
501 if (!mem->mtt_buf) {
502 ret = -ENOMEM;
503 goto error_ret;
504 }
505 phy_addr = mem->mtt_buf;
506 } else {
507 mem->mtt_type = ERDMA_MR_INLINE_MTT;
508 phy_addr = mem->mtt_entry;
509 }
510
511 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) {
512 *phy_addr = rdma_block_iter_dma_address(&biter);
513 phy_addr++;
514 }
515
516 if (mem->mtt_type == ERDMA_MR_INDIRECT_MTT) {
517 mem->mtt_entry[0] =
518 dma_map_single(&dev->pdev->dev, mem->mtt_buf,
519 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE);
520 if (dma_mapping_error(&dev->pdev->dev, mem->mtt_entry[0])) {
521 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt));
522 mem->mtt_buf = NULL;
523 ret = -ENOMEM;
524 goto error_ret;
525 }
526 }
527
528 return 0;
529
530 error_ret:
531 if (mem->umem) {
532 ib_umem_release(mem->umem);
533 mem->umem = NULL;
534 }
535
536 return ret;
537 }
538
put_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem)539 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
540 {
541 if (mem->mtt_buf) {
542 dma_unmap_single(&dev->pdev->dev, mem->mtt_entry[0],
543 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE);
544 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt));
545 }
546
547 if (mem->umem) {
548 ib_umem_release(mem->umem);
549 mem->umem = NULL;
550 }
551 }
552
erdma_map_user_dbrecords(struct erdma_ucontext * ctx,u64 dbrecords_va,struct erdma_user_dbrecords_page ** dbr_page,dma_addr_t * dma_addr)553 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
554 u64 dbrecords_va,
555 struct erdma_user_dbrecords_page **dbr_page,
556 dma_addr_t *dma_addr)
557 {
558 struct erdma_user_dbrecords_page *page = NULL;
559 int rv = 0;
560
561 mutex_lock(&ctx->dbrecords_page_mutex);
562
563 list_for_each_entry(page, &ctx->dbrecords_page_list, list)
564 if (page->va == (dbrecords_va & PAGE_MASK))
565 goto found;
566
567 page = kmalloc(sizeof(*page), GFP_KERNEL);
568 if (!page) {
569 rv = -ENOMEM;
570 goto out;
571 }
572
573 page->va = (dbrecords_va & PAGE_MASK);
574 page->refcnt = 0;
575
576 page->umem = ib_umem_get(ctx->ibucontext.device,
577 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
578 if (IS_ERR(page->umem)) {
579 rv = PTR_ERR(page->umem);
580 kfree(page);
581 goto out;
582 }
583
584 list_add(&page->list, &ctx->dbrecords_page_list);
585
586 found:
587 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
588 (dbrecords_va & ~PAGE_MASK);
589 *dbr_page = page;
590 page->refcnt++;
591
592 out:
593 mutex_unlock(&ctx->dbrecords_page_mutex);
594 return rv;
595 }
596
597 static void
erdma_unmap_user_dbrecords(struct erdma_ucontext * ctx,struct erdma_user_dbrecords_page ** dbr_page)598 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
599 struct erdma_user_dbrecords_page **dbr_page)
600 {
601 if (!ctx || !(*dbr_page))
602 return;
603
604 mutex_lock(&ctx->dbrecords_page_mutex);
605 if (--(*dbr_page)->refcnt == 0) {
606 list_del(&(*dbr_page)->list);
607 ib_umem_release((*dbr_page)->umem);
608 kfree(*dbr_page);
609 }
610
611 *dbr_page = NULL;
612 mutex_unlock(&ctx->dbrecords_page_mutex);
613 }
614
init_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx,u64 va,u32 len,u64 db_info_va)615 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
616 u64 va, u32 len, u64 db_info_va)
617 {
618 dma_addr_t db_info_dma_addr;
619 u32 rq_offset;
620 int ret;
621
622 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
623 qp->attrs.rq_size * RQE_SIZE))
624 return -EINVAL;
625
626 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mtt, va,
627 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
628 (SZ_1M - SZ_4K), 1);
629 if (ret)
630 return ret;
631
632 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
633 qp->user_qp.rq_offset = rq_offset;
634
635 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset,
636 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
637 (SZ_1M - SZ_4K), 1);
638 if (ret)
639 goto put_sq_mtt;
640
641 ret = erdma_map_user_dbrecords(uctx, db_info_va,
642 &qp->user_qp.user_dbr_page,
643 &db_info_dma_addr);
644 if (ret)
645 goto put_rq_mtt;
646
647 qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr;
648 qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE;
649
650 return 0;
651
652 put_rq_mtt:
653 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt);
654
655 put_sq_mtt:
656 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt);
657
658 return ret;
659 }
660
free_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx)661 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
662 {
663 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt);
664 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt);
665 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
666 }
667
erdma_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)668 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
669 struct ib_udata *udata)
670 {
671 struct erdma_qp *qp = to_eqp(ibqp);
672 struct erdma_dev *dev = to_edev(ibqp->device);
673 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
674 udata, struct erdma_ucontext, ibucontext);
675 struct erdma_ureq_create_qp ureq;
676 struct erdma_uresp_create_qp uresp;
677 int ret;
678
679 ret = erdma_qp_validate_cap(dev, attrs);
680 if (ret)
681 goto err_out;
682
683 ret = erdma_qp_validate_attr(dev, attrs);
684 if (ret)
685 goto err_out;
686
687 qp->scq = to_ecq(attrs->send_cq);
688 qp->rcq = to_ecq(attrs->recv_cq);
689 qp->dev = dev;
690 qp->attrs.cc = dev->attrs.cc;
691
692 init_rwsem(&qp->state_lock);
693 kref_init(&qp->ref);
694 init_completion(&qp->safe_free);
695
696 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
697 XA_LIMIT(1, dev->attrs.max_qp - 1),
698 &dev->next_alloc_qpn, GFP_KERNEL);
699 if (ret < 0) {
700 ret = -ENOMEM;
701 goto err_out;
702 }
703
704 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
705 ERDMA_MAX_WQEBB_PER_SQE);
706 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
707
708 if (uctx) {
709 ret = ib_copy_from_udata(&ureq, udata,
710 min(sizeof(ureq), udata->inlen));
711 if (ret)
712 goto err_out_xa;
713
714 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
715 ureq.db_record_va);
716 if (ret)
717 goto err_out_xa;
718
719 memset(&uresp, 0, sizeof(uresp));
720
721 uresp.num_sqe = qp->attrs.sq_size;
722 uresp.num_rqe = qp->attrs.rq_size;
723 uresp.qp_id = QP_ID(qp);
724 uresp.rq_offset = qp->user_qp.rq_offset;
725
726 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
727 if (ret)
728 goto err_out_cmd;
729 } else {
730 init_kernel_qp(dev, qp, attrs);
731 }
732
733 qp->attrs.max_send_sge = attrs->cap.max_send_sge;
734 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
735 qp->attrs.state = ERDMA_QP_STATE_IDLE;
736
737 ret = create_qp_cmd(dev, qp);
738 if (ret)
739 goto err_out_cmd;
740
741 spin_lock_init(&qp->lock);
742
743 return 0;
744
745 err_out_cmd:
746 if (uctx)
747 free_user_qp(qp, uctx);
748 else
749 free_kernel_qp(qp);
750 err_out_xa:
751 xa_erase(&dev->qp_xa, QP_ID(qp));
752 err_out:
753 return ret;
754 }
755
erdma_create_stag(struct erdma_dev * dev,u32 * stag)756 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
757 {
758 int stag_idx;
759
760 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
761 if (stag_idx < 0)
762 return stag_idx;
763
764 /* For now, we always let key field be zero. */
765 *stag = (stag_idx << 8);
766
767 return 0;
768 }
769
erdma_get_dma_mr(struct ib_pd * ibpd,int acc)770 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
771 {
772 struct erdma_dev *dev = to_edev(ibpd->device);
773 struct erdma_mr *mr;
774 u32 stag;
775 int ret;
776
777 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
778 if (!mr)
779 return ERR_PTR(-ENOMEM);
780
781 ret = erdma_create_stag(dev, &stag);
782 if (ret)
783 goto out_free;
784
785 mr->type = ERDMA_MR_TYPE_DMA;
786
787 mr->ibmr.lkey = stag;
788 mr->ibmr.rkey = stag;
789 mr->ibmr.pd = ibpd;
790 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
791 ret = regmr_cmd(dev, mr);
792 if (ret)
793 goto out_remove_stag;
794
795 return &mr->ibmr;
796
797 out_remove_stag:
798 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
799 mr->ibmr.lkey >> 8);
800
801 out_free:
802 kfree(mr);
803
804 return ERR_PTR(ret);
805 }
806
erdma_ib_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)807 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
808 u32 max_num_sg)
809 {
810 struct erdma_mr *mr;
811 struct erdma_dev *dev = to_edev(ibpd->device);
812 int ret;
813 u32 stag;
814
815 if (mr_type != IB_MR_TYPE_MEM_REG)
816 return ERR_PTR(-EOPNOTSUPP);
817
818 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
819 return ERR_PTR(-EINVAL);
820
821 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
822 if (!mr)
823 return ERR_PTR(-ENOMEM);
824
825 ret = erdma_create_stag(dev, &stag);
826 if (ret)
827 goto out_free;
828
829 mr->type = ERDMA_MR_TYPE_FRMR;
830
831 mr->ibmr.lkey = stag;
832 mr->ibmr.rkey = stag;
833 mr->ibmr.pd = ibpd;
834 /* update it in FRMR. */
835 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
836 ERDMA_MR_ACC_RW;
837
838 mr->mem.page_size = PAGE_SIZE; /* update it later. */
839 mr->mem.page_cnt = max_num_sg;
840 mr->mem.mtt_type = ERDMA_MR_INDIRECT_MTT;
841 mr->mem.mtt_buf =
842 alloc_pages_exact(MTT_SIZE(mr->mem.page_cnt), GFP_KERNEL);
843 if (!mr->mem.mtt_buf) {
844 ret = -ENOMEM;
845 goto out_remove_stag;
846 }
847
848 mr->mem.mtt_entry[0] =
849 dma_map_single(&dev->pdev->dev, mr->mem.mtt_buf,
850 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE);
851 if (dma_mapping_error(&dev->pdev->dev, mr->mem.mtt_entry[0])) {
852 ret = -ENOMEM;
853 goto out_free_mtt;
854 }
855
856 ret = regmr_cmd(dev, mr);
857 if (ret)
858 goto out_dma_unmap;
859
860 return &mr->ibmr;
861
862 out_dma_unmap:
863 dma_unmap_single(&dev->pdev->dev, mr->mem.mtt_entry[0],
864 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE);
865 out_free_mtt:
866 free_pages_exact(mr->mem.mtt_buf, MTT_SIZE(mr->mem.page_cnt));
867
868 out_remove_stag:
869 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
870 mr->ibmr.lkey >> 8);
871
872 out_free:
873 kfree(mr);
874
875 return ERR_PTR(ret);
876 }
877
erdma_set_page(struct ib_mr * ibmr,u64 addr)878 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
879 {
880 struct erdma_mr *mr = to_emr(ibmr);
881
882 if (mr->mem.mtt_nents >= mr->mem.page_cnt)
883 return -1;
884
885 *((u64 *)mr->mem.mtt_buf + mr->mem.mtt_nents) = addr;
886 mr->mem.mtt_nents++;
887
888 return 0;
889 }
890
erdma_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)891 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
892 unsigned int *sg_offset)
893 {
894 struct erdma_mr *mr = to_emr(ibmr);
895 int num;
896
897 mr->mem.mtt_nents = 0;
898
899 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
900 erdma_set_page);
901
902 return num;
903 }
904
erdma_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 virt,int access,struct ib_udata * udata)905 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
906 u64 virt, int access, struct ib_udata *udata)
907 {
908 struct erdma_mr *mr = NULL;
909 struct erdma_dev *dev = to_edev(ibpd->device);
910 u32 stag;
911 int ret;
912
913 if (!len || len > dev->attrs.max_mr_size)
914 return ERR_PTR(-EINVAL);
915
916 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
917 if (!mr)
918 return ERR_PTR(-ENOMEM);
919
920 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
921 SZ_2G - SZ_4K, 0);
922 if (ret)
923 goto err_out_free;
924
925 ret = erdma_create_stag(dev, &stag);
926 if (ret)
927 goto err_out_put_mtt;
928
929 mr->ibmr.lkey = mr->ibmr.rkey = stag;
930 mr->ibmr.pd = ibpd;
931 mr->mem.va = virt;
932 mr->mem.len = len;
933 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
934 mr->valid = 1;
935 mr->type = ERDMA_MR_TYPE_NORMAL;
936
937 ret = regmr_cmd(dev, mr);
938 if (ret)
939 goto err_out_mr;
940
941 return &mr->ibmr;
942
943 err_out_mr:
944 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
945 mr->ibmr.lkey >> 8);
946
947 err_out_put_mtt:
948 put_mtt_entries(dev, &mr->mem);
949
950 err_out_free:
951 kfree(mr);
952
953 return ERR_PTR(ret);
954 }
955
erdma_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)956 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
957 {
958 struct erdma_mr *mr;
959 struct erdma_dev *dev = to_edev(ibmr->device);
960 struct erdma_cmdq_dereg_mr_req req;
961 int ret;
962
963 mr = to_emr(ibmr);
964
965 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
966 CMDQ_OPCODE_DEREG_MR);
967
968 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
969 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
970
971 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
972 if (ret)
973 return ret;
974
975 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
976
977 put_mtt_entries(dev, &mr->mem);
978
979 kfree(mr);
980 return 0;
981 }
982
erdma_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)983 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
984 {
985 struct erdma_cq *cq = to_ecq(ibcq);
986 struct erdma_dev *dev = to_edev(ibcq->device);
987 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
988 udata, struct erdma_ucontext, ibucontext);
989 int err;
990 struct erdma_cmdq_destroy_cq_req req;
991
992 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
993 CMDQ_OPCODE_DESTROY_CQ);
994 req.cqn = cq->cqn;
995
996 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
997 if (err)
998 return err;
999
1000 if (rdma_is_kernel_res(&cq->ibcq.res)) {
1001 dma_free_coherent(&dev->pdev->dev,
1002 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1003 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1004 } else {
1005 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1006 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1007 }
1008
1009 xa_erase(&dev->cq_xa, cq->cqn);
1010
1011 return 0;
1012 }
1013
erdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)1014 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1015 {
1016 struct erdma_qp *qp = to_eqp(ibqp);
1017 struct erdma_dev *dev = to_edev(ibqp->device);
1018 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1019 udata, struct erdma_ucontext, ibucontext);
1020 struct erdma_qp_attrs qp_attrs;
1021 int err;
1022 struct erdma_cmdq_destroy_qp_req req;
1023
1024 down_write(&qp->state_lock);
1025 qp_attrs.state = ERDMA_QP_STATE_ERROR;
1026 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
1027 up_write(&qp->state_lock);
1028
1029 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1030 CMDQ_OPCODE_DESTROY_QP);
1031 req.qpn = QP_ID(qp);
1032
1033 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1034 if (err)
1035 return err;
1036
1037 erdma_qp_put(qp);
1038 wait_for_completion(&qp->safe_free);
1039
1040 if (rdma_is_kernel_res(&qp->ibqp.res)) {
1041 vfree(qp->kern_qp.swr_tbl);
1042 vfree(qp->kern_qp.rwr_tbl);
1043 dma_free_coherent(
1044 &dev->pdev->dev,
1045 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
1046 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
1047 dma_free_coherent(
1048 &dev->pdev->dev,
1049 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
1050 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
1051 } else {
1052 put_mtt_entries(dev, &qp->user_qp.sq_mtt);
1053 put_mtt_entries(dev, &qp->user_qp.rq_mtt);
1054 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1055 }
1056
1057 if (qp->cep)
1058 erdma_cep_put(qp->cep);
1059 xa_erase(&dev->qp_xa, QP_ID(qp));
1060
1061 return 0;
1062 }
1063
erdma_qp_get_ref(struct ib_qp * ibqp)1064 void erdma_qp_get_ref(struct ib_qp *ibqp)
1065 {
1066 erdma_qp_get(to_eqp(ibqp));
1067 }
1068
erdma_qp_put_ref(struct ib_qp * ibqp)1069 void erdma_qp_put_ref(struct ib_qp *ibqp)
1070 {
1071 erdma_qp_put(to_eqp(ibqp));
1072 }
1073
erdma_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)1074 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1075 {
1076 struct rdma_user_mmap_entry *rdma_entry;
1077 struct erdma_user_mmap_entry *entry;
1078 pgprot_t prot;
1079 int err;
1080
1081 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1082 if (!rdma_entry)
1083 return -EINVAL;
1084
1085 entry = to_emmap(rdma_entry);
1086
1087 switch (entry->mmap_flag) {
1088 case ERDMA_MMAP_IO_NC:
1089 /* map doorbell. */
1090 prot = pgprot_device(vma->vm_page_prot);
1091 break;
1092 default:
1093 err = -EINVAL;
1094 goto put_entry;
1095 }
1096
1097 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1098 prot, rdma_entry);
1099
1100 put_entry:
1101 rdma_user_mmap_entry_put(rdma_entry);
1102 return err;
1103 }
1104
erdma_mmap_free(struct rdma_user_mmap_entry * rdma_entry)1105 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1106 {
1107 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1108
1109 kfree(entry);
1110 }
1111
1112 #define ERDMA_SDB_PAGE 0
1113 #define ERDMA_SDB_ENTRY 1
1114 #define ERDMA_SDB_SHARED 2
1115
alloc_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx)1116 static void alloc_db_resources(struct erdma_dev *dev,
1117 struct erdma_ucontext *ctx)
1118 {
1119 u32 bitmap_idx;
1120 struct erdma_devattr *attrs = &dev->attrs;
1121
1122 if (attrs->disable_dwqe)
1123 goto alloc_normal_db;
1124
1125 /* Try to alloc independent SDB page. */
1126 spin_lock(&dev->db_bitmap_lock);
1127 bitmap_idx = find_first_zero_bit(dev->sdb_page, attrs->dwqe_pages);
1128 if (bitmap_idx != attrs->dwqe_pages) {
1129 set_bit(bitmap_idx, dev->sdb_page);
1130 spin_unlock(&dev->db_bitmap_lock);
1131
1132 ctx->sdb_type = ERDMA_SDB_PAGE;
1133 ctx->sdb_idx = bitmap_idx;
1134 ctx->sdb_page_idx = bitmap_idx;
1135 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET +
1136 (bitmap_idx << PAGE_SHIFT);
1137 ctx->sdb_page_off = 0;
1138
1139 return;
1140 }
1141
1142 bitmap_idx = find_first_zero_bit(dev->sdb_entry, attrs->dwqe_entries);
1143 if (bitmap_idx != attrs->dwqe_entries) {
1144 set_bit(bitmap_idx, dev->sdb_entry);
1145 spin_unlock(&dev->db_bitmap_lock);
1146
1147 ctx->sdb_type = ERDMA_SDB_ENTRY;
1148 ctx->sdb_idx = bitmap_idx;
1149 ctx->sdb_page_idx = attrs->dwqe_pages +
1150 bitmap_idx / ERDMA_DWQE_TYPE1_CNT_PER_PAGE;
1151 ctx->sdb_page_off = bitmap_idx % ERDMA_DWQE_TYPE1_CNT_PER_PAGE;
1152
1153 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET +
1154 (ctx->sdb_page_idx << PAGE_SHIFT);
1155
1156 return;
1157 }
1158
1159 spin_unlock(&dev->db_bitmap_lock);
1160
1161 alloc_normal_db:
1162 ctx->sdb_type = ERDMA_SDB_SHARED;
1163 ctx->sdb_idx = 0;
1164 ctx->sdb_page_idx = ERDMA_SDB_SHARED_PAGE_INDEX;
1165 ctx->sdb_page_off = 0;
1166
1167 ctx->sdb = dev->func_bar_addr + (ctx->sdb_page_idx << PAGE_SHIFT);
1168 }
1169
erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext * uctx)1170 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1171 {
1172 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1173 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1174 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1175 }
1176
erdma_alloc_ucontext(struct ib_ucontext * ibctx,struct ib_udata * udata)1177 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1178 {
1179 struct erdma_ucontext *ctx = to_ectx(ibctx);
1180 struct erdma_dev *dev = to_edev(ibctx->device);
1181 int ret;
1182 struct erdma_uresp_alloc_ctx uresp = {};
1183
1184 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1185 ret = -ENOMEM;
1186 goto err_out;
1187 }
1188
1189 INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1190 mutex_init(&ctx->dbrecords_page_mutex);
1191
1192 alloc_db_resources(dev, ctx);
1193
1194 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1195 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1196
1197 if (udata->outlen < sizeof(uresp)) {
1198 ret = -EINVAL;
1199 goto err_out;
1200 }
1201
1202 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1203 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1204 if (!ctx->sq_db_mmap_entry) {
1205 ret = -ENOMEM;
1206 goto err_out;
1207 }
1208
1209 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1210 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1211 if (!ctx->rq_db_mmap_entry) {
1212 ret = -EINVAL;
1213 goto err_out;
1214 }
1215
1216 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1217 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1218 if (!ctx->cq_db_mmap_entry) {
1219 ret = -EINVAL;
1220 goto err_out;
1221 }
1222
1223 uresp.dev_id = dev->pdev->device;
1224 uresp.sdb_type = ctx->sdb_type;
1225 uresp.sdb_offset = ctx->sdb_page_off;
1226
1227 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1228 if (ret)
1229 goto err_out;
1230
1231 return 0;
1232
1233 err_out:
1234 erdma_uctx_user_mmap_entries_remove(ctx);
1235 atomic_dec(&dev->num_ctx);
1236 return ret;
1237 }
1238
erdma_dealloc_ucontext(struct ib_ucontext * ibctx)1239 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1240 {
1241 struct erdma_ucontext *ctx = to_ectx(ibctx);
1242 struct erdma_dev *dev = to_edev(ibctx->device);
1243
1244 spin_lock(&dev->db_bitmap_lock);
1245 if (ctx->sdb_type == ERDMA_SDB_PAGE)
1246 clear_bit(ctx->sdb_idx, dev->sdb_page);
1247 else if (ctx->sdb_type == ERDMA_SDB_ENTRY)
1248 clear_bit(ctx->sdb_idx, dev->sdb_entry);
1249
1250 erdma_uctx_user_mmap_entries_remove(ctx);
1251
1252 spin_unlock(&dev->db_bitmap_lock);
1253
1254 atomic_dec(&dev->num_ctx);
1255 }
1256
1257 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = {
1258 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE,
1259 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE,
1260 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR,
1261 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS,
1262 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING,
1263 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE,
1264 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR
1265 };
1266
erdma_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1267 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1268 struct ib_udata *udata)
1269 {
1270 struct erdma_qp_attrs new_attrs;
1271 enum erdma_qp_attr_mask erdma_attr_mask = 0;
1272 struct erdma_qp *qp = to_eqp(ibqp);
1273 int ret = 0;
1274
1275 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1276 return -EOPNOTSUPP;
1277
1278 memset(&new_attrs, 0, sizeof(new_attrs));
1279
1280 if (attr_mask & IB_QP_STATE) {
1281 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state];
1282
1283 erdma_attr_mask |= ERDMA_QP_ATTR_STATE;
1284 }
1285
1286 down_write(&qp->state_lock);
1287
1288 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask);
1289
1290 up_write(&qp->state_lock);
1291
1292 return ret;
1293 }
1294
erdma_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1295 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1296 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1297 {
1298 struct erdma_qp *qp;
1299 struct erdma_dev *dev;
1300
1301 if (ibqp && qp_attr && qp_init_attr) {
1302 qp = to_eqp(ibqp);
1303 dev = to_edev(ibqp->device);
1304 } else {
1305 return -EINVAL;
1306 }
1307
1308 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1309 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1310
1311 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1312 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1313 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1314 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1315
1316 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1317 qp_attr->max_rd_atomic = qp->attrs.irq_size;
1318 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1319
1320 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1321 IB_ACCESS_REMOTE_WRITE |
1322 IB_ACCESS_REMOTE_READ;
1323
1324 qp_init_attr->cap = qp_attr->cap;
1325
1326 return 0;
1327 }
1328
erdma_init_user_cq(struct erdma_ucontext * ctx,struct erdma_cq * cq,struct erdma_ureq_create_cq * ureq)1329 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1330 struct erdma_ureq_create_cq *ureq)
1331 {
1332 int ret;
1333 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1334
1335 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mtt, ureq->qbuf_va,
1336 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1337 1);
1338 if (ret)
1339 return ret;
1340
1341 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1342 &cq->user_cq.user_dbr_page,
1343 &cq->user_cq.db_info_dma_addr);
1344 if (ret)
1345 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1346
1347 return ret;
1348 }
1349
erdma_init_kernel_cq(struct erdma_cq * cq)1350 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1351 {
1352 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1353
1354 cq->kern_cq.qbuf =
1355 dma_alloc_coherent(&dev->pdev->dev,
1356 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1357 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1358 if (!cq->kern_cq.qbuf)
1359 return -ENOMEM;
1360
1361 cq->kern_cq.db_record =
1362 (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT));
1363 spin_lock_init(&cq->kern_cq.lock);
1364 /* use default cqdb addr */
1365 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1366
1367 return 0;
1368 }
1369
erdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)1370 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1371 struct ib_udata *udata)
1372 {
1373 struct erdma_cq *cq = to_ecq(ibcq);
1374 struct erdma_dev *dev = to_edev(ibcq->device);
1375 unsigned int depth = attr->cqe;
1376 int ret;
1377 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1378 udata, struct erdma_ucontext, ibucontext);
1379
1380 if (depth > dev->attrs.max_cqe)
1381 return -EINVAL;
1382
1383 depth = roundup_pow_of_two(depth);
1384 cq->ibcq.cqe = depth;
1385 cq->depth = depth;
1386 cq->assoc_eqn = attr->comp_vector + 1;
1387
1388 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1389 XA_LIMIT(1, dev->attrs.max_cq - 1),
1390 &dev->next_alloc_cqn, GFP_KERNEL);
1391 if (ret < 0)
1392 return ret;
1393
1394 if (!rdma_is_kernel_res(&ibcq->res)) {
1395 struct erdma_ureq_create_cq ureq;
1396 struct erdma_uresp_create_cq uresp;
1397
1398 ret = ib_copy_from_udata(&ureq, udata,
1399 min(udata->inlen, sizeof(ureq)));
1400 if (ret)
1401 goto err_out_xa;
1402
1403 ret = erdma_init_user_cq(ctx, cq, &ureq);
1404 if (ret)
1405 goto err_out_xa;
1406
1407 uresp.cq_id = cq->cqn;
1408 uresp.num_cqe = depth;
1409
1410 ret = ib_copy_to_udata(udata, &uresp,
1411 min(sizeof(uresp), udata->outlen));
1412 if (ret)
1413 goto err_free_res;
1414 } else {
1415 ret = erdma_init_kernel_cq(cq);
1416 if (ret)
1417 goto err_out_xa;
1418 }
1419
1420 ret = create_cq_cmd(dev, cq);
1421 if (ret)
1422 goto err_free_res;
1423
1424 return 0;
1425
1426 err_free_res:
1427 if (!rdma_is_kernel_res(&ibcq->res)) {
1428 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1429 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1430 } else {
1431 dma_free_coherent(&dev->pdev->dev,
1432 WARPPED_BUFSIZE(depth << CQE_SHIFT),
1433 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1434 }
1435
1436 err_out_xa:
1437 xa_erase(&dev->cq_xa, cq->cqn);
1438
1439 return ret;
1440 }
1441
erdma_set_mtu(struct erdma_dev * dev,u32 mtu)1442 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
1443 {
1444 struct erdma_cmdq_config_mtu_req req;
1445
1446 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1447 CMDQ_OPCODE_CONF_MTU);
1448 req.mtu = mtu;
1449
1450 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1451 }
1452
erdma_port_event(struct erdma_dev * dev,enum ib_event_type reason)1453 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
1454 {
1455 struct ib_event event;
1456
1457 event.device = &dev->ibdev;
1458 event.element.port_num = 1;
1459 event.event = reason;
1460
1461 ib_dispatch_event(&event);
1462 }
1463