Lines Matching refs:srq
14 static void *get_wqe(struct mlx5_ib_srq *srq, int n) in get_wqe() argument
16 return mlx5_frag_buf_get_wqe(&srq->fbc, n); in get_wqe()
19 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) in mlx5_ib_srq_event() argument
22 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; in mlx5_ib_srq_event()
26 event.element.srq = ibsrq; in mlx5_ib_srq_event()
36 type, srq->srqn); in mlx5_ib_srq_event()
44 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, in create_srq_user() argument
81 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); in create_srq_user()
83 srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0); in create_srq_user()
84 if (IS_ERR(srq->umem)) { in create_srq_user()
86 err = PTR_ERR(srq->umem); in create_srq_user()
90 mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages, in create_srq_user()
105 mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0); in create_srq_user()
107 err = mlx5_ib_db_map_user(ucontext, udata, ucmd.db_addr, &srq->db); in create_srq_user()
126 ib_umem_release(srq->umem); in create_srq_user()
131 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, in create_srq_kernel() argument
138 err = mlx5_db_alloc(dev->mdev, &srq->db); in create_srq_kernel()
144 if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf, in create_srq_kernel()
151 mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max), in create_srq_kernel()
152 &srq->fbc); in create_srq_kernel()
154 srq->head = 0; in create_srq_kernel()
155 srq->tail = srq->msrq.max - 1; in create_srq_kernel()
156 srq->wqe_ctr = 0; in create_srq_kernel()
158 for (i = 0; i < srq->msrq.max; i++) { in create_srq_kernel()
159 next = get_wqe(srq, i); in create_srq_kernel()
161 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); in create_srq_kernel()
164 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); in create_srq_kernel()
165 in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); in create_srq_kernel()
170 mlx5_fill_page_frag_array(&srq->buf, in->pas); in create_srq_kernel()
172 srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); in create_srq_kernel()
173 if (!srq->wrid) { in create_srq_kernel()
177 srq->wq_sig = 0; in create_srq_kernel()
179 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; in create_srq_kernel()
190 mlx5_frag_buf_free(dev->mdev, &srq->buf); in create_srq_kernel()
193 mlx5_db_free(dev->mdev, &srq->db); in create_srq_kernel()
197 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, in destroy_srq_user() argument
205 &srq->db); in destroy_srq_user()
206 ib_umem_release(srq->umem); in destroy_srq_user()
210 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) in destroy_srq_kernel() argument
212 kvfree(srq->wrid); in destroy_srq_kernel()
213 mlx5_frag_buf_free(dev->mdev, &srq->buf); in destroy_srq_kernel()
214 mlx5_db_free(dev->mdev, &srq->db); in destroy_srq_kernel()
222 struct mlx5_ib_srq *srq = to_msrq(ib_srq); in mlx5_ib_create_srq() local
237 mutex_init(&srq->mutex); in mlx5_ib_create_srq()
238 spin_lock_init(&srq->lock); in mlx5_ib_create_srq()
239 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); in mlx5_ib_create_srq()
240 srq->msrq.max_gs = init_attr->attr.max_sge; in mlx5_ib_create_srq()
243 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); in mlx5_ib_create_srq()
244 if (desc_size == 0 || srq->msrq.max_gs > desc_size) in mlx5_ib_create_srq()
252 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / in mlx5_ib_create_srq()
254 srq->msrq.wqe_shift = ilog2(desc_size); in mlx5_ib_create_srq()
255 buf_size = srq->msrq.max * desc_size; in mlx5_ib_create_srq()
262 err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size); in mlx5_ib_create_srq()
264 err = create_srq_kernel(dev, srq, &in, buf_size); in mlx5_ib_create_srq()
272 in.log_size = ilog2(srq->msrq.max); in mlx5_ib_create_srq()
273 in.wqe_shift = srq->msrq.wqe_shift - 4; in mlx5_ib_create_srq()
274 if (srq->wq_sig) in mlx5_ib_create_srq()
300 in.db_record = srq->db.dma; in mlx5_ib_create_srq()
301 err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); in mlx5_ib_create_srq()
308 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); in mlx5_ib_create_srq()
310 srq->msrq.event = mlx5_ib_srq_event; in mlx5_ib_create_srq()
311 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; in mlx5_ib_create_srq()
315 .srqn = srq->msrq.srqn, in mlx5_ib_create_srq()
326 init_attr->attr.max_wr = srq->msrq.max - 1; in mlx5_ib_create_srq()
331 mlx5_cmd_destroy_srq(dev, &srq->msrq); in mlx5_ib_create_srq()
335 destroy_srq_user(ib_srq->pd, srq, udata); in mlx5_ib_create_srq()
337 destroy_srq_kernel(dev, srq); in mlx5_ib_create_srq()
346 struct mlx5_ib_srq *srq = to_msrq(ibsrq); in mlx5_ib_modify_srq() local
354 if (attr->srq_limit >= srq->msrq.max) in mlx5_ib_modify_srq()
357 mutex_lock(&srq->mutex); in mlx5_ib_modify_srq()
358 ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); in mlx5_ib_modify_srq()
359 mutex_unlock(&srq->mutex); in mlx5_ib_modify_srq()
371 struct mlx5_ib_srq *srq = to_msrq(ibsrq); in mlx5_ib_query_srq() local
379 ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); in mlx5_ib_query_srq()
384 srq_attr->max_wr = srq->msrq.max - 1; in mlx5_ib_query_srq()
385 srq_attr->max_sge = srq->msrq.max_gs; in mlx5_ib_query_srq()
392 int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) in mlx5_ib_destroy_srq() argument
394 struct mlx5_ib_dev *dev = to_mdev(srq->device); in mlx5_ib_destroy_srq()
395 struct mlx5_ib_srq *msrq = to_msrq(srq); in mlx5_ib_destroy_srq()
403 destroy_srq_user(srq->pd, msrq, udata); in mlx5_ib_destroy_srq()
409 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) in mlx5_ib_free_srq_wqe() argument
414 spin_lock(&srq->lock); in mlx5_ib_free_srq_wqe()
416 next = get_wqe(srq, srq->tail); in mlx5_ib_free_srq_wqe()
418 srq->tail = wqe_index; in mlx5_ib_free_srq_wqe()
420 spin_unlock(&srq->lock); in mlx5_ib_free_srq_wqe()
426 struct mlx5_ib_srq *srq = to_msrq(ibsrq); in mlx5_ib_post_srq_recv() local
436 spin_lock_irqsave(&srq->lock, flags); in mlx5_ib_post_srq_recv()
445 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { in mlx5_ib_post_srq_recv()
451 if (unlikely(srq->head == srq->tail)) { in mlx5_ib_post_srq_recv()
457 srq->wrid[srq->head] = wr->wr_id; in mlx5_ib_post_srq_recv()
459 next = get_wqe(srq, srq->head); in mlx5_ib_post_srq_recv()
460 srq->head = be16_to_cpu(next->next_wqe_index); in mlx5_ib_post_srq_recv()
469 if (i < srq->msrq.max_avail_gather) { in mlx5_ib_post_srq_recv()
477 srq->wqe_ctr += nreq; in mlx5_ib_post_srq_recv()
484 *srq->db.db = cpu_to_be32(srq->wqe_ctr); in mlx5_ib_post_srq_recv()
487 spin_unlock_irqrestore(&srq->lock, flags); in mlx5_ib_post_srq_recv()