Lines Matching refs:cq
42 static void mlx4_ib_cq_comp(struct mlx4_cq *cq) in mlx4_ib_cq_comp() argument
44 struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; in mlx4_ib_cq_comp()
48 static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) in mlx4_ib_cq_event() argument
55 "on CQ %06x\n", type, cq->cqn); in mlx4_ib_cq_event()
59 ibcq = &to_mibcq(cq)->ibcq; in mlx4_ib_cq_event()
63 event.element.cq = ibcq; in mlx4_ib_cq_event()
73 static void *get_cqe(struct mlx4_ib_cq *cq, int n) in get_cqe() argument
75 return get_cqe_from_buf(&cq->buf, n); in get_cqe()
78 static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) in get_sw_cqe() argument
80 struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); in get_sw_cqe()
81 struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe); in get_sw_cqe()
84 !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; in get_sw_cqe()
87 static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) in next_cqe_sw() argument
89 return get_sw_cqe(cq, cq->mcq.cons_index); in next_cqe_sw()
92 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) in mlx4_ib_modify_cq() argument
94 struct mlx4_ib_cq *mcq = to_mcq(cq); in mlx4_ib_modify_cq()
95 struct mlx4_ib_dev *dev = to_mdev(cq->device); in mlx4_ib_modify_cq()
178 struct mlx4_ib_cq *cq; in mlx4_ib_create_cq() local
188 cq = kmalloc(sizeof *cq, GFP_KERNEL); in mlx4_ib_create_cq()
189 if (!cq) in mlx4_ib_create_cq()
193 cq->ibcq.cqe = entries - 1; in mlx4_ib_create_cq()
194 mutex_init(&cq->resize_mutex); in mlx4_ib_create_cq()
195 spin_lock_init(&cq->lock); in mlx4_ib_create_cq()
196 cq->resize_buf = NULL; in mlx4_ib_create_cq()
197 cq->resize_umem = NULL; in mlx4_ib_create_cq()
198 cq->create_flags = attr->flags; in mlx4_ib_create_cq()
199 INIT_LIST_HEAD(&cq->send_qp_list); in mlx4_ib_create_cq()
200 INIT_LIST_HEAD(&cq->recv_qp_list); in mlx4_ib_create_cq()
210 err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem, in mlx4_ib_create_cq()
216 &cq->db); in mlx4_ib_create_cq()
222 err = mlx4_db_alloc(dev->dev, &cq->db, 1, GFP_KERNEL); in mlx4_ib_create_cq()
226 cq->mcq.set_ci_db = cq->db.db; in mlx4_ib_create_cq()
227 cq->mcq.arm_db = cq->db.db + 1; in mlx4_ib_create_cq()
228 *cq->mcq.set_ci_db = 0; in mlx4_ib_create_cq()
229 *cq->mcq.arm_db = 0; in mlx4_ib_create_cq()
231 err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries); in mlx4_ib_create_cq()
241 err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, in mlx4_ib_create_cq()
242 cq->db.dma, &cq->mcq, vector, 0, in mlx4_ib_create_cq()
243 !!(cq->create_flags & IB_CQ_FLAGS_TIMESTAMP_COMPLETION)); in mlx4_ib_create_cq()
248 cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp; in mlx4_ib_create_cq()
250 cq->mcq.comp = mlx4_ib_cq_comp; in mlx4_ib_create_cq()
251 cq->mcq.event = mlx4_ib_cq_event; in mlx4_ib_create_cq()
254 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { in mlx4_ib_create_cq()
259 return &cq->ibcq; in mlx4_ib_create_cq()
262 mlx4_cq_free(dev->dev, &cq->mcq); in mlx4_ib_create_cq()
266 mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); in mlx4_ib_create_cq()
269 mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); in mlx4_ib_create_cq()
272 ib_umem_release(cq->umem); in mlx4_ib_create_cq()
274 mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mlx4_ib_create_cq()
278 mlx4_db_free(dev->dev, &cq->db); in mlx4_ib_create_cq()
281 kfree(cq); in mlx4_ib_create_cq()
286 static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, in mlx4_alloc_resize_buf() argument
291 if (cq->resize_buf) in mlx4_alloc_resize_buf()
294 cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL); in mlx4_alloc_resize_buf()
295 if (!cq->resize_buf) in mlx4_alloc_resize_buf()
298 err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); in mlx4_alloc_resize_buf()
300 kfree(cq->resize_buf); in mlx4_alloc_resize_buf()
301 cq->resize_buf = NULL; in mlx4_alloc_resize_buf()
305 cq->resize_buf->cqe = entries - 1; in mlx4_alloc_resize_buf()
310 static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, in mlx4_alloc_resize_umem() argument
316 if (cq->resize_umem) in mlx4_alloc_resize_umem()
322 cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL); in mlx4_alloc_resize_umem()
323 if (!cq->resize_buf) in mlx4_alloc_resize_umem()
326 err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf, in mlx4_alloc_resize_umem()
327 &cq->resize_umem, ucmd.buf_addr, entries); in mlx4_alloc_resize_umem()
329 kfree(cq->resize_buf); in mlx4_alloc_resize_umem()
330 cq->resize_buf = NULL; in mlx4_alloc_resize_umem()
334 cq->resize_buf->cqe = entries - 1; in mlx4_alloc_resize_umem()
339 static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq) in mlx4_ib_get_outstanding_cqes() argument
343 i = cq->mcq.cons_index; in mlx4_ib_get_outstanding_cqes()
344 while (get_sw_cqe(cq, i)) in mlx4_ib_get_outstanding_cqes()
347 return i - cq->mcq.cons_index; in mlx4_ib_get_outstanding_cqes()
350 static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq) in mlx4_ib_cq_resize_copy_cqes() argument
354 int cqe_size = cq->buf.entry_size; in mlx4_ib_cq_resize_copy_cqes()
357 i = cq->mcq.cons_index; in mlx4_ib_cq_resize_copy_cqes()
358 cqe = get_cqe(cq, i & cq->ibcq.cqe); in mlx4_ib_cq_resize_copy_cqes()
362 new_cqe = get_cqe_from_buf(&cq->resize_buf->buf, in mlx4_ib_cq_resize_copy_cqes()
363 (i + 1) & cq->resize_buf->cqe); in mlx4_ib_cq_resize_copy_cqes()
364 memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size); in mlx4_ib_cq_resize_copy_cqes()
368 (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); in mlx4_ib_cq_resize_copy_cqes()
369 cqe = get_cqe(cq, ++i & cq->ibcq.cqe); in mlx4_ib_cq_resize_copy_cqes()
372 ++cq->mcq.cons_index; in mlx4_ib_cq_resize_copy_cqes()
378 struct mlx4_ib_cq *cq = to_mcq(ibcq); in mlx4_ib_resize_cq() local
383 mutex_lock(&cq->resize_mutex); in mlx4_ib_resize_cq()
401 err = mlx4_alloc_resize_umem(dev, cq, entries, udata); in mlx4_ib_resize_cq()
406 outst_cqe = mlx4_ib_get_outstanding_cqes(cq); in mlx4_ib_resize_cq()
412 err = mlx4_alloc_resize_buf(dev, cq, entries); in mlx4_ib_resize_cq()
417 mtt = cq->buf.mtt; in mlx4_ib_resize_cq()
419 err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt); in mlx4_ib_resize_cq()
425 cq->buf = cq->resize_buf->buf; in mlx4_ib_resize_cq()
426 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_resize_cq()
427 ib_umem_release(cq->umem); in mlx4_ib_resize_cq()
428 cq->umem = cq->resize_umem; in mlx4_ib_resize_cq()
430 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
431 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
432 cq->resize_umem = NULL; in mlx4_ib_resize_cq()
437 spin_lock_irq(&cq->lock); in mlx4_ib_resize_cq()
438 if (cq->resize_buf) { in mlx4_ib_resize_cq()
439 mlx4_ib_cq_resize_copy_cqes(cq); in mlx4_ib_resize_cq()
440 tmp_buf = cq->buf; in mlx4_ib_resize_cq()
441 tmp_cqe = cq->ibcq.cqe; in mlx4_ib_resize_cq()
442 cq->buf = cq->resize_buf->buf; in mlx4_ib_resize_cq()
443 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_resize_cq()
445 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
446 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
448 spin_unlock_irq(&cq->lock); in mlx4_ib_resize_cq()
457 mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt); in mlx4_ib_resize_cq()
459 mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf, in mlx4_ib_resize_cq()
460 cq->resize_buf->cqe); in mlx4_ib_resize_cq()
462 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
463 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
465 if (cq->resize_umem) { in mlx4_ib_resize_cq()
466 ib_umem_release(cq->resize_umem); in mlx4_ib_resize_cq()
467 cq->resize_umem = NULL; in mlx4_ib_resize_cq()
471 mutex_unlock(&cq->resize_mutex); in mlx4_ib_resize_cq()
476 int mlx4_ib_destroy_cq(struct ib_cq *cq) in mlx4_ib_destroy_cq() argument
478 struct mlx4_ib_dev *dev = to_mdev(cq->device); in mlx4_ib_destroy_cq()
479 struct mlx4_ib_cq *mcq = to_mcq(cq); in mlx4_ib_destroy_cq()
484 if (cq->uobject) { in mlx4_ib_destroy_cq()
485 mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db); in mlx4_ib_destroy_cq()
488 mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); in mlx4_ib_destroy_cq()
582 static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, in use_tunnel_data() argument
632 static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries, in mlx4_ib_poll_sw_comp() argument
641 list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) { in mlx4_ib_poll_sw_comp()
647 list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) { in mlx4_ib_poll_sw_comp()
657 static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, in mlx4_ib_poll_one() argument
674 cqe = next_cqe_sw(cq); in mlx4_ib_poll_one()
678 if (cq->buf.entry_size == 64) in mlx4_ib_poll_one()
681 ++cq->mcq.cons_index; in mlx4_ib_poll_one()
695 if (cq->resize_buf) { in mlx4_ib_poll_one()
696 struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device); in mlx4_ib_poll_one()
698 mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mlx4_ib_poll_one()
699 cq->buf = cq->resize_buf->buf; in mlx4_ib_poll_one()
700 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_poll_one()
702 kfree(cq->resize_buf); in mlx4_ib_poll_one()
703 cq->resize_buf = NULL; in mlx4_ib_poll_one()
716 mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev, in mlx4_ib_poll_one()
728 msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev, in mlx4_ib_poll_one()
836 if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) { in mlx4_ib_poll_one()
840 use_tunnel_data(*cur_qp, cq, wc, tail, cqe, in mlx4_ib_poll_one()
876 struct mlx4_ib_cq *cq = to_mcq(ibcq); in mlx4_ib_poll_cq() local
880 struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device); in mlx4_ib_poll_cq()
882 spin_lock_irqsave(&cq->lock, flags); in mlx4_ib_poll_cq()
884 mlx4_ib_poll_sw_comp(cq, num_entries, wc, &npolled); in mlx4_ib_poll_cq()
889 if (mlx4_ib_poll_one(cq, &cur_qp, wc + npolled)) in mlx4_ib_poll_cq()
893 mlx4_cq_set_ci(&cq->mcq); in mlx4_ib_poll_cq()
896 spin_unlock_irqrestore(&cq->lock, flags); in mlx4_ib_poll_cq()
912 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) in __mlx4_ib_cq_clean() argument
918 int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0; in __mlx4_ib_cq_clean()
927 for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) in __mlx4_ib_cq_clean()
928 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) in __mlx4_ib_cq_clean()
935 while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { in __mlx4_ib_cq_clean()
936 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); in __mlx4_ib_cq_clean()
944 dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); in __mlx4_ib_cq_clean()
955 cq->mcq.cons_index += nfreed; in __mlx4_ib_cq_clean()
961 mlx4_cq_set_ci(&cq->mcq); in __mlx4_ib_cq_clean()
965 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) in mlx4_ib_cq_clean() argument
967 spin_lock_irq(&cq->lock); in mlx4_ib_cq_clean()
968 __mlx4_ib_cq_clean(cq, qpn, srq); in mlx4_ib_cq_clean()
969 spin_unlock_irq(&cq->lock); in mlx4_ib_cq_clean()