Lines Matching refs:cq
41 static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq, struct mlx5_eqe *eqe) in mlx5_ib_cq_comp() argument
43 struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; in mlx5_ib_cq_comp()
50 struct mlx5_ib_cq *cq = container_of(mcq, struct mlx5_ib_cq, mcq); in mlx5_ib_cq_event() local
51 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); in mlx5_ib_cq_event()
52 struct ib_cq *ibcq = &cq->ibcq; in mlx5_ib_cq_event()
64 event.element.cq = ibcq; in mlx5_ib_cq_event()
69 static void *get_cqe(struct mlx5_ib_cq *cq, int n) in get_cqe() argument
71 return mlx5_frag_buf_get_wqe(&cq->buf.fbc, n); in get_cqe()
79 static void *get_sw_cqe(struct mlx5_ib_cq *cq, int n) in get_sw_cqe() argument
81 void *cqe = get_cqe(cq, n & cq->ibcq.cqe); in get_sw_cqe()
84 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; in get_sw_cqe()
87 !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ibcq.cqe + 1)))) { in get_sw_cqe()
94 static void *next_cqe_sw(struct mlx5_ib_cq *cq) in next_cqe_sw() argument
96 return get_sw_cqe(cq, cq->mcq.cons_index); in next_cqe_sw()
421 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq *cq, int num_entries, in mlx5_ib_poll_sw_comp() argument
428 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { in mlx5_ib_poll_sw_comp()
434 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { in mlx5_ib_poll_sw_comp()
441 static int mlx5_poll_one(struct mlx5_ib_cq *cq, in mlx5_poll_one() argument
445 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); in mlx5_poll_one()
457 cqe = next_cqe_sw(cq); in mlx5_poll_one()
461 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; in mlx5_poll_one()
463 ++cq->mcq.cons_index; in mlx5_poll_one()
472 if (likely(cq->resize_buf)) { in mlx5_poll_one()
473 free_cq_buf(dev, &cq->buf); in mlx5_poll_one()
474 cq->buf = *cq->resize_buf; in mlx5_poll_one()
475 kfree(cq->resize_buf); in mlx5_poll_one()
476 cq->resize_buf = NULL; in mlx5_poll_one()
520 "Requestor" : "Responder", cq->mcq.cqn); in mlx5_poll_one()
557 cq->mcq.cqn, sig->err_item.key, in mlx5_poll_one()
571 static int poll_soft_wc(struct mlx5_ib_cq *cq, int num_entries, in poll_soft_wc() argument
574 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); in poll_soft_wc()
578 list_for_each_entry_safe(soft_wc, next, &cq->wc_list, list) { in poll_soft_wc()
583 cq->mcq.cqn); in poll_soft_wc()
599 struct mlx5_ib_cq *cq = to_mcq(ibcq); in mlx5_ib_poll_cq() local
601 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); in mlx5_ib_poll_cq()
607 spin_lock_irqsave(&cq->lock, flags); in mlx5_ib_poll_cq()
610 if (unlikely(!list_empty(&cq->wc_list))) in mlx5_ib_poll_cq()
611 soft_polled = poll_soft_wc(cq, num_entries, wc, true); in mlx5_ib_poll_cq()
613 mlx5_ib_poll_sw_comp(cq, num_entries - soft_polled, in mlx5_ib_poll_cq()
618 if (unlikely(!list_empty(&cq->wc_list))) in mlx5_ib_poll_cq()
619 soft_polled = poll_soft_wc(cq, num_entries, wc, false); in mlx5_ib_poll_cq()
622 if (mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled)) in mlx5_ib_poll_cq()
627 mlx5_cq_set_ci(&cq->mcq); in mlx5_ib_poll_cq()
629 spin_unlock_irqrestore(&cq->lock, flags); in mlx5_ib_poll_cq()
637 struct mlx5_ib_cq *cq = to_mcq(ibcq); in mlx5_ib_arm_cq() local
642 spin_lock_irqsave(&cq->lock, irq_flags); in mlx5_ib_arm_cq()
643 if (cq->notify_flags != IB_CQ_NEXT_COMP) in mlx5_ib_arm_cq()
644 cq->notify_flags = flags & IB_CQ_SOLICITED_MASK; in mlx5_ib_arm_cq()
646 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !list_empty(&cq->wc_list)) in mlx5_ib_arm_cq()
648 spin_unlock_irqrestore(&cq->lock, irq_flags); in mlx5_ib_arm_cq()
650 mlx5_cq_arm(&cq->mcq, in mlx5_ib_arm_cq()
706 struct mlx5_ib_cq *cq, int entries, u32 **cqb, in create_cq_user() argument
737 cq->buf.umem = in create_cq_user()
740 if (IS_ERR(cq->buf.umem)) { in create_cq_user()
741 err = PTR_ERR(cq->buf.umem); in create_cq_user()
745 err = mlx5_ib_db_map_user(context, udata, ucmd.db_addr, &cq->db); in create_cq_user()
749 mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, 0, &npages, &page_shift, in create_cq_user()
763 mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, pas, 0); in create_cq_user()
815 cq->private_flags |= MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD; in create_cq_user()
825 mlx5_ib_db_unmap_user(context, &cq->db); in create_cq_user()
828 ib_umem_release(cq->buf.umem); in create_cq_user()
832 static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata) in destroy_cq_user() argument
837 mlx5_ib_db_unmap_user(context, &cq->db); in destroy_cq_user()
838 ib_umem_release(cq->buf.umem); in destroy_cq_user()
854 static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, in create_cq_kernel() argument
862 err = mlx5_db_alloc(dev->mdev, &cq->db); in create_cq_kernel()
866 cq->mcq.set_ci_db = cq->db.db; in create_cq_kernel()
867 cq->mcq.arm_db = cq->db.db + 1; in create_cq_kernel()
868 cq->mcq.cqe_sz = cqe_size; in create_cq_kernel()
870 err = alloc_cq_frag_buf(dev, &cq->buf, entries, cqe_size); in create_cq_kernel()
874 init_cq_frag_buf(&cq->buf); in create_cq_kernel()
878 cq->buf.frag_buf.npages; in create_cq_kernel()
886 mlx5_fill_page_frag_array(&cq->buf.frag_buf, pas); in create_cq_kernel()
890 cq->buf.frag_buf.page_shift - in create_cq_kernel()
898 free_cq_buf(dev, &cq->buf); in create_cq_kernel()
901 mlx5_db_free(dev->mdev, &cq->db); in create_cq_kernel()
905 static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) in destroy_cq_kernel() argument
907 free_cq_buf(dev, &cq->buf); in destroy_cq_kernel()
908 mlx5_db_free(dev->mdev, &cq->db); in destroy_cq_kernel()
913 struct mlx5_ib_cq *cq = container_of(work, struct mlx5_ib_cq, in notify_soft_wc_handler() local
916 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); in notify_soft_wc_handler()
926 struct mlx5_ib_cq *cq = to_mcq(ibcq); in mlx5_ib_create_cq() local
947 cq->ibcq.cqe = entries - 1; in mlx5_ib_create_cq()
948 mutex_init(&cq->resize_mutex); in mlx5_ib_create_cq()
949 spin_lock_init(&cq->lock); in mlx5_ib_create_cq()
950 cq->resize_buf = NULL; in mlx5_ib_create_cq()
951 cq->resize_umem = NULL; in mlx5_ib_create_cq()
952 cq->create_flags = attr->flags; in mlx5_ib_create_cq()
953 INIT_LIST_HEAD(&cq->list_send_qp); in mlx5_ib_create_cq()
954 INIT_LIST_HEAD(&cq->list_recv_qp); in mlx5_ib_create_cq()
957 err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size, in mlx5_ib_create_cq()
963 err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, in mlx5_ib_create_cq()
968 INIT_WORK(&cq->notify_work, notify_soft_wc_handler); in mlx5_ib_create_cq()
975 cq->cqe_size = cqe_size; in mlx5_ib_create_cq()
980 cq->private_flags & in mlx5_ib_create_cq()
985 MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma); in mlx5_ib_create_cq()
986 if (cq->create_flags & IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN) in mlx5_ib_create_cq()
989 err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen, out, sizeof(out)); in mlx5_ib_create_cq()
993 mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn); in mlx5_ib_create_cq()
995 cq->mcq.tasklet_ctx.comp = mlx5_ib_cq_comp; in mlx5_ib_create_cq()
997 cq->mcq.comp = mlx5_ib_cq_comp; in mlx5_ib_create_cq()
998 cq->mcq.event = mlx5_ib_cq_event; in mlx5_ib_create_cq()
1000 INIT_LIST_HEAD(&cq->wc_list); in mlx5_ib_create_cq()
1003 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { in mlx5_ib_create_cq()
1013 mlx5_core_destroy_cq(dev->mdev, &cq->mcq); in mlx5_ib_create_cq()
1018 destroy_cq_user(cq, udata); in mlx5_ib_create_cq()
1020 destroy_cq_kernel(dev, cq); in mlx5_ib_create_cq()
1024 int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) in mlx5_ib_destroy_cq() argument
1026 struct mlx5_ib_dev *dev = to_mdev(cq->device); in mlx5_ib_destroy_cq()
1027 struct mlx5_ib_cq *mcq = to_mcq(cq); in mlx5_ib_destroy_cq()
1046 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq) in __mlx5_ib_cq_clean() argument
1054 if (!cq) in __mlx5_ib_cq_clean()
1063 for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++) in __mlx5_ib_cq_clean()
1064 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) in __mlx5_ib_cq_clean()
1070 while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { in __mlx5_ib_cq_clean()
1071 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); in __mlx5_ib_cq_clean()
1072 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; in __mlx5_ib_cq_clean()
1078 dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); in __mlx5_ib_cq_clean()
1079 dest64 = (cq->mcq.cqe_sz == 64) ? dest : dest + 64; in __mlx5_ib_cq_clean()
1081 memcpy(dest, cqe, cq->mcq.cqe_sz); in __mlx5_ib_cq_clean()
1088 cq->mcq.cons_index += nfreed; in __mlx5_ib_cq_clean()
1093 mlx5_cq_set_ci(&cq->mcq); in __mlx5_ib_cq_clean()
1097 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq) in mlx5_ib_cq_clean() argument
1099 if (!cq) in mlx5_ib_cq_clean()
1102 spin_lock_irq(&cq->lock); in mlx5_ib_cq_clean()
1103 __mlx5_ib_cq_clean(cq, qpn, srq); in mlx5_ib_cq_clean()
1104 spin_unlock_irq(&cq->lock); in mlx5_ib_cq_clean()
1107 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) in mlx5_ib_modify_cq() argument
1109 struct mlx5_ib_dev *dev = to_mdev(cq->device); in mlx5_ib_modify_cq()
1110 struct mlx5_ib_cq *mcq = to_mcq(cq); in mlx5_ib_modify_cq()
1127 static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, in resize_user() argument
1158 cq->resize_umem = umem; in resize_user()
1164 static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, in resize_kernel() argument
1169 cq->resize_buf = kzalloc(sizeof(*cq->resize_buf), GFP_KERNEL); in resize_kernel()
1170 if (!cq->resize_buf) in resize_kernel()
1173 err = alloc_cq_frag_buf(dev, cq->resize_buf, entries, cqe_size); in resize_kernel()
1177 init_cq_frag_buf(cq->resize_buf); in resize_kernel()
1182 kfree(cq->resize_buf); in resize_kernel()
1186 static int copy_resize_cqes(struct mlx5_ib_cq *cq) in copy_resize_cqes() argument
1188 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); in copy_resize_cqes()
1199 ssize = cq->buf.cqe_size; in copy_resize_cqes()
1200 dsize = cq->resize_buf->cqe_size; in copy_resize_cqes()
1206 i = cq->mcq.cons_index; in copy_resize_cqes()
1207 scqe = get_sw_cqe(cq, i); in copy_resize_cqes()
1216 dcqe = mlx5_frag_buf_get_wqe(&cq->resize_buf->fbc, in copy_resize_cqes()
1217 (i + 1) & cq->resize_buf->nent); in copy_resize_cqes()
1219 sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent); in copy_resize_cqes()
1224 scqe = get_sw_cqe(cq, i); in copy_resize_cqes()
1233 cq->mcq.cqn); in copy_resize_cqes()
1237 ++cq->mcq.cons_index; in copy_resize_cqes()
1244 struct mlx5_ib_cq *cq = to_mcq(ibcq); in mlx5_ib_resize_cq() local
1275 mutex_lock(&cq->resize_mutex); in mlx5_ib_resize_cq()
1277 err = resize_user(dev, cq, entries, udata, &npas, &page_shift, in mlx5_ib_resize_cq()
1281 err = resize_kernel(dev, cq, entries, cqe_size); in mlx5_ib_resize_cq()
1283 struct mlx5_frag_buf *frag_buf = &cq->resize_buf->frag_buf; in mlx5_ib_resize_cq()
1304 mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift, in mlx5_ib_resize_cq()
1307 mlx5_fill_page_frag_array(&cq->resize_buf->frag_buf, pas); in mlx5_ib_resize_cq()
1321 cq->private_flags & in mlx5_ib_resize_cq()
1326 MLX5_SET(modify_cq_in, in, cqn, cq->mcq.cqn); in mlx5_ib_resize_cq()
1328 err = mlx5_core_modify_cq(dev->mdev, &cq->mcq, in, inlen); in mlx5_ib_resize_cq()
1333 cq->ibcq.cqe = entries - 1; in mlx5_ib_resize_cq()
1334 ib_umem_release(cq->buf.umem); in mlx5_ib_resize_cq()
1335 cq->buf.umem = cq->resize_umem; in mlx5_ib_resize_cq()
1336 cq->resize_umem = NULL; in mlx5_ib_resize_cq()
1341 spin_lock_irqsave(&cq->lock, flags); in mlx5_ib_resize_cq()
1342 if (cq->resize_buf) { in mlx5_ib_resize_cq()
1343 err = copy_resize_cqes(cq); in mlx5_ib_resize_cq()
1345 tbuf = cq->buf; in mlx5_ib_resize_cq()
1346 cq->buf = *cq->resize_buf; in mlx5_ib_resize_cq()
1347 kfree(cq->resize_buf); in mlx5_ib_resize_cq()
1348 cq->resize_buf = NULL; in mlx5_ib_resize_cq()
1352 cq->ibcq.cqe = entries - 1; in mlx5_ib_resize_cq()
1353 spin_unlock_irqrestore(&cq->lock, flags); in mlx5_ib_resize_cq()
1357 mutex_unlock(&cq->resize_mutex); in mlx5_ib_resize_cq()
1366 ib_umem_release(cq->resize_umem); in mlx5_ib_resize_cq()
1368 free_cq_buf(dev, cq->resize_buf); in mlx5_ib_resize_cq()
1369 cq->resize_buf = NULL; in mlx5_ib_resize_cq()
1372 mutex_unlock(&cq->resize_mutex); in mlx5_ib_resize_cq()
1378 struct mlx5_ib_cq *cq; in mlx5_ib_get_cqe_size() local
1383 cq = to_mcq(ibcq); in mlx5_ib_get_cqe_size()
1384 return cq->cqe_size; in mlx5_ib_get_cqe_size()
1391 struct mlx5_ib_cq *cq = to_mcq(ibcq); in mlx5_ib_generate_wc() local
1399 spin_lock_irqsave(&cq->lock, flags); in mlx5_ib_generate_wc()
1400 list_add_tail(&soft_wc->list, &cq->wc_list); in mlx5_ib_generate_wc()
1401 if (cq->notify_flags == IB_CQ_NEXT_COMP || in mlx5_ib_generate_wc()
1403 cq->notify_flags = 0; in mlx5_ib_generate_wc()
1404 schedule_work(&cq->notify_work); in mlx5_ib_generate_wc()
1406 spin_unlock_irqrestore(&cq->lock, flags); in mlx5_ib_generate_wc()