Lines Matching refs:cq
168 static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) in get_cqe() argument
170 return get_cqe_from_buf(&cq->buf, entry); in get_cqe()
178 static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq) in next_cqe_sw() argument
180 return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe)); in next_cqe_sw()
203 static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq, in update_cons_index() argument
207 *cq->set_ci_db = cpu_to_be32(cq->cons_index); in update_cons_index()
210 mthca_write64(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn, incr - 1, in update_cons_index()
223 struct mthca_cq *cq; in mthca_cq_completion() local
225 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); in mthca_cq_completion()
227 if (!cq) { in mthca_cq_completion()
232 ++cq->arm_sn; in mthca_cq_completion()
234 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); in mthca_cq_completion()
240 struct mthca_cq *cq; in mthca_cq_event() local
245 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); in mthca_cq_event()
246 if (cq) in mthca_cq_event()
247 ++cq->refcount; in mthca_cq_event()
251 if (!cq) { in mthca_cq_event()
258 event.element.cq = &cq->ibcq; in mthca_cq_event()
259 if (cq->ibcq.event_handler) in mthca_cq_event()
260 cq->ibcq.event_handler(&event, cq->ibcq.cq_context); in mthca_cq_event()
263 if (!--cq->refcount) in mthca_cq_event()
264 wake_up(&cq->wait); in mthca_cq_event()
277 void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, in mthca_cq_clean() argument
284 spin_lock_irq(&cq->lock); in mthca_cq_clean()
293 for (prod_index = cq->cons_index; in mthca_cq_clean()
294 cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe)); in mthca_cq_clean()
296 if (prod_index == cq->cons_index + cq->ibcq.cqe) in mthca_cq_clean()
301 qpn, cq->cqn, cq->cons_index, prod_index); in mthca_cq_clean()
307 while ((int) --prod_index - (int) cq->cons_index >= 0) { in mthca_cq_clean()
308 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); in mthca_cq_clean()
314 memcpy(get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe), in mthca_cq_clean()
320 set_cqe_hw(get_cqe(cq, (cq->cons_index + i) & cq->ibcq.cqe)); in mthca_cq_clean()
322 cq->cons_index += nfreed; in mthca_cq_clean()
323 update_cons_index(dev, cq, nfreed); in mthca_cq_clean()
326 spin_unlock_irq(&cq->lock); in mthca_cq_clean()
329 void mthca_cq_resize_copy_cqes(struct mthca_cq *cq) in mthca_cq_resize_copy_cqes() argument
339 if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) && in mthca_cq_resize_copy_cqes()
340 cq->ibcq.cqe < cq->resize_buf->cqe) { in mthca_cq_resize_copy_cqes()
341 cq->cons_index &= cq->ibcq.cqe; in mthca_cq_resize_copy_cqes()
342 if (cqe_sw(get_cqe(cq, cq->ibcq.cqe))) in mthca_cq_resize_copy_cqes()
343 cq->cons_index -= cq->ibcq.cqe + 1; in mthca_cq_resize_copy_cqes()
346 for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i) in mthca_cq_resize_copy_cqes()
347 memcpy(get_cqe_from_buf(&cq->resize_buf->buf, in mthca_cq_resize_copy_cqes()
348 i & cq->resize_buf->cqe), in mthca_cq_resize_copy_cqes()
349 get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE); in mthca_cq_resize_copy_cqes()
376 static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, in handle_error_cqe() argument
388 cq->cqn, cq->cons_index); in handle_error_cqe()
483 struct mthca_cq *cq, in mthca_poll_one() argument
497 cqe = next_cqe_sw(cq); in mthca_poll_one()
509 cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn), in mthca_poll_one()
575 handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send, in mthca_poll_one()
654 ++cq->cons_index; in mthca_poll_one()
664 struct mthca_cq *cq = to_mcq(ibcq); in mthca_poll_cq() local
671 spin_lock_irqsave(&cq->lock, flags); in mthca_poll_cq()
676 err = mthca_poll_one(dev, cq, &qp, in mthca_poll_cq()
685 update_cons_index(dev, cq, freed); in mthca_poll_cq()
694 if (unlikely(err == -EAGAIN && cq->resize_buf && in mthca_poll_cq()
695 cq->resize_buf->state == CQ_RESIZE_READY)) { in mthca_poll_cq()
704 cq->cons_index &= cq->ibcq.cqe; in mthca_poll_cq()
706 if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf, in mthca_poll_cq()
707 cq->cons_index & cq->resize_buf->cqe))) { in mthca_poll_cq()
711 tbuf = cq->buf; in mthca_poll_cq()
712 tcqe = cq->ibcq.cqe; in mthca_poll_cq()
713 cq->buf = cq->resize_buf->buf; in mthca_poll_cq()
714 cq->ibcq.cqe = cq->resize_buf->cqe; in mthca_poll_cq()
716 cq->resize_buf->buf = tbuf; in mthca_poll_cq()
717 cq->resize_buf->cqe = tcqe; in mthca_poll_cq()
718 cq->resize_buf->state = CQ_RESIZE_SWAPPED; in mthca_poll_cq()
724 spin_unlock_irqrestore(&cq->lock, flags); in mthca_poll_cq()
729 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags) in mthca_tavor_arm_cq() argument
734 to_mcq(cq)->cqn; in mthca_tavor_arm_cq()
736 mthca_write64(dbhi, 0xffffffff, to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL, in mthca_tavor_arm_cq()
737 MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock)); in mthca_tavor_arm_cq()
744 struct mthca_cq *cq = to_mcq(ibcq); in mthca_arbel_arm_cq() local
747 u32 sn = cq->arm_sn & 3; in mthca_arbel_arm_cq()
749 db_rec[0] = cpu_to_be32(cq->cons_index); in mthca_arbel_arm_cq()
750 db_rec[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) | in mthca_arbel_arm_cq()
754 mthca_write_db_rec(db_rec, cq->arm_db); in mthca_arbel_arm_cq()
765 MTHCA_ARBEL_CQ_DB_REQ_NOT) | cq->cqn; in mthca_arbel_arm_cq()
767 mthca_write64(dbhi, cq->cons_index, in mthca_arbel_arm_cq()
776 struct mthca_cq *cq) in mthca_init_cq() argument
783 cq->ibcq.cqe = nent - 1; in mthca_init_cq()
784 cq->is_kernel = !ctx; in mthca_init_cq()
786 cq->cqn = mthca_alloc(&dev->cq_table.alloc); in mthca_init_cq()
787 if (cq->cqn == -1) in mthca_init_cq()
791 err = mthca_table_get(dev, dev->cq_table.table, cq->cqn); in mthca_init_cq()
795 if (cq->is_kernel) { in mthca_init_cq()
796 cq->arm_sn = 1; in mthca_init_cq()
800 cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, in mthca_init_cq()
801 cq->cqn, &cq->set_ci_db); in mthca_init_cq()
802 if (cq->set_ci_db_index < 0) in mthca_init_cq()
805 cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM, in mthca_init_cq()
806 cq->cqn, &cq->arm_db); in mthca_init_cq()
807 if (cq->arm_db_index < 0) in mthca_init_cq()
818 if (cq->is_kernel) { in mthca_init_cq()
819 err = mthca_alloc_cq_buf(dev, &cq->buf, nent); in mthca_init_cq()
824 spin_lock_init(&cq->lock); in mthca_init_cq()
825 cq->refcount = 1; in mthca_init_cq()
826 init_waitqueue_head(&cq->wait); in mthca_init_cq()
827 mutex_init(&cq->mutex); in mthca_init_cq()
841 cq_context->lkey = cpu_to_be32(cq->buf.mr.ibmr.lkey); in mthca_init_cq()
842 cq_context->cqn = cpu_to_be32(cq->cqn); in mthca_init_cq()
845 cq_context->ci_db = cpu_to_be32(cq->set_ci_db_index); in mthca_init_cq()
846 cq_context->state_db = cpu_to_be32(cq->arm_db_index); in mthca_init_cq()
849 err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn, &status); in mthca_init_cq()
863 if (mthca_array_set(&dev->cq_table.cq, in mthca_init_cq()
864 cq->cqn & (dev->limits.num_cqs - 1), in mthca_init_cq()
865 cq)) { in mthca_init_cq()
871 cq->cons_index = 0; in mthca_init_cq()
878 if (cq->is_kernel) in mthca_init_cq()
879 mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mthca_init_cq()
885 if (cq->is_kernel && mthca_is_memfree(dev)) in mthca_init_cq()
886 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); in mthca_init_cq()
889 if (cq->is_kernel && mthca_is_memfree(dev)) in mthca_init_cq()
890 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); in mthca_init_cq()
893 mthca_table_put(dev, dev->cq_table.table, cq->cqn); in mthca_init_cq()
896 mthca_free(&dev->cq_table.alloc, cq->cqn); in mthca_init_cq()
901 static inline int get_cq_refcount(struct mthca_dev *dev, struct mthca_cq *cq) in get_cq_refcount() argument
906 c = cq->refcount; in get_cq_refcount()
913 struct mthca_cq *cq) in mthca_free_cq() argument
925 err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn, &status); in mthca_free_cq()
936 cq->cqn, cq->cons_index, in mthca_free_cq()
937 cq->is_kernel ? !!next_cqe_sw(cq) : 0); in mthca_free_cq()
943 mthca_array_clear(&dev->cq_table.cq, in mthca_free_cq()
944 cq->cqn & (dev->limits.num_cqs - 1)); in mthca_free_cq()
945 --cq->refcount; in mthca_free_cq()
953 wait_event(cq->wait, !get_cq_refcount(dev, cq)); in mthca_free_cq()
955 if (cq->is_kernel) { in mthca_free_cq()
956 mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mthca_free_cq()
958 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); in mthca_free_cq()
959 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); in mthca_free_cq()
963 mthca_table_put(dev, dev->cq_table.table, cq->cqn); in mthca_free_cq()
964 mthca_free(&dev->cq_table.alloc, cq->cqn); in mthca_free_cq()
981 err = mthca_array_init(&dev->cq_table.cq, in mthca_init_cq_table()
991 mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs); in mthca_cleanup_cq_table()