568 static inline void write_gts(struct t4_cq *cq, u32 val)  in write_gts()  argument
570 	if (cq->bar2_va)  in write_gts()
571 		writel(val | INGRESSQID_V(cq->bar2_qid),  in write_gts()
572 		       cq->bar2_va + SGE_UDB_GTS);  in write_gts()
574 		writel(val | INGRESSQID_V(cq->cqid), cq->gts);  in write_gts()
577 static inline int t4_clear_cq_armed(struct t4_cq *cq)  in t4_clear_cq_armed()  argument
579 	return test_and_clear_bit(CQ_ARMED, &cq->flags);  in t4_clear_cq_armed()
582 static inline int t4_arm_cq(struct t4_cq *cq, int se)  in t4_arm_cq()  argument
586 	set_bit(CQ_ARMED, &cq->flags);  in t4_arm_cq()
587 	while (cq->cidx_inc > CIDXINC_M) {  in t4_arm_cq()
589 		write_gts(cq, val);  in t4_arm_cq()
590 		cq->cidx_inc -= CIDXINC_M;  in t4_arm_cq()
592 	val = SEINTARM_V(se) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(6);  in t4_arm_cq()
593 	write_gts(cq, val);  in t4_arm_cq()
594 	cq->cidx_inc = 0;  in t4_arm_cq()
598 static inline void t4_swcq_produce(struct t4_cq *cq)  in t4_swcq_produce()  argument
600 	cq->sw_in_use++;  in t4_swcq_produce()
601 	if (cq->sw_in_use == cq->size) {  in t4_swcq_produce()
602 		PDBG("%s cxgb4 sw cq overflow cqid %u\n", __func__, cq->cqid);  in t4_swcq_produce()
603 		cq->error = 1;  in t4_swcq_produce()
606 	if (++cq->sw_pidx == cq->size)  in t4_swcq_produce()
607 		cq->sw_pidx = 0;  in t4_swcq_produce()
610 static inline void t4_swcq_consume(struct t4_cq *cq)  in t4_swcq_consume()  argument
612 	BUG_ON(cq->sw_in_use < 1);  in t4_swcq_consume()
613 	cq->sw_in_use--;  in t4_swcq_consume()
614 	if (++cq->sw_cidx == cq->size)  in t4_swcq_consume()
615 		cq->sw_cidx = 0;  in t4_swcq_consume()
618 static inline void t4_hwcq_consume(struct t4_cq *cq)  in t4_hwcq_consume()  argument
620 	cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts;  in t4_hwcq_consume()
621 	if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == CIDXINC_M) {  in t4_hwcq_consume()
624 		val = SEINTARM_V(0) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(7);  in t4_hwcq_consume()
625 		write_gts(cq, val);  in t4_hwcq_consume()
626 		cq->cidx_inc = 0;  in t4_hwcq_consume()
628 	if (++cq->cidx == cq->size) {  in t4_hwcq_consume()
629 		cq->cidx = 0;  in t4_hwcq_consume()
630 		cq->gen ^= 1;  in t4_hwcq_consume()
634 static inline int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe)  in t4_valid_cqe()  argument
636 	return (CQE_GENBIT(cqe) == cq->gen);  in t4_valid_cqe()
639 static inline int t4_cq_notempty(struct t4_cq *cq)  in t4_cq_notempty()  argument
641 	return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]);  in t4_cq_notempty()
644 static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)  in t4_next_hw_cqe()  argument
649 	if (cq->cidx == 0)  in t4_next_hw_cqe()
650 		prev_cidx = cq->size - 1;  in t4_next_hw_cqe()
652 		prev_cidx = cq->cidx - 1;  in t4_next_hw_cqe()
654 	if (cq->queue[prev_cidx].bits_type_ts != cq->bits_type_ts) {  in t4_next_hw_cqe()
656 		cq->error = 1;  in t4_next_hw_cqe()
657 		printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid);  in t4_next_hw_cqe()
659 	} else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {  in t4_next_hw_cqe()
663 		*cqe = &cq->queue[cq->cidx];  in t4_next_hw_cqe()
670 static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq)  in t4_next_sw_cqe()  argument
672 	if (cq->sw_in_use == cq->size) {  in t4_next_sw_cqe()
673 		PDBG("%s cxgb4 sw cq overflow cqid %u\n", __func__, cq->cqid);  in t4_next_sw_cqe()
674 		cq->error = 1;  in t4_next_sw_cqe()
678 	if (cq->sw_in_use)  in t4_next_sw_cqe()
679 		return &cq->sw_queue[cq->sw_cidx];  in t4_next_sw_cqe()
683 static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe)  in t4_next_cqe()  argument
687 	if (cq->error)  in t4_next_cqe()
689 	else if (cq->sw_in_use)  in t4_next_cqe()
690 		*cqe = &cq->sw_queue[cq->sw_cidx];  in t4_next_cqe()
692 		ret = t4_next_hw_cqe(cq, cqe);  in t4_next_cqe()
696 static inline int t4_cq_in_error(struct t4_cq *cq)  in t4_cq_in_error()  argument
698 	return ((struct t4_status_page *)&cq->queue[cq->size])->qp_err;  in t4_cq_in_error()
701 static inline void t4_set_cq_in_error(struct t4_cq *cq)  in t4_set_cq_in_error()  argument
703 	((struct t4_status_page *)&cq->queue[cq->size])->qp_err = 1;  in t4_set_cq_in_error()