108 	u32 space, prod, cons;  in queue_has_space()  local
110 	prod = Q_IDX(q, q->prod);  in queue_has_space()
113 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))  in queue_has_space()
114 		space = (1 << q->max_n_shift) - (prod - cons);  in queue_has_space()
116 		space = cons - prod;  in queue_has_space()
123 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_full()
124 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);  in queue_full()
129 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_empty()
130 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);  in queue_empty()
133 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)  in queue_consumed()  argument
135 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&  in queue_consumed()
136 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||  in queue_consumed()
137 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&  in queue_consumed()
138 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));  in queue_consumed()
161 	if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))  in queue_sync_cons_ovf()
164 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in queue_sync_cons_ovf()
171 	u32 prod;  in queue_sync_prod_in()  local
179 	prod = readl(q->prod_reg);  in queue_sync_prod_in()
181 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))  in queue_sync_prod_in()
184 	q->llq.prod = prod;  in queue_sync_prod_in()
190 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;  in queue_inc_prod_n()  local
191 	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);  in queue_inc_prod_n()
356 					 struct arm_smmu_queue *q, u32 prod)  in arm_smmu_cmdq_build_sync_cmd()  argument
367 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *  in arm_smmu_cmdq_build_sync_cmd()
543 		.prod		= sprod,  in __arm_smmu_cmdq_poll_set_valid_map()
549 	while (llq.prod != eprod) {  in __arm_smmu_cmdq_poll_set_valid_map()
554 		swidx = BIT_WORD(Q_IDX(&llq, llq.prod));  in __arm_smmu_cmdq_poll_set_valid_map()
555 		sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;  in __arm_smmu_cmdq_poll_set_valid_map()
575 			valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;  in __arm_smmu_cmdq_poll_set_valid_map()
579 		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);  in __arm_smmu_cmdq_poll_set_valid_map()
639 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));  in __arm_smmu_cmdq_poll_until_msi()
649 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);  in __arm_smmu_cmdq_poll_until_msi()
662 	u32 prod = llq->prod;  in __arm_smmu_cmdq_poll_until_consumed()  local
668 		if (queue_consumed(llq, prod))  in __arm_smmu_cmdq_poll_until_consumed()
717 					u32 prod, int n)  in arm_smmu_cmdq_write_entries()  argument
722 		.prod		= prod,  in arm_smmu_cmdq_write_entries()
728 		prod = queue_inc_prod_n(&llq, i);  in arm_smmu_cmdq_write_entries()
729 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_write_entries()
753 	u32 prod;  in arm_smmu_cmdq_issue_cmdlist()  local
776 		head.prod = queue_inc_prod_n(&llq, n + sync) |  in arm_smmu_cmdq_issue_cmdlist()
785 	owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);  in arm_smmu_cmdq_issue_cmdlist()
786 	head.prod &= ~CMDQ_PROD_OWNED_FLAG;  in arm_smmu_cmdq_issue_cmdlist()
787 	llq.prod &= ~CMDQ_PROD_OWNED_FLAG;  in arm_smmu_cmdq_issue_cmdlist()
793 	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);  in arm_smmu_cmdq_issue_cmdlist()
795 		prod = queue_inc_prod_n(&llq, n);  in arm_smmu_cmdq_issue_cmdlist()
796 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);  in arm_smmu_cmdq_issue_cmdlist()
797 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_issue_cmdlist()
810 	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);  in arm_smmu_cmdq_issue_cmdlist()
815 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);  in arm_smmu_cmdq_issue_cmdlist()
818 		prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,  in arm_smmu_cmdq_issue_cmdlist()
819 						   &cmdq->q.llq.atomic.prod);  in arm_smmu_cmdq_issue_cmdlist()
820 		prod &= ~CMDQ_PROD_OWNED_FLAG;  in arm_smmu_cmdq_issue_cmdlist()
827 		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
833 		writel_relaxed(prod, cmdq->q.prod_reg);  in arm_smmu_cmdq_issue_cmdlist()
840 		atomic_set_release(&cmdq->owner_prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
845 		llq.prod = queue_inc_prod_n(&llq, n);  in arm_smmu_cmdq_issue_cmdlist()
850 					    llq.prod,  in arm_smmu_cmdq_issue_cmdlist()
2950 	q->llq.prod = q->llq.cons = 0;  in arm_smmu_init_one_queue()
3386 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);  in arm_smmu_device_reset()
3412 	writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);  in arm_smmu_device_reset()
3427 		writel_relaxed(smmu->priq.q.llq.prod,  in arm_smmu_device_reset()