Lines Matching refs:llq
148 writel_relaxed(q->llq.cons, q->cons_reg); in queue_sync_cons_out()
159 struct arm_smmu_ll_queue *llq = &q->llq; in queue_sync_cons_ovf() local
161 if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons))) in queue_sync_cons_ovf()
164 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | in queue_sync_cons_ovf()
165 Q_IDX(llq, llq->cons); in queue_sync_cons_ovf()
181 if (Q_OVF(prod) != Q_OVF(q->llq.prod)) in queue_sync_prod_in()
184 q->llq.prod = prod; in queue_sync_prod_in()
239 if (queue_empty(&q->llq)) in queue_remove_raw()
242 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords); in queue_remove_raw()
243 queue_inc_cons(&q->llq); in queue_remove_raw()
367 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * in arm_smmu_cmdq_build_sync_cmd()
541 struct arm_smmu_ll_queue llq = { in __arm_smmu_cmdq_poll_set_valid_map() local
542 .max_n_shift = cmdq->q.llq.max_n_shift, in __arm_smmu_cmdq_poll_set_valid_map()
546 ewidx = BIT_WORD(Q_IDX(&llq, eprod)); in __arm_smmu_cmdq_poll_set_valid_map()
547 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG; in __arm_smmu_cmdq_poll_set_valid_map()
549 while (llq.prod != eprod) { in __arm_smmu_cmdq_poll_set_valid_map()
554 swidx = BIT_WORD(Q_IDX(&llq, llq.prod)); in __arm_smmu_cmdq_poll_set_valid_map()
555 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG; in __arm_smmu_cmdq_poll_set_valid_map()
575 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask; in __arm_smmu_cmdq_poll_set_valid_map()
579 llq.prod = queue_inc_prod_n(&llq, limit - sbidx); in __arm_smmu_cmdq_poll_set_valid_map()
599 struct arm_smmu_ll_queue *llq) in arm_smmu_cmdq_poll_until_not_full() argument
611 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg)); in arm_smmu_cmdq_poll_until_not_full()
613 llq->val = READ_ONCE(cmdq->q.llq.val); in arm_smmu_cmdq_poll_until_not_full()
619 llq->val = READ_ONCE(cmdq->q.llq.val); in arm_smmu_cmdq_poll_until_not_full()
620 if (!queue_full(llq)) in arm_smmu_cmdq_poll_until_not_full()
634 struct arm_smmu_ll_queue *llq) in __arm_smmu_cmdq_poll_until_msi() argument
639 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod)); in __arm_smmu_cmdq_poll_until_msi()
649 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1); in __arm_smmu_cmdq_poll_until_msi()
658 struct arm_smmu_ll_queue *llq) in __arm_smmu_cmdq_poll_until_consumed() argument
662 u32 prod = llq->prod; in __arm_smmu_cmdq_poll_until_consumed()
666 llq->val = READ_ONCE(cmdq->q.llq.val); in __arm_smmu_cmdq_poll_until_consumed()
668 if (queue_consumed(llq, prod)) in __arm_smmu_cmdq_poll_until_consumed()
701 llq->cons = readl(cmdq->q.cons_reg); in __arm_smmu_cmdq_poll_until_consumed()
708 struct arm_smmu_ll_queue *llq) in arm_smmu_cmdq_poll_until_sync() argument
711 return __arm_smmu_cmdq_poll_until_msi(smmu, llq); in arm_smmu_cmdq_poll_until_sync()
713 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq); in arm_smmu_cmdq_poll_until_sync()
720 struct arm_smmu_ll_queue llq = { in arm_smmu_cmdq_write_entries() local
721 .max_n_shift = cmdq->q.llq.max_n_shift, in arm_smmu_cmdq_write_entries()
728 prod = queue_inc_prod_n(&llq, i); in arm_smmu_cmdq_write_entries()
757 struct arm_smmu_ll_queue llq, head; in arm_smmu_cmdq_issue_cmdlist() local
760 llq.max_n_shift = cmdq->q.llq.max_n_shift; in arm_smmu_cmdq_issue_cmdlist()
764 llq.val = READ_ONCE(cmdq->q.llq.val); in arm_smmu_cmdq_issue_cmdlist()
768 while (!queue_has_space(&llq, n + sync)) { in arm_smmu_cmdq_issue_cmdlist()
770 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq)) in arm_smmu_cmdq_issue_cmdlist()
775 head.cons = llq.cons; in arm_smmu_cmdq_issue_cmdlist()
776 head.prod = queue_inc_prod_n(&llq, n + sync) | in arm_smmu_cmdq_issue_cmdlist()
779 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val); in arm_smmu_cmdq_issue_cmdlist()
780 if (old == llq.val) in arm_smmu_cmdq_issue_cmdlist()
783 llq.val = old; in arm_smmu_cmdq_issue_cmdlist()
785 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG); in arm_smmu_cmdq_issue_cmdlist()
787 llq.prod &= ~CMDQ_PROD_OWNED_FLAG; in arm_smmu_cmdq_issue_cmdlist()
793 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); in arm_smmu_cmdq_issue_cmdlist()
795 prod = queue_inc_prod_n(&llq, n); in arm_smmu_cmdq_issue_cmdlist()
810 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod); in arm_smmu_cmdq_issue_cmdlist()
815 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod); in arm_smmu_cmdq_issue_cmdlist()
819 &cmdq->q.llq.atomic.prod); in arm_smmu_cmdq_issue_cmdlist()
827 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod); in arm_smmu_cmdq_issue_cmdlist()
845 llq.prod = queue_inc_prod_n(&llq, n); in arm_smmu_cmdq_issue_cmdlist()
846 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq); in arm_smmu_cmdq_issue_cmdlist()
850 llq.prod, in arm_smmu_cmdq_issue_cmdlist()
860 WRITE_ONCE(cmdq->q.llq.cons, llq.cons); in arm_smmu_cmdq_issue_cmdlist()
1555 struct arm_smmu_ll_queue *llq = &q->llq; in arm_smmu_evtq_thread() local
1582 } while (!queue_empty(llq)); in arm_smmu_evtq_thread()
1631 struct arm_smmu_ll_queue *llq = &q->llq; in arm_smmu_priq_thread() local
1640 } while (!queue_empty(llq)); in arm_smmu_priq_thread()
2921 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3; in arm_smmu_init_one_queue()
2927 q->llq.max_n_shift--; in arm_smmu_init_one_queue()
2939 1 << q->llq.max_n_shift, name); in arm_smmu_init_one_queue()
2948 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift); in arm_smmu_init_one_queue()
2950 q->llq.prod = q->llq.cons = 0; in arm_smmu_init_one_queue()
2964 unsigned int nents = 1 << cmdq->q.llq.max_n_shift; in arm_smmu_cmdq_init()
3386 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); in arm_smmu_device_reset()
3387 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); in arm_smmu_device_reset()
3412 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); in arm_smmu_device_reset()
3413 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); in arm_smmu_device_reset()
3427 writel_relaxed(smmu->priq.q.llq.prod, in arm_smmu_device_reset()
3429 writel_relaxed(smmu->priq.q.llq.cons, in arm_smmu_device_reset()
3627 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
3629 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { in arm_smmu_device_hw_probe()
3641 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
3643 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()