Lines Matching refs:ring
106 int (*push_tail)(struct k3_ring *ring, void *elm);
107 int (*push_head)(struct k3_ring *ring, void *elm);
108 int (*pop_tail)(struct k3_ring *ring, void *elm);
109 int (*pop_head)(struct k3_ring *ring, void *elm);
221 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) in k3_ringacc_ring_get_fifo_pos() argument
224 (4 << ring->elm_size); in k3_ringacc_ring_get_fifo_pos()
227 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) in k3_ringacc_get_elm_addr() argument
229 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); in k3_ringacc_get_elm_addr()
232 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
233 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
240 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
241 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
242 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
243 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
252 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
253 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
254 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
255 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
264 static void k3_ringacc_ring_dump(struct k3_ring *ring) in k3_ringacc_ring_dump() argument
266 struct device *dev = ring->parent->dev; in k3_ringacc_ring_dump()
268 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); in k3_ringacc_ring_dump()
269 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, in k3_ringacc_ring_dump()
270 &ring->ring_mem_dma); in k3_ringacc_ring_dump()
272 ring->elm_size, ring->size, ring->mode, ring->proxy_id); in k3_ringacc_ring_dump()
273 dev_dbg(dev, "dump flags %08X\n", ring->flags); in k3_ringacc_ring_dump()
275 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); in k3_ringacc_ring_dump()
276 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); in k3_ringacc_ring_dump()
277 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); in k3_ringacc_ring_dump()
278 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); in k3_ringacc_ring_dump()
279 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); in k3_ringacc_ring_dump()
281 if (ring->ring_mem_virt) in k3_ringacc_ring_dump()
283 16, 1, ring->ring_mem_virt, 16 * 8, false); in k3_ringacc_ring_dump()
366 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) in k3_ringacc_ring_reset_sci() argument
368 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_reset_sci()
375 ring->ring_id, in k3_ringacc_ring_reset_sci()
378 ring->size, in k3_ringacc_ring_reset_sci()
384 ret, ring->ring_id); in k3_ringacc_ring_reset_sci()
387 void k3_ringacc_ring_reset(struct k3_ring *ring) in k3_ringacc_ring_reset() argument
389 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_reset()
392 memset(&ring->state, 0, sizeof(ring->state)); in k3_ringacc_ring_reset()
394 k3_ringacc_ring_reset_sci(ring); in k3_ringacc_ring_reset()
398 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, in k3_ringacc_ring_reconfig_qmode_sci() argument
401 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_reconfig_qmode_sci()
408 ring->ring_id, in k3_ringacc_ring_reconfig_qmode_sci()
417 ret, ring->ring_id); in k3_ringacc_ring_reconfig_qmode_sci()
420 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) in k3_ringacc_ring_reset_dma() argument
422 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_reset_dma()
425 if (!ring->parent->dma_ring_reset_quirk) in k3_ringacc_ring_reset_dma()
429 occ = readl(&ring->rt->occ); in k3_ringacc_ring_reset_dma()
434 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, in k3_ringacc_ring_reset_dma()
435 ring->ring_id, occ); in k3_ringacc_ring_reset_dma()
437 k3_ringacc_ring_reset_sci(ring); in k3_ringacc_ring_reset_dma()
443 if (ring->mode != K3_RINGACC_RING_MODE_RING) in k3_ringacc_ring_reset_dma()
445 ring, K3_RINGACC_RING_MODE_RING); in k3_ringacc_ring_reset_dma()
464 writel(db_ring_cnt_cur, &ring->rt->db); in k3_ringacc_ring_reset_dma()
469 if (ring->mode != K3_RINGACC_RING_MODE_RING) in k3_ringacc_ring_reset_dma()
470 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); in k3_ringacc_ring_reset_dma()
475 k3_ringacc_ring_reset(ring); in k3_ringacc_ring_reset_dma()
479 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) in k3_ringacc_ring_free_sci() argument
481 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_free_sci()
488 ring->ring_id, in k3_ringacc_ring_free_sci()
497 ret, ring->ring_id); in k3_ringacc_ring_free_sci()
500 int k3_ringacc_ring_free(struct k3_ring *ring) in k3_ringacc_ring_free() argument
504 if (!ring) in k3_ringacc_ring_free()
507 ringacc = ring->parent; in k3_ringacc_ring_free()
509 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); in k3_ringacc_ring_free()
511 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) in k3_ringacc_ring_free()
516 if (--ring->use_count) in k3_ringacc_ring_free()
519 if (!(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_free()
522 k3_ringacc_ring_free_sci(ring); in k3_ringacc_ring_free()
525 ring->size * (4 << ring->elm_size), in k3_ringacc_ring_free()
526 ring->ring_mem_virt, ring->ring_mem_dma); in k3_ringacc_ring_free()
527 ring->flags = 0; in k3_ringacc_ring_free()
528 ring->ops = NULL; in k3_ringacc_ring_free()
529 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { in k3_ringacc_ring_free()
530 clear_bit(ring->proxy_id, ringacc->proxy_inuse); in k3_ringacc_ring_free()
531 ring->proxy = NULL; in k3_ringacc_ring_free()
532 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; in k3_ringacc_ring_free()
536 clear_bit(ring->ring_id, ringacc->rings_inuse); in k3_ringacc_ring_free()
544 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) in k3_ringacc_get_ring_id() argument
546 if (!ring) in k3_ringacc_get_ring_id()
549 return ring->ring_id; in k3_ringacc_get_ring_id()
553 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) in k3_ringacc_get_tisci_dev_id() argument
555 if (!ring) in k3_ringacc_get_tisci_dev_id()
558 return ring->parent->tisci_dev_id; in k3_ringacc_get_tisci_dev_id()
562 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) in k3_ringacc_get_ring_irq_num() argument
566 if (!ring) in k3_ringacc_get_ring_irq_num()
569 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); in k3_ringacc_get_ring_irq_num()
576 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) in k3_ringacc_ring_cfg_sci() argument
578 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_cfg_sci()
585 ring_idx = ring->ring_id; in k3_ringacc_ring_cfg_sci()
591 lower_32_bits(ring->ring_mem_dma), in k3_ringacc_ring_cfg_sci()
592 upper_32_bits(ring->ring_mem_dma), in k3_ringacc_ring_cfg_sci()
593 ring->size, in k3_ringacc_ring_cfg_sci()
594 ring->mode, in k3_ringacc_ring_cfg_sci()
595 ring->elm_size, in k3_ringacc_ring_cfg_sci()
604 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) in k3_ringacc_ring_cfg() argument
609 if (!ring || !cfg) in k3_ringacc_ring_cfg()
611 ringacc = ring->parent; in k3_ringacc_ring_cfg()
616 !test_bit(ring->ring_id, ringacc->rings_inuse)) in k3_ringacc_ring_cfg()
620 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && in k3_ringacc_ring_cfg()
624 4 << ring->elm_size); in k3_ringacc_ring_cfg()
636 if (ring->use_count != 1) in k3_ringacc_ring_cfg()
639 ring->size = cfg->size; in k3_ringacc_ring_cfg()
640 ring->elm_size = cfg->elm_size; in k3_ringacc_ring_cfg()
641 ring->mode = cfg->mode; in k3_ringacc_ring_cfg()
642 memset(&ring->state, 0, sizeof(ring->state)); in k3_ringacc_ring_cfg()
644 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) in k3_ringacc_ring_cfg()
645 ring->proxy = ringacc->proxy_target_base + in k3_ringacc_ring_cfg()
646 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; in k3_ringacc_ring_cfg()
648 switch (ring->mode) { in k3_ringacc_ring_cfg()
650 ring->ops = &k3_ring_mode_ring_ops; in k3_ringacc_ring_cfg()
653 if (ring->proxy) in k3_ringacc_ring_cfg()
654 ring->ops = &k3_ring_mode_proxy_ops; in k3_ringacc_ring_cfg()
656 ring->ops = &k3_ring_mode_msg_ops; in k3_ringacc_ring_cfg()
659 ring->ops = NULL; in k3_ringacc_ring_cfg()
664 ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev, in k3_ringacc_ring_cfg()
665 ring->size * (4 << ring->elm_size), in k3_ringacc_ring_cfg()
666 &ring->ring_mem_dma, GFP_KERNEL); in k3_ringacc_ring_cfg()
667 if (!ring->ring_mem_virt) { in k3_ringacc_ring_cfg()
673 ret = k3_ringacc_ring_cfg_sci(ring); in k3_ringacc_ring_cfg()
678 ring->flags |= K3_RING_FLAG_BUSY; in k3_ringacc_ring_cfg()
679 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? in k3_ringacc_ring_cfg()
682 k3_ringacc_ring_dump(ring); in k3_ringacc_ring_cfg()
688 ring->size * (4 << ring->elm_size), in k3_ringacc_ring_cfg()
689 ring->ring_mem_virt, in k3_ringacc_ring_cfg()
690 ring->ring_mem_dma); in k3_ringacc_ring_cfg()
692 ring->ops = NULL; in k3_ringacc_ring_cfg()
694 ring->proxy = NULL; in k3_ringacc_ring_cfg()
699 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) in k3_ringacc_ring_get_size() argument
701 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_get_size()
704 return ring->size; in k3_ringacc_ring_get_size()
708 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) in k3_ringacc_ring_get_free() argument
710 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_get_free()
713 if (!ring->state.free) in k3_ringacc_ring_get_free()
714 ring->state.free = ring->size - readl(&ring->rt->occ); in k3_ringacc_ring_get_free()
716 return ring->state.free; in k3_ringacc_ring_get_free()
720 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) in k3_ringacc_ring_get_occ() argument
722 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_get_occ()
725 return readl(&ring->rt->occ); in k3_ringacc_ring_get_occ()
729 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) in k3_ringacc_ring_is_full() argument
731 return !k3_ringacc_ring_get_free(ring); in k3_ringacc_ring_is_full()
746 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, in k3_ringacc_ring_cfg_proxy() argument
751 val = ring->ring_id; in k3_ringacc_ring_cfg_proxy()
753 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); in k3_ringacc_ring_cfg_proxy()
754 writel(val, &ring->proxy->control); in k3_ringacc_ring_cfg_proxy()
758 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, in k3_ringacc_ring_access_proxy() argument
763 ptr = (void __iomem *)&ring->proxy->data; in k3_ringacc_ring_access_proxy()
768 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); in k3_ringacc_ring_access_proxy()
772 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); in k3_ringacc_ring_access_proxy()
778 ptr += k3_ringacc_ring_get_fifo_pos(ring); in k3_ringacc_ring_access_proxy()
783 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_proxy()
786 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); in k3_ringacc_ring_access_proxy()
787 ring->state.occ--; in k3_ringacc_ring_access_proxy()
791 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_proxy()
794 memcpy_toio(ptr, elem, (4 << ring->elm_size)); in k3_ringacc_ring_access_proxy()
795 ring->state.free--; in k3_ringacc_ring_access_proxy()
801 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, in k3_ringacc_ring_access_proxy()
802 ring->state.occ); in k3_ringacc_ring_access_proxy()
806 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_head_proxy() argument
808 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_push_head_proxy()
812 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_tail_proxy() argument
814 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_push_tail_proxy()
818 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_head_proxy() argument
820 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_pop_head_proxy()
824 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_tail_proxy() argument
826 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_pop_tail_proxy()
830 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, in k3_ringacc_ring_access_io() argument
838 ptr = (void __iomem *)&ring->fifos->head_data; in k3_ringacc_ring_access_io()
842 ptr = (void __iomem *)&ring->fifos->tail_data; in k3_ringacc_ring_access_io()
848 ptr += k3_ringacc_ring_get_fifo_pos(ring); in k3_ringacc_ring_access_io()
853 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_io()
856 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); in k3_ringacc_ring_access_io()
857 ring->state.occ--; in k3_ringacc_ring_access_io()
861 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_io()
864 memcpy_toio(ptr, elem, (4 << ring->elm_size)); in k3_ringacc_ring_access_io()
865 ring->state.free--; in k3_ringacc_ring_access_io()
871 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", in k3_ringacc_ring_access_io()
872 ring->state.free, ring->state.windex, ring->state.occ, in k3_ringacc_ring_access_io()
873 ring->state.rindex); in k3_ringacc_ring_access_io()
877 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_head_io() argument
879 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_push_head_io()
883 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_io() argument
885 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_push_io()
889 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_io() argument
891 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_pop_io()
895 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_tail_io() argument
897 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_pop_tail_io()
901 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_mem() argument
905 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); in k3_ringacc_ring_push_mem()
907 memcpy(elem_ptr, elem, (4 << ring->elm_size)); in k3_ringacc_ring_push_mem()
909 ring->state.windex = (ring->state.windex + 1) % ring->size; in k3_ringacc_ring_push_mem()
910 ring->state.free--; in k3_ringacc_ring_push_mem()
911 writel(1, &ring->rt->db); in k3_ringacc_ring_push_mem()
913 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", in k3_ringacc_ring_push_mem()
914 ring->state.free, ring->state.windex); in k3_ringacc_ring_push_mem()
919 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_mem() argument
923 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); in k3_ringacc_ring_pop_mem()
925 memcpy(elem, elem_ptr, (4 << ring->elm_size)); in k3_ringacc_ring_pop_mem()
927 ring->state.rindex = (ring->state.rindex + 1) % ring->size; in k3_ringacc_ring_pop_mem()
928 ring->state.occ--; in k3_ringacc_ring_pop_mem()
929 writel(-1, &ring->rt->db); in k3_ringacc_ring_pop_mem()
931 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", in k3_ringacc_ring_pop_mem()
932 ring->state.occ, ring->state.rindex, elem_ptr); in k3_ringacc_ring_pop_mem()
936 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push() argument
940 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_push()
943 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", in k3_ringacc_ring_push()
944 ring->state.free, ring->state.windex); in k3_ringacc_ring_push()
946 if (k3_ringacc_ring_is_full(ring)) in k3_ringacc_ring_push()
949 if (ring->ops && ring->ops->push_tail) in k3_ringacc_ring_push()
950 ret = ring->ops->push_tail(ring, elem); in k3_ringacc_ring_push()
956 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_head() argument
960 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_push_head()
963 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", in k3_ringacc_ring_push_head()
964 ring->state.free, ring->state.windex); in k3_ringacc_ring_push_head()
966 if (k3_ringacc_ring_is_full(ring)) in k3_ringacc_ring_push_head()
969 if (ring->ops && ring->ops->push_head) in k3_ringacc_ring_push_head()
970 ret = ring->ops->push_head(ring, elem); in k3_ringacc_ring_push_head()
976 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop() argument
980 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_pop()
983 if (!ring->state.occ) in k3_ringacc_ring_pop()
984 ring->state.occ = k3_ringacc_ring_get_occ(ring); in k3_ringacc_ring_pop()
986 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, in k3_ringacc_ring_pop()
987 ring->state.rindex); in k3_ringacc_ring_pop()
989 if (!ring->state.occ) in k3_ringacc_ring_pop()
992 if (ring->ops && ring->ops->pop_head) in k3_ringacc_ring_pop()
993 ret = ring->ops->pop_head(ring, elem); in k3_ringacc_ring_pop()
999 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_tail() argument
1003 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_pop_tail()
1006 if (!ring->state.occ) in k3_ringacc_ring_pop_tail()
1007 ring->state.occ = k3_ringacc_ring_get_occ(ring); in k3_ringacc_ring_pop_tail()
1009 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", in k3_ringacc_ring_pop_tail()
1010 ring->state.occ, ring->state.rindex); in k3_ringacc_ring_pop_tail()
1012 if (!ring->state.occ) in k3_ringacc_ring_pop_tail()
1015 if (ring->ops && ring->ops->pop_tail) in k3_ringacc_ring_pop_tail()
1016 ret = ring->ops->pop_tail(ring, elem); in k3_ringacc_ring_pop_tail()