Lines Matching +full:zynqmp +full:- +full:dpdma
1 // SPDX-License-Identifier: GPL-2.0
3 * Xilinx ZynqMP DPDMA Engine driver
5 * Copyright (C) 2015 - 2020 Xilinx, Inc.
27 #include <dt-bindings/dma/xlnx-zynqmp-dpdma.h>
30 #include "../virt-dma.h"
32 /* DPDMA registers */
118 /* DPDMA descriptor fields */
141 * struct xilinx_dpdma_hw_desc - DPDMA hardware descriptor
179 * struct xilinx_dpdma_sw_desc - DPDMA software descriptor
180 * @hw: DPDMA hardware descriptor
191 * struct xilinx_dpdma_tx_desc - DPDMA transaction descriptor
208 * struct xilinx_dpdma_chan - DPDMA channel
215 * @video_group: flag if multi-channel operation is needed for video channels
222 * @xdev: DPDMA device
250 * struct xilinx_dpdma_device - DPDMA device
256 * @chan: DPDMA channels
271 /* -----------------------------------------------------------------------------
302 if (chan->id == dpdma_debugfs.chan_id) in xilinx_dpdma_debugfs_desc_done_irq()
329 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
333 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
340 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
367 return -EINVAL; in xilinx_dpdma_debugfs_read()
372 return -ENOMEM; in xilinx_dpdma_debugfs_read()
387 ret = -EFAULT; in xilinx_dpdma_debugfs_read()
408 return -EINVAL; in xilinx_dpdma_debugfs_write()
412 return -EBUSY; in xilinx_dpdma_debugfs_write()
416 return -ENOMEM; in xilinx_dpdma_debugfs_write()
432 ret = -EINVAL; in xilinx_dpdma_debugfs_write()
459 dent = debugfs_create_file("testcase", 0444, xdev->common.dbg_dev_root, in xilinx_dpdma_debugfs_init()
462 dev_err(xdev->dev, "Failed to create debugfs testcase file\n"); in xilinx_dpdma_debugfs_init()
475 /* -----------------------------------------------------------------------------
499 /* -----------------------------------------------------------------------------
504 * xilinx_dpdma_sw_desc_set_dma_addrs - Set DMA addresses in the descriptor
505 * @xdev: DPDMA device
522 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_sw_desc_set_dma_addrs()
525 hw_desc->src_addr = lower_32_bits(dma_addr[0]); in xilinx_dpdma_sw_desc_set_dma_addrs()
526 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
527 hw_desc->addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
532 u32 *addr = &hw_desc->src_addr2; in xilinx_dpdma_sw_desc_set_dma_addrs()
534 addr[i-1] = lower_32_bits(dma_addr[i]); in xilinx_dpdma_sw_desc_set_dma_addrs()
536 if (xdev->ext_addr) { in xilinx_dpdma_sw_desc_set_dma_addrs()
537 u32 *addr_ext = &hw_desc->addr_ext_23; in xilinx_dpdma_sw_desc_set_dma_addrs()
541 addr_msb <<= 16 * ((i - 1) % 2); in xilinx_dpdma_sw_desc_set_dma_addrs()
542 addr_ext[(i - 1) / 2] |= addr_msb; in xilinx_dpdma_sw_desc_set_dma_addrs()
549 prev->hw.next_desc = lower_32_bits(sw_desc->dma_addr); in xilinx_dpdma_sw_desc_set_dma_addrs()
550 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
551 prev->hw.addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
553 upper_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_sw_desc_set_dma_addrs()
557 * xilinx_dpdma_chan_alloc_sw_desc - Allocate a software descriptor
558 * @chan: DPDMA channel
570 sw_desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &dma_addr); in xilinx_dpdma_chan_alloc_sw_desc()
574 sw_desc->dma_addr = dma_addr; in xilinx_dpdma_chan_alloc_sw_desc()
580 * xilinx_dpdma_chan_free_sw_desc - Free a software descriptor
581 * @chan: DPDMA channel
590 dma_pool_free(chan->desc_pool, sw_desc, sw_desc->dma_addr); in xilinx_dpdma_chan_free_sw_desc()
594 * xilinx_dpdma_chan_dump_tx_desc - Dump a tx descriptor
595 * @chan: DPDMA channel
604 struct device *dev = chan->xdev->dev; in xilinx_dpdma_chan_dump_tx_desc()
607 dev_dbg(dev, "------- TX descriptor dump start -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
608 dev_dbg(dev, "------- channel ID = %d -------\n", chan->id); in xilinx_dpdma_chan_dump_tx_desc()
610 list_for_each_entry(sw_desc, &tx_desc->descriptors, node) { in xilinx_dpdma_chan_dump_tx_desc()
611 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_dump_tx_desc()
613 dev_dbg(dev, "------- HW descriptor %d -------\n", i++); in xilinx_dpdma_chan_dump_tx_desc()
614 dev_dbg(dev, "descriptor DMA addr: %pad\n", &sw_desc->dma_addr); in xilinx_dpdma_chan_dump_tx_desc()
615 dev_dbg(dev, "control: 0x%08x\n", hw_desc->control); in xilinx_dpdma_chan_dump_tx_desc()
616 dev_dbg(dev, "desc_id: 0x%08x\n", hw_desc->desc_id); in xilinx_dpdma_chan_dump_tx_desc()
617 dev_dbg(dev, "xfer_size: 0x%08x\n", hw_desc->xfer_size); in xilinx_dpdma_chan_dump_tx_desc()
618 dev_dbg(dev, "hsize_stride: 0x%08x\n", hw_desc->hsize_stride); in xilinx_dpdma_chan_dump_tx_desc()
619 dev_dbg(dev, "timestamp_lsb: 0x%08x\n", hw_desc->timestamp_lsb); in xilinx_dpdma_chan_dump_tx_desc()
620 dev_dbg(dev, "timestamp_msb: 0x%08x\n", hw_desc->timestamp_msb); in xilinx_dpdma_chan_dump_tx_desc()
621 dev_dbg(dev, "addr_ext: 0x%08x\n", hw_desc->addr_ext); in xilinx_dpdma_chan_dump_tx_desc()
622 dev_dbg(dev, "next_desc: 0x%08x\n", hw_desc->next_desc); in xilinx_dpdma_chan_dump_tx_desc()
623 dev_dbg(dev, "src_addr: 0x%08x\n", hw_desc->src_addr); in xilinx_dpdma_chan_dump_tx_desc()
624 dev_dbg(dev, "addr_ext_23: 0x%08x\n", hw_desc->addr_ext_23); in xilinx_dpdma_chan_dump_tx_desc()
625 dev_dbg(dev, "addr_ext_45: 0x%08x\n", hw_desc->addr_ext_45); in xilinx_dpdma_chan_dump_tx_desc()
626 dev_dbg(dev, "src_addr2: 0x%08x\n", hw_desc->src_addr2); in xilinx_dpdma_chan_dump_tx_desc()
627 dev_dbg(dev, "src_addr3: 0x%08x\n", hw_desc->src_addr3); in xilinx_dpdma_chan_dump_tx_desc()
628 dev_dbg(dev, "src_addr4: 0x%08x\n", hw_desc->src_addr4); in xilinx_dpdma_chan_dump_tx_desc()
629 dev_dbg(dev, "src_addr5: 0x%08x\n", hw_desc->src_addr5); in xilinx_dpdma_chan_dump_tx_desc()
630 dev_dbg(dev, "crc: 0x%08x\n", hw_desc->crc); in xilinx_dpdma_chan_dump_tx_desc()
633 dev_dbg(dev, "------- TX descriptor dump end -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
637 * xilinx_dpdma_chan_alloc_tx_desc - Allocate a transaction descriptor
638 * @chan: DPDMA channel
653 INIT_LIST_HEAD(&tx_desc->descriptors); in xilinx_dpdma_chan_alloc_tx_desc()
654 tx_desc->chan = chan; in xilinx_dpdma_chan_alloc_tx_desc()
655 tx_desc->error = false; in xilinx_dpdma_chan_alloc_tx_desc()
661 * xilinx_dpdma_chan_free_tx_desc - Free a virtual DMA descriptor
676 list_for_each_entry_safe(sw_desc, next, &desc->descriptors, node) { in xilinx_dpdma_chan_free_tx_desc()
677 list_del(&sw_desc->node); in xilinx_dpdma_chan_free_tx_desc()
678 xilinx_dpdma_chan_free_sw_desc(desc->chan, sw_desc); in xilinx_dpdma_chan_free_tx_desc()
685 * xilinx_dpdma_chan_prep_interleaved_dma - Prepare an interleaved dma
687 * @chan: DPDMA channel
693 * Return: A DPDMA TX descriptor on success, or NULL.
702 size_t hsize = xt->sgl[0].size; in xilinx_dpdma_chan_prep_interleaved_dma()
703 size_t stride = hsize + xt->sgl[0].icg; in xilinx_dpdma_chan_prep_interleaved_dma()
705 if (!IS_ALIGNED(xt->src_start, XILINX_DPDMA_ALIGN_BYTES)) { in xilinx_dpdma_chan_prep_interleaved_dma()
706 dev_err(chan->xdev->dev, "buffer should be aligned at %d B\n", in xilinx_dpdma_chan_prep_interleaved_dma()
717 xilinx_dpdma_chan_free_tx_desc(&tx_desc->vdesc); in xilinx_dpdma_chan_prep_interleaved_dma()
721 xilinx_dpdma_sw_desc_set_dma_addrs(chan->xdev, sw_desc, sw_desc, in xilinx_dpdma_chan_prep_interleaved_dma()
722 &xt->src_start, 1); in xilinx_dpdma_chan_prep_interleaved_dma()
724 hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_prep_interleaved_dma()
726 hw_desc->xfer_size = hsize * xt->numf; in xilinx_dpdma_chan_prep_interleaved_dma()
727 hw_desc->hsize_stride = in xilinx_dpdma_chan_prep_interleaved_dma()
731 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_PREEMBLE; in xilinx_dpdma_chan_prep_interleaved_dma()
732 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_COMPLETE_INTR; in xilinx_dpdma_chan_prep_interleaved_dma()
733 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_IGNORE_DONE; in xilinx_dpdma_chan_prep_interleaved_dma()
734 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_LAST_OF_FRAME; in xilinx_dpdma_chan_prep_interleaved_dma()
736 list_add_tail(&sw_desc->node, &tx_desc->descriptors); in xilinx_dpdma_chan_prep_interleaved_dma()
741 /* -----------------------------------------------------------------------------
742 * DPDMA Channel Operations
746 * xilinx_dpdma_chan_enable - Enable the channel
747 * @chan: DPDMA channel
755 reg = (XILINX_DPDMA_INTR_CHAN_MASK << chan->id) in xilinx_dpdma_chan_enable()
757 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_enable()
758 reg = (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id) in xilinx_dpdma_chan_enable()
760 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_enable()
769 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, reg); in xilinx_dpdma_chan_enable()
773 * xilinx_dpdma_chan_disable - Disable the channel
774 * @chan: DPDMA channel
782 reg = XILINX_DPDMA_INTR_CHAN_MASK << chan->id; in xilinx_dpdma_chan_disable()
783 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_disable()
784 reg = XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id; in xilinx_dpdma_chan_disable()
785 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_disable()
787 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_ENABLE); in xilinx_dpdma_chan_disable()
791 * xilinx_dpdma_chan_pause - Pause the channel
792 * @chan: DPDMA channel
798 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_pause()
802 * xilinx_dpdma_chan_unpause - Unpause the channel
803 * @chan: DPDMA channel
809 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_unpause()
814 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_video_group_ready()
819 if (xdev->chan[i]->video_group && !xdev->chan[i]->running) in xilinx_dpdma_chan_video_group_ready()
822 if (xdev->chan[i]->video_group) in xilinx_dpdma_chan_video_group_ready()
830 * xilinx_dpdma_chan_queue_transfer - Queue the next transfer
831 * @chan: DPDMA channel
838 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_queue_transfer()
845 lockdep_assert_held(&chan->lock); in xilinx_dpdma_chan_queue_transfer()
847 if (chan->desc.pending) in xilinx_dpdma_chan_queue_transfer()
850 if (!chan->running) { in xilinx_dpdma_chan_queue_transfer()
853 chan->first_frame = true; in xilinx_dpdma_chan_queue_transfer()
854 chan->running = true; in xilinx_dpdma_chan_queue_transfer()
857 vdesc = vchan_next_desc(&chan->vchan); in xilinx_dpdma_chan_queue_transfer()
862 chan->desc.pending = desc; in xilinx_dpdma_chan_queue_transfer()
863 list_del(&desc->vdesc.node); in xilinx_dpdma_chan_queue_transfer()
869 list_for_each_entry(sw_desc, &desc->descriptors, node) in xilinx_dpdma_chan_queue_transfer()
870 sw_desc->hw.desc_id = desc->vdesc.tx.cookie in xilinx_dpdma_chan_queue_transfer()
873 sw_desc = list_first_entry(&desc->descriptors, in xilinx_dpdma_chan_queue_transfer()
875 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR, in xilinx_dpdma_chan_queue_transfer()
876 lower_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_chan_queue_transfer()
877 if (xdev->ext_addr) in xilinx_dpdma_chan_queue_transfer()
878 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE, in xilinx_dpdma_chan_queue_transfer()
880 upper_32_bits(sw_desc->dma_addr))); in xilinx_dpdma_chan_queue_transfer()
882 first_frame = chan->first_frame; in xilinx_dpdma_chan_queue_transfer()
883 chan->first_frame = false; in xilinx_dpdma_chan_queue_transfer()
885 if (chan->video_group) { in xilinx_dpdma_chan_queue_transfer()
894 channels = BIT(chan->id); in xilinx_dpdma_chan_queue_transfer()
902 dpdma_write(xdev->reg, XILINX_DPDMA_GBL, reg); in xilinx_dpdma_chan_queue_transfer()
906 * xilinx_dpdma_chan_ostand - Number of outstanding transactions
907 * @chan: DPDMA channel
916 dpdma_read(chan->reg, XILINX_DPDMA_CH_STATUS)); in xilinx_dpdma_chan_ostand()
920 * xilinx_dpdma_chan_no_ostand - Notify no outstanding transaction event
921 * @chan: DPDMA channel
926 * should be re-enabled when this event is handled. If the channel status
930 * Return: 0 on success. On failure, -EWOULDBLOCK if there's still outstanding
939 dev_dbg(chan->xdev->dev, "%d outstanding transactions\n", cnt); in xilinx_dpdma_chan_notify_no_ostand()
940 return -EWOULDBLOCK; in xilinx_dpdma_chan_notify_no_ostand()
944 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_chan_notify_no_ostand()
945 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_notify_no_ostand()
946 wake_up(&chan->wait_to_stop); in xilinx_dpdma_chan_notify_no_ostand()
952 * xilinx_dpdma_chan_wait_no_ostand - Wait for the no outstanding irq
953 * @chan: DPDMA channel
958 * Return: 0 on success. On failure, -ETIMEOUT for time out, or the error code
966 ret = wait_event_interruptible_timeout(chan->wait_to_stop, in xilinx_dpdma_chan_wait_no_ostand()
970 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_wait_no_ostand()
971 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_wait_no_ostand()
975 dev_err(chan->xdev->dev, "not ready to stop: %d trans\n", in xilinx_dpdma_chan_wait_no_ostand()
979 return -ETIMEDOUT; in xilinx_dpdma_chan_wait_no_ostand()
985 * xilinx_dpdma_chan_poll_no_ostand - Poll the outstanding transaction status
986 * @chan: DPDMA channel
992 * Return: 0 on success, or -ETIMEDOUT.
1002 } while (loop-- > 0 && cnt); in xilinx_dpdma_chan_poll_no_ostand()
1005 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_poll_no_ostand()
1006 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_poll_no_ostand()
1010 dev_err(chan->xdev->dev, "not ready to stop: %d trans\n", in xilinx_dpdma_chan_poll_no_ostand()
1013 return -ETIMEDOUT; in xilinx_dpdma_chan_poll_no_ostand()
1017 * xilinx_dpdma_chan_stop - Stop the channel
1018 * @chan: DPDMA channel
1023 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1034 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1036 chan->running = false; in xilinx_dpdma_chan_stop()
1037 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1043 * xilinx_dpdma_chan_done_irq - Handle hardware descriptor completion
1044 * @chan: DPDMA channel
1046 * Handle completion of the currently active descriptor (@chan->desc.active). As
1056 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_done_irq()
1060 active = chan->desc.active; in xilinx_dpdma_chan_done_irq()
1062 vchan_cyclic_callback(&active->vdesc); in xilinx_dpdma_chan_done_irq()
1064 dev_warn(chan->xdev->dev, in xilinx_dpdma_chan_done_irq()
1067 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_done_irq()
1071 * xilinx_dpdma_chan_vsync_irq - Handle hardware descriptor scheduling
1072 * @chan: DPDMA channel
1085 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_vsync_irq()
1087 pending = chan->desc.pending; in xilinx_dpdma_chan_vsync_irq()
1088 if (!chan->running || !pending) in xilinx_dpdma_chan_vsync_irq()
1091 desc_id = dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_ID) in xilinx_dpdma_chan_vsync_irq()
1095 sw_desc = list_first_entry(&pending->descriptors, in xilinx_dpdma_chan_vsync_irq()
1097 if (sw_desc->hw.desc_id != desc_id) in xilinx_dpdma_chan_vsync_irq()
1104 if (chan->desc.active) in xilinx_dpdma_chan_vsync_irq()
1105 vchan_cookie_complete(&chan->desc.active->vdesc); in xilinx_dpdma_chan_vsync_irq()
1106 chan->desc.active = pending; in xilinx_dpdma_chan_vsync_irq()
1107 chan->desc.pending = NULL; in xilinx_dpdma_chan_vsync_irq()
1112 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_vsync_irq()
1116 * xilinx_dpdma_chan_err - Detect any channel error
1117 * @chan: DPDMA channel
1129 if (chan->running && in xilinx_dpdma_chan_err()
1130 ((isr & (XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id)) || in xilinx_dpdma_chan_err()
1131 (eisr & (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id)))) in xilinx_dpdma_chan_err()
1138 * xilinx_dpdma_chan_handle_err - DPDMA channel error handling
1139 * @chan: DPDMA channel
1148 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_handle_err()
1152 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1154 dev_dbg(xdev->dev, "cur desc addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1155 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE), in xilinx_dpdma_chan_handle_err()
1156 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR)); in xilinx_dpdma_chan_handle_err()
1157 dev_dbg(xdev->dev, "cur payload addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1158 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDRE), in xilinx_dpdma_chan_handle_err()
1159 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDR)); in xilinx_dpdma_chan_handle_err()
1162 chan->running = false; in xilinx_dpdma_chan_handle_err()
1164 if (!chan->desc.active) in xilinx_dpdma_chan_handle_err()
1167 active = chan->desc.active; in xilinx_dpdma_chan_handle_err()
1168 chan->desc.active = NULL; in xilinx_dpdma_chan_handle_err()
1172 if (active->error) in xilinx_dpdma_chan_handle_err()
1173 dev_dbg(xdev->dev, "repeated error on desc\n"); in xilinx_dpdma_chan_handle_err()
1176 if (!chan->desc.pending && in xilinx_dpdma_chan_handle_err()
1177 list_empty(&chan->vchan.desc_issued)) { in xilinx_dpdma_chan_handle_err()
1178 active->error = true; in xilinx_dpdma_chan_handle_err()
1179 list_add_tail(&active->vdesc.node, in xilinx_dpdma_chan_handle_err()
1180 &chan->vchan.desc_issued); in xilinx_dpdma_chan_handle_err()
1182 xilinx_dpdma_chan_free_tx_desc(&active->vdesc); in xilinx_dpdma_chan_handle_err()
1186 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1189 /* -----------------------------------------------------------------------------
1201 if (xt->dir != DMA_MEM_TO_DEV) in xilinx_dpdma_prep_interleaved_dma()
1204 if (!xt->numf || !xt->sgl[0].size) in xilinx_dpdma_prep_interleaved_dma()
1214 vchan_tx_prep(&chan->vchan, &desc->vdesc, flags | DMA_CTRL_ACK); in xilinx_dpdma_prep_interleaved_dma()
1216 return &desc->vdesc.tx; in xilinx_dpdma_prep_interleaved_dma()
1220 * xilinx_dpdma_alloc_chan_resources - Allocate resources for the channel
1225 * Return: 0 on success, or -ENOMEM if failed to allocate a pool.
1232 chan->desc_pool = dma_pool_create(dev_name(chan->xdev->dev), in xilinx_dpdma_alloc_chan_resources()
1233 chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1236 if (!chan->desc_pool) { in xilinx_dpdma_alloc_chan_resources()
1237 dev_err(chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1239 return -ENOMEM; in xilinx_dpdma_alloc_chan_resources()
1246 * xilinx_dpdma_free_chan_resources - Free all resources for the channel
1256 vchan_free_chan_resources(&chan->vchan); in xilinx_dpdma_free_chan_resources()
1258 dma_pool_destroy(chan->desc_pool); in xilinx_dpdma_free_chan_resources()
1259 chan->desc_pool = NULL; in xilinx_dpdma_free_chan_resources()
1267 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_issue_pending()
1268 if (vchan_issue_pending(&chan->vchan)) in xilinx_dpdma_issue_pending()
1270 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_issue_pending()
1280 * The destination address doesn't need to be specified as the DPDMA is in xilinx_dpdma_config()
1283 * fixed both on the DPDMA side and on the DP controller side. in xilinx_dpdma_config()
1286 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_config()
1292 if (chan->id <= ZYNQMP_DPDMA_VIDEO2) in xilinx_dpdma_config()
1293 chan->video_group = config->slave_id != 0; in xilinx_dpdma_config()
1295 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_config()
1315 * xilinx_dpdma_terminate_all - Terminate the channel and descriptors
1327 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1332 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_terminate_all()
1338 if (chan->video_group) { in xilinx_dpdma_terminate_all()
1340 if (xdev->chan[i]->video_group && in xilinx_dpdma_terminate_all()
1341 xdev->chan[i]->running) { in xilinx_dpdma_terminate_all()
1342 xilinx_dpdma_chan_pause(xdev->chan[i]); in xilinx_dpdma_terminate_all()
1343 xdev->chan[i]->video_group = false; in xilinx_dpdma_terminate_all()
1351 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1352 vchan_get_all_descriptors(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1353 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1355 vchan_dma_desc_free_list(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1361 * xilinx_dpdma_synchronize - Synchronize callback execution
1380 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1381 if (chan->desc.pending) { in xilinx_dpdma_synchronize()
1382 vchan_terminate_vdesc(&chan->desc.pending->vdesc); in xilinx_dpdma_synchronize()
1383 chan->desc.pending = NULL; in xilinx_dpdma_synchronize()
1385 if (chan->desc.active) { in xilinx_dpdma_synchronize()
1386 vchan_terminate_vdesc(&chan->desc.active->vdesc); in xilinx_dpdma_synchronize()
1387 chan->desc.active = NULL; in xilinx_dpdma_synchronize()
1389 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1391 vchan_synchronize(&chan->vchan); in xilinx_dpdma_synchronize()
1394 /* -----------------------------------------------------------------------------
1399 * xilinx_dpdma_err - Detect any global error
1415 * xilinx_dpdma_handle_err_irq - Handle DPDMA error interrupt
1416 * @xdev: DPDMA device
1421 * corresponding error interrupts, and those should be re-enabled once handling
1430 dev_dbg_ratelimited(xdev->dev, in xilinx_dpdma_handle_err_irq()
1435 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_handle_err_irq()
1437 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, in xilinx_dpdma_handle_err_irq()
1440 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_handle_err_irq()
1441 if (err || xilinx_dpdma_chan_err(xdev->chan[i], isr, eisr)) in xilinx_dpdma_handle_err_irq()
1442 tasklet_schedule(&xdev->chan[i]->err_task); in xilinx_dpdma_handle_err_irq()
1446 * xilinx_dpdma_enable_irq - Enable interrupts
1447 * @xdev: DPDMA device
1453 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_enable_irq()
1454 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_enable_irq()
1458 * xilinx_dpdma_disable_irq - Disable interrupts
1459 * @xdev: DPDMA device
1465 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_disable_irq()
1466 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_disable_irq()
1470 * xilinx_dpdma_chan_err_task - Per channel tasklet for error handling
1475 * re-enable channel error interrupts, and restart the channel if needed.
1480 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_err_task()
1488 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_err_task()
1489 XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1490 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, in xilinx_dpdma_chan_err_task()
1491 XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1493 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1495 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1506 status = dpdma_read(xdev->reg, XILINX_DPDMA_ISR); in xilinx_dpdma_irq_handler()
1507 error = dpdma_read(xdev->reg, XILINX_DPDMA_EISR); in xilinx_dpdma_irq_handler()
1511 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, status); in xilinx_dpdma_irq_handler()
1512 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, error); in xilinx_dpdma_irq_handler()
1519 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in xilinx_dpdma_irq_handler()
1520 struct xilinx_dpdma_chan *chan = xdev->chan[i]; in xilinx_dpdma_irq_handler()
1529 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1530 xilinx_dpdma_chan_done_irq(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1535 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1536 xilinx_dpdma_chan_notify_no_ostand(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1546 /* -----------------------------------------------------------------------------
1555 chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); in xilinx_dpdma_chan_init()
1557 return -ENOMEM; in xilinx_dpdma_chan_init()
1559 chan->id = chan_id; in xilinx_dpdma_chan_init()
1560 chan->reg = xdev->reg + XILINX_DPDMA_CH_BASE in xilinx_dpdma_chan_init()
1561 + XILINX_DPDMA_CH_OFFSET * chan->id; in xilinx_dpdma_chan_init()
1562 chan->running = false; in xilinx_dpdma_chan_init()
1563 chan->xdev = xdev; in xilinx_dpdma_chan_init()
1565 spin_lock_init(&chan->lock); in xilinx_dpdma_chan_init()
1566 init_waitqueue_head(&chan->wait_to_stop); in xilinx_dpdma_chan_init()
1568 tasklet_setup(&chan->err_task, xilinx_dpdma_chan_err_task); in xilinx_dpdma_chan_init()
1570 chan->vchan.desc_free = xilinx_dpdma_chan_free_tx_desc; in xilinx_dpdma_chan_init()
1571 vchan_init(&chan->vchan, &xdev->common); in xilinx_dpdma_chan_init()
1573 xdev->chan[chan->id] = chan; in xilinx_dpdma_chan_init()
1583 tasklet_kill(&chan->err_task); in xilinx_dpdma_chan_remove()
1584 list_del(&chan->vchan.chan.device_node); in xilinx_dpdma_chan_remove()
1590 struct xilinx_dpdma_device *xdev = ofdma->of_dma_data; in of_dma_xilinx_xlate()
1591 uint32_t chan_id = dma_spec->args[0]; in of_dma_xilinx_xlate()
1593 if (chan_id >= ARRAY_SIZE(xdev->chan)) in of_dma_xilinx_xlate()
1596 if (!xdev->chan[chan_id]) in of_dma_xilinx_xlate()
1599 return dma_get_slave_channel(&xdev->chan[chan_id]->vchan.chan); in of_dma_xilinx_xlate()
1611 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in dpdma_hw_init()
1612 reg = xdev->reg + XILINX_DPDMA_CH_BASE in dpdma_hw_init()
1618 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, XILINX_DPDMA_INTR_ALL); in dpdma_hw_init()
1619 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, XILINX_DPDMA_EINTR_ALL); in dpdma_hw_init()
1629 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); in xilinx_dpdma_probe()
1631 return -ENOMEM; in xilinx_dpdma_probe()
1633 xdev->dev = &pdev->dev; in xilinx_dpdma_probe()
1634 xdev->ext_addr = sizeof(dma_addr_t) > 4; in xilinx_dpdma_probe()
1636 INIT_LIST_HEAD(&xdev->common.channels); in xilinx_dpdma_probe()
1640 xdev->axi_clk = devm_clk_get(xdev->dev, "axi_clk"); in xilinx_dpdma_probe()
1641 if (IS_ERR(xdev->axi_clk)) in xilinx_dpdma_probe()
1642 return PTR_ERR(xdev->axi_clk); in xilinx_dpdma_probe()
1644 xdev->reg = devm_platform_ioremap_resource(pdev, 0); in xilinx_dpdma_probe()
1645 if (IS_ERR(xdev->reg)) in xilinx_dpdma_probe()
1646 return PTR_ERR(xdev->reg); in xilinx_dpdma_probe()
1650 xdev->irq = platform_get_irq(pdev, 0); in xilinx_dpdma_probe()
1651 if (xdev->irq < 0) { in xilinx_dpdma_probe()
1652 dev_err(xdev->dev, "failed to get platform irq\n"); in xilinx_dpdma_probe()
1653 return xdev->irq; in xilinx_dpdma_probe()
1656 ret = request_irq(xdev->irq, xilinx_dpdma_irq_handler, IRQF_SHARED, in xilinx_dpdma_probe()
1657 dev_name(xdev->dev), xdev); in xilinx_dpdma_probe()
1659 dev_err(xdev->dev, "failed to request IRQ\n"); in xilinx_dpdma_probe()
1663 ddev = &xdev->common; in xilinx_dpdma_probe()
1664 ddev->dev = &pdev->dev; in xilinx_dpdma_probe()
1666 dma_cap_set(DMA_SLAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1667 dma_cap_set(DMA_PRIVATE, ddev->cap_mask); in xilinx_dpdma_probe()
1668 dma_cap_set(DMA_INTERLEAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1669 dma_cap_set(DMA_REPEAT, ddev->cap_mask); in xilinx_dpdma_probe()
1670 dma_cap_set(DMA_LOAD_EOT, ddev->cap_mask); in xilinx_dpdma_probe()
1671 ddev->copy_align = fls(XILINX_DPDMA_ALIGN_BYTES - 1); in xilinx_dpdma_probe()
1673 ddev->device_alloc_chan_resources = xilinx_dpdma_alloc_chan_resources; in xilinx_dpdma_probe()
1674 ddev->device_free_chan_resources = xilinx_dpdma_free_chan_resources; in xilinx_dpdma_probe()
1675 ddev->device_prep_interleaved_dma = xilinx_dpdma_prep_interleaved_dma; in xilinx_dpdma_probe()
1677 ddev->device_tx_status = dma_cookie_status; in xilinx_dpdma_probe()
1678 ddev->device_issue_pending = xilinx_dpdma_issue_pending; in xilinx_dpdma_probe()
1679 ddev->device_config = xilinx_dpdma_config; in xilinx_dpdma_probe()
1680 ddev->device_pause = xilinx_dpdma_pause; in xilinx_dpdma_probe()
1681 ddev->device_resume = xilinx_dpdma_resume; in xilinx_dpdma_probe()
1682 ddev->device_terminate_all = xilinx_dpdma_terminate_all; in xilinx_dpdma_probe()
1683 ddev->device_synchronize = xilinx_dpdma_synchronize; in xilinx_dpdma_probe()
1684 ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED); in xilinx_dpdma_probe()
1685 ddev->directions = BIT(DMA_MEM_TO_DEV); in xilinx_dpdma_probe()
1686 ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; in xilinx_dpdma_probe()
1688 for (i = 0; i < ARRAY_SIZE(xdev->chan); ++i) { in xilinx_dpdma_probe()
1691 dev_err(xdev->dev, "failed to initialize channel %u\n", in xilinx_dpdma_probe()
1697 ret = clk_prepare_enable(xdev->axi_clk); in xilinx_dpdma_probe()
1699 dev_err(xdev->dev, "failed to enable the axi clock\n"); in xilinx_dpdma_probe()
1705 dev_err(xdev->dev, "failed to register the dma device\n"); in xilinx_dpdma_probe()
1709 ret = of_dma_controller_register(xdev->dev->of_node, in xilinx_dpdma_probe()
1712 dev_err(xdev->dev, "failed to register DMA to DT DMA helper\n"); in xilinx_dpdma_probe()
1720 dev_info(&pdev->dev, "Xilinx DPDMA engine is probed\n"); in xilinx_dpdma_probe()
1727 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_probe()
1729 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_probe()
1730 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_probe()
1732 free_irq(xdev->irq, xdev); in xilinx_dpdma_probe()
1743 free_irq(xdev->irq, xdev); in xilinx_dpdma_remove()
1746 of_dma_controller_free(pdev->dev.of_node); in xilinx_dpdma_remove()
1747 dma_async_device_unregister(&xdev->common); in xilinx_dpdma_remove()
1748 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_remove()
1750 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_remove()
1751 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_remove()
1757 { .compatible = "xlnx,zynqmp-dpdma",},
1766 .name = "xilinx-zynqmp-dpdma",
1774 MODULE_DESCRIPTION("Xilinx ZynqMP DPDMA driver");