Lines Matching +full:zynqmp +full:- +full:dpdma
1 // SPDX-License-Identifier: GPL-2.0
3 * Xilinx ZynqMP DPDMA Engine driver
5 * Copyright (C) 2015 - 2020 Xilinx, Inc.
28 #include <dt-bindings/dma/xlnx-zynqmp-dpdma.h>
31 #include "../virt-dma.h"
33 /* DPDMA registers */
119 /* DPDMA descriptor fields */
142 * struct xilinx_dpdma_hw_desc - DPDMA hardware descriptor
180 * struct xilinx_dpdma_sw_desc - DPDMA software descriptor
181 * @hw: DPDMA hardware descriptor
192 * struct xilinx_dpdma_tx_desc - DPDMA transaction descriptor
209 * struct xilinx_dpdma_chan - DPDMA channel
216 * @video_group: flag if multi-channel operation is needed for video channels
223 * @xdev: DPDMA device
251 * struct xilinx_dpdma_device - DPDMA device
257 * @chan: DPDMA channels
272 /* -----------------------------------------------------------------------------
300 if (IS_ENABLED(CONFIG_DEBUG_FS) && chan->id == dpdma_debugfs.chan_id) in xilinx_dpdma_debugfs_desc_done_irq()
327 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
331 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
338 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
365 return -EINVAL; in xilinx_dpdma_debugfs_read()
370 return -ENOMEM; in xilinx_dpdma_debugfs_read()
385 ret = -EFAULT; in xilinx_dpdma_debugfs_read()
406 return -EINVAL; in xilinx_dpdma_debugfs_write()
410 return -EBUSY; in xilinx_dpdma_debugfs_write()
414 return -ENOMEM; in xilinx_dpdma_debugfs_write()
430 ret = -EINVAL; in xilinx_dpdma_debugfs_write()
457 dent = debugfs_create_file("testcase", 0444, xdev->common.dbg_dev_root, in xilinx_dpdma_debugfs_init()
460 dev_err(xdev->dev, "Failed to create debugfs testcase file\n"); in xilinx_dpdma_debugfs_init()
463 /* -----------------------------------------------------------------------------
487 /* -----------------------------------------------------------------------------
492 * xilinx_dpdma_sw_desc_set_dma_addrs - Set DMA addresses in the descriptor
493 * @xdev: DPDMA device
510 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_sw_desc_set_dma_addrs()
513 hw_desc->src_addr = lower_32_bits(dma_addr[0]); in xilinx_dpdma_sw_desc_set_dma_addrs()
514 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
515 hw_desc->addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
520 u32 *addr = &hw_desc->src_addr2; in xilinx_dpdma_sw_desc_set_dma_addrs()
522 addr[i - 1] = lower_32_bits(dma_addr[i]); in xilinx_dpdma_sw_desc_set_dma_addrs()
524 if (xdev->ext_addr) { in xilinx_dpdma_sw_desc_set_dma_addrs()
525 u32 *addr_ext = &hw_desc->addr_ext_23; in xilinx_dpdma_sw_desc_set_dma_addrs()
529 addr_msb <<= 16 * ((i - 1) % 2); in xilinx_dpdma_sw_desc_set_dma_addrs()
530 addr_ext[(i - 1) / 2] |= addr_msb; in xilinx_dpdma_sw_desc_set_dma_addrs()
537 prev->hw.next_desc = lower_32_bits(sw_desc->dma_addr); in xilinx_dpdma_sw_desc_set_dma_addrs()
538 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
539 prev->hw.addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
541 upper_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_sw_desc_set_dma_addrs()
545 * xilinx_dpdma_chan_alloc_sw_desc - Allocate a software descriptor
546 * @chan: DPDMA channel
558 sw_desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &dma_addr); in xilinx_dpdma_chan_alloc_sw_desc()
562 sw_desc->dma_addr = dma_addr; in xilinx_dpdma_chan_alloc_sw_desc()
568 * xilinx_dpdma_chan_free_sw_desc - Free a software descriptor
569 * @chan: DPDMA channel
578 dma_pool_free(chan->desc_pool, sw_desc, sw_desc->dma_addr); in xilinx_dpdma_chan_free_sw_desc()
582 * xilinx_dpdma_chan_dump_tx_desc - Dump a tx descriptor
583 * @chan: DPDMA channel
592 struct device *dev = chan->xdev->dev; in xilinx_dpdma_chan_dump_tx_desc()
595 dev_dbg(dev, "------- TX descriptor dump start -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
596 dev_dbg(dev, "------- channel ID = %d -------\n", chan->id); in xilinx_dpdma_chan_dump_tx_desc()
598 list_for_each_entry(sw_desc, &tx_desc->descriptors, node) { in xilinx_dpdma_chan_dump_tx_desc()
599 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_dump_tx_desc()
601 dev_dbg(dev, "------- HW descriptor %d -------\n", i++); in xilinx_dpdma_chan_dump_tx_desc()
602 dev_dbg(dev, "descriptor DMA addr: %pad\n", &sw_desc->dma_addr); in xilinx_dpdma_chan_dump_tx_desc()
603 dev_dbg(dev, "control: 0x%08x\n", hw_desc->control); in xilinx_dpdma_chan_dump_tx_desc()
604 dev_dbg(dev, "desc_id: 0x%08x\n", hw_desc->desc_id); in xilinx_dpdma_chan_dump_tx_desc()
605 dev_dbg(dev, "xfer_size: 0x%08x\n", hw_desc->xfer_size); in xilinx_dpdma_chan_dump_tx_desc()
606 dev_dbg(dev, "hsize_stride: 0x%08x\n", hw_desc->hsize_stride); in xilinx_dpdma_chan_dump_tx_desc()
607 dev_dbg(dev, "timestamp_lsb: 0x%08x\n", hw_desc->timestamp_lsb); in xilinx_dpdma_chan_dump_tx_desc()
608 dev_dbg(dev, "timestamp_msb: 0x%08x\n", hw_desc->timestamp_msb); in xilinx_dpdma_chan_dump_tx_desc()
609 dev_dbg(dev, "addr_ext: 0x%08x\n", hw_desc->addr_ext); in xilinx_dpdma_chan_dump_tx_desc()
610 dev_dbg(dev, "next_desc: 0x%08x\n", hw_desc->next_desc); in xilinx_dpdma_chan_dump_tx_desc()
611 dev_dbg(dev, "src_addr: 0x%08x\n", hw_desc->src_addr); in xilinx_dpdma_chan_dump_tx_desc()
612 dev_dbg(dev, "addr_ext_23: 0x%08x\n", hw_desc->addr_ext_23); in xilinx_dpdma_chan_dump_tx_desc()
613 dev_dbg(dev, "addr_ext_45: 0x%08x\n", hw_desc->addr_ext_45); in xilinx_dpdma_chan_dump_tx_desc()
614 dev_dbg(dev, "src_addr2: 0x%08x\n", hw_desc->src_addr2); in xilinx_dpdma_chan_dump_tx_desc()
615 dev_dbg(dev, "src_addr3: 0x%08x\n", hw_desc->src_addr3); in xilinx_dpdma_chan_dump_tx_desc()
616 dev_dbg(dev, "src_addr4: 0x%08x\n", hw_desc->src_addr4); in xilinx_dpdma_chan_dump_tx_desc()
617 dev_dbg(dev, "src_addr5: 0x%08x\n", hw_desc->src_addr5); in xilinx_dpdma_chan_dump_tx_desc()
618 dev_dbg(dev, "crc: 0x%08x\n", hw_desc->crc); in xilinx_dpdma_chan_dump_tx_desc()
621 dev_dbg(dev, "------- TX descriptor dump end -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
625 * xilinx_dpdma_chan_alloc_tx_desc - Allocate a transaction descriptor
626 * @chan: DPDMA channel
641 INIT_LIST_HEAD(&tx_desc->descriptors); in xilinx_dpdma_chan_alloc_tx_desc()
642 tx_desc->chan = chan; in xilinx_dpdma_chan_alloc_tx_desc()
643 tx_desc->error = false; in xilinx_dpdma_chan_alloc_tx_desc()
649 * xilinx_dpdma_chan_free_tx_desc - Free a virtual DMA descriptor
664 list_for_each_entry_safe(sw_desc, next, &desc->descriptors, node) { in xilinx_dpdma_chan_free_tx_desc()
665 list_del(&sw_desc->node); in xilinx_dpdma_chan_free_tx_desc()
666 xilinx_dpdma_chan_free_sw_desc(desc->chan, sw_desc); in xilinx_dpdma_chan_free_tx_desc()
673 * xilinx_dpdma_chan_prep_interleaved_dma - Prepare an interleaved dma
675 * @chan: DPDMA channel
681 * Return: A DPDMA TX descriptor on success, or NULL.
690 size_t hsize = xt->sgl[0].size; in xilinx_dpdma_chan_prep_interleaved_dma()
691 size_t stride = hsize + xt->sgl[0].icg; in xilinx_dpdma_chan_prep_interleaved_dma()
693 if (!IS_ALIGNED(xt->src_start, XILINX_DPDMA_ALIGN_BYTES)) { in xilinx_dpdma_chan_prep_interleaved_dma()
694 dev_err(chan->xdev->dev, in xilinx_dpdma_chan_prep_interleaved_dma()
696 chan->id, XILINX_DPDMA_ALIGN_BYTES); in xilinx_dpdma_chan_prep_interleaved_dma()
706 xilinx_dpdma_chan_free_tx_desc(&tx_desc->vdesc); in xilinx_dpdma_chan_prep_interleaved_dma()
710 xilinx_dpdma_sw_desc_set_dma_addrs(chan->xdev, sw_desc, sw_desc, in xilinx_dpdma_chan_prep_interleaved_dma()
711 &xt->src_start, 1); in xilinx_dpdma_chan_prep_interleaved_dma()
713 hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_prep_interleaved_dma()
715 hw_desc->xfer_size = hsize * xt->numf; in xilinx_dpdma_chan_prep_interleaved_dma()
716 hw_desc->hsize_stride = in xilinx_dpdma_chan_prep_interleaved_dma()
720 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_PREEMBLE; in xilinx_dpdma_chan_prep_interleaved_dma()
721 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_COMPLETE_INTR; in xilinx_dpdma_chan_prep_interleaved_dma()
722 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_IGNORE_DONE; in xilinx_dpdma_chan_prep_interleaved_dma()
723 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_LAST_OF_FRAME; in xilinx_dpdma_chan_prep_interleaved_dma()
725 list_add_tail(&sw_desc->node, &tx_desc->descriptors); in xilinx_dpdma_chan_prep_interleaved_dma()
730 /* -----------------------------------------------------------------------------
731 * DPDMA Channel Operations
735 * xilinx_dpdma_chan_enable - Enable the channel
736 * @chan: DPDMA channel
744 reg = (XILINX_DPDMA_INTR_CHAN_MASK << chan->id) in xilinx_dpdma_chan_enable()
746 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_enable()
747 reg = (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id) in xilinx_dpdma_chan_enable()
749 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_enable()
758 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, reg); in xilinx_dpdma_chan_enable()
762 * xilinx_dpdma_chan_disable - Disable the channel
763 * @chan: DPDMA channel
771 reg = XILINX_DPDMA_INTR_CHAN_MASK << chan->id; in xilinx_dpdma_chan_disable()
772 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_disable()
773 reg = XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id; in xilinx_dpdma_chan_disable()
774 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_disable()
776 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_ENABLE); in xilinx_dpdma_chan_disable()
780 * xilinx_dpdma_chan_pause - Pause the channel
781 * @chan: DPDMA channel
787 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_pause()
791 * xilinx_dpdma_chan_unpause - Unpause the channel
792 * @chan: DPDMA channel
798 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_unpause()
803 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_video_group_ready()
808 if (xdev->chan[i]->video_group && !xdev->chan[i]->running) in xilinx_dpdma_chan_video_group_ready()
811 if (xdev->chan[i]->video_group) in xilinx_dpdma_chan_video_group_ready()
819 * xilinx_dpdma_chan_queue_transfer - Queue the next transfer
820 * @chan: DPDMA channel
827 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_queue_transfer()
834 lockdep_assert_held(&chan->lock); in xilinx_dpdma_chan_queue_transfer()
836 if (chan->desc.pending) in xilinx_dpdma_chan_queue_transfer()
839 if (!chan->running) { in xilinx_dpdma_chan_queue_transfer()
842 chan->first_frame = true; in xilinx_dpdma_chan_queue_transfer()
843 chan->running = true; in xilinx_dpdma_chan_queue_transfer()
846 vdesc = vchan_next_desc(&chan->vchan); in xilinx_dpdma_chan_queue_transfer()
851 chan->desc.pending = desc; in xilinx_dpdma_chan_queue_transfer()
852 list_del(&desc->vdesc.node); in xilinx_dpdma_chan_queue_transfer()
858 list_for_each_entry(sw_desc, &desc->descriptors, node) in xilinx_dpdma_chan_queue_transfer()
859 sw_desc->hw.desc_id = desc->vdesc.tx.cookie in xilinx_dpdma_chan_queue_transfer()
862 sw_desc = list_first_entry(&desc->descriptors, in xilinx_dpdma_chan_queue_transfer()
864 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR, in xilinx_dpdma_chan_queue_transfer()
865 lower_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_chan_queue_transfer()
866 if (xdev->ext_addr) in xilinx_dpdma_chan_queue_transfer()
867 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE, in xilinx_dpdma_chan_queue_transfer()
869 upper_32_bits(sw_desc->dma_addr))); in xilinx_dpdma_chan_queue_transfer()
871 first_frame = chan->first_frame; in xilinx_dpdma_chan_queue_transfer()
872 chan->first_frame = false; in xilinx_dpdma_chan_queue_transfer()
874 if (chan->video_group) { in xilinx_dpdma_chan_queue_transfer()
883 channels = BIT(chan->id); in xilinx_dpdma_chan_queue_transfer()
891 dpdma_write(xdev->reg, XILINX_DPDMA_GBL, reg); in xilinx_dpdma_chan_queue_transfer()
895 * xilinx_dpdma_chan_ostand - Number of outstanding transactions
896 * @chan: DPDMA channel
905 dpdma_read(chan->reg, XILINX_DPDMA_CH_STATUS)); in xilinx_dpdma_chan_ostand()
909 * xilinx_dpdma_chan_notify_no_ostand - Notify no outstanding transaction event
910 * @chan: DPDMA channel
915 * should be re-enabled when this event is handled. If the channel status
919 * Return: 0 on success. On failure, -EWOULDBLOCK if there's still outstanding
928 dev_dbg(chan->xdev->dev, in xilinx_dpdma_chan_notify_no_ostand()
930 chan->id, cnt); in xilinx_dpdma_chan_notify_no_ostand()
931 return -EWOULDBLOCK; in xilinx_dpdma_chan_notify_no_ostand()
935 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_chan_notify_no_ostand()
936 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_notify_no_ostand()
937 wake_up(&chan->wait_to_stop); in xilinx_dpdma_chan_notify_no_ostand()
943 * xilinx_dpdma_chan_wait_no_ostand - Wait for the no outstanding irq
944 * @chan: DPDMA channel
949 * Return: 0 on success. On failure, -ETIMEOUT for time out, or the error code
957 ret = wait_event_interruptible_timeout(chan->wait_to_stop, in xilinx_dpdma_chan_wait_no_ostand()
961 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_wait_no_ostand()
962 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_wait_no_ostand()
966 dev_err(chan->xdev->dev, "chan%u: not ready to stop: %d trans\n", in xilinx_dpdma_chan_wait_no_ostand()
967 chan->id, xilinx_dpdma_chan_ostand(chan)); in xilinx_dpdma_chan_wait_no_ostand()
970 return -ETIMEDOUT; in xilinx_dpdma_chan_wait_no_ostand()
976 * xilinx_dpdma_chan_poll_no_ostand - Poll the outstanding transaction status
977 * @chan: DPDMA channel
983 * Return: 0 on success, or -ETIMEDOUT.
993 } while (loop-- > 0 && cnt); in xilinx_dpdma_chan_poll_no_ostand()
996 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_poll_no_ostand()
997 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_poll_no_ostand()
1001 dev_err(chan->xdev->dev, "chan%u: not ready to stop: %d trans\n", in xilinx_dpdma_chan_poll_no_ostand()
1002 chan->id, xilinx_dpdma_chan_ostand(chan)); in xilinx_dpdma_chan_poll_no_ostand()
1004 return -ETIMEDOUT; in xilinx_dpdma_chan_poll_no_ostand()
1008 * xilinx_dpdma_chan_stop - Stop the channel
1009 * @chan: DPDMA channel
1014 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1025 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1027 chan->running = false; in xilinx_dpdma_chan_stop()
1028 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1034 * xilinx_dpdma_chan_done_irq - Handle hardware descriptor completion
1035 * @chan: DPDMA channel
1037 * Handle completion of the currently active descriptor (@chan->desc.active). As
1047 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_done_irq()
1051 active = chan->desc.active; in xilinx_dpdma_chan_done_irq()
1053 vchan_cyclic_callback(&active->vdesc); in xilinx_dpdma_chan_done_irq()
1055 dev_warn(chan->xdev->dev, in xilinx_dpdma_chan_done_irq()
1057 chan->id); in xilinx_dpdma_chan_done_irq()
1059 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_done_irq()
1063 * xilinx_dpdma_chan_vsync_irq - Handle hardware descriptor scheduling
1064 * @chan: DPDMA channel
1077 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_vsync_irq()
1079 pending = chan->desc.pending; in xilinx_dpdma_chan_vsync_irq()
1080 if (!chan->running || !pending) in xilinx_dpdma_chan_vsync_irq()
1083 desc_id = dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_ID) in xilinx_dpdma_chan_vsync_irq()
1087 sw_desc = list_first_entry(&pending->descriptors, in xilinx_dpdma_chan_vsync_irq()
1089 if (sw_desc->hw.desc_id != desc_id) { in xilinx_dpdma_chan_vsync_irq()
1090 dev_dbg(chan->xdev->dev, in xilinx_dpdma_chan_vsync_irq()
1092 chan->id, sw_desc->hw.desc_id, desc_id); in xilinx_dpdma_chan_vsync_irq()
1100 if (chan->desc.active) in xilinx_dpdma_chan_vsync_irq()
1101 vchan_cookie_complete(&chan->desc.active->vdesc); in xilinx_dpdma_chan_vsync_irq()
1102 chan->desc.active = pending; in xilinx_dpdma_chan_vsync_irq()
1103 chan->desc.pending = NULL; in xilinx_dpdma_chan_vsync_irq()
1108 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_vsync_irq()
1112 * xilinx_dpdma_chan_err - Detect any channel error
1113 * @chan: DPDMA channel
1125 if (chan->running && in xilinx_dpdma_chan_err()
1126 ((isr & (XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id)) || in xilinx_dpdma_chan_err()
1127 (eisr & (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id)))) in xilinx_dpdma_chan_err()
1134 * xilinx_dpdma_chan_handle_err - DPDMA channel error handling
1135 * @chan: DPDMA channel
1144 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_handle_err()
1148 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1150 dev_dbg(xdev->dev, "chan%u: cur desc addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1151 chan->id, in xilinx_dpdma_chan_handle_err()
1152 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE), in xilinx_dpdma_chan_handle_err()
1153 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR)); in xilinx_dpdma_chan_handle_err()
1154 dev_dbg(xdev->dev, "chan%u: cur payload addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1155 chan->id, in xilinx_dpdma_chan_handle_err()
1156 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDRE), in xilinx_dpdma_chan_handle_err()
1157 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDR)); in xilinx_dpdma_chan_handle_err()
1160 chan->running = false; in xilinx_dpdma_chan_handle_err()
1162 if (!chan->desc.active) in xilinx_dpdma_chan_handle_err()
1165 active = chan->desc.active; in xilinx_dpdma_chan_handle_err()
1166 chan->desc.active = NULL; in xilinx_dpdma_chan_handle_err()
1170 if (active->error) in xilinx_dpdma_chan_handle_err()
1171 dev_dbg(xdev->dev, "chan%u: repeated error on desc\n", in xilinx_dpdma_chan_handle_err()
1172 chan->id); in xilinx_dpdma_chan_handle_err()
1175 if (!chan->desc.pending && in xilinx_dpdma_chan_handle_err()
1176 list_empty(&chan->vchan.desc_issued)) { in xilinx_dpdma_chan_handle_err()
1177 active->error = true; in xilinx_dpdma_chan_handle_err()
1178 list_add_tail(&active->vdesc.node, in xilinx_dpdma_chan_handle_err()
1179 &chan->vchan.desc_issued); in xilinx_dpdma_chan_handle_err()
1181 xilinx_dpdma_chan_free_tx_desc(&active->vdesc); in xilinx_dpdma_chan_handle_err()
1185 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1188 /* -----------------------------------------------------------------------------
1200 if (xt->dir != DMA_MEM_TO_DEV) in xilinx_dpdma_prep_interleaved_dma()
1203 if (!xt->numf || !xt->sgl[0].size) in xilinx_dpdma_prep_interleaved_dma()
1213 vchan_tx_prep(&chan->vchan, &desc->vdesc, flags | DMA_CTRL_ACK); in xilinx_dpdma_prep_interleaved_dma()
1215 return &desc->vdesc.tx; in xilinx_dpdma_prep_interleaved_dma()
1219 * xilinx_dpdma_alloc_chan_resources - Allocate resources for the channel
1224 * Return: 0 on success, or -ENOMEM if failed to allocate a pool.
1231 chan->desc_pool = dma_pool_create(dev_name(chan->xdev->dev), in xilinx_dpdma_alloc_chan_resources()
1232 chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1235 if (!chan->desc_pool) { in xilinx_dpdma_alloc_chan_resources()
1236 dev_err(chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1238 chan->id); in xilinx_dpdma_alloc_chan_resources()
1239 return -ENOMEM; in xilinx_dpdma_alloc_chan_resources()
1246 * xilinx_dpdma_free_chan_resources - Free all resources for the channel
1256 vchan_free_chan_resources(&chan->vchan); in xilinx_dpdma_free_chan_resources()
1258 dma_pool_destroy(chan->desc_pool); in xilinx_dpdma_free_chan_resources()
1259 chan->desc_pool = NULL; in xilinx_dpdma_free_chan_resources()
1267 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_issue_pending()
1268 if (vchan_issue_pending(&chan->vchan)) in xilinx_dpdma_issue_pending()
1270 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_issue_pending()
1281 * The destination address doesn't need to be specified as the DPDMA is in xilinx_dpdma_config()
1284 * fixed both on the DPDMA side and on the DP controller side. in xilinx_dpdma_config()
1292 pconfig = config->peripheral_config; in xilinx_dpdma_config()
1293 if (WARN_ON(pconfig && config->peripheral_size != sizeof(*pconfig))) in xilinx_dpdma_config()
1294 return -EINVAL; in xilinx_dpdma_config()
1296 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_config()
1297 if (chan->id <= ZYNQMP_DPDMA_VIDEO2 && pconfig) in xilinx_dpdma_config()
1298 chan->video_group = pconfig->video_group; in xilinx_dpdma_config()
1299 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_config()
1319 * xilinx_dpdma_terminate_all - Terminate the channel and descriptors
1331 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1336 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_terminate_all()
1342 if (chan->video_group) { in xilinx_dpdma_terminate_all()
1344 if (xdev->chan[i]->video_group && in xilinx_dpdma_terminate_all()
1345 xdev->chan[i]->running) { in xilinx_dpdma_terminate_all()
1346 xilinx_dpdma_chan_pause(xdev->chan[i]); in xilinx_dpdma_terminate_all()
1347 xdev->chan[i]->video_group = false; in xilinx_dpdma_terminate_all()
1355 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1356 vchan_get_all_descriptors(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1357 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1359 vchan_dma_desc_free_list(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1365 * xilinx_dpdma_synchronize - Synchronize callback execution
1384 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1385 if (chan->desc.pending) { in xilinx_dpdma_synchronize()
1386 vchan_terminate_vdesc(&chan->desc.pending->vdesc); in xilinx_dpdma_synchronize()
1387 chan->desc.pending = NULL; in xilinx_dpdma_synchronize()
1389 if (chan->desc.active) { in xilinx_dpdma_synchronize()
1390 vchan_terminate_vdesc(&chan->desc.active->vdesc); in xilinx_dpdma_synchronize()
1391 chan->desc.active = NULL; in xilinx_dpdma_synchronize()
1393 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1395 vchan_synchronize(&chan->vchan); in xilinx_dpdma_synchronize()
1398 /* -----------------------------------------------------------------------------
1403 * xilinx_dpdma_err - Detect any global error
1419 * xilinx_dpdma_handle_err_irq - Handle DPDMA error interrupt
1420 * @xdev: DPDMA device
1425 * corresponding error interrupts, and those should be re-enabled once handling
1434 dev_dbg_ratelimited(xdev->dev, in xilinx_dpdma_handle_err_irq()
1439 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_handle_err_irq()
1441 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, in xilinx_dpdma_handle_err_irq()
1444 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_handle_err_irq()
1445 if (err || xilinx_dpdma_chan_err(xdev->chan[i], isr, eisr)) in xilinx_dpdma_handle_err_irq()
1446 tasklet_schedule(&xdev->chan[i]->err_task); in xilinx_dpdma_handle_err_irq()
1450 * xilinx_dpdma_enable_irq - Enable interrupts
1451 * @xdev: DPDMA device
1457 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_enable_irq()
1458 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_enable_irq()
1462 * xilinx_dpdma_disable_irq - Disable interrupts
1463 * @xdev: DPDMA device
1469 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_disable_irq()
1470 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_disable_irq()
1474 * xilinx_dpdma_chan_err_task - Per channel tasklet for error handling
1479 * re-enable channel error interrupts, and restart the channel if needed.
1484 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_err_task()
1492 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_err_task()
1493 XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1494 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, in xilinx_dpdma_chan_err_task()
1495 XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1497 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1499 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1510 status = dpdma_read(xdev->reg, XILINX_DPDMA_ISR); in xilinx_dpdma_irq_handler()
1511 error = dpdma_read(xdev->reg, XILINX_DPDMA_EISR); in xilinx_dpdma_irq_handler()
1515 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, status); in xilinx_dpdma_irq_handler()
1516 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, error); in xilinx_dpdma_irq_handler()
1523 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in xilinx_dpdma_irq_handler()
1524 struct xilinx_dpdma_chan *chan = xdev->chan[i]; in xilinx_dpdma_irq_handler()
1533 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1534 xilinx_dpdma_chan_done_irq(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1539 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1540 xilinx_dpdma_chan_notify_no_ostand(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1550 /* -----------------------------------------------------------------------------
1559 chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); in xilinx_dpdma_chan_init()
1561 return -ENOMEM; in xilinx_dpdma_chan_init()
1563 chan->id = chan_id; in xilinx_dpdma_chan_init()
1564 chan->reg = xdev->reg + XILINX_DPDMA_CH_BASE in xilinx_dpdma_chan_init()
1565 + XILINX_DPDMA_CH_OFFSET * chan->id; in xilinx_dpdma_chan_init()
1566 chan->running = false; in xilinx_dpdma_chan_init()
1567 chan->xdev = xdev; in xilinx_dpdma_chan_init()
1569 spin_lock_init(&chan->lock); in xilinx_dpdma_chan_init()
1570 init_waitqueue_head(&chan->wait_to_stop); in xilinx_dpdma_chan_init()
1572 tasklet_setup(&chan->err_task, xilinx_dpdma_chan_err_task); in xilinx_dpdma_chan_init()
1574 chan->vchan.desc_free = xilinx_dpdma_chan_free_tx_desc; in xilinx_dpdma_chan_init()
1575 vchan_init(&chan->vchan, &xdev->common); in xilinx_dpdma_chan_init()
1577 xdev->chan[chan->id] = chan; in xilinx_dpdma_chan_init()
1587 tasklet_kill(&chan->err_task); in xilinx_dpdma_chan_remove()
1588 list_del(&chan->vchan.chan.device_node); in xilinx_dpdma_chan_remove()
1594 struct xilinx_dpdma_device *xdev = ofdma->of_dma_data; in of_dma_xilinx_xlate()
1595 u32 chan_id = dma_spec->args[0]; in of_dma_xilinx_xlate()
1597 if (chan_id >= ARRAY_SIZE(xdev->chan)) in of_dma_xilinx_xlate()
1600 if (!xdev->chan[chan_id]) in of_dma_xilinx_xlate()
1603 return dma_get_slave_channel(&xdev->chan[chan_id]->vchan.chan); in of_dma_xilinx_xlate()
1615 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in dpdma_hw_init()
1616 reg = xdev->reg + XILINX_DPDMA_CH_BASE in dpdma_hw_init()
1622 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, XILINX_DPDMA_INTR_ALL); in dpdma_hw_init()
1623 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, XILINX_DPDMA_EINTR_ALL); in dpdma_hw_init()
1633 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); in xilinx_dpdma_probe()
1635 return -ENOMEM; in xilinx_dpdma_probe()
1637 xdev->dev = &pdev->dev; in xilinx_dpdma_probe()
1638 xdev->ext_addr = sizeof(dma_addr_t) > 4; in xilinx_dpdma_probe()
1640 INIT_LIST_HEAD(&xdev->common.channels); in xilinx_dpdma_probe()
1644 xdev->axi_clk = devm_clk_get(xdev->dev, "axi_clk"); in xilinx_dpdma_probe()
1645 if (IS_ERR(xdev->axi_clk)) in xilinx_dpdma_probe()
1646 return PTR_ERR(xdev->axi_clk); in xilinx_dpdma_probe()
1648 xdev->reg = devm_platform_ioremap_resource(pdev, 0); in xilinx_dpdma_probe()
1649 if (IS_ERR(xdev->reg)) in xilinx_dpdma_probe()
1650 return PTR_ERR(xdev->reg); in xilinx_dpdma_probe()
1654 xdev->irq = platform_get_irq(pdev, 0); in xilinx_dpdma_probe()
1655 if (xdev->irq < 0) in xilinx_dpdma_probe()
1656 return xdev->irq; in xilinx_dpdma_probe()
1658 ret = request_irq(xdev->irq, xilinx_dpdma_irq_handler, IRQF_SHARED, in xilinx_dpdma_probe()
1659 dev_name(xdev->dev), xdev); in xilinx_dpdma_probe()
1661 dev_err(xdev->dev, "failed to request IRQ\n"); in xilinx_dpdma_probe()
1665 ddev = &xdev->common; in xilinx_dpdma_probe()
1666 ddev->dev = &pdev->dev; in xilinx_dpdma_probe()
1668 dma_cap_set(DMA_SLAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1669 dma_cap_set(DMA_PRIVATE, ddev->cap_mask); in xilinx_dpdma_probe()
1670 dma_cap_set(DMA_INTERLEAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1671 dma_cap_set(DMA_REPEAT, ddev->cap_mask); in xilinx_dpdma_probe()
1672 dma_cap_set(DMA_LOAD_EOT, ddev->cap_mask); in xilinx_dpdma_probe()
1673 ddev->copy_align = fls(XILINX_DPDMA_ALIGN_BYTES - 1); in xilinx_dpdma_probe()
1675 ddev->device_alloc_chan_resources = xilinx_dpdma_alloc_chan_resources; in xilinx_dpdma_probe()
1676 ddev->device_free_chan_resources = xilinx_dpdma_free_chan_resources; in xilinx_dpdma_probe()
1677 ddev->device_prep_interleaved_dma = xilinx_dpdma_prep_interleaved_dma; in xilinx_dpdma_probe()
1679 ddev->device_tx_status = dma_cookie_status; in xilinx_dpdma_probe()
1680 ddev->device_issue_pending = xilinx_dpdma_issue_pending; in xilinx_dpdma_probe()
1681 ddev->device_config = xilinx_dpdma_config; in xilinx_dpdma_probe()
1682 ddev->device_pause = xilinx_dpdma_pause; in xilinx_dpdma_probe()
1683 ddev->device_resume = xilinx_dpdma_resume; in xilinx_dpdma_probe()
1684 ddev->device_terminate_all = xilinx_dpdma_terminate_all; in xilinx_dpdma_probe()
1685 ddev->device_synchronize = xilinx_dpdma_synchronize; in xilinx_dpdma_probe()
1686 ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED); in xilinx_dpdma_probe()
1687 ddev->directions = BIT(DMA_MEM_TO_DEV); in xilinx_dpdma_probe()
1688 ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; in xilinx_dpdma_probe()
1690 for (i = 0; i < ARRAY_SIZE(xdev->chan); ++i) { in xilinx_dpdma_probe()
1693 dev_err(xdev->dev, "failed to initialize channel %u\n", in xilinx_dpdma_probe()
1699 ret = clk_prepare_enable(xdev->axi_clk); in xilinx_dpdma_probe()
1701 dev_err(xdev->dev, "failed to enable the axi clock\n"); in xilinx_dpdma_probe()
1707 dev_err(xdev->dev, "failed to register the dma device\n"); in xilinx_dpdma_probe()
1711 ret = of_dma_controller_register(xdev->dev->of_node, in xilinx_dpdma_probe()
1714 dev_err(xdev->dev, "failed to register DMA to DT DMA helper\n"); in xilinx_dpdma_probe()
1722 dev_info(&pdev->dev, "Xilinx DPDMA engine is probed\n"); in xilinx_dpdma_probe()
1729 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_probe()
1731 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_probe()
1732 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_probe()
1734 free_irq(xdev->irq, xdev); in xilinx_dpdma_probe()
1745 free_irq(xdev->irq, xdev); in xilinx_dpdma_remove()
1748 of_dma_controller_free(pdev->dev.of_node); in xilinx_dpdma_remove()
1749 dma_async_device_unregister(&xdev->common); in xilinx_dpdma_remove()
1750 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_remove()
1752 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_remove()
1753 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_remove()
1759 { .compatible = "xlnx,zynqmp-dpdma",},
1768 .name = "xilinx-zynqmp-dpdma",
1776 MODULE_DESCRIPTION("Xilinx ZynqMP DPDMA driver");