2  * DMA driver for Xilinx ZynqMP DMA Engine
25 #include <linux/io-64-nonatomic-lo-hi.h>
60 #define ZYNQMP_DMA_AXI_RD_DATA		BIT(8)
86 #define ZYNQMP_DMA_AWCACHE		GENMASK(11, 8)
87 #define ZYNQMP_DMA_AWCACHE_OFST		8
94 #define ZYNQMP_DMA_AXCOHRNT		BIT(8)
147 #define ZYNQMP_DMA_DESC_SIZE(chan)	(chan->desc_size)
155  * struct zynqmp_dma_desc_ll - Hw linked list descriptor
171  * struct zynqmp_dma_desc_sw - Per Transaction structure
197  * struct zynqmp_dma_chan - Driver specific DMA channel structure
247  * struct zynqmp_dma_device - DMA device structure
265 	lo_hi_writeq(value, chan->regs + reg);  in zynqmp_dma_writeq()
269  * zynqmp_dma_update_desc_to_ctrlr - Updates descriptor to the controller
270  * @chan: ZynqMP DMA DMA channel pointer
278 	addr = desc->src_p;  in zynqmp_dma_update_desc_to_ctrlr()
280 	addr = desc->dst_p;  in zynqmp_dma_update_desc_to_ctrlr()
285  * zynqmp_dma_desc_config_eod - Mark the descriptor as end descriptor
286  * @chan: ZynqMP DMA channel pointer
294 	hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_desc_config_eod()
296 	hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_COMP_INT | ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_desc_config_eod()
300  * zynqmp_dma_config_sg_ll_desc - Configure the linked list descriptor
301  * @chan: ZynqMP DMA channel pointer
315 	sdesc->size = ddesc->size = len;  in zynqmp_dma_config_sg_ll_desc()
316 	sdesc->addr = src;  in zynqmp_dma_config_sg_ll_desc()
317 	ddesc->addr = dst;  in zynqmp_dma_config_sg_ll_desc()
319 	sdesc->ctrl = ddesc->ctrl = ZYNQMP_DMA_DESC_CTRL_SIZE_256;  in zynqmp_dma_config_sg_ll_desc()
320 	if (chan->is_dmacoherent) {  in zynqmp_dma_config_sg_ll_desc()
321 		sdesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;  in zynqmp_dma_config_sg_ll_desc()
322 		ddesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;  in zynqmp_dma_config_sg_ll_desc()
326 		dma_addr_t addr = chan->desc_pool_p +  in zynqmp_dma_config_sg_ll_desc()
327 			    ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v);  in zynqmp_dma_config_sg_ll_desc()
329 		prev->nxtdscraddr = addr;  in zynqmp_dma_config_sg_ll_desc()
330 		ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan);  in zynqmp_dma_config_sg_ll_desc()
335  * zynqmp_dma_init - Initialize the channel
336  * @chan: ZynqMP DMA channel pointer
342 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_init()
343 	val = readl(chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_init()
344 	writel(val, chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_init()
346 	if (chan->is_dmacoherent) {  in zynqmp_dma_init()
350 		writel(val, chan->regs + ZYNQMP_DMA_DSCR_ATTR);  in zynqmp_dma_init()
353 	val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_init()
354 	if (chan->is_dmacoherent) {  in zynqmp_dma_init()
360 	writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_init()
363 	val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);  in zynqmp_dma_init()
364 	val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_init()
366 	chan->idle = true;  in zynqmp_dma_init()
370  * zynqmp_dma_tx_submit - Submit DMA transaction
377 	struct zynqmp_dma_chan *chan = to_chan(tx->chan);  in zynqmp_dma_tx_submit()
382 	spin_lock_bh(&chan->lock);  in zynqmp_dma_tx_submit()
385 	if (!list_empty(&chan->pending_list)) {  in zynqmp_dma_tx_submit()
386 		desc = list_last_entry(&chan->pending_list,  in zynqmp_dma_tx_submit()
388 		if (!list_empty(&desc->tx_list))  in zynqmp_dma_tx_submit()
389 			desc = list_last_entry(&desc->tx_list,  in zynqmp_dma_tx_submit()
391 		desc->src_v->nxtdscraddr = new->src_p;  in zynqmp_dma_tx_submit()
392 		desc->src_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_tx_submit()
393 		desc->dst_v->nxtdscraddr = new->dst_p;  in zynqmp_dma_tx_submit()
394 		desc->dst_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_tx_submit()
397 	list_add_tail(&new->node, &chan->pending_list);  in zynqmp_dma_tx_submit()
398 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_tx_submit()
404  * zynqmp_dma_get_descriptor - Get the sw descriptor from the pool
405  * @chan: ZynqMP DMA channel pointer
414 	spin_lock_bh(&chan->lock);  in zynqmp_dma_get_descriptor()
415 	desc = list_first_entry(&chan->free_list,  in zynqmp_dma_get_descriptor()
417 	list_del(&desc->node);  in zynqmp_dma_get_descriptor()
418 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_get_descriptor()
420 	INIT_LIST_HEAD(&desc->tx_list);  in zynqmp_dma_get_descriptor()
422 	memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));  in zynqmp_dma_get_descriptor()
423 	memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));  in zynqmp_dma_get_descriptor()
429  * zynqmp_dma_free_descriptor - Issue pending transactions
430  * @chan: ZynqMP DMA channel pointer
438 	chan->desc_free_cnt++;  in zynqmp_dma_free_descriptor()
439 	list_add_tail(&sdesc->node, &chan->free_list);  in zynqmp_dma_free_descriptor()
440 	list_for_each_entry_safe(child, next, &sdesc->tx_list, node) {  in zynqmp_dma_free_descriptor()
441 		chan->desc_free_cnt++;  in zynqmp_dma_free_descriptor()
442 		list_move_tail(&child->node, &chan->free_list);  in zynqmp_dma_free_descriptor()
447  * zynqmp_dma_free_desc_list - Free descriptors list
448  * @chan: ZynqMP DMA channel pointer
461  * zynqmp_dma_alloc_chan_resources - Allocate channel resources
472 	ret = pm_runtime_get_sync(chan->dev);  in zynqmp_dma_alloc_chan_resources()
476 	chan->sw_desc_pool = kcalloc(ZYNQMP_DMA_NUM_DESCS, sizeof(*desc),  in zynqmp_dma_alloc_chan_resources()
478 	if (!chan->sw_desc_pool)  in zynqmp_dma_alloc_chan_resources()
479 		return -ENOMEM;  in zynqmp_dma_alloc_chan_resources()
481 	chan->idle = true;  in zynqmp_dma_alloc_chan_resources()
482 	chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS;  in zynqmp_dma_alloc_chan_resources()
484 	INIT_LIST_HEAD(&chan->free_list);  in zynqmp_dma_alloc_chan_resources()
487 		desc = chan->sw_desc_pool + i;  in zynqmp_dma_alloc_chan_resources()
488 		dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in zynqmp_dma_alloc_chan_resources()
489 		desc->async_tx.tx_submit = zynqmp_dma_tx_submit;  in zynqmp_dma_alloc_chan_resources()
490 		list_add_tail(&desc->node, &chan->free_list);  in zynqmp_dma_alloc_chan_resources()
493 	chan->desc_pool_v = dma_zalloc_coherent(chan->dev,  in zynqmp_dma_alloc_chan_resources()
494 				(2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS),  in zynqmp_dma_alloc_chan_resources()
495 				&chan->desc_pool_p, GFP_KERNEL);  in zynqmp_dma_alloc_chan_resources()
496 	if (!chan->desc_pool_v)  in zynqmp_dma_alloc_chan_resources()
497 		return -ENOMEM;  in zynqmp_dma_alloc_chan_resources()
500 		desc = chan->sw_desc_pool + i;  in zynqmp_dma_alloc_chan_resources()
501 		desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v +  in zynqmp_dma_alloc_chan_resources()
503 		desc->dst_v = (struct zynqmp_dma_desc_ll *) (desc->src_v + 1);  in zynqmp_dma_alloc_chan_resources()
504 		desc->src_p = chan->desc_pool_p +  in zynqmp_dma_alloc_chan_resources()
506 		desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan);  in zynqmp_dma_alloc_chan_resources()
513  * zynqmp_dma_start - Start DMA channel
514  * @chan: ZynqMP DMA channel pointer
518 	writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER);  in zynqmp_dma_start()
519 	writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);  in zynqmp_dma_start()
520 	chan->idle = false;  in zynqmp_dma_start()
521 	writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2);  in zynqmp_dma_start()
525  * zynqmp_dma_handle_ovfl_int - Process the overflow interrupt
526  * @chan: ZynqMP DMA channel pointer
532 		writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);  in zynqmp_dma_handle_ovfl_int()
534 		readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_handle_ovfl_int()
536 		readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);  in zynqmp_dma_handle_ovfl_int()
543 	val = readl(chan->regs + ZYNQMP_DMA_CTRL0);  in zynqmp_dma_config()
545 	writel(val, chan->regs + ZYNQMP_DMA_CTRL0);  in zynqmp_dma_config()
547 	val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_config()
548 	burst_val = __ilog2_u32(chan->src_burst_len);  in zynqmp_dma_config()
551 	burst_val = __ilog2_u32(chan->dst_burst_len);  in zynqmp_dma_config()
554 	writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_config()
558  * zynqmp_dma_device_config - Zynqmp dma device configuration
569 	chan->src_burst_len = clamp(config->src_maxburst, 1U,  in zynqmp_dma_device_config()
571 	chan->dst_burst_len = clamp(config->dst_maxburst, 1U,  in zynqmp_dma_device_config()
578  * zynqmp_dma_start_transfer - Initiate the new transfer
579  * @chan: ZynqMP DMA channel pointer
585 	if (!chan->idle)  in zynqmp_dma_start_transfer()
590 	desc = list_first_entry_or_null(&chan->pending_list,  in zynqmp_dma_start_transfer()
595 	list_splice_tail_init(&chan->pending_list, &chan->active_list);  in zynqmp_dma_start_transfer()
602  * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors
603  * @chan: ZynqMP DMA channel
609 	list_for_each_entry_safe(desc, next, &chan->done_list, node) {  in zynqmp_dma_chan_desc_cleanup()
613 		list_del(&desc->node);  in zynqmp_dma_chan_desc_cleanup()
615 		callback = desc->async_tx.callback;  in zynqmp_dma_chan_desc_cleanup()
616 		callback_param = desc->async_tx.callback_param;  in zynqmp_dma_chan_desc_cleanup()
618 			spin_unlock(&chan->lock);  in zynqmp_dma_chan_desc_cleanup()
620 			spin_lock(&chan->lock);  in zynqmp_dma_chan_desc_cleanup()
629  * zynqmp_dma_complete_descriptor - Mark the active descriptor as complete
630  * @chan: ZynqMP DMA channel pointer
636 	desc = list_first_entry_or_null(&chan->active_list,  in zynqmp_dma_complete_descriptor()
640 	list_del(&desc->node);  in zynqmp_dma_complete_descriptor()
641 	dma_cookie_complete(&desc->async_tx);  in zynqmp_dma_complete_descriptor()
642 	list_add_tail(&desc->node, &chan->done_list);  in zynqmp_dma_complete_descriptor()
646  * zynqmp_dma_issue_pending - Issue pending transactions
653 	spin_lock_bh(&chan->lock);  in zynqmp_dma_issue_pending()
655 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_issue_pending()
659  * zynqmp_dma_free_descriptors - Free channel descriptors
660  * @chan: ZynqMP DMA channel pointer
664 	zynqmp_dma_free_desc_list(chan, &chan->active_list);  in zynqmp_dma_free_descriptors()
665 	zynqmp_dma_free_desc_list(chan, &chan->pending_list);  in zynqmp_dma_free_descriptors()
666 	zynqmp_dma_free_desc_list(chan, &chan->done_list);  in zynqmp_dma_free_descriptors()
670  * zynqmp_dma_free_chan_resources - Free channel resources
677 	spin_lock_bh(&chan->lock);  in zynqmp_dma_free_chan_resources()
679 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_free_chan_resources()
680 	dma_free_coherent(chan->dev,  in zynqmp_dma_free_chan_resources()
682 		chan->desc_pool_v, chan->desc_pool_p);  in zynqmp_dma_free_chan_resources()
683 	kfree(chan->sw_desc_pool);  in zynqmp_dma_free_chan_resources()
684 	pm_runtime_mark_last_busy(chan->dev);  in zynqmp_dma_free_chan_resources()
685 	pm_runtime_put_autosuspend(chan->dev);  in zynqmp_dma_free_chan_resources()
689  * zynqmp_dma_reset - Reset the channel
690  * @chan: ZynqMP DMA channel pointer
694 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_reset()
703  * zynqmp_dma_irq_handler - ZynqMP DMA Interrupt handler
705  * @data: Pointer to the ZynqMP DMA channel structure
715 	isr = readl(chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_irq_handler()
716 	imr = readl(chan->regs + ZYNQMP_DMA_IMR);  in zynqmp_dma_irq_handler()
719 	writel(isr, chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_irq_handler()
721 		tasklet_schedule(&chan->tasklet);  in zynqmp_dma_irq_handler()
726 		chan->idle = true;  in zynqmp_dma_irq_handler()
729 		chan->err = true;  in zynqmp_dma_irq_handler()
730 		tasklet_schedule(&chan->tasklet);  in zynqmp_dma_irq_handler()
731 		dev_err(chan->dev, "Channel %p has errors\n", chan);  in zynqmp_dma_irq_handler()
737 		dev_dbg(chan->dev, "Channel %p overflow interrupt\n", chan);  in zynqmp_dma_irq_handler()
745  * zynqmp_dma_do_tasklet - Schedule completion tasklet
746  * @data: Pointer to the ZynqMP DMA channel structure
753 	spin_lock(&chan->lock);  in zynqmp_dma_do_tasklet()
755 	if (chan->err) {  in zynqmp_dma_do_tasklet()
757 		chan->err = false;  in zynqmp_dma_do_tasklet()
761 	count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_do_tasklet()
766 		count--;  in zynqmp_dma_do_tasklet()
769 	if (chan->idle)  in zynqmp_dma_do_tasklet()
773 	spin_unlock(&chan->lock);  in zynqmp_dma_do_tasklet()
777  * zynqmp_dma_device_terminate_all - Aborts all transfers on a channel
786 	spin_lock_bh(&chan->lock);  in zynqmp_dma_device_terminate_all()
787 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_device_terminate_all()
789 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_device_terminate_all()
795  * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction
818 	spin_lock_bh(&chan->lock);  in zynqmp_dma_prep_memcpy()
819 	if (desc_cnt > chan->desc_free_cnt) {  in zynqmp_dma_prep_memcpy()
820 		spin_unlock_bh(&chan->lock);  in zynqmp_dma_prep_memcpy()
821 		dev_dbg(chan->dev, "chan %p descs are not available\n", chan);  in zynqmp_dma_prep_memcpy()
824 	chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt;  in zynqmp_dma_prep_memcpy()
825 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_prep_memcpy()
832 		desc = (struct zynqmp_dma_desc_ll *)new->src_v;  in zynqmp_dma_prep_memcpy()
836 		len -= copy;  in zynqmp_dma_prep_memcpy()
842 			list_add_tail(&new->node, &first->tx_list);  in zynqmp_dma_prep_memcpy()
846 	async_tx_ack(&first->async_tx);  in zynqmp_dma_prep_memcpy()
847 	first->async_tx.flags = flags;  in zynqmp_dma_prep_memcpy()
848 	return &first->async_tx;  in zynqmp_dma_prep_memcpy()
852  * zynqmp_dma_chan_remove - Channel remove function
853  * @chan: ZynqMP DMA channel pointer
860 	if (chan->irq)  in zynqmp_dma_chan_remove()
861 		devm_free_irq(chan->zdev->dev, chan->irq, chan);  in zynqmp_dma_chan_remove()
862 	tasklet_kill(&chan->tasklet);  in zynqmp_dma_chan_remove()
863 	list_del(&chan->common.device_node);  in zynqmp_dma_chan_remove()
867  * zynqmp_dma_chan_probe - Per Channel Probing
878 	struct device_node *node = pdev->dev.of_node;  in zynqmp_dma_chan_probe()
881 	chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL);  in zynqmp_dma_chan_probe()
883 		return -ENOMEM;  in zynqmp_dma_chan_probe()
884 	chan->dev = zdev->dev;  in zynqmp_dma_chan_probe()
885 	chan->zdev = zdev;  in zynqmp_dma_chan_probe()
888 	chan->regs = devm_ioremap_resource(&pdev->dev, res);  in zynqmp_dma_chan_probe()
889 	if (IS_ERR(chan->regs))  in zynqmp_dma_chan_probe()
890 		return PTR_ERR(chan->regs);  in zynqmp_dma_chan_probe()
892 	chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64;  in zynqmp_dma_chan_probe()
893 	chan->dst_burst_len = ZYNQMP_DMA_MAX_DST_BURST_LEN;  in zynqmp_dma_chan_probe()
894 	chan->src_burst_len = ZYNQMP_DMA_MAX_SRC_BURST_LEN;  in zynqmp_dma_chan_probe()
895 	err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width);  in zynqmp_dma_chan_probe()
897 		dev_err(&pdev->dev, "missing xlnx,bus-width property\n");  in zynqmp_dma_chan_probe()
901 	if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 &&  in zynqmp_dma_chan_probe()
902 	    chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) {  in zynqmp_dma_chan_probe()
903 		dev_err(zdev->dev, "invalid bus-width value");  in zynqmp_dma_chan_probe()
904 		return -EINVAL;  in zynqmp_dma_chan_probe()
907 	chan->is_dmacoherent =  of_property_read_bool(node, "dma-coherent");  in zynqmp_dma_chan_probe()
908 	zdev->chan = chan;  in zynqmp_dma_chan_probe()
909 	tasklet_init(&chan->tasklet, zynqmp_dma_do_tasklet, (ulong)chan);  in zynqmp_dma_chan_probe()
910 	spin_lock_init(&chan->lock);  in zynqmp_dma_chan_probe()
911 	INIT_LIST_HEAD(&chan->active_list);  in zynqmp_dma_chan_probe()
912 	INIT_LIST_HEAD(&chan->pending_list);  in zynqmp_dma_chan_probe()
913 	INIT_LIST_HEAD(&chan->done_list);  in zynqmp_dma_chan_probe()
914 	INIT_LIST_HEAD(&chan->free_list);  in zynqmp_dma_chan_probe()
916 	dma_cookie_init(&chan->common);  in zynqmp_dma_chan_probe()
917 	chan->common.device = &zdev->common;  in zynqmp_dma_chan_probe()
918 	list_add_tail(&chan->common.device_node, &zdev->common.channels);  in zynqmp_dma_chan_probe()
921 	chan->irq = platform_get_irq(pdev, 0);  in zynqmp_dma_chan_probe()
922 	if (chan->irq < 0)  in zynqmp_dma_chan_probe()
923 		return -ENXIO;  in zynqmp_dma_chan_probe()
924 	err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0,  in zynqmp_dma_chan_probe()
925 			       "zynqmp-dma", chan);  in zynqmp_dma_chan_probe()
929 	chan->desc_size = sizeof(struct zynqmp_dma_desc_ll);  in zynqmp_dma_chan_probe()
930 	chan->idle = true;  in zynqmp_dma_chan_probe()
935  * of_zynqmp_dma_xlate - Translation function
944 	struct zynqmp_dma_device *zdev = ofdma->of_dma_data;  in of_zynqmp_dma_xlate()
946 	return dma_get_slave_channel(&zdev->chan->common);  in of_zynqmp_dma_xlate()
950  * zynqmp_dma_suspend - Suspend method for the driver
965  * zynqmp_dma_resume - Resume from suspend
980  * zynqmp_dma_runtime_suspend - Runtime suspend method for the driver
990 	clk_disable_unprepare(zdev->clk_main);  in zynqmp_dma_runtime_suspend()
991 	clk_disable_unprepare(zdev->clk_apb);  in zynqmp_dma_runtime_suspend()
997  * zynqmp_dma_runtime_resume - Runtime suspend method for the driver
1008 	err = clk_prepare_enable(zdev->clk_main);  in zynqmp_dma_runtime_resume()
1014 	err = clk_prepare_enable(zdev->clk_apb);  in zynqmp_dma_runtime_resume()
1017 		clk_disable_unprepare(zdev->clk_main);  in zynqmp_dma_runtime_resume()
1031  * zynqmp_dma_probe - Driver probe function
1042 	zdev = devm_kzalloc(&pdev->dev, sizeof(*zdev), GFP_KERNEL);  in zynqmp_dma_probe()
1044 		return -ENOMEM;  in zynqmp_dma_probe()
1046 	zdev->dev = &pdev->dev;  in zynqmp_dma_probe()
1047 	INIT_LIST_HEAD(&zdev->common.channels);  in zynqmp_dma_probe()
1049 	dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));  in zynqmp_dma_probe()
1050 	dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask);  in zynqmp_dma_probe()
1052 	p = &zdev->common;  in zynqmp_dma_probe()
1053 	p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy;  in zynqmp_dma_probe()
1054 	p->device_terminate_all = zynqmp_dma_device_terminate_all;  in zynqmp_dma_probe()
1055 	p->device_issue_pending = zynqmp_dma_issue_pending;  in zynqmp_dma_probe()
1056 	p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources;  in zynqmp_dma_probe()
1057 	p->device_free_chan_resources = zynqmp_dma_free_chan_resources;  in zynqmp_dma_probe()
1058 	p->device_tx_status = dma_cookie_status;  in zynqmp_dma_probe()
1059 	p->device_config = zynqmp_dma_device_config;  in zynqmp_dma_probe()
1060 	p->dev = &pdev->dev;  in zynqmp_dma_probe()
1062 	zdev->clk_main = devm_clk_get(&pdev->dev, "clk_main");  in zynqmp_dma_probe()
1063 	if (IS_ERR(zdev->clk_main)) {  in zynqmp_dma_probe()
1064 		dev_err(&pdev->dev, "main clock not found.\n");  in zynqmp_dma_probe()
1065 		return PTR_ERR(zdev->clk_main);  in zynqmp_dma_probe()
1068 	zdev->clk_apb = devm_clk_get(&pdev->dev, "clk_apb");  in zynqmp_dma_probe()
1069 	if (IS_ERR(zdev->clk_apb)) {  in zynqmp_dma_probe()
1070 		dev_err(&pdev->dev, "apb clock not found.\n");  in zynqmp_dma_probe()
1071 		return PTR_ERR(zdev->clk_apb);  in zynqmp_dma_probe()
1075 	pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT);  in zynqmp_dma_probe()
1076 	pm_runtime_use_autosuspend(zdev->dev);  in zynqmp_dma_probe()
1077 	pm_runtime_enable(zdev->dev);  in zynqmp_dma_probe()
1078 	pm_runtime_get_sync(zdev->dev);  in zynqmp_dma_probe()
1079 	if (!pm_runtime_enabled(zdev->dev)) {  in zynqmp_dma_probe()
1080 		ret = zynqmp_dma_runtime_resume(zdev->dev);  in zynqmp_dma_probe()
1087 		dev_err(&pdev->dev, "Probing channel failed\n");  in zynqmp_dma_probe()
1091 	p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);  in zynqmp_dma_probe()
1092 	p->src_addr_widths = BIT(zdev->chan->bus_width / 8);  in zynqmp_dma_probe()
1094 	dma_async_device_register(&zdev->common);  in zynqmp_dma_probe()
1096 	ret = of_dma_controller_register(pdev->dev.of_node,  in zynqmp_dma_probe()
1099 		dev_err(&pdev->dev, "Unable to register DMA to DT\n");  in zynqmp_dma_probe()
1100 		dma_async_device_unregister(&zdev->common);  in zynqmp_dma_probe()
1104 	pm_runtime_mark_last_busy(zdev->dev);  in zynqmp_dma_probe()
1105 	pm_runtime_put_sync_autosuspend(zdev->dev);  in zynqmp_dma_probe()
1107 	dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n");  in zynqmp_dma_probe()
1112 	zynqmp_dma_chan_remove(zdev->chan);  in zynqmp_dma_probe()
1114 	if (!pm_runtime_enabled(zdev->dev))  in zynqmp_dma_probe()
1115 		zynqmp_dma_runtime_suspend(zdev->dev);  in zynqmp_dma_probe()
1116 	pm_runtime_disable(zdev->dev);  in zynqmp_dma_probe()
1121  * zynqmp_dma_remove - Driver remove function
1130 	of_dma_controller_free(pdev->dev.of_node);  in zynqmp_dma_remove()
1131 	dma_async_device_unregister(&zdev->common);  in zynqmp_dma_remove()
1133 	zynqmp_dma_chan_remove(zdev->chan);  in zynqmp_dma_remove()
1134 	pm_runtime_disable(zdev->dev);  in zynqmp_dma_remove()
1135 	if (!pm_runtime_enabled(zdev->dev))  in zynqmp_dma_remove()
1136 		zynqmp_dma_runtime_suspend(zdev->dev);  in zynqmp_dma_remove()
1142 	{ .compatible = "xlnx,zynqmp-dma-1.0", },
1149 		.name = "xilinx-zynqmp-dma",
1161 MODULE_DESCRIPTION("Xilinx ZynqMP DMA driver");