1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * DMA driver for Xilinx ZynqMP DMA Engine
21 #include <linux/io-64-nonatomic-lo-hi.h>
56 #define ZYNQMP_DMA_AXI_RD_DATA		BIT(8)
82 #define ZYNQMP_DMA_AWCACHE		GENMASK(11, 8)
83 #define ZYNQMP_DMA_AWCACHE_OFST		8
90 #define ZYNQMP_DMA_AXCOHRNT		BIT(8)
143 #define ZYNQMP_DMA_DESC_SIZE(chan)	(chan->desc_size)
151  * struct zynqmp_dma_desc_ll - Hw linked list descriptor
167  * struct zynqmp_dma_desc_sw - Per Transaction structure
193  * struct zynqmp_dma_chan - Driver specific DMA channel structure
243  * struct zynqmp_dma_device - DMA device structure
261 	lo_hi_writeq(value, chan->regs + reg);  in zynqmp_dma_writeq()
265  * zynqmp_dma_update_desc_to_ctrlr - Updates descriptor to the controller
266  * @chan: ZynqMP DMA DMA channel pointer
274 	addr = desc->src_p;  in zynqmp_dma_update_desc_to_ctrlr()
276 	addr = desc->dst_p;  in zynqmp_dma_update_desc_to_ctrlr()
281  * zynqmp_dma_desc_config_eod - Mark the descriptor as end descriptor
282  * @chan: ZynqMP DMA channel pointer
290 	hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_desc_config_eod()
292 	hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_COMP_INT | ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_desc_config_eod()
296  * zynqmp_dma_config_sg_ll_desc - Configure the linked list descriptor
297  * @chan: ZynqMP DMA channel pointer
311 	sdesc->size = ddesc->size = len;  in zynqmp_dma_config_sg_ll_desc()
312 	sdesc->addr = src;  in zynqmp_dma_config_sg_ll_desc()
313 	ddesc->addr = dst;  in zynqmp_dma_config_sg_ll_desc()
315 	sdesc->ctrl = ddesc->ctrl = ZYNQMP_DMA_DESC_CTRL_SIZE_256;  in zynqmp_dma_config_sg_ll_desc()
316 	if (chan->is_dmacoherent) {  in zynqmp_dma_config_sg_ll_desc()
317 		sdesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;  in zynqmp_dma_config_sg_ll_desc()
318 		ddesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;  in zynqmp_dma_config_sg_ll_desc()
322 		dma_addr_t addr = chan->desc_pool_p +  in zynqmp_dma_config_sg_ll_desc()
323 			    ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v);  in zynqmp_dma_config_sg_ll_desc()
325 		prev->nxtdscraddr = addr;  in zynqmp_dma_config_sg_ll_desc()
326 		ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan);  in zynqmp_dma_config_sg_ll_desc()
331  * zynqmp_dma_init - Initialize the channel
332  * @chan: ZynqMP DMA channel pointer
338 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_init()
339 	val = readl(chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_init()
340 	writel(val, chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_init()
342 	if (chan->is_dmacoherent) {  in zynqmp_dma_init()
346 		writel(val, chan->regs + ZYNQMP_DMA_DSCR_ATTR);  in zynqmp_dma_init()
349 	val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_init()
350 	if (chan->is_dmacoherent) {  in zynqmp_dma_init()
356 	writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_init()
359 	val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);  in zynqmp_dma_init()
360 	val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_init()
362 	chan->idle = true;  in zynqmp_dma_init()
366  * zynqmp_dma_tx_submit - Submit DMA transaction
373 	struct zynqmp_dma_chan *chan = to_chan(tx->chan);  in zynqmp_dma_tx_submit()
379 	spin_lock_irqsave(&chan->lock, irqflags);  in zynqmp_dma_tx_submit()
382 	if (!list_empty(&chan->pending_list)) {  in zynqmp_dma_tx_submit()
383 		desc = list_last_entry(&chan->pending_list,  in zynqmp_dma_tx_submit()
385 		if (!list_empty(&desc->tx_list))  in zynqmp_dma_tx_submit()
386 			desc = list_last_entry(&desc->tx_list,  in zynqmp_dma_tx_submit()
388 		desc->src_v->nxtdscraddr = new->src_p;  in zynqmp_dma_tx_submit()
389 		desc->src_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_tx_submit()
390 		desc->dst_v->nxtdscraddr = new->dst_p;  in zynqmp_dma_tx_submit()
391 		desc->dst_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;  in zynqmp_dma_tx_submit()
394 	list_add_tail(&new->node, &chan->pending_list);  in zynqmp_dma_tx_submit()
395 	spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_tx_submit()
401  * zynqmp_dma_get_descriptor - Get the sw descriptor from the pool
402  * @chan: ZynqMP DMA channel pointer
412 	spin_lock_irqsave(&chan->lock, irqflags);  in zynqmp_dma_get_descriptor()
413 	desc = list_first_entry(&chan->free_list,  in zynqmp_dma_get_descriptor()
415 	list_del(&desc->node);  in zynqmp_dma_get_descriptor()
416 	spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_get_descriptor()
418 	INIT_LIST_HEAD(&desc->tx_list);  in zynqmp_dma_get_descriptor()
420 	memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));  in zynqmp_dma_get_descriptor()
421 	memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));  in zynqmp_dma_get_descriptor()
427  * zynqmp_dma_free_descriptor - Issue pending transactions
428  * @chan: ZynqMP DMA channel pointer
436 	chan->desc_free_cnt++;  in zynqmp_dma_free_descriptor()
437 	list_del(&sdesc->node);  in zynqmp_dma_free_descriptor()
438 	list_add_tail(&sdesc->node, &chan->free_list);  in zynqmp_dma_free_descriptor()
439 	list_for_each_entry_safe(child, next, &sdesc->tx_list, node) {  in zynqmp_dma_free_descriptor()
440 		chan->desc_free_cnt++;  in zynqmp_dma_free_descriptor()
441 		list_move_tail(&child->node, &chan->free_list);  in zynqmp_dma_free_descriptor()
446  * zynqmp_dma_free_desc_list - Free descriptors list
447  * @chan: ZynqMP DMA channel pointer
460  * zynqmp_dma_alloc_chan_resources - Allocate channel resources
471 	ret = pm_runtime_resume_and_get(chan->dev);  in zynqmp_dma_alloc_chan_resources()
475 	chan->sw_desc_pool = kcalloc(ZYNQMP_DMA_NUM_DESCS, sizeof(*desc),  in zynqmp_dma_alloc_chan_resources()
477 	if (!chan->sw_desc_pool)  in zynqmp_dma_alloc_chan_resources()
478 		return -ENOMEM;  in zynqmp_dma_alloc_chan_resources()
480 	chan->idle = true;  in zynqmp_dma_alloc_chan_resources()
481 	chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS;  in zynqmp_dma_alloc_chan_resources()
483 	INIT_LIST_HEAD(&chan->free_list);  in zynqmp_dma_alloc_chan_resources()
486 		desc = chan->sw_desc_pool + i;  in zynqmp_dma_alloc_chan_resources()
487 		dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in zynqmp_dma_alloc_chan_resources()
488 		desc->async_tx.tx_submit = zynqmp_dma_tx_submit;  in zynqmp_dma_alloc_chan_resources()
489 		list_add_tail(&desc->node, &chan->free_list);  in zynqmp_dma_alloc_chan_resources()
492 	chan->desc_pool_v = dma_alloc_coherent(chan->dev,  in zynqmp_dma_alloc_chan_resources()
493 					       (2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS),  in zynqmp_dma_alloc_chan_resources()
494 					       &chan->desc_pool_p, GFP_KERNEL);  in zynqmp_dma_alloc_chan_resources()
495 	if (!chan->desc_pool_v)  in zynqmp_dma_alloc_chan_resources()
496 		return -ENOMEM;  in zynqmp_dma_alloc_chan_resources()
499 		desc = chan->sw_desc_pool + i;  in zynqmp_dma_alloc_chan_resources()
500 		desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v +  in zynqmp_dma_alloc_chan_resources()
502 		desc->dst_v = (struct zynqmp_dma_desc_ll *) (desc->src_v + 1);  in zynqmp_dma_alloc_chan_resources()
503 		desc->src_p = chan->desc_pool_p +  in zynqmp_dma_alloc_chan_resources()
505 		desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan);  in zynqmp_dma_alloc_chan_resources()
512  * zynqmp_dma_start - Start DMA channel
513  * @chan: ZynqMP DMA channel pointer
517 	writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER);  in zynqmp_dma_start()
518 	writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);  in zynqmp_dma_start()
519 	chan->idle = false;  in zynqmp_dma_start()
520 	writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2);  in zynqmp_dma_start()
524  * zynqmp_dma_handle_ovfl_int - Process the overflow interrupt
525  * @chan: ZynqMP DMA channel pointer
531 		writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);  in zynqmp_dma_handle_ovfl_int()
533 		readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_handle_ovfl_int()
535 		readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);  in zynqmp_dma_handle_ovfl_int()
542 	val = readl(chan->regs + ZYNQMP_DMA_CTRL0);  in zynqmp_dma_config()
544 	writel(val, chan->regs + ZYNQMP_DMA_CTRL0);  in zynqmp_dma_config()
546 	val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_config()
547 	burst_val = __ilog2_u32(chan->src_burst_len);  in zynqmp_dma_config()
550 	burst_val = __ilog2_u32(chan->dst_burst_len);  in zynqmp_dma_config()
553 	writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_config()
557  * zynqmp_dma_device_config - Zynqmp dma device configuration
568 	chan->src_burst_len = clamp(config->src_maxburst, 1U,  in zynqmp_dma_device_config()
570 	chan->dst_burst_len = clamp(config->dst_maxburst, 1U,  in zynqmp_dma_device_config()
577  * zynqmp_dma_start_transfer - Initiate the new transfer
578  * @chan: ZynqMP DMA channel pointer
584 	if (!chan->idle)  in zynqmp_dma_start_transfer()
589 	desc = list_first_entry_or_null(&chan->pending_list,  in zynqmp_dma_start_transfer()
594 	list_splice_tail_init(&chan->pending_list, &chan->active_list);  in zynqmp_dma_start_transfer()
601  * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors
602  * @chan: ZynqMP DMA channel
608 	list_for_each_entry_safe(desc, next, &chan->done_list, node) {  in zynqmp_dma_chan_desc_cleanup()
612 		callback = desc->async_tx.callback;  in zynqmp_dma_chan_desc_cleanup()
613 		callback_param = desc->async_tx.callback_param;  in zynqmp_dma_chan_desc_cleanup()
615 			spin_unlock(&chan->lock);  in zynqmp_dma_chan_desc_cleanup()
617 			spin_lock(&chan->lock);  in zynqmp_dma_chan_desc_cleanup()
626  * zynqmp_dma_complete_descriptor - Mark the active descriptor as complete
627  * @chan: ZynqMP DMA channel pointer
633 	desc = list_first_entry_or_null(&chan->active_list,  in zynqmp_dma_complete_descriptor()
637 	list_del(&desc->node);  in zynqmp_dma_complete_descriptor()
638 	dma_cookie_complete(&desc->async_tx);  in zynqmp_dma_complete_descriptor()
639 	list_add_tail(&desc->node, &chan->done_list);  in zynqmp_dma_complete_descriptor()
643  * zynqmp_dma_issue_pending - Issue pending transactions
651 	spin_lock_irqsave(&chan->lock, irqflags);  in zynqmp_dma_issue_pending()
653 	spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_issue_pending()
657  * zynqmp_dma_free_descriptors - Free channel descriptors
658  * @chan: ZynqMP DMA channel pointer
662 	zynqmp_dma_free_desc_list(chan, &chan->active_list);  in zynqmp_dma_free_descriptors()
663 	zynqmp_dma_free_desc_list(chan, &chan->pending_list);  in zynqmp_dma_free_descriptors()
664 	zynqmp_dma_free_desc_list(chan, &chan->done_list);  in zynqmp_dma_free_descriptors()
668  * zynqmp_dma_free_chan_resources - Free channel resources
676 	spin_lock_irqsave(&chan->lock, irqflags);  in zynqmp_dma_free_chan_resources()
678 	spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_free_chan_resources()
679 	dma_free_coherent(chan->dev,  in zynqmp_dma_free_chan_resources()
681 		chan->desc_pool_v, chan->desc_pool_p);  in zynqmp_dma_free_chan_resources()
682 	kfree(chan->sw_desc_pool);  in zynqmp_dma_free_chan_resources()
683 	pm_runtime_mark_last_busy(chan->dev);  in zynqmp_dma_free_chan_resources()
684 	pm_runtime_put_autosuspend(chan->dev);  in zynqmp_dma_free_chan_resources()
688  * zynqmp_dma_reset - Reset the channel
689  * @chan: ZynqMP DMA channel pointer
693 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_reset()
702  * zynqmp_dma_irq_handler - ZynqMP DMA Interrupt handler
704  * @data: Pointer to the ZynqMP DMA channel structure
714 	isr = readl(chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_irq_handler()
715 	imr = readl(chan->regs + ZYNQMP_DMA_IMR);  in zynqmp_dma_irq_handler()
718 	writel(isr, chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_irq_handler()
720 		tasklet_schedule(&chan->tasklet);  in zynqmp_dma_irq_handler()
725 		chan->idle = true;  in zynqmp_dma_irq_handler()
728 		chan->err = true;  in zynqmp_dma_irq_handler()
729 		tasklet_schedule(&chan->tasklet);  in zynqmp_dma_irq_handler()
730 		dev_err(chan->dev, "Channel %p has errors\n", chan);  in zynqmp_dma_irq_handler()
736 		dev_dbg(chan->dev, "Channel %p overflow interrupt\n", chan);  in zynqmp_dma_irq_handler()
744  * zynqmp_dma_do_tasklet - Schedule completion tasklet
745  * @t: Pointer to the ZynqMP DMA channel structure
753 	spin_lock_irqsave(&chan->lock, irqflags);  in zynqmp_dma_do_tasklet()
755 	if (chan->err) {  in zynqmp_dma_do_tasklet()
757 		chan->err = false;  in zynqmp_dma_do_tasklet()
761 	count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_do_tasklet()
766 		count--;  in zynqmp_dma_do_tasklet()
769 	if (chan->idle)  in zynqmp_dma_do_tasklet()
773 	spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_do_tasklet()
777  * zynqmp_dma_device_terminate_all - Aborts all transfers on a channel
787 	spin_lock_irqsave(&chan->lock, irqflags);  in zynqmp_dma_device_terminate_all()
788 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_device_terminate_all()
790 	spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_device_terminate_all()
796  * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction
820 	spin_lock_irqsave(&chan->lock, irqflags);  in zynqmp_dma_prep_memcpy()
821 	if (desc_cnt > chan->desc_free_cnt) {  in zynqmp_dma_prep_memcpy()
822 		spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_prep_memcpy()
823 		dev_dbg(chan->dev, "chan %p descs are not available\n", chan);  in zynqmp_dma_prep_memcpy()
826 	chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt;  in zynqmp_dma_prep_memcpy()
827 	spin_unlock_irqrestore(&chan->lock, irqflags);  in zynqmp_dma_prep_memcpy()
834 		desc = (struct zynqmp_dma_desc_ll *)new->src_v;  in zynqmp_dma_prep_memcpy()
838 		len -= copy;  in zynqmp_dma_prep_memcpy()
844 			list_add_tail(&new->node, &first->tx_list);  in zynqmp_dma_prep_memcpy()
848 	async_tx_ack(&first->async_tx);  in zynqmp_dma_prep_memcpy()
849 	first->async_tx.flags = flags;  in zynqmp_dma_prep_memcpy()
850 	return &first->async_tx;  in zynqmp_dma_prep_memcpy()
854  * zynqmp_dma_chan_remove - Channel remove function
855  * @chan: ZynqMP DMA channel pointer
862 	if (chan->irq)  in zynqmp_dma_chan_remove()
863 		devm_free_irq(chan->zdev->dev, chan->irq, chan);  in zynqmp_dma_chan_remove()
864 	tasklet_kill(&chan->tasklet);  in zynqmp_dma_chan_remove()
865 	list_del(&chan->common.device_node);  in zynqmp_dma_chan_remove()
869  * zynqmp_dma_chan_probe - Per Channel Probing
880 	struct device_node *node = pdev->dev.of_node;  in zynqmp_dma_chan_probe()
883 	chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL);  in zynqmp_dma_chan_probe()
885 		return -ENOMEM;  in zynqmp_dma_chan_probe()
886 	chan->dev = zdev->dev;  in zynqmp_dma_chan_probe()
887 	chan->zdev = zdev;  in zynqmp_dma_chan_probe()
890 	chan->regs = devm_ioremap_resource(&pdev->dev, res);  in zynqmp_dma_chan_probe()
891 	if (IS_ERR(chan->regs))  in zynqmp_dma_chan_probe()
892 		return PTR_ERR(chan->regs);  in zynqmp_dma_chan_probe()
894 	chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64;  in zynqmp_dma_chan_probe()
895 	chan->dst_burst_len = ZYNQMP_DMA_MAX_DST_BURST_LEN;  in zynqmp_dma_chan_probe()
896 	chan->src_burst_len = ZYNQMP_DMA_MAX_SRC_BURST_LEN;  in zynqmp_dma_chan_probe()
897 	err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width);  in zynqmp_dma_chan_probe()
899 		dev_err(&pdev->dev, "missing xlnx,bus-width property\n");  in zynqmp_dma_chan_probe()
903 	if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 &&  in zynqmp_dma_chan_probe()
904 	    chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) {  in zynqmp_dma_chan_probe()
905 		dev_err(zdev->dev, "invalid bus-width value");  in zynqmp_dma_chan_probe()
906 		return -EINVAL;  in zynqmp_dma_chan_probe()
909 	chan->is_dmacoherent =  of_property_read_bool(node, "dma-coherent");  in zynqmp_dma_chan_probe()
910 	zdev->chan = chan;  in zynqmp_dma_chan_probe()
911 	tasklet_setup(&chan->tasklet, zynqmp_dma_do_tasklet);  in zynqmp_dma_chan_probe()
912 	spin_lock_init(&chan->lock);  in zynqmp_dma_chan_probe()
913 	INIT_LIST_HEAD(&chan->active_list);  in zynqmp_dma_chan_probe()
914 	INIT_LIST_HEAD(&chan->pending_list);  in zynqmp_dma_chan_probe()
915 	INIT_LIST_HEAD(&chan->done_list);  in zynqmp_dma_chan_probe()
916 	INIT_LIST_HEAD(&chan->free_list);  in zynqmp_dma_chan_probe()
918 	dma_cookie_init(&chan->common);  in zynqmp_dma_chan_probe()
919 	chan->common.device = &zdev->common;  in zynqmp_dma_chan_probe()
920 	list_add_tail(&chan->common.device_node, &zdev->common.channels);  in zynqmp_dma_chan_probe()
923 	chan->irq = platform_get_irq(pdev, 0);  in zynqmp_dma_chan_probe()
924 	if (chan->irq < 0)  in zynqmp_dma_chan_probe()
925 		return -ENXIO;  in zynqmp_dma_chan_probe()
926 	err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0,  in zynqmp_dma_chan_probe()
927 			       "zynqmp-dma", chan);  in zynqmp_dma_chan_probe()
931 	chan->desc_size = sizeof(struct zynqmp_dma_desc_ll);  in zynqmp_dma_chan_probe()
932 	chan->idle = true;  in zynqmp_dma_chan_probe()
937  * of_zynqmp_dma_xlate - Translation function
946 	struct zynqmp_dma_device *zdev = ofdma->of_dma_data;  in of_zynqmp_dma_xlate()
948 	return dma_get_slave_channel(&zdev->chan->common);  in of_zynqmp_dma_xlate()
952  * zynqmp_dma_suspend - Suspend method for the driver
967  * zynqmp_dma_resume - Resume from suspend
982  * zynqmp_dma_runtime_suspend - Runtime suspend method for the driver
992 	clk_disable_unprepare(zdev->clk_main);  in zynqmp_dma_runtime_suspend()
993 	clk_disable_unprepare(zdev->clk_apb);  in zynqmp_dma_runtime_suspend()
999  * zynqmp_dma_runtime_resume - Runtime suspend method for the driver
1010 	err = clk_prepare_enable(zdev->clk_main);  in zynqmp_dma_runtime_resume()
1016 	err = clk_prepare_enable(zdev->clk_apb);  in zynqmp_dma_runtime_resume()
1019 		clk_disable_unprepare(zdev->clk_main);  in zynqmp_dma_runtime_resume()
1033  * zynqmp_dma_probe - Driver probe function
1044 	zdev = devm_kzalloc(&pdev->dev, sizeof(*zdev), GFP_KERNEL);  in zynqmp_dma_probe()
1046 		return -ENOMEM;  in zynqmp_dma_probe()
1048 	zdev->dev = &pdev->dev;  in zynqmp_dma_probe()
1049 	INIT_LIST_HEAD(&zdev->common.channels);  in zynqmp_dma_probe()
1051 	dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));  in zynqmp_dma_probe()
1052 	dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask);  in zynqmp_dma_probe()
1054 	p = &zdev->common;  in zynqmp_dma_probe()
1055 	p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy;  in zynqmp_dma_probe()
1056 	p->device_terminate_all = zynqmp_dma_device_terminate_all;  in zynqmp_dma_probe()
1057 	p->device_issue_pending = zynqmp_dma_issue_pending;  in zynqmp_dma_probe()
1058 	p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources;  in zynqmp_dma_probe()
1059 	p->device_free_chan_resources = zynqmp_dma_free_chan_resources;  in zynqmp_dma_probe()
1060 	p->device_tx_status = dma_cookie_status;  in zynqmp_dma_probe()
1061 	p->device_config = zynqmp_dma_device_config;  in zynqmp_dma_probe()
1062 	p->dev = &pdev->dev;  in zynqmp_dma_probe()
1064 	zdev->clk_main = devm_clk_get(&pdev->dev, "clk_main");  in zynqmp_dma_probe()
1065 	if (IS_ERR(zdev->clk_main)) {  in zynqmp_dma_probe()
1066 		dev_err(&pdev->dev, "main clock not found.\n");  in zynqmp_dma_probe()
1067 		return PTR_ERR(zdev->clk_main);  in zynqmp_dma_probe()
1070 	zdev->clk_apb = devm_clk_get(&pdev->dev, "clk_apb");  in zynqmp_dma_probe()
1071 	if (IS_ERR(zdev->clk_apb)) {  in zynqmp_dma_probe()
1072 		dev_err(&pdev->dev, "apb clock not found.\n");  in zynqmp_dma_probe()
1073 		return PTR_ERR(zdev->clk_apb);  in zynqmp_dma_probe()
1077 	pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT);  in zynqmp_dma_probe()
1078 	pm_runtime_use_autosuspend(zdev->dev);  in zynqmp_dma_probe()
1079 	pm_runtime_enable(zdev->dev);  in zynqmp_dma_probe()
1080 	pm_runtime_get_sync(zdev->dev);  in zynqmp_dma_probe()
1081 	if (!pm_runtime_enabled(zdev->dev)) {  in zynqmp_dma_probe()
1082 		ret = zynqmp_dma_runtime_resume(zdev->dev);  in zynqmp_dma_probe()
1089 		dev_err(&pdev->dev, "Probing channel failed\n");  in zynqmp_dma_probe()
1093 	p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);  in zynqmp_dma_probe()
1094 	p->src_addr_widths = BIT(zdev->chan->bus_width / 8);  in zynqmp_dma_probe()
1096 	dma_async_device_register(&zdev->common);  in zynqmp_dma_probe()
1098 	ret = of_dma_controller_register(pdev->dev.of_node,  in zynqmp_dma_probe()
1101 		dev_err(&pdev->dev, "Unable to register DMA to DT\n");  in zynqmp_dma_probe()
1102 		dma_async_device_unregister(&zdev->common);  in zynqmp_dma_probe()
1106 	pm_runtime_mark_last_busy(zdev->dev);  in zynqmp_dma_probe()
1107 	pm_runtime_put_sync_autosuspend(zdev->dev);  in zynqmp_dma_probe()
1109 	dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n");  in zynqmp_dma_probe()
1114 	zynqmp_dma_chan_remove(zdev->chan);  in zynqmp_dma_probe()
1116 	if (!pm_runtime_enabled(zdev->dev))  in zynqmp_dma_probe()
1117 		zynqmp_dma_runtime_suspend(zdev->dev);  in zynqmp_dma_probe()
1118 	pm_runtime_disable(zdev->dev);  in zynqmp_dma_probe()
1123  * zynqmp_dma_remove - Driver remove function
1132 	of_dma_controller_free(pdev->dev.of_node);  in zynqmp_dma_remove()
1133 	dma_async_device_unregister(&zdev->common);  in zynqmp_dma_remove()
1135 	zynqmp_dma_chan_remove(zdev->chan);  in zynqmp_dma_remove()
1136 	pm_runtime_disable(zdev->dev);  in zynqmp_dma_remove()
1137 	if (!pm_runtime_enabled(zdev->dev))  in zynqmp_dma_remove()
1138 		zynqmp_dma_runtime_suspend(zdev->dev);  in zynqmp_dma_remove()
1144 	{ .compatible = "xlnx,zynqmp-dma-1.0", },
1151 		.name = "xilinx-zynqmp-dma",
1163 MODULE_DESCRIPTION("Xilinx ZynqMP DMA driver");