Lines Matching +full:uni +full:- +full:directional
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Driver for the Analog Devices AXI-DMAC core
5 * Copyright 2013-2019 Analog Devices Inc.
6 * Author: Lars-Peter Clausen <lars@metafoo.de>
12 #include <linux/dma-mapping.h>
24 #include <linux/fpga/adi-axi-common.h>
26 #include <dt-bindings/dma/axi-dmac.h>
29 #include "virt-dma.h"
32 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has
43 * runtime. By extension this means that each channel is uni-directional. It can
152 return container_of(chan->vchan.chan.device, struct axi_dmac, in chan_to_axi_dmac()
169 writel(val, axi_dmac->base + reg); in axi_dmac_write()
174 return readl(axi_dmac->base + reg); in axi_dmac_read()
179 return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM; in axi_dmac_src_is_mem()
184 return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM; in axi_dmac_dest_is_mem()
191 if ((len & chan->length_align_mask) != 0) /* Not aligned */ in axi_dmac_check_len()
198 if ((addr & chan->address_align_mask) != 0) /* Not aligned */ in axi_dmac_check_addr()
216 desc = chan->next_desc; in axi_dmac_start_transfer()
219 vdesc = vchan_next_desc(&chan->vchan); in axi_dmac_start_transfer()
222 list_move_tail(&vdesc->node, &chan->active_descs); in axi_dmac_start_transfer()
225 sg = &desc->sg[desc->num_submitted]; in axi_dmac_start_transfer()
228 if (sg->id != AXI_DMAC_SG_UNUSED) { in axi_dmac_start_transfer()
229 sg->schedule_when_free = true; in axi_dmac_start_transfer()
233 desc->num_submitted++; in axi_dmac_start_transfer()
234 if (desc->num_submitted == desc->num_sgs || in axi_dmac_start_transfer()
235 desc->have_partial_xfer) { in axi_dmac_start_transfer()
236 if (desc->cyclic) in axi_dmac_start_transfer()
237 desc->num_submitted = 0; /* Start again */ in axi_dmac_start_transfer()
239 chan->next_desc = NULL; in axi_dmac_start_transfer()
242 chan->next_desc = desc; in axi_dmac_start_transfer()
245 sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); in axi_dmac_start_transfer()
248 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr); in axi_dmac_start_transfer()
249 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride); in axi_dmac_start_transfer()
253 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr); in axi_dmac_start_transfer()
254 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride); in axi_dmac_start_transfer()
262 if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback && in axi_dmac_start_transfer()
263 desc->num_sgs == 1) in axi_dmac_start_transfer()
266 if (chan->hw_partial_xfer) in axi_dmac_start_transfer()
269 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); in axi_dmac_start_transfer()
270 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); in axi_dmac_start_transfer()
277 return list_first_entry_or_null(&chan->active_descs, in axi_dmac_active_desc()
284 if (chan->hw_2d) in axi_dmac_total_sg_bytes()
285 return sg->x_len * sg->y_len; in axi_dmac_total_sg_bytes()
287 return sg->x_len; in axi_dmac_total_sg_bytes()
303 list_for_each_entry(desc, &chan->active_descs, vdesc.node) { in axi_dmac_dequeue_partial_xfers()
304 for (i = 0; i < desc->num_sgs; i++) { in axi_dmac_dequeue_partial_xfers()
305 sg = &desc->sg[i]; in axi_dmac_dequeue_partial_xfers()
306 if (sg->id == AXI_DMAC_SG_UNUSED) in axi_dmac_dequeue_partial_xfers()
308 if (sg->id == id) { in axi_dmac_dequeue_partial_xfers()
309 desc->have_partial_xfer = true; in axi_dmac_dequeue_partial_xfers()
310 sg->partial_len = len; in axi_dmac_dequeue_partial_xfers()
320 dev_dbg(dmac->dma_dev.dev, in axi_dmac_dequeue_partial_xfers()
324 dev_warn(dmac->dma_dev.dev, in axi_dmac_dequeue_partial_xfers()
339 struct dmaengine_result *rslt = &active->vdesc.tx_result; in axi_dmac_compute_residue()
340 unsigned int start = active->num_completed - 1; in axi_dmac_compute_residue()
344 rslt->result = DMA_TRANS_NOERROR; in axi_dmac_compute_residue()
345 rslt->residue = 0; in axi_dmac_compute_residue()
351 for (i = start; i < active->num_sgs; i++) { in axi_dmac_compute_residue()
352 sg = &active->sg[i]; in axi_dmac_compute_residue()
354 rslt->residue += (total - sg->partial_len); in axi_dmac_compute_residue()
369 if (chan->hw_partial_xfer && in axi_dmac_transfer_done()
374 sg = &active->sg[active->num_completed]; in axi_dmac_transfer_done()
375 if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ in axi_dmac_transfer_done()
377 if (!(BIT(sg->id) & completed_transfers)) in axi_dmac_transfer_done()
379 active->num_completed++; in axi_dmac_transfer_done()
380 sg->id = AXI_DMAC_SG_UNUSED; in axi_dmac_transfer_done()
381 if (sg->schedule_when_free) { in axi_dmac_transfer_done()
382 sg->schedule_when_free = false; in axi_dmac_transfer_done()
386 if (sg->partial_len) in axi_dmac_transfer_done()
389 if (active->cyclic) in axi_dmac_transfer_done()
390 vchan_cyclic_callback(&active->vdesc); in axi_dmac_transfer_done()
392 if (active->num_completed == active->num_sgs || in axi_dmac_transfer_done()
393 sg->partial_len) { in axi_dmac_transfer_done()
394 if (active->cyclic) { in axi_dmac_transfer_done()
395 active->num_completed = 0; /* wrap around */ in axi_dmac_transfer_done()
397 list_del(&active->vdesc.node); in axi_dmac_transfer_done()
398 vchan_cookie_complete(&active->vdesc); in axi_dmac_transfer_done()
419 spin_lock(&dmac->chan.vchan.lock); in axi_dmac_interrupt_handler()
425 start_next = axi_dmac_transfer_done(&dmac->chan, completed); in axi_dmac_interrupt_handler()
429 axi_dmac_start_transfer(&dmac->chan); in axi_dmac_interrupt_handler()
430 spin_unlock(&dmac->chan.vchan.lock); in axi_dmac_interrupt_handler()
442 spin_lock_irqsave(&chan->vchan.lock, flags); in axi_dmac_terminate_all()
444 chan->next_desc = NULL; in axi_dmac_terminate_all()
445 vchan_get_all_descriptors(&chan->vchan, &head); in axi_dmac_terminate_all()
446 list_splice_tail_init(&chan->active_descs, &head); in axi_dmac_terminate_all()
447 spin_unlock_irqrestore(&chan->vchan.lock, flags); in axi_dmac_terminate_all()
449 vchan_dma_desc_free_list(&chan->vchan, &head); in axi_dmac_terminate_all()
458 vchan_synchronize(&chan->vchan); in axi_dmac_synchronize()
469 spin_lock_irqsave(&chan->vchan.lock, flags); in axi_dmac_issue_pending()
470 if (vchan_issue_pending(&chan->vchan)) in axi_dmac_issue_pending()
472 spin_unlock_irqrestore(&chan->vchan.lock, flags); in axi_dmac_issue_pending()
485 desc->sg[i].id = AXI_DMAC_SG_UNUSED; in axi_dmac_alloc_desc()
487 desc->num_sgs = num_sgs; in axi_dmac_alloc_desc()
502 num_segments = DIV_ROUND_UP(period_len, chan->max_length); in axi_dmac_fill_linear_sg()
505 segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; in axi_dmac_fill_linear_sg()
512 sg->dest_addr = addr; in axi_dmac_fill_linear_sg()
514 sg->src_addr = addr; in axi_dmac_fill_linear_sg()
515 sg->x_len = segment_size; in axi_dmac_fill_linear_sg()
516 sg->y_len = 1; in axi_dmac_fill_linear_sg()
519 len -= segment_size; in axi_dmac_fill_linear_sg()
523 sg->dest_addr = addr; in axi_dmac_fill_linear_sg()
525 sg->src_addr = addr; in axi_dmac_fill_linear_sg()
526 sg->x_len = len; in axi_dmac_fill_linear_sg()
527 sg->y_len = 1; in axi_dmac_fill_linear_sg()
547 if (direction != chan->direction) in axi_dmac_prep_slave_sg()
552 num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length); in axi_dmac_prep_slave_sg()
558 dsg = desc->sg; in axi_dmac_prep_slave_sg()
571 desc->cyclic = false; in axi_dmac_prep_slave_sg()
573 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); in axi_dmac_prep_slave_sg()
585 if (direction != chan->direction) in axi_dmac_prep_dma_cyclic()
596 num_segments = DIV_ROUND_UP(period_len, chan->max_length); in axi_dmac_prep_dma_cyclic()
603 period_len, desc->sg); in axi_dmac_prep_dma_cyclic()
605 desc->cyclic = true; in axi_dmac_prep_dma_cyclic()
607 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); in axi_dmac_prep_dma_cyclic()
618 if (xt->frame_size != 1) in axi_dmac_prep_interleaved()
621 if (xt->dir != chan->direction) in axi_dmac_prep_interleaved()
625 if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start)) in axi_dmac_prep_interleaved()
630 if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start)) in axi_dmac_prep_interleaved()
634 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); in axi_dmac_prep_interleaved()
635 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); in axi_dmac_prep_interleaved()
637 if (chan->hw_2d) { in axi_dmac_prep_interleaved()
638 if (!axi_dmac_check_len(chan, xt->sgl[0].size) || in axi_dmac_prep_interleaved()
639 xt->numf == 0) in axi_dmac_prep_interleaved()
641 if (xt->sgl[0].size + dst_icg > chan->max_length || in axi_dmac_prep_interleaved()
642 xt->sgl[0].size + src_icg > chan->max_length) in axi_dmac_prep_interleaved()
647 if (chan->max_length / xt->sgl[0].size < xt->numf) in axi_dmac_prep_interleaved()
649 if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf)) in axi_dmac_prep_interleaved()
658 desc->sg[0].src_addr = xt->src_start; in axi_dmac_prep_interleaved()
659 desc->sg[0].src_stride = xt->sgl[0].size + src_icg; in axi_dmac_prep_interleaved()
663 desc->sg[0].dest_addr = xt->dst_start; in axi_dmac_prep_interleaved()
664 desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg; in axi_dmac_prep_interleaved()
667 if (chan->hw_2d) { in axi_dmac_prep_interleaved()
668 desc->sg[0].x_len = xt->sgl[0].size; in axi_dmac_prep_interleaved()
669 desc->sg[0].y_len = xt->numf; in axi_dmac_prep_interleaved()
671 desc->sg[0].x_len = xt->sgl[0].size * xt->numf; in axi_dmac_prep_interleaved()
672 desc->sg[0].y_len = 1; in axi_dmac_prep_interleaved()
676 desc->cyclic = true; in axi_dmac_prep_interleaved()
678 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); in axi_dmac_prep_interleaved()
731 chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1; in axi_dmac_adjust_chan_params()
734 chan->direction = DMA_MEM_TO_MEM; in axi_dmac_adjust_chan_params()
736 chan->direction = DMA_MEM_TO_DEV; in axi_dmac_adjust_chan_params()
738 chan->direction = DMA_DEV_TO_MEM; in axi_dmac_adjust_chan_params()
740 chan->direction = DMA_DEV_TO_DEV; in axi_dmac_adjust_chan_params()
760 return -EINVAL; in axi_dmac_parse_chan_dt()
762 ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val); in axi_dmac_parse_chan_dt()
766 return -EINVAL; in axi_dmac_parse_chan_dt()
767 chan->src_type = val; in axi_dmac_parse_chan_dt()
769 ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val); in axi_dmac_parse_chan_dt()
773 return -EINVAL; in axi_dmac_parse_chan_dt()
774 chan->dest_type = val; in axi_dmac_parse_chan_dt()
776 ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val); in axi_dmac_parse_chan_dt()
779 chan->src_width = val / 8; in axi_dmac_parse_chan_dt()
781 ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val); in axi_dmac_parse_chan_dt()
784 chan->dest_width = val / 8; in axi_dmac_parse_chan_dt()
796 of_channels = of_get_child_by_name(dev->of_node, "adi,channels"); in axi_dmac_parse_dt()
798 return -ENODEV; in axi_dmac_parse_dt()
801 ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan); in axi_dmac_parse_dt()
805 return -EINVAL; in axi_dmac_parse_dt()
815 struct axi_dmac_chan *chan = &dmac->chan; in axi_dmac_read_chan_config()
821 return -EFAULT; in axi_dmac_read_chan_config()
827 return -EINVAL; in axi_dmac_read_chan_config()
829 chan->src_type = val; in axi_dmac_read_chan_config()
834 return -EINVAL; in axi_dmac_read_chan_config()
836 chan->dest_type = val; in axi_dmac_read_chan_config()
841 return -EINVAL; in axi_dmac_read_chan_config()
844 chan->src_width = 1 << val; in axi_dmac_read_chan_config()
849 return -EINVAL; in axi_dmac_read_chan_config()
851 chan->dest_width = 1 << val; in axi_dmac_read_chan_config()
860 struct axi_dmac_chan *chan = &dmac->chan; in axi_dmac_detect_caps()
864 chan->hw_cyclic = true; in axi_dmac_detect_caps()
868 chan->hw_2d = true; in axi_dmac_detect_caps()
871 chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); in axi_dmac_detect_caps()
872 if (chan->max_length != UINT_MAX) in axi_dmac_detect_caps()
873 chan->max_length++; in axi_dmac_detect_caps()
877 chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) { in axi_dmac_detect_caps()
878 dev_err(dmac->dma_dev.dev, in axi_dmac_detect_caps()
879 "Destination memory-mapped interface not supported."); in axi_dmac_detect_caps()
880 return -ENODEV; in axi_dmac_detect_caps()
885 chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) { in axi_dmac_detect_caps()
886 dev_err(dmac->dma_dev.dev, in axi_dmac_detect_caps()
887 "Source memory-mapped interface not supported."); in axi_dmac_detect_caps()
888 return -ENODEV; in axi_dmac_detect_caps()
892 chan->hw_partial_xfer = true; in axi_dmac_detect_caps()
896 chan->length_align_mask = in axi_dmac_detect_caps()
899 chan->length_align_mask = chan->address_align_mask; in axi_dmac_detect_caps()
914 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); in axi_dmac_probe()
916 return -ENOMEM; in axi_dmac_probe()
918 dmac->irq = platform_get_irq(pdev, 0); in axi_dmac_probe()
919 if (dmac->irq < 0) in axi_dmac_probe()
920 return dmac->irq; in axi_dmac_probe()
921 if (dmac->irq == 0) in axi_dmac_probe()
922 return -EINVAL; in axi_dmac_probe()
925 dmac->base = devm_ioremap_resource(&pdev->dev, res); in axi_dmac_probe()
926 if (IS_ERR(dmac->base)) in axi_dmac_probe()
927 return PTR_ERR(dmac->base); in axi_dmac_probe()
929 dmac->clk = devm_clk_get(&pdev->dev, NULL); in axi_dmac_probe()
930 if (IS_ERR(dmac->clk)) in axi_dmac_probe()
931 return PTR_ERR(dmac->clk); in axi_dmac_probe()
933 ret = clk_prepare_enable(dmac->clk); in axi_dmac_probe()
940 ret = axi_dmac_read_chan_config(&pdev->dev, dmac); in axi_dmac_probe()
942 ret = axi_dmac_parse_dt(&pdev->dev, dmac); in axi_dmac_probe()
947 INIT_LIST_HEAD(&dmac->chan.active_descs); in axi_dmac_probe()
949 dma_set_max_seg_size(&pdev->dev, UINT_MAX); in axi_dmac_probe()
951 dma_dev = &dmac->dma_dev; in axi_dmac_probe()
952 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); in axi_dmac_probe()
953 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); in axi_dmac_probe()
954 dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask); in axi_dmac_probe()
955 dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources; in axi_dmac_probe()
956 dma_dev->device_tx_status = dma_cookie_status; in axi_dmac_probe()
957 dma_dev->device_issue_pending = axi_dmac_issue_pending; in axi_dmac_probe()
958 dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg; in axi_dmac_probe()
959 dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic; in axi_dmac_probe()
960 dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved; in axi_dmac_probe()
961 dma_dev->device_terminate_all = axi_dmac_terminate_all; in axi_dmac_probe()
962 dma_dev->device_synchronize = axi_dmac_synchronize; in axi_dmac_probe()
963 dma_dev->dev = &pdev->dev; in axi_dmac_probe()
964 dma_dev->chancnt = 1; in axi_dmac_probe()
965 dma_dev->src_addr_widths = BIT(dmac->chan.src_width); in axi_dmac_probe()
966 dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); in axi_dmac_probe()
967 dma_dev->directions = BIT(dmac->chan.direction); in axi_dmac_probe()
968 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; in axi_dmac_probe()
969 INIT_LIST_HEAD(&dma_dev->channels); in axi_dmac_probe()
971 dmac->chan.vchan.desc_free = axi_dmac_desc_free; in axi_dmac_probe()
972 vchan_init(&dmac->chan.vchan, dma_dev); in axi_dmac_probe()
978 dma_dev->copy_align = (dmac->chan.address_align_mask + 1); in axi_dmac_probe()
986 ret = of_dma_controller_register(pdev->dev.of_node, in axi_dmac_probe()
991 ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED, in axi_dmac_probe()
992 dev_name(&pdev->dev), dmac); in axi_dmac_probe()
998 regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base, in axi_dmac_probe()
1008 free_irq(dmac->irq, dmac); in axi_dmac_probe()
1010 of_dma_controller_free(pdev->dev.of_node); in axi_dmac_probe()
1012 dma_async_device_unregister(&dmac->dma_dev); in axi_dmac_probe()
1014 clk_disable_unprepare(dmac->clk); in axi_dmac_probe()
1023 of_dma_controller_free(pdev->dev.of_node); in axi_dmac_remove()
1024 free_irq(dmac->irq, dmac); in axi_dmac_remove()
1025 tasklet_kill(&dmac->chan.vchan.task); in axi_dmac_remove()
1026 dma_async_device_unregister(&dmac->dma_dev); in axi_dmac_remove()
1027 clk_disable_unprepare(dmac->clk); in axi_dmac_remove()
1033 { .compatible = "adi,axi-dmac-1.00.a" },
1040 .name = "dma-axi-dmac",
1048 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
1049 MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller");