Lines Matching +full:max +full:- +full:burst +full:- +full:len
22 #include <linux/dma-mapping.h>
47 CCTRL6, /* Cacheable write-through, allocate on writes only */
48 CCTRL7, /* Cacheable write-back, allocate on writes only */
246 * at 1byte/burst for P<->M and M<->M respectively.
247 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
248 * should be enough for P<->M and M<->M respectively.
304 * and burst size/length are assumed same.
352 BURST, enumerator
383 /* Index of the last submitted request or -1 if the DMA is stopped */
418 /* DMA-Engine Channel */
444 /* For D-to-M and M-to-D channels */
446 int burst_len; /* the number of burst */
448 /* DMA-mapped view of the FIFO; may differ if an IOMMU is present */
460 /* DMA-Engine Device */
508 .quirk = "arm,pl330-broken-no-flushp",
547 return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL; in _queue_full()
552 return thrd->dmac->manager == thrd; in is_manager()
555 /* If manager of the thread is in Non-Secure mode */
558 return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false; in _manager_ns()
603 else if (cond == BURST) in _emit_LD()
607 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); in _emit_LD()
620 if (cond == BURST) in _emit_LDP()
644 cnt--; /* DMAC increments by 1 internally */ in _emit_LP()
662 enum pl330_cond cond = arg->cond; in _emit_LPEND()
663 bool forever = arg->forever; in _emit_LPEND()
664 unsigned loop = arg->loop; in _emit_LPEND()
665 u8 bjump = arg->bjump; in _emit_LPEND()
680 else if (cond == BURST) in _emit_LPEND()
687 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'), in _emit_LPEND()
760 else if (cond == BURST) in _emit_ST()
764 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); in _emit_ST()
777 if (cond == BURST) in _emit_STP()
800 else if (cond == BURST) in _emit_WFP()
810 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3); in _emit_WFP()
836 u8 chan = arg->chan; in _emit_GO()
837 u32 addr = arg->addr; in _emit_GO()
838 unsigned ns = arg->ns; in _emit_GO()
856 /* Returns Time-Out */
859 void __iomem *regs = thrd->dmac->base; in _until_dmac_idle()
868 } while (--loops); in _until_dmac_idle()
879 void __iomem *regs = thrd->dmac->base; in _execute_DBGINSN()
885 val |= (thrd->id << 8); /* Channel Number */ in _execute_DBGINSN()
892 /* If timed out due to halted state-machine */ in _execute_DBGINSN()
894 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n"); in _execute_DBGINSN()
904 void __iomem *regs = thrd->dmac->base; in _state()
910 val = readl(regs + CS(thrd->id)) & 0xf; in _state()
962 void __iomem *regs = thrd->dmac->base; in _stop()
980 if (inten & (1 << thrd->ev)) in _stop()
981 writel(1 << thrd->ev, regs + INTCLR); in _stop()
983 writel(inten & ~(1 << thrd->ev), regs + INTEN); in _stop()
989 void __iomem *regs = thrd->dmac->base; in _trigger()
1001 idx = 1 - thrd->lstenq; in _trigger()
1002 if (thrd->req[idx].desc != NULL) { in _trigger()
1003 req = &thrd->req[idx]; in _trigger()
1005 idx = thrd->lstenq; in _trigger()
1006 if (thrd->req[idx].desc != NULL) in _trigger()
1007 req = &thrd->req[idx]; in _trigger()
1017 if (idx == thrd->req_running) in _trigger()
1020 desc = req->desc; in _trigger()
1022 ns = desc->rqcfg.nonsecure ? 1 : 0; in _trigger()
1024 /* See 'Abort Sources' point-4 at Page 2-25 */ in _trigger()
1026 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n", in _trigger()
1029 go.chan = thrd->id; in _trigger()
1030 go.addr = req->mc_bus; in _trigger()
1035 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN); in _trigger()
1040 thrd->req_running = idx; in _trigger()
1085 struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg; in _ldst_memtomem()
1087 /* check lock-up free version */ in _ldst_memtomem()
1088 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) { in _ldst_memtomem()
1089 while (cyc--) { in _ldst_memtomem()
1094 while (cyc--) { in _ldst_memtomem()
1122 off += _emit_LDP(dry_run, &buf[off], BURST, in _emit_load()
1156 off += _emit_STP(dry_run, &buf[off], BURST, in _emit_store()
1180 if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) in _ldst_peripheral()
1181 cond = BURST; in _ldst_peripheral()
1187 if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)) in _ldst_peripheral()
1188 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); in _ldst_peripheral()
1189 while (cyc--) { in _ldst_peripheral()
1190 off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); in _ldst_peripheral()
1191 off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype, in _ldst_peripheral()
1192 pxs->desc->peri); in _ldst_peripheral()
1193 off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype, in _ldst_peripheral()
1194 pxs->desc->peri); in _ldst_peripheral()
1204 enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE; in _bursts()
1206 switch (pxs->desc->rqtype) { in _bursts()
1228 * transfer dregs with single transfers to peripheral, or a reduced size burst
1229 * for mem-to-mem.
1240 switch (pxs->desc->rqtype) { in _dregs()
1249 dregs_ccr = pxs->ccr; in _dregs()
1252 dregs_ccr |= (((transfer_length - 1) & 0xf) << in _dregs()
1254 dregs_ccr |= (((transfer_length - 1) & 0xf) << in _dregs()
1280 /* Max iterations possible in DMALP is 256 */ in _loop()
1310 * Max bursts that we can unroll due to limit on the in _loop()
1312 * which is 8-bits and hence 255 in _loop()
1314 cycmax = (255 - (szlp + szlpend)) / szbrst; in _loop()
1333 lpend.bjump = off - ljmp1; in _loop()
1340 lpend.bjump = off - ljmp0; in _loop()
1355 struct pl330_xfer *x = &pxs->desc->px; in _setup_loops()
1356 u32 ccr = pxs->ccr; in _setup_loops()
1357 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr); in _setup_loops()
1358 int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) / in _setup_loops()
1365 bursts -= c; in _setup_loops()
1376 struct pl330_xfer *x = &pxs->desc->px; in _setup_xfer()
1379 /* DMAMOV SAR, x->src_addr */ in _setup_xfer()
1380 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr); in _setup_xfer()
1381 /* DMAMOV DAR, x->dst_addr */ in _setup_xfer()
1382 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr); in _setup_xfer()
1398 struct _pl330_req *req = &thrd->req[index]; in _setup_req()
1399 u8 *buf = req->mc_cpu; in _setup_req()
1402 PL330_DBGMC_START(req->mc_bus); in _setup_req()
1405 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr); in _setup_req()
1410 off += _emit_SEV(dry_run, &buf[off], thrd->ev); in _setup_req()
1421 if (rqc->src_inc) in _prepare_ccr()
1424 if (rqc->dst_inc) in _prepare_ccr()
1428 if (rqc->privileged) in _prepare_ccr()
1430 if (rqc->nonsecure) in _prepare_ccr()
1432 if (rqc->insnaccess) in _prepare_ccr()
1435 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT); in _prepare_ccr()
1436 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT); in _prepare_ccr()
1438 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT); in _prepare_ccr()
1439 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT); in _prepare_ccr()
1441 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT); in _prepare_ccr()
1442 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT); in _prepare_ccr()
1444 ccr |= (rqc->swap << CC_SWAP_SHFT); in _prepare_ccr()
1457 struct pl330_dmac *pl330 = thrd->dmac; in pl330_submit_req()
1464 switch (desc->rqtype) { in pl330_submit_req()
1475 return -ENOTSUPP; in pl330_submit_req()
1478 if (pl330->state == DYING in pl330_submit_req()
1479 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) { in pl330_submit_req()
1480 dev_info(thrd->dmac->ddma.dev, "%s:%d\n", in pl330_submit_req()
1482 return -EAGAIN; in pl330_submit_req()
1485 /* If request for non-existing peripheral */ in pl330_submit_req()
1486 if (desc->rqtype != DMA_MEM_TO_MEM && in pl330_submit_req()
1487 desc->peri >= pl330->pcfg.num_peri) { in pl330_submit_req()
1488 dev_info(thrd->dmac->ddma.dev, in pl330_submit_req()
1490 __func__, __LINE__, desc->peri); in pl330_submit_req()
1491 return -EINVAL; in pl330_submit_req()
1494 spin_lock_irqsave(&pl330->lock, flags); in pl330_submit_req()
1497 ret = -EAGAIN; in pl330_submit_req()
1503 desc->rqcfg.nonsecure = 0; in pl330_submit_req()
1505 desc->rqcfg.nonsecure = 1; in pl330_submit_req()
1507 ccr = _prepare_ccr(&desc->rqcfg); in pl330_submit_req()
1509 idx = thrd->req[0].desc == NULL ? 0 : 1; in pl330_submit_req()
1519 if (ret > pl330->mcbufsz / 2) { in pl330_submit_req()
1520 dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n", in pl330_submit_req()
1521 __func__, __LINE__, ret, pl330->mcbufsz / 2); in pl330_submit_req()
1522 ret = -ENOMEM; in pl330_submit_req()
1527 thrd->lstenq = idx; in pl330_submit_req()
1528 thrd->req[idx].desc = desc; in pl330_submit_req()
1534 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_submit_req()
1547 pch = desc->pchan; in dma_pl330_rqcb()
1553 spin_lock_irqsave(&pch->lock, flags); in dma_pl330_rqcb()
1555 desc->status = DONE; in dma_pl330_rqcb()
1557 spin_unlock_irqrestore(&pch->lock, flags); in dma_pl330_rqcb()
1559 tasklet_schedule(&pch->task); in dma_pl330_rqcb()
1568 spin_lock_irqsave(&pl330->lock, flags); in pl330_dotask()
1571 if (pl330->dmac_tbd.reset_dmac) { in pl330_dotask()
1572 pl330->state = DYING; in pl330_dotask()
1574 pl330->dmac_tbd.reset_mngr = true; in pl330_dotask()
1576 pl330->dmac_tbd.reset_dmac = false; in pl330_dotask()
1579 if (pl330->dmac_tbd.reset_mngr) { in pl330_dotask()
1580 _stop(pl330->manager); in pl330_dotask()
1582 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1; in pl330_dotask()
1584 pl330->dmac_tbd.reset_mngr = false; in pl330_dotask()
1587 for (i = 0; i < pl330->pcfg.num_chan; i++) { in pl330_dotask()
1589 if (pl330->dmac_tbd.reset_chan & (1 << i)) { in pl330_dotask()
1590 struct pl330_thread *thrd = &pl330->channels[i]; in pl330_dotask()
1591 void __iomem *regs = pl330->base; in pl330_dotask()
1596 if (readl(regs + FSC) & (1 << thrd->id)) in pl330_dotask()
1601 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_dotask()
1602 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err); in pl330_dotask()
1603 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err); in pl330_dotask()
1604 spin_lock_irqsave(&pl330->lock, flags); in pl330_dotask()
1606 thrd->req[0].desc = NULL; in pl330_dotask()
1607 thrd->req[1].desc = NULL; in pl330_dotask()
1608 thrd->req_running = -1; in pl330_dotask()
1611 pl330->dmac_tbd.reset_chan &= ~(1 << i); in pl330_dotask()
1615 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_dotask()
1629 regs = pl330->base; in pl330_update()
1631 spin_lock_irqsave(&pl330->lock, flags); in pl330_update()
1635 pl330->dmac_tbd.reset_mngr = true; in pl330_update()
1637 pl330->dmac_tbd.reset_mngr = false; in pl330_update()
1639 val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1); in pl330_update()
1640 pl330->dmac_tbd.reset_chan |= val; in pl330_update()
1643 while (i < pl330->pcfg.num_chan) { in pl330_update()
1645 dev_info(pl330->ddma.dev, in pl330_update()
1646 "Reset Channel-%d\t CS-%x FTC-%x\n", in pl330_update()
1649 _stop(&pl330->channels[i]); in pl330_update()
1657 if (pl330->pcfg.num_events < 32 in pl330_update()
1658 && val & ~((1 << pl330->pcfg.num_events) - 1)) { in pl330_update()
1659 pl330->dmac_tbd.reset_dmac = true; in pl330_update()
1660 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__, in pl330_update()
1666 for (ev = 0; ev < pl330->pcfg.num_events; ev++) { in pl330_update()
1678 id = pl330->events[ev]; in pl330_update()
1680 thrd = &pl330->channels[id]; in pl330_update()
1682 active = thrd->req_running; in pl330_update()
1683 if (active == -1) /* Aborted */ in pl330_update()
1687 descdone = thrd->req[active].desc; in pl330_update()
1688 thrd->req[active].desc = NULL; in pl330_update()
1690 thrd->req_running = -1; in pl330_update()
1696 list_add_tail(&descdone->rqd, &pl330->req_done); in pl330_update()
1701 while (!list_empty(&pl330->req_done)) { in pl330_update()
1702 descdone = list_first_entry(&pl330->req_done, in pl330_update()
1704 list_del(&descdone->rqd); in pl330_update()
1705 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_update()
1707 spin_lock_irqsave(&pl330->lock, flags); in pl330_update()
1711 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_update()
1713 if (pl330->dmac_tbd.reset_dmac in pl330_update()
1714 || pl330->dmac_tbd.reset_mngr in pl330_update()
1715 || pl330->dmac_tbd.reset_chan) { in pl330_update()
1717 tasklet_schedule(&pl330->tasks); in pl330_update()
1726 struct pl330_dmac *pl330 = thrd->dmac; in _alloc_event()
1729 for (ev = 0; ev < pl330->pcfg.num_events; ev++) in _alloc_event()
1730 if (pl330->events[ev] == -1) { in _alloc_event()
1731 pl330->events[ev] = thrd->id; in _alloc_event()
1735 return -1; in _alloc_event()
1740 return pl330->pcfg.irq_ns & (1 << i); in _chan_ns()
1751 if (pl330->state == DYING) in pl330_request_channel()
1754 chans = pl330->pcfg.num_chan; in pl330_request_channel()
1757 thrd = &pl330->channels[i]; in pl330_request_channel()
1758 if ((thrd->free) && (!_manager_ns(thrd) || in pl330_request_channel()
1760 thrd->ev = _alloc_event(thrd); in pl330_request_channel()
1761 if (thrd->ev >= 0) { in pl330_request_channel()
1762 thrd->free = false; in pl330_request_channel()
1763 thrd->lstenq = 1; in pl330_request_channel()
1764 thrd->req[0].desc = NULL; in pl330_request_channel()
1765 thrd->req[1].desc = NULL; in pl330_request_channel()
1766 thrd->req_running = -1; in pl330_request_channel()
1779 struct pl330_dmac *pl330 = thrd->dmac; in _free_event()
1782 if (ev >= 0 && ev < pl330->pcfg.num_events in _free_event()
1783 && pl330->events[ev] == thrd->id) in _free_event()
1784 pl330->events[ev] = -1; in _free_event()
1789 if (!thrd || thrd->free) in pl330_release_channel()
1794 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT); in pl330_release_channel()
1795 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT); in pl330_release_channel()
1797 _free_event(thrd, thrd->ev); in pl330_release_channel()
1798 thrd->free = true; in pl330_release_channel()
1806 void __iomem *regs = pl330->base; in read_dmac_config()
1811 pl330->pcfg.data_bus_width = 8 * (1 << val); in read_dmac_config()
1815 pl330->pcfg.data_buf_dep = val + 1; in read_dmac_config()
1820 pl330->pcfg.num_chan = val; in read_dmac_config()
1826 pl330->pcfg.num_peri = val; in read_dmac_config()
1827 pl330->pcfg.peri_ns = readl(regs + CR4); in read_dmac_config()
1829 pl330->pcfg.num_peri = 0; in read_dmac_config()
1834 pl330->pcfg.mode |= DMAC_MODE_NS; in read_dmac_config()
1836 pl330->pcfg.mode &= ~DMAC_MODE_NS; in read_dmac_config()
1841 pl330->pcfg.num_events = val; in read_dmac_config()
1843 pl330->pcfg.irq_ns = readl(regs + CR3); in read_dmac_config()
1848 struct pl330_dmac *pl330 = thrd->dmac; in _reset_thread()
1850 thrd->req[0].mc_cpu = pl330->mcode_cpu in _reset_thread()
1851 + (thrd->id * pl330->mcbufsz); in _reset_thread()
1852 thrd->req[0].mc_bus = pl330->mcode_bus in _reset_thread()
1853 + (thrd->id * pl330->mcbufsz); in _reset_thread()
1854 thrd->req[0].desc = NULL; in _reset_thread()
1856 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu in _reset_thread()
1857 + pl330->mcbufsz / 2; in _reset_thread()
1858 thrd->req[1].mc_bus = thrd->req[0].mc_bus in _reset_thread()
1859 + pl330->mcbufsz / 2; in _reset_thread()
1860 thrd->req[1].desc = NULL; in _reset_thread()
1862 thrd->req_running = -1; in _reset_thread()
1867 int chans = pl330->pcfg.num_chan; in dmac_alloc_threads()
1872 pl330->channels = kcalloc(1 + chans, sizeof(*thrd), in dmac_alloc_threads()
1874 if (!pl330->channels) in dmac_alloc_threads()
1875 return -ENOMEM; in dmac_alloc_threads()
1879 thrd = &pl330->channels[i]; in dmac_alloc_threads()
1880 thrd->id = i; in dmac_alloc_threads()
1881 thrd->dmac = pl330; in dmac_alloc_threads()
1883 thrd->free = true; in dmac_alloc_threads()
1887 thrd = &pl330->channels[chans]; in dmac_alloc_threads()
1888 thrd->id = chans; in dmac_alloc_threads()
1889 thrd->dmac = pl330; in dmac_alloc_threads()
1890 thrd->free = false; in dmac_alloc_threads()
1891 pl330->manager = thrd; in dmac_alloc_threads()
1898 int chans = pl330->pcfg.num_chan; in dmac_alloc_resources()
1905 pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev, in dmac_alloc_resources()
1906 chans * pl330->mcbufsz, in dmac_alloc_resources()
1907 &pl330->mcode_bus, GFP_KERNEL, in dmac_alloc_resources()
1909 if (!pl330->mcode_cpu) { in dmac_alloc_resources()
1910 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n", in dmac_alloc_resources()
1912 return -ENOMEM; in dmac_alloc_resources()
1917 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n", in dmac_alloc_resources()
1919 dma_free_coherent(pl330->ddma.dev, in dmac_alloc_resources()
1920 chans * pl330->mcbufsz, in dmac_alloc_resources()
1921 pl330->mcode_cpu, pl330->mcode_bus); in dmac_alloc_resources()
1933 if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) { in pl330_add()
1934 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n", in pl330_add()
1935 pl330->pcfg.periph_id); in pl330_add()
1936 return -EINVAL; in pl330_add()
1942 if (pl330->pcfg.num_events == 0) { in pl330_add()
1943 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n", in pl330_add()
1945 return -EINVAL; in pl330_add()
1948 spin_lock_init(&pl330->lock); in pl330_add()
1950 INIT_LIST_HEAD(&pl330->req_done); in pl330_add()
1953 if (!pl330->mcbufsz) in pl330_add()
1954 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2; in pl330_add()
1957 for (i = 0; i < pl330->pcfg.num_events; i++) in pl330_add()
1958 pl330->events[i] = -1; in pl330_add()
1963 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n"); in pl330_add()
1967 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330); in pl330_add()
1969 pl330->state = INIT; in pl330_add()
1980 for (i = 0; i < pl330->pcfg.num_chan; i++) { in dmac_free_threads()
1981 thrd = &pl330->channels[i]; in dmac_free_threads()
1986 kfree(pl330->channels); in dmac_free_threads()
1993 pl330->state = UNINIT; in pl330_del()
1995 tasklet_kill(&pl330->tasks); in pl330_del()
2000 dma_free_coherent(pl330->ddma.dev, in pl330_del()
2001 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu, in pl330_del()
2002 pl330->mcode_bus); in pl330_del()
2028 list_for_each_entry(desc, &pch->work_list, node) { in fill_queue()
2031 if (desc->status == BUSY) in fill_queue()
2034 ret = pl330_submit_req(pch->thread, desc); in fill_queue()
2036 desc->status = BUSY; in fill_queue()
2037 } else if (ret == -EAGAIN) { in fill_queue()
2042 desc->status = DONE; in fill_queue()
2043 dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n", in fill_queue()
2044 __func__, __LINE__, desc->txd.cookie); in fill_queue()
2045 tasklet_schedule(&pch->task); in fill_queue()
2057 spin_lock_irqsave(&pch->lock, flags); in pl330_tasklet()
2060 list_for_each_entry_safe(desc, _dt, &pch->work_list, node) in pl330_tasklet()
2061 if (desc->status == DONE) { in pl330_tasklet()
2062 if (!pch->cyclic) in pl330_tasklet()
2063 dma_cookie_complete(&desc->txd); in pl330_tasklet()
2064 list_move_tail(&desc->node, &pch->completed_list); in pl330_tasklet()
2070 if (list_empty(&pch->work_list)) { in pl330_tasklet()
2071 spin_lock(&pch->thread->dmac->lock); in pl330_tasklet()
2072 _stop(pch->thread); in pl330_tasklet()
2073 spin_unlock(&pch->thread->dmac->lock); in pl330_tasklet()
2075 pch->active = false; in pl330_tasklet()
2078 spin_lock(&pch->thread->dmac->lock); in pl330_tasklet()
2079 _start(pch->thread); in pl330_tasklet()
2080 spin_unlock(&pch->thread->dmac->lock); in pl330_tasklet()
2083 while (!list_empty(&pch->completed_list)) { in pl330_tasklet()
2086 desc = list_first_entry(&pch->completed_list, in pl330_tasklet()
2089 dmaengine_desc_get_callback(&desc->txd, &cb); in pl330_tasklet()
2091 if (pch->cyclic) { in pl330_tasklet()
2092 desc->status = PREP; in pl330_tasklet()
2093 list_move_tail(&desc->node, &pch->work_list); in pl330_tasklet()
2095 pch->active = true; in pl330_tasklet()
2096 spin_lock(&pch->thread->dmac->lock); in pl330_tasklet()
2097 _start(pch->thread); in pl330_tasklet()
2098 spin_unlock(&pch->thread->dmac->lock); in pl330_tasklet()
2102 desc->status = FREE; in pl330_tasklet()
2103 list_move_tail(&desc->node, &pch->dmac->desc_pool); in pl330_tasklet()
2106 dma_descriptor_unmap(&desc->txd); in pl330_tasklet()
2109 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tasklet()
2111 spin_lock_irqsave(&pch->lock, flags); in pl330_tasklet()
2114 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tasklet()
2118 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); in pl330_tasklet()
2119 pm_runtime_put_autosuspend(pch->dmac->ddma.dev); in pl330_tasklet()
2126 int count = dma_spec->args_count; in of_dma_pl330_xlate()
2127 struct pl330_dmac *pl330 = ofdma->of_dma_data; in of_dma_pl330_xlate()
2136 chan_id = dma_spec->args[0]; in of_dma_pl330_xlate()
2137 if (chan_id >= pl330->num_peripherals) in of_dma_pl330_xlate()
2140 return dma_get_slave_channel(&pl330->peripherals[chan_id].chan); in of_dma_pl330_xlate()
2146 struct pl330_dmac *pl330 = pch->dmac; in pl330_alloc_chan_resources()
2149 spin_lock_irqsave(&pl330->lock, flags); in pl330_alloc_chan_resources()
2152 pch->cyclic = false; in pl330_alloc_chan_resources()
2154 pch->thread = pl330_request_channel(pl330); in pl330_alloc_chan_resources()
2155 if (!pch->thread) { in pl330_alloc_chan_resources()
2156 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_alloc_chan_resources()
2157 return -ENOMEM; in pl330_alloc_chan_resources()
2160 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch); in pl330_alloc_chan_resources()
2162 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_alloc_chan_resources()
2168 * We need the data direction between the DMAC (the dma-mapping "device") and
2188 if (pch->dir != DMA_NONE) in pl330_unprep_slave_fifo()
2189 dma_unmap_resource(pch->chan.device->dev, pch->fifo_dma, in pl330_unprep_slave_fifo()
2190 1 << pch->burst_sz, pch->dir, 0); in pl330_unprep_slave_fifo()
2191 pch->dir = DMA_NONE; in pl330_unprep_slave_fifo()
2198 struct device *dev = pch->chan.device->dev; in pl330_prep_slave_fifo()
2202 if (pch->dir == dma_dir) in pl330_prep_slave_fifo()
2206 pch->fifo_dma = dma_map_resource(dev, pch->fifo_addr, in pl330_prep_slave_fifo()
2207 1 << pch->burst_sz, dma_dir, 0); in pl330_prep_slave_fifo()
2208 if (dma_mapping_error(dev, pch->fifo_dma)) in pl330_prep_slave_fifo()
2211 pch->dir = dma_dir; in pl330_prep_slave_fifo()
2233 if (slave_config->direction == DMA_MEM_TO_DEV) { in pl330_config()
2234 if (slave_config->dst_addr) in pl330_config()
2235 pch->fifo_addr = slave_config->dst_addr; in pl330_config()
2236 if (slave_config->dst_addr_width) in pl330_config()
2237 pch->burst_sz = __ffs(slave_config->dst_addr_width); in pl330_config()
2238 pch->burst_len = fixup_burst_len(slave_config->dst_maxburst, in pl330_config()
2239 pch->dmac->quirks); in pl330_config()
2240 } else if (slave_config->direction == DMA_DEV_TO_MEM) { in pl330_config()
2241 if (slave_config->src_addr) in pl330_config()
2242 pch->fifo_addr = slave_config->src_addr; in pl330_config()
2243 if (slave_config->src_addr_width) in pl330_config()
2244 pch->burst_sz = __ffs(slave_config->src_addr_width); in pl330_config()
2245 pch->burst_len = fixup_burst_len(slave_config->src_maxburst, in pl330_config()
2246 pch->dmac->quirks); in pl330_config()
2257 struct pl330_dmac *pl330 = pch->dmac; in pl330_terminate_all()
2261 pm_runtime_get_sync(pl330->ddma.dev); in pl330_terminate_all()
2262 spin_lock_irqsave(&pch->lock, flags); in pl330_terminate_all()
2264 spin_lock(&pl330->lock); in pl330_terminate_all()
2265 _stop(pch->thread); in pl330_terminate_all()
2266 pch->thread->req[0].desc = NULL; in pl330_terminate_all()
2267 pch->thread->req[1].desc = NULL; in pl330_terminate_all()
2268 pch->thread->req_running = -1; in pl330_terminate_all()
2269 spin_unlock(&pl330->lock); in pl330_terminate_all()
2271 power_down = pch->active; in pl330_terminate_all()
2272 pch->active = false; in pl330_terminate_all()
2275 list_for_each_entry(desc, &pch->submitted_list, node) { in pl330_terminate_all()
2276 desc->status = FREE; in pl330_terminate_all()
2277 dma_cookie_complete(&desc->txd); in pl330_terminate_all()
2280 list_for_each_entry(desc, &pch->work_list , node) { in pl330_terminate_all()
2281 desc->status = FREE; in pl330_terminate_all()
2282 dma_cookie_complete(&desc->txd); in pl330_terminate_all()
2285 list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool); in pl330_terminate_all()
2286 list_splice_tail_init(&pch->work_list, &pl330->desc_pool); in pl330_terminate_all()
2287 list_splice_tail_init(&pch->completed_list, &pl330->desc_pool); in pl330_terminate_all()
2288 spin_unlock_irqrestore(&pch->lock, flags); in pl330_terminate_all()
2289 pm_runtime_mark_last_busy(pl330->ddma.dev); in pl330_terminate_all()
2291 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_terminate_all()
2292 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_terminate_all()
2307 struct pl330_dmac *pl330 = pch->dmac; in pl330_pause()
2310 pm_runtime_get_sync(pl330->ddma.dev); in pl330_pause()
2311 spin_lock_irqsave(&pch->lock, flags); in pl330_pause()
2313 spin_lock(&pl330->lock); in pl330_pause()
2314 _stop(pch->thread); in pl330_pause()
2315 spin_unlock(&pl330->lock); in pl330_pause()
2317 spin_unlock_irqrestore(&pch->lock, flags); in pl330_pause()
2318 pm_runtime_mark_last_busy(pl330->ddma.dev); in pl330_pause()
2319 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_pause()
2327 struct pl330_dmac *pl330 = pch->dmac; in pl330_free_chan_resources()
2330 tasklet_kill(&pch->task); in pl330_free_chan_resources()
2332 pm_runtime_get_sync(pch->dmac->ddma.dev); in pl330_free_chan_resources()
2333 spin_lock_irqsave(&pl330->lock, flags); in pl330_free_chan_resources()
2335 pl330_release_channel(pch->thread); in pl330_free_chan_resources()
2336 pch->thread = NULL; in pl330_free_chan_resources()
2338 if (pch->cyclic) in pl330_free_chan_resources()
2339 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); in pl330_free_chan_resources()
2341 spin_unlock_irqrestore(&pl330->lock, flags); in pl330_free_chan_resources()
2342 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); in pl330_free_chan_resources()
2343 pm_runtime_put_autosuspend(pch->dmac->ddma.dev); in pl330_free_chan_resources()
2350 struct pl330_thread *thrd = pch->thread; in pl330_get_current_xferred_count()
2351 struct pl330_dmac *pl330 = pch->dmac; in pl330_get_current_xferred_count()
2352 void __iomem *regs = thrd->dmac->base; in pl330_get_current_xferred_count()
2355 pm_runtime_get_sync(pl330->ddma.dev); in pl330_get_current_xferred_count()
2357 if (desc->rqcfg.src_inc) { in pl330_get_current_xferred_count()
2358 val = readl(regs + SA(thrd->id)); in pl330_get_current_xferred_count()
2359 addr = desc->px.src_addr; in pl330_get_current_xferred_count()
2361 val = readl(regs + DA(thrd->id)); in pl330_get_current_xferred_count()
2362 addr = desc->px.dst_addr; in pl330_get_current_xferred_count()
2364 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); in pl330_get_current_xferred_count()
2365 pm_runtime_put_autosuspend(pl330->ddma.dev); in pl330_get_current_xferred_count()
2371 return val - addr; in pl330_get_current_xferred_count()
2392 spin_lock_irqsave(&pch->lock, flags); in pl330_tx_status()
2393 spin_lock(&pch->thread->dmac->lock); in pl330_tx_status()
2395 if (pch->thread->req_running != -1) in pl330_tx_status()
2396 running = pch->thread->req[pch->thread->req_running].desc; in pl330_tx_status()
2398 last_enq = pch->thread->req[pch->thread->lstenq].desc; in pl330_tx_status()
2401 list_for_each_entry(desc, &pch->work_list, node) { in pl330_tx_status()
2402 if (desc->status == DONE) in pl330_tx_status()
2403 transferred = desc->bytes_requested; in pl330_tx_status()
2407 else if (desc->status == BUSY) in pl330_tx_status()
2415 transferred = desc->bytes_requested; in pl330_tx_status()
2418 residual += desc->bytes_requested - transferred; in pl330_tx_status()
2419 if (desc->txd.cookie == cookie) { in pl330_tx_status()
2420 switch (desc->status) { in pl330_tx_status()
2433 if (desc->last) in pl330_tx_status()
2436 spin_unlock(&pch->thread->dmac->lock); in pl330_tx_status()
2437 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tx_status()
2450 spin_lock_irqsave(&pch->lock, flags); in pl330_issue_pending()
2451 if (list_empty(&pch->work_list)) { in pl330_issue_pending()
2457 WARN_ON(list_empty(&pch->submitted_list)); in pl330_issue_pending()
2458 pch->active = true; in pl330_issue_pending()
2459 pm_runtime_get_sync(pch->dmac->ddma.dev); in pl330_issue_pending()
2461 list_splice_tail_init(&pch->submitted_list, &pch->work_list); in pl330_issue_pending()
2462 spin_unlock_irqrestore(&pch->lock, flags); in pl330_issue_pending()
2475 struct dma_pl330_chan *pch = to_pchan(tx->chan); in pl330_tx_submit()
2479 spin_lock_irqsave(&pch->lock, flags); in pl330_tx_submit()
2482 while (!list_empty(&last->node)) { in pl330_tx_submit()
2483 desc = list_entry(last->node.next, struct dma_pl330_desc, node); in pl330_tx_submit()
2484 if (pch->cyclic) { in pl330_tx_submit()
2485 desc->txd.callback = last->txd.callback; in pl330_tx_submit()
2486 desc->txd.callback_param = last->txd.callback_param; in pl330_tx_submit()
2488 desc->last = false; in pl330_tx_submit()
2490 dma_cookie_assign(&desc->txd); in pl330_tx_submit()
2492 list_move_tail(&desc->node, &pch->submitted_list); in pl330_tx_submit()
2495 last->last = true; in pl330_tx_submit()
2496 cookie = dma_cookie_assign(&last->txd); in pl330_tx_submit()
2497 list_add_tail(&last->node, &pch->submitted_list); in pl330_tx_submit()
2498 spin_unlock_irqrestore(&pch->lock, flags); in pl330_tx_submit()
2505 desc->rqcfg.swap = SWAP_NO; in _init_desc()
2506 desc->rqcfg.scctl = CCTRL0; in _init_desc()
2507 desc->rqcfg.dcctl = CCTRL0; in _init_desc()
2508 desc->txd.tx_submit = pl330_tx_submit; in _init_desc()
2510 INIT_LIST_HEAD(&desc->node); in _init_desc()
2546 desc = list_entry(pool->next, in pluck_desc()
2549 list_del_init(&desc->node); in pluck_desc()
2551 desc->status = PREP; in pluck_desc()
2552 desc->txd.callback = NULL; in pluck_desc()
2562 struct pl330_dmac *pl330 = pch->dmac; in pl330_get_desc()
2563 u8 *peri_id = pch->chan.private; in pl330_get_desc()
2567 desc = pluck_desc(&pl330->desc_pool, &pl330->pool_lock); in pl330_get_desc()
2582 desc->pchan = pch; in pl330_get_desc()
2583 desc->txd.cookie = 0; in pl330_get_desc()
2584 async_tx_ack(&desc->txd); in pl330_get_desc()
2586 desc->peri = peri_id ? pch->chan.chan_id : 0; in pl330_get_desc()
2587 desc->rqcfg.pcfg = &pch->dmac->pcfg; in pl330_get_desc()
2589 dma_async_tx_descriptor_init(&desc->txd, &pch->chan); in pl330_get_desc()
2595 dma_addr_t dst, dma_addr_t src, size_t len) in fill_px() argument
2597 px->bytes = len; in fill_px()
2598 px->dst_addr = dst; in fill_px()
2599 px->src_addr = src; in fill_px()
2604 dma_addr_t src, size_t len) in __pl330_prep_dma_memcpy() argument
2609 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", in __pl330_prep_dma_memcpy()
2618 * going to be word-unaligned and more than 200MB, in __pl330_prep_dma_memcpy()
2624 fill_px(&desc->px, dst, src, len); in __pl330_prep_dma_memcpy()
2629 /* Call after fixing burst size */
2630 static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len) in get_burst_len() argument
2632 struct dma_pl330_chan *pch = desc->pchan; in get_burst_len()
2633 struct pl330_dmac *pl330 = pch->dmac; in get_burst_len()
2636 burst_len = pl330->pcfg.data_bus_width / 8; in get_burst_len()
2637 burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan; in get_burst_len()
2638 burst_len >>= desc->rqcfg.brst_size; in get_burst_len()
2648 struct dma_chan *chan, dma_addr_t dma_addr, size_t len, in pl330_prep_dma_cyclic() argument
2654 struct pl330_dmac *pl330 = pch->dmac; in pl330_prep_dma_cyclic()
2659 if (len % period_len != 0) in pl330_prep_dma_cyclic()
2663 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n", in pl330_prep_dma_cyclic()
2671 for (i = 0; i < len / period_len; i++) { in pl330_prep_dma_cyclic()
2674 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", in pl330_prep_dma_cyclic()
2680 spin_lock_irqsave(&pl330->pool_lock, flags); in pl330_prep_dma_cyclic()
2682 while (!list_empty(&first->node)) { in pl330_prep_dma_cyclic()
2683 desc = list_entry(first->node.next, in pl330_prep_dma_cyclic()
2685 list_move_tail(&desc->node, &pl330->desc_pool); in pl330_prep_dma_cyclic()
2688 list_move_tail(&first->node, &pl330->desc_pool); in pl330_prep_dma_cyclic()
2690 spin_unlock_irqrestore(&pl330->pool_lock, flags); in pl330_prep_dma_cyclic()
2697 desc->rqcfg.src_inc = 1; in pl330_prep_dma_cyclic()
2698 desc->rqcfg.dst_inc = 0; in pl330_prep_dma_cyclic()
2700 dst = pch->fifo_dma; in pl330_prep_dma_cyclic()
2703 desc->rqcfg.src_inc = 0; in pl330_prep_dma_cyclic()
2704 desc->rqcfg.dst_inc = 1; in pl330_prep_dma_cyclic()
2705 src = pch->fifo_dma; in pl330_prep_dma_cyclic()
2712 desc->rqtype = direction; in pl330_prep_dma_cyclic()
2713 desc->rqcfg.brst_size = pch->burst_sz; in pl330_prep_dma_cyclic()
2714 desc->rqcfg.brst_len = pch->burst_len; in pl330_prep_dma_cyclic()
2715 desc->bytes_requested = period_len; in pl330_prep_dma_cyclic()
2716 fill_px(&desc->px, dst, src, period_len); in pl330_prep_dma_cyclic()
2721 list_add_tail(&desc->node, &first->node); in pl330_prep_dma_cyclic()
2729 pch->cyclic = true; in pl330_prep_dma_cyclic()
2730 desc->txd.flags = flags; in pl330_prep_dma_cyclic()
2732 return &desc->txd; in pl330_prep_dma_cyclic()
2737 dma_addr_t src, size_t len, unsigned long flags) in pl330_prep_dma_memcpy() argument
2742 int burst; in pl330_prep_dma_memcpy() local
2744 if (unlikely(!pch || !len)) in pl330_prep_dma_memcpy()
2747 pl330 = pch->dmac; in pl330_prep_dma_memcpy()
2749 desc = __pl330_prep_dma_memcpy(pch, dst, src, len); in pl330_prep_dma_memcpy()
2753 desc->rqcfg.src_inc = 1; in pl330_prep_dma_memcpy()
2754 desc->rqcfg.dst_inc = 1; in pl330_prep_dma_memcpy()
2755 desc->rqtype = DMA_MEM_TO_MEM; in pl330_prep_dma_memcpy()
2757 /* Select max possible burst size */ in pl330_prep_dma_memcpy()
2758 burst = pl330->pcfg.data_bus_width / 8; in pl330_prep_dma_memcpy()
2761 * Make sure we use a burst size that aligns with all the memcpy in pl330_prep_dma_memcpy()
2765 while ((src | dst | len) & (burst - 1)) in pl330_prep_dma_memcpy()
2766 burst /= 2; in pl330_prep_dma_memcpy()
2768 desc->rqcfg.brst_size = 0; in pl330_prep_dma_memcpy()
2769 while (burst != (1 << desc->rqcfg.brst_size)) in pl330_prep_dma_memcpy()
2770 desc->rqcfg.brst_size++; in pl330_prep_dma_memcpy()
2772 desc->rqcfg.brst_len = get_burst_len(desc, len); in pl330_prep_dma_memcpy()
2774 * If burst size is smaller than bus width then make sure we only in pl330_prep_dma_memcpy()
2775 * transfer one at a time to avoid a burst stradling an MFIFO entry. in pl330_prep_dma_memcpy()
2777 if (desc->rqcfg.brst_size * 8 < pl330->pcfg.data_bus_width) in pl330_prep_dma_memcpy()
2778 desc->rqcfg.brst_len = 1; in pl330_prep_dma_memcpy()
2780 desc->bytes_requested = len; in pl330_prep_dma_memcpy()
2782 desc->txd.flags = flags; in pl330_prep_dma_memcpy()
2784 return &desc->txd; in pl330_prep_dma_memcpy()
2796 spin_lock_irqsave(&pl330->pool_lock, flags); in __pl330_giveback_desc()
2798 while (!list_empty(&first->node)) { in __pl330_giveback_desc()
2799 desc = list_entry(first->node.next, in __pl330_giveback_desc()
2801 list_move_tail(&desc->node, &pl330->desc_pool); in __pl330_giveback_desc()
2804 list_move_tail(&first->node, &pl330->desc_pool); in __pl330_giveback_desc()
2806 spin_unlock_irqrestore(&pl330->pool_lock, flags); in __pl330_giveback_desc()
2831 struct pl330_dmac *pl330 = pch->dmac; in pl330_prep_slave_sg()
2833 dev_err(pch->dmac->ddma.dev, in pl330_prep_slave_sg()
2844 list_add_tail(&desc->node, &first->node); in pl330_prep_slave_sg()
2847 desc->rqcfg.src_inc = 1; in pl330_prep_slave_sg()
2848 desc->rqcfg.dst_inc = 0; in pl330_prep_slave_sg()
2849 fill_px(&desc->px, pch->fifo_dma, sg_dma_address(sg), in pl330_prep_slave_sg()
2852 desc->rqcfg.src_inc = 0; in pl330_prep_slave_sg()
2853 desc->rqcfg.dst_inc = 1; in pl330_prep_slave_sg()
2854 fill_px(&desc->px, sg_dma_address(sg), pch->fifo_dma, in pl330_prep_slave_sg()
2858 desc->rqcfg.brst_size = pch->burst_sz; in pl330_prep_slave_sg()
2859 desc->rqcfg.brst_len = pch->burst_len; in pl330_prep_slave_sg()
2860 desc->rqtype = direction; in pl330_prep_slave_sg()
2861 desc->bytes_requested = sg_dma_len(sg); in pl330_prep_slave_sg()
2865 desc->txd.flags = flg; in pl330_prep_slave_sg()
2866 return &desc->txd; in pl330_prep_slave_sg()
2934 struct device_node *np = adev->dev.of_node; in pl330_probe()
2936 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); in pl330_probe()
2941 pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL); in pl330_probe()
2943 return -ENOMEM; in pl330_probe()
2945 pd = &pl330->ddma; in pl330_probe()
2946 pd->dev = &adev->dev; in pl330_probe()
2948 pl330->mcbufsz = 0; in pl330_probe()
2953 pl330->quirks |= of_quirks[i].id; in pl330_probe()
2955 res = &adev->res; in pl330_probe()
2956 pl330->base = devm_ioremap_resource(&adev->dev, res); in pl330_probe()
2957 if (IS_ERR(pl330->base)) in pl330_probe()
2958 return PTR_ERR(pl330->base); in pl330_probe()
2963 irq = adev->irq[i]; in pl330_probe()
2965 ret = devm_request_irq(&adev->dev, irq, in pl330_probe()
2967 dev_name(&adev->dev), pl330); in pl330_probe()
2975 pcfg = &pl330->pcfg; in pl330_probe()
2977 pcfg->periph_id = adev->periphid; in pl330_probe()
2982 INIT_LIST_HEAD(&pl330->desc_pool); in pl330_probe()
2983 spin_lock_init(&pl330->pool_lock); in pl330_probe()
2986 if (!add_desc(&pl330->desc_pool, &pl330->pool_lock, in pl330_probe()
2988 dev_warn(&adev->dev, "unable to allocate desc\n"); in pl330_probe()
2990 INIT_LIST_HEAD(&pd->channels); in pl330_probe()
2993 num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan); in pl330_probe()
2995 pl330->num_peripherals = num_chan; in pl330_probe()
2997 pl330->peripherals = kcalloc(num_chan, sizeof(*pch), GFP_KERNEL); in pl330_probe()
2998 if (!pl330->peripherals) { in pl330_probe()
2999 ret = -ENOMEM; in pl330_probe()
3004 pch = &pl330->peripherals[i]; in pl330_probe()
3006 pch->chan.private = adev->dev.of_node; in pl330_probe()
3007 INIT_LIST_HEAD(&pch->submitted_list); in pl330_probe()
3008 INIT_LIST_HEAD(&pch->work_list); in pl330_probe()
3009 INIT_LIST_HEAD(&pch->completed_list); in pl330_probe()
3010 spin_lock_init(&pch->lock); in pl330_probe()
3011 pch->thread = NULL; in pl330_probe()
3012 pch->chan.device = pd; in pl330_probe()
3013 pch->dmac = pl330; in pl330_probe()
3014 pch->dir = DMA_NONE; in pl330_probe()
3017 list_add_tail(&pch->chan.device_node, &pd->channels); in pl330_probe()
3020 dma_cap_set(DMA_MEMCPY, pd->cap_mask); in pl330_probe()
3021 if (pcfg->num_peri) { in pl330_probe()
3022 dma_cap_set(DMA_SLAVE, pd->cap_mask); in pl330_probe()
3023 dma_cap_set(DMA_CYCLIC, pd->cap_mask); in pl330_probe()
3024 dma_cap_set(DMA_PRIVATE, pd->cap_mask); in pl330_probe()
3027 pd->device_alloc_chan_resources = pl330_alloc_chan_resources; in pl330_probe()
3028 pd->device_free_chan_resources = pl330_free_chan_resources; in pl330_probe()
3029 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy; in pl330_probe()
3030 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic; in pl330_probe()
3031 pd->device_tx_status = pl330_tx_status; in pl330_probe()
3032 pd->device_prep_slave_sg = pl330_prep_slave_sg; in pl330_probe()
3033 pd->device_config = pl330_config; in pl330_probe()
3034 pd->device_pause = pl330_pause; in pl330_probe()
3035 pd->device_terminate_all = pl330_terminate_all; in pl330_probe()
3036 pd->device_issue_pending = pl330_issue_pending; in pl330_probe()
3037 pd->src_addr_widths = PL330_DMA_BUSWIDTHS; in pl330_probe()
3038 pd->dst_addr_widths = PL330_DMA_BUSWIDTHS; in pl330_probe()
3039 pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in pl330_probe()
3040 pd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in pl330_probe()
3041 pd->max_burst = ((pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) ? in pl330_probe()
3046 dev_err(&adev->dev, "unable to register DMAC\n"); in pl330_probe()
3050 if (adev->dev.of_node) { in pl330_probe()
3051 ret = of_dma_controller_register(adev->dev.of_node, in pl330_probe()
3054 dev_err(&adev->dev, in pl330_probe()
3059 adev->dev.dma_parms = &pl330->dma_parms; in pl330_probe()
3065 ret = dma_set_max_seg_size(&adev->dev, 1900800); in pl330_probe()
3067 dev_err(&adev->dev, "unable to set the seg size\n"); in pl330_probe()
3070 dev_info(&adev->dev, in pl330_probe()
3071 "Loaded driver for PL330 DMAC-%x\n", adev->periphid); in pl330_probe()
3072 dev_info(&adev->dev, in pl330_probe()
3073 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n", in pl330_probe()
3074 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan, in pl330_probe()
3075 pcfg->num_peri, pcfg->num_events); in pl330_probe()
3077 pm_runtime_irq_safe(&adev->dev); in pl330_probe()
3078 pm_runtime_use_autosuspend(&adev->dev); in pl330_probe()
3079 pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY); in pl330_probe()
3080 pm_runtime_mark_last_busy(&adev->dev); in pl330_probe()
3081 pm_runtime_put_autosuspend(&adev->dev); in pl330_probe()
3086 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels, in pl330_probe()
3090 list_del(&pch->chan.device_node); in pl330_probe()
3093 if (pch->thread) { in pl330_probe()
3094 pl330_terminate_all(&pch->chan); in pl330_probe()
3095 pl330_free_chan_resources(&pch->chan); in pl330_probe()
3110 pm_runtime_get_noresume(pl330->ddma.dev); in pl330_remove()
3112 if (adev->dev.of_node) in pl330_remove()
3113 of_dma_controller_free(adev->dev.of_node); in pl330_remove()
3116 irq = adev->irq[i]; in pl330_remove()
3118 devm_free_irq(&adev->dev, irq, pl330); in pl330_remove()
3121 dma_async_device_unregister(&pl330->ddma); in pl330_remove()
3124 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels, in pl330_remove()
3128 list_del(&pch->chan.device_node); in pl330_remove()
3131 if (pch->thread) { in pl330_remove()
3132 pl330_terminate_all(&pch->chan); in pl330_remove()
3133 pl330_free_chan_resources(&pch->chan); in pl330_remove()
3155 .name = "dma-pl330",