Lines Matching refs:atchan
63 static struct at_desc *atc_first_active(struct at_dma_chan *atchan) in atc_first_active() argument
65 return list_first_entry(&atchan->active_list, in atc_first_active()
69 static struct at_desc *atc_first_queued(struct at_dma_chan *atchan) in atc_first_queued() argument
71 return list_first_entry(&atchan->queue, in atc_first_queued()
110 static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) in atc_desc_get() argument
118 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
119 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_desc_get()
126 dev_dbg(chan2dev(&atchan->chan_common), in atc_desc_get()
129 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
130 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_get()
135 ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); in atc_desc_get()
137 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
138 atchan->descs_allocated++; in atc_desc_get()
139 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
141 dev_err(chan2dev(&atchan->chan_common), in atc_desc_get()
154 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) in atc_desc_put() argument
160 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_put()
162 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
165 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_desc_put()
166 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
168 list_add(&desc->desc_node, &atchan->free_list); in atc_desc_put()
169 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_put()
203 static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) in atc_dostart() argument
205 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_dostart()
208 if (atc_chan_is_enabled(atchan)) { in atc_dostart()
209 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
211 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
213 channel_readl(atchan, SADDR), in atc_dostart()
214 channel_readl(atchan, DADDR), in atc_dostart()
215 channel_readl(atchan, CTRLA), in atc_dostart()
216 channel_readl(atchan, CTRLB), in atc_dostart()
217 channel_readl(atchan, DSCR)); in atc_dostart()
223 vdbg_dump_regs(atchan); in atc_dostart()
225 channel_writel(atchan, SADDR, 0); in atc_dostart()
226 channel_writel(atchan, DADDR, 0); in atc_dostart()
227 channel_writel(atchan, CTRLA, 0); in atc_dostart()
228 channel_writel(atchan, CTRLB, 0); in atc_dostart()
229 channel_writel(atchan, DSCR, first->txd.phys); in atc_dostart()
230 dma_writel(atdma, CHER, atchan->mask); in atc_dostart()
232 vdbg_dump_regs(atchan); in atc_dostart()
241 static struct at_desc *atc_get_current_descriptors(struct at_dma_chan *atchan, in atc_get_current_descriptors() argument
246 list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { in atc_get_current_descriptors()
270 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_get_bytes_left() local
272 int chan_id = atchan->chan_common.chan_id; in atc_get_bytes_left()
273 struct at_desc *desc_first = atc_first_active(atchan); in atc_get_bytes_left()
281 if (atchan->remain_desc == 0) in atc_get_bytes_left()
283 atchan->remain_desc = desc_first->len; in atc_get_bytes_left()
289 if (unlikely(test_bit(ATC_IS_BTC, &atchan->status))) { in atc_get_bytes_left()
290 clear_bit(ATC_IS_BTC, &atchan->status); in atc_get_bytes_left()
291 desc_cur = atc_get_current_descriptors(atchan, in atc_get_bytes_left()
292 channel_readl(atchan, DSCR)); in atc_get_bytes_left()
300 if (atchan->remain_desc < count) { in atc_get_bytes_left()
305 atchan->remain_desc -= count; in atc_get_bytes_left()
306 ret = atchan->remain_desc; in atc_get_bytes_left()
312 count = (channel_readl(atchan, CTRLA) & ATC_BTSIZE_MAX) in atc_get_bytes_left()
314 ret = atchan->remain_desc - count; in atc_get_bytes_left()
333 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) in atc_chain_complete() argument
337 dev_vdbg(chan2dev(&atchan->chan_common), in atc_chain_complete()
341 if (!atc_chan_is_cyclic(atchan)) in atc_chain_complete()
345 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_chain_complete()
347 list_move(&desc->desc_node, &atchan->free_list); in atc_chain_complete()
352 if (!atc_chan_is_cyclic(atchan)) { in atc_chain_complete()
376 static void atc_complete_all(struct at_dma_chan *atchan) in atc_complete_all() argument
381 dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); in atc_complete_all()
387 if (!list_empty(&atchan->queue)) in atc_complete_all()
388 atc_dostart(atchan, atc_first_queued(atchan)); in atc_complete_all()
390 list_splice_init(&atchan->active_list, &list); in atc_complete_all()
392 list_splice_init(&atchan->queue, &atchan->active_list); in atc_complete_all()
395 atc_chain_complete(atchan, desc); in atc_complete_all()
404 static void atc_advance_work(struct at_dma_chan *atchan) in atc_advance_work() argument
406 dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); in atc_advance_work()
408 if (atc_chan_is_enabled(atchan)) in atc_advance_work()
411 if (list_empty(&atchan->active_list) || in atc_advance_work()
412 list_is_singular(&atchan->active_list)) { in atc_advance_work()
413 atc_complete_all(atchan); in atc_advance_work()
415 atc_chain_complete(atchan, atc_first_active(atchan)); in atc_advance_work()
417 atc_dostart(atchan, atc_first_active(atchan)); in atc_advance_work()
428 static void atc_handle_error(struct at_dma_chan *atchan) in atc_handle_error() argument
438 bad_desc = atc_first_active(atchan); in atc_handle_error()
443 list_splice_init(&atchan->queue, atchan->active_list.prev); in atc_handle_error()
446 if (!list_empty(&atchan->active_list)) in atc_handle_error()
447 atc_dostart(atchan, atc_first_active(atchan)); in atc_handle_error()
456 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
458 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
460 atc_dump_lli(atchan, &bad_desc->lli); in atc_handle_error()
462 atc_dump_lli(atchan, &child->lli); in atc_handle_error()
465 atc_chain_complete(atchan, bad_desc); in atc_handle_error()
474 static void atc_handle_cyclic(struct at_dma_chan *atchan) in atc_handle_cyclic() argument
476 struct at_desc *first = atc_first_active(atchan); in atc_handle_cyclic()
481 dev_vdbg(chan2dev(&atchan->chan_common), in atc_handle_cyclic()
483 channel_readl(atchan, DSCR)); in atc_handle_cyclic()
493 struct at_dma_chan *atchan = (struct at_dma_chan *)data; in atc_tasklet() local
496 spin_lock_irqsave(&atchan->lock, flags); in atc_tasklet()
497 if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) in atc_tasklet()
498 atc_handle_error(atchan); in atc_tasklet()
499 else if (atc_chan_is_cyclic(atchan)) in atc_tasklet()
500 atc_handle_cyclic(atchan); in atc_tasklet()
502 atc_advance_work(atchan); in atc_tasklet()
504 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tasklet()
510 struct at_dma_chan *atchan; in at_dma_interrupt() local
528 atchan = &atdma->chan[i]; in at_dma_interrupt()
533 AT_DMA_RES(i) | atchan->mask); in at_dma_interrupt()
535 set_bit(ATC_IS_ERROR, &atchan->status); in at_dma_interrupt()
538 set_bit(ATC_IS_BTC, &atchan->status); in at_dma_interrupt()
539 tasklet_schedule(&atchan->tasklet); in at_dma_interrupt()
563 struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); in atc_tx_submit() local
567 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_submit()
570 if (list_empty(&atchan->active_list)) { in atc_tx_submit()
573 atc_dostart(atchan, desc); in atc_tx_submit()
574 list_add_tail(&desc->desc_node, &atchan->active_list); in atc_tx_submit()
578 list_add_tail(&desc->desc_node, &atchan->queue); in atc_tx_submit()
581 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_submit()
598 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memcpy() local
641 desc = atc_desc_get(atchan); in atc_prep_dma_memcpy()
668 atc_desc_put(atchan, first); in atc_prep_dma_memcpy()
687 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_slave_sg() local
689 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_slave_sg()
722 | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); in atc_prep_slave_sg()
729 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
761 | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); in atc_prep_slave_sg()
769 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
815 atc_desc_put(atchan, first); in atc_prep_slave_sg()
849 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_dma_cyclic_fill_desc() local
850 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_dma_cyclic_fill_desc()
868 | ATC_SIF(atchan->mem_if) in atc_dma_cyclic_fill_desc()
869 | ATC_DIF(atchan->per_if); in atc_dma_cyclic_fill_desc()
879 | ATC_SIF(atchan->per_if) in atc_dma_cyclic_fill_desc()
880 | ATC_DIF(atchan->mem_if); in atc_dma_cyclic_fill_desc()
904 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_cyclic() local
906 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_dma_cyclic()
924 was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
946 desc = atc_desc_get(atchan); in atc_prep_dma_cyclic()
969 atc_desc_put(atchan, first); in atc_prep_dma_cyclic()
971 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
978 struct at_dma_chan *atchan = to_at_dma_chan(chan); in set_runtime_config() local
984 memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig)); in set_runtime_config()
986 convert_burst(&atchan->dma_sconfig.src_maxburst); in set_runtime_config()
987 convert_burst(&atchan->dma_sconfig.dst_maxburst); in set_runtime_config()
996 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_control() local
998 int chan_id = atchan->chan_common.chan_id; in atc_control()
1006 spin_lock_irqsave(&atchan->lock, flags); in atc_control()
1009 set_bit(ATC_IS_PAUSED, &atchan->status); in atc_control()
1011 spin_unlock_irqrestore(&atchan->lock, flags); in atc_control()
1013 if (!atc_chan_is_paused(atchan)) in atc_control()
1016 spin_lock_irqsave(&atchan->lock, flags); in atc_control()
1019 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_control()
1021 spin_unlock_irqrestore(&atchan->lock, flags); in atc_control()
1030 spin_lock_irqsave(&atchan->lock, flags); in atc_control()
1033 dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); in atc_control()
1036 while (dma_readl(atdma, CHSR) & atchan->mask) in atc_control()
1040 list_splice_init(&atchan->queue, &list); in atc_control()
1041 list_splice_init(&atchan->active_list, &list); in atc_control()
1045 atc_chain_complete(atchan, desc); in atc_control()
1047 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_control()
1049 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_control()
1051 spin_unlock_irqrestore(&atchan->lock, flags); in atc_control()
1076 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_tx_status() local
1091 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_status()
1096 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_status()
1117 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_issue_pending() local
1123 if (atc_chan_is_cyclic(atchan)) in atc_issue_pending()
1126 spin_lock_irqsave(&atchan->lock, flags); in atc_issue_pending()
1127 atc_advance_work(atchan); in atc_issue_pending()
1128 spin_unlock_irqrestore(&atchan->lock, flags); in atc_issue_pending()
1140 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_alloc_chan_resources() local
1152 if (atc_chan_is_enabled(atchan)) { in atc_alloc_chan_resources()
1174 if (!list_empty(&atchan->free_list)) in atc_alloc_chan_resources()
1175 return atchan->descs_allocated; in atc_alloc_chan_resources()
1188 spin_lock_irqsave(&atchan->lock, flags); in atc_alloc_chan_resources()
1189 atchan->descs_allocated = i; in atc_alloc_chan_resources()
1190 atchan->remain_desc = 0; in atc_alloc_chan_resources()
1191 list_splice(&tmp_list, &atchan->free_list); in atc_alloc_chan_resources()
1193 spin_unlock_irqrestore(&atchan->lock, flags); in atc_alloc_chan_resources()
1196 channel_writel(atchan, CFG, cfg); in atc_alloc_chan_resources()
1200 atchan->descs_allocated); in atc_alloc_chan_resources()
1202 return atchan->descs_allocated; in atc_alloc_chan_resources()
1211 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_free_chan_resources() local
1217 atchan->descs_allocated); in atc_free_chan_resources()
1220 BUG_ON(!list_empty(&atchan->active_list)); in atc_free_chan_resources()
1221 BUG_ON(!list_empty(&atchan->queue)); in atc_free_chan_resources()
1222 BUG_ON(atc_chan_is_enabled(atchan)); in atc_free_chan_resources()
1224 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_free_chan_resources()
1230 list_splice_init(&atchan->free_list, &list); in atc_free_chan_resources()
1231 atchan->descs_allocated = 0; in atc_free_chan_resources()
1232 atchan->status = 0; in atc_free_chan_resources()
1233 atchan->remain_desc = 0; in atc_free_chan_resources()
1255 struct at_dma_chan *atchan; in at_dma_xlate() local
1303 atchan = to_at_dma_chan(chan); in at_dma_xlate()
1304 atchan->per_if = dma_spec->args[0] & 0xff; in at_dma_xlate()
1305 atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff; in at_dma_xlate()
1470 struct at_dma_chan *atchan = &atdma->chan[i]; in at_dma_probe() local
1472 atchan->mem_if = AT_DMA_MEM_IF; in at_dma_probe()
1473 atchan->per_if = AT_DMA_PER_IF; in at_dma_probe()
1474 atchan->chan_common.device = &atdma->dma_common; in at_dma_probe()
1475 dma_cookie_init(&atchan->chan_common); in at_dma_probe()
1476 list_add_tail(&atchan->chan_common.device_node, in at_dma_probe()
1479 atchan->ch_regs = atdma->regs + ch_regs(i); in at_dma_probe()
1480 spin_lock_init(&atchan->lock); in at_dma_probe()
1481 atchan->mask = 1 << i; in at_dma_probe()
1483 INIT_LIST_HEAD(&atchan->active_list); in at_dma_probe()
1484 INIT_LIST_HEAD(&atchan->queue); in at_dma_probe()
1485 INIT_LIST_HEAD(&atchan->free_list); in at_dma_probe()
1487 tasklet_init(&atchan->tasklet, atc_tasklet, in at_dma_probe()
1488 (unsigned long)atchan); in at_dma_probe()
1569 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_remove() local
1574 tasklet_kill(&atchan->tasklet); in at_dma_remove()
1608 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_prepare() local
1610 if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan)) in at_dma_prepare()
1616 static void atc_suspend_cyclic(struct at_dma_chan *atchan) in atc_suspend_cyclic() argument
1618 struct dma_chan *chan = &atchan->chan_common; in atc_suspend_cyclic()
1622 if (!atc_chan_is_paused(atchan)) { in atc_suspend_cyclic()
1630 atchan->save_dscr = channel_readl(atchan, DSCR); in atc_suspend_cyclic()
1632 vdbg_dump_regs(atchan); in atc_suspend_cyclic()
1644 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_suspend_noirq() local
1646 if (atc_chan_is_cyclic(atchan)) in at_dma_suspend_noirq()
1647 atc_suspend_cyclic(atchan); in at_dma_suspend_noirq()
1648 atchan->save_cfg = channel_readl(atchan, CFG); in at_dma_suspend_noirq()
1658 static void atc_resume_cyclic(struct at_dma_chan *atchan) in atc_resume_cyclic() argument
1660 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_resume_cyclic()
1664 channel_writel(atchan, SADDR, 0); in atc_resume_cyclic()
1665 channel_writel(atchan, DADDR, 0); in atc_resume_cyclic()
1666 channel_writel(atchan, CTRLA, 0); in atc_resume_cyclic()
1667 channel_writel(atchan, CTRLB, 0); in atc_resume_cyclic()
1668 channel_writel(atchan, DSCR, atchan->save_dscr); in atc_resume_cyclic()
1669 dma_writel(atdma, CHER, atchan->mask); in atc_resume_cyclic()
1674 vdbg_dump_regs(atchan); in atc_resume_cyclic()
1695 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_resume_noirq() local
1697 channel_writel(atchan, CFG, atchan->save_cfg); in at_dma_resume_noirq()
1698 if (atc_chan_is_cyclic(atchan)) in at_dma_resume_noirq()
1699 atc_resume_cyclic(atchan); in at_dma_resume_noirq()