1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
4 * Synopsys DesignWare eDMA core driver
5 *
6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
7 */
8
9 #include <linux/module.h>
10 #include <linux/device.h>
11 #include <linux/kernel.h>
12 #include <linux/dmaengine.h>
13 #include <linux/err.h>
14 #include <linux/interrupt.h>
15 #include <linux/irq.h>
16 #include <linux/dma/edma.h>
17 #include <linux/dma-mapping.h>
18
19 #include "dw-edma-core.h"
20 #include "dw-edma-v0-core.h"
21 #include "dw-hdma-v0-core.h"
22 #include "../dmaengine.h"
23 #include "../virt-dma.h"
24
25 static inline
dchan2dev(struct dma_chan * dchan)26 struct device *dchan2dev(struct dma_chan *dchan)
27 {
28 return &dchan->dev->device;
29 }
30
31 static inline
chan2dev(struct dw_edma_chan * chan)32 struct device *chan2dev(struct dw_edma_chan *chan)
33 {
34 return &chan->vc.chan.dev->device;
35 }
36
37 static inline
vd2dw_edma_desc(struct virt_dma_desc * vd)38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
39 {
40 return container_of(vd, struct dw_edma_desc, vd);
41 }
42
43 static inline
dw_edma_get_pci_address(struct dw_edma_chan * chan,phys_addr_t cpu_addr)44 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
45 {
46 struct dw_edma_chip *chip = chan->dw->chip;
47
48 if (chip->ops->pci_address)
49 return chip->ops->pci_address(chip->dev, cpu_addr);
50
51 return cpu_addr;
52 }
53
dw_edma_alloc_burst(struct dw_edma_chunk * chunk)54 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
55 {
56 struct dw_edma_burst *burst;
57
58 burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
59 if (unlikely(!burst))
60 return NULL;
61
62 INIT_LIST_HEAD(&burst->list);
63 if (chunk->burst) {
64 /* Create and add new element into the linked list */
65 chunk->bursts_alloc++;
66 list_add_tail(&burst->list, &chunk->burst->list);
67 } else {
68 /* List head */
69 chunk->bursts_alloc = 0;
70 chunk->burst = burst;
71 }
72
73 return burst;
74 }
75
dw_edma_alloc_chunk(struct dw_edma_desc * desc)76 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
77 {
78 struct dw_edma_chip *chip = desc->chan->dw->chip;
79 struct dw_edma_chan *chan = desc->chan;
80 struct dw_edma_chunk *chunk;
81
82 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
83 if (unlikely(!chunk))
84 return NULL;
85
86 INIT_LIST_HEAD(&chunk->list);
87 chunk->chan = chan;
88 /* Toggling change bit (CB) in each chunk, this is a mechanism to
89 * inform the eDMA HW block that this is a new linked list ready
90 * to be consumed.
91 * - Odd chunks originate CB equal to 0
92 * - Even chunks originate CB equal to 1
93 */
94 chunk->cb = !(desc->chunks_alloc % 2);
95 if (chan->dir == EDMA_DIR_WRITE) {
96 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;
97 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;
98 } else {
99 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;
100 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;
101 }
102
103 if (desc->chunk) {
104 /* Create and add new element into the linked list */
105 if (!dw_edma_alloc_burst(chunk)) {
106 kfree(chunk);
107 return NULL;
108 }
109 desc->chunks_alloc++;
110 list_add_tail(&chunk->list, &desc->chunk->list);
111 } else {
112 /* List head */
113 chunk->burst = NULL;
114 desc->chunks_alloc = 0;
115 desc->chunk = chunk;
116 }
117
118 return chunk;
119 }
120
dw_edma_alloc_desc(struct dw_edma_chan * chan)121 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
122 {
123 struct dw_edma_desc *desc;
124
125 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
126 if (unlikely(!desc))
127 return NULL;
128
129 desc->chan = chan;
130 if (!dw_edma_alloc_chunk(desc)) {
131 kfree(desc);
132 return NULL;
133 }
134
135 return desc;
136 }
137
dw_edma_free_burst(struct dw_edma_chunk * chunk)138 static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
139 {
140 struct dw_edma_burst *child, *_next;
141
142 /* Remove all the list elements */
143 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
144 list_del(&child->list);
145 kfree(child);
146 chunk->bursts_alloc--;
147 }
148
149 /* Remove the list head */
150 kfree(child);
151 chunk->burst = NULL;
152 }
153
dw_edma_free_chunk(struct dw_edma_desc * desc)154 static void dw_edma_free_chunk(struct dw_edma_desc *desc)
155 {
156 struct dw_edma_chunk *child, *_next;
157
158 if (!desc->chunk)
159 return;
160
161 /* Remove all the list elements */
162 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
163 dw_edma_free_burst(child);
164 list_del(&child->list);
165 kfree(child);
166 desc->chunks_alloc--;
167 }
168
169 /* Remove the list head */
170 kfree(child);
171 desc->chunk = NULL;
172 }
173
dw_edma_free_desc(struct dw_edma_desc * desc)174 static void dw_edma_free_desc(struct dw_edma_desc *desc)
175 {
176 dw_edma_free_chunk(desc);
177 kfree(desc);
178 }
179
vchan_free_desc(struct virt_dma_desc * vdesc)180 static void vchan_free_desc(struct virt_dma_desc *vdesc)
181 {
182 dw_edma_free_desc(vd2dw_edma_desc(vdesc));
183 }
184
dw_edma_start_transfer(struct dw_edma_chan * chan)185 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
186 {
187 struct dw_edma *dw = chan->dw;
188 struct dw_edma_chunk *child;
189 struct dw_edma_desc *desc;
190 struct virt_dma_desc *vd;
191
192 vd = vchan_next_desc(&chan->vc);
193 if (!vd)
194 return 0;
195
196 desc = vd2dw_edma_desc(vd);
197 if (!desc)
198 return 0;
199
200 child = list_first_entry_or_null(&desc->chunk->list,
201 struct dw_edma_chunk, list);
202 if (!child)
203 return 0;
204
205 dw_edma_core_start(dw, child, !desc->xfer_sz);
206 desc->xfer_sz += child->ll_region.sz;
207 dw_edma_free_burst(child);
208 list_del(&child->list);
209 kfree(child);
210 desc->chunks_alloc--;
211
212 return 1;
213 }
214
dw_edma_device_caps(struct dma_chan * dchan,struct dma_slave_caps * caps)215 static void dw_edma_device_caps(struct dma_chan *dchan,
216 struct dma_slave_caps *caps)
217 {
218 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
219
220 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
221 if (chan->dir == EDMA_DIR_READ)
222 caps->directions = BIT(DMA_DEV_TO_MEM);
223 else
224 caps->directions = BIT(DMA_MEM_TO_DEV);
225 } else {
226 if (chan->dir == EDMA_DIR_WRITE)
227 caps->directions = BIT(DMA_DEV_TO_MEM);
228 else
229 caps->directions = BIT(DMA_MEM_TO_DEV);
230 }
231 }
232
dw_edma_device_config(struct dma_chan * dchan,struct dma_slave_config * config)233 static int dw_edma_device_config(struct dma_chan *dchan,
234 struct dma_slave_config *config)
235 {
236 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
237
238 memcpy(&chan->config, config, sizeof(*config));
239 chan->configured = true;
240
241 return 0;
242 }
243
dw_edma_device_pause(struct dma_chan * dchan)244 static int dw_edma_device_pause(struct dma_chan *dchan)
245 {
246 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
247 int err = 0;
248
249 if (!chan->configured)
250 err = -EPERM;
251 else if (chan->status != EDMA_ST_BUSY)
252 err = -EPERM;
253 else if (chan->request != EDMA_REQ_NONE)
254 err = -EPERM;
255 else
256 chan->request = EDMA_REQ_PAUSE;
257
258 return err;
259 }
260
dw_edma_device_resume(struct dma_chan * dchan)261 static int dw_edma_device_resume(struct dma_chan *dchan)
262 {
263 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
264 int err = 0;
265
266 if (!chan->configured) {
267 err = -EPERM;
268 } else if (chan->status != EDMA_ST_PAUSE) {
269 err = -EPERM;
270 } else if (chan->request != EDMA_REQ_NONE) {
271 err = -EPERM;
272 } else {
273 chan->status = EDMA_ST_BUSY;
274 dw_edma_start_transfer(chan);
275 }
276
277 return err;
278 }
279
dw_edma_device_terminate_all(struct dma_chan * dchan)280 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
281 {
282 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
283 int err = 0;
284
285 if (!chan->configured) {
286 /* Do nothing */
287 } else if (chan->status == EDMA_ST_PAUSE) {
288 chan->status = EDMA_ST_IDLE;
289 chan->configured = false;
290 } else if (chan->status == EDMA_ST_IDLE) {
291 chan->configured = false;
292 } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
293 /*
294 * The channel is in a false BUSY state, probably didn't
295 * receive or lost an interrupt
296 */
297 chan->status = EDMA_ST_IDLE;
298 chan->configured = false;
299 } else if (chan->request > EDMA_REQ_PAUSE) {
300 err = -EPERM;
301 } else {
302 chan->request = EDMA_REQ_STOP;
303 }
304
305 return err;
306 }
307
dw_edma_device_issue_pending(struct dma_chan * dchan)308 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
309 {
310 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
311 unsigned long flags;
312
313 if (!chan->configured)
314 return;
315
316 spin_lock_irqsave(&chan->vc.lock, flags);
317 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE &&
318 chan->status == EDMA_ST_IDLE) {
319 chan->status = EDMA_ST_BUSY;
320 dw_edma_start_transfer(chan);
321 }
322 spin_unlock_irqrestore(&chan->vc.lock, flags);
323 }
324
325 static enum dma_status
dw_edma_device_tx_status(struct dma_chan * dchan,dma_cookie_t cookie,struct dma_tx_state * txstate)326 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
327 struct dma_tx_state *txstate)
328 {
329 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
330 struct dw_edma_desc *desc;
331 struct virt_dma_desc *vd;
332 unsigned long flags;
333 enum dma_status ret;
334 u32 residue = 0;
335
336 ret = dma_cookie_status(dchan, cookie, txstate);
337 if (ret == DMA_COMPLETE)
338 return ret;
339
340 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
341 ret = DMA_PAUSED;
342
343 if (!txstate)
344 goto ret_residue;
345
346 spin_lock_irqsave(&chan->vc.lock, flags);
347 vd = vchan_find_desc(&chan->vc, cookie);
348 if (vd) {
349 desc = vd2dw_edma_desc(vd);
350 if (desc)
351 residue = desc->alloc_sz - desc->xfer_sz;
352 }
353 spin_unlock_irqrestore(&chan->vc.lock, flags);
354
355 ret_residue:
356 dma_set_residue(txstate, residue);
357
358 return ret;
359 }
360
361 static struct dma_async_tx_descriptor *
dw_edma_device_transfer(struct dw_edma_transfer * xfer)362 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
363 {
364 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
365 enum dma_transfer_direction dir = xfer->direction;
366 struct scatterlist *sg = NULL;
367 struct dw_edma_chunk *chunk;
368 struct dw_edma_burst *burst;
369 struct dw_edma_desc *desc;
370 u64 src_addr, dst_addr;
371 size_t fsz = 0;
372 u32 cnt = 0;
373 int i;
374
375 if (!chan->configured)
376 return NULL;
377
378 /*
379 * Local Root Port/End-point Remote End-point
380 * +-----------------------+ PCIe bus +----------------------+
381 * | | +-+ | |
382 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM |
383 * | | | | | |
384 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV |
385 * | | +-+ | |
386 * +-----------------------+ +----------------------+
387 *
388 * 1. Normal logic:
389 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the
390 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used
391 * for the device read operations (DEV_TO_MEM) and the Tx channel
392 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV).
393 *
394 * 2. Inverted logic:
395 * If eDMA is embedded into a Remote PCIe EP and is controlled by the
396 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx
397 * channel (EDMA_DIR_WRITE) will be used for the device read operations
398 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write
399 * operations (MEM_TO_DEV).
400 *
401 * It is the client driver responsibility to choose a proper channel
402 * for the DMA transfers.
403 */
404 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
405 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) ||
406 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV))
407 return NULL;
408 } else {
409 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) ||
410 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV))
411 return NULL;
412 }
413
414 if (xfer->type == EDMA_XFER_CYCLIC) {
415 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
416 return NULL;
417 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
418 if (xfer->xfer.sg.len < 1)
419 return NULL;
420 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
421 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1)
422 return NULL;
423 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)
424 return NULL;
425 } else {
426 return NULL;
427 }
428
429 desc = dw_edma_alloc_desc(chan);
430 if (unlikely(!desc))
431 goto err_alloc;
432
433 chunk = dw_edma_alloc_chunk(desc);
434 if (unlikely(!chunk))
435 goto err_alloc;
436
437 if (xfer->type == EDMA_XFER_INTERLEAVED) {
438 src_addr = xfer->xfer.il->src_start;
439 dst_addr = xfer->xfer.il->dst_start;
440 } else {
441 src_addr = chan->config.src_addr;
442 dst_addr = chan->config.dst_addr;
443 }
444
445 if (dir == DMA_DEV_TO_MEM)
446 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
447 else
448 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
449
450 if (xfer->type == EDMA_XFER_CYCLIC) {
451 cnt = xfer->xfer.cyclic.cnt;
452 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
453 cnt = xfer->xfer.sg.len;
454 sg = xfer->xfer.sg.sgl;
455 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
456 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;
457 fsz = xfer->xfer.il->frame_size;
458 }
459
460 for (i = 0; i < cnt; i++) {
461 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
462 break;
463
464 if (chunk->bursts_alloc == chan->ll_max) {
465 chunk = dw_edma_alloc_chunk(desc);
466 if (unlikely(!chunk))
467 goto err_alloc;
468 }
469
470 burst = dw_edma_alloc_burst(chunk);
471 if (unlikely(!burst))
472 goto err_alloc;
473
474 if (xfer->type == EDMA_XFER_CYCLIC)
475 burst->sz = xfer->xfer.cyclic.len;
476 else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
477 burst->sz = sg_dma_len(sg);
478 else if (xfer->type == EDMA_XFER_INTERLEAVED)
479 burst->sz = xfer->xfer.il->sgl[i % fsz].size;
480
481 chunk->ll_region.sz += burst->sz;
482 desc->alloc_sz += burst->sz;
483
484 if (dir == DMA_DEV_TO_MEM) {
485 burst->sar = src_addr;
486 if (xfer->type == EDMA_XFER_CYCLIC) {
487 burst->dar = xfer->xfer.cyclic.paddr;
488 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
489 src_addr += sg_dma_len(sg);
490 burst->dar = sg_dma_address(sg);
491 /* Unlike the typical assumption by other
492 * drivers/IPs the peripheral memory isn't
493 * a FIFO memory, in this case, it's a
494 * linear memory and that why the source
495 * and destination addresses are increased
496 * by the same portion (data length)
497 */
498 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
499 burst->dar = dst_addr;
500 }
501 } else {
502 burst->dar = dst_addr;
503 if (xfer->type == EDMA_XFER_CYCLIC) {
504 burst->sar = xfer->xfer.cyclic.paddr;
505 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
506 dst_addr += sg_dma_len(sg);
507 burst->sar = sg_dma_address(sg);
508 /* Unlike the typical assumption by other
509 * drivers/IPs the peripheral memory isn't
510 * a FIFO memory, in this case, it's a
511 * linear memory and that why the source
512 * and destination addresses are increased
513 * by the same portion (data length)
514 */
515 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
516 burst->sar = src_addr;
517 }
518 }
519
520 if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
521 sg = sg_next(sg);
522 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
523 struct dma_interleaved_template *il = xfer->xfer.il;
524 struct data_chunk *dc = &il->sgl[i % fsz];
525
526 src_addr += burst->sz;
527 if (il->src_sgl)
528 src_addr += dmaengine_get_src_icg(il, dc);
529
530 dst_addr += burst->sz;
531 if (il->dst_sgl)
532 dst_addr += dmaengine_get_dst_icg(il, dc);
533 }
534 }
535
536 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
537
538 err_alloc:
539 if (desc)
540 dw_edma_free_desc(desc);
541
542 return NULL;
543 }
544
545 static struct dma_async_tx_descriptor *
dw_edma_device_prep_slave_sg(struct dma_chan * dchan,struct scatterlist * sgl,unsigned int len,enum dma_transfer_direction direction,unsigned long flags,void * context)546 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
547 unsigned int len,
548 enum dma_transfer_direction direction,
549 unsigned long flags, void *context)
550 {
551 struct dw_edma_transfer xfer;
552
553 xfer.dchan = dchan;
554 xfer.direction = direction;
555 xfer.xfer.sg.sgl = sgl;
556 xfer.xfer.sg.len = len;
557 xfer.flags = flags;
558 xfer.type = EDMA_XFER_SCATTER_GATHER;
559
560 return dw_edma_device_transfer(&xfer);
561 }
562
563 static struct dma_async_tx_descriptor *
dw_edma_device_prep_dma_cyclic(struct dma_chan * dchan,dma_addr_t paddr,size_t len,size_t count,enum dma_transfer_direction direction,unsigned long flags)564 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
565 size_t len, size_t count,
566 enum dma_transfer_direction direction,
567 unsigned long flags)
568 {
569 struct dw_edma_transfer xfer;
570
571 xfer.dchan = dchan;
572 xfer.direction = direction;
573 xfer.xfer.cyclic.paddr = paddr;
574 xfer.xfer.cyclic.len = len;
575 xfer.xfer.cyclic.cnt = count;
576 xfer.flags = flags;
577 xfer.type = EDMA_XFER_CYCLIC;
578
579 return dw_edma_device_transfer(&xfer);
580 }
581
582 static struct dma_async_tx_descriptor *
dw_edma_device_prep_interleaved_dma(struct dma_chan * dchan,struct dma_interleaved_template * ilt,unsigned long flags)583 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
584 struct dma_interleaved_template *ilt,
585 unsigned long flags)
586 {
587 struct dw_edma_transfer xfer;
588
589 xfer.dchan = dchan;
590 xfer.direction = ilt->dir;
591 xfer.xfer.il = ilt;
592 xfer.flags = flags;
593 xfer.type = EDMA_XFER_INTERLEAVED;
594
595 return dw_edma_device_transfer(&xfer);
596 }
597
dw_edma_done_interrupt(struct dw_edma_chan * chan)598 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
599 {
600 struct dw_edma_desc *desc;
601 struct virt_dma_desc *vd;
602 unsigned long flags;
603
604 spin_lock_irqsave(&chan->vc.lock, flags);
605 vd = vchan_next_desc(&chan->vc);
606 if (vd) {
607 switch (chan->request) {
608 case EDMA_REQ_NONE:
609 desc = vd2dw_edma_desc(vd);
610 if (!desc->chunks_alloc) {
611 list_del(&vd->node);
612 vchan_cookie_complete(vd);
613 }
614
615 /* Continue transferring if there are remaining chunks or issued requests.
616 */
617 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
618 break;
619
620 case EDMA_REQ_STOP:
621 list_del(&vd->node);
622 vchan_cookie_complete(vd);
623 chan->request = EDMA_REQ_NONE;
624 chan->status = EDMA_ST_IDLE;
625 break;
626
627 case EDMA_REQ_PAUSE:
628 chan->request = EDMA_REQ_NONE;
629 chan->status = EDMA_ST_PAUSE;
630 break;
631
632 default:
633 break;
634 }
635 }
636 spin_unlock_irqrestore(&chan->vc.lock, flags);
637 }
638
dw_edma_abort_interrupt(struct dw_edma_chan * chan)639 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
640 {
641 struct virt_dma_desc *vd;
642 unsigned long flags;
643
644 spin_lock_irqsave(&chan->vc.lock, flags);
645 vd = vchan_next_desc(&chan->vc);
646 if (vd) {
647 list_del(&vd->node);
648 vchan_cookie_complete(vd);
649 }
650 spin_unlock_irqrestore(&chan->vc.lock, flags);
651 chan->request = EDMA_REQ_NONE;
652 chan->status = EDMA_ST_IDLE;
653 }
654
dw_edma_interrupt_write(int irq,void * data)655 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
656 {
657 struct dw_edma_irq *dw_irq = data;
658
659 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
660 dw_edma_done_interrupt,
661 dw_edma_abort_interrupt);
662 }
663
dw_edma_interrupt_read(int irq,void * data)664 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
665 {
666 struct dw_edma_irq *dw_irq = data;
667
668 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
669 dw_edma_done_interrupt,
670 dw_edma_abort_interrupt);
671 }
672
dw_edma_interrupt_common(int irq,void * data)673 static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
674 {
675 irqreturn_t ret = IRQ_NONE;
676
677 ret |= dw_edma_interrupt_write(irq, data);
678 ret |= dw_edma_interrupt_read(irq, data);
679
680 return ret;
681 }
682
dw_edma_alloc_chan_resources(struct dma_chan * dchan)683 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
684 {
685 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
686
687 if (chan->status != EDMA_ST_IDLE)
688 return -EBUSY;
689
690 return 0;
691 }
692
dw_edma_free_chan_resources(struct dma_chan * dchan)693 static void dw_edma_free_chan_resources(struct dma_chan *dchan)
694 {
695 unsigned long timeout = jiffies + msecs_to_jiffies(5000);
696 int ret;
697
698 while (time_before(jiffies, timeout)) {
699 ret = dw_edma_device_terminate_all(dchan);
700 if (!ret)
701 break;
702
703 if (time_after_eq(jiffies, timeout))
704 return;
705
706 cpu_relax();
707 }
708 }
709
dw_edma_channel_setup(struct dw_edma * dw,u32 wr_alloc,u32 rd_alloc)710 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
711 {
712 struct dw_edma_chip *chip = dw->chip;
713 struct device *dev = chip->dev;
714 struct dw_edma_chan *chan;
715 struct dw_edma_irq *irq;
716 struct dma_device *dma;
717 u32 i, ch_cnt;
718 u32 pos;
719
720 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
721 dma = &dw->dma;
722
723 INIT_LIST_HEAD(&dma->channels);
724
725 for (i = 0; i < ch_cnt; i++) {
726 chan = &dw->chan[i];
727
728 chan->dw = dw;
729
730 if (i < dw->wr_ch_cnt) {
731 chan->id = i;
732 chan->dir = EDMA_DIR_WRITE;
733 } else {
734 chan->id = i - dw->wr_ch_cnt;
735 chan->dir = EDMA_DIR_READ;
736 }
737
738 chan->configured = false;
739 chan->request = EDMA_REQ_NONE;
740 chan->status = EDMA_ST_IDLE;
741
742 if (chan->dir == EDMA_DIR_WRITE)
743 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
744 else
745 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
746 chan->ll_max -= 1;
747
748 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
749 chan->dir == EDMA_DIR_WRITE ? "write" : "read",
750 chan->id, chan->ll_max);
751
752 if (dw->nr_irqs == 1)
753 pos = 0;
754 else if (chan->dir == EDMA_DIR_WRITE)
755 pos = chan->id % wr_alloc;
756 else
757 pos = wr_alloc + chan->id % rd_alloc;
758
759 irq = &dw->irq[pos];
760
761 if (chan->dir == EDMA_DIR_WRITE)
762 irq->wr_mask |= BIT(chan->id);
763 else
764 irq->rd_mask |= BIT(chan->id);
765
766 irq->dw = dw;
767 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
768
769 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
770 chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id,
771 chan->msi.address_hi, chan->msi.address_lo,
772 chan->msi.data);
773
774 chan->vc.desc_free = vchan_free_desc;
775 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ?
776 &dw->chip->dt_region_wr[chan->id] :
777 &dw->chip->dt_region_rd[chan->id];
778
779 vchan_init(&chan->vc, dma);
780
781 dw_edma_core_ch_config(chan);
782 }
783
784 /* Set DMA channel capabilities */
785 dma_cap_zero(dma->cap_mask);
786 dma_cap_set(DMA_SLAVE, dma->cap_mask);
787 dma_cap_set(DMA_CYCLIC, dma->cap_mask);
788 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
789 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
790 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
791 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
792 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
793 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
794
795 /* Set DMA channel callbacks */
796 dma->dev = chip->dev;
797 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
798 dma->device_free_chan_resources = dw_edma_free_chan_resources;
799 dma->device_caps = dw_edma_device_caps;
800 dma->device_config = dw_edma_device_config;
801 dma->device_pause = dw_edma_device_pause;
802 dma->device_resume = dw_edma_device_resume;
803 dma->device_terminate_all = dw_edma_device_terminate_all;
804 dma->device_issue_pending = dw_edma_device_issue_pending;
805 dma->device_tx_status = dw_edma_device_tx_status;
806 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
807 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
808 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
809
810 dma_set_max_seg_size(dma->dev, U32_MAX);
811
812 /* Register DMA device */
813 return dma_async_device_register(dma);
814 }
815
dw_edma_dec_irq_alloc(int * nr_irqs,u32 * alloc,u16 cnt)816 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
817 {
818 if (*nr_irqs && *alloc < cnt) {
819 (*alloc)++;
820 (*nr_irqs)--;
821 }
822 }
823
dw_edma_add_irq_mask(u32 * mask,u32 alloc,u16 cnt)824 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
825 {
826 while (*mask * alloc < cnt)
827 (*mask)++;
828 }
829
dw_edma_irq_request(struct dw_edma * dw,u32 * wr_alloc,u32 * rd_alloc)830 static int dw_edma_irq_request(struct dw_edma *dw,
831 u32 *wr_alloc, u32 *rd_alloc)
832 {
833 struct dw_edma_chip *chip = dw->chip;
834 struct device *dev = dw->chip->dev;
835 u32 wr_mask = 1;
836 u32 rd_mask = 1;
837 int i, err = 0;
838 u32 ch_cnt;
839 int irq;
840
841 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
842
843 if (chip->nr_irqs < 1 || !chip->ops->irq_vector)
844 return -EINVAL;
845
846 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL);
847 if (!dw->irq)
848 return -ENOMEM;
849
850 if (chip->nr_irqs == 1) {
851 /* Common IRQ shared among all channels */
852 irq = chip->ops->irq_vector(dev, 0);
853 err = request_irq(irq, dw_edma_interrupt_common,
854 IRQF_SHARED, dw->name, &dw->irq[0]);
855 if (err) {
856 dw->nr_irqs = 0;
857 return err;
858 }
859
860 if (irq_get_msi_desc(irq))
861 get_cached_msi_msg(irq, &dw->irq[0].msi);
862
863 dw->nr_irqs = 1;
864 } else {
865 /* Distribute IRQs equally among all channels */
866 int tmp = chip->nr_irqs;
867
868 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
869 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
870 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
871 }
872
873 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
874 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
875
876 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
877 irq = chip->ops->irq_vector(dev, i);
878 err = request_irq(irq,
879 i < *wr_alloc ?
880 dw_edma_interrupt_write :
881 dw_edma_interrupt_read,
882 IRQF_SHARED, dw->name,
883 &dw->irq[i]);
884 if (err)
885 goto err_irq_free;
886
887 if (irq_get_msi_desc(irq))
888 get_cached_msi_msg(irq, &dw->irq[i].msi);
889 }
890
891 dw->nr_irqs = i;
892 }
893
894 return 0;
895
896 err_irq_free:
897 for (i--; i >= 0; i--) {
898 irq = chip->ops->irq_vector(dev, i);
899 free_irq(irq, &dw->irq[i]);
900 }
901
902 return err;
903 }
904
dw_edma_probe(struct dw_edma_chip * chip)905 int dw_edma_probe(struct dw_edma_chip *chip)
906 {
907 struct device *dev;
908 struct dw_edma *dw;
909 u32 wr_alloc = 0;
910 u32 rd_alloc = 0;
911 int i, err;
912
913 if (!chip)
914 return -EINVAL;
915
916 dev = chip->dev;
917 if (!dev || !chip->ops)
918 return -EINVAL;
919
920 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
921 if (!dw)
922 return -ENOMEM;
923
924 dw->chip = chip;
925
926 if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)
927 dw_hdma_v0_core_register(dw);
928 else
929 dw_edma_v0_core_register(dw);
930
931 raw_spin_lock_init(&dw->lock);
932
933 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,
934 dw_edma_core_ch_count(dw, EDMA_DIR_WRITE));
935 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
936
937 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,
938 dw_edma_core_ch_count(dw, EDMA_DIR_READ));
939 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
940
941 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
942 return -EINVAL;
943
944 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
945 dw->wr_ch_cnt, dw->rd_ch_cnt);
946
947 /* Allocate channels */
948 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
949 sizeof(*dw->chan), GFP_KERNEL);
950 if (!dw->chan)
951 return -ENOMEM;
952
953 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",
954 dev_name(chip->dev));
955
956 /* Disable eDMA, only to establish the ideal initial conditions */
957 dw_edma_core_off(dw);
958
959 /* Request IRQs */
960 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc);
961 if (err)
962 return err;
963
964 /* Setup write/read channels */
965 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);
966 if (err)
967 goto err_irq_free;
968
969 /* Turn debugfs on */
970 dw_edma_core_debugfs_on(dw);
971
972 chip->dw = dw;
973
974 return 0;
975
976 err_irq_free:
977 for (i = (dw->nr_irqs - 1); i >= 0; i--)
978 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
979
980 return err;
981 }
982 EXPORT_SYMBOL_GPL(dw_edma_probe);
983
dw_edma_remove(struct dw_edma_chip * chip)984 int dw_edma_remove(struct dw_edma_chip *chip)
985 {
986 struct dw_edma_chan *chan, *_chan;
987 struct device *dev = chip->dev;
988 struct dw_edma *dw = chip->dw;
989 int i;
990
991 /* Skip removal if no private data found */
992 if (!dw)
993 return -ENODEV;
994
995 /* Disable eDMA */
996 dw_edma_core_off(dw);
997
998 /* Free irqs */
999 for (i = (dw->nr_irqs - 1); i >= 0; i--)
1000 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
1001
1002 /* Deregister eDMA device */
1003 dma_async_device_unregister(&dw->dma);
1004 list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
1005 vc.chan.device_node) {
1006 tasklet_kill(&chan->vc.task);
1007 list_del(&chan->vc.chan.device_node);
1008 }
1009
1010 return 0;
1011 }
1012 EXPORT_SYMBOL_GPL(dw_edma_remove);
1013
1014 MODULE_LICENSE("GPL v2");
1015 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
1016 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
1017