• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2012 Marvell International Ltd.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  */
8 
9 #include <linux/err.h>
10 #include <linux/module.h>
11 #include <linux/init.h>
12 #include <linux/types.h>
13 #include <linux/interrupt.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/slab.h>
16 #include <linux/dmaengine.h>
17 #include <linux/platform_device.h>
18 #include <linux/device.h>
19 #include <linux/platform_data/mmp_dma.h>
20 #include <linux/dmapool.h>
21 #include <linux/of_device.h>
22 #include <linux/of_dma.h>
23 #include <linux/of.h>
24 #include <linux/dma/mmp-pdma.h>
25 
26 #include "dmaengine.h"
27 
28 #define DCSR		0x0000
29 #define DALGN		0x00a0
30 #define DINT		0x00f0
31 #define DDADR		0x0200
32 #define DSADR(n)	(0x0204 + ((n) << 4))
33 #define DTADR(n)	(0x0208 + ((n) << 4))
34 #define DCMD		0x020c
35 
36 #define DCSR_RUN	BIT(31)	/* Run Bit (read / write) */
37 #define DCSR_NODESC	BIT(30)	/* No-Descriptor Fetch (read / write) */
38 #define DCSR_STOPIRQEN	BIT(29)	/* Stop Interrupt Enable (read / write) */
39 #define DCSR_REQPEND	BIT(8)	/* Request Pending (read-only) */
40 #define DCSR_STOPSTATE	BIT(3)	/* Stop State (read-only) */
41 #define DCSR_ENDINTR	BIT(2)	/* End Interrupt (read / write) */
42 #define DCSR_STARTINTR	BIT(1)	/* Start Interrupt (read / write) */
43 #define DCSR_BUSERR	BIT(0)	/* Bus Error Interrupt (read / write) */
44 
45 #define DCSR_EORIRQEN	BIT(28)	/* End of Receive Interrupt Enable (R/W) */
46 #define DCSR_EORJMPEN	BIT(27)	/* Jump to next descriptor on EOR */
47 #define DCSR_EORSTOPEN	BIT(26)	/* STOP on an EOR */
48 #define DCSR_SETCMPST	BIT(25)	/* Set Descriptor Compare Status */
49 #define DCSR_CLRCMPST	BIT(24)	/* Clear Descriptor Compare Status */
50 #define DCSR_CMPST	BIT(10)	/* The Descriptor Compare Status */
51 #define DCSR_EORINTR	BIT(9)	/* The end of Receive */
52 
53 #define DRCMR(n)	((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2))
54 #define DRCMR_MAPVLD	BIT(7)	/* Map Valid (read / write) */
55 #define DRCMR_CHLNUM	0x1f	/* mask for Channel Number (read / write) */
56 
57 #define DDADR_DESCADDR	0xfffffff0	/* Address of next descriptor (mask) */
58 #define DDADR_STOP	BIT(0)	/* Stop (read / write) */
59 
60 #define DCMD_INCSRCADDR	BIT(31)	/* Source Address Increment Setting. */
61 #define DCMD_INCTRGADDR	BIT(30)	/* Target Address Increment Setting. */
62 #define DCMD_FLOWSRC	BIT(29)	/* Flow Control by the source. */
63 #define DCMD_FLOWTRG	BIT(28)	/* Flow Control by the target. */
64 #define DCMD_STARTIRQEN	BIT(22)	/* Start Interrupt Enable */
65 #define DCMD_ENDIRQEN	BIT(21)	/* End Interrupt Enable */
66 #define DCMD_ENDIAN	BIT(18)	/* Device Endian-ness. */
67 #define DCMD_BURST8	(1 << 16)	/* 8 byte burst */
68 #define DCMD_BURST16	(2 << 16)	/* 16 byte burst */
69 #define DCMD_BURST32	(3 << 16)	/* 32 byte burst */
70 #define DCMD_WIDTH1	(1 << 14)	/* 1 byte width */
71 #define DCMD_WIDTH2	(2 << 14)	/* 2 byte width (HalfWord) */
72 #define DCMD_WIDTH4	(3 << 14)	/* 4 byte width (Word) */
73 #define DCMD_LENGTH	0x01fff		/* length mask (max = 8K - 1) */
74 
75 #define PDMA_ALIGNMENT		3
76 #define PDMA_MAX_DESC_BYTES	DCMD_LENGTH
77 
78 struct mmp_pdma_desc_hw {
79 	u32 ddadr;	/* Points to the next descriptor + flags */
80 	u32 dsadr;	/* DSADR value for the current transfer */
81 	u32 dtadr;	/* DTADR value for the current transfer */
82 	u32 dcmd;	/* DCMD value for the current transfer */
83 } __aligned(32);
84 
85 struct mmp_pdma_desc_sw {
86 	struct mmp_pdma_desc_hw desc;
87 	struct list_head node;
88 	struct list_head tx_list;
89 	struct dma_async_tx_descriptor async_tx;
90 };
91 
92 struct mmp_pdma_phy;
93 
94 struct mmp_pdma_chan {
95 	struct device *dev;
96 	struct dma_chan chan;
97 	struct dma_async_tx_descriptor desc;
98 	struct mmp_pdma_phy *phy;
99 	enum dma_transfer_direction dir;
100 
101 	struct mmp_pdma_desc_sw *cyclic_first;	/* first desc_sw if channel
102 						 * is in cyclic mode */
103 
104 	/* channel's basic info */
105 	struct tasklet_struct tasklet;
106 	u32 dcmd;
107 	u32 drcmr;
108 	u32 dev_addr;
109 
110 	/* list for desc */
111 	spinlock_t desc_lock;		/* Descriptor list lock */
112 	struct list_head chain_pending;	/* Link descriptors queue for pending */
113 	struct list_head chain_running;	/* Link descriptors queue for running */
114 	bool idle;			/* channel statue machine */
115 	bool byte_align;
116 
117 	struct dma_pool *desc_pool;	/* Descriptors pool */
118 };
119 
120 struct mmp_pdma_phy {
121 	int idx;
122 	void __iomem *base;
123 	struct mmp_pdma_chan *vchan;
124 };
125 
126 struct mmp_pdma_device {
127 	int				dma_channels;
128 	void __iomem			*base;
129 	struct device			*dev;
130 	struct dma_device		device;
131 	struct mmp_pdma_phy		*phy;
132 	spinlock_t phy_lock; /* protect alloc/free phy channels */
133 };
134 
135 #define tx_to_mmp_pdma_desc(tx)					\
136 	container_of(tx, struct mmp_pdma_desc_sw, async_tx)
137 #define to_mmp_pdma_desc(lh)					\
138 	container_of(lh, struct mmp_pdma_desc_sw, node)
139 #define to_mmp_pdma_chan(dchan)					\
140 	container_of(dchan, struct mmp_pdma_chan, chan)
141 #define to_mmp_pdma_dev(dmadev)					\
142 	container_of(dmadev, struct mmp_pdma_device, device)
143 
set_desc(struct mmp_pdma_phy * phy,dma_addr_t addr)144 static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr)
145 {
146 	u32 reg = (phy->idx << 4) + DDADR;
147 
148 	writel(addr, phy->base + reg);
149 }
150 
enable_chan(struct mmp_pdma_phy * phy)151 static void enable_chan(struct mmp_pdma_phy *phy)
152 {
153 	u32 reg, dalgn;
154 
155 	if (!phy->vchan)
156 		return;
157 
158 	reg = DRCMR(phy->vchan->drcmr);
159 	writel(DRCMR_MAPVLD | phy->idx, phy->base + reg);
160 
161 	dalgn = readl(phy->base + DALGN);
162 	if (phy->vchan->byte_align)
163 		dalgn |= 1 << phy->idx;
164 	else
165 		dalgn &= ~(1 << phy->idx);
166 	writel(dalgn, phy->base + DALGN);
167 
168 	reg = (phy->idx << 2) + DCSR;
169 	writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg);
170 }
171 
disable_chan(struct mmp_pdma_phy * phy)172 static void disable_chan(struct mmp_pdma_phy *phy)
173 {
174 	u32 reg;
175 
176 	if (!phy)
177 		return;
178 
179 	reg = (phy->idx << 2) + DCSR;
180 	writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg);
181 }
182 
clear_chan_irq(struct mmp_pdma_phy * phy)183 static int clear_chan_irq(struct mmp_pdma_phy *phy)
184 {
185 	u32 dcsr;
186 	u32 dint = readl(phy->base + DINT);
187 	u32 reg = (phy->idx << 2) + DCSR;
188 
189 	if (!(dint & BIT(phy->idx)))
190 		return -EAGAIN;
191 
192 	/* clear irq */
193 	dcsr = readl(phy->base + reg);
194 	writel(dcsr, phy->base + reg);
195 	if ((dcsr & DCSR_BUSERR) && (phy->vchan))
196 		dev_warn(phy->vchan->dev, "DCSR_BUSERR\n");
197 
198 	return 0;
199 }
200 
mmp_pdma_chan_handler(int irq,void * dev_id)201 static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id)
202 {
203 	struct mmp_pdma_phy *phy = dev_id;
204 
205 	if (clear_chan_irq(phy) != 0)
206 		return IRQ_NONE;
207 
208 	tasklet_schedule(&phy->vchan->tasklet);
209 	return IRQ_HANDLED;
210 }
211 
mmp_pdma_int_handler(int irq,void * dev_id)212 static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id)
213 {
214 	struct mmp_pdma_device *pdev = dev_id;
215 	struct mmp_pdma_phy *phy;
216 	u32 dint = readl(pdev->base + DINT);
217 	int i, ret;
218 	int irq_num = 0;
219 
220 	while (dint) {
221 		i = __ffs(dint);
222 		dint &= (dint - 1);
223 		phy = &pdev->phy[i];
224 		ret = mmp_pdma_chan_handler(irq, phy);
225 		if (ret == IRQ_HANDLED)
226 			irq_num++;
227 	}
228 
229 	if (irq_num)
230 		return IRQ_HANDLED;
231 
232 	return IRQ_NONE;
233 }
234 
235 /* lookup free phy channel as descending priority */
lookup_phy(struct mmp_pdma_chan * pchan)236 static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan)
237 {
238 	int prio, i;
239 	struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
240 	struct mmp_pdma_phy *phy, *found = NULL;
241 	unsigned long flags;
242 
243 	/*
244 	 * dma channel priorities
245 	 * ch 0 - 3,  16 - 19  <--> (0)
246 	 * ch 4 - 7,  20 - 23  <--> (1)
247 	 * ch 8 - 11, 24 - 27  <--> (2)
248 	 * ch 12 - 15, 28 - 31  <--> (3)
249 	 */
250 
251 	spin_lock_irqsave(&pdev->phy_lock, flags);
252 	for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) {
253 		for (i = 0; i < pdev->dma_channels; i++) {
254 			if (prio != (i & 0xf) >> 2)
255 				continue;
256 			phy = &pdev->phy[i];
257 			if (!phy->vchan) {
258 				phy->vchan = pchan;
259 				found = phy;
260 				goto out_unlock;
261 			}
262 		}
263 	}
264 
265 out_unlock:
266 	spin_unlock_irqrestore(&pdev->phy_lock, flags);
267 	return found;
268 }
269 
mmp_pdma_free_phy(struct mmp_pdma_chan * pchan)270 static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
271 {
272 	struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
273 	unsigned long flags;
274 	u32 reg;
275 
276 	if (!pchan->phy)
277 		return;
278 
279 	/* clear the channel mapping in DRCMR */
280 	reg = DRCMR(pchan->drcmr);
281 	writel(0, pchan->phy->base + reg);
282 
283 	spin_lock_irqsave(&pdev->phy_lock, flags);
284 	pchan->phy->vchan = NULL;
285 	pchan->phy = NULL;
286 	spin_unlock_irqrestore(&pdev->phy_lock, flags);
287 }
288 
289 /**
290  * start_pending_queue - transfer any pending transactions
291  * pending list ==> running list
292  */
start_pending_queue(struct mmp_pdma_chan * chan)293 static void start_pending_queue(struct mmp_pdma_chan *chan)
294 {
295 	struct mmp_pdma_desc_sw *desc;
296 
297 	/* still in running, irq will start the pending list */
298 	if (!chan->idle) {
299 		dev_dbg(chan->dev, "DMA controller still busy\n");
300 		return;
301 	}
302 
303 	if (list_empty(&chan->chain_pending)) {
304 		/* chance to re-fetch phy channel with higher prio */
305 		mmp_pdma_free_phy(chan);
306 		dev_dbg(chan->dev, "no pending list\n");
307 		return;
308 	}
309 
310 	if (!chan->phy) {
311 		chan->phy = lookup_phy(chan);
312 		if (!chan->phy) {
313 			dev_dbg(chan->dev, "no free dma channel\n");
314 			return;
315 		}
316 	}
317 
318 	/*
319 	 * pending -> running
320 	 * reintilize pending list
321 	 */
322 	desc = list_first_entry(&chan->chain_pending,
323 				struct mmp_pdma_desc_sw, node);
324 	list_splice_tail_init(&chan->chain_pending, &chan->chain_running);
325 
326 	/*
327 	 * Program the descriptor's address into the DMA controller,
328 	 * then start the DMA transaction
329 	 */
330 	set_desc(chan->phy, desc->async_tx.phys);
331 	enable_chan(chan->phy);
332 	chan->idle = false;
333 }
334 
335 
336 /* desc->tx_list ==> pending list */
mmp_pdma_tx_submit(struct dma_async_tx_descriptor * tx)337 static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx)
338 {
339 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan);
340 	struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx);
341 	struct mmp_pdma_desc_sw *child;
342 	unsigned long flags;
343 	dma_cookie_t cookie = -EBUSY;
344 
345 	spin_lock_irqsave(&chan->desc_lock, flags);
346 
347 	list_for_each_entry(child, &desc->tx_list, node) {
348 		cookie = dma_cookie_assign(&child->async_tx);
349 	}
350 
351 	/* softly link to pending list - desc->tx_list ==> pending list */
352 	list_splice_tail_init(&desc->tx_list, &chan->chain_pending);
353 
354 	spin_unlock_irqrestore(&chan->desc_lock, flags);
355 
356 	return cookie;
357 }
358 
359 static struct mmp_pdma_desc_sw *
mmp_pdma_alloc_descriptor(struct mmp_pdma_chan * chan)360 mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan)
361 {
362 	struct mmp_pdma_desc_sw *desc;
363 	dma_addr_t pdesc;
364 
365 	desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
366 	if (!desc) {
367 		dev_err(chan->dev, "out of memory for link descriptor\n");
368 		return NULL;
369 	}
370 
371 	memset(desc, 0, sizeof(*desc));
372 	INIT_LIST_HEAD(&desc->tx_list);
373 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
374 	/* each desc has submit */
375 	desc->async_tx.tx_submit = mmp_pdma_tx_submit;
376 	desc->async_tx.phys = pdesc;
377 
378 	return desc;
379 }
380 
381 /**
382  * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel.
383  *
384  * This function will create a dma pool for descriptor allocation.
385  * Request irq only when channel is requested
386  * Return - The number of allocated descriptors.
387  */
388 
mmp_pdma_alloc_chan_resources(struct dma_chan * dchan)389 static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan)
390 {
391 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
392 
393 	if (chan->desc_pool)
394 		return 1;
395 
396 	chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device),
397 					  chan->dev,
398 					  sizeof(struct mmp_pdma_desc_sw),
399 					  __alignof__(struct mmp_pdma_desc_sw),
400 					  0);
401 	if (!chan->desc_pool) {
402 		dev_err(chan->dev, "unable to allocate descriptor pool\n");
403 		return -ENOMEM;
404 	}
405 
406 	mmp_pdma_free_phy(chan);
407 	chan->idle = true;
408 	chan->dev_addr = 0;
409 	return 1;
410 }
411 
mmp_pdma_free_desc_list(struct mmp_pdma_chan * chan,struct list_head * list)412 static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan,
413 				    struct list_head *list)
414 {
415 	struct mmp_pdma_desc_sw *desc, *_desc;
416 
417 	list_for_each_entry_safe(desc, _desc, list, node) {
418 		list_del(&desc->node);
419 		dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
420 	}
421 }
422 
mmp_pdma_free_chan_resources(struct dma_chan * dchan)423 static void mmp_pdma_free_chan_resources(struct dma_chan *dchan)
424 {
425 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
426 	unsigned long flags;
427 
428 	spin_lock_irqsave(&chan->desc_lock, flags);
429 	mmp_pdma_free_desc_list(chan, &chan->chain_pending);
430 	mmp_pdma_free_desc_list(chan, &chan->chain_running);
431 	spin_unlock_irqrestore(&chan->desc_lock, flags);
432 
433 	dma_pool_destroy(chan->desc_pool);
434 	chan->desc_pool = NULL;
435 	chan->idle = true;
436 	chan->dev_addr = 0;
437 	mmp_pdma_free_phy(chan);
438 	return;
439 }
440 
441 static struct dma_async_tx_descriptor *
mmp_pdma_prep_memcpy(struct dma_chan * dchan,dma_addr_t dma_dst,dma_addr_t dma_src,size_t len,unsigned long flags)442 mmp_pdma_prep_memcpy(struct dma_chan *dchan,
443 		     dma_addr_t dma_dst, dma_addr_t dma_src,
444 		     size_t len, unsigned long flags)
445 {
446 	struct mmp_pdma_chan *chan;
447 	struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
448 	size_t copy = 0;
449 
450 	if (!dchan)
451 		return NULL;
452 
453 	if (!len)
454 		return NULL;
455 
456 	chan = to_mmp_pdma_chan(dchan);
457 	chan->byte_align = false;
458 
459 	if (!chan->dir) {
460 		chan->dir = DMA_MEM_TO_MEM;
461 		chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR;
462 		chan->dcmd |= DCMD_BURST32;
463 	}
464 
465 	do {
466 		/* Allocate the link descriptor from DMA pool */
467 		new = mmp_pdma_alloc_descriptor(chan);
468 		if (!new) {
469 			dev_err(chan->dev, "no memory for desc\n");
470 			goto fail;
471 		}
472 
473 		copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES);
474 		if (dma_src & 0x7 || dma_dst & 0x7)
475 			chan->byte_align = true;
476 
477 		new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy);
478 		new->desc.dsadr = dma_src;
479 		new->desc.dtadr = dma_dst;
480 
481 		if (!first)
482 			first = new;
483 		else
484 			prev->desc.ddadr = new->async_tx.phys;
485 
486 		new->async_tx.cookie = 0;
487 		async_tx_ack(&new->async_tx);
488 
489 		prev = new;
490 		len -= copy;
491 
492 		if (chan->dir == DMA_MEM_TO_DEV) {
493 			dma_src += copy;
494 		} else if (chan->dir == DMA_DEV_TO_MEM) {
495 			dma_dst += copy;
496 		} else if (chan->dir == DMA_MEM_TO_MEM) {
497 			dma_src += copy;
498 			dma_dst += copy;
499 		}
500 
501 		/* Insert the link descriptor to the LD ring */
502 		list_add_tail(&new->node, &first->tx_list);
503 	} while (len);
504 
505 	first->async_tx.flags = flags; /* client is in control of this ack */
506 	first->async_tx.cookie = -EBUSY;
507 
508 	/* last desc and fire IRQ */
509 	new->desc.ddadr = DDADR_STOP;
510 	new->desc.dcmd |= DCMD_ENDIRQEN;
511 
512 	chan->cyclic_first = NULL;
513 
514 	return &first->async_tx;
515 
516 fail:
517 	if (first)
518 		mmp_pdma_free_desc_list(chan, &first->tx_list);
519 	return NULL;
520 }
521 
522 static struct dma_async_tx_descriptor *
mmp_pdma_prep_slave_sg(struct dma_chan * dchan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction dir,unsigned long flags,void * context)523 mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
524 		       unsigned int sg_len, enum dma_transfer_direction dir,
525 		       unsigned long flags, void *context)
526 {
527 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
528 	struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL;
529 	size_t len, avail;
530 	struct scatterlist *sg;
531 	dma_addr_t addr;
532 	int i;
533 
534 	if ((sgl == NULL) || (sg_len == 0))
535 		return NULL;
536 
537 	chan->byte_align = false;
538 
539 	for_each_sg(sgl, sg, sg_len, i) {
540 		addr = sg_dma_address(sg);
541 		avail = sg_dma_len(sgl);
542 
543 		do {
544 			len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
545 			if (addr & 0x7)
546 				chan->byte_align = true;
547 
548 			/* allocate and populate the descriptor */
549 			new = mmp_pdma_alloc_descriptor(chan);
550 			if (!new) {
551 				dev_err(chan->dev, "no memory for desc\n");
552 				goto fail;
553 			}
554 
555 			new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len);
556 			if (dir == DMA_MEM_TO_DEV) {
557 				new->desc.dsadr = addr;
558 				new->desc.dtadr = chan->dev_addr;
559 			} else {
560 				new->desc.dsadr = chan->dev_addr;
561 				new->desc.dtadr = addr;
562 			}
563 
564 			if (!first)
565 				first = new;
566 			else
567 				prev->desc.ddadr = new->async_tx.phys;
568 
569 			new->async_tx.cookie = 0;
570 			async_tx_ack(&new->async_tx);
571 			prev = new;
572 
573 			/* Insert the link descriptor to the LD ring */
574 			list_add_tail(&new->node, &first->tx_list);
575 
576 			/* update metadata */
577 			addr += len;
578 			avail -= len;
579 		} while (avail);
580 	}
581 
582 	first->async_tx.cookie = -EBUSY;
583 	first->async_tx.flags = flags;
584 
585 	/* last desc and fire IRQ */
586 	new->desc.ddadr = DDADR_STOP;
587 	new->desc.dcmd |= DCMD_ENDIRQEN;
588 
589 	chan->dir = dir;
590 	chan->cyclic_first = NULL;
591 
592 	return &first->async_tx;
593 
594 fail:
595 	if (first)
596 		mmp_pdma_free_desc_list(chan, &first->tx_list);
597 	return NULL;
598 }
599 
600 static struct dma_async_tx_descriptor *
mmp_pdma_prep_dma_cyclic(struct dma_chan * dchan,dma_addr_t buf_addr,size_t len,size_t period_len,enum dma_transfer_direction direction,unsigned long flags)601 mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan,
602 			 dma_addr_t buf_addr, size_t len, size_t period_len,
603 			 enum dma_transfer_direction direction,
604 			 unsigned long flags)
605 {
606 	struct mmp_pdma_chan *chan;
607 	struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
608 	dma_addr_t dma_src, dma_dst;
609 
610 	if (!dchan || !len || !period_len)
611 		return NULL;
612 
613 	/* the buffer length must be a multiple of period_len */
614 	if (len % period_len != 0)
615 		return NULL;
616 
617 	if (period_len > PDMA_MAX_DESC_BYTES)
618 		return NULL;
619 
620 	chan = to_mmp_pdma_chan(dchan);
621 
622 	switch (direction) {
623 	case DMA_MEM_TO_DEV:
624 		dma_src = buf_addr;
625 		dma_dst = chan->dev_addr;
626 		break;
627 	case DMA_DEV_TO_MEM:
628 		dma_dst = buf_addr;
629 		dma_src = chan->dev_addr;
630 		break;
631 	default:
632 		dev_err(chan->dev, "Unsupported direction for cyclic DMA\n");
633 		return NULL;
634 	}
635 
636 	chan->dir = direction;
637 
638 	do {
639 		/* Allocate the link descriptor from DMA pool */
640 		new = mmp_pdma_alloc_descriptor(chan);
641 		if (!new) {
642 			dev_err(chan->dev, "no memory for desc\n");
643 			goto fail;
644 		}
645 
646 		new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN |
647 				  (DCMD_LENGTH & period_len));
648 		new->desc.dsadr = dma_src;
649 		new->desc.dtadr = dma_dst;
650 
651 		if (!first)
652 			first = new;
653 		else
654 			prev->desc.ddadr = new->async_tx.phys;
655 
656 		new->async_tx.cookie = 0;
657 		async_tx_ack(&new->async_tx);
658 
659 		prev = new;
660 		len -= period_len;
661 
662 		if (chan->dir == DMA_MEM_TO_DEV)
663 			dma_src += period_len;
664 		else
665 			dma_dst += period_len;
666 
667 		/* Insert the link descriptor to the LD ring */
668 		list_add_tail(&new->node, &first->tx_list);
669 	} while (len);
670 
671 	first->async_tx.flags = flags; /* client is in control of this ack */
672 	first->async_tx.cookie = -EBUSY;
673 
674 	/* make the cyclic link */
675 	new->desc.ddadr = first->async_tx.phys;
676 	chan->cyclic_first = first;
677 
678 	return &first->async_tx;
679 
680 fail:
681 	if (first)
682 		mmp_pdma_free_desc_list(chan, &first->tx_list);
683 	return NULL;
684 }
685 
mmp_pdma_control(struct dma_chan * dchan,enum dma_ctrl_cmd cmd,unsigned long arg)686 static int mmp_pdma_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd,
687 			    unsigned long arg)
688 {
689 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
690 	struct dma_slave_config *cfg = (void *)arg;
691 	unsigned long flags;
692 	u32 maxburst = 0, addr = 0;
693 	enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
694 
695 	if (!dchan)
696 		return -EINVAL;
697 
698 	switch (cmd) {
699 	case DMA_TERMINATE_ALL:
700 		disable_chan(chan->phy);
701 		mmp_pdma_free_phy(chan);
702 		spin_lock_irqsave(&chan->desc_lock, flags);
703 		mmp_pdma_free_desc_list(chan, &chan->chain_pending);
704 		mmp_pdma_free_desc_list(chan, &chan->chain_running);
705 		spin_unlock_irqrestore(&chan->desc_lock, flags);
706 		chan->idle = true;
707 		break;
708 	case DMA_SLAVE_CONFIG:
709 		if (cfg->direction == DMA_DEV_TO_MEM) {
710 			chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC;
711 			maxburst = cfg->src_maxburst;
712 			width = cfg->src_addr_width;
713 			addr = cfg->src_addr;
714 		} else if (cfg->direction == DMA_MEM_TO_DEV) {
715 			chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG;
716 			maxburst = cfg->dst_maxburst;
717 			width = cfg->dst_addr_width;
718 			addr = cfg->dst_addr;
719 		}
720 
721 		if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
722 			chan->dcmd |= DCMD_WIDTH1;
723 		else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
724 			chan->dcmd |= DCMD_WIDTH2;
725 		else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES)
726 			chan->dcmd |= DCMD_WIDTH4;
727 
728 		if (maxburst == 8)
729 			chan->dcmd |= DCMD_BURST8;
730 		else if (maxburst == 16)
731 			chan->dcmd |= DCMD_BURST16;
732 		else if (maxburst == 32)
733 			chan->dcmd |= DCMD_BURST32;
734 
735 		chan->dir = cfg->direction;
736 		chan->dev_addr = addr;
737 		/* FIXME: drivers should be ported over to use the filter
738 		 * function. Once that's done, the following two lines can
739 		 * be removed.
740 		 */
741 		if (cfg->slave_id)
742 			chan->drcmr = cfg->slave_id;
743 		break;
744 	default:
745 		return -ENOSYS;
746 	}
747 
748 	return 0;
749 }
750 
mmp_pdma_residue(struct mmp_pdma_chan * chan,dma_cookie_t cookie)751 static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
752 				     dma_cookie_t cookie)
753 {
754 	struct mmp_pdma_desc_sw *sw;
755 	u32 curr, residue = 0;
756 	bool passed = false;
757 	bool cyclic = chan->cyclic_first != NULL;
758 
759 	/*
760 	 * If the channel does not have a phy pointer anymore, it has already
761 	 * been completed. Therefore, its residue is 0.
762 	 */
763 	if (!chan->phy)
764 		return 0;
765 
766 	if (chan->dir == DMA_DEV_TO_MEM)
767 		curr = readl(chan->phy->base + DTADR(chan->phy->idx));
768 	else
769 		curr = readl(chan->phy->base + DSADR(chan->phy->idx));
770 
771 	list_for_each_entry(sw, &chan->chain_running, node) {
772 		u32 start, end, len;
773 
774 		if (chan->dir == DMA_DEV_TO_MEM)
775 			start = sw->desc.dtadr;
776 		else
777 			start = sw->desc.dsadr;
778 
779 		len = sw->desc.dcmd & DCMD_LENGTH;
780 		end = start + len;
781 
782 		/*
783 		 * 'passed' will be latched once we found the descriptor which
784 		 * lies inside the boundaries of the curr pointer. All
785 		 * descriptors that occur in the list _after_ we found that
786 		 * partially handled descriptor are still to be processed and
787 		 * are hence added to the residual bytes counter.
788 		 */
789 
790 		if (passed) {
791 			residue += len;
792 		} else if (curr >= start && curr <= end) {
793 			residue += end - curr;
794 			passed = true;
795 		}
796 
797 		/*
798 		 * Descriptors that have the ENDIRQEN bit set mark the end of a
799 		 * transaction chain, and the cookie assigned with it has been
800 		 * returned previously from mmp_pdma_tx_submit().
801 		 *
802 		 * In case we have multiple transactions in the running chain,
803 		 * and the cookie does not match the one the user asked us
804 		 * about, reset the state variables and start over.
805 		 *
806 		 * This logic does not apply to cyclic transactions, where all
807 		 * descriptors have the ENDIRQEN bit set, and for which we
808 		 * can't have multiple transactions on one channel anyway.
809 		 */
810 		if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN))
811 			continue;
812 
813 		if (sw->async_tx.cookie == cookie) {
814 			return residue;
815 		} else {
816 			residue = 0;
817 			passed = false;
818 		}
819 	}
820 
821 	/* We should only get here in case of cyclic transactions */
822 	return residue;
823 }
824 
mmp_pdma_tx_status(struct dma_chan * dchan,dma_cookie_t cookie,struct dma_tx_state * txstate)825 static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan,
826 					  dma_cookie_t cookie,
827 					  struct dma_tx_state *txstate)
828 {
829 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
830 	enum dma_status ret;
831 
832 	ret = dma_cookie_status(dchan, cookie, txstate);
833 	if (likely(ret != DMA_ERROR))
834 		dma_set_residue(txstate, mmp_pdma_residue(chan, cookie));
835 
836 	return ret;
837 }
838 
839 /**
840  * mmp_pdma_issue_pending - Issue the DMA start command
841  * pending list ==> running list
842  */
mmp_pdma_issue_pending(struct dma_chan * dchan)843 static void mmp_pdma_issue_pending(struct dma_chan *dchan)
844 {
845 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
846 	unsigned long flags;
847 
848 	spin_lock_irqsave(&chan->desc_lock, flags);
849 	start_pending_queue(chan);
850 	spin_unlock_irqrestore(&chan->desc_lock, flags);
851 }
852 
853 /*
854  * dma_do_tasklet
855  * Do call back
856  * Start pending list
857  */
dma_do_tasklet(unsigned long data)858 static void dma_do_tasklet(unsigned long data)
859 {
860 	struct mmp_pdma_chan *chan = (struct mmp_pdma_chan *)data;
861 	struct mmp_pdma_desc_sw *desc, *_desc;
862 	LIST_HEAD(chain_cleanup);
863 	unsigned long flags;
864 
865 	if (chan->cyclic_first) {
866 		dma_async_tx_callback cb = NULL;
867 		void *cb_data = NULL;
868 
869 		spin_lock_irqsave(&chan->desc_lock, flags);
870 		desc = chan->cyclic_first;
871 		cb = desc->async_tx.callback;
872 		cb_data = desc->async_tx.callback_param;
873 		spin_unlock_irqrestore(&chan->desc_lock, flags);
874 
875 		if (cb)
876 			cb(cb_data);
877 
878 		return;
879 	}
880 
881 	/* submit pending list; callback for each desc; free desc */
882 	spin_lock_irqsave(&chan->desc_lock, flags);
883 
884 	list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) {
885 		/*
886 		 * move the descriptors to a temporary list so we can drop
887 		 * the lock during the entire cleanup operation
888 		 */
889 		list_move(&desc->node, &chain_cleanup);
890 
891 		/*
892 		 * Look for the first list entry which has the ENDIRQEN flag
893 		 * set. That is the descriptor we got an interrupt for, so
894 		 * complete that transaction and its cookie.
895 		 */
896 		if (desc->desc.dcmd & DCMD_ENDIRQEN) {
897 			dma_cookie_t cookie = desc->async_tx.cookie;
898 			dma_cookie_complete(&desc->async_tx);
899 			dev_dbg(chan->dev, "completed_cookie=%d\n", cookie);
900 			break;
901 		}
902 	}
903 
904 	/*
905 	 * The hardware is idle and ready for more when the
906 	 * chain_running list is empty.
907 	 */
908 	chan->idle = list_empty(&chan->chain_running);
909 
910 	/* Start any pending transactions automatically */
911 	start_pending_queue(chan);
912 	spin_unlock_irqrestore(&chan->desc_lock, flags);
913 
914 	/* Run the callback for each descriptor, in order */
915 	list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) {
916 		struct dma_async_tx_descriptor *txd = &desc->async_tx;
917 
918 		/* Remove from the list of transactions */
919 		list_del(&desc->node);
920 		/* Run the link descriptor callback function */
921 		if (txd->callback)
922 			txd->callback(txd->callback_param);
923 
924 		dma_pool_free(chan->desc_pool, desc, txd->phys);
925 	}
926 }
927 
mmp_pdma_remove(struct platform_device * op)928 static int mmp_pdma_remove(struct platform_device *op)
929 {
930 	struct mmp_pdma_device *pdev = platform_get_drvdata(op);
931 
932 	dma_async_device_unregister(&pdev->device);
933 	return 0;
934 }
935 
mmp_pdma_chan_init(struct mmp_pdma_device * pdev,int idx,int irq)936 static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq)
937 {
938 	struct mmp_pdma_phy *phy  = &pdev->phy[idx];
939 	struct mmp_pdma_chan *chan;
940 	int ret;
941 
942 	chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL);
943 	if (chan == NULL)
944 		return -ENOMEM;
945 
946 	phy->idx = idx;
947 	phy->base = pdev->base;
948 
949 	if (irq) {
950 		ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler,
951 				       IRQF_SHARED, "pdma", phy);
952 		if (ret) {
953 			dev_err(pdev->dev, "channel request irq fail!\n");
954 			return ret;
955 		}
956 	}
957 
958 	spin_lock_init(&chan->desc_lock);
959 	chan->dev = pdev->dev;
960 	chan->chan.device = &pdev->device;
961 	tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);
962 	INIT_LIST_HEAD(&chan->chain_pending);
963 	INIT_LIST_HEAD(&chan->chain_running);
964 
965 	/* register virt channel to dma engine */
966 	list_add_tail(&chan->chan.device_node, &pdev->device.channels);
967 
968 	return 0;
969 }
970 
971 static struct of_device_id mmp_pdma_dt_ids[] = {
972 	{ .compatible = "marvell,pdma-1.0", },
973 	{}
974 };
975 MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids);
976 
mmp_pdma_dma_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)977 static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec,
978 					   struct of_dma *ofdma)
979 {
980 	struct mmp_pdma_device *d = ofdma->of_dma_data;
981 	struct dma_chan *chan;
982 
983 	chan = dma_get_any_slave_channel(&d->device);
984 	if (!chan)
985 		return NULL;
986 
987 	to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0];
988 
989 	return chan;
990 }
991 
mmp_pdma_probe(struct platform_device * op)992 static int mmp_pdma_probe(struct platform_device *op)
993 {
994 	struct mmp_pdma_device *pdev;
995 	const struct of_device_id *of_id;
996 	struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev);
997 	struct resource *iores;
998 	int i, ret, irq = 0;
999 	int dma_channels = 0, irq_num = 0;
1000 
1001 	pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL);
1002 	if (!pdev)
1003 		return -ENOMEM;
1004 
1005 	pdev->dev = &op->dev;
1006 
1007 	spin_lock_init(&pdev->phy_lock);
1008 
1009 	iores = platform_get_resource(op, IORESOURCE_MEM, 0);
1010 	pdev->base = devm_ioremap_resource(pdev->dev, iores);
1011 	if (IS_ERR(pdev->base))
1012 		return PTR_ERR(pdev->base);
1013 
1014 	of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev);
1015 	if (of_id)
1016 		of_property_read_u32(pdev->dev->of_node, "#dma-channels",
1017 				     &dma_channels);
1018 	else if (pdata && pdata->dma_channels)
1019 		dma_channels = pdata->dma_channels;
1020 	else
1021 		dma_channels = 32;	/* default 32 channel */
1022 	pdev->dma_channels = dma_channels;
1023 
1024 	for (i = 0; i < dma_channels; i++) {
1025 		if (platform_get_irq(op, i) > 0)
1026 			irq_num++;
1027 	}
1028 
1029 	pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy),
1030 				 GFP_KERNEL);
1031 	if (pdev->phy == NULL)
1032 		return -ENOMEM;
1033 
1034 	INIT_LIST_HEAD(&pdev->device.channels);
1035 
1036 	if (irq_num != dma_channels) {
1037 		/* all chan share one irq, demux inside */
1038 		irq = platform_get_irq(op, 0);
1039 		ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler,
1040 				       IRQF_SHARED, "pdma", pdev);
1041 		if (ret)
1042 			return ret;
1043 	}
1044 
1045 	for (i = 0; i < dma_channels; i++) {
1046 		irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i);
1047 		ret = mmp_pdma_chan_init(pdev, i, irq);
1048 		if (ret)
1049 			return ret;
1050 	}
1051 
1052 	dma_cap_set(DMA_SLAVE, pdev->device.cap_mask);
1053 	dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask);
1054 	dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask);
1055 	dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask);
1056 	pdev->device.dev = &op->dev;
1057 	pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources;
1058 	pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources;
1059 	pdev->device.device_tx_status = mmp_pdma_tx_status;
1060 	pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy;
1061 	pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg;
1062 	pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic;
1063 	pdev->device.device_issue_pending = mmp_pdma_issue_pending;
1064 	pdev->device.device_control = mmp_pdma_control;
1065 	pdev->device.copy_align = PDMA_ALIGNMENT;
1066 
1067 	if (pdev->dev->coherent_dma_mask)
1068 		dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask);
1069 	else
1070 		dma_set_mask(pdev->dev, DMA_BIT_MASK(64));
1071 
1072 	ret = dma_async_device_register(&pdev->device);
1073 	if (ret) {
1074 		dev_err(pdev->device.dev, "unable to register\n");
1075 		return ret;
1076 	}
1077 
1078 	if (op->dev.of_node) {
1079 		/* Device-tree DMA controller registration */
1080 		ret = of_dma_controller_register(op->dev.of_node,
1081 						 mmp_pdma_dma_xlate, pdev);
1082 		if (ret < 0) {
1083 			dev_err(&op->dev, "of_dma_controller_register failed\n");
1084 			return ret;
1085 		}
1086 	}
1087 
1088 	platform_set_drvdata(op, pdev);
1089 	dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels);
1090 	return 0;
1091 }
1092 
1093 static const struct platform_device_id mmp_pdma_id_table[] = {
1094 	{ "mmp-pdma", },
1095 	{ },
1096 };
1097 
1098 static struct platform_driver mmp_pdma_driver = {
1099 	.driver		= {
1100 		.name	= "mmp-pdma",
1101 		.owner  = THIS_MODULE,
1102 		.of_match_table = mmp_pdma_dt_ids,
1103 	},
1104 	.id_table	= mmp_pdma_id_table,
1105 	.probe		= mmp_pdma_probe,
1106 	.remove		= mmp_pdma_remove,
1107 };
1108 
mmp_pdma_filter_fn(struct dma_chan * chan,void * param)1109 bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param)
1110 {
1111 	struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan);
1112 
1113 	if (chan->device->dev->driver != &mmp_pdma_driver.driver)
1114 		return false;
1115 
1116 	c->drcmr = *(unsigned int *)param;
1117 
1118 	return true;
1119 }
1120 EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn);
1121 
1122 module_platform_driver(mmp_pdma_driver);
1123 
1124 MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver");
1125 MODULE_AUTHOR("Marvell International Ltd.");
1126 MODULE_LICENSE("GPL v2");
1127