1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2013-2014 Allwinner Tech Co., Ltd
4 * Author: Sugar <shuge@allwinnertech.com>
5 *
6 * Copyright (C) 2014 Maxime Ripard
7 * Maxime Ripard <maxime.ripard@free-electrons.com>
8 */
9
10 #include <linux/clk.h>
11 #include <linux/delay.h>
12 #include <linux/dmaengine.h>
13 #include <linux/dmapool.h>
14 #include <linux/interrupt.h>
15 #include <linux/module.h>
16 #include <linux/of_dma.h>
17 #include <linux/of_device.h>
18 #include <linux/platform_device.h>
19 #include <linux/reset.h>
20 #include <linux/slab.h>
21 #include <linux/types.h>
22
23 #include "virt-dma.h"
24
25 /*
26 * Common registers
27 */
28 #define DMA_IRQ_EN(x) ((x) * 0x04)
29 #define DMA_IRQ_HALF BIT(0)
30 #define DMA_IRQ_PKG BIT(1)
31 #define DMA_IRQ_QUEUE BIT(2)
32
33 #define DMA_IRQ_CHAN_NR 8
34 #define DMA_IRQ_CHAN_WIDTH 4
35
36
37 #define DMA_IRQ_STAT(x) ((x) * 0x04 + 0x10)
38
39 #define DMA_STAT 0x30
40
41 /* Offset between DMA_IRQ_EN and DMA_IRQ_STAT limits number of channels */
42 #define DMA_MAX_CHANNELS (DMA_IRQ_CHAN_NR * 0x10 / 4)
43
44 /*
45 * sun8i specific registers
46 */
47 #define SUN8I_DMA_GATE 0x20
48 #define SUN8I_DMA_GATE_ENABLE 0x4
49
50 #define SUNXI_H3_SECURE_REG 0x20
51 #define SUNXI_H3_DMA_GATE 0x28
52 #define SUNXI_H3_DMA_GATE_ENABLE 0x4
53 /*
54 * Channels specific registers
55 */
56 #define DMA_CHAN_ENABLE 0x00
57 #define DMA_CHAN_ENABLE_START BIT(0)
58 #define DMA_CHAN_ENABLE_STOP 0
59
60 #define DMA_CHAN_PAUSE 0x04
61 #define DMA_CHAN_PAUSE_PAUSE BIT(0)
62 #define DMA_CHAN_PAUSE_RESUME 0
63
64 #define DMA_CHAN_LLI_ADDR 0x08
65
66 #define DMA_CHAN_CUR_CFG 0x0c
67 #define DMA_CHAN_MAX_DRQ_A31 0x1f
68 #define DMA_CHAN_MAX_DRQ_H6 0x3f
69 #define DMA_CHAN_CFG_SRC_DRQ_A31(x) ((x) & DMA_CHAN_MAX_DRQ_A31)
70 #define DMA_CHAN_CFG_SRC_DRQ_H6(x) ((x) & DMA_CHAN_MAX_DRQ_H6)
71 #define DMA_CHAN_CFG_SRC_MODE_A31(x) (((x) & 0x1) << 5)
72 #define DMA_CHAN_CFG_SRC_MODE_H6(x) (((x) & 0x1) << 8)
73 #define DMA_CHAN_CFG_SRC_BURST_A31(x) (((x) & 0x3) << 7)
74 #define DMA_CHAN_CFG_SRC_BURST_H3(x) (((x) & 0x3) << 6)
75 #define DMA_CHAN_CFG_SRC_WIDTH(x) (((x) & 0x3) << 9)
76
77 #define DMA_CHAN_CFG_DST_DRQ_A31(x) (DMA_CHAN_CFG_SRC_DRQ_A31(x) << 16)
78 #define DMA_CHAN_CFG_DST_DRQ_H6(x) (DMA_CHAN_CFG_SRC_DRQ_H6(x) << 16)
79 #define DMA_CHAN_CFG_DST_MODE_A31(x) (DMA_CHAN_CFG_SRC_MODE_A31(x) << 16)
80 #define DMA_CHAN_CFG_DST_MODE_H6(x) (DMA_CHAN_CFG_SRC_MODE_H6(x) << 16)
81 #define DMA_CHAN_CFG_DST_BURST_A31(x) (DMA_CHAN_CFG_SRC_BURST_A31(x) << 16)
82 #define DMA_CHAN_CFG_DST_BURST_H3(x) (DMA_CHAN_CFG_SRC_BURST_H3(x) << 16)
83 #define DMA_CHAN_CFG_DST_WIDTH(x) (DMA_CHAN_CFG_SRC_WIDTH(x) << 16)
84
85 #define DMA_CHAN_CUR_SRC 0x10
86
87 #define DMA_CHAN_CUR_DST 0x14
88
89 #define DMA_CHAN_CUR_CNT 0x18
90
91 #define DMA_CHAN_CUR_PARA 0x1c
92
93
94 /*
95 * Various hardware related defines
96 */
97 #define LLI_LAST_ITEM 0xfffff800
98 #define NORMAL_WAIT 8
99 #define DRQ_SDRAM 1
100 #define LINEAR_MODE 0
101 #define IO_MODE 1
102
103 #define SET_DST_HIGH_ADDR(x) ((((u64)x >> 32) & 0x3UL) << 18)
104 #define SET_SRC_HIGH_ADDR(x) ((((u64)x >> 32) & 0x3UL) << 16)
105 #define SET_DESC_HIGH_ADDR(x) ((((u64)x >> 32) & 0x3UL) | (x & 0xFFFFFFFC))
106
107 /* forward declaration */
108 struct sun6i_dma_dev;
109
110 /*
111 * Hardware channels / ports representation
112 *
113 * The hardware is used in several SoCs, with differing numbers
114 * of channels and endpoints. This structure ties those numbers
115 * to a certain compatible string.
116 */
117 struct sun6i_dma_config {
118 u32 nr_max_channels;
119 u32 nr_max_requests;
120 u32 nr_max_vchans;
121 /*
122 * In the datasheets/user manuals of newer Allwinner SoCs, a special
123 * bit (bit 2 at register 0x20) is present.
124 * It's named "DMA MCLK interface circuit auto gating bit" in the
125 * documents, and the footnote of this register says that this bit
126 * should be set up when initializing the DMA controller.
127 * Allwinner A23/A33 user manuals do not have this bit documented,
128 * however these SoCs really have and need this bit, as seen in the
129 * BSP kernel source code.
130 */
131 void (*clock_autogate_enable)(struct sun6i_dma_dev *);
132 void (*set_burst_length)(u32 *p_cfg, s8 src_burst, s8 dst_burst);
133 void (*set_drq)(u32 *p_cfg, s8 src_drq, s8 dst_drq);
134 void (*set_mode)(u32 *p_cfg, s8 src_mode, s8 dst_mode);
135 u32 src_burst_lengths;
136 u32 dst_burst_lengths;
137 u32 src_addr_widths;
138 u32 dst_addr_widths;
139 bool has_mbus_clk;
140 };
141
142 /*
143 * Hardware representation of the LLI
144 *
145 * The hardware will be fed the physical address of this structure,
146 * and read its content in order to start the transfer.
147 */
148 struct sun6i_dma_lli {
149 u32 cfg;
150 u32 src;
151 u32 dst;
152 u32 len;
153 u32 para;
154 u32 p_lli_next;
155
156 /*
157 * This field is not used by the DMA controller, but will be
158 * used by the CPU to go through the list (mostly for dumping
159 * or freeing it).
160 */
161 struct sun6i_dma_lli *v_lli_next;
162
163 /*
164 * This param is used to store the physical address of the
165 * coherent cache requested by dma_pool_alloc.
166 */
167 dma_addr_t this_phy;
168 };
169
170
171 struct sun6i_desc {
172 struct virt_dma_desc vd;
173 dma_addr_t p_lli;
174 struct sun6i_dma_lli *v_lli;
175 };
176
177 struct sun6i_pchan {
178 u32 idx;
179 void __iomem *base;
180 struct sun6i_vchan *vchan;
181 struct sun6i_desc *desc;
182 struct sun6i_desc *done;
183 };
184
185 struct sun6i_vchan {
186 struct virt_dma_chan vc;
187 struct list_head node;
188 struct dma_slave_config cfg;
189 struct sun6i_pchan *phy;
190 u8 port;
191 u8 irq_type;
192 bool cyclic;
193 };
194
195 struct sun6i_dma_dev {
196 struct dma_device slave;
197 void __iomem *base;
198 struct clk *clk;
199 struct clk *clk_mbus;
200 int irq;
201 spinlock_t lock;
202 struct reset_control *rstc;
203 struct tasklet_struct task;
204 atomic_t tasklet_shutdown;
205 struct list_head pending;
206 struct dma_pool *pool;
207 struct sun6i_pchan *pchans;
208 struct sun6i_vchan *vchans;
209 const struct sun6i_dma_config *cfg;
210 u32 num_pchans;
211 u32 num_vchans;
212 u32 max_request;
213 };
214
chan2dev(struct dma_chan * chan)215 static struct device *chan2dev(struct dma_chan *chan)
216 {
217 return &chan->dev->device;
218 }
219
to_sun6i_dma_dev(struct dma_device * d)220 static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d)
221 {
222 return container_of(d, struct sun6i_dma_dev, slave);
223 }
224
to_sun6i_vchan(struct dma_chan * chan)225 static inline struct sun6i_vchan *to_sun6i_vchan(struct dma_chan *chan)
226 {
227 return container_of(chan, struct sun6i_vchan, vc.chan);
228 }
229
230 static inline struct sun6i_desc *
to_sun6i_desc(struct dma_async_tx_descriptor * tx)231 to_sun6i_desc(struct dma_async_tx_descriptor *tx)
232 {
233 return container_of(tx, struct sun6i_desc, vd.tx);
234 }
235
sun6i_dma_dump_com_regs(struct sun6i_dma_dev * sdev)236 static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev)
237 {
238 dev_dbg(sdev->slave.dev, "Common register:\n"
239 "\tmask0(%04x): 0x%08x\n"
240 "\tmask1(%04x): 0x%08x\n"
241 "\tpend0(%04x): 0x%08x\n"
242 "\tpend1(%04x): 0x%08x\n"
243 "\tstats(%04x): 0x%08x\n",
244 DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)),
245 DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)),
246 DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)),
247 DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)),
248 DMA_STAT, readl(sdev->base + DMA_STAT));
249 }
250
sun6i_dma_dump_chan_regs(struct sun6i_dma_dev * sdev,struct sun6i_pchan * pchan)251 static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev,
252 struct sun6i_pchan *pchan)
253 {
254
255 dev_dbg(sdev->slave.dev, "Chan %d\n"
256 "\t___en(%04x): \t0x%08x\n"
257 "\tpause(%04x): \t0x%08x\n"
258 "\tstart(%04x): \t0x%08x\n"
259 "\t__cfg(%04x): \t0x%08x\n"
260 "\t__src(%04x): \t0x%08x\n"
261 "\t__dst(%04x): \t0x%08x\n"
262 "\tcount(%04x): \t0x%08x\n"
263 "\t_para(%04x): \t0x%08x\n\n",
264 pchan->idx,
265 DMA_CHAN_ENABLE,
266 readl(pchan->base + DMA_CHAN_ENABLE),
267 DMA_CHAN_PAUSE,
268 readl(pchan->base + DMA_CHAN_PAUSE),
269 DMA_CHAN_LLI_ADDR,
270 readl(pchan->base + DMA_CHAN_LLI_ADDR),
271 DMA_CHAN_CUR_CFG,
272 readl(pchan->base + DMA_CHAN_CUR_CFG),
273 DMA_CHAN_CUR_SRC,
274 readl(pchan->base + DMA_CHAN_CUR_SRC),
275 DMA_CHAN_CUR_DST,
276 readl(pchan->base + DMA_CHAN_CUR_DST),
277 DMA_CHAN_CUR_CNT,
278 readl(pchan->base + DMA_CHAN_CUR_CNT),
279 DMA_CHAN_CUR_PARA,
280 readl(pchan->base + DMA_CHAN_CUR_PARA));
281 }
282
convert_burst(u32 maxburst)283 static inline s8 convert_burst(u32 maxburst)
284 {
285 switch (maxburst) {
286 case 1:
287 return 0;
288 case 4:
289 return 1;
290 case 8:
291 return 2;
292 case 16:
293 return 3;
294 default:
295 return -EINVAL;
296 }
297 }
298
convert_buswidth(enum dma_slave_buswidth addr_width)299 static inline s8 convert_buswidth(enum dma_slave_buswidth addr_width)
300 {
301 return ilog2(addr_width);
302 }
303
sun6i_enable_clock_autogate_a23(struct sun6i_dma_dev * sdev)304 static void sun6i_enable_clock_autogate_a23(struct sun6i_dma_dev *sdev)
305 {
306 writel(SUN8I_DMA_GATE_ENABLE, sdev->base + SUN8I_DMA_GATE);
307 }
308
sun6i_enable_clock_autogate_h3(struct sun6i_dma_dev * sdev)309 static void sun6i_enable_clock_autogate_h3(struct sun6i_dma_dev *sdev)
310 {
311 writel(SUNXI_H3_DMA_GATE_ENABLE, sdev->base + SUNXI_H3_DMA_GATE);
312 }
313
sun6i_set_burst_length_a31(u32 * p_cfg,s8 src_burst,s8 dst_burst)314 static void sun6i_set_burst_length_a31(u32 *p_cfg, s8 src_burst, s8 dst_burst)
315 {
316 *p_cfg |= DMA_CHAN_CFG_SRC_BURST_A31(src_burst) |
317 DMA_CHAN_CFG_DST_BURST_A31(dst_burst);
318 }
319
sun6i_set_burst_length_h3(u32 * p_cfg,s8 src_burst,s8 dst_burst)320 static void sun6i_set_burst_length_h3(u32 *p_cfg, s8 src_burst, s8 dst_burst)
321 {
322 *p_cfg |= DMA_CHAN_CFG_SRC_BURST_H3(src_burst) |
323 DMA_CHAN_CFG_DST_BURST_H3(dst_burst);
324 }
325
sun6i_set_drq_a31(u32 * p_cfg,s8 src_drq,s8 dst_drq)326 static void sun6i_set_drq_a31(u32 *p_cfg, s8 src_drq, s8 dst_drq)
327 {
328 *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_A31(src_drq) |
329 DMA_CHAN_CFG_DST_DRQ_A31(dst_drq);
330 }
331
sun6i_set_drq_h6(u32 * p_cfg,s8 src_drq,s8 dst_drq)332 static void sun6i_set_drq_h6(u32 *p_cfg, s8 src_drq, s8 dst_drq)
333 {
334 *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_H6(src_drq) |
335 DMA_CHAN_CFG_DST_DRQ_H6(dst_drq);
336 }
337
sun6i_set_mode_a31(u32 * p_cfg,s8 src_mode,s8 dst_mode)338 static void sun6i_set_mode_a31(u32 *p_cfg, s8 src_mode, s8 dst_mode)
339 {
340 *p_cfg |= DMA_CHAN_CFG_SRC_MODE_A31(src_mode) |
341 DMA_CHAN_CFG_DST_MODE_A31(dst_mode);
342 }
343
sun6i_set_mode_h6(u32 * p_cfg,s8 src_mode,s8 dst_mode)344 static void sun6i_set_mode_h6(u32 *p_cfg, s8 src_mode, s8 dst_mode)
345 {
346 *p_cfg |= DMA_CHAN_CFG_SRC_MODE_H6(src_mode) |
347 DMA_CHAN_CFG_DST_MODE_H6(dst_mode);
348 }
349
sun6i_get_chan_size(struct sun6i_pchan * pchan)350 static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan)
351 {
352 struct sun6i_desc *txd = pchan->desc;
353 struct sun6i_dma_lli *lli;
354 size_t bytes;
355 dma_addr_t pos;
356
357 pos = readl(pchan->base + DMA_CHAN_LLI_ADDR);
358 bytes = readl(pchan->base + DMA_CHAN_CUR_CNT);
359
360 if (pos == LLI_LAST_ITEM)
361 return bytes;
362
363 for (lli = txd->v_lli; lli; lli = lli->v_lli_next) {
364 if (lli->p_lli_next == pos) {
365 for (lli = lli->v_lli_next; lli; lli = lli->v_lli_next)
366 bytes += lli->len;
367 break;
368 }
369 }
370
371 return bytes;
372 }
373
sun6i_dma_lli_add(struct sun6i_dma_lli * prev,struct sun6i_dma_lli * next,dma_addr_t next_phy,struct sun6i_desc * txd)374 static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev,
375 struct sun6i_dma_lli *next,
376 dma_addr_t next_phy,
377 struct sun6i_desc *txd)
378 {
379 if ((!prev && !txd) || !next)
380 return NULL;
381
382 if (!prev) {
383 txd->p_lli = next_phy;
384 txd->v_lli = next;
385 } else {
386 prev->p_lli_next = next_phy;
387 prev->v_lli_next = next;
388 }
389
390 next->p_lli_next = LLI_LAST_ITEM;
391 next->v_lli_next = NULL;
392
393 return next;
394 }
395
sun6i_dma_dump_lli(struct sun6i_vchan * vchan,struct sun6i_dma_lli * lli)396 static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan,
397 struct sun6i_dma_lli *lli)
398 {
399 dev_dbg(chan2dev(&vchan->vc.chan),
400 "\n\tdesc: p - %pad v - 0x%p\n"
401 "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n"
402 "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n",
403 &lli->this_phy, lli,
404 lli->cfg, lli->src, lli->dst,
405 lli->len, lli->para, lli->p_lli_next);
406
407 }
408
sun6i_dma_free_desc(struct virt_dma_desc * vd)409 static void sun6i_dma_free_desc(struct virt_dma_desc *vd)
410 {
411 struct sun6i_desc *txd = to_sun6i_desc(&vd->tx);
412 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device);
413 struct sun6i_dma_lli *v_lli, *v_next;
414 dma_addr_t p_lli, p_next;
415
416 if (unlikely(!txd))
417 return;
418
419 p_lli = txd->p_lli;
420 v_lli = txd->v_lli;
421
422 while (v_lli) {
423 v_next = v_lli->v_lli_next;
424 p_next = v_lli->p_lli_next;
425
426 dma_pool_free(sdev->pool, v_lli, p_lli);
427
428 v_lli = v_next;
429 p_lli = p_next;
430 }
431
432 txd->vd.tx.callback = NULL;
433 txd->vd.tx.callback_result = NULL;
434 txd->vd.tx.callback_param = NULL;
435 kfree(txd);
436 txd = NULL;
437 }
438
sun6i_dma_start_desc(struct sun6i_vchan * vchan)439 static int sun6i_dma_start_desc(struct sun6i_vchan *vchan)
440 {
441 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device);
442 struct virt_dma_desc *desc = vchan_next_desc(&vchan->vc);
443 struct sun6i_pchan *pchan = vchan->phy;
444 u32 irq_val, irq_reg, irq_offset;
445
446 if (!pchan)
447 return -EAGAIN;
448
449 if (!desc) {
450 pchan->desc = NULL;
451 pchan->done = NULL;
452 return -EAGAIN;
453 }
454
455 list_del(&desc->node);
456
457 pchan->desc = to_sun6i_desc(&desc->tx);
458 pchan->done = NULL;
459
460 sun6i_dma_dump_lli(vchan, pchan->desc->v_lli);
461
462 irq_reg = pchan->idx / DMA_IRQ_CHAN_NR;
463 irq_offset = pchan->idx % DMA_IRQ_CHAN_NR;
464
465 vchan->irq_type = vchan->cyclic ? DMA_IRQ_PKG : DMA_IRQ_QUEUE;
466
467 irq_val = readl(sdev->base + DMA_IRQ_EN(irq_reg));
468 irq_val &= ~((DMA_IRQ_HALF | DMA_IRQ_PKG | DMA_IRQ_QUEUE) <<
469 (irq_offset * DMA_IRQ_CHAN_WIDTH));
470 irq_val |= vchan->irq_type << (irq_offset * DMA_IRQ_CHAN_WIDTH);
471 writel(irq_val, sdev->base + DMA_IRQ_EN(irq_reg));
472
473 writel(pchan->desc->p_lli, pchan->base + DMA_CHAN_LLI_ADDR);
474 writel(DMA_CHAN_ENABLE_START, pchan->base + DMA_CHAN_ENABLE);
475
476 sun6i_dma_dump_com_regs(sdev);
477 sun6i_dma_dump_chan_regs(sdev, pchan);
478
479 return 0;
480 }
481
sun6i_dma_tasklet(unsigned long data)482 static void sun6i_dma_tasklet(unsigned long data)
483 {
484 struct sun6i_dma_dev *sdev = (struct sun6i_dma_dev *)data;
485 struct sun6i_vchan *vchan;
486 struct sun6i_pchan *pchan;
487 unsigned int pchan_alloc = 0;
488 unsigned int pchan_idx;
489
490 list_for_each_entry(vchan, &sdev->slave.channels, vc.chan.device_node) {
491 spin_lock_irq(&vchan->vc.lock);
492
493 pchan = vchan->phy;
494
495 if (pchan && pchan->done) {
496 if (sun6i_dma_start_desc(vchan)) {
497 /*
498 * No current txd associated with this channel
499 */
500 dev_dbg(sdev->slave.dev, "pchan %u: free\n",
501 pchan->idx);
502
503 /* Mark this channel free */
504 vchan->phy = NULL;
505 pchan->vchan = NULL;
506 }
507 }
508 spin_unlock_irq(&vchan->vc.lock);
509 }
510
511 spin_lock_irq(&sdev->lock);
512 for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) {
513 pchan = &sdev->pchans[pchan_idx];
514
515 if (pchan->vchan || list_empty(&sdev->pending))
516 continue;
517
518 vchan = list_first_entry(&sdev->pending,
519 struct sun6i_vchan, node);
520
521 /* Remove from pending channels */
522 list_del_init(&vchan->node);
523 pchan_alloc |= BIT(pchan_idx);
524
525 /* Mark this channel allocated */
526 pchan->vchan = vchan;
527 vchan->phy = pchan;
528 dev_dbg(sdev->slave.dev, "pchan %u: alloc vchan %p\n",
529 pchan->idx, &vchan->vc);
530 }
531 spin_unlock_irq(&sdev->lock);
532
533 for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) {
534 if (!(pchan_alloc & BIT(pchan_idx)))
535 continue;
536
537 pchan = sdev->pchans + pchan_idx;
538 vchan = pchan->vchan;
539 if (vchan) {
540 spin_lock_irq(&vchan->vc.lock);
541 sun6i_dma_start_desc(vchan);
542 spin_unlock_irq(&vchan->vc.lock);
543 }
544 }
545 }
546
sun6i_dma_interrupt(int irq,void * dev_id)547 static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id)
548 {
549 struct sun6i_dma_dev *sdev = dev_id;
550 struct sun6i_vchan *vchan;
551 struct sun6i_pchan *pchan;
552 int i, j, ret = IRQ_NONE;
553 u32 status;
554
555 for (i = 0; i < sdev->num_pchans / DMA_IRQ_CHAN_NR; i++) {
556 status = readl(sdev->base + DMA_IRQ_STAT(i));
557 if (!status)
558 continue;
559
560 dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n",
561 i ? "high" : "low", status);
562
563 writel(status, sdev->base + DMA_IRQ_STAT(i));
564
565 for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) {
566 pchan = sdev->pchans + j;
567 vchan = pchan->vchan;
568 if (!pchan->desc)
569 goto next;
570
571 if (vchan && (status & vchan->irq_type)) {
572 if (vchan->cyclic) {
573 struct virt_dma_desc *vd;
574 dma_async_tx_callback cb = NULL;
575 void *cb_data = NULL;
576
577 vd = &(pchan->desc->vd);
578 if (vd) {
579 cb = vd->tx.callback;
580 cb_data = vd->tx.callback_param;
581 }
582 if (cb)
583 cb(cb_data);
584 } else {
585 spin_lock(&vchan->vc.lock);
586 if (pchan->desc) {
587 vchan_cookie_complete(&pchan->desc->vd);
588 pchan->done = pchan->desc;
589 }
590 spin_unlock(&vchan->vc.lock);
591 }
592 }
593 next:
594 status = status >> DMA_IRQ_CHAN_WIDTH;
595 }
596
597 if (!atomic_read(&sdev->tasklet_shutdown))
598 tasklet_schedule(&sdev->task);
599 ret = IRQ_HANDLED;
600 }
601
602 return ret;
603 }
604
set_config(struct sun6i_dma_dev * sdev,struct dma_slave_config * sconfig,enum dma_transfer_direction direction,u32 * p_cfg)605 static int set_config(struct sun6i_dma_dev *sdev,
606 struct dma_slave_config *sconfig,
607 enum dma_transfer_direction direction,
608 u32 *p_cfg)
609 {
610 enum dma_slave_buswidth src_addr_width, dst_addr_width;
611 u32 src_maxburst, dst_maxburst;
612 s8 src_width, dst_width, src_burst, dst_burst;
613
614 src_addr_width = sconfig->src_addr_width;
615 dst_addr_width = sconfig->dst_addr_width;
616 src_maxburst = sconfig->src_maxburst;
617 dst_maxburst = sconfig->dst_maxburst;
618
619 switch (direction) {
620 case DMA_MEM_TO_DEV:
621 if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
622 src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
623 src_maxburst = src_maxburst ? src_maxburst : 8;
624 break;
625 case DMA_DEV_TO_MEM:
626 if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
627 dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
628 dst_maxburst = dst_maxburst ? dst_maxburst : 8;
629 break;
630 default:
631 return -EINVAL;
632 }
633
634 if (!(BIT(src_addr_width) & sdev->slave.src_addr_widths))
635 return -EINVAL;
636 if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths))
637 return -EINVAL;
638 if (!(BIT(src_maxburst) & sdev->cfg->src_burst_lengths))
639 return -EINVAL;
640 if (!(BIT(dst_maxburst) & sdev->cfg->dst_burst_lengths))
641 return -EINVAL;
642
643 src_width = convert_buswidth(src_addr_width);
644 dst_width = convert_buswidth(dst_addr_width);
645 dst_burst = convert_burst(dst_maxburst);
646 src_burst = convert_burst(src_maxburst);
647
648 *p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) |
649 DMA_CHAN_CFG_DST_WIDTH(dst_width);
650
651 sdev->cfg->set_burst_length(p_cfg, src_burst, dst_burst);
652
653 return 0;
654 }
655
sun6i_dma_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)656 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy(
657 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
658 size_t len, unsigned long flags)
659 {
660 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
661 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
662 struct sun6i_dma_lli *v_lli;
663 struct sun6i_desc *txd;
664 dma_addr_t p_lli;
665 s8 burst, width;
666
667 dev_dbg(chan2dev(chan),
668 "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n",
669 __func__, vchan->vc.chan.chan_id, &dest, &src, len, flags);
670
671 if (!len)
672 return NULL;
673
674 txd = kzalloc(sizeof(*txd), GFP_KERNEL);
675 if (!txd)
676 return NULL;
677
678
679 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
680 if (!v_lli) {
681 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
682 goto err_txd_free;
683 }
684 v_lli->this_phy = p_lli;
685
686 v_lli->src = src;
687 v_lli->dst = dest;
688 v_lli->len = len;
689 v_lli->para = SET_DST_HIGH_ADDR(dest)
690 | SET_SRC_HIGH_ADDR(src)
691 | NORMAL_WAIT;
692
693 burst = convert_burst(8);
694 width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES);
695 v_lli->cfg = DMA_CHAN_CFG_SRC_WIDTH(width) |
696 DMA_CHAN_CFG_DST_WIDTH(width);
697
698 sdev->cfg->set_burst_length(&v_lli->cfg, burst, burst);
699 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, DRQ_SDRAM);
700 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, LINEAR_MODE);
701
702 sun6i_dma_lli_add(NULL, v_lli, p_lli, txd);
703
704 sun6i_dma_dump_lli(vchan, v_lli);
705
706 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
707
708 err_txd_free:
709 kfree(txd);
710 return NULL;
711 }
712
to_sun6i_dma_chan(struct dma_chan * c)713 static inline struct sun6i_vchan *to_sun6i_dma_chan(struct dma_chan *c)
714 {
715 return container_of(c, struct sun6i_vchan, vc.chan);
716 }
717
sun6i_dma_synchronize(struct dma_chan * chan)718 static void sun6i_dma_synchronize(struct dma_chan *chan)
719 {
720 struct sun6i_vchan *c = to_sun6i_dma_chan(chan);
721
722 vchan_synchronize(&c->vc);
723 }
724
sun6i_dma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction dir,unsigned long flags,void * context)725 static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
726 struct dma_chan *chan, struct scatterlist *sgl,
727 unsigned int sg_len, enum dma_transfer_direction dir,
728 unsigned long flags, void *context)
729 {
730 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
731 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
732 struct dma_slave_config *sconfig = &vchan->cfg;
733 struct sun6i_dma_lli *v_lli, *prev = NULL;
734 struct sun6i_desc *txd;
735 struct scatterlist *sg;
736 dma_addr_t p_lli;
737 u32 lli_cfg;
738 int i, ret;
739
740 if (!sgl)
741 return NULL;
742
743 ret = set_config(sdev, sconfig, dir, &lli_cfg);
744 if (ret) {
745 dev_err(chan2dev(chan), "Invalid DMA configuration\n");
746 return NULL;
747 }
748
749 txd = kzalloc(sizeof(*txd), GFP_KERNEL);
750 if (!txd)
751 return NULL;
752
753 for_each_sg(sgl, sg, sg_len, i) {
754 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
755 if (!v_lli)
756 goto err_lli_free;
757
758 v_lli->this_phy = p_lli;
759 p_lli = (u32)SET_DESC_HIGH_ADDR(p_lli);
760 v_lli->len = sg_dma_len(sg);
761
762 if (dir == DMA_MEM_TO_DEV) {
763 v_lli->src = sg_dma_address(sg);
764 v_lli->dst = sconfig->dst_addr;
765 v_lli->cfg = lli_cfg;
766 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
767 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
768
769 dev_dbg(chan2dev(chan),
770 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
771 __func__, vchan->vc.chan.chan_id,
772 &sconfig->dst_addr, &sg_dma_address(sg),
773 sg_dma_len(sg), flags);
774
775 } else {
776 v_lli->src = sconfig->src_addr;
777 v_lli->dst = sg_dma_address(sg);
778 v_lli->cfg = lli_cfg;
779 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
780 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
781
782 dev_dbg(chan2dev(chan),
783 "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
784 __func__, vchan->vc.chan.chan_id,
785 &sg_dma_address(sg), &sconfig->src_addr,
786 sg_dma_len(sg), flags);
787 }
788
789 v_lli->para = SET_DST_HIGH_ADDR(v_lli->dst)
790 | SET_SRC_HIGH_ADDR(v_lli->src)
791 | NORMAL_WAIT;
792
793 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
794 }
795
796 dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli);
797 for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
798 sun6i_dma_dump_lli(vchan, prev);
799
800 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
801
802 err_lli_free:
803 for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
804 dma_pool_free(sdev->pool, prev, prev->this_phy);
805 kfree(txd);
806 return NULL;
807 }
808
sun6i_dma_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t buf_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction dir,unsigned long flags)809 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
810 struct dma_chan *chan,
811 dma_addr_t buf_addr,
812 size_t buf_len,
813 size_t period_len,
814 enum dma_transfer_direction dir,
815 unsigned long flags)
816 {
817 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
818 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
819 struct dma_slave_config *sconfig = &vchan->cfg;
820 struct sun6i_dma_lli *v_lli, *prev = NULL;
821 struct sun6i_desc *txd;
822 dma_addr_t p_lli;
823 u32 lli_cfg;
824 unsigned int i, periods = buf_len / period_len;
825 int ret;
826
827 ret = set_config(sdev, sconfig, dir, &lli_cfg);
828 if (ret) {
829 dev_err(chan2dev(chan), "Invalid DMA configuration\n");
830 return NULL;
831 }
832
833 txd = kzalloc(sizeof(*txd), GFP_KERNEL);
834 if (!txd)
835 return NULL;
836
837 for (i = 0; i < periods; i++) {
838 v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
839 if (!v_lli) {
840 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
841 goto err_lli_free;
842 }
843 v_lli->this_phy = p_lli;
844 v_lli->len = period_len;
845
846 if (dir == DMA_MEM_TO_DEV) {
847 v_lli->src = buf_addr + period_len * i;
848 v_lli->dst = sconfig->dst_addr;
849 v_lli->cfg = lli_cfg;
850 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
851 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
852 } else {
853 v_lli->src = sconfig->src_addr;
854 v_lli->dst = buf_addr + period_len * i;
855 v_lli->cfg = lli_cfg;
856 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
857 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
858 }
859
860 v_lli->para = SET_DST_HIGH_ADDR(v_lli->dst)
861 | SET_SRC_HIGH_ADDR(v_lli->src)
862 | NORMAL_WAIT;
863
864 prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
865 }
866
867 prev->p_lli_next = txd->p_lli; /* cyclic list */
868
869 vchan->cyclic = true;
870
871 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
872
873 err_lli_free:
874 for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
875 dma_pool_free(sdev->pool, prev, prev->this_phy);
876 kfree(txd);
877 return NULL;
878 }
879
sun6i_dma_config(struct dma_chan * chan,struct dma_slave_config * config)880 static int sun6i_dma_config(struct dma_chan *chan,
881 struct dma_slave_config *config)
882 {
883 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
884
885 memcpy(&vchan->cfg, config, sizeof(*config));
886
887 return 0;
888 }
889
sun6i_dma_pause(struct dma_chan * chan)890 static int sun6i_dma_pause(struct dma_chan *chan)
891 {
892 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
893 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
894 struct sun6i_pchan *pchan = vchan->phy;
895
896 dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc);
897
898 if (pchan) {
899 writel(DMA_CHAN_PAUSE_PAUSE,
900 pchan->base + DMA_CHAN_PAUSE);
901 } else {
902 spin_lock(&sdev->lock);
903 list_del_init(&vchan->node);
904 spin_unlock(&sdev->lock);
905 }
906
907 return 0;
908 }
909
sun6i_dma_resume(struct dma_chan * chan)910 static int sun6i_dma_resume(struct dma_chan *chan)
911 {
912 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
913 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
914 struct sun6i_pchan *pchan = vchan->phy;
915 unsigned long flags;
916
917 dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
918
919 spin_lock_irqsave(&vchan->vc.lock, flags);
920
921 if (pchan) {
922 writel(DMA_CHAN_PAUSE_RESUME,
923 pchan->base + DMA_CHAN_PAUSE);
924 } else if (!list_empty(&vchan->vc.desc_issued)) {
925 spin_lock(&sdev->lock);
926 list_add_tail(&vchan->node, &sdev->pending);
927 spin_unlock(&sdev->lock);
928 }
929
930 spin_unlock_irqrestore(&vchan->vc.lock, flags);
931
932 return 0;
933 }
934
sun6i_dma_terminate_all(struct dma_chan * chan)935 static int sun6i_dma_terminate_all(struct dma_chan *chan)
936 {
937 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
938 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
939 struct sun6i_pchan *pchan = vchan->phy;
940 unsigned long flags;
941 LIST_HEAD(head);
942
943 spin_lock(&sdev->lock);
944 list_del_init(&vchan->node);
945 spin_unlock(&sdev->lock);
946
947 spin_lock_irqsave(&vchan->vc.lock, flags);
948
949 if (vchan->cyclic) {
950 vchan->cyclic = false;
951 if (pchan && pchan->desc) {
952 struct virt_dma_desc *vd = &pchan->desc->vd;
953 struct virt_dma_chan *vc = &vchan->vc;
954
955 list_add_tail(&vd->node, &vc->desc_completed);
956 }
957 }
958
959 vchan_get_all_descriptors(&vchan->vc, &head);
960
961 if (pchan) {
962 writel(DMA_CHAN_PAUSE_PAUSE, pchan->base + DMA_CHAN_PAUSE);
963 writel(DMA_CHAN_ENABLE_STOP, pchan->base + DMA_CHAN_ENABLE);
964 writel(DMA_CHAN_PAUSE_RESUME, pchan->base + DMA_CHAN_PAUSE);
965
966 vchan->phy = NULL;
967 pchan->vchan = NULL;
968 pchan->desc = NULL;
969 pchan->done = NULL;
970 }
971
972 spin_unlock_irqrestore(&vchan->vc.lock, flags);
973
974 vchan_dma_desc_free_list(&vchan->vc, &head);
975
976 return 0;
977 }
978
sun6i_dma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * state)979 static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan,
980 dma_cookie_t cookie,
981 struct dma_tx_state *state)
982 {
983 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
984 struct sun6i_pchan *pchan = vchan->phy;
985 struct sun6i_dma_lli *lli;
986 struct virt_dma_desc *vd;
987 struct sun6i_desc *txd;
988 enum dma_status ret;
989 unsigned long flags;
990 size_t bytes = 0;
991
992 ret = dma_cookie_status(chan, cookie, state);
993 if (ret == DMA_COMPLETE || !state)
994 return ret;
995
996 spin_lock_irqsave(&vchan->vc.lock, flags);
997
998 vd = vchan_find_desc(&vchan->vc, cookie);
999 txd = to_sun6i_desc(&vd->tx);
1000
1001 if (vd) {
1002 for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next)
1003 bytes += lli->len;
1004 } else if (!pchan || !pchan->desc) {
1005 bytes = 0;
1006 } else {
1007 bytes = sun6i_get_chan_size(pchan);
1008 }
1009
1010 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1011
1012 dma_set_residue(state, bytes);
1013
1014 return ret;
1015 }
1016
sun6i_dma_issue_pending(struct dma_chan * chan)1017 static void sun6i_dma_issue_pending(struct dma_chan *chan)
1018 {
1019 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
1020 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
1021 unsigned long flags;
1022
1023 spin_lock_irqsave(&vchan->vc.lock, flags);
1024
1025 if (vchan_issue_pending(&vchan->vc)) {
1026 spin_lock(&sdev->lock);
1027
1028 if (!vchan->phy && list_empty(&vchan->node)) {
1029 list_add_tail(&vchan->node, &sdev->pending);
1030 tasklet_schedule(&sdev->task);
1031 dev_dbg(chan2dev(chan), "vchan %p: issued\n",
1032 &vchan->vc);
1033 }
1034
1035 spin_unlock(&sdev->lock);
1036 } else {
1037 dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n",
1038 &vchan->vc);
1039 }
1040
1041 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1042 }
1043
sun6i_dma_free_chan_resources(struct dma_chan * chan)1044 static void sun6i_dma_free_chan_resources(struct dma_chan *chan)
1045 {
1046 struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
1047 struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
1048 unsigned long flags;
1049
1050 spin_lock_irqsave(&sdev->lock, flags);
1051 list_del_init(&vchan->node);
1052 spin_unlock_irqrestore(&sdev->lock, flags);
1053
1054 vchan_free_chan_resources(&vchan->vc);
1055 }
1056
sun6i_dma_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)1057 static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec,
1058 struct of_dma *ofdma)
1059 {
1060 struct sun6i_dma_dev *sdev = ofdma->of_dma_data;
1061 struct sun6i_vchan *vchan;
1062 struct dma_chan *chan;
1063 u8 port = dma_spec->args[0];
1064
1065 if (port > sdev->max_request)
1066 return NULL;
1067
1068 chan = dma_get_any_slave_channel(&sdev->slave);
1069 if (!chan)
1070 return NULL;
1071
1072 vchan = to_sun6i_vchan(chan);
1073 vchan->port = port;
1074
1075 return chan;
1076 }
1077
sun6i_kill_tasklet(struct sun6i_dma_dev * sdev)1078 static inline void sun6i_kill_tasklet(struct sun6i_dma_dev *sdev)
1079 {
1080 /* Disable all interrupts from DMA */
1081 writel(0, sdev->base + DMA_IRQ_EN(0));
1082 writel(0, sdev->base + DMA_IRQ_EN(1));
1083
1084 /* Prevent spurious interrupts from scheduling the tasklet */
1085 atomic_inc(&sdev->tasklet_shutdown);
1086
1087 /* Make sure we won't have any further interrupts */
1088 devm_free_irq(sdev->slave.dev, sdev->irq, sdev);
1089
1090 /* Actually prevent the tasklet from being scheduled */
1091 tasklet_kill(&sdev->task);
1092 }
1093
sun6i_dma_free(struct sun6i_dma_dev * sdev)1094 static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev)
1095 {
1096 int i;
1097
1098 for (i = 0; i < sdev->num_vchans; i++) {
1099 struct sun6i_vchan *vchan = &sdev->vchans[i];
1100
1101 list_del(&vchan->vc.chan.device_node);
1102 tasklet_kill(&vchan->vc.task);
1103 }
1104 }
1105
1106 /*
1107 * For A31:
1108 *
1109 * There's 16 physical channels that can work in parallel.
1110 *
1111 * However we have 30 different endpoints for our requests.
1112 *
1113 * Since the channels are able to handle only an unidirectional
1114 * transfer, we need to allocate more virtual channels so that
1115 * everyone can grab one channel.
1116 *
1117 * Some devices can't work in both direction (mostly because it
1118 * wouldn't make sense), so we have a bit fewer virtual channels than
1119 * 2 channels per endpoints.
1120 */
1121
1122 static struct sun6i_dma_config sun6i_a31_dma_cfg = {
1123 .nr_max_channels = 16,
1124 .nr_max_requests = 30,
1125 .nr_max_vchans = 53,
1126 .set_burst_length = sun6i_set_burst_length_a31,
1127 .set_drq = sun6i_set_drq_a31,
1128 .set_mode = sun6i_set_mode_a31,
1129 .src_burst_lengths = BIT(1) | BIT(8),
1130 .dst_burst_lengths = BIT(1) | BIT(8),
1131 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1132 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1133 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1134 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1135 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1136 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1137 };
1138
1139 /*
1140 * The A23 only has 8 physical channels, a maximum DRQ port id of 24,
1141 * and a total of 37 usable source and destination endpoints.
1142 */
1143
1144 static struct sun6i_dma_config sun8i_a23_dma_cfg = {
1145 .nr_max_channels = 8,
1146 .nr_max_requests = 24,
1147 .nr_max_vchans = 37,
1148 .clock_autogate_enable = sun6i_enable_clock_autogate_a23,
1149 .set_burst_length = sun6i_set_burst_length_a31,
1150 .set_drq = sun6i_set_drq_a31,
1151 .set_mode = sun6i_set_mode_a31,
1152 .src_burst_lengths = BIT(1) | BIT(8),
1153 .dst_burst_lengths = BIT(1) | BIT(8),
1154 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1155 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1156 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1157 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1158 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1159 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1160 };
1161
1162 static struct sun6i_dma_config sun8i_a83t_dma_cfg = {
1163 .nr_max_channels = 8,
1164 .nr_max_requests = 28,
1165 .nr_max_vchans = 39,
1166 .clock_autogate_enable = sun6i_enable_clock_autogate_a23,
1167 .set_burst_length = sun6i_set_burst_length_a31,
1168 .set_drq = sun6i_set_drq_a31,
1169 .set_mode = sun6i_set_mode_a31,
1170 .src_burst_lengths = BIT(1) | BIT(8),
1171 .dst_burst_lengths = BIT(1) | BIT(8),
1172 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1173 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1174 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1175 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1176 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1177 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1178 };
1179
1180 /*
1181 * The H3 has 12 physical channels, a maximum DRQ port id of 27,
1182 * and a total of 34 usable source and destination endpoints.
1183 * It also supports additional burst lengths and bus widths,
1184 * and the burst length fields have different offsets.
1185 */
1186
1187 static struct sun6i_dma_config sun8i_h3_dma_cfg = {
1188 .nr_max_channels = 12,
1189 .nr_max_requests = 27,
1190 .nr_max_vchans = 34,
1191 .clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1192 .set_burst_length = sun6i_set_burst_length_h3,
1193 .set_drq = sun6i_set_drq_a31,
1194 .set_mode = sun6i_set_mode_a31,
1195 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1196 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1197 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1198 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1199 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1200 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1201 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1202 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1203 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1204 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1205 };
1206
1207 /*
1208 * The A64 binding uses the number of dma channels from the
1209 * device tree node.
1210 */
1211 static struct sun6i_dma_config sun50i_a64_dma_cfg = {
1212 .clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1213 .set_burst_length = sun6i_set_burst_length_h3,
1214 .set_drq = sun6i_set_drq_a31,
1215 .set_mode = sun6i_set_mode_a31,
1216 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1217 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1218 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1219 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1220 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1221 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1222 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1223 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1224 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1225 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1226 };
1227
1228 /*
1229 * The H6 binding uses the number of dma channels from the
1230 * device tree node.
1231 */
1232 static struct sun6i_dma_config sun50i_h6_dma_cfg = {
1233 .clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1234 .set_burst_length = sun6i_set_burst_length_h3,
1235 .set_drq = sun6i_set_drq_h6,
1236 .set_mode = sun6i_set_mode_h6,
1237 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1238 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1239 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1240 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1241 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1242 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1243 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1244 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1245 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1246 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1247 .has_mbus_clk = true,
1248 };
1249
1250 /*
1251 * The sun50iw9 binding uses the number of dma channels from the
1252 * device tree node.
1253 */
1254 static struct sun6i_dma_config sun50iw9_dma_cfg = {
1255 .clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1256 .set_burst_length = sun6i_set_burst_length_h3,
1257 .set_drq = sun6i_set_drq_h6,
1258 .set_mode = sun6i_set_mode_h6,
1259 .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1260 .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1261 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1262 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1263 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1264 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1265 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1266 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1267 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1268 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1269 .has_mbus_clk = true,
1270 };
1271
1272 /*
1273 * The V3s have only 8 physical channels, a maximum DRQ port id of 23,
1274 * and a total of 24 usable source and destination endpoints.
1275 */
1276 static struct sun6i_dma_config sun8i_v3s_dma_cfg = {
1277 .nr_max_channels = 8,
1278 .nr_max_requests = 23,
1279 .nr_max_vchans = 24,
1280 .clock_autogate_enable = sun6i_enable_clock_autogate_a23,
1281 .set_burst_length = sun6i_set_burst_length_a31,
1282 .set_drq = sun6i_set_drq_a31,
1283 .set_mode = sun6i_set_mode_a31,
1284 .src_burst_lengths = BIT(1) | BIT(8),
1285 .dst_burst_lengths = BIT(1) | BIT(8),
1286 .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1287 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1288 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1289 .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1290 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1291 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1292 };
1293
1294 static const struct of_device_id sun6i_dma_match[] = {
1295 { .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg },
1296 { .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg },
1297 { .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg },
1298 { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
1299 { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
1300 { .compatible = "allwinner,sun8iw20-dma", .data = &sun50iw9_dma_cfg },
1301 { .compatible = "allwinner,sun8i-riscv-dma", .data = &sun50iw9_dma_cfg },
1302 { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg },
1303 { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg },
1304 { .compatible = "allwinner,sun50iw9-dma", .data = &sun50iw9_dma_cfg },
1305 { .compatible = "allwinner,sun50iw10-dma", .data = &sun50iw9_dma_cfg },
1306 { .compatible = "allwinner,sun50iw12-dma", .data = &sun50iw9_dma_cfg },
1307 { /* sentinel */ }
1308 };
1309 MODULE_DEVICE_TABLE(of, sun6i_dma_match);
1310
sun6i_dma_probe(struct platform_device * pdev)1311 static int sun6i_dma_probe(struct platform_device *pdev)
1312 {
1313 struct device_node *np = pdev->dev.of_node;
1314 struct sun6i_dma_dev *sdc;
1315 struct resource *res;
1316 int ret, i;
1317
1318 sdc = devm_kzalloc(&pdev->dev, sizeof(*sdc), GFP_KERNEL);
1319 if (!sdc)
1320 return -ENOMEM;
1321
1322 sdc->cfg = of_device_get_match_data(&pdev->dev);
1323 if (!sdc->cfg)
1324 return -ENODEV;
1325
1326 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1327 sdc->base = devm_ioremap_resource(&pdev->dev, res);
1328 if (IS_ERR(sdc->base))
1329 return PTR_ERR(sdc->base);
1330
1331 sdc->irq = platform_get_irq(pdev, 0);
1332 if (sdc->irq < 0)
1333 return sdc->irq;
1334
1335 sdc->clk = devm_clk_get(&pdev->dev, NULL);
1336 if (IS_ERR(sdc->clk)) {
1337 dev_err(&pdev->dev, "No clock specified\n");
1338 return PTR_ERR(sdc->clk);
1339 }
1340
1341 if (sdc->cfg->has_mbus_clk) {
1342 sdc->clk_mbus = devm_clk_get(&pdev->dev, "mbus");
1343 if (IS_ERR(sdc->clk_mbus)) {
1344 dev_err(&pdev->dev, "No mbus clock specified\n");
1345 return PTR_ERR(sdc->clk_mbus);
1346 }
1347 }
1348
1349 sdc->rstc = devm_reset_control_get(&pdev->dev, NULL);
1350 if (IS_ERR(sdc->rstc)) {
1351 dev_err(&pdev->dev, "No reset controller specified\n");
1352 return PTR_ERR(sdc->rstc);
1353 }
1354
1355 sdc->pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
1356 sizeof(struct sun6i_dma_lli), 4, 0);
1357 if (!sdc->pool) {
1358 dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
1359 return -ENOMEM;
1360 }
1361
1362 platform_set_drvdata(pdev, sdc);
1363 INIT_LIST_HEAD(&sdc->pending);
1364 spin_lock_init(&sdc->lock);
1365
1366 dma_cap_set(DMA_PRIVATE, sdc->slave.cap_mask);
1367 dma_cap_set(DMA_MEMCPY, sdc->slave.cap_mask);
1368 dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask);
1369 dma_cap_set(DMA_CYCLIC, sdc->slave.cap_mask);
1370
1371 INIT_LIST_HEAD(&sdc->slave.channels);
1372 sdc->slave.device_free_chan_resources = sun6i_dma_free_chan_resources;
1373 sdc->slave.device_tx_status = sun6i_dma_tx_status;
1374 sdc->slave.device_issue_pending = sun6i_dma_issue_pending;
1375 sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg;
1376 sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy;
1377 sdc->slave.device_prep_dma_cyclic = sun6i_dma_prep_dma_cyclic;
1378 sdc->slave.copy_align = DMAENGINE_ALIGN_4_BYTES;
1379 sdc->slave.device_config = sun6i_dma_config;
1380 sdc->slave.device_pause = sun6i_dma_pause;
1381 sdc->slave.device_resume = sun6i_dma_resume;
1382 sdc->slave.device_terminate_all = sun6i_dma_terminate_all;
1383 sdc->slave.device_synchronize = sun6i_dma_synchronize;
1384 sdc->slave.src_addr_widths = sdc->cfg->src_addr_widths;
1385 sdc->slave.dst_addr_widths = sdc->cfg->dst_addr_widths;
1386 sdc->slave.directions = BIT(DMA_DEV_TO_MEM) |
1387 BIT(DMA_MEM_TO_DEV);
1388 sdc->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1389 sdc->slave.dev = &pdev->dev;
1390
1391 sdc->num_pchans = sdc->cfg->nr_max_channels;
1392 sdc->num_vchans = sdc->cfg->nr_max_vchans;
1393 sdc->max_request = sdc->cfg->nr_max_requests;
1394
1395 ret = of_property_read_u32(np, "dma-channels", &sdc->num_pchans);
1396 if (ret && !sdc->num_pchans) {
1397 dev_err(&pdev->dev, "Can't get dma-channels.\n");
1398 return ret;
1399 }
1400
1401 ret = of_property_read_u32(np, "dma-requests", &sdc->max_request);
1402 if (ret && !sdc->max_request) {
1403 dev_info(&pdev->dev, "Missing dma-requests, using %u.\n",
1404 DMA_CHAN_MAX_DRQ_A31);
1405 sdc->max_request = DMA_CHAN_MAX_DRQ_A31;
1406 }
1407
1408 /*
1409 * If the number of vchans is not specified, derive it from the
1410 * highest port number, at most one channel per port and direction.
1411 */
1412 if (!sdc->num_vchans)
1413 sdc->num_vchans = 2 * (sdc->max_request + 1);
1414
1415 sdc->pchans = devm_kcalloc(&pdev->dev, sdc->num_pchans,
1416 sizeof(struct sun6i_pchan), GFP_KERNEL);
1417 if (!sdc->pchans)
1418 return -ENOMEM;
1419
1420 sdc->vchans = devm_kcalloc(&pdev->dev, sdc->num_vchans,
1421 sizeof(struct sun6i_vchan), GFP_KERNEL);
1422 if (!sdc->vchans)
1423 return -ENOMEM;
1424
1425 tasklet_init(&sdc->task, sun6i_dma_tasklet, (unsigned long)sdc);
1426
1427 for (i = 0; i < sdc->num_pchans; i++) {
1428 struct sun6i_pchan *pchan = &sdc->pchans[i];
1429
1430 pchan->idx = i;
1431 pchan->base = sdc->base + 0x100 + i * 0x40;
1432 }
1433
1434 for (i = 0; i < sdc->num_vchans; i++) {
1435 struct sun6i_vchan *vchan = &sdc->vchans[i];
1436
1437 INIT_LIST_HEAD(&vchan->node);
1438 vchan->vc.desc_free = sun6i_dma_free_desc;
1439 vchan_init(&vchan->vc, &sdc->slave);
1440 }
1441
1442 ret = reset_control_assert(sdc->rstc);
1443 if (ret) {
1444 dev_err(&pdev->dev, "Couldn't assert the device from reset\n");
1445 goto err_chan_free;
1446 }
1447 usleep_range(20, 25); /* ensure dma controller is reset */
1448
1449 ret = reset_control_deassert(sdc->rstc);
1450 if (ret) {
1451 dev_err(&pdev->dev, "Couldn't deassert the device from reset\n");
1452 goto err_chan_free;
1453 }
1454
1455 ret = clk_prepare_enable(sdc->clk);
1456 if (ret) {
1457 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1458 goto err_reset_assert;
1459 }
1460
1461 if (sdc->cfg->has_mbus_clk) {
1462 ret = clk_prepare_enable(sdc->clk_mbus);
1463 if (ret) {
1464 dev_err(&pdev->dev, "Couldn't enable mbus clock\n");
1465 goto err_clk_disable;
1466 }
1467 }
1468
1469 ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0,
1470 dev_name(&pdev->dev), sdc);
1471 if (ret) {
1472 dev_err(&pdev->dev, "Cannot request IRQ\n");
1473 goto err_mbus_clk_disable;
1474 }
1475
1476 ret = dma_async_device_register(&sdc->slave);
1477 if (ret) {
1478 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1479 goto err_irq_disable;
1480 }
1481
1482 ret = of_dma_controller_register(pdev->dev.of_node, sun6i_dma_of_xlate,
1483 sdc);
1484 if (ret) {
1485 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1486 goto err_dma_unregister;
1487 }
1488
1489 if (sdc->cfg->clock_autogate_enable)
1490 sdc->cfg->clock_autogate_enable(sdc);
1491
1492 dev_info(&pdev->dev, "sunxi dma probed\n");
1493
1494 return 0;
1495
1496 err_dma_unregister:
1497 dma_async_device_unregister(&sdc->slave);
1498 err_irq_disable:
1499 sun6i_kill_tasklet(sdc);
1500 err_mbus_clk_disable:
1501 clk_disable_unprepare(sdc->clk_mbus);
1502 err_clk_disable:
1503 clk_disable_unprepare(sdc->clk);
1504 err_reset_assert:
1505 reset_control_assert(sdc->rstc);
1506 err_chan_free:
1507 sun6i_dma_free(sdc);
1508 return ret;
1509 }
1510
sun6i_dma_remove(struct platform_device * pdev)1511 static int sun6i_dma_remove(struct platform_device *pdev)
1512 {
1513 struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev);
1514
1515 of_dma_controller_free(pdev->dev.of_node);
1516 dma_async_device_unregister(&sdc->slave);
1517
1518 sun6i_kill_tasklet(sdc);
1519
1520 clk_disable_unprepare(sdc->clk_mbus);
1521 clk_disable_unprepare(sdc->clk);
1522 reset_control_assert(sdc->rstc);
1523
1524 sun6i_dma_free(sdc);
1525
1526 return 0;
1527 }
1528
1529 static struct platform_driver sun6i_dma_driver = {
1530 .probe = sun6i_dma_probe,
1531 .remove = sun6i_dma_remove,
1532 .driver = {
1533 .name = "sun6i-dma",
1534 .of_match_table = sun6i_dma_match,
1535 },
1536 };
1537
sun6i_dma_init(void)1538 static int __init sun6i_dma_init(void)
1539 {
1540 return platform_driver_register(&sun6i_dma_driver);
1541 }
1542 subsys_initcall(sun6i_dma_init);
1543
sun6i_dma_exit(void)1544 static void __exit sun6i_dma_exit(void)
1545 {
1546 platform_driver_unregister(&sun6i_dma_driver);
1547 }
1548 module_exit(sun6i_dma_exit);
1549
1550 MODULE_DESCRIPTION("Allwinner A31 DMA Controller Driver");
1551 MODULE_AUTHOR("Sugar <shuge@allwinnertech.com>");
1552 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
1553 MODULE_LICENSE("GPL");
1554 MODULE_VERSION("1.0.0");
1555