1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 *
4 * Copyright (C) STMicroelectronics SA 2017
5 * Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com>
6 * Pierre-Yves Mordret <pierre-yves.mordret@st.com>
7 *
8 * Driver for STM32 MDMA controller
9 *
10 * Inspired by stm32-dma.c and dma-jz4780.c
11 */
12
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/dmaengine.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmapool.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/iopoll.h>
21 #include <linux/jiffies.h>
22 #include <linux/list.h>
23 #include <linux/log2.h>
24 #include <linux/module.h>
25 #include <linux/of.h>
26 #include <linux/of_device.h>
27 #include <linux/of_dma.h>
28 #include <linux/platform_device.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/reset.h>
31 #include <linux/slab.h>
32
33 #include "virt-dma.h"
34
35 /* MDMA Generic getter/setter */
36 #define STM32_MDMA_SHIFT(n) (ffs(n) - 1)
37 #define STM32_MDMA_SET(n, mask) (((n) << STM32_MDMA_SHIFT(mask)) & \
38 (mask))
39 #define STM32_MDMA_GET(n, mask) (((n) & (mask)) >> \
40 STM32_MDMA_SHIFT(mask))
41
42 #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */
43
44 /* MDMA Channel x interrupt/status register */
45 #define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */
46 #define STM32_MDMA_CISR_CRQA BIT(16)
47 #define STM32_MDMA_CISR_TCIF BIT(4)
48 #define STM32_MDMA_CISR_BTIF BIT(3)
49 #define STM32_MDMA_CISR_BRTIF BIT(2)
50 #define STM32_MDMA_CISR_CTCIF BIT(1)
51 #define STM32_MDMA_CISR_TEIF BIT(0)
52
53 /* MDMA Channel x interrupt flag clear register */
54 #define STM32_MDMA_CIFCR(x) (0x44 + 0x40 * (x))
55 #define STM32_MDMA_CIFCR_CLTCIF BIT(4)
56 #define STM32_MDMA_CIFCR_CBTIF BIT(3)
57 #define STM32_MDMA_CIFCR_CBRTIF BIT(2)
58 #define STM32_MDMA_CIFCR_CCTCIF BIT(1)
59 #define STM32_MDMA_CIFCR_CTEIF BIT(0)
60 #define STM32_MDMA_CIFCR_CLEAR_ALL (STM32_MDMA_CIFCR_CLTCIF \
61 | STM32_MDMA_CIFCR_CBTIF \
62 | STM32_MDMA_CIFCR_CBRTIF \
63 | STM32_MDMA_CIFCR_CCTCIF \
64 | STM32_MDMA_CIFCR_CTEIF)
65
66 /* MDMA Channel x error status register */
67 #define STM32_MDMA_CESR(x) (0x48 + 0x40 * (x))
68 #define STM32_MDMA_CESR_BSE BIT(11)
69 #define STM32_MDMA_CESR_ASR BIT(10)
70 #define STM32_MDMA_CESR_TEMD BIT(9)
71 #define STM32_MDMA_CESR_TELD BIT(8)
72 #define STM32_MDMA_CESR_TED BIT(7)
73 #define STM32_MDMA_CESR_TEA_MASK GENMASK(6, 0)
74
75 /* MDMA Channel x control register */
76 #define STM32_MDMA_CCR(x) (0x4C + 0x40 * (x))
77 #define STM32_MDMA_CCR_SWRQ BIT(16)
78 #define STM32_MDMA_CCR_WEX BIT(14)
79 #define STM32_MDMA_CCR_HEX BIT(13)
80 #define STM32_MDMA_CCR_BEX BIT(12)
81 #define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6)
82 #define STM32_MDMA_CCR_PL(n) STM32_MDMA_SET(n, \
83 STM32_MDMA_CCR_PL_MASK)
84 #define STM32_MDMA_CCR_TCIE BIT(5)
85 #define STM32_MDMA_CCR_BTIE BIT(4)
86 #define STM32_MDMA_CCR_BRTIE BIT(3)
87 #define STM32_MDMA_CCR_CTCIE BIT(2)
88 #define STM32_MDMA_CCR_TEIE BIT(1)
89 #define STM32_MDMA_CCR_EN BIT(0)
90 #define STM32_MDMA_CCR_IRQ_MASK (STM32_MDMA_CCR_TCIE \
91 | STM32_MDMA_CCR_BTIE \
92 | STM32_MDMA_CCR_BRTIE \
93 | STM32_MDMA_CCR_CTCIE \
94 | STM32_MDMA_CCR_TEIE)
95
96 /* MDMA Channel x transfer configuration register */
97 #define STM32_MDMA_CTCR(x) (0x50 + 0x40 * (x))
98 #define STM32_MDMA_CTCR_BWM BIT(31)
99 #define STM32_MDMA_CTCR_SWRM BIT(30)
100 #define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28)
101 #define STM32_MDMA_CTCR_TRGM(n) STM32_MDMA_SET((n), \
102 STM32_MDMA_CTCR_TRGM_MSK)
103 #define STM32_MDMA_CTCR_TRGM_GET(n) STM32_MDMA_GET((n), \
104 STM32_MDMA_CTCR_TRGM_MSK)
105 #define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26)
106 #define STM32_MDMA_CTCR_PAM(n) STM32_MDMA_SET(n, \
107 STM32_MDMA_CTCR_PAM_MASK)
108 #define STM32_MDMA_CTCR_PKE BIT(25)
109 #define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18)
110 #define STM32_MDMA_CTCR_TLEN(n) STM32_MDMA_SET((n), \
111 STM32_MDMA_CTCR_TLEN_MSK)
112 #define STM32_MDMA_CTCR_TLEN_GET(n) STM32_MDMA_GET((n), \
113 STM32_MDMA_CTCR_TLEN_MSK)
114 #define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18)
115 #define STM32_MDMA_CTCR_LEN2(n) STM32_MDMA_SET((n), \
116 STM32_MDMA_CTCR_LEN2_MSK)
117 #define STM32_MDMA_CTCR_LEN2_GET(n) STM32_MDMA_GET((n), \
118 STM32_MDMA_CTCR_LEN2_MSK)
119 #define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15)
120 #define STM32_MDMA_CTCR_DBURST(n) STM32_MDMA_SET(n, \
121 STM32_MDMA_CTCR_DBURST_MASK)
122 #define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12)
123 #define STM32_MDMA_CTCR_SBURST(n) STM32_MDMA_SET(n, \
124 STM32_MDMA_CTCR_SBURST_MASK)
125 #define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10)
126 #define STM32_MDMA_CTCR_DINCOS(n) STM32_MDMA_SET((n), \
127 STM32_MDMA_CTCR_DINCOS_MASK)
128 #define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8)
129 #define STM32_MDMA_CTCR_SINCOS(n) STM32_MDMA_SET((n), \
130 STM32_MDMA_CTCR_SINCOS_MASK)
131 #define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6)
132 #define STM32_MDMA_CTCR_DSIZE(n) STM32_MDMA_SET(n, \
133 STM32_MDMA_CTCR_DSIZE_MASK)
134 #define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4)
135 #define STM32_MDMA_CTCR_SSIZE(n) STM32_MDMA_SET(n, \
136 STM32_MDMA_CTCR_SSIZE_MASK)
137 #define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2)
138 #define STM32_MDMA_CTCR_DINC(n) STM32_MDMA_SET((n), \
139 STM32_MDMA_CTCR_DINC_MASK)
140 #define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0)
141 #define STM32_MDMA_CTCR_SINC(n) STM32_MDMA_SET((n), \
142 STM32_MDMA_CTCR_SINC_MASK)
143 #define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \
144 | STM32_MDMA_CTCR_DINC_MASK \
145 | STM32_MDMA_CTCR_SINCOS_MASK \
146 | STM32_MDMA_CTCR_DINCOS_MASK \
147 | STM32_MDMA_CTCR_LEN2_MSK \
148 | STM32_MDMA_CTCR_TRGM_MSK)
149
150 /* MDMA Channel x block number of data register */
151 #define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x))
152 #define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20)
153 #define STM32_MDMA_CBNDTR_BRC(n) STM32_MDMA_SET(n, \
154 STM32_MDMA_CBNDTR_BRC_MK)
155 #define STM32_MDMA_CBNDTR_BRC_GET(n) STM32_MDMA_GET((n), \
156 STM32_MDMA_CBNDTR_BRC_MK)
157
158 #define STM32_MDMA_CBNDTR_BRDUM BIT(19)
159 #define STM32_MDMA_CBNDTR_BRSUM BIT(18)
160 #define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0)
161 #define STM32_MDMA_CBNDTR_BNDT(n) STM32_MDMA_SET(n, \
162 STM32_MDMA_CBNDTR_BNDT_MASK)
163
164 /* MDMA Channel x source address register */
165 #define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x))
166
167 /* MDMA Channel x destination address register */
168 #define STM32_MDMA_CDAR(x) (0x5C + 0x40 * (x))
169
170 /* MDMA Channel x block repeat address update register */
171 #define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x))
172 #define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16)
173 #define STM32_MDMA_CBRUR_DUV(n) STM32_MDMA_SET(n, \
174 STM32_MDMA_CBRUR_DUV_MASK)
175 #define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0)
176 #define STM32_MDMA_CBRUR_SUV(n) STM32_MDMA_SET(n, \
177 STM32_MDMA_CBRUR_SUV_MASK)
178
179 /* MDMA Channel x link address register */
180 #define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x))
181
182 /* MDMA Channel x trigger and bus selection register */
183 #define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x))
184 #define STM32_MDMA_CTBR_DBUS BIT(17)
185 #define STM32_MDMA_CTBR_SBUS BIT(16)
186 #define STM32_MDMA_CTBR_TSEL_MASK GENMASK(5, 0)
187 #define STM32_MDMA_CTBR_TSEL(n) STM32_MDMA_SET(n, \
188 STM32_MDMA_CTBR_TSEL_MASK)
189
190 /* MDMA Channel x mask address register */
191 #define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x))
192
193 /* MDMA Channel x mask data register */
194 #define STM32_MDMA_CMDR(x) (0x74 + 0x40 * (x))
195
196 #define STM32_MDMA_MAX_BUF_LEN 128
197 #define STM32_MDMA_MAX_BLOCK_LEN 65536
198 #define STM32_MDMA_MAX_CHANNELS 32
199 #define STM32_MDMA_MAX_REQUESTS 256
200 #define STM32_MDMA_MAX_BURST 128
201 #define STM32_MDMA_VERY_HIGH_PRIORITY 0x11
202
203 enum stm32_mdma_trigger_mode {
204 STM32_MDMA_BUFFER,
205 STM32_MDMA_BLOCK,
206 STM32_MDMA_BLOCK_REP,
207 STM32_MDMA_LINKED_LIST,
208 };
209
210 enum stm32_mdma_width {
211 STM32_MDMA_BYTE,
212 STM32_MDMA_HALF_WORD,
213 STM32_MDMA_WORD,
214 STM32_MDMA_DOUBLE_WORD,
215 };
216
217 enum stm32_mdma_inc_mode {
218 STM32_MDMA_FIXED = 0,
219 STM32_MDMA_INC = 2,
220 STM32_MDMA_DEC = 3,
221 };
222
223 struct stm32_mdma_chan_config {
224 u32 request;
225 u32 priority_level;
226 u32 transfer_config;
227 u32 mask_addr;
228 u32 mask_data;
229 };
230
231 struct stm32_mdma_hwdesc {
232 u32 ctcr;
233 u32 cbndtr;
234 u32 csar;
235 u32 cdar;
236 u32 cbrur;
237 u32 clar;
238 u32 ctbr;
239 u32 dummy;
240 u32 cmar;
241 u32 cmdr;
242 } __aligned(64);
243
244 struct stm32_mdma_desc_node {
245 struct stm32_mdma_hwdesc *hwdesc;
246 dma_addr_t hwdesc_phys;
247 };
248
249 struct stm32_mdma_desc {
250 struct virt_dma_desc vdesc;
251 u32 ccr;
252 bool cyclic;
253 u32 count;
254 struct stm32_mdma_desc_node node[];
255 };
256
257 struct stm32_mdma_chan {
258 struct virt_dma_chan vchan;
259 struct dma_pool *desc_pool;
260 u32 id;
261 struct stm32_mdma_desc *desc;
262 u32 curr_hwdesc;
263 struct dma_slave_config dma_config;
264 struct stm32_mdma_chan_config chan_config;
265 bool busy;
266 u32 mem_burst;
267 u32 mem_width;
268 };
269
270 struct stm32_mdma_device {
271 struct dma_device ddev;
272 void __iomem *base;
273 struct clk *clk;
274 int irq;
275 u32 nr_channels;
276 u32 nr_requests;
277 u32 nr_ahb_addr_masks;
278 struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS];
279 u32 ahb_addr_masks[];
280 };
281
stm32_mdma_get_dev(struct stm32_mdma_chan * chan)282 static struct stm32_mdma_device *stm32_mdma_get_dev(
283 struct stm32_mdma_chan *chan)
284 {
285 return container_of(chan->vchan.chan.device, struct stm32_mdma_device,
286 ddev);
287 }
288
to_stm32_mdma_chan(struct dma_chan * c)289 static struct stm32_mdma_chan *to_stm32_mdma_chan(struct dma_chan *c)
290 {
291 return container_of(c, struct stm32_mdma_chan, vchan.chan);
292 }
293
to_stm32_mdma_desc(struct virt_dma_desc * vdesc)294 static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc)
295 {
296 return container_of(vdesc, struct stm32_mdma_desc, vdesc);
297 }
298
chan2dev(struct stm32_mdma_chan * chan)299 static struct device *chan2dev(struct stm32_mdma_chan *chan)
300 {
301 return &chan->vchan.chan.dev->device;
302 }
303
mdma2dev(struct stm32_mdma_device * mdma_dev)304 static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev)
305 {
306 return mdma_dev->ddev.dev;
307 }
308
stm32_mdma_read(struct stm32_mdma_device * dmadev,u32 reg)309 static u32 stm32_mdma_read(struct stm32_mdma_device *dmadev, u32 reg)
310 {
311 return readl_relaxed(dmadev->base + reg);
312 }
313
stm32_mdma_write(struct stm32_mdma_device * dmadev,u32 reg,u32 val)314 static void stm32_mdma_write(struct stm32_mdma_device *dmadev, u32 reg, u32 val)
315 {
316 writel_relaxed(val, dmadev->base + reg);
317 }
318
stm32_mdma_set_bits(struct stm32_mdma_device * dmadev,u32 reg,u32 mask)319 static void stm32_mdma_set_bits(struct stm32_mdma_device *dmadev, u32 reg,
320 u32 mask)
321 {
322 void __iomem *addr = dmadev->base + reg;
323
324 writel_relaxed(readl_relaxed(addr) | mask, addr);
325 }
326
stm32_mdma_clr_bits(struct stm32_mdma_device * dmadev,u32 reg,u32 mask)327 static void stm32_mdma_clr_bits(struct stm32_mdma_device *dmadev, u32 reg,
328 u32 mask)
329 {
330 void __iomem *addr = dmadev->base + reg;
331
332 writel_relaxed(readl_relaxed(addr) & ~mask, addr);
333 }
334
stm32_mdma_alloc_desc(struct stm32_mdma_chan * chan,u32 count)335 static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
336 struct stm32_mdma_chan *chan, u32 count)
337 {
338 struct stm32_mdma_desc *desc;
339 int i;
340
341 desc = kzalloc(offsetof(typeof(*desc), node[count]), GFP_NOWAIT);
342 if (!desc)
343 return NULL;
344
345 for (i = 0; i < count; i++) {
346 desc->node[i].hwdesc =
347 dma_pool_alloc(chan->desc_pool, GFP_NOWAIT,
348 &desc->node[i].hwdesc_phys);
349 if (!desc->node[i].hwdesc)
350 goto err;
351 }
352
353 desc->count = count;
354
355 return desc;
356
357 err:
358 dev_err(chan2dev(chan), "Failed to allocate descriptor\n");
359 while (--i >= 0)
360 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
361 desc->node[i].hwdesc_phys);
362 kfree(desc);
363 return NULL;
364 }
365
stm32_mdma_desc_free(struct virt_dma_desc * vdesc)366 static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc)
367 {
368 struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc);
369 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(vdesc->tx.chan);
370 int i;
371
372 for (i = 0; i < desc->count; i++)
373 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
374 desc->node[i].hwdesc_phys);
375 kfree(desc);
376 }
377
stm32_mdma_get_width(struct stm32_mdma_chan * chan,enum dma_slave_buswidth width)378 static int stm32_mdma_get_width(struct stm32_mdma_chan *chan,
379 enum dma_slave_buswidth width)
380 {
381 switch (width) {
382 case DMA_SLAVE_BUSWIDTH_1_BYTE:
383 case DMA_SLAVE_BUSWIDTH_2_BYTES:
384 case DMA_SLAVE_BUSWIDTH_4_BYTES:
385 case DMA_SLAVE_BUSWIDTH_8_BYTES:
386 return ffs(width) - 1;
387 default:
388 dev_err(chan2dev(chan), "Dma bus width %i not supported\n",
389 width);
390 return -EINVAL;
391 }
392 }
393
stm32_mdma_get_max_width(dma_addr_t addr,u32 buf_len,u32 tlen)394 static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr,
395 u32 buf_len, u32 tlen)
396 {
397 enum dma_slave_buswidth max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
398
399 for (max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
400 max_width > DMA_SLAVE_BUSWIDTH_1_BYTE;
401 max_width >>= 1) {
402 /*
403 * Address and buffer length both have to be aligned on
404 * bus width
405 */
406 if ((((buf_len | addr) & (max_width - 1)) == 0) &&
407 tlen >= max_width)
408 break;
409 }
410
411 return max_width;
412 }
413
stm32_mdma_get_best_burst(u32 buf_len,u32 tlen,u32 max_burst,enum dma_slave_buswidth width)414 static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst,
415 enum dma_slave_buswidth width)
416 {
417 u32 best_burst;
418
419 best_burst = min((u32)1 << __ffs(tlen | buf_len),
420 max_burst * width) / width;
421
422 return (best_burst > 0) ? best_burst : 1;
423 }
424
stm32_mdma_disable_chan(struct stm32_mdma_chan * chan)425 static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan)
426 {
427 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
428 u32 ccr, cisr, id, reg;
429 int ret;
430
431 id = chan->id;
432 reg = STM32_MDMA_CCR(id);
433
434 /* Disable interrupts */
435 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_IRQ_MASK);
436
437 ccr = stm32_mdma_read(dmadev, reg);
438 if (ccr & STM32_MDMA_CCR_EN) {
439 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_EN);
440
441 /* Ensure that any ongoing transfer has been completed */
442 ret = readl_relaxed_poll_timeout_atomic(
443 dmadev->base + STM32_MDMA_CISR(id), cisr,
444 (cisr & STM32_MDMA_CISR_CTCIF), 10, 1000);
445 if (ret) {
446 dev_err(chan2dev(chan), "%s: timeout!\n", __func__);
447 return -EBUSY;
448 }
449 }
450
451 return 0;
452 }
453
stm32_mdma_stop(struct stm32_mdma_chan * chan)454 static void stm32_mdma_stop(struct stm32_mdma_chan *chan)
455 {
456 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
457 u32 status;
458 int ret;
459
460 /* Disable DMA */
461 ret = stm32_mdma_disable_chan(chan);
462 if (ret < 0)
463 return;
464
465 /* Clear interrupt status if it is there */
466 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
467 if (status) {
468 dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",
469 __func__, status);
470 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);
471 }
472
473 chan->busy = false;
474 }
475
stm32_mdma_set_bus(struct stm32_mdma_device * dmadev,u32 * ctbr,u32 ctbr_mask,u32 src_addr)476 static void stm32_mdma_set_bus(struct stm32_mdma_device *dmadev, u32 *ctbr,
477 u32 ctbr_mask, u32 src_addr)
478 {
479 u32 mask;
480 int i;
481
482 /* Check if memory device is on AHB or AXI */
483 *ctbr &= ~ctbr_mask;
484 mask = src_addr & 0xF0000000;
485 for (i = 0; i < dmadev->nr_ahb_addr_masks; i++) {
486 if (mask == dmadev->ahb_addr_masks[i]) {
487 *ctbr |= ctbr_mask;
488 break;
489 }
490 }
491 }
492
stm32_mdma_set_xfer_param(struct stm32_mdma_chan * chan,enum dma_transfer_direction direction,u32 * mdma_ccr,u32 * mdma_ctcr,u32 * mdma_ctbr,dma_addr_t addr,u32 buf_len)493 static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
494 enum dma_transfer_direction direction,
495 u32 *mdma_ccr, u32 *mdma_ctcr,
496 u32 *mdma_ctbr, dma_addr_t addr,
497 u32 buf_len)
498 {
499 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
500 struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
501 enum dma_slave_buswidth src_addr_width, dst_addr_width;
502 phys_addr_t src_addr, dst_addr;
503 int src_bus_width, dst_bus_width;
504 u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst;
505 u32 ccr, ctcr, ctbr, tlen;
506
507 src_addr_width = chan->dma_config.src_addr_width;
508 dst_addr_width = chan->dma_config.dst_addr_width;
509 src_maxburst = chan->dma_config.src_maxburst;
510 dst_maxburst = chan->dma_config.dst_maxburst;
511
512 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN;
513 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
514 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
515
516 /* Enable HW request mode */
517 ctcr &= ~STM32_MDMA_CTCR_SWRM;
518
519 /* Set DINC, SINC, DINCOS, SINCOS, TRGM and TLEN retrieve from DT */
520 ctcr &= ~STM32_MDMA_CTCR_CFG_MASK;
521 ctcr |= chan_config->transfer_config & STM32_MDMA_CTCR_CFG_MASK;
522
523 /*
524 * For buffer transfer length (TLEN) we have to set
525 * the number of bytes - 1 in CTCR register
526 */
527 tlen = STM32_MDMA_CTCR_LEN2_GET(ctcr);
528 ctcr &= ~STM32_MDMA_CTCR_LEN2_MSK;
529 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
530
531 /* Disable Pack Enable */
532 ctcr &= ~STM32_MDMA_CTCR_PKE;
533
534 /* Check burst size constraints */
535 if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST ||
536 dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) {
537 dev_err(chan2dev(chan),
538 "burst size * bus width higher than %d bytes\n",
539 STM32_MDMA_MAX_BURST);
540 return -EINVAL;
541 }
542
543 if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) ||
544 (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) {
545 dev_err(chan2dev(chan), "burst size must be a power of 2\n");
546 return -EINVAL;
547 }
548
549 /*
550 * Configure channel control:
551 * - Clear SW request as in this case this is a HW one
552 * - Clear WEX, HEX and BEX bits
553 * - Set priority level
554 */
555 ccr &= ~(STM32_MDMA_CCR_SWRQ | STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
556 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK);
557 ccr |= STM32_MDMA_CCR_PL(chan_config->priority_level);
558
559 /* Configure Trigger selection */
560 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
561 ctbr |= STM32_MDMA_CTBR_TSEL(chan_config->request);
562
563 switch (direction) {
564 case DMA_MEM_TO_DEV:
565 dst_addr = chan->dma_config.dst_addr;
566
567 /* Set device data size */
568 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);
569 if (dst_bus_width < 0)
570 return dst_bus_width;
571 ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK;
572 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width);
573
574 /* Set device burst value */
575 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
576 dst_maxburst,
577 dst_addr_width);
578 chan->mem_burst = dst_best_burst;
579 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
580 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
581
582 /* Set memory data size */
583 src_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
584 chan->mem_width = src_addr_width;
585 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);
586 if (src_bus_width < 0)
587 return src_bus_width;
588 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK |
589 STM32_MDMA_CTCR_SINCOS_MASK;
590 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width) |
591 STM32_MDMA_CTCR_SINCOS(src_bus_width);
592
593 /* Set memory burst value */
594 src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;
595 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
596 src_maxburst,
597 src_addr_width);
598 chan->mem_burst = src_best_burst;
599 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
600 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
601
602 /* Select bus */
603 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
604 dst_addr);
605
606 if (dst_bus_width != src_bus_width)
607 ctcr |= STM32_MDMA_CTCR_PKE;
608
609 /* Set destination address */
610 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(chan->id), dst_addr);
611 break;
612
613 case DMA_DEV_TO_MEM:
614 src_addr = chan->dma_config.src_addr;
615
616 /* Set device data size */
617 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);
618 if (src_bus_width < 0)
619 return src_bus_width;
620 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK;
621 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width);
622
623 /* Set device burst value */
624 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
625 src_maxburst,
626 src_addr_width);
627 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
628 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
629
630 /* Set memory data size */
631 dst_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
632 chan->mem_width = dst_addr_width;
633 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);
634 if (dst_bus_width < 0)
635 return dst_bus_width;
636 ctcr &= ~(STM32_MDMA_CTCR_DSIZE_MASK |
637 STM32_MDMA_CTCR_DINCOS_MASK);
638 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
639 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
640
641 /* Set memory burst value */
642 dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;
643 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
644 dst_maxburst,
645 dst_addr_width);
646 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
647 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
648
649 /* Select bus */
650 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
651 src_addr);
652
653 if (dst_bus_width != src_bus_width)
654 ctcr |= STM32_MDMA_CTCR_PKE;
655
656 /* Set source address */
657 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(chan->id), src_addr);
658 break;
659
660 default:
661 dev_err(chan2dev(chan), "Dma direction is not supported\n");
662 return -EINVAL;
663 }
664
665 *mdma_ccr = ccr;
666 *mdma_ctcr = ctcr;
667 *mdma_ctbr = ctbr;
668
669 return 0;
670 }
671
stm32_mdma_dump_hwdesc(struct stm32_mdma_chan * chan,struct stm32_mdma_desc_node * node)672 static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan,
673 struct stm32_mdma_desc_node *node)
674 {
675 dev_dbg(chan2dev(chan), "hwdesc: %pad\n", &node->hwdesc_phys);
676 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", node->hwdesc->ctcr);
677 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr);
678 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", node->hwdesc->csar);
679 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", node->hwdesc->cdar);
680 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur);
681 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", node->hwdesc->clar);
682 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", node->hwdesc->ctbr);
683 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", node->hwdesc->cmar);
684 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr);
685 }
686
stm32_mdma_setup_hwdesc(struct stm32_mdma_chan * chan,struct stm32_mdma_desc * desc,enum dma_transfer_direction dir,u32 count,dma_addr_t src_addr,dma_addr_t dst_addr,u32 len,u32 ctcr,u32 ctbr,bool is_last,bool is_first,bool is_cyclic)687 static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
688 struct stm32_mdma_desc *desc,
689 enum dma_transfer_direction dir, u32 count,
690 dma_addr_t src_addr, dma_addr_t dst_addr,
691 u32 len, u32 ctcr, u32 ctbr, bool is_last,
692 bool is_first, bool is_cyclic)
693 {
694 struct stm32_mdma_chan_config *config = &chan->chan_config;
695 struct stm32_mdma_hwdesc *hwdesc;
696 u32 next = count + 1;
697
698 hwdesc = desc->node[count].hwdesc;
699 hwdesc->ctcr = ctcr;
700 hwdesc->cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK |
701 STM32_MDMA_CBNDTR_BRDUM |
702 STM32_MDMA_CBNDTR_BRSUM |
703 STM32_MDMA_CBNDTR_BNDT_MASK);
704 hwdesc->cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
705 hwdesc->csar = src_addr;
706 hwdesc->cdar = dst_addr;
707 hwdesc->cbrur = 0;
708 hwdesc->ctbr = ctbr;
709 hwdesc->cmar = config->mask_addr;
710 hwdesc->cmdr = config->mask_data;
711
712 if (is_last) {
713 if (is_cyclic)
714 hwdesc->clar = desc->node[0].hwdesc_phys;
715 else
716 hwdesc->clar = 0;
717 } else {
718 hwdesc->clar = desc->node[next].hwdesc_phys;
719 }
720
721 stm32_mdma_dump_hwdesc(chan, &desc->node[count]);
722 }
723
stm32_mdma_setup_xfer(struct stm32_mdma_chan * chan,struct stm32_mdma_desc * desc,struct scatterlist * sgl,u32 sg_len,enum dma_transfer_direction direction)724 static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan,
725 struct stm32_mdma_desc *desc,
726 struct scatterlist *sgl, u32 sg_len,
727 enum dma_transfer_direction direction)
728 {
729 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
730 struct dma_slave_config *dma_config = &chan->dma_config;
731 struct scatterlist *sg;
732 dma_addr_t src_addr, dst_addr;
733 u32 ccr, ctcr, ctbr;
734 int i, ret = 0;
735
736 for_each_sg(sgl, sg, sg_len, i) {
737 if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) {
738 dev_err(chan2dev(chan), "Invalid block len\n");
739 return -EINVAL;
740 }
741
742 if (direction == DMA_MEM_TO_DEV) {
743 src_addr = sg_dma_address(sg);
744 dst_addr = dma_config->dst_addr;
745 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
746 &ctcr, &ctbr, src_addr,
747 sg_dma_len(sg));
748 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
749 src_addr);
750 } else {
751 src_addr = dma_config->src_addr;
752 dst_addr = sg_dma_address(sg);
753 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
754 &ctcr, &ctbr, dst_addr,
755 sg_dma_len(sg));
756 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
757 dst_addr);
758 }
759
760 if (ret < 0)
761 return ret;
762
763 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,
764 dst_addr, sg_dma_len(sg), ctcr, ctbr,
765 i == sg_len - 1, i == 0, false);
766 }
767
768 /* Enable interrupts */
769 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
770 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE;
771 if (sg_len > 1)
772 ccr |= STM32_MDMA_CCR_BTIE;
773 desc->ccr = ccr;
774
775 return 0;
776 }
777
778 static struct dma_async_tx_descriptor *
stm32_mdma_prep_slave_sg(struct dma_chan * c,struct scatterlist * sgl,u32 sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)779 stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,
780 u32 sg_len, enum dma_transfer_direction direction,
781 unsigned long flags, void *context)
782 {
783 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
784 struct stm32_mdma_desc *desc;
785 int i, ret;
786
787 /*
788 * Once DMA is in setup cyclic mode the channel we cannot assign this
789 * channel anymore. The DMA channel needs to be aborted or terminated
790 * for allowing another request.
791 */
792 if (chan->desc && chan->desc->cyclic) {
793 dev_err(chan2dev(chan),
794 "Request not allowed when dma in cyclic mode\n");
795 return NULL;
796 }
797
798 desc = stm32_mdma_alloc_desc(chan, sg_len);
799 if (!desc)
800 return NULL;
801
802 ret = stm32_mdma_setup_xfer(chan, desc, sgl, sg_len, direction);
803 if (ret < 0)
804 goto xfer_setup_err;
805
806 desc->cyclic = false;
807
808 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
809
810 xfer_setup_err:
811 for (i = 0; i < desc->count; i++)
812 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
813 desc->node[i].hwdesc_phys);
814 kfree(desc);
815 return NULL;
816 }
817
818 static struct dma_async_tx_descriptor *
stm32_mdma_prep_dma_cyclic(struct dma_chan * c,dma_addr_t buf_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction direction,unsigned long flags)819 stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr,
820 size_t buf_len, size_t period_len,
821 enum dma_transfer_direction direction,
822 unsigned long flags)
823 {
824 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
825 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
826 struct dma_slave_config *dma_config = &chan->dma_config;
827 struct stm32_mdma_desc *desc;
828 dma_addr_t src_addr, dst_addr;
829 u32 ccr, ctcr, ctbr, count;
830 int i, ret;
831
832 /*
833 * Once DMA is in setup cyclic mode the channel we cannot assign this
834 * channel anymore. The DMA channel needs to be aborted or terminated
835 * for allowing another request.
836 */
837 if (chan->desc && chan->desc->cyclic) {
838 dev_err(chan2dev(chan),
839 "Request not allowed when dma in cyclic mode\n");
840 return NULL;
841 }
842
843 if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) {
844 dev_err(chan2dev(chan), "Invalid buffer/period len\n");
845 return NULL;
846 }
847
848 if (buf_len % period_len) {
849 dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");
850 return NULL;
851 }
852
853 count = buf_len / period_len;
854
855 desc = stm32_mdma_alloc_desc(chan, count);
856 if (!desc)
857 return NULL;
858
859 /* Select bus */
860 if (direction == DMA_MEM_TO_DEV) {
861 src_addr = buf_addr;
862 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,
863 &ctbr, src_addr, period_len);
864 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
865 src_addr);
866 } else {
867 dst_addr = buf_addr;
868 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,
869 &ctbr, dst_addr, period_len);
870 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
871 dst_addr);
872 }
873
874 if (ret < 0)
875 goto xfer_setup_err;
876
877 /* Enable interrupts */
878 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
879 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE | STM32_MDMA_CCR_BTIE;
880 desc->ccr = ccr;
881
882 /* Configure hwdesc list */
883 for (i = 0; i < count; i++) {
884 if (direction == DMA_MEM_TO_DEV) {
885 src_addr = buf_addr + i * period_len;
886 dst_addr = dma_config->dst_addr;
887 } else {
888 src_addr = dma_config->src_addr;
889 dst_addr = buf_addr + i * period_len;
890 }
891
892 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,
893 dst_addr, period_len, ctcr, ctbr,
894 i == count - 1, i == 0, true);
895 }
896
897 desc->cyclic = true;
898
899 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
900
901 xfer_setup_err:
902 for (i = 0; i < desc->count; i++)
903 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
904 desc->node[i].hwdesc_phys);
905 kfree(desc);
906 return NULL;
907 }
908
909 static struct dma_async_tx_descriptor *
stm32_mdma_prep_dma_memcpy(struct dma_chan * c,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)910 stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
911 size_t len, unsigned long flags)
912 {
913 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
914 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
915 enum dma_slave_buswidth max_width;
916 struct stm32_mdma_desc *desc;
917 struct stm32_mdma_hwdesc *hwdesc;
918 u32 ccr, ctcr, ctbr, cbndtr, count, max_burst, mdma_burst;
919 u32 best_burst, tlen;
920 size_t xfer_count, offset;
921 int src_bus_width, dst_bus_width;
922 int i;
923
924 /*
925 * Once DMA is in setup cyclic mode the channel we cannot assign this
926 * channel anymore. The DMA channel needs to be aborted or terminated
927 * to allow another request
928 */
929 if (chan->desc && chan->desc->cyclic) {
930 dev_err(chan2dev(chan),
931 "Request not allowed when dma in cyclic mode\n");
932 return NULL;
933 }
934
935 count = DIV_ROUND_UP(len, STM32_MDMA_MAX_BLOCK_LEN);
936 desc = stm32_mdma_alloc_desc(chan, count);
937 if (!desc)
938 return NULL;
939
940 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN;
941 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
942 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
943 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
944
945 /* Enable sw req, some interrupts and clear other bits */
946 ccr &= ~(STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
947 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK |
948 STM32_MDMA_CCR_IRQ_MASK);
949 ccr |= STM32_MDMA_CCR_TEIE;
950
951 /* Enable SW request mode, dest/src inc and clear other bits */
952 ctcr &= ~(STM32_MDMA_CTCR_BWM | STM32_MDMA_CTCR_TRGM_MSK |
953 STM32_MDMA_CTCR_PAM_MASK | STM32_MDMA_CTCR_PKE |
954 STM32_MDMA_CTCR_TLEN_MSK | STM32_MDMA_CTCR_DBURST_MASK |
955 STM32_MDMA_CTCR_SBURST_MASK | STM32_MDMA_CTCR_DINCOS_MASK |
956 STM32_MDMA_CTCR_SINCOS_MASK | STM32_MDMA_CTCR_DSIZE_MASK |
957 STM32_MDMA_CTCR_SSIZE_MASK | STM32_MDMA_CTCR_DINC_MASK |
958 STM32_MDMA_CTCR_SINC_MASK);
959 ctcr |= STM32_MDMA_CTCR_SWRM | STM32_MDMA_CTCR_SINC(STM32_MDMA_INC) |
960 STM32_MDMA_CTCR_DINC(STM32_MDMA_INC);
961
962 /* Reset HW request */
963 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
964
965 /* Select bus */
966 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, src);
967 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, dest);
968
969 /* Clear CBNDTR registers */
970 cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK | STM32_MDMA_CBNDTR_BRDUM |
971 STM32_MDMA_CBNDTR_BRSUM | STM32_MDMA_CBNDTR_BNDT_MASK);
972
973 if (len <= STM32_MDMA_MAX_BLOCK_LEN) {
974 cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
975 if (len <= STM32_MDMA_MAX_BUF_LEN) {
976 /* Setup a buffer transfer */
977 ccr |= STM32_MDMA_CCR_TCIE | STM32_MDMA_CCR_CTCIE;
978 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BUFFER);
979 } else {
980 /* Setup a block transfer */
981 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
982 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BLOCK);
983 }
984
985 tlen = STM32_MDMA_MAX_BUF_LEN;
986 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
987
988 /* Set source best burst size */
989 max_width = stm32_mdma_get_max_width(src, len, tlen);
990 src_bus_width = stm32_mdma_get_width(chan, max_width);
991
992 max_burst = tlen / max_width;
993 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,
994 max_width);
995 mdma_burst = ilog2(best_burst);
996
997 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
998 STM32_MDMA_CTCR_SSIZE(src_bus_width) |
999 STM32_MDMA_CTCR_SINCOS(src_bus_width);
1000
1001 /* Set destination best burst size */
1002 max_width = stm32_mdma_get_max_width(dest, len, tlen);
1003 dst_bus_width = stm32_mdma_get_width(chan, max_width);
1004
1005 max_burst = tlen / max_width;
1006 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,
1007 max_width);
1008 mdma_burst = ilog2(best_burst);
1009
1010 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1011 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1012 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1013
1014 if (dst_bus_width != src_bus_width)
1015 ctcr |= STM32_MDMA_CTCR_PKE;
1016
1017 /* Prepare hardware descriptor */
1018 hwdesc = desc->node[0].hwdesc;
1019 hwdesc->ctcr = ctcr;
1020 hwdesc->cbndtr = cbndtr;
1021 hwdesc->csar = src;
1022 hwdesc->cdar = dest;
1023 hwdesc->cbrur = 0;
1024 hwdesc->clar = 0;
1025 hwdesc->ctbr = ctbr;
1026 hwdesc->cmar = 0;
1027 hwdesc->cmdr = 0;
1028
1029 stm32_mdma_dump_hwdesc(chan, &desc->node[0]);
1030 } else {
1031 /* Setup a LLI transfer */
1032 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_LINKED_LIST) |
1033 STM32_MDMA_CTCR_TLEN((STM32_MDMA_MAX_BUF_LEN - 1));
1034 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
1035 tlen = STM32_MDMA_MAX_BUF_LEN;
1036
1037 for (i = 0, offset = 0; offset < len;
1038 i++, offset += xfer_count) {
1039 xfer_count = min_t(size_t, len - offset,
1040 STM32_MDMA_MAX_BLOCK_LEN);
1041
1042 /* Set source best burst size */
1043 max_width = stm32_mdma_get_max_width(src, len, tlen);
1044 src_bus_width = stm32_mdma_get_width(chan, max_width);
1045
1046 max_burst = tlen / max_width;
1047 best_burst = stm32_mdma_get_best_burst(len, tlen,
1048 max_burst,
1049 max_width);
1050 mdma_burst = ilog2(best_burst);
1051
1052 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
1053 STM32_MDMA_CTCR_SSIZE(src_bus_width) |
1054 STM32_MDMA_CTCR_SINCOS(src_bus_width);
1055
1056 /* Set destination best burst size */
1057 max_width = stm32_mdma_get_max_width(dest, len, tlen);
1058 dst_bus_width = stm32_mdma_get_width(chan, max_width);
1059
1060 max_burst = tlen / max_width;
1061 best_burst = stm32_mdma_get_best_burst(len, tlen,
1062 max_burst,
1063 max_width);
1064 mdma_burst = ilog2(best_burst);
1065
1066 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1067 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1068 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1069
1070 if (dst_bus_width != src_bus_width)
1071 ctcr |= STM32_MDMA_CTCR_PKE;
1072
1073 /* Prepare hardware descriptor */
1074 stm32_mdma_setup_hwdesc(chan, desc, DMA_MEM_TO_MEM, i,
1075 src + offset, dest + offset,
1076 xfer_count, ctcr, ctbr,
1077 i == count - 1, i == 0, false);
1078 }
1079 }
1080
1081 desc->ccr = ccr;
1082
1083 desc->cyclic = false;
1084
1085 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
1086 }
1087
stm32_mdma_dump_reg(struct stm32_mdma_chan * chan)1088 static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan)
1089 {
1090 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1091
1092 dev_dbg(chan2dev(chan), "CCR: 0x%08x\n",
1093 stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)));
1094 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n",
1095 stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)));
1096 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n",
1097 stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)));
1098 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n",
1099 stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id)));
1100 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n",
1101 stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id)));
1102 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n",
1103 stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id)));
1104 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n",
1105 stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)));
1106 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n",
1107 stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)));
1108 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n",
1109 stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id)));
1110 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n",
1111 stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id)));
1112 }
1113
stm32_mdma_start_transfer(struct stm32_mdma_chan * chan)1114 static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)
1115 {
1116 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1117 struct virt_dma_desc *vdesc;
1118 struct stm32_mdma_hwdesc *hwdesc;
1119 u32 id = chan->id;
1120 u32 status, reg;
1121
1122 vdesc = vchan_next_desc(&chan->vchan);
1123 if (!vdesc) {
1124 chan->desc = NULL;
1125 return;
1126 }
1127
1128 list_del(&vdesc->node);
1129
1130 chan->desc = to_stm32_mdma_desc(vdesc);
1131 hwdesc = chan->desc->node[0].hwdesc;
1132 chan->curr_hwdesc = 0;
1133
1134 stm32_mdma_write(dmadev, STM32_MDMA_CCR(id), chan->desc->ccr);
1135 stm32_mdma_write(dmadev, STM32_MDMA_CTCR(id), hwdesc->ctcr);
1136 stm32_mdma_write(dmadev, STM32_MDMA_CBNDTR(id), hwdesc->cbndtr);
1137 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(id), hwdesc->csar);
1138 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(id), hwdesc->cdar);
1139 stm32_mdma_write(dmadev, STM32_MDMA_CBRUR(id), hwdesc->cbrur);
1140 stm32_mdma_write(dmadev, STM32_MDMA_CLAR(id), hwdesc->clar);
1141 stm32_mdma_write(dmadev, STM32_MDMA_CTBR(id), hwdesc->ctbr);
1142 stm32_mdma_write(dmadev, STM32_MDMA_CMAR(id), hwdesc->cmar);
1143 stm32_mdma_write(dmadev, STM32_MDMA_CMDR(id), hwdesc->cmdr);
1144
1145 /* Clear interrupt status if it is there */
1146 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
1147 if (status)
1148 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(id), status);
1149
1150 stm32_mdma_dump_reg(chan);
1151
1152 /* Start DMA */
1153 stm32_mdma_set_bits(dmadev, STM32_MDMA_CCR(id), STM32_MDMA_CCR_EN);
1154
1155 /* Set SW request in case of MEM2MEM transfer */
1156 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM) {
1157 reg = STM32_MDMA_CCR(id);
1158 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1159 }
1160
1161 chan->busy = true;
1162
1163 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
1164 }
1165
stm32_mdma_issue_pending(struct dma_chan * c)1166 static void stm32_mdma_issue_pending(struct dma_chan *c)
1167 {
1168 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1169 unsigned long flags;
1170
1171 spin_lock_irqsave(&chan->vchan.lock, flags);
1172
1173 if (!vchan_issue_pending(&chan->vchan))
1174 goto end;
1175
1176 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
1177
1178 if (!chan->desc && !chan->busy)
1179 stm32_mdma_start_transfer(chan);
1180
1181 end:
1182 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1183 }
1184
stm32_mdma_pause(struct dma_chan * c)1185 static int stm32_mdma_pause(struct dma_chan *c)
1186 {
1187 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1188 unsigned long flags;
1189 int ret;
1190
1191 spin_lock_irqsave(&chan->vchan.lock, flags);
1192 ret = stm32_mdma_disable_chan(chan);
1193 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1194
1195 if (!ret)
1196 dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan);
1197
1198 return ret;
1199 }
1200
stm32_mdma_resume(struct dma_chan * c)1201 static int stm32_mdma_resume(struct dma_chan *c)
1202 {
1203 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1204 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1205 struct stm32_mdma_hwdesc *hwdesc;
1206 unsigned long flags;
1207 u32 status, reg;
1208
1209 /* Transfer can be terminated */
1210 if (!chan->desc || (stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & STM32_MDMA_CCR_EN))
1211 return -EPERM;
1212
1213 hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc;
1214
1215 spin_lock_irqsave(&chan->vchan.lock, flags);
1216
1217 /* Re-configure control register */
1218 stm32_mdma_write(dmadev, STM32_MDMA_CCR(chan->id), chan->desc->ccr);
1219
1220 /* Clear interrupt status if it is there */
1221 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
1222 if (status)
1223 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);
1224
1225 stm32_mdma_dump_reg(chan);
1226
1227 /* Re-start DMA */
1228 reg = STM32_MDMA_CCR(chan->id);
1229 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_EN);
1230
1231 /* Set SW request in case of MEM2MEM transfer */
1232 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM)
1233 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1234
1235 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1236
1237 dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan);
1238
1239 return 0;
1240 }
1241
stm32_mdma_terminate_all(struct dma_chan * c)1242 static int stm32_mdma_terminate_all(struct dma_chan *c)
1243 {
1244 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1245 unsigned long flags;
1246 LIST_HEAD(head);
1247
1248 spin_lock_irqsave(&chan->vchan.lock, flags);
1249 if (chan->desc) {
1250 vchan_terminate_vdesc(&chan->desc->vdesc);
1251 if (chan->busy)
1252 stm32_mdma_stop(chan);
1253 chan->desc = NULL;
1254 }
1255 vchan_get_all_descriptors(&chan->vchan, &head);
1256 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1257
1258 vchan_dma_desc_free_list(&chan->vchan, &head);
1259
1260 return 0;
1261 }
1262
stm32_mdma_synchronize(struct dma_chan * c)1263 static void stm32_mdma_synchronize(struct dma_chan *c)
1264 {
1265 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1266
1267 vchan_synchronize(&chan->vchan);
1268 }
1269
stm32_mdma_slave_config(struct dma_chan * c,struct dma_slave_config * config)1270 static int stm32_mdma_slave_config(struct dma_chan *c,
1271 struct dma_slave_config *config)
1272 {
1273 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1274
1275 memcpy(&chan->dma_config, config, sizeof(*config));
1276
1277 return 0;
1278 }
1279
stm32_mdma_desc_residue(struct stm32_mdma_chan * chan,struct stm32_mdma_desc * desc,u32 curr_hwdesc)1280 static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan,
1281 struct stm32_mdma_desc *desc,
1282 u32 curr_hwdesc)
1283 {
1284 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1285 struct stm32_mdma_hwdesc *hwdesc = desc->node[0].hwdesc;
1286 u32 cbndtr, residue, modulo, burst_size;
1287 int i;
1288
1289 residue = 0;
1290 for (i = curr_hwdesc + 1; i < desc->count; i++) {
1291 hwdesc = desc->node[i].hwdesc;
1292 residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr);
1293 }
1294 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
1295 residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK;
1296
1297 if (!chan->mem_burst)
1298 return residue;
1299
1300 burst_size = chan->mem_burst * chan->mem_width;
1301 modulo = residue % burst_size;
1302 if (modulo)
1303 residue = residue - modulo + burst_size;
1304
1305 return residue;
1306 }
1307
stm32_mdma_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * state)1308 static enum dma_status stm32_mdma_tx_status(struct dma_chan *c,
1309 dma_cookie_t cookie,
1310 struct dma_tx_state *state)
1311 {
1312 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1313 struct virt_dma_desc *vdesc;
1314 enum dma_status status;
1315 unsigned long flags;
1316 u32 residue = 0;
1317
1318 status = dma_cookie_status(c, cookie, state);
1319 if ((status == DMA_COMPLETE) || (!state))
1320 return status;
1321
1322 spin_lock_irqsave(&chan->vchan.lock, flags);
1323
1324 vdesc = vchan_find_desc(&chan->vchan, cookie);
1325 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie)
1326 residue = stm32_mdma_desc_residue(chan, chan->desc,
1327 chan->curr_hwdesc);
1328 else if (vdesc)
1329 residue = stm32_mdma_desc_residue(chan,
1330 to_stm32_mdma_desc(vdesc), 0);
1331 dma_set_residue(state, residue);
1332
1333 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1334
1335 return status;
1336 }
1337
stm32_mdma_xfer_end(struct stm32_mdma_chan * chan)1338 static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan)
1339 {
1340 vchan_cookie_complete(&chan->desc->vdesc);
1341 chan->desc = NULL;
1342 chan->busy = false;
1343
1344 /* Start the next transfer if this driver has a next desc */
1345 stm32_mdma_start_transfer(chan);
1346 }
1347
stm32_mdma_irq_handler(int irq,void * devid)1348 static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
1349 {
1350 struct stm32_mdma_device *dmadev = devid;
1351 struct stm32_mdma_chan *chan;
1352 u32 reg, id, ccr, ien, status;
1353
1354 /* Find out which channel generates the interrupt */
1355 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);
1356 if (!status) {
1357 dev_dbg(mdma2dev(dmadev), "spurious it\n");
1358 return IRQ_NONE;
1359 }
1360 id = __ffs(status);
1361
1362 chan = &dmadev->chan[id];
1363 if (!chan) {
1364 dev_warn(mdma2dev(dmadev), "MDMA channel not initialized\n");
1365 return IRQ_NONE;
1366 }
1367
1368 /* Handle interrupt for the channel */
1369 spin_lock(&chan->vchan.lock);
1370 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
1371 /* Mask Channel ReQuest Active bit which can be set in case of MEM2MEM */
1372 status &= ~STM32_MDMA_CISR_CRQA;
1373 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));
1374 ien = (ccr & STM32_MDMA_CCR_IRQ_MASK) >> 1;
1375
1376 if (!(status & ien)) {
1377 spin_unlock(&chan->vchan.lock);
1378 dev_warn(chan2dev(chan),
1379 "spurious it (status=0x%04x, ien=0x%04x)\n",
1380 status, ien);
1381 return IRQ_NONE;
1382 }
1383
1384 reg = STM32_MDMA_CIFCR(id);
1385
1386 if (status & STM32_MDMA_CISR_TEIF) {
1387 dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n",
1388 readl_relaxed(dmadev->base + STM32_MDMA_CESR(id)));
1389 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);
1390 status &= ~STM32_MDMA_CISR_TEIF;
1391 }
1392
1393 if (status & STM32_MDMA_CISR_CTCIF) {
1394 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF);
1395 status &= ~STM32_MDMA_CISR_CTCIF;
1396 stm32_mdma_xfer_end(chan);
1397 }
1398
1399 if (status & STM32_MDMA_CISR_BRTIF) {
1400 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF);
1401 status &= ~STM32_MDMA_CISR_BRTIF;
1402 }
1403
1404 if (status & STM32_MDMA_CISR_BTIF) {
1405 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF);
1406 status &= ~STM32_MDMA_CISR_BTIF;
1407 chan->curr_hwdesc++;
1408 if (chan->desc && chan->desc->cyclic) {
1409 if (chan->curr_hwdesc == chan->desc->count)
1410 chan->curr_hwdesc = 0;
1411 vchan_cyclic_callback(&chan->desc->vdesc);
1412 }
1413 }
1414
1415 if (status & STM32_MDMA_CISR_TCIF) {
1416 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF);
1417 status &= ~STM32_MDMA_CISR_TCIF;
1418 }
1419
1420 if (status) {
1421 stm32_mdma_set_bits(dmadev, reg, status);
1422 dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
1423 if (!(ccr & STM32_MDMA_CCR_EN))
1424 dev_err(chan2dev(chan), "chan disabled by HW\n");
1425 }
1426
1427 spin_unlock(&chan->vchan.lock);
1428
1429 return IRQ_HANDLED;
1430 }
1431
stm32_mdma_alloc_chan_resources(struct dma_chan * c)1432 static int stm32_mdma_alloc_chan_resources(struct dma_chan *c)
1433 {
1434 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1435 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1436 int ret;
1437
1438 chan->desc_pool = dmam_pool_create(dev_name(&c->dev->device),
1439 c->device->dev,
1440 sizeof(struct stm32_mdma_hwdesc),
1441 __alignof__(struct stm32_mdma_hwdesc),
1442 0);
1443 if (!chan->desc_pool) {
1444 dev_err(chan2dev(chan), "failed to allocate descriptor pool\n");
1445 return -ENOMEM;
1446 }
1447
1448 ret = pm_runtime_resume_and_get(dmadev->ddev.dev);
1449 if (ret < 0)
1450 return ret;
1451
1452 ret = stm32_mdma_disable_chan(chan);
1453 if (ret < 0)
1454 pm_runtime_put(dmadev->ddev.dev);
1455
1456 return ret;
1457 }
1458
stm32_mdma_free_chan_resources(struct dma_chan * c)1459 static void stm32_mdma_free_chan_resources(struct dma_chan *c)
1460 {
1461 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1462 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1463 unsigned long flags;
1464
1465 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);
1466
1467 if (chan->busy) {
1468 spin_lock_irqsave(&chan->vchan.lock, flags);
1469 stm32_mdma_stop(chan);
1470 chan->desc = NULL;
1471 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1472 }
1473
1474 pm_runtime_put(dmadev->ddev.dev);
1475 vchan_free_chan_resources(to_virt_chan(c));
1476 dmam_pool_destroy(chan->desc_pool);
1477 chan->desc_pool = NULL;
1478 }
1479
stm32_mdma_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)1480 static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec,
1481 struct of_dma *ofdma)
1482 {
1483 struct stm32_mdma_device *dmadev = ofdma->of_dma_data;
1484 struct stm32_mdma_chan *chan;
1485 struct dma_chan *c;
1486 struct stm32_mdma_chan_config config;
1487
1488 if (dma_spec->args_count < 5) {
1489 dev_err(mdma2dev(dmadev), "Bad number of args\n");
1490 return NULL;
1491 }
1492
1493 config.request = dma_spec->args[0];
1494 config.priority_level = dma_spec->args[1];
1495 config.transfer_config = dma_spec->args[2];
1496 config.mask_addr = dma_spec->args[3];
1497 config.mask_data = dma_spec->args[4];
1498
1499 if (config.request >= dmadev->nr_requests) {
1500 dev_err(mdma2dev(dmadev), "Bad request line\n");
1501 return NULL;
1502 }
1503
1504 if (config.priority_level > STM32_MDMA_VERY_HIGH_PRIORITY) {
1505 dev_err(mdma2dev(dmadev), "Priority level not supported\n");
1506 return NULL;
1507 }
1508
1509 c = dma_get_any_slave_channel(&dmadev->ddev);
1510 if (!c) {
1511 dev_err(mdma2dev(dmadev), "No more channels available\n");
1512 return NULL;
1513 }
1514
1515 chan = to_stm32_mdma_chan(c);
1516 chan->chan_config = config;
1517
1518 return c;
1519 }
1520
1521 static const struct of_device_id stm32_mdma_of_match[] = {
1522 { .compatible = "st,stm32h7-mdma", },
1523 { /* sentinel */ },
1524 };
1525 MODULE_DEVICE_TABLE(of, stm32_mdma_of_match);
1526
stm32_mdma_probe(struct platform_device * pdev)1527 static int stm32_mdma_probe(struct platform_device *pdev)
1528 {
1529 struct stm32_mdma_chan *chan;
1530 struct stm32_mdma_device *dmadev;
1531 struct dma_device *dd;
1532 struct device_node *of_node;
1533 struct resource *res;
1534 struct reset_control *rst;
1535 u32 nr_channels, nr_requests;
1536 int i, count, ret;
1537
1538 of_node = pdev->dev.of_node;
1539 if (!of_node)
1540 return -ENODEV;
1541
1542 ret = device_property_read_u32(&pdev->dev, "dma-channels",
1543 &nr_channels);
1544 if (ret) {
1545 nr_channels = STM32_MDMA_MAX_CHANNELS;
1546 dev_warn(&pdev->dev, "MDMA defaulting on %i channels\n",
1547 nr_channels);
1548 }
1549
1550 ret = device_property_read_u32(&pdev->dev, "dma-requests",
1551 &nr_requests);
1552 if (ret) {
1553 nr_requests = STM32_MDMA_MAX_REQUESTS;
1554 dev_warn(&pdev->dev, "MDMA defaulting on %i request lines\n",
1555 nr_requests);
1556 }
1557
1558 count = device_property_count_u32(&pdev->dev, "st,ahb-addr-masks");
1559 if (count < 0)
1560 count = 0;
1561
1562 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev) + sizeof(u32) * count,
1563 GFP_KERNEL);
1564 if (!dmadev)
1565 return -ENOMEM;
1566
1567 dmadev->nr_channels = nr_channels;
1568 dmadev->nr_requests = nr_requests;
1569 device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks",
1570 dmadev->ahb_addr_masks,
1571 count);
1572 dmadev->nr_ahb_addr_masks = count;
1573
1574 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1575 dmadev->base = devm_ioremap_resource(&pdev->dev, res);
1576 if (IS_ERR(dmadev->base))
1577 return PTR_ERR(dmadev->base);
1578
1579 dmadev->clk = devm_clk_get(&pdev->dev, NULL);
1580 if (IS_ERR(dmadev->clk))
1581 return dev_err_probe(&pdev->dev, PTR_ERR(dmadev->clk),
1582 "Missing clock controller\n");
1583
1584 ret = clk_prepare_enable(dmadev->clk);
1585 if (ret < 0) {
1586 dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret);
1587 return ret;
1588 }
1589
1590 rst = devm_reset_control_get(&pdev->dev, NULL);
1591 if (IS_ERR(rst)) {
1592 ret = PTR_ERR(rst);
1593 if (ret == -EPROBE_DEFER)
1594 goto err_clk;
1595 } else {
1596 reset_control_assert(rst);
1597 udelay(2);
1598 reset_control_deassert(rst);
1599 }
1600
1601 dd = &dmadev->ddev;
1602 dma_cap_set(DMA_SLAVE, dd->cap_mask);
1603 dma_cap_set(DMA_PRIVATE, dd->cap_mask);
1604 dma_cap_set(DMA_CYCLIC, dd->cap_mask);
1605 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
1606 dd->device_alloc_chan_resources = stm32_mdma_alloc_chan_resources;
1607 dd->device_free_chan_resources = stm32_mdma_free_chan_resources;
1608 dd->device_tx_status = stm32_mdma_tx_status;
1609 dd->device_issue_pending = stm32_mdma_issue_pending;
1610 dd->device_prep_slave_sg = stm32_mdma_prep_slave_sg;
1611 dd->device_prep_dma_cyclic = stm32_mdma_prep_dma_cyclic;
1612 dd->device_prep_dma_memcpy = stm32_mdma_prep_dma_memcpy;
1613 dd->device_config = stm32_mdma_slave_config;
1614 dd->device_pause = stm32_mdma_pause;
1615 dd->device_resume = stm32_mdma_resume;
1616 dd->device_terminate_all = stm32_mdma_terminate_all;
1617 dd->device_synchronize = stm32_mdma_synchronize;
1618 dd->descriptor_reuse = true;
1619
1620 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1621 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1622 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1623 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1624 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1625 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1626 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1627 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1628 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
1629 BIT(DMA_MEM_TO_MEM);
1630 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1631 dd->max_burst = STM32_MDMA_MAX_BURST;
1632 dd->dev = &pdev->dev;
1633 INIT_LIST_HEAD(&dd->channels);
1634
1635 for (i = 0; i < dmadev->nr_channels; i++) {
1636 chan = &dmadev->chan[i];
1637 chan->id = i;
1638 chan->vchan.desc_free = stm32_mdma_desc_free;
1639 vchan_init(&chan->vchan, dd);
1640 }
1641
1642 dmadev->irq = platform_get_irq(pdev, 0);
1643 if (dmadev->irq < 0) {
1644 ret = dmadev->irq;
1645 goto err_clk;
1646 }
1647
1648 ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler,
1649 0, dev_name(&pdev->dev), dmadev);
1650 if (ret) {
1651 dev_err(&pdev->dev, "failed to request IRQ\n");
1652 goto err_clk;
1653 }
1654
1655 ret = dmaenginem_async_device_register(dd);
1656 if (ret)
1657 goto err_clk;
1658
1659 ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev);
1660 if (ret < 0) {
1661 dev_err(&pdev->dev,
1662 "STM32 MDMA DMA OF registration failed %d\n", ret);
1663 goto err_clk;
1664 }
1665
1666 platform_set_drvdata(pdev, dmadev);
1667 pm_runtime_set_active(&pdev->dev);
1668 pm_runtime_enable(&pdev->dev);
1669 pm_runtime_get_noresume(&pdev->dev);
1670 pm_runtime_put(&pdev->dev);
1671
1672 dev_info(&pdev->dev, "STM32 MDMA driver registered\n");
1673
1674 return 0;
1675
1676 err_clk:
1677 clk_disable_unprepare(dmadev->clk);
1678
1679 return ret;
1680 }
1681
1682 #ifdef CONFIG_PM
stm32_mdma_runtime_suspend(struct device * dev)1683 static int stm32_mdma_runtime_suspend(struct device *dev)
1684 {
1685 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1686
1687 clk_disable_unprepare(dmadev->clk);
1688
1689 return 0;
1690 }
1691
stm32_mdma_runtime_resume(struct device * dev)1692 static int stm32_mdma_runtime_resume(struct device *dev)
1693 {
1694 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1695 int ret;
1696
1697 ret = clk_prepare_enable(dmadev->clk);
1698 if (ret) {
1699 dev_err(dev, "failed to prepare_enable clock\n");
1700 return ret;
1701 }
1702
1703 return 0;
1704 }
1705 #endif
1706
1707 #ifdef CONFIG_PM_SLEEP
stm32_mdma_pm_suspend(struct device * dev)1708 static int stm32_mdma_pm_suspend(struct device *dev)
1709 {
1710 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1711 u32 ccr, id;
1712 int ret;
1713
1714 ret = pm_runtime_resume_and_get(dev);
1715 if (ret < 0)
1716 return ret;
1717
1718 for (id = 0; id < dmadev->nr_channels; id++) {
1719 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));
1720 if (ccr & STM32_MDMA_CCR_EN) {
1721 dev_warn(dev, "Suspend is prevented by Chan %i\n", id);
1722 return -EBUSY;
1723 }
1724 }
1725
1726 pm_runtime_put_sync(dev);
1727
1728 pm_runtime_force_suspend(dev);
1729
1730 return 0;
1731 }
1732
stm32_mdma_pm_resume(struct device * dev)1733 static int stm32_mdma_pm_resume(struct device *dev)
1734 {
1735 return pm_runtime_force_resume(dev);
1736 }
1737 #endif
1738
1739 static const struct dev_pm_ops stm32_mdma_pm_ops = {
1740 SET_SYSTEM_SLEEP_PM_OPS(stm32_mdma_pm_suspend, stm32_mdma_pm_resume)
1741 SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend,
1742 stm32_mdma_runtime_resume, NULL)
1743 };
1744
1745 static struct platform_driver stm32_mdma_driver = {
1746 .probe = stm32_mdma_probe,
1747 .driver = {
1748 .name = "stm32-mdma",
1749 .of_match_table = stm32_mdma_of_match,
1750 .pm = &stm32_mdma_pm_ops,
1751 },
1752 };
1753
stm32_mdma_init(void)1754 static int __init stm32_mdma_init(void)
1755 {
1756 return platform_driver_register(&stm32_mdma_driver);
1757 }
1758
1759 subsys_initcall(stm32_mdma_init);
1760
1761 MODULE_DESCRIPTION("Driver for STM32 MDMA controller");
1762 MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");
1763 MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>");
1764 MODULE_LICENSE("GPL v2");
1765