1 /*
2 * bfin_dma_5xx.c - Blackfin DMA implementation
3 *
4 * Copyright 2004-2008 Analog Devices Inc.
5 * Licensed under the GPL-2 or later.
6 */
7
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/param.h>
13 #include <linux/proc_fs.h>
14 #include <linux/sched.h>
15 #include <linux/seq_file.h>
16 #include <linux/spinlock.h>
17
18 #include <asm/blackfin.h>
19 #include <asm/cacheflush.h>
20 #include <asm/dma.h>
21 #include <asm/uaccess.h>
22
23 struct dma_channel dma_ch[MAX_DMA_CHANNELS];
24 EXPORT_SYMBOL(dma_ch);
25
blackfin_dma_init(void)26 static int __init blackfin_dma_init(void)
27 {
28 int i;
29
30 printk(KERN_INFO "Blackfin DMA Controller\n");
31
32 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
33 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
34 dma_ch[i].regs = dma_io_base_addr[i];
35 mutex_init(&(dma_ch[i].dmalock));
36 }
37 /* Mark MEMDMA Channel 0 as requested since we're using it internally */
38 request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
39 request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
40
41 #if defined(CONFIG_DEB_DMA_URGENT)
42 bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
43 | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
44 #endif
45
46 return 0;
47 }
48 arch_initcall(blackfin_dma_init);
49
50 #ifdef CONFIG_PROC_FS
proc_dma_show(struct seq_file * m,void * v)51 static int proc_dma_show(struct seq_file *m, void *v)
52 {
53 int i;
54
55 for (i = 0; i < MAX_DMA_CHANNELS; ++i)
56 if (dma_ch[i].chan_status != DMA_CHANNEL_FREE)
57 seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
58
59 return 0;
60 }
61
proc_dma_open(struct inode * inode,struct file * file)62 static int proc_dma_open(struct inode *inode, struct file *file)
63 {
64 return single_open(file, proc_dma_show, NULL);
65 }
66
67 static const struct file_operations proc_dma_operations = {
68 .open = proc_dma_open,
69 .read = seq_read,
70 .llseek = seq_lseek,
71 .release = single_release,
72 };
73
proc_dma_init(void)74 static int __init proc_dma_init(void)
75 {
76 return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
77 }
78 late_initcall(proc_dma_init);
79 #endif
80
81 /**
82 * request_dma - request a DMA channel
83 *
84 * Request the specific DMA channel from the system if it's available.
85 */
request_dma(unsigned int channel,const char * device_id)86 int request_dma(unsigned int channel, const char *device_id)
87 {
88 pr_debug("request_dma() : BEGIN \n");
89
90 if (device_id == NULL)
91 printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
92
93 #if defined(CONFIG_BF561) && ANOMALY_05000182
94 if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
95 if (get_cclk() > 500000000) {
96 printk(KERN_WARNING
97 "Request IMDMA failed due to ANOMALY 05000182\n");
98 return -EFAULT;
99 }
100 }
101 #endif
102
103 mutex_lock(&(dma_ch[channel].dmalock));
104
105 if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
106 || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
107 mutex_unlock(&(dma_ch[channel].dmalock));
108 pr_debug("DMA CHANNEL IN USE \n");
109 return -EBUSY;
110 } else {
111 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
112 pr_debug("DMA CHANNEL IS ALLOCATED \n");
113 }
114
115 mutex_unlock(&(dma_ch[channel].dmalock));
116
117 #ifdef CONFIG_BF54x
118 if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
119 unsigned int per_map;
120 per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
121 if (strncmp(device_id, "BFIN_UART", 9) == 0)
122 dma_ch[channel].regs->peripheral_map = per_map |
123 ((channel - CH_UART2_RX + 0xC)<<12);
124 else
125 dma_ch[channel].regs->peripheral_map = per_map |
126 ((channel - CH_UART2_RX + 0x6)<<12);
127 }
128 #endif
129
130 dma_ch[channel].device_id = device_id;
131 dma_ch[channel].irq = 0;
132
133 /* This is to be enabled by putting a restriction -
134 * you have to request DMA, before doing any operations on
135 * descriptor/channel
136 */
137 pr_debug("request_dma() : END \n");
138 return 0;
139 }
140 EXPORT_SYMBOL(request_dma);
141
set_dma_callback(unsigned int channel,irq_handler_t callback,void * data)142 int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data)
143 {
144 BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
145 && channel < MAX_DMA_CHANNELS));
146
147 if (callback != NULL) {
148 int ret;
149 unsigned int irq = channel2irq(channel);
150
151 ret = request_irq(irq, callback, IRQF_DISABLED,
152 dma_ch[channel].device_id, data);
153 if (ret)
154 return ret;
155
156 dma_ch[channel].irq = irq;
157 dma_ch[channel].data = data;
158 }
159 return 0;
160 }
161 EXPORT_SYMBOL(set_dma_callback);
162
163 /**
164 * clear_dma_buffer - clear DMA fifos for specified channel
165 *
166 * Set the Buffer Clear bit in the Configuration register of specific DMA
167 * channel. This will stop the descriptor based DMA operation.
168 */
clear_dma_buffer(unsigned int channel)169 static void clear_dma_buffer(unsigned int channel)
170 {
171 dma_ch[channel].regs->cfg |= RESTART;
172 SSYNC();
173 dma_ch[channel].regs->cfg &= ~RESTART;
174 }
175
free_dma(unsigned int channel)176 void free_dma(unsigned int channel)
177 {
178 pr_debug("freedma() : BEGIN \n");
179 BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
180 && channel < MAX_DMA_CHANNELS));
181
182 /* Halt the DMA */
183 disable_dma(channel);
184 clear_dma_buffer(channel);
185
186 if (dma_ch[channel].irq)
187 free_irq(dma_ch[channel].irq, dma_ch[channel].data);
188
189 /* Clear the DMA Variable in the Channel */
190 mutex_lock(&(dma_ch[channel].dmalock));
191 dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
192 mutex_unlock(&(dma_ch[channel].dmalock));
193
194 pr_debug("freedma() : END \n");
195 }
196 EXPORT_SYMBOL(free_dma);
197
198 #ifdef CONFIG_PM
199 # ifndef MAX_DMA_SUSPEND_CHANNELS
200 # define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
201 # endif
blackfin_dma_suspend(void)202 int blackfin_dma_suspend(void)
203 {
204 int i;
205
206 for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i) {
207 if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
208 printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
209 return -EBUSY;
210 }
211
212 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
213 }
214
215 return 0;
216 }
217
blackfin_dma_resume(void)218 void blackfin_dma_resume(void)
219 {
220 int i;
221 for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i)
222 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
223 }
224 #endif
225
226 /**
227 * blackfin_dma_early_init - minimal DMA init
228 *
229 * Setup a few DMA registers so we can safely do DMA transfers early on in
230 * the kernel booting process. Really this just means using dma_memcpy().
231 */
blackfin_dma_early_init(void)232 void __init blackfin_dma_early_init(void)
233 {
234 bfin_write_MDMA_S0_CONFIG(0);
235 }
236
237 /**
238 * __dma_memcpy - program the MDMA registers
239 *
240 * Actually program MDMA0 and wait for the transfer to finish. Disable IRQs
241 * while programming registers so that everything is fully configured. Wait
242 * for DMA to finish with IRQs enabled. If interrupted, the initial DMA_DONE
243 * check will make sure we don't clobber any existing transfer.
244 */
__dma_memcpy(u32 daddr,s16 dmod,u32 saddr,s16 smod,size_t cnt,u32 conf)245 static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf)
246 {
247 static DEFINE_SPINLOCK(mdma_lock);
248 unsigned long flags;
249
250 spin_lock_irqsave(&mdma_lock, flags);
251
252 /* Force a sync in case a previous config reset on this channel
253 * occurred. This is needed so subsequent writes to DMA registers
254 * are not spuriously lost/corrupted. Do it under irq lock and
255 * without the anomaly version (because we are atomic already).
256 */
257 __builtin_bfin_ssync();
258
259 if (bfin_read_MDMA_S0_CONFIG())
260 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
261 continue;
262
263 if (conf & DMA2D) {
264 /* For larger bit sizes, we've already divided down cnt so it
265 * is no longer a multiple of 64k. So we have to break down
266 * the limit here so it is a multiple of the incoming size.
267 * There is no limitation here in terms of total size other
268 * than the hardware though as the bits lost in the shift are
269 * made up by MODIFY (== we can hit the whole address space).
270 * X: (2^(16 - 0)) * 1 == (2^(16 - 1)) * 2 == (2^(16 - 2)) * 4
271 */
272 u32 shift = abs(dmod) >> 1;
273 size_t ycnt = cnt >> (16 - shift);
274 cnt = 1 << (16 - shift);
275 bfin_write_MDMA_D0_Y_COUNT(ycnt);
276 bfin_write_MDMA_S0_Y_COUNT(ycnt);
277 bfin_write_MDMA_D0_Y_MODIFY(dmod);
278 bfin_write_MDMA_S0_Y_MODIFY(smod);
279 }
280
281 bfin_write_MDMA_D0_START_ADDR(daddr);
282 bfin_write_MDMA_D0_X_COUNT(cnt);
283 bfin_write_MDMA_D0_X_MODIFY(dmod);
284 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
285
286 bfin_write_MDMA_S0_START_ADDR(saddr);
287 bfin_write_MDMA_S0_X_COUNT(cnt);
288 bfin_write_MDMA_S0_X_MODIFY(smod);
289 bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
290
291 bfin_write_MDMA_S0_CONFIG(DMAEN | conf);
292 bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | conf);
293
294 spin_unlock_irqrestore(&mdma_lock, flags);
295
296 SSYNC();
297
298 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
299 if (bfin_read_MDMA_S0_CONFIG())
300 continue;
301 else
302 return;
303
304 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
305
306 bfin_write_MDMA_S0_CONFIG(0);
307 bfin_write_MDMA_D0_CONFIG(0);
308 }
309
310 /**
311 * _dma_memcpy - translate C memcpy settings into MDMA settings
312 *
313 * Handle all the high level steps before we touch the MDMA registers. So
314 * handle direction, tweaking of sizes, and formatting of addresses.
315 */
_dma_memcpy(void * pdst,const void * psrc,size_t size)316 static void *_dma_memcpy(void *pdst, const void *psrc, size_t size)
317 {
318 u32 conf, shift;
319 s16 mod;
320 unsigned long dst = (unsigned long)pdst;
321 unsigned long src = (unsigned long)psrc;
322
323 if (size == 0)
324 return NULL;
325
326 if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) {
327 conf = WDSIZE_32;
328 shift = 2;
329 } else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) {
330 conf = WDSIZE_16;
331 shift = 1;
332 } else {
333 conf = WDSIZE_8;
334 shift = 0;
335 }
336
337 /* If the two memory regions have a chance of overlapping, make
338 * sure the memcpy still works as expected. Do this by having the
339 * copy run backwards instead.
340 */
341 mod = 1 << shift;
342 if (src < dst) {
343 mod *= -1;
344 dst += size + mod;
345 src += size + mod;
346 }
347 size >>= shift;
348
349 if (size > 0x10000)
350 conf |= DMA2D;
351
352 __dma_memcpy(dst, mod, src, mod, size, conf);
353
354 return pdst;
355 }
356
357 /**
358 * dma_memcpy - DMA memcpy under mutex lock
359 *
360 * Do not check arguments before starting the DMA memcpy. Break the transfer
361 * up into two pieces. The first transfer is in multiples of 64k and the
362 * second transfer is the piece smaller than 64k.
363 */
dma_memcpy(void * pdst,const void * psrc,size_t size)364 void *dma_memcpy(void *pdst, const void *psrc, size_t size)
365 {
366 unsigned long dst = (unsigned long)pdst;
367 unsigned long src = (unsigned long)psrc;
368 size_t bulk, rest;
369
370 if (bfin_addr_dcachable(src))
371 blackfin_dcache_flush_range(src, src + size);
372
373 if (bfin_addr_dcachable(dst))
374 blackfin_dcache_invalidate_range(dst, dst + size);
375
376 bulk = size & ~0xffff;
377 rest = size - bulk;
378 if (bulk)
379 _dma_memcpy(pdst, psrc, bulk);
380 _dma_memcpy(pdst + bulk, psrc + bulk, rest);
381 return pdst;
382 }
383 EXPORT_SYMBOL(dma_memcpy);
384
385 /**
386 * safe_dma_memcpy - DMA memcpy w/argument checking
387 *
388 * Verify arguments are safe before heading to dma_memcpy().
389 */
safe_dma_memcpy(void * dst,const void * src,size_t size)390 void *safe_dma_memcpy(void *dst, const void *src, size_t size)
391 {
392 if (!access_ok(VERIFY_WRITE, dst, size))
393 return NULL;
394 if (!access_ok(VERIFY_READ, src, size))
395 return NULL;
396 return dma_memcpy(dst, src, size);
397 }
398 EXPORT_SYMBOL(safe_dma_memcpy);
399
_dma_out(unsigned long addr,unsigned long buf,unsigned short len,u16 size,u16 dma_size)400 static void _dma_out(unsigned long addr, unsigned long buf, unsigned short len,
401 u16 size, u16 dma_size)
402 {
403 blackfin_dcache_flush_range(buf, buf + len * size);
404 __dma_memcpy(addr, 0, buf, size, len, dma_size);
405 }
406
_dma_in(unsigned long addr,unsigned long buf,unsigned short len,u16 size,u16 dma_size)407 static void _dma_in(unsigned long addr, unsigned long buf, unsigned short len,
408 u16 size, u16 dma_size)
409 {
410 blackfin_dcache_invalidate_range(buf, buf + len * size);
411 __dma_memcpy(buf, size, addr, 0, len, dma_size);
412 }
413
414 #define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \
415 void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned short len) \
416 { \
417 _dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \
418 } \
419 EXPORT_SYMBOL(dma_##io##s##bwl)
420 MAKE_DMA_IO(out, b, 1, 8, const);
421 MAKE_DMA_IO(in, b, 1, 8, );
422 MAKE_DMA_IO(out, w, 2, 16, const);
423 MAKE_DMA_IO(in, w, 2, 16, );
424 MAKE_DMA_IO(out, l, 4, 32, const);
425 MAKE_DMA_IO(in, l, 4, 32, );
426