1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2014 Intel Corporation.
6 *
7 * Intel MIC X100 DMA Driver.
8 *
9 * Adapted from IOAT dma driver.
10 */
11 #include <linux/module.h>
12 #include <linux/io.h>
13 #include <linux/seq_file.h>
14 #include <linux/vmalloc.h>
15
16 #include "mic_x100_dma.h"
17
18 #define MIC_DMA_MAX_XFER_SIZE_CARD (1 * 1024 * 1024 -\
19 MIC_DMA_ALIGN_BYTES)
20 #define MIC_DMA_MAX_XFER_SIZE_HOST (1 * 1024 * 1024 >> 1)
21 #define MIC_DMA_DESC_TYPE_SHIFT 60
22 #define MIC_DMA_MEMCPY_LEN_SHIFT 46
23 #define MIC_DMA_STAT_INTR_SHIFT 59
24
25 /* high-water mark for pushing dma descriptors */
26 static int mic_dma_pending_level = 4;
27
28 /* Status descriptor is used to write a 64 bit value to a memory location */
29 enum mic_dma_desc_format_type {
30 MIC_DMA_MEMCPY = 1,
31 MIC_DMA_STATUS,
32 };
33
mic_dma_hw_ring_inc(u32 val)34 static inline u32 mic_dma_hw_ring_inc(u32 val)
35 {
36 return (val + 1) % MIC_DMA_DESC_RX_SIZE;
37 }
38
mic_dma_hw_ring_dec(u32 val)39 static inline u32 mic_dma_hw_ring_dec(u32 val)
40 {
41 return val ? val - 1 : MIC_DMA_DESC_RX_SIZE - 1;
42 }
43
mic_dma_hw_ring_inc_head(struct mic_dma_chan * ch)44 static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch)
45 {
46 ch->head = mic_dma_hw_ring_inc(ch->head);
47 }
48
49 /* Prepare a memcpy desc */
mic_dma_memcpy_desc(struct mic_dma_desc * desc,dma_addr_t src_phys,dma_addr_t dst_phys,u64 size)50 static inline void mic_dma_memcpy_desc(struct mic_dma_desc *desc,
51 dma_addr_t src_phys, dma_addr_t dst_phys, u64 size)
52 {
53 u64 qw0, qw1;
54
55 qw0 = src_phys;
56 qw0 |= (size >> MIC_DMA_ALIGN_SHIFT) << MIC_DMA_MEMCPY_LEN_SHIFT;
57 qw1 = MIC_DMA_MEMCPY;
58 qw1 <<= MIC_DMA_DESC_TYPE_SHIFT;
59 qw1 |= dst_phys;
60 desc->qw0 = qw0;
61 desc->qw1 = qw1;
62 }
63
64 /* Prepare a status desc. with @data to be written at @dst_phys */
mic_dma_prep_status_desc(struct mic_dma_desc * desc,u64 data,dma_addr_t dst_phys,bool generate_intr)65 static inline void mic_dma_prep_status_desc(struct mic_dma_desc *desc, u64 data,
66 dma_addr_t dst_phys, bool generate_intr)
67 {
68 u64 qw0, qw1;
69
70 qw0 = data;
71 qw1 = (u64) MIC_DMA_STATUS << MIC_DMA_DESC_TYPE_SHIFT | dst_phys;
72 if (generate_intr)
73 qw1 |= (1ULL << MIC_DMA_STAT_INTR_SHIFT);
74 desc->qw0 = qw0;
75 desc->qw1 = qw1;
76 }
77
mic_dma_cleanup(struct mic_dma_chan * ch)78 static void mic_dma_cleanup(struct mic_dma_chan *ch)
79 {
80 struct dma_async_tx_descriptor *tx;
81 u32 tail;
82 u32 last_tail;
83
84 spin_lock(&ch->cleanup_lock);
85 tail = mic_dma_read_cmp_cnt(ch);
86 /*
87 * This is the barrier pair for smp_wmb() in fn.
88 * mic_dma_tx_submit_unlock. It's required so that we read the
89 * updated cookie value from tx->cookie.
90 */
91 smp_rmb();
92 for (last_tail = ch->last_tail; tail != last_tail;) {
93 tx = &ch->tx_array[last_tail];
94 if (tx->cookie) {
95 dma_cookie_complete(tx);
96 dmaengine_desc_get_callback_invoke(tx, NULL);
97 tx->callback = NULL;
98 }
99 last_tail = mic_dma_hw_ring_inc(last_tail);
100 }
101 /* finish all completion callbacks before incrementing tail */
102 smp_mb();
103 ch->last_tail = last_tail;
104 spin_unlock(&ch->cleanup_lock);
105 }
106
mic_dma_ring_count(u32 head,u32 tail)107 static u32 mic_dma_ring_count(u32 head, u32 tail)
108 {
109 u32 count;
110
111 if (head >= tail)
112 count = (tail - 0) + (MIC_DMA_DESC_RX_SIZE - head);
113 else
114 count = tail - head;
115 return count - 1;
116 }
117
118 /* Returns the num. of free descriptors on success, -ENOMEM on failure */
mic_dma_avail_desc_ring_space(struct mic_dma_chan * ch,int required)119 static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required)
120 {
121 struct device *dev = mic_dma_ch_to_device(ch);
122 u32 count;
123
124 count = mic_dma_ring_count(ch->head, ch->last_tail);
125 if (count < required) {
126 mic_dma_cleanup(ch);
127 count = mic_dma_ring_count(ch->head, ch->last_tail);
128 }
129
130 if (count < required) {
131 dev_dbg(dev, "Not enough desc space");
132 dev_dbg(dev, "%s %d required=%u, avail=%u\n",
133 __func__, __LINE__, required, count);
134 return -ENOMEM;
135 } else {
136 return count;
137 }
138 }
139
140 /* Program memcpy descriptors into the descriptor ring and update s/w head ptr*/
mic_dma_prog_memcpy_desc(struct mic_dma_chan * ch,dma_addr_t src,dma_addr_t dst,size_t len)141 static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src,
142 dma_addr_t dst, size_t len)
143 {
144 size_t current_transfer_len;
145 size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size;
146 /* 3 is added to make sure we have enough space for status desc */
147 int num_desc = len / max_xfer_size + 3;
148 int ret;
149
150 if (len % max_xfer_size)
151 num_desc++;
152
153 ret = mic_dma_avail_desc_ring_space(ch, num_desc);
154 if (ret < 0)
155 return ret;
156 do {
157 current_transfer_len = min(len, max_xfer_size);
158 mic_dma_memcpy_desc(&ch->desc_ring[ch->head],
159 src, dst, current_transfer_len);
160 mic_dma_hw_ring_inc_head(ch);
161 len -= current_transfer_len;
162 dst = dst + current_transfer_len;
163 src = src + current_transfer_len;
164 } while (len > 0);
165 return 0;
166 }
167
168 /* It's a h/w quirk and h/w needs 2 status descriptors for every status desc */
mic_dma_prog_intr(struct mic_dma_chan * ch)169 static void mic_dma_prog_intr(struct mic_dma_chan *ch)
170 {
171 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
172 ch->status_dest_micpa, false);
173 mic_dma_hw_ring_inc_head(ch);
174 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
175 ch->status_dest_micpa, true);
176 mic_dma_hw_ring_inc_head(ch);
177 }
178
179 /* Wrapper function to program memcpy descriptors/status descriptors */
mic_dma_do_dma(struct mic_dma_chan * ch,int flags,dma_addr_t src,dma_addr_t dst,size_t len)180 static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src,
181 dma_addr_t dst, size_t len)
182 {
183 if (len && -ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len)) {
184 return -ENOMEM;
185 } else {
186 /* 3 is the maximum number of status descriptors */
187 int ret = mic_dma_avail_desc_ring_space(ch, 3);
188
189 if (ret < 0)
190 return ret;
191 }
192
193 /* Above mic_dma_prog_memcpy_desc() makes sure we have enough space */
194 if (flags & DMA_PREP_FENCE) {
195 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
196 ch->status_dest_micpa, false);
197 mic_dma_hw_ring_inc_head(ch);
198 }
199
200 if (flags & DMA_PREP_INTERRUPT)
201 mic_dma_prog_intr(ch);
202
203 return 0;
204 }
205
mic_dma_issue_pending(struct dma_chan * ch)206 static inline void mic_dma_issue_pending(struct dma_chan *ch)
207 {
208 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
209
210 spin_lock(&mic_ch->issue_lock);
211 /*
212 * Write to head triggers h/w to act on the descriptors.
213 * On MIC, writing the same head value twice causes
214 * a h/w error. On second write, h/w assumes we filled
215 * the entire ring & overwrote some of the descriptors.
216 */
217 if (mic_ch->issued == mic_ch->submitted)
218 goto out;
219 mic_ch->issued = mic_ch->submitted;
220 /*
221 * make descriptor updates visible before advancing head,
222 * this is purposefully not smp_wmb() since we are also
223 * publishing the descriptor updates to a dma device
224 */
225 wmb();
226 mic_dma_write_reg(mic_ch, MIC_DMA_REG_DHPR, mic_ch->issued);
227 out:
228 spin_unlock(&mic_ch->issue_lock);
229 }
230
mic_dma_update_pending(struct mic_dma_chan * ch)231 static inline void mic_dma_update_pending(struct mic_dma_chan *ch)
232 {
233 if (mic_dma_ring_count(ch->issued, ch->submitted)
234 > mic_dma_pending_level)
235 mic_dma_issue_pending(&ch->api_ch);
236 }
237
mic_dma_tx_submit_unlock(struct dma_async_tx_descriptor * tx)238 static dma_cookie_t mic_dma_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
239 {
240 struct mic_dma_chan *mic_ch = to_mic_dma_chan(tx->chan);
241 dma_cookie_t cookie;
242
243 dma_cookie_assign(tx);
244 cookie = tx->cookie;
245 /*
246 * We need an smp write barrier here because another CPU might see
247 * an update to submitted and update h/w head even before we
248 * assigned a cookie to this tx.
249 */
250 smp_wmb();
251 mic_ch->submitted = mic_ch->head;
252 spin_unlock(&mic_ch->prep_lock);
253 mic_dma_update_pending(mic_ch);
254 return cookie;
255 }
256
257 static inline struct dma_async_tx_descriptor *
allocate_tx(struct mic_dma_chan * ch)258 allocate_tx(struct mic_dma_chan *ch)
259 {
260 u32 idx = mic_dma_hw_ring_dec(ch->head);
261 struct dma_async_tx_descriptor *tx = &ch->tx_array[idx];
262
263 dma_async_tx_descriptor_init(tx, &ch->api_ch);
264 tx->tx_submit = mic_dma_tx_submit_unlock;
265 return tx;
266 }
267
268 /* Program a status descriptor with dst as address and value to be written */
269 static struct dma_async_tx_descriptor *
mic_dma_prep_status_lock(struct dma_chan * ch,dma_addr_t dst,u64 src_val,unsigned long flags)270 mic_dma_prep_status_lock(struct dma_chan *ch, dma_addr_t dst, u64 src_val,
271 unsigned long flags)
272 {
273 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
274 int result;
275
276 spin_lock(&mic_ch->prep_lock);
277 result = mic_dma_avail_desc_ring_space(mic_ch, 4);
278 if (result < 0)
279 goto error;
280 mic_dma_prep_status_desc(&mic_ch->desc_ring[mic_ch->head], src_val, dst,
281 false);
282 mic_dma_hw_ring_inc_head(mic_ch);
283 result = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
284 if (result < 0)
285 goto error;
286
287 return allocate_tx(mic_ch);
288 error:
289 dev_err(mic_dma_ch_to_device(mic_ch),
290 "Error enqueueing dma status descriptor, error=%d\n", result);
291 spin_unlock(&mic_ch->prep_lock);
292 return NULL;
293 }
294
295 /*
296 * Prepare a memcpy descriptor to be added to the ring.
297 * Note that the temporary descriptor adds an extra overhead of copying the
298 * descriptor to ring. So, we copy directly to the descriptor ring
299 */
300 static struct dma_async_tx_descriptor *
mic_dma_prep_memcpy_lock(struct dma_chan * ch,dma_addr_t dma_dest,dma_addr_t dma_src,size_t len,unsigned long flags)301 mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest,
302 dma_addr_t dma_src, size_t len, unsigned long flags)
303 {
304 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
305 struct device *dev = mic_dma_ch_to_device(mic_ch);
306 int result;
307
308 if (!len && !flags)
309 return NULL;
310
311 spin_lock(&mic_ch->prep_lock);
312 result = mic_dma_do_dma(mic_ch, flags, dma_src, dma_dest, len);
313 if (result >= 0)
314 return allocate_tx(mic_ch);
315 dev_err(dev, "Error enqueueing dma, error=%d\n", result);
316 spin_unlock(&mic_ch->prep_lock);
317 return NULL;
318 }
319
320 static struct dma_async_tx_descriptor *
mic_dma_prep_interrupt_lock(struct dma_chan * ch,unsigned long flags)321 mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags)
322 {
323 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
324 int ret;
325
326 spin_lock(&mic_ch->prep_lock);
327 ret = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
328 if (!ret)
329 return allocate_tx(mic_ch);
330 spin_unlock(&mic_ch->prep_lock);
331 return NULL;
332 }
333
334 /* Return the status of the transaction */
335 static enum dma_status
mic_dma_tx_status(struct dma_chan * ch,dma_cookie_t cookie,struct dma_tx_state * txstate)336 mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie,
337 struct dma_tx_state *txstate)
338 {
339 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
340
341 if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate))
342 mic_dma_cleanup(mic_ch);
343
344 return dma_cookie_status(ch, cookie, txstate);
345 }
346
mic_dma_thread_fn(int irq,void * data)347 static irqreturn_t mic_dma_thread_fn(int irq, void *data)
348 {
349 mic_dma_cleanup((struct mic_dma_chan *)data);
350 return IRQ_HANDLED;
351 }
352
mic_dma_intr_handler(int irq,void * data)353 static irqreturn_t mic_dma_intr_handler(int irq, void *data)
354 {
355 struct mic_dma_chan *ch = ((struct mic_dma_chan *)data);
356
357 mic_dma_ack_interrupt(ch);
358 return IRQ_WAKE_THREAD;
359 }
360
mic_dma_alloc_desc_ring(struct mic_dma_chan * ch)361 static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch)
362 {
363 u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
364 struct device *dev = &to_mbus_device(ch)->dev;
365
366 desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
367 ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL);
368
369 if (!ch->desc_ring)
370 return -ENOMEM;
371
372 ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring,
373 desc_ring_size, DMA_BIDIRECTIONAL);
374 if (dma_mapping_error(dev, ch->desc_ring_micpa))
375 goto map_error;
376
377 ch->tx_array = vzalloc(array_size(MIC_DMA_DESC_RX_SIZE,
378 sizeof(*ch->tx_array)));
379 if (!ch->tx_array)
380 goto tx_error;
381 return 0;
382 tx_error:
383 dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size,
384 DMA_BIDIRECTIONAL);
385 map_error:
386 kfree(ch->desc_ring);
387 return -ENOMEM;
388 }
389
mic_dma_free_desc_ring(struct mic_dma_chan * ch)390 static void mic_dma_free_desc_ring(struct mic_dma_chan *ch)
391 {
392 u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
393
394 vfree(ch->tx_array);
395 desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
396 dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa,
397 desc_ring_size, DMA_BIDIRECTIONAL);
398 kfree(ch->desc_ring);
399 ch->desc_ring = NULL;
400 }
401
mic_dma_free_status_dest(struct mic_dma_chan * ch)402 static void mic_dma_free_status_dest(struct mic_dma_chan *ch)
403 {
404 dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa,
405 L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
406 kfree(ch->status_dest);
407 }
408
mic_dma_alloc_status_dest(struct mic_dma_chan * ch)409 static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch)
410 {
411 struct device *dev = &to_mbus_device(ch)->dev;
412
413 ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL);
414 if (!ch->status_dest)
415 return -ENOMEM;
416 ch->status_dest_micpa = dma_map_single(dev, ch->status_dest,
417 L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
418 if (dma_mapping_error(dev, ch->status_dest_micpa)) {
419 kfree(ch->status_dest);
420 ch->status_dest = NULL;
421 return -ENOMEM;
422 }
423 return 0;
424 }
425
mic_dma_check_chan(struct mic_dma_chan * ch)426 static int mic_dma_check_chan(struct mic_dma_chan *ch)
427 {
428 if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) ||
429 mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) {
430 mic_dma_disable_chan(ch);
431 mic_dma_chan_mask_intr(ch);
432 dev_err(mic_dma_ch_to_device(ch),
433 "%s %d error setting up mic dma chan %d\n",
434 __func__, __LINE__, ch->ch_num);
435 return -EBUSY;
436 }
437 return 0;
438 }
439
mic_dma_chan_setup(struct mic_dma_chan * ch)440 static int mic_dma_chan_setup(struct mic_dma_chan *ch)
441 {
442 if (MIC_DMA_CHAN_MIC == ch->owner)
443 mic_dma_chan_set_owner(ch);
444 mic_dma_disable_chan(ch);
445 mic_dma_chan_mask_intr(ch);
446 mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0);
447 mic_dma_chan_set_desc_ring(ch);
448 ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR);
449 ch->head = ch->last_tail;
450 ch->issued = 0;
451 mic_dma_chan_unmask_intr(ch);
452 mic_dma_enable_chan(ch);
453 return mic_dma_check_chan(ch);
454 }
455
mic_dma_chan_destroy(struct mic_dma_chan * ch)456 static void mic_dma_chan_destroy(struct mic_dma_chan *ch)
457 {
458 mic_dma_disable_chan(ch);
459 mic_dma_chan_mask_intr(ch);
460 }
461
mic_dma_setup_irq(struct mic_dma_chan * ch)462 static int mic_dma_setup_irq(struct mic_dma_chan *ch)
463 {
464 ch->cookie =
465 to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),
466 mic_dma_intr_handler, mic_dma_thread_fn,
467 "mic dma_channel", ch, ch->ch_num);
468 return PTR_ERR_OR_ZERO(ch->cookie);
469 }
470
mic_dma_free_irq(struct mic_dma_chan * ch)471 static inline void mic_dma_free_irq(struct mic_dma_chan *ch)
472 {
473 to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch);
474 }
475
mic_dma_chan_init(struct mic_dma_chan * ch)476 static int mic_dma_chan_init(struct mic_dma_chan *ch)
477 {
478 int ret = mic_dma_alloc_desc_ring(ch);
479
480 if (ret)
481 goto ring_error;
482 ret = mic_dma_alloc_status_dest(ch);
483 if (ret)
484 goto status_error;
485 ret = mic_dma_chan_setup(ch);
486 if (ret)
487 goto chan_error;
488 return ret;
489 chan_error:
490 mic_dma_free_status_dest(ch);
491 status_error:
492 mic_dma_free_desc_ring(ch);
493 ring_error:
494 return ret;
495 }
496
mic_dma_drain_chan(struct mic_dma_chan * ch)497 static int mic_dma_drain_chan(struct mic_dma_chan *ch)
498 {
499 struct dma_async_tx_descriptor *tx;
500 int err = 0;
501 dma_cookie_t cookie;
502
503 tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE);
504 if (!tx) {
505 err = -ENOMEM;
506 goto error;
507 }
508
509 cookie = tx->tx_submit(tx);
510 if (dma_submit_error(cookie))
511 err = -ENOMEM;
512 else
513 err = dma_sync_wait(&ch->api_ch, cookie);
514 if (err) {
515 dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n",
516 __func__, __LINE__, ch->ch_num);
517 err = -EIO;
518 }
519 error:
520 mic_dma_cleanup(ch);
521 return err;
522 }
523
mic_dma_chan_uninit(struct mic_dma_chan * ch)524 static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch)
525 {
526 mic_dma_chan_destroy(ch);
527 mic_dma_cleanup(ch);
528 mic_dma_free_status_dest(ch);
529 mic_dma_free_desc_ring(ch);
530 }
531
mic_dma_init(struct mic_dma_device * mic_dma_dev,enum mic_dma_chan_owner owner)532 static int mic_dma_init(struct mic_dma_device *mic_dma_dev,
533 enum mic_dma_chan_owner owner)
534 {
535 int i, first_chan = mic_dma_dev->start_ch;
536 struct mic_dma_chan *ch;
537 int ret;
538
539 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
540 ch = &mic_dma_dev->mic_ch[i];
541 ch->ch_num = i;
542 ch->owner = owner;
543 spin_lock_init(&ch->cleanup_lock);
544 spin_lock_init(&ch->prep_lock);
545 spin_lock_init(&ch->issue_lock);
546 ret = mic_dma_setup_irq(ch);
547 if (ret)
548 goto error;
549 }
550 return 0;
551 error:
552 for (i = i - 1; i >= first_chan; i--)
553 mic_dma_free_irq(ch);
554 return ret;
555 }
556
mic_dma_uninit(struct mic_dma_device * mic_dma_dev)557 static void mic_dma_uninit(struct mic_dma_device *mic_dma_dev)
558 {
559 int i, first_chan = mic_dma_dev->start_ch;
560 struct mic_dma_chan *ch;
561
562 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
563 ch = &mic_dma_dev->mic_ch[i];
564 mic_dma_free_irq(ch);
565 }
566 }
567
mic_dma_alloc_chan_resources(struct dma_chan * ch)568 static int mic_dma_alloc_chan_resources(struct dma_chan *ch)
569 {
570 int ret = mic_dma_chan_init(to_mic_dma_chan(ch));
571 if (ret)
572 return ret;
573 return MIC_DMA_DESC_RX_SIZE;
574 }
575
mic_dma_free_chan_resources(struct dma_chan * ch)576 static void mic_dma_free_chan_resources(struct dma_chan *ch)
577 {
578 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
579 mic_dma_drain_chan(mic_ch);
580 mic_dma_chan_uninit(mic_ch);
581 }
582
583 /* Set the fn. handlers and register the dma device with dma api */
mic_dma_register_dma_device(struct mic_dma_device * mic_dma_dev,enum mic_dma_chan_owner owner)584 static int mic_dma_register_dma_device(struct mic_dma_device *mic_dma_dev,
585 enum mic_dma_chan_owner owner)
586 {
587 int i, first_chan = mic_dma_dev->start_ch;
588
589 dma_cap_zero(mic_dma_dev->dma_dev.cap_mask);
590 /*
591 * This dma engine is not capable of host memory to host memory
592 * transfers
593 */
594 dma_cap_set(DMA_MEMCPY, mic_dma_dev->dma_dev.cap_mask);
595
596 if (MIC_DMA_CHAN_HOST == owner)
597 dma_cap_set(DMA_PRIVATE, mic_dma_dev->dma_dev.cap_mask);
598 mic_dma_dev->dma_dev.device_alloc_chan_resources =
599 mic_dma_alloc_chan_resources;
600 mic_dma_dev->dma_dev.device_free_chan_resources =
601 mic_dma_free_chan_resources;
602 mic_dma_dev->dma_dev.device_tx_status = mic_dma_tx_status;
603 mic_dma_dev->dma_dev.device_prep_dma_memcpy = mic_dma_prep_memcpy_lock;
604 mic_dma_dev->dma_dev.device_prep_dma_imm_data =
605 mic_dma_prep_status_lock;
606 mic_dma_dev->dma_dev.device_prep_dma_interrupt =
607 mic_dma_prep_interrupt_lock;
608 mic_dma_dev->dma_dev.device_issue_pending = mic_dma_issue_pending;
609 mic_dma_dev->dma_dev.copy_align = MIC_DMA_ALIGN_SHIFT;
610 INIT_LIST_HEAD(&mic_dma_dev->dma_dev.channels);
611 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
612 mic_dma_dev->mic_ch[i].api_ch.device = &mic_dma_dev->dma_dev;
613 dma_cookie_init(&mic_dma_dev->mic_ch[i].api_ch);
614 list_add_tail(&mic_dma_dev->mic_ch[i].api_ch.device_node,
615 &mic_dma_dev->dma_dev.channels);
616 }
617 return dmaenginem_async_device_register(&mic_dma_dev->dma_dev);
618 }
619
620 /*
621 * Initializes dma channels and registers the dma device with the
622 * dma engine api.
623 */
mic_dma_dev_reg(struct mbus_device * mbdev,enum mic_dma_chan_owner owner)624 static struct mic_dma_device *mic_dma_dev_reg(struct mbus_device *mbdev,
625 enum mic_dma_chan_owner owner)
626 {
627 struct mic_dma_device *mic_dma_dev;
628 int ret;
629 struct device *dev = &mbdev->dev;
630
631 mic_dma_dev = devm_kzalloc(dev, sizeof(*mic_dma_dev), GFP_KERNEL);
632 if (!mic_dma_dev) {
633 ret = -ENOMEM;
634 goto alloc_error;
635 }
636 mic_dma_dev->mbdev = mbdev;
637 mic_dma_dev->dma_dev.dev = dev;
638 mic_dma_dev->mmio = mbdev->mmio_va;
639 if (MIC_DMA_CHAN_HOST == owner) {
640 mic_dma_dev->start_ch = 0;
641 mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_HOST;
642 } else {
643 mic_dma_dev->start_ch = 4;
644 mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_CARD;
645 }
646 ret = mic_dma_init(mic_dma_dev, owner);
647 if (ret)
648 goto init_error;
649 ret = mic_dma_register_dma_device(mic_dma_dev, owner);
650 if (ret)
651 goto reg_error;
652 return mic_dma_dev;
653 reg_error:
654 mic_dma_uninit(mic_dma_dev);
655 init_error:
656 mic_dma_dev = NULL;
657 alloc_error:
658 dev_err(dev, "Error at %s %d ret=%d\n", __func__, __LINE__, ret);
659 return mic_dma_dev;
660 }
661
mic_dma_dev_unreg(struct mic_dma_device * mic_dma_dev)662 static void mic_dma_dev_unreg(struct mic_dma_device *mic_dma_dev)
663 {
664 mic_dma_uninit(mic_dma_dev);
665 }
666
667 /* DEBUGFS CODE */
mic_dma_reg_show(struct seq_file * s,void * pos)668 static int mic_dma_reg_show(struct seq_file *s, void *pos)
669 {
670 struct mic_dma_device *mic_dma_dev = s->private;
671 int i, chan_num, first_chan = mic_dma_dev->start_ch;
672 struct mic_dma_chan *ch;
673
674 seq_printf(s, "SBOX_DCR: %#x\n",
675 mic_dma_mmio_read(&mic_dma_dev->mic_ch[first_chan],
676 MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR));
677 seq_puts(s, "DMA Channel Registers\n");
678 seq_printf(s, "%-10s| %-10s %-10s %-10s %-10s %-10s",
679 "Channel", "DCAR", "DTPR", "DHPR", "DRAR_HI", "DRAR_LO");
680 seq_printf(s, " %-11s %-14s %-10s\n", "DCHERR", "DCHERRMSK", "DSTAT");
681 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
682 ch = &mic_dma_dev->mic_ch[i];
683 chan_num = ch->ch_num;
684 seq_printf(s, "%-10i| %-#10x %-#10x %-#10x %-#10x",
685 chan_num,
686 mic_dma_read_reg(ch, MIC_DMA_REG_DCAR),
687 mic_dma_read_reg(ch, MIC_DMA_REG_DTPR),
688 mic_dma_read_reg(ch, MIC_DMA_REG_DHPR),
689 mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI));
690 seq_printf(s, " %-#10x %-#10x %-#14x %-#10x\n",
691 mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO),
692 mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR),
693 mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK),
694 mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT));
695 }
696 return 0;
697 }
698
699 DEFINE_SHOW_ATTRIBUTE(mic_dma_reg);
700
701 /* Debugfs parent dir */
702 static struct dentry *mic_dma_dbg;
703
mic_dma_driver_probe(struct mbus_device * mbdev)704 static int mic_dma_driver_probe(struct mbus_device *mbdev)
705 {
706 struct mic_dma_device *mic_dma_dev;
707 enum mic_dma_chan_owner owner;
708
709 if (MBUS_DEV_DMA_MIC == mbdev->id.device)
710 owner = MIC_DMA_CHAN_MIC;
711 else
712 owner = MIC_DMA_CHAN_HOST;
713
714 mic_dma_dev = mic_dma_dev_reg(mbdev, owner);
715 dev_set_drvdata(&mbdev->dev, mic_dma_dev);
716
717 if (mic_dma_dbg) {
718 mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev),
719 mic_dma_dbg);
720 debugfs_create_file("mic_dma_reg", 0444, mic_dma_dev->dbg_dir,
721 mic_dma_dev, &mic_dma_reg_fops);
722 }
723 return 0;
724 }
725
mic_dma_driver_remove(struct mbus_device * mbdev)726 static void mic_dma_driver_remove(struct mbus_device *mbdev)
727 {
728 struct mic_dma_device *mic_dma_dev;
729
730 mic_dma_dev = dev_get_drvdata(&mbdev->dev);
731 debugfs_remove_recursive(mic_dma_dev->dbg_dir);
732 mic_dma_dev_unreg(mic_dma_dev);
733 }
734
735 static struct mbus_device_id id_table[] = {
736 {MBUS_DEV_DMA_MIC, MBUS_DEV_ANY_ID},
737 {MBUS_DEV_DMA_HOST, MBUS_DEV_ANY_ID},
738 {0},
739 };
740
741 static struct mbus_driver mic_dma_driver = {
742 .driver.name = KBUILD_MODNAME,
743 .driver.owner = THIS_MODULE,
744 .id_table = id_table,
745 .probe = mic_dma_driver_probe,
746 .remove = mic_dma_driver_remove,
747 };
748
mic_x100_dma_init(void)749 static int __init mic_x100_dma_init(void)
750 {
751 int rc = mbus_register_driver(&mic_dma_driver);
752 if (rc)
753 return rc;
754 mic_dma_dbg = debugfs_create_dir(KBUILD_MODNAME, NULL);
755 return 0;
756 }
757
mic_x100_dma_exit(void)758 static void __exit mic_x100_dma_exit(void)
759 {
760 debugfs_remove_recursive(mic_dma_dbg);
761 mbus_unregister_driver(&mic_dma_driver);
762 }
763
764 module_init(mic_x100_dma_init);
765 module_exit(mic_x100_dma_exit);
766
767 MODULE_DEVICE_TABLE(mbus, id_table);
768 MODULE_AUTHOR("Intel Corporation");
769 MODULE_DESCRIPTION("Intel(R) MIC X100 DMA Driver");
770 MODULE_LICENSE("GPL v2");
771