1 /*
2 * Copyright © 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
16 *
17 */
18 #ifndef _ADMA_H
19 #define _ADMA_H
20 #include <linux/types.h>
21 #include <linux/io.h>
22 #include <mach/hardware.h>
23 #include <asm/hardware/iop_adma.h>
24
25 /* Memory copy units */
26 #define DMA_CCR(chan) (chan->mmr_base + 0x0)
27 #define DMA_CSR(chan) (chan->mmr_base + 0x4)
28 #define DMA_DAR(chan) (chan->mmr_base + 0xc)
29 #define DMA_NDAR(chan) (chan->mmr_base + 0x10)
30 #define DMA_PADR(chan) (chan->mmr_base + 0x14)
31 #define DMA_PUADR(chan) (chan->mmr_base + 0x18)
32 #define DMA_LADR(chan) (chan->mmr_base + 0x1c)
33 #define DMA_BCR(chan) (chan->mmr_base + 0x20)
34 #define DMA_DCR(chan) (chan->mmr_base + 0x24)
35
36 /* Application accelerator unit */
37 #define AAU_ACR(chan) (chan->mmr_base + 0x0)
38 #define AAU_ASR(chan) (chan->mmr_base + 0x4)
39 #define AAU_ADAR(chan) (chan->mmr_base + 0x8)
40 #define AAU_ANDAR(chan) (chan->mmr_base + 0xc)
41 #define AAU_SAR(src, chan) (chan->mmr_base + (0x10 + ((src) << 2)))
42 #define AAU_DAR(chan) (chan->mmr_base + 0x20)
43 #define AAU_ABCR(chan) (chan->mmr_base + 0x24)
44 #define AAU_ADCR(chan) (chan->mmr_base + 0x28)
45 #define AAU_SAR_EDCR(src_edc) (chan->mmr_base + (0x02c + ((src_edc-4) << 2)))
46 #define AAU_EDCR0_IDX 8
47 #define AAU_EDCR1_IDX 17
48 #define AAU_EDCR2_IDX 26
49
50 #define DMA0_ID 0
51 #define DMA1_ID 1
52 #define AAU_ID 2
53
54 struct iop3xx_aau_desc_ctrl {
55 unsigned int int_en:1;
56 unsigned int blk1_cmd_ctrl:3;
57 unsigned int blk2_cmd_ctrl:3;
58 unsigned int blk3_cmd_ctrl:3;
59 unsigned int blk4_cmd_ctrl:3;
60 unsigned int blk5_cmd_ctrl:3;
61 unsigned int blk6_cmd_ctrl:3;
62 unsigned int blk7_cmd_ctrl:3;
63 unsigned int blk8_cmd_ctrl:3;
64 unsigned int blk_ctrl:2;
65 unsigned int dual_xor_en:1;
66 unsigned int tx_complete:1;
67 unsigned int zero_result_err:1;
68 unsigned int zero_result_en:1;
69 unsigned int dest_write_en:1;
70 };
71
72 struct iop3xx_aau_e_desc_ctrl {
73 unsigned int reserved:1;
74 unsigned int blk1_cmd_ctrl:3;
75 unsigned int blk2_cmd_ctrl:3;
76 unsigned int blk3_cmd_ctrl:3;
77 unsigned int blk4_cmd_ctrl:3;
78 unsigned int blk5_cmd_ctrl:3;
79 unsigned int blk6_cmd_ctrl:3;
80 unsigned int blk7_cmd_ctrl:3;
81 unsigned int blk8_cmd_ctrl:3;
82 unsigned int reserved2:7;
83 };
84
85 struct iop3xx_dma_desc_ctrl {
86 unsigned int pci_transaction:4;
87 unsigned int int_en:1;
88 unsigned int dac_cycle_en:1;
89 unsigned int mem_to_mem_en:1;
90 unsigned int crc_data_tx_en:1;
91 unsigned int crc_gen_en:1;
92 unsigned int crc_seed_dis:1;
93 unsigned int reserved:21;
94 unsigned int crc_tx_complete:1;
95 };
96
97 struct iop3xx_desc_dma {
98 u32 next_desc;
99 union {
100 u32 pci_src_addr;
101 u32 pci_dest_addr;
102 u32 src_addr;
103 };
104 union {
105 u32 upper_pci_src_addr;
106 u32 upper_pci_dest_addr;
107 };
108 union {
109 u32 local_pci_src_addr;
110 u32 local_pci_dest_addr;
111 u32 dest_addr;
112 };
113 u32 byte_count;
114 union {
115 u32 desc_ctrl;
116 struct iop3xx_dma_desc_ctrl desc_ctrl_field;
117 };
118 u32 crc_addr;
119 };
120
121 struct iop3xx_desc_aau {
122 u32 next_desc;
123 u32 src[4];
124 u32 dest_addr;
125 u32 byte_count;
126 union {
127 u32 desc_ctrl;
128 struct iop3xx_aau_desc_ctrl desc_ctrl_field;
129 };
130 union {
131 u32 src_addr;
132 u32 e_desc_ctrl;
133 struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
134 } src_edc[31];
135 };
136
137 struct iop3xx_aau_gfmr {
138 unsigned int gfmr1:8;
139 unsigned int gfmr2:8;
140 unsigned int gfmr3:8;
141 unsigned int gfmr4:8;
142 };
143
144 struct iop3xx_desc_pq_xor {
145 u32 next_desc;
146 u32 src[3];
147 union {
148 u32 data_mult1;
149 struct iop3xx_aau_gfmr data_mult1_field;
150 };
151 u32 dest_addr;
152 u32 byte_count;
153 union {
154 u32 desc_ctrl;
155 struct iop3xx_aau_desc_ctrl desc_ctrl_field;
156 };
157 union {
158 u32 src_addr;
159 u32 e_desc_ctrl;
160 struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
161 u32 data_multiplier;
162 struct iop3xx_aau_gfmr data_mult_field;
163 u32 reserved;
164 } src_edc_gfmr[19];
165 };
166
167 struct iop3xx_desc_dual_xor {
168 u32 next_desc;
169 u32 src0_addr;
170 u32 src1_addr;
171 u32 h_src_addr;
172 u32 d_src_addr;
173 u32 h_dest_addr;
174 u32 byte_count;
175 union {
176 u32 desc_ctrl;
177 struct iop3xx_aau_desc_ctrl desc_ctrl_field;
178 };
179 u32 d_dest_addr;
180 };
181
182 union iop3xx_desc {
183 struct iop3xx_desc_aau *aau;
184 struct iop3xx_desc_dma *dma;
185 struct iop3xx_desc_pq_xor *pq_xor;
186 struct iop3xx_desc_dual_xor *dual_xor;
187 void *ptr;
188 };
189
190 /* No support for p+q operations */
191 static inline int
iop_chan_pq_slot_count(size_t len,int src_cnt,int * slots_per_op)192 iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op)
193 {
194 BUG();
195 return 0;
196 }
197
198 static inline void
iop_desc_init_pq(struct iop_adma_desc_slot * desc,int src_cnt,unsigned long flags)199 iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
200 unsigned long flags)
201 {
202 BUG();
203 }
204
205 static inline void
iop_desc_set_pq_addr(struct iop_adma_desc_slot * desc,dma_addr_t * addr)206 iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr)
207 {
208 BUG();
209 }
210
211 static inline void
iop_desc_set_pq_src_addr(struct iop_adma_desc_slot * desc,int src_idx,dma_addr_t addr,unsigned char coef)212 iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
213 dma_addr_t addr, unsigned char coef)
214 {
215 BUG();
216 }
217
218 static inline int
iop_chan_pq_zero_sum_slot_count(size_t len,int src_cnt,int * slots_per_op)219 iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op)
220 {
221 BUG();
222 return 0;
223 }
224
225 static inline void
iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot * desc,int src_cnt,unsigned long flags)226 iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
227 unsigned long flags)
228 {
229 BUG();
230 }
231
232 static inline void
iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot * desc,u32 len)233 iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
234 {
235 BUG();
236 }
237
238 #define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
239
240 static inline void
iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot * desc,int pq_idx,dma_addr_t * src)241 iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
242 dma_addr_t *src)
243 {
244 BUG();
245 }
246
iop_adma_get_max_xor(void)247 static inline int iop_adma_get_max_xor(void)
248 {
249 return 32;
250 }
251
iop_adma_get_max_pq(void)252 static inline int iop_adma_get_max_pq(void)
253 {
254 BUG();
255 return 0;
256 }
257
iop_chan_get_current_descriptor(struct iop_adma_chan * chan)258 static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
259 {
260 int id = chan->device->id;
261
262 switch (id) {
263 case DMA0_ID:
264 case DMA1_ID:
265 return __raw_readl(DMA_DAR(chan));
266 case AAU_ID:
267 return __raw_readl(AAU_ADAR(chan));
268 default:
269 BUG();
270 }
271 return 0;
272 }
273
iop_chan_set_next_descriptor(struct iop_adma_chan * chan,u32 next_desc_addr)274 static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan,
275 u32 next_desc_addr)
276 {
277 int id = chan->device->id;
278
279 switch (id) {
280 case DMA0_ID:
281 case DMA1_ID:
282 __raw_writel(next_desc_addr, DMA_NDAR(chan));
283 break;
284 case AAU_ID:
285 __raw_writel(next_desc_addr, AAU_ANDAR(chan));
286 break;
287 }
288
289 }
290
291 #define IOP_ADMA_STATUS_BUSY (1 << 10)
292 #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024)
293 #define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024)
294 #define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024)
295
iop_chan_is_busy(struct iop_adma_chan * chan)296 static inline int iop_chan_is_busy(struct iop_adma_chan *chan)
297 {
298 u32 status = __raw_readl(DMA_CSR(chan));
299 return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0;
300 }
301
iop_desc_is_aligned(struct iop_adma_desc_slot * desc,int num_slots)302 static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc,
303 int num_slots)
304 {
305 /* num_slots will only ever be 1, 2, 4, or 8 */
306 return (desc->idx & (num_slots - 1)) ? 0 : 1;
307 }
308
309 /* to do: support large (i.e. > hw max) buffer sizes */
iop_chan_memcpy_slot_count(size_t len,int * slots_per_op)310 static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op)
311 {
312 *slots_per_op = 1;
313 return 1;
314 }
315
316 /* to do: support large (i.e. > hw max) buffer sizes */
iop_chan_memset_slot_count(size_t len,int * slots_per_op)317 static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op)
318 {
319 *slots_per_op = 1;
320 return 1;
321 }
322
iop3xx_aau_xor_slot_count(size_t len,int src_cnt,int * slots_per_op)323 static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt,
324 int *slots_per_op)
325 {
326 static const char slot_count_table[] = {
327 1, 1, 1, 1, /* 01 - 04 */
328 2, 2, 2, 2, /* 05 - 08 */
329 4, 4, 4, 4, /* 09 - 12 */
330 4, 4, 4, 4, /* 13 - 16 */
331 8, 8, 8, 8, /* 17 - 20 */
332 8, 8, 8, 8, /* 21 - 24 */
333 8, 8, 8, 8, /* 25 - 28 */
334 8, 8, 8, 8, /* 29 - 32 */
335 };
336 *slots_per_op = slot_count_table[src_cnt - 1];
337 return *slots_per_op;
338 }
339
340 static inline int
iop_chan_interrupt_slot_count(int * slots_per_op,struct iop_adma_chan * chan)341 iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan)
342 {
343 switch (chan->device->id) {
344 case DMA0_ID:
345 case DMA1_ID:
346 return iop_chan_memcpy_slot_count(0, slots_per_op);
347 case AAU_ID:
348 return iop3xx_aau_xor_slot_count(0, 2, slots_per_op);
349 default:
350 BUG();
351 }
352 return 0;
353 }
354
iop_chan_xor_slot_count(size_t len,int src_cnt,int * slots_per_op)355 static inline int iop_chan_xor_slot_count(size_t len, int src_cnt,
356 int *slots_per_op)
357 {
358 int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
359
360 if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT)
361 return slot_cnt;
362
363 len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
364 while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) {
365 len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
366 slot_cnt += *slots_per_op;
367 }
368
369 slot_cnt += *slots_per_op;
370
371 return slot_cnt;
372 }
373
374 /* zero sum on iop3xx is limited to 1k at a time so it requires multiple
375 * descriptors
376 */
iop_chan_zero_sum_slot_count(size_t len,int src_cnt,int * slots_per_op)377 static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,
378 int *slots_per_op)
379 {
380 int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
381
382 if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT)
383 return slot_cnt;
384
385 len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
386 while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
387 len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
388 slot_cnt += *slots_per_op;
389 }
390
391 slot_cnt += *slots_per_op;
392
393 return slot_cnt;
394 }
395
iop_desc_get_byte_count(struct iop_adma_desc_slot * desc,struct iop_adma_chan * chan)396 static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
397 struct iop_adma_chan *chan)
398 {
399 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
400
401 switch (chan->device->id) {
402 case DMA0_ID:
403 case DMA1_ID:
404 return hw_desc.dma->byte_count;
405 case AAU_ID:
406 return hw_desc.aau->byte_count;
407 default:
408 BUG();
409 }
410 return 0;
411 }
412
413 /* translate the src_idx to a descriptor word index */
__desc_idx(int src_idx)414 static inline int __desc_idx(int src_idx)
415 {
416 static const int desc_idx_table[] = { 0, 0, 0, 0,
417 0, 1, 2, 3,
418 5, 6, 7, 8,
419 9, 10, 11, 12,
420 14, 15, 16, 17,
421 18, 19, 20, 21,
422 23, 24, 25, 26,
423 27, 28, 29, 30,
424 };
425
426 return desc_idx_table[src_idx];
427 }
428
iop_desc_get_src_addr(struct iop_adma_desc_slot * desc,struct iop_adma_chan * chan,int src_idx)429 static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc,
430 struct iop_adma_chan *chan,
431 int src_idx)
432 {
433 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
434
435 switch (chan->device->id) {
436 case DMA0_ID:
437 case DMA1_ID:
438 return hw_desc.dma->src_addr;
439 case AAU_ID:
440 break;
441 default:
442 BUG();
443 }
444
445 if (src_idx < 4)
446 return hw_desc.aau->src[src_idx];
447 else
448 return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr;
449 }
450
iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau * hw_desc,int src_idx,dma_addr_t addr)451 static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc,
452 int src_idx, dma_addr_t addr)
453 {
454 if (src_idx < 4)
455 hw_desc->src[src_idx] = addr;
456 else
457 hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr;
458 }
459
460 static inline void
iop_desc_init_memcpy(struct iop_adma_desc_slot * desc,unsigned long flags)461 iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags)
462 {
463 struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
464 union {
465 u32 value;
466 struct iop3xx_dma_desc_ctrl field;
467 } u_desc_ctrl;
468
469 u_desc_ctrl.value = 0;
470 u_desc_ctrl.field.mem_to_mem_en = 1;
471 u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */
472 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
473 hw_desc->desc_ctrl = u_desc_ctrl.value;
474 hw_desc->upper_pci_src_addr = 0;
475 hw_desc->crc_addr = 0;
476 }
477
478 static inline void
iop_desc_init_memset(struct iop_adma_desc_slot * desc,unsigned long flags)479 iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags)
480 {
481 struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
482 union {
483 u32 value;
484 struct iop3xx_aau_desc_ctrl field;
485 } u_desc_ctrl;
486
487 u_desc_ctrl.value = 0;
488 u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */
489 u_desc_ctrl.field.dest_write_en = 1;
490 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
491 hw_desc->desc_ctrl = u_desc_ctrl.value;
492 }
493
494 static inline u32
iop3xx_desc_init_xor(struct iop3xx_desc_aau * hw_desc,int src_cnt,unsigned long flags)495 iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt,
496 unsigned long flags)
497 {
498 int i, shift;
499 u32 edcr;
500 union {
501 u32 value;
502 struct iop3xx_aau_desc_ctrl field;
503 } u_desc_ctrl;
504
505 u_desc_ctrl.value = 0;
506 switch (src_cnt) {
507 case 25 ... 32:
508 u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
509 edcr = 0;
510 shift = 1;
511 for (i = 24; i < src_cnt; i++) {
512 edcr |= (1 << shift);
513 shift += 3;
514 }
515 hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr;
516 src_cnt = 24;
517 /* fall through */
518 case 17 ... 24:
519 if (!u_desc_ctrl.field.blk_ctrl) {
520 hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
521 u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
522 }
523 edcr = 0;
524 shift = 1;
525 for (i = 16; i < src_cnt; i++) {
526 edcr |= (1 << shift);
527 shift += 3;
528 }
529 hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr;
530 src_cnt = 16;
531 /* fall through */
532 case 9 ... 16:
533 if (!u_desc_ctrl.field.blk_ctrl)
534 u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
535 edcr = 0;
536 shift = 1;
537 for (i = 8; i < src_cnt; i++) {
538 edcr |= (1 << shift);
539 shift += 3;
540 }
541 hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr;
542 src_cnt = 8;
543 /* fall through */
544 case 2 ... 8:
545 shift = 1;
546 for (i = 0; i < src_cnt; i++) {
547 u_desc_ctrl.value |= (1 << shift);
548 shift += 3;
549 }
550
551 if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
552 u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
553 }
554
555 u_desc_ctrl.field.dest_write_en = 1;
556 u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */
557 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
558 hw_desc->desc_ctrl = u_desc_ctrl.value;
559
560 return u_desc_ctrl.value;
561 }
562
563 static inline void
iop_desc_init_xor(struct iop_adma_desc_slot * desc,int src_cnt,unsigned long flags)564 iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt,
565 unsigned long flags)
566 {
567 iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags);
568 }
569
570 /* return the number of operations */
571 static inline int
iop_desc_init_zero_sum(struct iop_adma_desc_slot * desc,int src_cnt,unsigned long flags)572 iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
573 unsigned long flags)
574 {
575 int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
576 struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter;
577 union {
578 u32 value;
579 struct iop3xx_aau_desc_ctrl field;
580 } u_desc_ctrl;
581 int i, j;
582
583 hw_desc = desc->hw_desc;
584
585 for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0;
586 i += slots_per_op, j++) {
587 iter = iop_hw_desc_slot_idx(hw_desc, i);
588 u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags);
589 u_desc_ctrl.field.dest_write_en = 0;
590 u_desc_ctrl.field.zero_result_en = 1;
591 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
592 iter->desc_ctrl = u_desc_ctrl.value;
593
594 /* for the subsequent descriptors preserve the store queue
595 * and chain them together
596 */
597 if (i) {
598 prev_hw_desc =
599 iop_hw_desc_slot_idx(hw_desc, i - slots_per_op);
600 prev_hw_desc->next_desc =
601 (u32) (desc->async_tx.phys + (i << 5));
602 }
603 }
604
605 return j;
606 }
607
608 static inline void
iop_desc_init_null_xor(struct iop_adma_desc_slot * desc,int src_cnt,unsigned long flags)609 iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt,
610 unsigned long flags)
611 {
612 struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
613 union {
614 u32 value;
615 struct iop3xx_aau_desc_ctrl field;
616 } u_desc_ctrl;
617
618 u_desc_ctrl.value = 0;
619 switch (src_cnt) {
620 case 25 ... 32:
621 u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
622 hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
623 /* fall through */
624 case 17 ... 24:
625 if (!u_desc_ctrl.field.blk_ctrl) {
626 hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
627 u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
628 }
629 hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0;
630 /* fall through */
631 case 9 ... 16:
632 if (!u_desc_ctrl.field.blk_ctrl)
633 u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
634 hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0;
635 /* fall through */
636 case 1 ... 8:
637 if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
638 u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
639 }
640
641 u_desc_ctrl.field.dest_write_en = 0;
642 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
643 hw_desc->desc_ctrl = u_desc_ctrl.value;
644 }
645
iop_desc_set_byte_count(struct iop_adma_desc_slot * desc,struct iop_adma_chan * chan,u32 byte_count)646 static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc,
647 struct iop_adma_chan *chan,
648 u32 byte_count)
649 {
650 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
651
652 switch (chan->device->id) {
653 case DMA0_ID:
654 case DMA1_ID:
655 hw_desc.dma->byte_count = byte_count;
656 break;
657 case AAU_ID:
658 hw_desc.aau->byte_count = byte_count;
659 break;
660 default:
661 BUG();
662 }
663 }
664
665 static inline void
iop_desc_init_interrupt(struct iop_adma_desc_slot * desc,struct iop_adma_chan * chan)666 iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
667 struct iop_adma_chan *chan)
668 {
669 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
670
671 switch (chan->device->id) {
672 case DMA0_ID:
673 case DMA1_ID:
674 iop_desc_init_memcpy(desc, 1);
675 hw_desc.dma->byte_count = 0;
676 hw_desc.dma->dest_addr = 0;
677 hw_desc.dma->src_addr = 0;
678 break;
679 case AAU_ID:
680 iop_desc_init_null_xor(desc, 2, 1);
681 hw_desc.aau->byte_count = 0;
682 hw_desc.aau->dest_addr = 0;
683 hw_desc.aau->src[0] = 0;
684 hw_desc.aau->src[1] = 0;
685 break;
686 default:
687 BUG();
688 }
689 }
690
691 static inline void
iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot * desc,u32 len)692 iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
693 {
694 int slots_per_op = desc->slots_per_op;
695 struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
696 int i = 0;
697
698 if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
699 hw_desc->byte_count = len;
700 } else {
701 do {
702 iter = iop_hw_desc_slot_idx(hw_desc, i);
703 iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
704 len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
705 i += slots_per_op;
706 } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT);
707
708 iter = iop_hw_desc_slot_idx(hw_desc, i);
709 iter->byte_count = len;
710 }
711 }
712
iop_desc_set_dest_addr(struct iop_adma_desc_slot * desc,struct iop_adma_chan * chan,dma_addr_t addr)713 static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
714 struct iop_adma_chan *chan,
715 dma_addr_t addr)
716 {
717 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
718
719 switch (chan->device->id) {
720 case DMA0_ID:
721 case DMA1_ID:
722 hw_desc.dma->dest_addr = addr;
723 break;
724 case AAU_ID:
725 hw_desc.aau->dest_addr = addr;
726 break;
727 default:
728 BUG();
729 }
730 }
731
iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot * desc,dma_addr_t addr)732 static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc,
733 dma_addr_t addr)
734 {
735 struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
736 hw_desc->src_addr = addr;
737 }
738
739 static inline void
iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot * desc,int src_idx,dma_addr_t addr)740 iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
741 dma_addr_t addr)
742 {
743
744 struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
745 int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
746 int i;
747
748 for (i = 0; (slot_cnt -= slots_per_op) >= 0;
749 i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
750 iter = iop_hw_desc_slot_idx(hw_desc, i);
751 iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
752 }
753 }
754
iop_desc_set_xor_src_addr(struct iop_adma_desc_slot * desc,int src_idx,dma_addr_t addr)755 static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc,
756 int src_idx, dma_addr_t addr)
757 {
758
759 struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
760 int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
761 int i;
762
763 for (i = 0; (slot_cnt -= slots_per_op) >= 0;
764 i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) {
765 iter = iop_hw_desc_slot_idx(hw_desc, i);
766 iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
767 }
768 }
769
iop_desc_set_next_desc(struct iop_adma_desc_slot * desc,u32 next_desc_addr)770 static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc,
771 u32 next_desc_addr)
772 {
773 /* hw_desc->next_desc is the same location for all channels */
774 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
775
776 iop_paranoia(hw_desc.dma->next_desc);
777 hw_desc.dma->next_desc = next_desc_addr;
778 }
779
iop_desc_get_next_desc(struct iop_adma_desc_slot * desc)780 static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc)
781 {
782 /* hw_desc->next_desc is the same location for all channels */
783 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
784 return hw_desc.dma->next_desc;
785 }
786
iop_desc_clear_next_desc(struct iop_adma_desc_slot * desc)787 static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc)
788 {
789 /* hw_desc->next_desc is the same location for all channels */
790 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
791 hw_desc.dma->next_desc = 0;
792 }
793
iop_desc_set_block_fill_val(struct iop_adma_desc_slot * desc,u32 val)794 static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc,
795 u32 val)
796 {
797 struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
798 hw_desc->src[0] = val;
799 }
800
801 static inline enum sum_check_flags
iop_desc_get_zero_result(struct iop_adma_desc_slot * desc)802 iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
803 {
804 struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
805 struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field;
806
807 iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en));
808 return desc_ctrl.zero_result_err << SUM_CHECK_P;
809 }
810
iop_chan_append(struct iop_adma_chan * chan)811 static inline void iop_chan_append(struct iop_adma_chan *chan)
812 {
813 u32 dma_chan_ctrl;
814
815 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
816 dma_chan_ctrl |= 0x2;
817 __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
818 }
819
iop_chan_get_status(struct iop_adma_chan * chan)820 static inline u32 iop_chan_get_status(struct iop_adma_chan *chan)
821 {
822 return __raw_readl(DMA_CSR(chan));
823 }
824
iop_chan_disable(struct iop_adma_chan * chan)825 static inline void iop_chan_disable(struct iop_adma_chan *chan)
826 {
827 u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
828 dma_chan_ctrl &= ~1;
829 __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
830 }
831
iop_chan_enable(struct iop_adma_chan * chan)832 static inline void iop_chan_enable(struct iop_adma_chan *chan)
833 {
834 u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
835
836 dma_chan_ctrl |= 1;
837 __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
838 }
839
iop_adma_device_clear_eot_status(struct iop_adma_chan * chan)840 static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan)
841 {
842 u32 status = __raw_readl(DMA_CSR(chan));
843 status &= (1 << 9);
844 __raw_writel(status, DMA_CSR(chan));
845 }
846
iop_adma_device_clear_eoc_status(struct iop_adma_chan * chan)847 static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan)
848 {
849 u32 status = __raw_readl(DMA_CSR(chan));
850 status &= (1 << 8);
851 __raw_writel(status, DMA_CSR(chan));
852 }
853
iop_adma_device_clear_err_status(struct iop_adma_chan * chan)854 static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan)
855 {
856 u32 status = __raw_readl(DMA_CSR(chan));
857
858 switch (chan->device->id) {
859 case DMA0_ID:
860 case DMA1_ID:
861 status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1);
862 break;
863 case AAU_ID:
864 status &= (1 << 5);
865 break;
866 default:
867 BUG();
868 }
869
870 __raw_writel(status, DMA_CSR(chan));
871 }
872
873 static inline int
iop_is_err_int_parity(unsigned long status,struct iop_adma_chan * chan)874 iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan)
875 {
876 return 0;
877 }
878
879 static inline int
iop_is_err_mcu_abort(unsigned long status,struct iop_adma_chan * chan)880 iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan)
881 {
882 return 0;
883 }
884
885 static inline int
iop_is_err_int_tabort(unsigned long status,struct iop_adma_chan * chan)886 iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan)
887 {
888 return 0;
889 }
890
891 static inline int
iop_is_err_int_mabort(unsigned long status,struct iop_adma_chan * chan)892 iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan)
893 {
894 return test_bit(5, &status);
895 }
896
897 static inline int
iop_is_err_pci_tabort(unsigned long status,struct iop_adma_chan * chan)898 iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan)
899 {
900 switch (chan->device->id) {
901 case DMA0_ID:
902 case DMA1_ID:
903 return test_bit(2, &status);
904 default:
905 return 0;
906 }
907 }
908
909 static inline int
iop_is_err_pci_mabort(unsigned long status,struct iop_adma_chan * chan)910 iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan)
911 {
912 switch (chan->device->id) {
913 case DMA0_ID:
914 case DMA1_ID:
915 return test_bit(3, &status);
916 default:
917 return 0;
918 }
919 }
920
921 static inline int
iop_is_err_split_tx(unsigned long status,struct iop_adma_chan * chan)922 iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan)
923 {
924 switch (chan->device->id) {
925 case DMA0_ID:
926 case DMA1_ID:
927 return test_bit(1, &status);
928 default:
929 return 0;
930 }
931 }
932 #endif /* _ADMA_H */
933