1 /*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
16 *
17 */
18
19 /*
20 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
21 * copy operations.
22 */
23
24 #include <linux/init.h>
25 #include <linux/module.h>
26 #include <linux/slab.h>
27 #include <linux/pci.h>
28 #include <linux/interrupt.h>
29 #include <linux/dmaengine.h>
30 #include <linux/delay.h>
31 #include <linux/dma-mapping.h>
32 #include <linux/workqueue.h>
33 #include <linux/prefetch.h>
34 #include <linux/sizes.h>
35 #include "dma.h"
36 #include "registers.h"
37 #include "hw.h"
38
39 #include "../dmaengine.h"
40
41 static char *chanerr_str[] = {
42 "DMA Transfer Source Address Error",
43 "DMA Transfer Destination Address Error",
44 "Next Descriptor Address Error",
45 "Descriptor Error",
46 "Chan Address Value Error",
47 "CHANCMD Error",
48 "Chipset Uncorrectable Data Integrity Error",
49 "DMA Uncorrectable Data Integrity Error",
50 "Read Data Error",
51 "Write Data Error",
52 "Descriptor Control Error",
53 "Descriptor Transfer Size Error",
54 "Completion Address Error",
55 "Interrupt Configuration Error",
56 "Super extended descriptor Address Error",
57 "Unaffiliated Error",
58 "CRC or XOR P Error",
59 "XOR Q Error",
60 "Descriptor Count Error",
61 "DIF All F detect Error",
62 "Guard Tag verification Error",
63 "Application Tag verification Error",
64 "Reference Tag verification Error",
65 "Bundle Bit Error",
66 "Result DIF All F detect Error",
67 "Result Guard Tag verification Error",
68 "Result Application Tag verification Error",
69 "Result Reference Tag verification Error",
70 };
71
72 static void ioat_eh(struct ioatdma_chan *ioat_chan);
73
ioat_print_chanerrs(struct ioatdma_chan * ioat_chan,u32 chanerr)74 static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
75 {
76 int i;
77
78 for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) {
79 if ((chanerr >> i) & 1) {
80 dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
81 i, chanerr_str[i]);
82 }
83 }
84 }
85
86 /**
87 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
88 * @irq: interrupt id
89 * @data: interrupt data
90 */
ioat_dma_do_interrupt(int irq,void * data)91 irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
92 {
93 struct ioatdma_device *instance = data;
94 struct ioatdma_chan *ioat_chan;
95 unsigned long attnstatus;
96 int bit;
97 u8 intrctrl;
98
99 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
100
101 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
102 return IRQ_NONE;
103
104 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
105 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
106 return IRQ_NONE;
107 }
108
109 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
110 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
111 ioat_chan = ioat_chan_by_index(instance, bit);
112 if (test_bit(IOAT_RUN, &ioat_chan->state))
113 tasklet_schedule(&ioat_chan->cleanup_task);
114 }
115
116 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
117 return IRQ_HANDLED;
118 }
119
120 /**
121 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
122 * @irq: interrupt id
123 * @data: interrupt data
124 */
ioat_dma_do_interrupt_msix(int irq,void * data)125 irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
126 {
127 struct ioatdma_chan *ioat_chan = data;
128
129 if (test_bit(IOAT_RUN, &ioat_chan->state))
130 tasklet_schedule(&ioat_chan->cleanup_task);
131
132 return IRQ_HANDLED;
133 }
134
ioat_stop(struct ioatdma_chan * ioat_chan)135 void ioat_stop(struct ioatdma_chan *ioat_chan)
136 {
137 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
138 struct pci_dev *pdev = ioat_dma->pdev;
139 int chan_id = chan_num(ioat_chan);
140 struct msix_entry *msix;
141
142 /* 1/ stop irq from firing tasklets
143 * 2/ stop the tasklet from re-arming irqs
144 */
145 clear_bit(IOAT_RUN, &ioat_chan->state);
146
147 /* flush inflight interrupts */
148 switch (ioat_dma->irq_mode) {
149 case IOAT_MSIX:
150 msix = &ioat_dma->msix_entries[chan_id];
151 synchronize_irq(msix->vector);
152 break;
153 case IOAT_MSI:
154 case IOAT_INTX:
155 synchronize_irq(pdev->irq);
156 break;
157 default:
158 break;
159 }
160
161 /* flush inflight timers */
162 del_timer_sync(&ioat_chan->timer);
163
164 /* flush inflight tasklet runs */
165 tasklet_kill(&ioat_chan->cleanup_task);
166
167 /* final cleanup now that everything is quiesced and can't re-arm */
168 ioat_cleanup_event((unsigned long)&ioat_chan->dma_chan);
169 }
170
__ioat_issue_pending(struct ioatdma_chan * ioat_chan)171 static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
172 {
173 ioat_chan->dmacount += ioat_ring_pending(ioat_chan);
174 ioat_chan->issued = ioat_chan->head;
175 writew(ioat_chan->dmacount,
176 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
177 dev_dbg(to_dev(ioat_chan),
178 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
179 __func__, ioat_chan->head, ioat_chan->tail,
180 ioat_chan->issued, ioat_chan->dmacount);
181 }
182
ioat_issue_pending(struct dma_chan * c)183 void ioat_issue_pending(struct dma_chan *c)
184 {
185 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
186
187 if (ioat_ring_pending(ioat_chan)) {
188 spin_lock_bh(&ioat_chan->prep_lock);
189 __ioat_issue_pending(ioat_chan);
190 spin_unlock_bh(&ioat_chan->prep_lock);
191 }
192 }
193
194 /**
195 * ioat_update_pending - log pending descriptors
196 * @ioat: ioat+ channel
197 *
198 * Check if the number of unsubmitted descriptors has exceeded the
199 * watermark. Called with prep_lock held
200 */
ioat_update_pending(struct ioatdma_chan * ioat_chan)201 static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
202 {
203 if (ioat_ring_pending(ioat_chan) > ioat_pending_level)
204 __ioat_issue_pending(ioat_chan);
205 }
206
__ioat_start_null_desc(struct ioatdma_chan * ioat_chan)207 static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
208 {
209 struct ioat_ring_ent *desc;
210 struct ioat_dma_descriptor *hw;
211
212 if (ioat_ring_space(ioat_chan) < 1) {
213 dev_err(to_dev(ioat_chan),
214 "Unable to start null desc - ring full\n");
215 return;
216 }
217
218 dev_dbg(to_dev(ioat_chan),
219 "%s: head: %#x tail: %#x issued: %#x\n",
220 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
221 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
222
223 hw = desc->hw;
224 hw->ctl = 0;
225 hw->ctl_f.null = 1;
226 hw->ctl_f.int_en = 1;
227 hw->ctl_f.compl_write = 1;
228 /* set size to non-zero value (channel returns error when size is 0) */
229 hw->size = NULL_DESC_BUFFER_SIZE;
230 hw->src_addr = 0;
231 hw->dst_addr = 0;
232 async_tx_ack(&desc->txd);
233 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
234 dump_desc_dbg(ioat_chan, desc);
235 /* make sure descriptors are written before we submit */
236 wmb();
237 ioat_chan->head += 1;
238 __ioat_issue_pending(ioat_chan);
239 }
240
ioat_start_null_desc(struct ioatdma_chan * ioat_chan)241 void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
242 {
243 spin_lock_bh(&ioat_chan->prep_lock);
244 if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
245 __ioat_start_null_desc(ioat_chan);
246 spin_unlock_bh(&ioat_chan->prep_lock);
247 }
248
__ioat_restart_chan(struct ioatdma_chan * ioat_chan)249 static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
250 {
251 /* set the tail to be re-issued */
252 ioat_chan->issued = ioat_chan->tail;
253 ioat_chan->dmacount = 0;
254 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
255
256 dev_dbg(to_dev(ioat_chan),
257 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
258 __func__, ioat_chan->head, ioat_chan->tail,
259 ioat_chan->issued, ioat_chan->dmacount);
260
261 if (ioat_ring_pending(ioat_chan)) {
262 struct ioat_ring_ent *desc;
263
264 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
265 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
266 __ioat_issue_pending(ioat_chan);
267 } else
268 __ioat_start_null_desc(ioat_chan);
269 }
270
ioat_quiesce(struct ioatdma_chan * ioat_chan,unsigned long tmo)271 static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
272 {
273 unsigned long end = jiffies + tmo;
274 int err = 0;
275 u32 status;
276
277 status = ioat_chansts(ioat_chan);
278 if (is_ioat_active(status) || is_ioat_idle(status))
279 ioat_suspend(ioat_chan);
280 while (is_ioat_active(status) || is_ioat_idle(status)) {
281 if (tmo && time_after(jiffies, end)) {
282 err = -ETIMEDOUT;
283 break;
284 }
285 status = ioat_chansts(ioat_chan);
286 cpu_relax();
287 }
288
289 return err;
290 }
291
ioat_reset_sync(struct ioatdma_chan * ioat_chan,unsigned long tmo)292 static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
293 {
294 unsigned long end = jiffies + tmo;
295 int err = 0;
296
297 ioat_reset(ioat_chan);
298 while (ioat_reset_pending(ioat_chan)) {
299 if (end && time_after(jiffies, end)) {
300 err = -ETIMEDOUT;
301 break;
302 }
303 cpu_relax();
304 }
305
306 return err;
307 }
308
ioat_tx_submit_unlock(struct dma_async_tx_descriptor * tx)309 static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
310 __releases(&ioat_chan->prep_lock)
311 {
312 struct dma_chan *c = tx->chan;
313 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
314 dma_cookie_t cookie;
315
316 cookie = dma_cookie_assign(tx);
317 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
318
319 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
320 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
321
322 /* make descriptor updates visible before advancing ioat->head,
323 * this is purposefully not smp_wmb() since we are also
324 * publishing the descriptor updates to a dma device
325 */
326 wmb();
327
328 ioat_chan->head += ioat_chan->produce;
329
330 ioat_update_pending(ioat_chan);
331 spin_unlock_bh(&ioat_chan->prep_lock);
332
333 return cookie;
334 }
335
336 static struct ioat_ring_ent *
ioat_alloc_ring_ent(struct dma_chan * chan,int idx,gfp_t flags)337 ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
338 {
339 struct ioat_dma_descriptor *hw;
340 struct ioat_ring_ent *desc;
341 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
342 int chunk;
343 dma_addr_t phys;
344 u8 *pos;
345 off_t offs;
346
347 chunk = idx / IOAT_DESCS_PER_2M;
348 idx &= (IOAT_DESCS_PER_2M - 1);
349 offs = idx * IOAT_DESC_SZ;
350 pos = (u8 *)ioat_chan->descs[chunk].virt + offs;
351 phys = ioat_chan->descs[chunk].hw + offs;
352 hw = (struct ioat_dma_descriptor *)pos;
353 memset(hw, 0, sizeof(*hw));
354
355 desc = kmem_cache_zalloc(ioat_cache, flags);
356 if (!desc)
357 return NULL;
358
359 dma_async_tx_descriptor_init(&desc->txd, chan);
360 desc->txd.tx_submit = ioat_tx_submit_unlock;
361 desc->hw = hw;
362 desc->txd.phys = phys;
363 return desc;
364 }
365
ioat_free_ring_ent(struct ioat_ring_ent * desc,struct dma_chan * chan)366 void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
367 {
368 kmem_cache_free(ioat_cache, desc);
369 }
370
371 struct ioat_ring_ent **
ioat_alloc_ring(struct dma_chan * c,int order,gfp_t flags)372 ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
373 {
374 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
375 struct ioat_ring_ent **ring;
376 int total_descs = 1 << order;
377 int i, chunks;
378
379 /* allocate the array to hold the software ring */
380 ring = kcalloc(total_descs, sizeof(*ring), flags);
381 if (!ring)
382 return NULL;
383
384 ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M;
385
386 for (i = 0; i < chunks; i++) {
387 struct ioat_descs *descs = &ioat_chan->descs[i];
388
389 descs->virt = dma_alloc_coherent(to_dev(ioat_chan),
390 SZ_2M, &descs->hw, flags);
391 if (!descs->virt) {
392 int idx;
393
394 for (idx = 0; idx < i; idx++) {
395 descs = &ioat_chan->descs[idx];
396 dma_free_coherent(to_dev(ioat_chan), SZ_2M,
397 descs->virt, descs->hw);
398 descs->virt = NULL;
399 descs->hw = 0;
400 }
401
402 ioat_chan->desc_chunks = 0;
403 kfree(ring);
404 return NULL;
405 }
406 }
407
408 for (i = 0; i < total_descs; i++) {
409 ring[i] = ioat_alloc_ring_ent(c, i, flags);
410 if (!ring[i]) {
411 int idx;
412
413 while (i--)
414 ioat_free_ring_ent(ring[i], c);
415
416 for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
417 dma_free_coherent(to_dev(ioat_chan),
418 SZ_2M,
419 ioat_chan->descs[idx].virt,
420 ioat_chan->descs[idx].hw);
421 ioat_chan->descs[idx].virt = NULL;
422 ioat_chan->descs[idx].hw = 0;
423 }
424
425 ioat_chan->desc_chunks = 0;
426 kfree(ring);
427 return NULL;
428 }
429 set_desc_id(ring[i], i);
430 }
431
432 /* link descs */
433 for (i = 0; i < total_descs-1; i++) {
434 struct ioat_ring_ent *next = ring[i+1];
435 struct ioat_dma_descriptor *hw = ring[i]->hw;
436
437 hw->next = next->txd.phys;
438 }
439 ring[i]->hw->next = ring[0]->txd.phys;
440
441 return ring;
442 }
443
444 /**
445 * ioat_check_space_lock - verify space and grab ring producer lock
446 * @ioat: ioat,3 channel (ring) to operate on
447 * @num_descs: allocation length
448 */
ioat_check_space_lock(struct ioatdma_chan * ioat_chan,int num_descs)449 int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
450 __acquires(&ioat_chan->prep_lock)
451 {
452 spin_lock_bh(&ioat_chan->prep_lock);
453 /* never allow the last descriptor to be consumed, we need at
454 * least one free at all times to allow for on-the-fly ring
455 * resizing.
456 */
457 if (likely(ioat_ring_space(ioat_chan) > num_descs)) {
458 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
459 __func__, num_descs, ioat_chan->head,
460 ioat_chan->tail, ioat_chan->issued);
461 ioat_chan->produce = num_descs;
462 return 0; /* with ioat->prep_lock held */
463 }
464 spin_unlock_bh(&ioat_chan->prep_lock);
465
466 dev_dbg_ratelimited(to_dev(ioat_chan),
467 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
468 __func__, num_descs, ioat_chan->head,
469 ioat_chan->tail, ioat_chan->issued);
470
471 /* progress reclaim in the allocation failure case we may be
472 * called under bh_disabled so we need to trigger the timer
473 * event directly
474 */
475 if (time_is_before_jiffies(ioat_chan->timer.expires)
476 && timer_pending(&ioat_chan->timer)) {
477 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
478 ioat_timer_event((unsigned long)ioat_chan);
479 }
480
481 return -ENOMEM;
482 }
483
desc_has_ext(struct ioat_ring_ent * desc)484 static bool desc_has_ext(struct ioat_ring_ent *desc)
485 {
486 struct ioat_dma_descriptor *hw = desc->hw;
487
488 if (hw->ctl_f.op == IOAT_OP_XOR ||
489 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
490 struct ioat_xor_descriptor *xor = desc->xor;
491
492 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
493 return true;
494 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
495 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
496 struct ioat_pq_descriptor *pq = desc->pq;
497
498 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
499 return true;
500 }
501
502 return false;
503 }
504
505 static void
ioat_free_sed(struct ioatdma_device * ioat_dma,struct ioat_sed_ent * sed)506 ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
507 {
508 if (!sed)
509 return;
510
511 dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
512 kmem_cache_free(ioat_sed_cache, sed);
513 }
514
ioat_get_current_completion(struct ioatdma_chan * ioat_chan)515 static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
516 {
517 u64 phys_complete;
518 u64 completion;
519
520 completion = *ioat_chan->completion;
521 phys_complete = ioat_chansts_to_addr(completion);
522
523 dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
524 (unsigned long long) phys_complete);
525
526 return phys_complete;
527 }
528
ioat_cleanup_preamble(struct ioatdma_chan * ioat_chan,u64 * phys_complete)529 static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
530 u64 *phys_complete)
531 {
532 *phys_complete = ioat_get_current_completion(ioat_chan);
533 if (*phys_complete == ioat_chan->last_completion)
534 return false;
535
536 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
537 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
538
539 return true;
540 }
541
542 static void
desc_get_errstat(struct ioatdma_chan * ioat_chan,struct ioat_ring_ent * desc)543 desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
544 {
545 struct ioat_dma_descriptor *hw = desc->hw;
546
547 switch (hw->ctl_f.op) {
548 case IOAT_OP_PQ_VAL:
549 case IOAT_OP_PQ_VAL_16S:
550 {
551 struct ioat_pq_descriptor *pq = desc->pq;
552
553 /* check if there's error written */
554 if (!pq->dwbes_f.wbes)
555 return;
556
557 /* need to set a chanerr var for checking to clear later */
558
559 if (pq->dwbes_f.p_val_err)
560 *desc->result |= SUM_CHECK_P_RESULT;
561
562 if (pq->dwbes_f.q_val_err)
563 *desc->result |= SUM_CHECK_Q_RESULT;
564
565 return;
566 }
567 default:
568 return;
569 }
570 }
571
572 /**
573 * __cleanup - reclaim used descriptors
574 * @ioat: channel (ring) to clean
575 */
__cleanup(struct ioatdma_chan * ioat_chan,dma_addr_t phys_complete)576 static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
577 {
578 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
579 struct ioat_ring_ent *desc;
580 bool seen_current = false;
581 int idx = ioat_chan->tail, i;
582 u16 active;
583
584 dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
585 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
586
587 /*
588 * At restart of the channel, the completion address and the
589 * channel status will be 0 due to starting a new chain. Since
590 * it's new chain and the first descriptor "fails", there is
591 * nothing to clean up. We do not want to reap the entire submitted
592 * chain due to this 0 address value and then BUG.
593 */
594 if (!phys_complete)
595 return;
596
597 active = ioat_ring_active(ioat_chan);
598 for (i = 0; i < active && !seen_current; i++) {
599 struct dma_async_tx_descriptor *tx;
600
601 smp_read_barrier_depends();
602 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
603 desc = ioat_get_ring_ent(ioat_chan, idx + i);
604 dump_desc_dbg(ioat_chan, desc);
605
606 /* set err stat if we are using dwbes */
607 if (ioat_dma->cap & IOAT_CAP_DWBES)
608 desc_get_errstat(ioat_chan, desc);
609
610 tx = &desc->txd;
611 if (tx->cookie) {
612 dma_cookie_complete(tx);
613 dma_descriptor_unmap(tx);
614 dmaengine_desc_get_callback_invoke(tx, NULL);
615 tx->callback = NULL;
616 tx->callback_result = NULL;
617 }
618
619 if (tx->phys == phys_complete)
620 seen_current = true;
621
622 /* skip extended descriptors */
623 if (desc_has_ext(desc)) {
624 BUG_ON(i + 1 >= active);
625 i++;
626 }
627
628 /* cleanup super extended descriptors */
629 if (desc->sed) {
630 ioat_free_sed(ioat_dma, desc->sed);
631 desc->sed = NULL;
632 }
633 }
634
635 /* finish all descriptor reads before incrementing tail */
636 smp_mb();
637 ioat_chan->tail = idx + i;
638 /* no active descs have written a completion? */
639 BUG_ON(active && !seen_current);
640 ioat_chan->last_completion = phys_complete;
641
642 if (active - i == 0) {
643 dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
644 __func__);
645 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
646 }
647
648 /* microsecond delay by sysfs variable per pending descriptor */
649 if (ioat_chan->intr_coalesce != ioat_chan->prev_intr_coalesce) {
650 writew(min((ioat_chan->intr_coalesce * (active - i)),
651 IOAT_INTRDELAY_MASK),
652 ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
653 ioat_chan->prev_intr_coalesce = ioat_chan->intr_coalesce;
654 }
655 }
656
ioat_cleanup(struct ioatdma_chan * ioat_chan)657 static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
658 {
659 u64 phys_complete;
660
661 spin_lock_bh(&ioat_chan->cleanup_lock);
662
663 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
664 __cleanup(ioat_chan, phys_complete);
665
666 if (is_ioat_halted(*ioat_chan->completion)) {
667 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
668
669 if (chanerr &
670 (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) {
671 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
672 ioat_eh(ioat_chan);
673 }
674 }
675
676 spin_unlock_bh(&ioat_chan->cleanup_lock);
677 }
678
ioat_cleanup_event(unsigned long data)679 void ioat_cleanup_event(unsigned long data)
680 {
681 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
682
683 ioat_cleanup(ioat_chan);
684 if (!test_bit(IOAT_RUN, &ioat_chan->state))
685 return;
686 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
687 }
688
ioat_restart_channel(struct ioatdma_chan * ioat_chan)689 static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
690 {
691 u64 phys_complete;
692
693 ioat_quiesce(ioat_chan, 0);
694 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
695 __cleanup(ioat_chan, phys_complete);
696
697 __ioat_restart_chan(ioat_chan);
698 }
699
700
ioat_abort_descs(struct ioatdma_chan * ioat_chan)701 static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
702 {
703 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
704 struct ioat_ring_ent *desc;
705 u16 active;
706 int idx = ioat_chan->tail, i;
707
708 /*
709 * We assume that the failed descriptor has been processed.
710 * Now we are just returning all the remaining submitted
711 * descriptors to abort.
712 */
713 active = ioat_ring_active(ioat_chan);
714
715 /* we skip the failed descriptor that tail points to */
716 for (i = 1; i < active; i++) {
717 struct dma_async_tx_descriptor *tx;
718
719 smp_read_barrier_depends();
720 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
721 desc = ioat_get_ring_ent(ioat_chan, idx + i);
722
723 tx = &desc->txd;
724 if (tx->cookie) {
725 struct dmaengine_result res;
726
727 dma_cookie_complete(tx);
728 dma_descriptor_unmap(tx);
729 res.result = DMA_TRANS_ABORTED;
730 dmaengine_desc_get_callback_invoke(tx, &res);
731 tx->callback = NULL;
732 tx->callback_result = NULL;
733 }
734
735 /* skip extended descriptors */
736 if (desc_has_ext(desc)) {
737 WARN_ON(i + 1 >= active);
738 i++;
739 }
740
741 /* cleanup super extended descriptors */
742 if (desc->sed) {
743 ioat_free_sed(ioat_dma, desc->sed);
744 desc->sed = NULL;
745 }
746 }
747
748 smp_mb(); /* finish all descriptor reads before incrementing tail */
749 ioat_chan->tail = idx + active;
750
751 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
752 ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys;
753 }
754
ioat_eh(struct ioatdma_chan * ioat_chan)755 static void ioat_eh(struct ioatdma_chan *ioat_chan)
756 {
757 struct pci_dev *pdev = to_pdev(ioat_chan);
758 struct ioat_dma_descriptor *hw;
759 struct dma_async_tx_descriptor *tx;
760 u64 phys_complete;
761 struct ioat_ring_ent *desc;
762 u32 err_handled = 0;
763 u32 chanerr_int;
764 u32 chanerr;
765 bool abort = false;
766 struct dmaengine_result res;
767
768 /* cleanup so tail points to descriptor that caused the error */
769 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
770 __cleanup(ioat_chan, phys_complete);
771
772 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
773 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
774
775 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
776 __func__, chanerr, chanerr_int);
777
778 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
779 hw = desc->hw;
780 dump_desc_dbg(ioat_chan, desc);
781
782 switch (hw->ctl_f.op) {
783 case IOAT_OP_XOR_VAL:
784 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
785 *desc->result |= SUM_CHECK_P_RESULT;
786 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
787 }
788 break;
789 case IOAT_OP_PQ_VAL:
790 case IOAT_OP_PQ_VAL_16S:
791 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
792 *desc->result |= SUM_CHECK_P_RESULT;
793 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
794 }
795 if (chanerr & IOAT_CHANERR_XOR_Q_ERR) {
796 *desc->result |= SUM_CHECK_Q_RESULT;
797 err_handled |= IOAT_CHANERR_XOR_Q_ERR;
798 }
799 break;
800 }
801
802 if (chanerr & IOAT_CHANERR_RECOVER_MASK) {
803 if (chanerr & IOAT_CHANERR_READ_DATA_ERR) {
804 res.result = DMA_TRANS_READ_FAILED;
805 err_handled |= IOAT_CHANERR_READ_DATA_ERR;
806 } else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) {
807 res.result = DMA_TRANS_WRITE_FAILED;
808 err_handled |= IOAT_CHANERR_WRITE_DATA_ERR;
809 }
810
811 abort = true;
812 } else
813 res.result = DMA_TRANS_NOERROR;
814
815 /* fault on unhandled error or spurious halt */
816 if (chanerr ^ err_handled || chanerr == 0) {
817 dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
818 __func__, chanerr, err_handled);
819 dev_err(to_dev(ioat_chan), "Errors handled:\n");
820 ioat_print_chanerrs(ioat_chan, err_handled);
821 dev_err(to_dev(ioat_chan), "Errors not handled:\n");
822 ioat_print_chanerrs(ioat_chan, (chanerr & ~err_handled));
823
824 BUG();
825 }
826
827 /* cleanup the faulty descriptor since we are continuing */
828 tx = &desc->txd;
829 if (tx->cookie) {
830 dma_cookie_complete(tx);
831 dma_descriptor_unmap(tx);
832 dmaengine_desc_get_callback_invoke(tx, &res);
833 tx->callback = NULL;
834 tx->callback_result = NULL;
835 }
836
837 /* mark faulting descriptor as complete */
838 *ioat_chan->completion = desc->txd.phys;
839
840 spin_lock_bh(&ioat_chan->prep_lock);
841 /* we need abort all descriptors */
842 if (abort) {
843 ioat_abort_descs(ioat_chan);
844 /* clean up the channel, we could be in weird state */
845 ioat_reset_hw(ioat_chan);
846 }
847
848 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
849 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
850
851 ioat_restart_channel(ioat_chan);
852 spin_unlock_bh(&ioat_chan->prep_lock);
853 }
854
check_active(struct ioatdma_chan * ioat_chan)855 static void check_active(struct ioatdma_chan *ioat_chan)
856 {
857 if (ioat_ring_active(ioat_chan)) {
858 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
859 return;
860 }
861
862 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
863 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
864 }
865
ioat_timer_event(unsigned long data)866 void ioat_timer_event(unsigned long data)
867 {
868 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
869 dma_addr_t phys_complete;
870 u64 status;
871
872 status = ioat_chansts(ioat_chan);
873
874 /* when halted due to errors check for channel
875 * programming errors before advancing the completion state
876 */
877 if (is_ioat_halted(status)) {
878 u32 chanerr;
879
880 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
881 dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
882 __func__, chanerr);
883 dev_err(to_dev(ioat_chan), "Errors:\n");
884 ioat_print_chanerrs(ioat_chan, chanerr);
885
886 if (test_bit(IOAT_RUN, &ioat_chan->state)) {
887 spin_lock_bh(&ioat_chan->cleanup_lock);
888 spin_lock_bh(&ioat_chan->prep_lock);
889 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
890 spin_unlock_bh(&ioat_chan->prep_lock);
891
892 ioat_abort_descs(ioat_chan);
893 dev_warn(to_dev(ioat_chan), "Reset channel...\n");
894 ioat_reset_hw(ioat_chan);
895 dev_warn(to_dev(ioat_chan), "Restart channel...\n");
896 ioat_restart_channel(ioat_chan);
897
898 spin_lock_bh(&ioat_chan->prep_lock);
899 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
900 spin_unlock_bh(&ioat_chan->prep_lock);
901 spin_unlock_bh(&ioat_chan->cleanup_lock);
902 }
903
904 return;
905 }
906
907 spin_lock_bh(&ioat_chan->cleanup_lock);
908
909 /* handle the no-actives case */
910 if (!ioat_ring_active(ioat_chan)) {
911 spin_lock_bh(&ioat_chan->prep_lock);
912 check_active(ioat_chan);
913 spin_unlock_bh(&ioat_chan->prep_lock);
914 spin_unlock_bh(&ioat_chan->cleanup_lock);
915 return;
916 }
917
918 /* if we haven't made progress and we have already
919 * acknowledged a pending completion once, then be more
920 * forceful with a restart
921 */
922 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
923 __cleanup(ioat_chan, phys_complete);
924 else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
925 u32 chanerr;
926
927 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
928 dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n",
929 status, chanerr);
930 dev_err(to_dev(ioat_chan), "Errors:\n");
931 ioat_print_chanerrs(ioat_chan, chanerr);
932
933 dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n",
934 ioat_ring_active(ioat_chan));
935
936 spin_lock_bh(&ioat_chan->prep_lock);
937 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
938 spin_unlock_bh(&ioat_chan->prep_lock);
939
940 ioat_abort_descs(ioat_chan);
941 dev_warn(to_dev(ioat_chan), "Resetting channel...\n");
942 ioat_reset_hw(ioat_chan);
943 dev_warn(to_dev(ioat_chan), "Restarting channel...\n");
944 ioat_restart_channel(ioat_chan);
945
946 spin_lock_bh(&ioat_chan->prep_lock);
947 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
948 spin_unlock_bh(&ioat_chan->prep_lock);
949 spin_unlock_bh(&ioat_chan->cleanup_lock);
950 return;
951 } else
952 set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
953
954 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
955 spin_unlock_bh(&ioat_chan->cleanup_lock);
956 }
957
958 enum dma_status
ioat_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)959 ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
960 struct dma_tx_state *txstate)
961 {
962 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
963 enum dma_status ret;
964
965 ret = dma_cookie_status(c, cookie, txstate);
966 if (ret == DMA_COMPLETE)
967 return ret;
968
969 ioat_cleanup(ioat_chan);
970
971 return dma_cookie_status(c, cookie, txstate);
972 }
973
ioat_reset_hw(struct ioatdma_chan * ioat_chan)974 int ioat_reset_hw(struct ioatdma_chan *ioat_chan)
975 {
976 /* throw away whatever the channel was doing and get it
977 * initialized, with ioat3 specific workarounds
978 */
979 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
980 struct pci_dev *pdev = ioat_dma->pdev;
981 u32 chanerr;
982 u16 dev_id;
983 int err;
984
985 ioat_quiesce(ioat_chan, msecs_to_jiffies(100));
986
987 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
988 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
989
990 if (ioat_dma->version < IOAT_VER_3_3) {
991 /* clear any pending errors */
992 err = pci_read_config_dword(pdev,
993 IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
994 if (err) {
995 dev_err(&pdev->dev,
996 "channel error register unreachable\n");
997 return err;
998 }
999 pci_write_config_dword(pdev,
1000 IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
1001
1002 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1003 * (workaround for spurious config parity error after restart)
1004 */
1005 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1006 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
1007 pci_write_config_dword(pdev,
1008 IOAT_PCI_DMAUNCERRSTS_OFFSET,
1009 0x10);
1010 }
1011 }
1012
1013 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1014 ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000);
1015 ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008);
1016 ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800);
1017 }
1018
1019
1020 err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200));
1021 if (!err) {
1022 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1023 writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000);
1024 writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008);
1025 writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800);
1026 }
1027 }
1028
1029 if (err)
1030 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1031
1032 return err;
1033 }
1034