1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel I/OAT DMA Linux driver
4 * Copyright(c) 2004 - 2015 Intel Corporation.
5 */
6
7 /*
8 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
9 * copy operations.
10 */
11
12 #include <linux/init.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/pci.h>
16 #include <linux/interrupt.h>
17 #include <linux/dmaengine.h>
18 #include <linux/delay.h>
19 #include <linux/dma-mapping.h>
20 #include <linux/workqueue.h>
21 #include <linux/prefetch.h>
22 #include <linux/sizes.h>
23 #include "dma.h"
24 #include "registers.h"
25 #include "hw.h"
26
27 #include "../dmaengine.h"
28
29 int completion_timeout = 200;
30 module_param(completion_timeout, int, 0644);
31 MODULE_PARM_DESC(completion_timeout,
32 "set ioat completion timeout [msec] (default 200 [msec])");
33 int idle_timeout = 2000;
34 module_param(idle_timeout, int, 0644);
35 MODULE_PARM_DESC(idle_timeout,
36 "set ioat idel timeout [msec] (default 2000 [msec])");
37
38 #define IDLE_TIMEOUT msecs_to_jiffies(idle_timeout)
39 #define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout)
40
41 static char *chanerr_str[] = {
42 "DMA Transfer Source Address Error",
43 "DMA Transfer Destination Address Error",
44 "Next Descriptor Address Error",
45 "Descriptor Error",
46 "Chan Address Value Error",
47 "CHANCMD Error",
48 "Chipset Uncorrectable Data Integrity Error",
49 "DMA Uncorrectable Data Integrity Error",
50 "Read Data Error",
51 "Write Data Error",
52 "Descriptor Control Error",
53 "Descriptor Transfer Size Error",
54 "Completion Address Error",
55 "Interrupt Configuration Error",
56 "Super extended descriptor Address Error",
57 "Unaffiliated Error",
58 "CRC or XOR P Error",
59 "XOR Q Error",
60 "Descriptor Count Error",
61 "DIF All F detect Error",
62 "Guard Tag verification Error",
63 "Application Tag verification Error",
64 "Reference Tag verification Error",
65 "Bundle Bit Error",
66 "Result DIF All F detect Error",
67 "Result Guard Tag verification Error",
68 "Result Application Tag verification Error",
69 "Result Reference Tag verification Error",
70 };
71
72 static void ioat_eh(struct ioatdma_chan *ioat_chan);
73
ioat_print_chanerrs(struct ioatdma_chan * ioat_chan,u32 chanerr)74 static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
75 {
76 int i;
77
78 for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) {
79 if ((chanerr >> i) & 1) {
80 dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
81 i, chanerr_str[i]);
82 }
83 }
84 }
85
86 /**
87 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
88 * @irq: interrupt id
89 * @data: interrupt data
90 */
ioat_dma_do_interrupt(int irq,void * data)91 irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
92 {
93 struct ioatdma_device *instance = data;
94 struct ioatdma_chan *ioat_chan;
95 unsigned long attnstatus;
96 int bit;
97 u8 intrctrl;
98
99 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
100
101 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
102 return IRQ_NONE;
103
104 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
105 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
106 return IRQ_NONE;
107 }
108
109 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
110 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
111 ioat_chan = ioat_chan_by_index(instance, bit);
112 if (test_bit(IOAT_RUN, &ioat_chan->state))
113 tasklet_schedule(&ioat_chan->cleanup_task);
114 }
115
116 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
117 return IRQ_HANDLED;
118 }
119
120 /**
121 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
122 * @irq: interrupt id
123 * @data: interrupt data
124 */
ioat_dma_do_interrupt_msix(int irq,void * data)125 irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
126 {
127 struct ioatdma_chan *ioat_chan = data;
128
129 if (test_bit(IOAT_RUN, &ioat_chan->state))
130 tasklet_schedule(&ioat_chan->cleanup_task);
131
132 return IRQ_HANDLED;
133 }
134
ioat_stop(struct ioatdma_chan * ioat_chan)135 void ioat_stop(struct ioatdma_chan *ioat_chan)
136 {
137 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
138 struct pci_dev *pdev = ioat_dma->pdev;
139 int chan_id = chan_num(ioat_chan);
140 struct msix_entry *msix;
141
142 /* 1/ stop irq from firing tasklets
143 * 2/ stop the tasklet from re-arming irqs
144 */
145 clear_bit(IOAT_RUN, &ioat_chan->state);
146
147 /* flush inflight interrupts */
148 switch (ioat_dma->irq_mode) {
149 case IOAT_MSIX:
150 msix = &ioat_dma->msix_entries[chan_id];
151 synchronize_irq(msix->vector);
152 break;
153 case IOAT_MSI:
154 case IOAT_INTX:
155 synchronize_irq(pdev->irq);
156 break;
157 default:
158 break;
159 }
160
161 /* flush inflight timers */
162 del_timer_sync(&ioat_chan->timer);
163
164 /* flush inflight tasklet runs */
165 tasklet_kill(&ioat_chan->cleanup_task);
166
167 /* final cleanup now that everything is quiesced and can't re-arm */
168 ioat_cleanup_event((unsigned long)&ioat_chan->dma_chan);
169 }
170
__ioat_issue_pending(struct ioatdma_chan * ioat_chan)171 static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
172 {
173 ioat_chan->dmacount += ioat_ring_pending(ioat_chan);
174 ioat_chan->issued = ioat_chan->head;
175 writew(ioat_chan->dmacount,
176 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
177 dev_dbg(to_dev(ioat_chan),
178 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
179 __func__, ioat_chan->head, ioat_chan->tail,
180 ioat_chan->issued, ioat_chan->dmacount);
181 }
182
ioat_issue_pending(struct dma_chan * c)183 void ioat_issue_pending(struct dma_chan *c)
184 {
185 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
186
187 if (ioat_ring_pending(ioat_chan)) {
188 spin_lock_bh(&ioat_chan->prep_lock);
189 __ioat_issue_pending(ioat_chan);
190 spin_unlock_bh(&ioat_chan->prep_lock);
191 }
192 }
193
194 /**
195 * ioat_update_pending - log pending descriptors
196 * @ioat: ioat+ channel
197 *
198 * Check if the number of unsubmitted descriptors has exceeded the
199 * watermark. Called with prep_lock held
200 */
ioat_update_pending(struct ioatdma_chan * ioat_chan)201 static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
202 {
203 if (ioat_ring_pending(ioat_chan) > ioat_pending_level)
204 __ioat_issue_pending(ioat_chan);
205 }
206
__ioat_start_null_desc(struct ioatdma_chan * ioat_chan)207 static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
208 {
209 struct ioat_ring_ent *desc;
210 struct ioat_dma_descriptor *hw;
211
212 if (ioat_ring_space(ioat_chan) < 1) {
213 dev_err(to_dev(ioat_chan),
214 "Unable to start null desc - ring full\n");
215 return;
216 }
217
218 dev_dbg(to_dev(ioat_chan),
219 "%s: head: %#x tail: %#x issued: %#x\n",
220 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
221 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
222
223 hw = desc->hw;
224 hw->ctl = 0;
225 hw->ctl_f.null = 1;
226 hw->ctl_f.int_en = 1;
227 hw->ctl_f.compl_write = 1;
228 /* set size to non-zero value (channel returns error when size is 0) */
229 hw->size = NULL_DESC_BUFFER_SIZE;
230 hw->src_addr = 0;
231 hw->dst_addr = 0;
232 async_tx_ack(&desc->txd);
233 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
234 dump_desc_dbg(ioat_chan, desc);
235 /* make sure descriptors are written before we submit */
236 wmb();
237 ioat_chan->head += 1;
238 __ioat_issue_pending(ioat_chan);
239 }
240
ioat_start_null_desc(struct ioatdma_chan * ioat_chan)241 void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
242 {
243 spin_lock_bh(&ioat_chan->prep_lock);
244 if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
245 __ioat_start_null_desc(ioat_chan);
246 spin_unlock_bh(&ioat_chan->prep_lock);
247 }
248
__ioat_restart_chan(struct ioatdma_chan * ioat_chan)249 static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
250 {
251 /* set the tail to be re-issued */
252 ioat_chan->issued = ioat_chan->tail;
253 ioat_chan->dmacount = 0;
254 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
255
256 dev_dbg(to_dev(ioat_chan),
257 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
258 __func__, ioat_chan->head, ioat_chan->tail,
259 ioat_chan->issued, ioat_chan->dmacount);
260
261 if (ioat_ring_pending(ioat_chan)) {
262 struct ioat_ring_ent *desc;
263
264 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
265 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
266 __ioat_issue_pending(ioat_chan);
267 } else
268 __ioat_start_null_desc(ioat_chan);
269 }
270
ioat_quiesce(struct ioatdma_chan * ioat_chan,unsigned long tmo)271 static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
272 {
273 unsigned long end = jiffies + tmo;
274 int err = 0;
275 u32 status;
276
277 status = ioat_chansts(ioat_chan);
278 if (is_ioat_active(status) || is_ioat_idle(status))
279 ioat_suspend(ioat_chan);
280 while (is_ioat_active(status) || is_ioat_idle(status)) {
281 if (tmo && time_after(jiffies, end)) {
282 err = -ETIMEDOUT;
283 break;
284 }
285 status = ioat_chansts(ioat_chan);
286 cpu_relax();
287 }
288
289 return err;
290 }
291
ioat_reset_sync(struct ioatdma_chan * ioat_chan,unsigned long tmo)292 static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
293 {
294 unsigned long end = jiffies + tmo;
295 int err = 0;
296
297 ioat_reset(ioat_chan);
298 while (ioat_reset_pending(ioat_chan)) {
299 if (end && time_after(jiffies, end)) {
300 err = -ETIMEDOUT;
301 break;
302 }
303 cpu_relax();
304 }
305
306 return err;
307 }
308
ioat_tx_submit_unlock(struct dma_async_tx_descriptor * tx)309 static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
310 __releases(&ioat_chan->prep_lock)
311 {
312 struct dma_chan *c = tx->chan;
313 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
314 dma_cookie_t cookie;
315
316 cookie = dma_cookie_assign(tx);
317 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
318
319 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
320 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
321
322 /* make descriptor updates visible before advancing ioat->head,
323 * this is purposefully not smp_wmb() since we are also
324 * publishing the descriptor updates to a dma device
325 */
326 wmb();
327
328 ioat_chan->head += ioat_chan->produce;
329
330 ioat_update_pending(ioat_chan);
331 spin_unlock_bh(&ioat_chan->prep_lock);
332
333 return cookie;
334 }
335
336 static struct ioat_ring_ent *
ioat_alloc_ring_ent(struct dma_chan * chan,int idx,gfp_t flags)337 ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
338 {
339 struct ioat_dma_descriptor *hw;
340 struct ioat_ring_ent *desc;
341 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
342 int chunk;
343 dma_addr_t phys;
344 u8 *pos;
345 off_t offs;
346
347 chunk = idx / IOAT_DESCS_PER_2M;
348 idx &= (IOAT_DESCS_PER_2M - 1);
349 offs = idx * IOAT_DESC_SZ;
350 pos = (u8 *)ioat_chan->descs[chunk].virt + offs;
351 phys = ioat_chan->descs[chunk].hw + offs;
352 hw = (struct ioat_dma_descriptor *)pos;
353 memset(hw, 0, sizeof(*hw));
354
355 desc = kmem_cache_zalloc(ioat_cache, flags);
356 if (!desc)
357 return NULL;
358
359 dma_async_tx_descriptor_init(&desc->txd, chan);
360 desc->txd.tx_submit = ioat_tx_submit_unlock;
361 desc->hw = hw;
362 desc->txd.phys = phys;
363 return desc;
364 }
365
ioat_free_ring_ent(struct ioat_ring_ent * desc,struct dma_chan * chan)366 void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
367 {
368 kmem_cache_free(ioat_cache, desc);
369 }
370
371 struct ioat_ring_ent **
ioat_alloc_ring(struct dma_chan * c,int order,gfp_t flags)372 ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
373 {
374 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
375 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
376 struct ioat_ring_ent **ring;
377 int total_descs = 1 << order;
378 int i, chunks;
379
380 /* allocate the array to hold the software ring */
381 ring = kcalloc(total_descs, sizeof(*ring), flags);
382 if (!ring)
383 return NULL;
384
385 ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M;
386
387 for (i = 0; i < chunks; i++) {
388 struct ioat_descs *descs = &ioat_chan->descs[i];
389
390 descs->virt = dma_alloc_coherent(to_dev(ioat_chan),
391 SZ_2M, &descs->hw, flags);
392 if (!descs->virt) {
393 int idx;
394
395 for (idx = 0; idx < i; idx++) {
396 descs = &ioat_chan->descs[idx];
397 dma_free_coherent(to_dev(ioat_chan), SZ_2M,
398 descs->virt, descs->hw);
399 descs->virt = NULL;
400 descs->hw = 0;
401 }
402
403 ioat_chan->desc_chunks = 0;
404 kfree(ring);
405 return NULL;
406 }
407 }
408
409 for (i = 0; i < total_descs; i++) {
410 ring[i] = ioat_alloc_ring_ent(c, i, flags);
411 if (!ring[i]) {
412 int idx;
413
414 while (i--)
415 ioat_free_ring_ent(ring[i], c);
416
417 for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
418 dma_free_coherent(to_dev(ioat_chan),
419 SZ_2M,
420 ioat_chan->descs[idx].virt,
421 ioat_chan->descs[idx].hw);
422 ioat_chan->descs[idx].virt = NULL;
423 ioat_chan->descs[idx].hw = 0;
424 }
425
426 ioat_chan->desc_chunks = 0;
427 kfree(ring);
428 return NULL;
429 }
430 set_desc_id(ring[i], i);
431 }
432
433 /* link descs */
434 for (i = 0; i < total_descs-1; i++) {
435 struct ioat_ring_ent *next = ring[i+1];
436 struct ioat_dma_descriptor *hw = ring[i]->hw;
437
438 hw->next = next->txd.phys;
439 }
440 ring[i]->hw->next = ring[0]->txd.phys;
441
442 /* setup descriptor pre-fetching for v3.4 */
443 if (ioat_dma->cap & IOAT_CAP_DPS) {
444 u16 drsctl = IOAT_CHAN_DRSZ_2MB | IOAT_CHAN_DRS_EN;
445
446 if (chunks == 1)
447 drsctl |= IOAT_CHAN_DRS_AUTOWRAP;
448
449 writew(drsctl, ioat_chan->reg_base + IOAT_CHAN_DRSCTL_OFFSET);
450
451 }
452
453 return ring;
454 }
455
456 /**
457 * ioat_check_space_lock - verify space and grab ring producer lock
458 * @ioat: ioat,3 channel (ring) to operate on
459 * @num_descs: allocation length
460 */
ioat_check_space_lock(struct ioatdma_chan * ioat_chan,int num_descs)461 int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
462 __acquires(&ioat_chan->prep_lock)
463 {
464 spin_lock_bh(&ioat_chan->prep_lock);
465 /* never allow the last descriptor to be consumed, we need at
466 * least one free at all times to allow for on-the-fly ring
467 * resizing.
468 */
469 if (likely(ioat_ring_space(ioat_chan) > num_descs)) {
470 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
471 __func__, num_descs, ioat_chan->head,
472 ioat_chan->tail, ioat_chan->issued);
473 ioat_chan->produce = num_descs;
474 return 0; /* with ioat->prep_lock held */
475 }
476 spin_unlock_bh(&ioat_chan->prep_lock);
477
478 dev_dbg_ratelimited(to_dev(ioat_chan),
479 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
480 __func__, num_descs, ioat_chan->head,
481 ioat_chan->tail, ioat_chan->issued);
482
483 /* progress reclaim in the allocation failure case we may be
484 * called under bh_disabled so we need to trigger the timer
485 * event directly
486 */
487 if (time_is_before_jiffies(ioat_chan->timer.expires)
488 && timer_pending(&ioat_chan->timer)) {
489 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
490 ioat_timer_event(&ioat_chan->timer);
491 }
492
493 return -ENOMEM;
494 }
495
desc_has_ext(struct ioat_ring_ent * desc)496 static bool desc_has_ext(struct ioat_ring_ent *desc)
497 {
498 struct ioat_dma_descriptor *hw = desc->hw;
499
500 if (hw->ctl_f.op == IOAT_OP_XOR ||
501 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
502 struct ioat_xor_descriptor *xor = desc->xor;
503
504 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
505 return true;
506 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
507 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
508 struct ioat_pq_descriptor *pq = desc->pq;
509
510 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
511 return true;
512 }
513
514 return false;
515 }
516
517 static void
ioat_free_sed(struct ioatdma_device * ioat_dma,struct ioat_sed_ent * sed)518 ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
519 {
520 if (!sed)
521 return;
522
523 dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
524 kmem_cache_free(ioat_sed_cache, sed);
525 }
526
ioat_get_current_completion(struct ioatdma_chan * ioat_chan)527 static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
528 {
529 u64 phys_complete;
530 u64 completion;
531
532 completion = *ioat_chan->completion;
533 phys_complete = ioat_chansts_to_addr(completion);
534
535 dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
536 (unsigned long long) phys_complete);
537
538 return phys_complete;
539 }
540
ioat_cleanup_preamble(struct ioatdma_chan * ioat_chan,u64 * phys_complete)541 static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
542 u64 *phys_complete)
543 {
544 *phys_complete = ioat_get_current_completion(ioat_chan);
545 if (*phys_complete == ioat_chan->last_completion)
546 return false;
547
548 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
549 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
550
551 return true;
552 }
553
554 static void
desc_get_errstat(struct ioatdma_chan * ioat_chan,struct ioat_ring_ent * desc)555 desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
556 {
557 struct ioat_dma_descriptor *hw = desc->hw;
558
559 switch (hw->ctl_f.op) {
560 case IOAT_OP_PQ_VAL:
561 case IOAT_OP_PQ_VAL_16S:
562 {
563 struct ioat_pq_descriptor *pq = desc->pq;
564
565 /* check if there's error written */
566 if (!pq->dwbes_f.wbes)
567 return;
568
569 /* need to set a chanerr var for checking to clear later */
570
571 if (pq->dwbes_f.p_val_err)
572 *desc->result |= SUM_CHECK_P_RESULT;
573
574 if (pq->dwbes_f.q_val_err)
575 *desc->result |= SUM_CHECK_Q_RESULT;
576
577 return;
578 }
579 default:
580 return;
581 }
582 }
583
584 /**
585 * __cleanup - reclaim used descriptors
586 * @ioat: channel (ring) to clean
587 */
__cleanup(struct ioatdma_chan * ioat_chan,dma_addr_t phys_complete)588 static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
589 {
590 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
591 struct ioat_ring_ent *desc;
592 bool seen_current = false;
593 int idx = ioat_chan->tail, i;
594 u16 active;
595
596 dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
597 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
598
599 /*
600 * At restart of the channel, the completion address and the
601 * channel status will be 0 due to starting a new chain. Since
602 * it's new chain and the first descriptor "fails", there is
603 * nothing to clean up. We do not want to reap the entire submitted
604 * chain due to this 0 address value and then BUG.
605 */
606 if (!phys_complete)
607 return;
608
609 active = ioat_ring_active(ioat_chan);
610 for (i = 0; i < active && !seen_current; i++) {
611 struct dma_async_tx_descriptor *tx;
612
613 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
614 desc = ioat_get_ring_ent(ioat_chan, idx + i);
615 dump_desc_dbg(ioat_chan, desc);
616
617 /* set err stat if we are using dwbes */
618 if (ioat_dma->cap & IOAT_CAP_DWBES)
619 desc_get_errstat(ioat_chan, desc);
620
621 tx = &desc->txd;
622 if (tx->cookie) {
623 dma_cookie_complete(tx);
624 dma_descriptor_unmap(tx);
625 dmaengine_desc_get_callback_invoke(tx, NULL);
626 tx->callback = NULL;
627 tx->callback_result = NULL;
628 }
629
630 if (tx->phys == phys_complete)
631 seen_current = true;
632
633 /* skip extended descriptors */
634 if (desc_has_ext(desc)) {
635 BUG_ON(i + 1 >= active);
636 i++;
637 }
638
639 /* cleanup super extended descriptors */
640 if (desc->sed) {
641 ioat_free_sed(ioat_dma, desc->sed);
642 desc->sed = NULL;
643 }
644 }
645
646 /* finish all descriptor reads before incrementing tail */
647 smp_mb();
648 ioat_chan->tail = idx + i;
649 /* no active descs have written a completion? */
650 BUG_ON(active && !seen_current);
651 ioat_chan->last_completion = phys_complete;
652
653 if (active - i == 0) {
654 dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
655 __func__);
656 mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
657 }
658
659 /* microsecond delay by sysfs variable per pending descriptor */
660 if (ioat_chan->intr_coalesce != ioat_chan->prev_intr_coalesce) {
661 writew(min((ioat_chan->intr_coalesce * (active - i)),
662 IOAT_INTRDELAY_MASK),
663 ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
664 ioat_chan->prev_intr_coalesce = ioat_chan->intr_coalesce;
665 }
666 }
667
ioat_cleanup(struct ioatdma_chan * ioat_chan)668 static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
669 {
670 u64 phys_complete;
671
672 spin_lock_bh(&ioat_chan->cleanup_lock);
673
674 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
675 __cleanup(ioat_chan, phys_complete);
676
677 if (is_ioat_halted(*ioat_chan->completion)) {
678 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
679
680 if (chanerr &
681 (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) {
682 mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
683 ioat_eh(ioat_chan);
684 }
685 }
686
687 spin_unlock_bh(&ioat_chan->cleanup_lock);
688 }
689
ioat_cleanup_event(unsigned long data)690 void ioat_cleanup_event(unsigned long data)
691 {
692 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
693
694 ioat_cleanup(ioat_chan);
695 if (!test_bit(IOAT_RUN, &ioat_chan->state))
696 return;
697 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
698 }
699
ioat_restart_channel(struct ioatdma_chan * ioat_chan)700 static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
701 {
702 u64 phys_complete;
703
704 /* set the completion address register again */
705 writel(lower_32_bits(ioat_chan->completion_dma),
706 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
707 writel(upper_32_bits(ioat_chan->completion_dma),
708 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
709
710 ioat_quiesce(ioat_chan, 0);
711 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
712 __cleanup(ioat_chan, phys_complete);
713
714 __ioat_restart_chan(ioat_chan);
715 }
716
717
ioat_abort_descs(struct ioatdma_chan * ioat_chan)718 static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
719 {
720 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
721 struct ioat_ring_ent *desc;
722 u16 active;
723 int idx = ioat_chan->tail, i;
724
725 /*
726 * We assume that the failed descriptor has been processed.
727 * Now we are just returning all the remaining submitted
728 * descriptors to abort.
729 */
730 active = ioat_ring_active(ioat_chan);
731
732 /* we skip the failed descriptor that tail points to */
733 for (i = 1; i < active; i++) {
734 struct dma_async_tx_descriptor *tx;
735
736 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
737 desc = ioat_get_ring_ent(ioat_chan, idx + i);
738
739 tx = &desc->txd;
740 if (tx->cookie) {
741 struct dmaengine_result res;
742
743 dma_cookie_complete(tx);
744 dma_descriptor_unmap(tx);
745 res.result = DMA_TRANS_ABORTED;
746 dmaengine_desc_get_callback_invoke(tx, &res);
747 tx->callback = NULL;
748 tx->callback_result = NULL;
749 }
750
751 /* skip extended descriptors */
752 if (desc_has_ext(desc)) {
753 WARN_ON(i + 1 >= active);
754 i++;
755 }
756
757 /* cleanup super extended descriptors */
758 if (desc->sed) {
759 ioat_free_sed(ioat_dma, desc->sed);
760 desc->sed = NULL;
761 }
762 }
763
764 smp_mb(); /* finish all descriptor reads before incrementing tail */
765 ioat_chan->tail = idx + active;
766
767 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
768 ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys;
769 }
770
ioat_eh(struct ioatdma_chan * ioat_chan)771 static void ioat_eh(struct ioatdma_chan *ioat_chan)
772 {
773 struct pci_dev *pdev = to_pdev(ioat_chan);
774 struct ioat_dma_descriptor *hw;
775 struct dma_async_tx_descriptor *tx;
776 u64 phys_complete;
777 struct ioat_ring_ent *desc;
778 u32 err_handled = 0;
779 u32 chanerr_int;
780 u32 chanerr;
781 bool abort = false;
782 struct dmaengine_result res;
783
784 /* cleanup so tail points to descriptor that caused the error */
785 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
786 __cleanup(ioat_chan, phys_complete);
787
788 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
789 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
790
791 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
792 __func__, chanerr, chanerr_int);
793
794 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
795 hw = desc->hw;
796 dump_desc_dbg(ioat_chan, desc);
797
798 switch (hw->ctl_f.op) {
799 case IOAT_OP_XOR_VAL:
800 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
801 *desc->result |= SUM_CHECK_P_RESULT;
802 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
803 }
804 break;
805 case IOAT_OP_PQ_VAL:
806 case IOAT_OP_PQ_VAL_16S:
807 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
808 *desc->result |= SUM_CHECK_P_RESULT;
809 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
810 }
811 if (chanerr & IOAT_CHANERR_XOR_Q_ERR) {
812 *desc->result |= SUM_CHECK_Q_RESULT;
813 err_handled |= IOAT_CHANERR_XOR_Q_ERR;
814 }
815 break;
816 }
817
818 if (chanerr & IOAT_CHANERR_RECOVER_MASK) {
819 if (chanerr & IOAT_CHANERR_READ_DATA_ERR) {
820 res.result = DMA_TRANS_READ_FAILED;
821 err_handled |= IOAT_CHANERR_READ_DATA_ERR;
822 } else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) {
823 res.result = DMA_TRANS_WRITE_FAILED;
824 err_handled |= IOAT_CHANERR_WRITE_DATA_ERR;
825 }
826
827 abort = true;
828 } else
829 res.result = DMA_TRANS_NOERROR;
830
831 /* fault on unhandled error or spurious halt */
832 if (chanerr ^ err_handled || chanerr == 0) {
833 dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
834 __func__, chanerr, err_handled);
835 dev_err(to_dev(ioat_chan), "Errors handled:\n");
836 ioat_print_chanerrs(ioat_chan, err_handled);
837 dev_err(to_dev(ioat_chan), "Errors not handled:\n");
838 ioat_print_chanerrs(ioat_chan, (chanerr & ~err_handled));
839
840 BUG();
841 }
842
843 /* cleanup the faulty descriptor since we are continuing */
844 tx = &desc->txd;
845 if (tx->cookie) {
846 dma_cookie_complete(tx);
847 dma_descriptor_unmap(tx);
848 dmaengine_desc_get_callback_invoke(tx, &res);
849 tx->callback = NULL;
850 tx->callback_result = NULL;
851 }
852
853 /* mark faulting descriptor as complete */
854 *ioat_chan->completion = desc->txd.phys;
855
856 spin_lock_bh(&ioat_chan->prep_lock);
857 /* we need abort all descriptors */
858 if (abort) {
859 ioat_abort_descs(ioat_chan);
860 /* clean up the channel, we could be in weird state */
861 ioat_reset_hw(ioat_chan);
862 }
863
864 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
865 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
866
867 ioat_restart_channel(ioat_chan);
868 spin_unlock_bh(&ioat_chan->prep_lock);
869 }
870
check_active(struct ioatdma_chan * ioat_chan)871 static void check_active(struct ioatdma_chan *ioat_chan)
872 {
873 if (ioat_ring_active(ioat_chan)) {
874 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
875 return;
876 }
877
878 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
879 mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
880 }
881
ioat_timer_event(struct timer_list * t)882 void ioat_timer_event(struct timer_list *t)
883 {
884 struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer);
885 dma_addr_t phys_complete;
886 u64 status;
887
888 status = ioat_chansts(ioat_chan);
889
890 /* when halted due to errors check for channel
891 * programming errors before advancing the completion state
892 */
893 if (is_ioat_halted(status)) {
894 u32 chanerr;
895
896 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
897 dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
898 __func__, chanerr);
899 dev_err(to_dev(ioat_chan), "Errors:\n");
900 ioat_print_chanerrs(ioat_chan, chanerr);
901
902 if (test_bit(IOAT_RUN, &ioat_chan->state)) {
903 spin_lock_bh(&ioat_chan->cleanup_lock);
904 spin_lock_bh(&ioat_chan->prep_lock);
905 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
906 spin_unlock_bh(&ioat_chan->prep_lock);
907
908 ioat_abort_descs(ioat_chan);
909 dev_warn(to_dev(ioat_chan), "Reset channel...\n");
910 ioat_reset_hw(ioat_chan);
911 dev_warn(to_dev(ioat_chan), "Restart channel...\n");
912 ioat_restart_channel(ioat_chan);
913
914 spin_lock_bh(&ioat_chan->prep_lock);
915 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
916 spin_unlock_bh(&ioat_chan->prep_lock);
917 spin_unlock_bh(&ioat_chan->cleanup_lock);
918 }
919
920 return;
921 }
922
923 spin_lock_bh(&ioat_chan->cleanup_lock);
924
925 /* handle the no-actives case */
926 if (!ioat_ring_active(ioat_chan)) {
927 spin_lock_bh(&ioat_chan->prep_lock);
928 check_active(ioat_chan);
929 spin_unlock_bh(&ioat_chan->prep_lock);
930 spin_unlock_bh(&ioat_chan->cleanup_lock);
931 return;
932 }
933
934 /* if we haven't made progress and we have already
935 * acknowledged a pending completion once, then be more
936 * forceful with a restart
937 */
938 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
939 __cleanup(ioat_chan, phys_complete);
940 else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
941 u32 chanerr;
942
943 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
944 dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n",
945 status, chanerr);
946 dev_err(to_dev(ioat_chan), "Errors:\n");
947 ioat_print_chanerrs(ioat_chan, chanerr);
948
949 dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n",
950 ioat_ring_active(ioat_chan));
951
952 spin_lock_bh(&ioat_chan->prep_lock);
953 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
954 spin_unlock_bh(&ioat_chan->prep_lock);
955
956 ioat_abort_descs(ioat_chan);
957 dev_warn(to_dev(ioat_chan), "Resetting channel...\n");
958 ioat_reset_hw(ioat_chan);
959 dev_warn(to_dev(ioat_chan), "Restarting channel...\n");
960 ioat_restart_channel(ioat_chan);
961
962 spin_lock_bh(&ioat_chan->prep_lock);
963 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
964 spin_unlock_bh(&ioat_chan->prep_lock);
965 spin_unlock_bh(&ioat_chan->cleanup_lock);
966 return;
967 } else
968 set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
969
970 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
971 spin_unlock_bh(&ioat_chan->cleanup_lock);
972 }
973
974 enum dma_status
ioat_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)975 ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
976 struct dma_tx_state *txstate)
977 {
978 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
979 enum dma_status ret;
980
981 ret = dma_cookie_status(c, cookie, txstate);
982 if (ret == DMA_COMPLETE)
983 return ret;
984
985 ioat_cleanup(ioat_chan);
986
987 return dma_cookie_status(c, cookie, txstate);
988 }
989
ioat_reset_hw(struct ioatdma_chan * ioat_chan)990 int ioat_reset_hw(struct ioatdma_chan *ioat_chan)
991 {
992 /* throw away whatever the channel was doing and get it
993 * initialized, with ioat3 specific workarounds
994 */
995 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
996 struct pci_dev *pdev = ioat_dma->pdev;
997 u32 chanerr;
998 u16 dev_id;
999 int err;
1000
1001 ioat_quiesce(ioat_chan, msecs_to_jiffies(100));
1002
1003 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1004 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1005
1006 if (ioat_dma->version < IOAT_VER_3_3) {
1007 /* clear any pending errors */
1008 err = pci_read_config_dword(pdev,
1009 IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
1010 if (err) {
1011 dev_err(&pdev->dev,
1012 "channel error register unreachable\n");
1013 return err;
1014 }
1015 pci_write_config_dword(pdev,
1016 IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
1017
1018 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1019 * (workaround for spurious config parity error after restart)
1020 */
1021 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1022 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
1023 pci_write_config_dword(pdev,
1024 IOAT_PCI_DMAUNCERRSTS_OFFSET,
1025 0x10);
1026 }
1027 }
1028
1029 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1030 ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000);
1031 ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008);
1032 ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800);
1033 }
1034
1035
1036 err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200));
1037 if (!err) {
1038 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1039 writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000);
1040 writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008);
1041 writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800);
1042 }
1043 }
1044
1045 if (err)
1046 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1047
1048 return err;
1049 }
1050