1 // SPDX-License-Identifier: ISC
2 /*
3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
4 */
5
6 #include <linux/dma-mapping.h>
7 #include "mt76.h"
8 #include "dma.h"
9
10 static struct mt76_txwi_cache *
mt76_alloc_txwi(struct mt76_dev * dev)11 mt76_alloc_txwi(struct mt76_dev *dev)
12 {
13 struct mt76_txwi_cache *t;
14 dma_addr_t addr;
15 u8 *txwi;
16 int size;
17
18 size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
19 txwi = devm_kzalloc(dev->dev, size, GFP_ATOMIC);
20 if (!txwi)
21 return NULL;
22
23 addr = dma_map_single(dev->dev, txwi, dev->drv->txwi_size,
24 DMA_TO_DEVICE);
25 t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
26 t->dma_addr = addr;
27
28 return t;
29 }
30
31 static struct mt76_txwi_cache *
__mt76_get_txwi(struct mt76_dev * dev)32 __mt76_get_txwi(struct mt76_dev *dev)
33 {
34 struct mt76_txwi_cache *t = NULL;
35
36 spin_lock(&dev->lock);
37 if (!list_empty(&dev->txwi_cache)) {
38 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
39 list);
40 list_del(&t->list);
41 }
42 spin_unlock(&dev->lock);
43
44 return t;
45 }
46
47 static struct mt76_txwi_cache *
mt76_get_txwi(struct mt76_dev * dev)48 mt76_get_txwi(struct mt76_dev *dev)
49 {
50 struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
51
52 if (t)
53 return t;
54
55 return mt76_alloc_txwi(dev);
56 }
57
58 void
mt76_put_txwi(struct mt76_dev * dev,struct mt76_txwi_cache * t)59 mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
60 {
61 if (!t)
62 return;
63
64 spin_lock(&dev->lock);
65 list_add(&t->list, &dev->txwi_cache);
66 spin_unlock(&dev->lock);
67 }
68 EXPORT_SYMBOL_GPL(mt76_put_txwi);
69
70 static void
mt76_free_pending_txwi(struct mt76_dev * dev)71 mt76_free_pending_txwi(struct mt76_dev *dev)
72 {
73 struct mt76_txwi_cache *t;
74
75 local_bh_disable();
76 while ((t = __mt76_get_txwi(dev)) != NULL)
77 dma_unmap_single(dev->dev, t->dma_addr, dev->drv->txwi_size,
78 DMA_TO_DEVICE);
79 local_bh_enable();
80 }
81
82 static int
mt76_dma_alloc_queue(struct mt76_dev * dev,struct mt76_queue * q,int idx,int n_desc,int bufsize,u32 ring_base)83 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
84 int idx, int n_desc, int bufsize,
85 u32 ring_base)
86 {
87 int size;
88 int i;
89
90 spin_lock_init(&q->lock);
91
92 q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
93 q->ndesc = n_desc;
94 q->buf_size = bufsize;
95 q->hw_idx = idx;
96
97 size = q->ndesc * sizeof(struct mt76_desc);
98 q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
99 if (!q->desc)
100 return -ENOMEM;
101
102 size = q->ndesc * sizeof(*q->entry);
103 q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
104 if (!q->entry)
105 return -ENOMEM;
106
107 /* clear descriptors */
108 for (i = 0; i < q->ndesc; i++)
109 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
110
111 writel(q->desc_dma, &q->regs->desc_base);
112 writel(0, &q->regs->cpu_idx);
113 writel(0, &q->regs->dma_idx);
114 writel(q->ndesc, &q->regs->ring_size);
115
116 return 0;
117 }
118
119 static int
mt76_dma_add_buf(struct mt76_dev * dev,struct mt76_queue * q,struct mt76_queue_buf * buf,int nbufs,u32 info,struct sk_buff * skb,void * txwi)120 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
121 struct mt76_queue_buf *buf, int nbufs, u32 info,
122 struct sk_buff *skb, void *txwi)
123 {
124 struct mt76_queue_entry *entry;
125 struct mt76_desc *desc;
126 u32 ctrl;
127 int i, idx = -1;
128
129 if (txwi) {
130 q->entry[q->head].txwi = DMA_DUMMY_DATA;
131 q->entry[q->head].skip_buf0 = true;
132 }
133
134 for (i = 0; i < nbufs; i += 2, buf += 2) {
135 u32 buf0 = buf[0].addr, buf1 = 0;
136
137 idx = q->head;
138 q->head = (q->head + 1) % q->ndesc;
139
140 desc = &q->desc[idx];
141 entry = &q->entry[idx];
142
143 if (buf[0].skip_unmap)
144 entry->skip_buf0 = true;
145 entry->skip_buf1 = i == nbufs - 1;
146
147 entry->dma_addr[0] = buf[0].addr;
148 entry->dma_len[0] = buf[0].len;
149
150 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
151 if (i < nbufs - 1) {
152 entry->dma_addr[1] = buf[1].addr;
153 entry->dma_len[1] = buf[1].len;
154 buf1 = buf[1].addr;
155 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
156 if (buf[1].skip_unmap)
157 entry->skip_buf1 = true;
158 }
159
160 if (i == nbufs - 1)
161 ctrl |= MT_DMA_CTL_LAST_SEC0;
162 else if (i == nbufs - 2)
163 ctrl |= MT_DMA_CTL_LAST_SEC1;
164
165 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
166 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
167 WRITE_ONCE(desc->info, cpu_to_le32(info));
168 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
169
170 q->queued++;
171 }
172
173 q->entry[idx].txwi = txwi;
174 q->entry[idx].skb = skb;
175
176 return idx;
177 }
178
179 static void
mt76_dma_tx_cleanup_idx(struct mt76_dev * dev,struct mt76_queue * q,int idx,struct mt76_queue_entry * prev_e)180 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
181 struct mt76_queue_entry *prev_e)
182 {
183 struct mt76_queue_entry *e = &q->entry[idx];
184
185 if (!e->skip_buf0)
186 dma_unmap_single(dev->dev, e->dma_addr[0], e->dma_len[0],
187 DMA_TO_DEVICE);
188
189 if (!e->skip_buf1)
190 dma_unmap_single(dev->dev, e->dma_addr[1], e->dma_len[1],
191 DMA_TO_DEVICE);
192
193 if (e->txwi == DMA_DUMMY_DATA)
194 e->txwi = NULL;
195
196 if (e->skb == DMA_DUMMY_DATA)
197 e->skb = NULL;
198
199 *prev_e = *e;
200 memset(e, 0, sizeof(*e));
201 }
202
203 static void
mt76_dma_sync_idx(struct mt76_dev * dev,struct mt76_queue * q)204 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
205 {
206 writel(q->desc_dma, &q->regs->desc_base);
207 writel(q->ndesc, &q->regs->ring_size);
208 q->head = readl(&q->regs->dma_idx);
209 q->tail = q->head;
210 }
211
212 static void
mt76_dma_kick_queue(struct mt76_dev * dev,struct mt76_queue * q)213 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
214 {
215 wmb();
216 writel(q->head, &q->regs->cpu_idx);
217 }
218
219 static void
mt76_dma_tx_cleanup(struct mt76_dev * dev,enum mt76_txq_id qid,bool flush)220 mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
221 {
222 struct mt76_queue *q = dev->q_tx[qid];
223 struct mt76_queue_entry entry;
224 bool wake = false;
225 int last;
226
227 if (!q)
228 return;
229
230 if (flush)
231 last = -1;
232 else
233 last = readl(&q->regs->dma_idx);
234
235 while (q->queued > 0 && q->tail != last) {
236 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
237 mt76_queue_tx_complete(dev, q, &entry);
238
239 if (entry.txwi) {
240 if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
241 mt76_put_txwi(dev, entry.txwi);
242 wake = !flush;
243 }
244
245 if (!flush && q->tail == last)
246 last = readl(&q->regs->dma_idx);
247
248 }
249
250 if (flush) {
251 spin_lock_bh(&q->lock);
252 mt76_dma_sync_idx(dev, q);
253 mt76_dma_kick_queue(dev, q);
254 spin_unlock_bh(&q->lock);
255 }
256
257 wake = wake && q->stopped &&
258 qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
259 if (wake)
260 q->stopped = false;
261
262 if (!q->queued)
263 wake_up(&dev->tx_wait);
264
265 if (wake)
266 ieee80211_wake_queue(dev->hw, qid);
267 }
268
269 static void *
mt76_dma_get_buf(struct mt76_dev * dev,struct mt76_queue * q,int idx,int * len,u32 * info,bool * more)270 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
271 int *len, u32 *info, bool *more)
272 {
273 struct mt76_queue_entry *e = &q->entry[idx];
274 struct mt76_desc *desc = &q->desc[idx];
275 dma_addr_t buf_addr;
276 void *buf = e->buf;
277 int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
278
279 buf_addr = e->dma_addr[0];
280 if (len) {
281 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
282 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
283 *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
284 }
285
286 if (info)
287 *info = le32_to_cpu(desc->info);
288
289 dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
290 e->buf = NULL;
291
292 return buf;
293 }
294
295 static void *
mt76_dma_dequeue(struct mt76_dev * dev,struct mt76_queue * q,bool flush,int * len,u32 * info,bool * more)296 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
297 int *len, u32 *info, bool *more)
298 {
299 int idx = q->tail;
300
301 *more = false;
302 if (!q->queued)
303 return NULL;
304
305 if (flush)
306 q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
307 else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
308 return NULL;
309
310 q->tail = (q->tail + 1) % q->ndesc;
311 q->queued--;
312
313 return mt76_dma_get_buf(dev, q, idx, len, info, more);
314 }
315
316 static int
mt76_dma_tx_queue_skb_raw(struct mt76_dev * dev,enum mt76_txq_id qid,struct sk_buff * skb,u32 tx_info)317 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, enum mt76_txq_id qid,
318 struct sk_buff *skb, u32 tx_info)
319 {
320 struct mt76_queue *q = dev->q_tx[qid];
321 struct mt76_queue_buf buf = {};
322 dma_addr_t addr;
323
324 if (q->queued + 1 >= q->ndesc - 1)
325 goto error;
326
327 addr = dma_map_single(dev->dev, skb->data, skb->len,
328 DMA_TO_DEVICE);
329 if (unlikely(dma_mapping_error(dev->dev, addr)))
330 goto error;
331
332 buf.addr = addr;
333 buf.len = skb->len;
334
335 spin_lock_bh(&q->lock);
336 mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
337 mt76_dma_kick_queue(dev, q);
338 spin_unlock_bh(&q->lock);
339
340 return 0;
341
342 error:
343 dev_kfree_skb(skb);
344 return -ENOMEM;
345 }
346
347 static int
mt76_dma_tx_queue_skb(struct mt76_dev * dev,enum mt76_txq_id qid,struct sk_buff * skb,struct mt76_wcid * wcid,struct ieee80211_sta * sta)348 mt76_dma_tx_queue_skb(struct mt76_dev *dev, enum mt76_txq_id qid,
349 struct sk_buff *skb, struct mt76_wcid *wcid,
350 struct ieee80211_sta *sta)
351 {
352 struct mt76_queue *q = dev->q_tx[qid];
353 struct mt76_tx_info tx_info = {
354 .skb = skb,
355 };
356 struct ieee80211_hw *hw;
357 int len, n = 0, ret = -ENOMEM;
358 struct mt76_txwi_cache *t;
359 struct sk_buff *iter;
360 dma_addr_t addr;
361 u8 *txwi;
362
363 t = mt76_get_txwi(dev);
364 if (!t) {
365 hw = mt76_tx_status_get_hw(dev, skb);
366 ieee80211_free_txskb(hw, skb);
367 return -ENOMEM;
368 }
369 txwi = mt76_get_txwi_ptr(dev, t);
370
371 skb->prev = skb->next = NULL;
372 if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
373 mt76_insert_hdr_pad(skb);
374
375 len = skb_headlen(skb);
376 addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
377 if (unlikely(dma_mapping_error(dev->dev, addr)))
378 goto free;
379
380 tx_info.buf[n].addr = t->dma_addr;
381 tx_info.buf[n++].len = dev->drv->txwi_size;
382 tx_info.buf[n].addr = addr;
383 tx_info.buf[n++].len = len;
384
385 skb_walk_frags(skb, iter) {
386 if (n == ARRAY_SIZE(tx_info.buf))
387 goto unmap;
388
389 addr = dma_map_single(dev->dev, iter->data, iter->len,
390 DMA_TO_DEVICE);
391 if (unlikely(dma_mapping_error(dev->dev, addr)))
392 goto unmap;
393
394 tx_info.buf[n].addr = addr;
395 tx_info.buf[n++].len = iter->len;
396 }
397 tx_info.nbuf = n;
398
399 if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
400 ret = -ENOMEM;
401 goto unmap;
402 }
403
404 dma_sync_single_for_cpu(dev->dev, t->dma_addr, dev->drv->txwi_size,
405 DMA_TO_DEVICE);
406 ret = dev->drv->tx_prepare_skb(dev, txwi, qid, wcid, sta, &tx_info);
407 dma_sync_single_for_device(dev->dev, t->dma_addr, dev->drv->txwi_size,
408 DMA_TO_DEVICE);
409 if (ret < 0)
410 goto unmap;
411
412 return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
413 tx_info.info, tx_info.skb, t);
414
415 unmap:
416 for (n--; n > 0; n--)
417 dma_unmap_single(dev->dev, tx_info.buf[n].addr,
418 tx_info.buf[n].len, DMA_TO_DEVICE);
419
420 free:
421 #ifdef CONFIG_NL80211_TESTMODE
422 /* fix tx_done accounting on queue overflow */
423 if (tx_info.skb == dev->test.tx_skb)
424 dev->test.tx_done--;
425 #endif
426
427 dev_kfree_skb(tx_info.skb);
428 mt76_put_txwi(dev, t);
429 return ret;
430 }
431
432 static int
mt76_dma_rx_fill(struct mt76_dev * dev,struct mt76_queue * q)433 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
434 {
435 dma_addr_t addr;
436 void *buf;
437 int frames = 0;
438 int len = SKB_WITH_OVERHEAD(q->buf_size);
439 int offset = q->buf_offset;
440
441 spin_lock_bh(&q->lock);
442
443 while (q->queued < q->ndesc - 1) {
444 struct mt76_queue_buf qbuf;
445
446 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
447 if (!buf)
448 break;
449
450 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
451 if (unlikely(dma_mapping_error(dev->dev, addr))) {
452 skb_free_frag(buf);
453 break;
454 }
455
456 qbuf.addr = addr + offset;
457 qbuf.len = len - offset;
458 mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
459 frames++;
460 }
461
462 if (frames)
463 mt76_dma_kick_queue(dev, q);
464
465 spin_unlock_bh(&q->lock);
466
467 return frames;
468 }
469
470 static void
mt76_dma_rx_cleanup(struct mt76_dev * dev,struct mt76_queue * q)471 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
472 {
473 struct page *page;
474 void *buf;
475 bool more;
476
477 spin_lock_bh(&q->lock);
478 do {
479 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
480 if (!buf)
481 break;
482
483 skb_free_frag(buf);
484 } while (1);
485 spin_unlock_bh(&q->lock);
486
487 if (!q->rx_page.va)
488 return;
489
490 page = virt_to_page(q->rx_page.va);
491 __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
492 memset(&q->rx_page, 0, sizeof(q->rx_page));
493 }
494
495 static void
mt76_dma_rx_reset(struct mt76_dev * dev,enum mt76_rxq_id qid)496 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
497 {
498 struct mt76_queue *q = &dev->q_rx[qid];
499 int i;
500
501 for (i = 0; i < q->ndesc; i++)
502 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
503
504 mt76_dma_rx_cleanup(dev, q);
505 mt76_dma_sync_idx(dev, q);
506 mt76_dma_rx_fill(dev, q);
507
508 if (!q->rx_head)
509 return;
510
511 dev_kfree_skb(q->rx_head);
512 q->rx_head = NULL;
513 }
514
515 static void
mt76_add_fragment(struct mt76_dev * dev,struct mt76_queue * q,void * data,int len,bool more)516 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
517 int len, bool more)
518 {
519 struct sk_buff *skb = q->rx_head;
520 struct skb_shared_info *shinfo = skb_shinfo(skb);
521 int nr_frags = shinfo->nr_frags;
522
523 if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
524 struct page *page = virt_to_head_page(data);
525 int offset = data - page_address(page) + q->buf_offset;
526
527 skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
528 } else {
529 skb_free_frag(data);
530 }
531
532 if (more)
533 return;
534
535 q->rx_head = NULL;
536 if (nr_frags < ARRAY_SIZE(shinfo->frags))
537 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
538 else
539 dev_kfree_skb(skb);
540 }
541
542 static int
mt76_dma_rx_process(struct mt76_dev * dev,struct mt76_queue * q,int budget)543 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
544 {
545 int len, data_len, done = 0;
546 struct sk_buff *skb;
547 unsigned char *data;
548 bool more;
549
550 while (done < budget) {
551 u32 info;
552
553 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
554 if (!data)
555 break;
556
557 if (q->rx_head)
558 data_len = q->buf_size;
559 else
560 data_len = SKB_WITH_OVERHEAD(q->buf_size);
561
562 if (data_len < len + q->buf_offset) {
563 dev_kfree_skb(q->rx_head);
564 q->rx_head = NULL;
565
566 skb_free_frag(data);
567 continue;
568 }
569
570 if (q->rx_head) {
571 mt76_add_fragment(dev, q, data, len, more);
572 continue;
573 }
574
575 skb = build_skb(data, q->buf_size);
576 if (!skb) {
577 skb_free_frag(data);
578 continue;
579 }
580 skb_reserve(skb, q->buf_offset);
581
582 if (q == &dev->q_rx[MT_RXQ_MCU]) {
583 u32 *rxfce = (u32 *)skb->cb;
584 *rxfce = info;
585 }
586
587 __skb_put(skb, len);
588 done++;
589
590 if (more) {
591 q->rx_head = skb;
592 continue;
593 }
594
595 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
596 }
597
598 mt76_dma_rx_fill(dev, q);
599 return done;
600 }
601
602 static int
mt76_dma_rx_poll(struct napi_struct * napi,int budget)603 mt76_dma_rx_poll(struct napi_struct *napi, int budget)
604 {
605 struct mt76_dev *dev;
606 int qid, done = 0, cur;
607
608 dev = container_of(napi->dev, struct mt76_dev, napi_dev);
609 qid = napi - dev->napi;
610
611 local_bh_disable();
612 rcu_read_lock();
613
614 do {
615 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
616 mt76_rx_poll_complete(dev, qid, napi);
617 done += cur;
618 } while (cur && done < budget);
619
620 rcu_read_unlock();
621 local_bh_enable();
622
623 if (done < budget && napi_complete(napi))
624 dev->drv->rx_poll_complete(dev, qid);
625
626 return done;
627 }
628
629 static int
mt76_dma_init(struct mt76_dev * dev)630 mt76_dma_init(struct mt76_dev *dev)
631 {
632 int i;
633
634 init_dummy_netdev(&dev->napi_dev);
635
636 mt76_for_each_q_rx(dev, i) {
637 netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
638 64);
639 mt76_dma_rx_fill(dev, &dev->q_rx[i]);
640 napi_enable(&dev->napi[i]);
641 }
642
643 return 0;
644 }
645
646 static const struct mt76_queue_ops mt76_dma_ops = {
647 .init = mt76_dma_init,
648 .alloc = mt76_dma_alloc_queue,
649 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
650 .tx_queue_skb = mt76_dma_tx_queue_skb,
651 .tx_cleanup = mt76_dma_tx_cleanup,
652 .rx_reset = mt76_dma_rx_reset,
653 .kick = mt76_dma_kick_queue,
654 };
655
mt76_dma_attach(struct mt76_dev * dev)656 void mt76_dma_attach(struct mt76_dev *dev)
657 {
658 dev->queue_ops = &mt76_dma_ops;
659 }
660 EXPORT_SYMBOL_GPL(mt76_dma_attach);
661
mt76_dma_cleanup(struct mt76_dev * dev)662 void mt76_dma_cleanup(struct mt76_dev *dev)
663 {
664 int i;
665
666 mt76_worker_disable(&dev->tx_worker);
667 netif_napi_del(&dev->tx_napi);
668 for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
669 mt76_dma_tx_cleanup(dev, i, true);
670
671 mt76_for_each_q_rx(dev, i) {
672 netif_napi_del(&dev->napi[i]);
673 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
674 }
675
676 mt76_free_pending_txwi(dev);
677 }
678 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);
679