1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2018-2019 Realtek Corporation
3 */
4
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include "main.h"
8 #include "pci.h"
9 #include "reg.h"
10 #include "tx.h"
11 #include "rx.h"
12 #include "fw.h"
13 #include "ps.h"
14 #include "debug.h"
15
16 static bool rtw_disable_msi;
17 static bool rtw_pci_disable_aspm;
18 module_param_named(disable_msi, rtw_disable_msi, bool, 0644);
19 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644);
20 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support");
21 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support");
22
23 static u32 rtw_pci_tx_queue_idx_addr[] = {
24 [RTW_TX_QUEUE_BK] = RTK_PCI_TXBD_IDX_BKQ,
25 [RTW_TX_QUEUE_BE] = RTK_PCI_TXBD_IDX_BEQ,
26 [RTW_TX_QUEUE_VI] = RTK_PCI_TXBD_IDX_VIQ,
27 [RTW_TX_QUEUE_VO] = RTK_PCI_TXBD_IDX_VOQ,
28 [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ,
29 [RTW_TX_QUEUE_HI0] = RTK_PCI_TXBD_IDX_HI0Q,
30 [RTW_TX_QUEUE_H2C] = RTK_PCI_TXBD_IDX_H2CQ,
31 };
32
rtw_pci_get_tx_qsel(struct sk_buff * skb,u8 queue)33 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, u8 queue)
34 {
35 switch (queue) {
36 case RTW_TX_QUEUE_BCN:
37 return TX_DESC_QSEL_BEACON;
38 case RTW_TX_QUEUE_H2C:
39 return TX_DESC_QSEL_H2C;
40 case RTW_TX_QUEUE_MGMT:
41 return TX_DESC_QSEL_MGMT;
42 case RTW_TX_QUEUE_HI0:
43 return TX_DESC_QSEL_HIGH;
44 default:
45 return skb->priority;
46 }
47 };
48
rtw_pci_read8(struct rtw_dev * rtwdev,u32 addr)49 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr)
50 {
51 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
52
53 return readb(rtwpci->mmap + addr);
54 }
55
rtw_pci_read16(struct rtw_dev * rtwdev,u32 addr)56 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr)
57 {
58 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
59
60 return readw(rtwpci->mmap + addr);
61 }
62
rtw_pci_read32(struct rtw_dev * rtwdev,u32 addr)63 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr)
64 {
65 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
66
67 return readl(rtwpci->mmap + addr);
68 }
69
rtw_pci_write8(struct rtw_dev * rtwdev,u32 addr,u8 val)70 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val)
71 {
72 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
73
74 writeb(val, rtwpci->mmap + addr);
75 }
76
rtw_pci_write16(struct rtw_dev * rtwdev,u32 addr,u16 val)77 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val)
78 {
79 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
80
81 writew(val, rtwpci->mmap + addr);
82 }
83
rtw_pci_write32(struct rtw_dev * rtwdev,u32 addr,u32 val)84 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
85 {
86 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
87
88 writel(val, rtwpci->mmap + addr);
89 }
90
rtw_pci_get_tx_desc(struct rtw_pci_tx_ring * tx_ring,u8 idx)91 static inline void *rtw_pci_get_tx_desc(struct rtw_pci_tx_ring *tx_ring, u8 idx)
92 {
93 int offset = tx_ring->r.desc_size * idx;
94
95 return tx_ring->r.head + offset;
96 }
97
rtw_pci_free_tx_ring_skbs(struct rtw_dev * rtwdev,struct rtw_pci_tx_ring * tx_ring)98 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev,
99 struct rtw_pci_tx_ring *tx_ring)
100 {
101 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
102 struct rtw_pci_tx_data *tx_data;
103 struct sk_buff *skb, *tmp;
104 dma_addr_t dma;
105
106 /* free every skb remained in tx list */
107 skb_queue_walk_safe(&tx_ring->queue, skb, tmp) {
108 __skb_unlink(skb, &tx_ring->queue);
109 tx_data = rtw_pci_get_tx_data(skb);
110 dma = tx_data->dma;
111
112 dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE);
113 dev_kfree_skb_any(skb);
114 }
115 }
116
rtw_pci_free_tx_ring(struct rtw_dev * rtwdev,struct rtw_pci_tx_ring * tx_ring)117 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev,
118 struct rtw_pci_tx_ring *tx_ring)
119 {
120 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
121 u8 *head = tx_ring->r.head;
122 u32 len = tx_ring->r.len;
123 int ring_sz = len * tx_ring->r.desc_size;
124
125 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
126
127 /* free the ring itself */
128 dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma);
129 tx_ring->r.head = NULL;
130 }
131
rtw_pci_free_rx_ring_skbs(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring)132 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev,
133 struct rtw_pci_rx_ring *rx_ring)
134 {
135 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
136 struct sk_buff *skb;
137 int buf_sz = RTK_PCI_RX_BUF_SIZE;
138 dma_addr_t dma;
139 int i;
140
141 for (i = 0; i < rx_ring->r.len; i++) {
142 skb = rx_ring->buf[i];
143 if (!skb)
144 continue;
145
146 dma = *((dma_addr_t *)skb->cb);
147 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
148 dev_kfree_skb(skb);
149 rx_ring->buf[i] = NULL;
150 }
151 }
152
rtw_pci_free_rx_ring(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring)153 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev,
154 struct rtw_pci_rx_ring *rx_ring)
155 {
156 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
157 u8 *head = rx_ring->r.head;
158 int ring_sz = rx_ring->r.desc_size * rx_ring->r.len;
159
160 rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring);
161
162 dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma);
163 }
164
rtw_pci_free_trx_ring(struct rtw_dev * rtwdev)165 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev)
166 {
167 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
168 struct rtw_pci_tx_ring *tx_ring;
169 struct rtw_pci_rx_ring *rx_ring;
170 int i;
171
172 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
173 tx_ring = &rtwpci->tx_rings[i];
174 rtw_pci_free_tx_ring(rtwdev, tx_ring);
175 }
176
177 for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) {
178 rx_ring = &rtwpci->rx_rings[i];
179 rtw_pci_free_rx_ring(rtwdev, rx_ring);
180 }
181 }
182
rtw_pci_init_tx_ring(struct rtw_dev * rtwdev,struct rtw_pci_tx_ring * tx_ring,u8 desc_size,u32 len)183 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev,
184 struct rtw_pci_tx_ring *tx_ring,
185 u8 desc_size, u32 len)
186 {
187 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
188 int ring_sz = desc_size * len;
189 dma_addr_t dma;
190 u8 *head;
191
192 if (len > TRX_BD_IDX_MASK) {
193 rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len);
194 return -EINVAL;
195 }
196
197 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
198 if (!head) {
199 rtw_err(rtwdev, "failed to allocate tx ring\n");
200 return -ENOMEM;
201 }
202
203 skb_queue_head_init(&tx_ring->queue);
204 tx_ring->r.head = head;
205 tx_ring->r.dma = dma;
206 tx_ring->r.len = len;
207 tx_ring->r.desc_size = desc_size;
208 tx_ring->r.wp = 0;
209 tx_ring->r.rp = 0;
210
211 return 0;
212 }
213
rtw_pci_reset_rx_desc(struct rtw_dev * rtwdev,struct sk_buff * skb,struct rtw_pci_rx_ring * rx_ring,u32 idx,u32 desc_sz)214 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb,
215 struct rtw_pci_rx_ring *rx_ring,
216 u32 idx, u32 desc_sz)
217 {
218 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
219 struct rtw_pci_rx_buffer_desc *buf_desc;
220 int buf_sz = RTK_PCI_RX_BUF_SIZE;
221 dma_addr_t dma;
222
223 if (!skb)
224 return -EINVAL;
225
226 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
227 if (dma_mapping_error(&pdev->dev, dma))
228 return -EBUSY;
229
230 *((dma_addr_t *)skb->cb) = dma;
231 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
232 idx * desc_sz);
233 memset(buf_desc, 0, sizeof(*buf_desc));
234 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
235 buf_desc->dma = cpu_to_le32(dma);
236
237 return 0;
238 }
239
rtw_pci_sync_rx_desc_device(struct rtw_dev * rtwdev,dma_addr_t dma,struct rtw_pci_rx_ring * rx_ring,u32 idx,u32 desc_sz)240 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma,
241 struct rtw_pci_rx_ring *rx_ring,
242 u32 idx, u32 desc_sz)
243 {
244 struct device *dev = rtwdev->dev;
245 struct rtw_pci_rx_buffer_desc *buf_desc;
246 int buf_sz = RTK_PCI_RX_BUF_SIZE;
247
248 dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE);
249
250 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
251 idx * desc_sz);
252 memset(buf_desc, 0, sizeof(*buf_desc));
253 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
254 buf_desc->dma = cpu_to_le32(dma);
255 }
256
rtw_pci_init_rx_ring(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring,u8 desc_size,u32 len)257 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev,
258 struct rtw_pci_rx_ring *rx_ring,
259 u8 desc_size, u32 len)
260 {
261 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
262 struct sk_buff *skb = NULL;
263 dma_addr_t dma;
264 u8 *head;
265 int ring_sz = desc_size * len;
266 int buf_sz = RTK_PCI_RX_BUF_SIZE;
267 int i, allocated;
268 int ret = 0;
269
270 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
271 if (!head) {
272 rtw_err(rtwdev, "failed to allocate rx ring\n");
273 return -ENOMEM;
274 }
275 rx_ring->r.head = head;
276
277 for (i = 0; i < len; i++) {
278 skb = dev_alloc_skb(buf_sz);
279 if (!skb) {
280 allocated = i;
281 ret = -ENOMEM;
282 goto err_out;
283 }
284
285 memset(skb->data, 0, buf_sz);
286 rx_ring->buf[i] = skb;
287 ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size);
288 if (ret) {
289 allocated = i;
290 dev_kfree_skb_any(skb);
291 goto err_out;
292 }
293 }
294
295 rx_ring->r.dma = dma;
296 rx_ring->r.len = len;
297 rx_ring->r.desc_size = desc_size;
298 rx_ring->r.wp = 0;
299 rx_ring->r.rp = 0;
300
301 return 0;
302
303 err_out:
304 for (i = 0; i < allocated; i++) {
305 skb = rx_ring->buf[i];
306 if (!skb)
307 continue;
308 dma = *((dma_addr_t *)skb->cb);
309 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
310 dev_kfree_skb_any(skb);
311 rx_ring->buf[i] = NULL;
312 }
313 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
314
315 rtw_err(rtwdev, "failed to init rx buffer\n");
316
317 return ret;
318 }
319
rtw_pci_init_trx_ring(struct rtw_dev * rtwdev)320 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev)
321 {
322 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
323 struct rtw_pci_tx_ring *tx_ring;
324 struct rtw_pci_rx_ring *rx_ring;
325 const struct rtw_chip_info *chip = rtwdev->chip;
326 int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0;
327 int tx_desc_size, rx_desc_size;
328 u32 len;
329 int ret;
330
331 tx_desc_size = chip->tx_buf_desc_sz;
332
333 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
334 tx_ring = &rtwpci->tx_rings[i];
335 len = max_num_of_tx_queue(i);
336 ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len);
337 if (ret)
338 goto out;
339 }
340
341 rx_desc_size = chip->rx_buf_desc_sz;
342
343 for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) {
344 rx_ring = &rtwpci->rx_rings[j];
345 ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size,
346 RTK_MAX_RX_DESC_NUM);
347 if (ret)
348 goto out;
349 }
350
351 return 0;
352
353 out:
354 tx_alloced = i;
355 for (i = 0; i < tx_alloced; i++) {
356 tx_ring = &rtwpci->tx_rings[i];
357 rtw_pci_free_tx_ring(rtwdev, tx_ring);
358 }
359
360 rx_alloced = j;
361 for (j = 0; j < rx_alloced; j++) {
362 rx_ring = &rtwpci->rx_rings[j];
363 rtw_pci_free_rx_ring(rtwdev, rx_ring);
364 }
365
366 return ret;
367 }
368
rtw_pci_deinit(struct rtw_dev * rtwdev)369 static void rtw_pci_deinit(struct rtw_dev *rtwdev)
370 {
371 rtw_pci_free_trx_ring(rtwdev);
372 }
373
rtw_pci_init(struct rtw_dev * rtwdev)374 static int rtw_pci_init(struct rtw_dev *rtwdev)
375 {
376 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
377 int ret = 0;
378
379 rtwpci->irq_mask[0] = IMR_HIGHDOK |
380 IMR_MGNTDOK |
381 IMR_BKDOK |
382 IMR_BEDOK |
383 IMR_VIDOK |
384 IMR_VODOK |
385 IMR_ROK |
386 IMR_BCNDMAINT_E |
387 IMR_C2HCMD |
388 0;
389 rtwpci->irq_mask[1] = IMR_TXFOVW |
390 0;
391 rtwpci->irq_mask[3] = IMR_H2CDOK |
392 0;
393 spin_lock_init(&rtwpci->irq_lock);
394 spin_lock_init(&rtwpci->hwirq_lock);
395 ret = rtw_pci_init_trx_ring(rtwdev);
396
397 return ret;
398 }
399
rtw_pci_reset_buf_desc(struct rtw_dev * rtwdev)400 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev)
401 {
402 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
403 u32 len;
404 u8 tmp;
405 dma_addr_t dma;
406
407 tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3);
408 rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7);
409
410 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;
411 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma);
412
413 if (!rtw_chip_wcpu_11n(rtwdev)) {
414 len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;
415 dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;
416 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;
417 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0;
418 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK);
419 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma);
420 }
421
422 len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len;
423 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma;
424 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0;
425 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0;
426 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK);
427 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma);
428
429 len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len;
430 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma;
431 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0;
432 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0;
433 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK);
434 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma);
435
436 len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len;
437 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma;
438 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0;
439 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0;
440 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK);
441 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma);
442
443 len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len;
444 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma;
445 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0;
446 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0;
447 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK);
448 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma);
449
450 len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len;
451 dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma;
452 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0;
453 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0;
454 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK);
455 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma);
456
457 len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len;
458 dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma;
459 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0;
460 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0;
461 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK);
462 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma);
463
464 len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len;
465 dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma;
466 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0;
467 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0;
468 rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK);
469 rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma);
470
471 /* reset read/write point */
472 rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff);
473
474 /* reset H2C Queue index in a single write */
475 if (rtw_chip_wcpu_11ac(rtwdev))
476 rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR,
477 BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX);
478 }
479
rtw_pci_reset_trx_ring(struct rtw_dev * rtwdev)480 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev)
481 {
482 rtw_pci_reset_buf_desc(rtwdev);
483 }
484
rtw_pci_enable_interrupt(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,bool exclude_rx)485 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev,
486 struct rtw_pci *rtwpci, bool exclude_rx)
487 {
488 unsigned long flags;
489 u32 imr0_unmask = exclude_rx ? IMR_ROK : 0;
490
491 spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
492
493 rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);
494 rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);
495 if (rtw_chip_wcpu_11ac(rtwdev))
496 rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);
497
498 rtwpci->irq_enabled = true;
499
500 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
501 }
502
rtw_pci_disable_interrupt(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)503 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev,
504 struct rtw_pci *rtwpci)
505 {
506 unsigned long flags;
507
508 spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
509
510 if (!rtwpci->irq_enabled)
511 goto out;
512
513 rtw_write32(rtwdev, RTK_PCI_HIMR0, 0);
514 rtw_write32(rtwdev, RTK_PCI_HIMR1, 0);
515 if (rtw_chip_wcpu_11ac(rtwdev))
516 rtw_write32(rtwdev, RTK_PCI_HIMR3, 0);
517
518 rtwpci->irq_enabled = false;
519
520 out:
521 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
522 }
523
rtw_pci_dma_reset(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)524 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
525 {
526 /* reset dma and rx tag */
527 rtw_write32_set(rtwdev, RTK_PCI_CTRL,
528 BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN);
529 rtwpci->rx_tag = 0;
530 }
531
rtw_pci_setup(struct rtw_dev * rtwdev)532 static int rtw_pci_setup(struct rtw_dev *rtwdev)
533 {
534 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
535
536 rtw_pci_reset_trx_ring(rtwdev);
537 rtw_pci_dma_reset(rtwdev, rtwpci);
538
539 return 0;
540 }
541
rtw_pci_dma_release(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)542 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
543 {
544 struct rtw_pci_tx_ring *tx_ring;
545 u8 queue;
546
547 rtw_pci_reset_trx_ring(rtwdev);
548 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
549 tx_ring = &rtwpci->tx_rings[queue];
550 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
551 }
552 }
553
rtw_pci_napi_start(struct rtw_dev * rtwdev)554 static void rtw_pci_napi_start(struct rtw_dev *rtwdev)
555 {
556 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
557
558 if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
559 return;
560
561 napi_enable(&rtwpci->napi);
562 }
563
rtw_pci_napi_stop(struct rtw_dev * rtwdev)564 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev)
565 {
566 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
567
568 if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
569 return;
570
571 napi_synchronize(&rtwpci->napi);
572 napi_disable(&rtwpci->napi);
573 }
574
rtw_pci_start(struct rtw_dev * rtwdev)575 static int rtw_pci_start(struct rtw_dev *rtwdev)
576 {
577 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
578
579 rtw_pci_napi_start(rtwdev);
580
581 spin_lock_bh(&rtwpci->irq_lock);
582 rtwpci->running = true;
583 rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
584 spin_unlock_bh(&rtwpci->irq_lock);
585
586 return 0;
587 }
588
rtw_pci_stop(struct rtw_dev * rtwdev)589 static void rtw_pci_stop(struct rtw_dev *rtwdev)
590 {
591 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
592 struct pci_dev *pdev = rtwpci->pdev;
593
594 spin_lock_bh(&rtwpci->irq_lock);
595 rtwpci->running = false;
596 rtw_pci_disable_interrupt(rtwdev, rtwpci);
597 spin_unlock_bh(&rtwpci->irq_lock);
598
599 synchronize_irq(pdev->irq);
600 rtw_pci_napi_stop(rtwdev);
601
602 spin_lock_bh(&rtwpci->irq_lock);
603 rtw_pci_dma_release(rtwdev, rtwpci);
604 spin_unlock_bh(&rtwpci->irq_lock);
605 }
606
rtw_pci_deep_ps_enter(struct rtw_dev * rtwdev)607 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev)
608 {
609 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
610 struct rtw_pci_tx_ring *tx_ring;
611 bool tx_empty = true;
612 u8 queue;
613
614 if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
615 goto enter_deep_ps;
616
617 lockdep_assert_held(&rtwpci->irq_lock);
618
619 /* Deep PS state is not allowed to TX-DMA */
620 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
621 /* BCN queue is rsvd page, does not have DMA interrupt
622 * H2C queue is managed by firmware
623 */
624 if (queue == RTW_TX_QUEUE_BCN ||
625 queue == RTW_TX_QUEUE_H2C)
626 continue;
627
628 tx_ring = &rtwpci->tx_rings[queue];
629
630 /* check if there is any skb DMAing */
631 if (skb_queue_len(&tx_ring->queue)) {
632 tx_empty = false;
633 break;
634 }
635 }
636
637 if (!tx_empty) {
638 rtw_dbg(rtwdev, RTW_DBG_PS,
639 "TX path not empty, cannot enter deep power save state\n");
640 return;
641 }
642 enter_deep_ps:
643 set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags);
644 rtw_power_mode_change(rtwdev, true);
645 }
646
rtw_pci_deep_ps_leave(struct rtw_dev * rtwdev)647 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev)
648 {
649 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
650
651 lockdep_assert_held(&rtwpci->irq_lock);
652
653 if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
654 rtw_power_mode_change(rtwdev, false);
655 }
656
rtw_pci_deep_ps(struct rtw_dev * rtwdev,bool enter)657 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter)
658 {
659 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
660
661 spin_lock_bh(&rtwpci->irq_lock);
662
663 if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
664 rtw_pci_deep_ps_enter(rtwdev);
665
666 if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
667 rtw_pci_deep_ps_leave(rtwdev);
668
669 spin_unlock_bh(&rtwpci->irq_lock);
670 }
671
672 static u8 ac_to_hwq[] = {
673 [IEEE80211_AC_VO] = RTW_TX_QUEUE_VO,
674 [IEEE80211_AC_VI] = RTW_TX_QUEUE_VI,
675 [IEEE80211_AC_BE] = RTW_TX_QUEUE_BE,
676 [IEEE80211_AC_BK] = RTW_TX_QUEUE_BK,
677 };
678
679 static_assert(ARRAY_SIZE(ac_to_hwq) == IEEE80211_NUM_ACS);
680
rtw_hw_queue_mapping(struct sk_buff * skb)681 static u8 rtw_hw_queue_mapping(struct sk_buff *skb)
682 {
683 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
684 __le16 fc = hdr->frame_control;
685 u8 q_mapping = skb_get_queue_mapping(skb);
686 u8 queue;
687
688 if (unlikely(ieee80211_is_beacon(fc)))
689 queue = RTW_TX_QUEUE_BCN;
690 else if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc)))
691 queue = RTW_TX_QUEUE_MGMT;
692 else if (is_broadcast_ether_addr(hdr->addr1) ||
693 is_multicast_ether_addr(hdr->addr1))
694 queue = RTW_TX_QUEUE_HI0;
695 else if (WARN_ON_ONCE(q_mapping >= ARRAY_SIZE(ac_to_hwq)))
696 queue = ac_to_hwq[IEEE80211_AC_BE];
697 else
698 queue = ac_to_hwq[q_mapping];
699
700 return queue;
701 }
702
rtw_pci_release_rsvd_page(struct rtw_pci * rtwpci,struct rtw_pci_tx_ring * ring)703 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci,
704 struct rtw_pci_tx_ring *ring)
705 {
706 struct sk_buff *prev = skb_dequeue(&ring->queue);
707 struct rtw_pci_tx_data *tx_data;
708 dma_addr_t dma;
709
710 if (!prev)
711 return;
712
713 tx_data = rtw_pci_get_tx_data(prev);
714 dma = tx_data->dma;
715 dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE);
716 dev_kfree_skb_any(prev);
717 }
718
rtw_pci_dma_check(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring,u32 idx)719 static void rtw_pci_dma_check(struct rtw_dev *rtwdev,
720 struct rtw_pci_rx_ring *rx_ring,
721 u32 idx)
722 {
723 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
724 const struct rtw_chip_info *chip = rtwdev->chip;
725 struct rtw_pci_rx_buffer_desc *buf_desc;
726 u32 desc_sz = chip->rx_buf_desc_sz;
727 u16 total_pkt_size;
728
729 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
730 idx * desc_sz);
731 total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size);
732
733 /* rx tag mismatch, throw a warning */
734 if (total_pkt_size != rtwpci->rx_tag)
735 rtw_warn(rtwdev, "pci bus timeout, check dma status\n");
736
737 rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX;
738 }
739
__pci_get_hw_tx_ring_rp(struct rtw_dev * rtwdev,u8 pci_q)740 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q)
741 {
742 u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q];
743 u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2);
744
745 return FIELD_GET(TRX_BD_IDX_MASK, bd_idx);
746 }
747
__pci_flush_queue(struct rtw_dev * rtwdev,u8 pci_q,bool drop)748 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop)
749 {
750 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
751 struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q];
752 u32 cur_rp;
753 u8 i;
754
755 /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a
756 * bit dynamic, it's hard to define a reasonable fixed total timeout to
757 * use read_poll_timeout* helper. Instead, we can ensure a reasonable
758 * polling times, so we just use for loop with udelay here.
759 */
760 for (i = 0; i < 30; i++) {
761 cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q);
762 if (cur_rp == ring->r.wp)
763 return;
764
765 udelay(1);
766 }
767
768 if (!drop)
769 rtw_warn(rtwdev, "timed out to flush pci tx ring[%d]\n", pci_q);
770 }
771
__rtw_pci_flush_queues(struct rtw_dev * rtwdev,u32 pci_queues,bool drop)772 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues,
773 bool drop)
774 {
775 u8 q;
776
777 for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) {
778 /* It may be not necessary to flush BCN and H2C tx queues. */
779 if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C)
780 continue;
781
782 if (pci_queues & BIT(q))
783 __pci_flush_queue(rtwdev, q, drop);
784 }
785 }
786
rtw_pci_flush_queues(struct rtw_dev * rtwdev,u32 queues,bool drop)787 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop)
788 {
789 u32 pci_queues = 0;
790 u8 i;
791
792 /* If all of the hardware queues are requested to flush,
793 * flush all of the pci queues.
794 */
795 if (queues == BIT(rtwdev->hw->queues) - 1) {
796 pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1;
797 } else {
798 for (i = 0; i < rtwdev->hw->queues; i++)
799 if (queues & BIT(i))
800 pci_queues |= BIT(ac_to_hwq[i]);
801 }
802
803 __rtw_pci_flush_queues(rtwdev, pci_queues, drop);
804 }
805
rtw_pci_tx_kick_off_queue(struct rtw_dev * rtwdev,u8 queue)806 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, u8 queue)
807 {
808 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
809 struct rtw_pci_tx_ring *ring;
810 u32 bd_idx;
811
812 ring = &rtwpci->tx_rings[queue];
813 bd_idx = rtw_pci_tx_queue_idx_addr[queue];
814
815 spin_lock_bh(&rtwpci->irq_lock);
816 if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
817 rtw_pci_deep_ps_leave(rtwdev);
818 rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK);
819 spin_unlock_bh(&rtwpci->irq_lock);
820 }
821
rtw_pci_tx_kick_off(struct rtw_dev * rtwdev)822 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev)
823 {
824 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
825 u8 queue;
826
827 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++)
828 if (test_and_clear_bit(queue, rtwpci->tx_queued))
829 rtw_pci_tx_kick_off_queue(rtwdev, queue);
830 }
831
rtw_pci_tx_write_data(struct rtw_dev * rtwdev,struct rtw_tx_pkt_info * pkt_info,struct sk_buff * skb,u8 queue)832 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev,
833 struct rtw_tx_pkt_info *pkt_info,
834 struct sk_buff *skb, u8 queue)
835 {
836 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
837 const struct rtw_chip_info *chip = rtwdev->chip;
838 struct rtw_pci_tx_ring *ring;
839 struct rtw_pci_tx_data *tx_data;
840 dma_addr_t dma;
841 u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz;
842 u32 tx_buf_desc_sz = chip->tx_buf_desc_sz;
843 u32 size;
844 u32 psb_len;
845 u8 *pkt_desc;
846 struct rtw_pci_tx_buffer_desc *buf_desc;
847
848 ring = &rtwpci->tx_rings[queue];
849
850 size = skb->len;
851
852 if (queue == RTW_TX_QUEUE_BCN)
853 rtw_pci_release_rsvd_page(rtwpci, ring);
854 else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len))
855 return -ENOSPC;
856
857 pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz);
858 memset(pkt_desc, 0, tx_pkt_desc_sz);
859 pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue);
860 rtw_tx_fill_tx_desc(pkt_info, skb);
861 dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len,
862 DMA_TO_DEVICE);
863 if (dma_mapping_error(&rtwpci->pdev->dev, dma))
864 return -EBUSY;
865
866 /* after this we got dma mapped, there is no way back */
867 buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz);
868 memset(buf_desc, 0, tx_buf_desc_sz);
869 psb_len = (skb->len - 1) / 128 + 1;
870 if (queue == RTW_TX_QUEUE_BCN)
871 psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET;
872
873 buf_desc[0].psb_len = cpu_to_le16(psb_len);
874 buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz);
875 buf_desc[0].dma = cpu_to_le32(dma);
876 buf_desc[1].buf_size = cpu_to_le16(size);
877 buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz);
878
879 tx_data = rtw_pci_get_tx_data(skb);
880 tx_data->dma = dma;
881 tx_data->sn = pkt_info->sn;
882
883 spin_lock_bh(&rtwpci->irq_lock);
884
885 skb_queue_tail(&ring->queue, skb);
886
887 if (queue == RTW_TX_QUEUE_BCN)
888 goto out_unlock;
889
890 /* update write-index, and kick it off later */
891 set_bit(queue, rtwpci->tx_queued);
892 if (++ring->r.wp >= ring->r.len)
893 ring->r.wp = 0;
894
895 out_unlock:
896 spin_unlock_bh(&rtwpci->irq_lock);
897
898 return 0;
899 }
900
rtw_pci_write_data_rsvd_page(struct rtw_dev * rtwdev,u8 * buf,u32 size)901 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf,
902 u32 size)
903 {
904 struct sk_buff *skb;
905 struct rtw_tx_pkt_info pkt_info = {0};
906 u8 reg_bcn_work;
907 int ret;
908
909 skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size);
910 if (!skb)
911 return -ENOMEM;
912
913 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN);
914 if (ret) {
915 rtw_err(rtwdev, "failed to write rsvd page data\n");
916 return ret;
917 }
918
919 /* reserved pages go through beacon queue */
920 reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK);
921 reg_bcn_work |= BIT_PCI_BCNQ_FLAG;
922 rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work);
923
924 return 0;
925 }
926
rtw_pci_write_data_h2c(struct rtw_dev * rtwdev,u8 * buf,u32 size)927 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size)
928 {
929 struct sk_buff *skb;
930 struct rtw_tx_pkt_info pkt_info = {0};
931 int ret;
932
933 skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size);
934 if (!skb)
935 return -ENOMEM;
936
937 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C);
938 if (ret) {
939 rtw_err(rtwdev, "failed to write h2c data\n");
940 return ret;
941 }
942
943 rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C);
944
945 return 0;
946 }
947
rtw_pci_tx_write(struct rtw_dev * rtwdev,struct rtw_tx_pkt_info * pkt_info,struct sk_buff * skb)948 static int rtw_pci_tx_write(struct rtw_dev *rtwdev,
949 struct rtw_tx_pkt_info *pkt_info,
950 struct sk_buff *skb)
951 {
952 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
953 struct rtw_pci_tx_ring *ring;
954 u8 queue = rtw_hw_queue_mapping(skb);
955 int ret;
956
957 ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue);
958 if (ret)
959 return ret;
960
961 ring = &rtwpci->tx_rings[queue];
962 spin_lock_bh(&rtwpci->irq_lock);
963 if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) {
964 ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb));
965 ring->queue_stopped = true;
966 }
967 spin_unlock_bh(&rtwpci->irq_lock);
968
969 return 0;
970 }
971
rtw_pci_tx_isr(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,u8 hw_queue)972 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
973 u8 hw_queue)
974 {
975 struct ieee80211_hw *hw = rtwdev->hw;
976 struct ieee80211_tx_info *info;
977 struct rtw_pci_tx_ring *ring;
978 struct rtw_pci_tx_data *tx_data;
979 struct sk_buff *skb;
980 u32 count;
981 u32 bd_idx_addr;
982 u32 bd_idx, cur_rp, rp_idx;
983 u16 q_map;
984
985 ring = &rtwpci->tx_rings[hw_queue];
986
987 bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue];
988 bd_idx = rtw_read32(rtwdev, bd_idx_addr);
989 cur_rp = bd_idx >> 16;
990 cur_rp &= TRX_BD_IDX_MASK;
991 rp_idx = ring->r.rp;
992 if (cur_rp >= ring->r.rp)
993 count = cur_rp - ring->r.rp;
994 else
995 count = ring->r.len - (ring->r.rp - cur_rp);
996
997 while (count--) {
998 skb = skb_dequeue(&ring->queue);
999 if (!skb) {
1000 rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n",
1001 count, hw_queue, bd_idx, ring->r.rp, cur_rp);
1002 break;
1003 }
1004 tx_data = rtw_pci_get_tx_data(skb);
1005 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
1006 DMA_TO_DEVICE);
1007
1008 /* just free command packets from host to card */
1009 if (hw_queue == RTW_TX_QUEUE_H2C) {
1010 dev_kfree_skb_irq(skb);
1011 continue;
1012 }
1013
1014 if (ring->queue_stopped &&
1015 avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) {
1016 q_map = skb_get_queue_mapping(skb);
1017 ieee80211_wake_queue(hw, q_map);
1018 ring->queue_stopped = false;
1019 }
1020
1021 if (++rp_idx >= ring->r.len)
1022 rp_idx = 0;
1023
1024 skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz);
1025
1026 info = IEEE80211_SKB_CB(skb);
1027
1028 /* enqueue to wait for tx report */
1029 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
1030 rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn);
1031 continue;
1032 }
1033
1034 /* always ACK for others, then they won't be marked as drop */
1035 if (info->flags & IEEE80211_TX_CTL_NO_ACK)
1036 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
1037 else
1038 info->flags |= IEEE80211_TX_STAT_ACK;
1039
1040 ieee80211_tx_info_clear_status(info);
1041 ieee80211_tx_status_irqsafe(hw, skb);
1042 }
1043
1044 ring->r.rp = cur_rp;
1045 }
1046
rtw_pci_rx_isr(struct rtw_dev * rtwdev)1047 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev)
1048 {
1049 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1050 struct napi_struct *napi = &rtwpci->napi;
1051
1052 napi_schedule(napi);
1053 }
1054
rtw_pci_get_hw_rx_ring_nr(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)1055 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev,
1056 struct rtw_pci *rtwpci)
1057 {
1058 struct rtw_pci_rx_ring *ring;
1059 int count = 0;
1060 u32 tmp, cur_wp;
1061
1062 ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1063 tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ);
1064 cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK);
1065 if (cur_wp >= ring->r.wp)
1066 count = cur_wp - ring->r.wp;
1067 else
1068 count = ring->r.len - (ring->r.wp - cur_wp);
1069
1070 return count;
1071 }
1072
rtw_pci_rx_napi(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,u8 hw_queue,u32 limit)1073 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
1074 u8 hw_queue, u32 limit)
1075 {
1076 const struct rtw_chip_info *chip = rtwdev->chip;
1077 struct napi_struct *napi = &rtwpci->napi;
1078 struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1079 struct rtw_rx_pkt_stat pkt_stat;
1080 struct ieee80211_rx_status rx_status;
1081 struct sk_buff *skb, *new;
1082 u32 cur_rp = ring->r.rp;
1083 u32 count, rx_done = 0;
1084 u32 pkt_offset;
1085 u32 pkt_desc_sz = chip->rx_pkt_desc_sz;
1086 u32 buf_desc_sz = chip->rx_buf_desc_sz;
1087 u32 new_len;
1088 u8 *rx_desc;
1089 dma_addr_t dma;
1090
1091 count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);
1092 count = min(count, limit);
1093
1094 while (count--) {
1095 rtw_pci_dma_check(rtwdev, ring, cur_rp);
1096 skb = ring->buf[cur_rp];
1097 dma = *((dma_addr_t *)skb->cb);
1098 dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE,
1099 DMA_FROM_DEVICE);
1100 rx_desc = skb->data;
1101 chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status);
1102
1103 /* offset from rx_desc to payload */
1104 pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz +
1105 pkt_stat.shift;
1106
1107 /* allocate a new skb for this frame,
1108 * discard the frame if none available
1109 */
1110 new_len = pkt_stat.pkt_len + pkt_offset;
1111 new = dev_alloc_skb(new_len);
1112 if (WARN_ONCE(!new, "rx routine starvation\n"))
1113 goto next_rp;
1114
1115 /* put the DMA data including rx_desc from phy to new skb */
1116 skb_put_data(new, skb->data, new_len);
1117
1118 if (pkt_stat.is_c2h) {
1119 rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new);
1120 } else {
1121 /* remove rx_desc */
1122 skb_pull(new, pkt_offset);
1123
1124 rtw_rx_stats(rtwdev, pkt_stat.vif, new);
1125 memcpy(new->cb, &rx_status, sizeof(rx_status));
1126 ieee80211_rx_napi(rtwdev->hw, NULL, new, napi);
1127 rx_done++;
1128 }
1129
1130 next_rp:
1131 /* new skb delivered to mac80211, re-enable original skb DMA */
1132 rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp,
1133 buf_desc_sz);
1134
1135 /* host read next element in ring */
1136 if (++cur_rp >= ring->r.len)
1137 cur_rp = 0;
1138 }
1139
1140 ring->r.rp = cur_rp;
1141 /* 'rp', the last position we have read, is seen as previous posistion
1142 * of 'wp' that is used to calculate 'count' next time.
1143 */
1144 ring->r.wp = cur_rp;
1145 rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp);
1146
1147 return rx_done;
1148 }
1149
rtw_pci_irq_recognized(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,u32 * irq_status)1150 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev,
1151 struct rtw_pci *rtwpci, u32 *irq_status)
1152 {
1153 unsigned long flags;
1154
1155 spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
1156
1157 irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0);
1158 irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1);
1159 if (rtw_chip_wcpu_11ac(rtwdev))
1160 irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3);
1161 else
1162 irq_status[3] = 0;
1163 irq_status[0] &= rtwpci->irq_mask[0];
1164 irq_status[1] &= rtwpci->irq_mask[1];
1165 irq_status[3] &= rtwpci->irq_mask[3];
1166 rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]);
1167 rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]);
1168 if (rtw_chip_wcpu_11ac(rtwdev))
1169 rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]);
1170
1171 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
1172 }
1173
rtw_pci_interrupt_handler(int irq,void * dev)1174 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev)
1175 {
1176 struct rtw_dev *rtwdev = dev;
1177 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1178
1179 /* disable RTW PCI interrupt to avoid more interrupts before the end of
1180 * thread function
1181 *
1182 * disable HIMR here to also avoid new HISR flag being raised before
1183 * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs
1184 * are cleared, the edge-triggered interrupt will not be generated when
1185 * a new HISR flag is set.
1186 */
1187 rtw_pci_disable_interrupt(rtwdev, rtwpci);
1188
1189 return IRQ_WAKE_THREAD;
1190 }
1191
rtw_pci_interrupt_threadfn(int irq,void * dev)1192 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev)
1193 {
1194 struct rtw_dev *rtwdev = dev;
1195 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1196 u32 irq_status[4];
1197 bool rx = false;
1198
1199 spin_lock_bh(&rtwpci->irq_lock);
1200 rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);
1201
1202 if (irq_status[0] & IMR_MGNTDOK)
1203 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);
1204 if (irq_status[0] & IMR_HIGHDOK)
1205 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);
1206 if (irq_status[0] & IMR_BEDOK)
1207 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);
1208 if (irq_status[0] & IMR_BKDOK)
1209 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);
1210 if (irq_status[0] & IMR_VODOK)
1211 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);
1212 if (irq_status[0] & IMR_VIDOK)
1213 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);
1214 if (irq_status[3] & IMR_H2CDOK)
1215 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);
1216 if (irq_status[0] & IMR_ROK) {
1217 rtw_pci_rx_isr(rtwdev);
1218 rx = true;
1219 }
1220 if (unlikely(irq_status[0] & IMR_C2HCMD))
1221 rtw_fw_c2h_cmd_isr(rtwdev);
1222
1223 /* all of the jobs for this interrupt have been done */
1224 if (rtwpci->running)
1225 rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);
1226 spin_unlock_bh(&rtwpci->irq_lock);
1227
1228 return IRQ_HANDLED;
1229 }
1230
rtw_pci_io_mapping(struct rtw_dev * rtwdev,struct pci_dev * pdev)1231 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev,
1232 struct pci_dev *pdev)
1233 {
1234 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1235 unsigned long len;
1236 u8 bar_id = 2;
1237 int ret;
1238
1239 ret = pci_request_regions(pdev, KBUILD_MODNAME);
1240 if (ret) {
1241 rtw_err(rtwdev, "failed to request pci regions\n");
1242 return ret;
1243 }
1244
1245 len = pci_resource_len(pdev, bar_id);
1246 rtwpci->mmap = pci_iomap(pdev, bar_id, len);
1247 if (!rtwpci->mmap) {
1248 pci_release_regions(pdev);
1249 rtw_err(rtwdev, "failed to map pci memory\n");
1250 return -ENOMEM;
1251 }
1252
1253 return 0;
1254 }
1255
rtw_pci_io_unmapping(struct rtw_dev * rtwdev,struct pci_dev * pdev)1256 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev,
1257 struct pci_dev *pdev)
1258 {
1259 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1260
1261 if (rtwpci->mmap) {
1262 pci_iounmap(pdev, rtwpci->mmap);
1263 pci_release_regions(pdev);
1264 }
1265 }
1266
rtw_dbi_write8(struct rtw_dev * rtwdev,u16 addr,u8 data)1267 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data)
1268 {
1269 u16 write_addr;
1270 u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK);
1271 u8 flag;
1272 u8 cnt;
1273
1274 write_addr = addr & BITS_DBI_ADDR_MASK;
1275 write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN);
1276 rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data);
1277 rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr);
1278 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16);
1279
1280 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1281 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1282 if (flag == 0)
1283 return;
1284
1285 udelay(10);
1286 }
1287
1288 WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr);
1289 }
1290
rtw_dbi_read8(struct rtw_dev * rtwdev,u16 addr,u8 * value)1291 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value)
1292 {
1293 u16 read_addr = addr & BITS_DBI_ADDR_MASK;
1294 u8 flag;
1295 u8 cnt;
1296
1297 rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr);
1298 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16);
1299
1300 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1301 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1302 if (flag == 0) {
1303 read_addr = REG_DBI_RDATA_V1 + (addr & 3);
1304 *value = rtw_read8(rtwdev, read_addr);
1305 return 0;
1306 }
1307
1308 udelay(10);
1309 }
1310
1311 WARN(1, "failed to read DBI register, addr=0x%04x\n", addr);
1312 return -EIO;
1313 }
1314
rtw_mdio_write(struct rtw_dev * rtwdev,u8 addr,u16 data,bool g1)1315 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1)
1316 {
1317 u8 page;
1318 u8 wflag;
1319 u8 cnt;
1320
1321 rtw_write16(rtwdev, REG_MDIO_V1, data);
1322
1323 page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1;
1324 page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2;
1325 rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK);
1326 rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page);
1327 rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1);
1328
1329 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1330 wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG,
1331 BIT_MDIO_WFLAG_V1);
1332 if (wflag == 0)
1333 return;
1334
1335 udelay(10);
1336 }
1337
1338 WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr);
1339 }
1340
rtw_pci_clkreq_set(struct rtw_dev * rtwdev,bool enable)1341 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable)
1342 {
1343 u8 value;
1344 int ret;
1345
1346 if (rtw_pci_disable_aspm)
1347 return;
1348
1349 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1350 if (ret) {
1351 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1352 return;
1353 }
1354
1355 if (enable)
1356 value |= BIT_CLKREQ_SW_EN;
1357 else
1358 value &= ~BIT_CLKREQ_SW_EN;
1359
1360 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1361 }
1362
rtw_pci_clkreq_pad_low(struct rtw_dev * rtwdev,bool enable)1363 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable)
1364 {
1365 u8 value;
1366 int ret;
1367
1368 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1369 if (ret) {
1370 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1371 return;
1372 }
1373
1374 if (enable)
1375 value &= ~BIT_CLKREQ_N_PAD;
1376 else
1377 value |= BIT_CLKREQ_N_PAD;
1378
1379 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1380 }
1381
rtw_pci_aspm_set(struct rtw_dev * rtwdev,bool enable)1382 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable)
1383 {
1384 u8 value;
1385 int ret;
1386
1387 if (rtw_pci_disable_aspm)
1388 return;
1389
1390 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1391 if (ret) {
1392 rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret);
1393 return;
1394 }
1395
1396 if (enable)
1397 value |= BIT_L1_SW_EN;
1398 else
1399 value &= ~BIT_L1_SW_EN;
1400
1401 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1402 }
1403
rtw_pci_link_ps(struct rtw_dev * rtwdev,bool enter)1404 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter)
1405 {
1406 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1407
1408 /* Like CLKREQ, ASPM is also implemented by two HW modules, and can
1409 * only be enabled when host supports it.
1410 *
1411 * And ASPM mechanism should be enabled when driver/firmware enters
1412 * power save mode, without having heavy traffic. Because we've
1413 * experienced some inter-operability issues that the link tends
1414 * to enter L1 state on the fly even when driver is having high
1415 * throughput. This is probably because the ASPM behavior slightly
1416 * varies from different SOC.
1417 */
1418 if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))
1419 return;
1420
1421 if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||
1422 (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))
1423 rtw_pci_aspm_set(rtwdev, enter);
1424 }
1425
rtw_pci_link_cfg(struct rtw_dev * rtwdev)1426 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev)
1427 {
1428 const struct rtw_chip_info *chip = rtwdev->chip;
1429 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1430 struct pci_dev *pdev = rtwpci->pdev;
1431 u16 link_ctrl;
1432 int ret;
1433
1434 /* RTL8822CE has enabled REFCLK auto calibration, it does not need
1435 * to add clock delay to cover the REFCLK timing gap.
1436 */
1437 if (chip->id == RTW_CHIP_TYPE_8822C)
1438 rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0);
1439
1440 /* Though there is standard PCIE configuration space to set the
1441 * link control register, but by Realtek's design, driver should
1442 * check if host supports CLKREQ/ASPM to enable the HW module.
1443 *
1444 * These functions are implemented by two HW modules associated,
1445 * one is responsible to access PCIE configuration space to
1446 * follow the host settings, and another is in charge of doing
1447 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
1448 * the host does not support it, and due to some reasons or wrong
1449 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
1450 * loss if HW misbehaves on the link.
1451 *
1452 * Hence it's designed that driver should first check the PCIE
1453 * configuration space is sync'ed and enabled, then driver can turn
1454 * on the other module that is actually working on the mechanism.
1455 */
1456 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
1457 if (ret) {
1458 rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
1459 return;
1460 }
1461
1462 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
1463 rtw_pci_clkreq_set(rtwdev, true);
1464
1465 rtwpci->link_ctrl = link_ctrl;
1466 }
1467
rtw_pci_interface_cfg(struct rtw_dev * rtwdev)1468 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev)
1469 {
1470 const struct rtw_chip_info *chip = rtwdev->chip;
1471
1472 switch (chip->id) {
1473 case RTW_CHIP_TYPE_8822C:
1474 if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D)
1475 rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG,
1476 BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1);
1477 break;
1478 default:
1479 break;
1480 }
1481 }
1482
rtw_pci_phy_cfg(struct rtw_dev * rtwdev)1483 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev)
1484 {
1485 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1486 const struct rtw_chip_info *chip = rtwdev->chip;
1487 struct pci_dev *pdev = rtwpci->pdev;
1488 const struct rtw_intf_phy_para *para;
1489 u16 cut;
1490 u16 value;
1491 u16 offset;
1492 int i;
1493 int ret;
1494
1495 cut = BIT(0) << rtwdev->hal.cut_version;
1496
1497 for (i = 0; i < chip->intf_table->n_gen1_para; i++) {
1498 para = &chip->intf_table->gen1_para[i];
1499 if (!(para->cut_mask & cut))
1500 continue;
1501 if (para->offset == 0xffff)
1502 break;
1503 offset = para->offset;
1504 value = para->value;
1505 if (para->ip_sel == RTW_IP_SEL_PHY)
1506 rtw_mdio_write(rtwdev, offset, value, true);
1507 else
1508 rtw_dbi_write8(rtwdev, offset, value);
1509 }
1510
1511 for (i = 0; i < chip->intf_table->n_gen2_para; i++) {
1512 para = &chip->intf_table->gen2_para[i];
1513 if (!(para->cut_mask & cut))
1514 continue;
1515 if (para->offset == 0xffff)
1516 break;
1517 offset = para->offset;
1518 value = para->value;
1519 if (para->ip_sel == RTW_IP_SEL_PHY)
1520 rtw_mdio_write(rtwdev, offset, value, false);
1521 else
1522 rtw_dbi_write8(rtwdev, offset, value);
1523 }
1524
1525 rtw_pci_link_cfg(rtwdev);
1526
1527 /* Disable 8821ce completion timeout by default */
1528 if (chip->id == RTW_CHIP_TYPE_8821C) {
1529 ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
1530 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS);
1531 if (ret)
1532 rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n",
1533 ret);
1534 }
1535 }
1536
rtw_pci_suspend(struct device * dev)1537 static int __maybe_unused rtw_pci_suspend(struct device *dev)
1538 {
1539 struct ieee80211_hw *hw = dev_get_drvdata(dev);
1540 struct rtw_dev *rtwdev = hw->priv;
1541 const struct rtw_chip_info *chip = rtwdev->chip;
1542 struct rtw_efuse *efuse = &rtwdev->efuse;
1543
1544 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1545 rtw_pci_clkreq_pad_low(rtwdev, true);
1546 return 0;
1547 }
1548
rtw_pci_resume(struct device * dev)1549 static int __maybe_unused rtw_pci_resume(struct device *dev)
1550 {
1551 struct ieee80211_hw *hw = dev_get_drvdata(dev);
1552 struct rtw_dev *rtwdev = hw->priv;
1553 const struct rtw_chip_info *chip = rtwdev->chip;
1554 struct rtw_efuse *efuse = &rtwdev->efuse;
1555
1556 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1557 rtw_pci_clkreq_pad_low(rtwdev, false);
1558 return 0;
1559 }
1560
1561 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume);
1562 EXPORT_SYMBOL(rtw_pm_ops);
1563
rtw_pci_claim(struct rtw_dev * rtwdev,struct pci_dev * pdev)1564 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1565 {
1566 int ret;
1567
1568 ret = pci_enable_device(pdev);
1569 if (ret) {
1570 rtw_err(rtwdev, "failed to enable pci device\n");
1571 return ret;
1572 }
1573
1574 pci_set_master(pdev);
1575 pci_set_drvdata(pdev, rtwdev->hw);
1576 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
1577
1578 return 0;
1579 }
1580
rtw_pci_declaim(struct rtw_dev * rtwdev,struct pci_dev * pdev)1581 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1582 {
1583 pci_clear_master(pdev);
1584 pci_disable_device(pdev);
1585 }
1586
rtw_pci_setup_resource(struct rtw_dev * rtwdev,struct pci_dev * pdev)1587 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1588 {
1589 struct rtw_pci *rtwpci;
1590 int ret;
1591
1592 rtwpci = (struct rtw_pci *)rtwdev->priv;
1593 rtwpci->pdev = pdev;
1594
1595 /* after this driver can access to hw registers */
1596 ret = rtw_pci_io_mapping(rtwdev, pdev);
1597 if (ret) {
1598 rtw_err(rtwdev, "failed to request pci io region\n");
1599 goto err_out;
1600 }
1601
1602 ret = rtw_pci_init(rtwdev);
1603 if (ret) {
1604 rtw_err(rtwdev, "failed to allocate pci resources\n");
1605 goto err_io_unmap;
1606 }
1607
1608 return 0;
1609
1610 err_io_unmap:
1611 rtw_pci_io_unmapping(rtwdev, pdev);
1612
1613 err_out:
1614 return ret;
1615 }
1616
rtw_pci_destroy(struct rtw_dev * rtwdev,struct pci_dev * pdev)1617 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1618 {
1619 rtw_pci_deinit(rtwdev);
1620 rtw_pci_io_unmapping(rtwdev, pdev);
1621 }
1622
1623 static struct rtw_hci_ops rtw_pci_ops = {
1624 .tx_write = rtw_pci_tx_write,
1625 .tx_kick_off = rtw_pci_tx_kick_off,
1626 .flush_queues = rtw_pci_flush_queues,
1627 .setup = rtw_pci_setup,
1628 .start = rtw_pci_start,
1629 .stop = rtw_pci_stop,
1630 .deep_ps = rtw_pci_deep_ps,
1631 .link_ps = rtw_pci_link_ps,
1632 .interface_cfg = rtw_pci_interface_cfg,
1633
1634 .read8 = rtw_pci_read8,
1635 .read16 = rtw_pci_read16,
1636 .read32 = rtw_pci_read32,
1637 .write8 = rtw_pci_write8,
1638 .write16 = rtw_pci_write16,
1639 .write32 = rtw_pci_write32,
1640 .write_data_rsvd_page = rtw_pci_write_data_rsvd_page,
1641 .write_data_h2c = rtw_pci_write_data_h2c,
1642 };
1643
rtw_pci_request_irq(struct rtw_dev * rtwdev,struct pci_dev * pdev)1644 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1645 {
1646 unsigned int flags = PCI_IRQ_LEGACY;
1647 int ret;
1648
1649 if (!rtw_disable_msi)
1650 flags |= PCI_IRQ_MSI;
1651
1652 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
1653 if (ret < 0) {
1654 rtw_err(rtwdev, "failed to alloc PCI irq vectors\n");
1655 return ret;
1656 }
1657
1658 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
1659 rtw_pci_interrupt_handler,
1660 rtw_pci_interrupt_threadfn,
1661 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
1662 if (ret) {
1663 rtw_err(rtwdev, "failed to request irq %d\n", ret);
1664 pci_free_irq_vectors(pdev);
1665 }
1666
1667 return ret;
1668 }
1669
rtw_pci_free_irq(struct rtw_dev * rtwdev,struct pci_dev * pdev)1670 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1671 {
1672 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
1673 pci_free_irq_vectors(pdev);
1674 }
1675
rtw_pci_napi_poll(struct napi_struct * napi,int budget)1676 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget)
1677 {
1678 struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi);
1679 struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,
1680 priv);
1681 int work_done = 0;
1682
1683 if (rtwpci->rx_no_aspm)
1684 rtw_pci_link_ps(rtwdev, false);
1685
1686 while (work_done < budget) {
1687 u32 work_done_once;
1688
1689 work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,
1690 budget - work_done);
1691 if (work_done_once == 0)
1692 break;
1693 work_done += work_done_once;
1694 }
1695 if (work_done < budget) {
1696 napi_complete_done(napi, work_done);
1697 spin_lock_bh(&rtwpci->irq_lock);
1698 if (rtwpci->running)
1699 rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
1700 spin_unlock_bh(&rtwpci->irq_lock);
1701 /* When ISR happens during polling and before napi_complete
1702 * while no further data is received. Data on the dma_ring will
1703 * not be processed immediately. Check whether dma ring is
1704 * empty and perform napi_schedule accordingly.
1705 */
1706 if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))
1707 napi_schedule(napi);
1708 }
1709 if (rtwpci->rx_no_aspm)
1710 rtw_pci_link_ps(rtwdev, true);
1711
1712 return work_done;
1713 }
1714
rtw_pci_napi_init(struct rtw_dev * rtwdev)1715 static void rtw_pci_napi_init(struct rtw_dev *rtwdev)
1716 {
1717 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1718
1719 init_dummy_netdev(&rtwpci->netdev);
1720 netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll);
1721 }
1722
rtw_pci_napi_deinit(struct rtw_dev * rtwdev)1723 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)
1724 {
1725 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1726
1727 rtw_pci_napi_stop(rtwdev);
1728 netif_napi_del(&rtwpci->napi);
1729 }
1730
rtw_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)1731 int rtw_pci_probe(struct pci_dev *pdev,
1732 const struct pci_device_id *id)
1733 {
1734 struct pci_dev *bridge = pci_upstream_bridge(pdev);
1735 struct ieee80211_hw *hw;
1736 struct rtw_dev *rtwdev;
1737 struct rtw_pci *rtwpci;
1738 int drv_data_size;
1739 int ret;
1740
1741 drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci);
1742 hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops);
1743 if (!hw) {
1744 dev_err(&pdev->dev, "failed to allocate hw\n");
1745 return -ENOMEM;
1746 }
1747
1748 rtwdev = hw->priv;
1749 rtwdev->hw = hw;
1750 rtwdev->dev = &pdev->dev;
1751 rtwdev->chip = (struct rtw_chip_info *)id->driver_data;
1752 rtwdev->hci.ops = &rtw_pci_ops;
1753 rtwdev->hci.type = RTW_HCI_TYPE_PCIE;
1754
1755 rtwpci = (struct rtw_pci *)rtwdev->priv;
1756 atomic_set(&rtwpci->link_usage, 1);
1757
1758 ret = rtw_core_init(rtwdev);
1759 if (ret)
1760 goto err_release_hw;
1761
1762 rtw_dbg(rtwdev, RTW_DBG_PCI,
1763 "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n",
1764 pdev->vendor, pdev->device, pdev->revision);
1765
1766 ret = rtw_pci_claim(rtwdev, pdev);
1767 if (ret) {
1768 rtw_err(rtwdev, "failed to claim pci device\n");
1769 goto err_deinit_core;
1770 }
1771
1772 ret = rtw_pci_setup_resource(rtwdev, pdev);
1773 if (ret) {
1774 rtw_err(rtwdev, "failed to setup pci resources\n");
1775 goto err_pci_declaim;
1776 }
1777
1778 rtw_pci_napi_init(rtwdev);
1779
1780 ret = rtw_chip_info_setup(rtwdev);
1781 if (ret) {
1782 rtw_err(rtwdev, "failed to setup chip information\n");
1783 goto err_destroy_pci;
1784 }
1785
1786 /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */
1787 if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL)
1788 rtwpci->rx_no_aspm = true;
1789
1790 rtw_pci_phy_cfg(rtwdev);
1791
1792 ret = rtw_register_hw(rtwdev, hw);
1793 if (ret) {
1794 rtw_err(rtwdev, "failed to register hw\n");
1795 goto err_destroy_pci;
1796 }
1797
1798 ret = rtw_pci_request_irq(rtwdev, pdev);
1799 if (ret) {
1800 ieee80211_unregister_hw(hw);
1801 goto err_destroy_pci;
1802 }
1803
1804 return 0;
1805
1806 err_destroy_pci:
1807 rtw_pci_napi_deinit(rtwdev);
1808 rtw_pci_destroy(rtwdev, pdev);
1809
1810 err_pci_declaim:
1811 rtw_pci_declaim(rtwdev, pdev);
1812
1813 err_deinit_core:
1814 rtw_core_deinit(rtwdev);
1815
1816 err_release_hw:
1817 ieee80211_free_hw(hw);
1818
1819 return ret;
1820 }
1821 EXPORT_SYMBOL(rtw_pci_probe);
1822
rtw_pci_remove(struct pci_dev * pdev)1823 void rtw_pci_remove(struct pci_dev *pdev)
1824 {
1825 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1826 struct rtw_dev *rtwdev;
1827 struct rtw_pci *rtwpci;
1828
1829 if (!hw)
1830 return;
1831
1832 rtwdev = hw->priv;
1833 rtwpci = (struct rtw_pci *)rtwdev->priv;
1834
1835 rtw_unregister_hw(rtwdev, hw);
1836 rtw_pci_disable_interrupt(rtwdev, rtwpci);
1837 rtw_pci_napi_deinit(rtwdev);
1838 rtw_pci_destroy(rtwdev, pdev);
1839 rtw_pci_declaim(rtwdev, pdev);
1840 rtw_pci_free_irq(rtwdev, pdev);
1841 rtw_core_deinit(rtwdev);
1842 ieee80211_free_hw(hw);
1843 }
1844 EXPORT_SYMBOL(rtw_pci_remove);
1845
rtw_pci_shutdown(struct pci_dev * pdev)1846 void rtw_pci_shutdown(struct pci_dev *pdev)
1847 {
1848 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1849 struct rtw_dev *rtwdev;
1850 const struct rtw_chip_info *chip;
1851
1852 if (!hw)
1853 return;
1854
1855 rtwdev = hw->priv;
1856 chip = rtwdev->chip;
1857
1858 if (chip->ops->shutdown)
1859 chip->ops->shutdown(rtwdev);
1860
1861 pci_set_power_state(pdev, PCI_D3hot);
1862 }
1863 EXPORT_SYMBOL(rtw_pci_shutdown);
1864
1865 MODULE_AUTHOR("Realtek Corporation");
1866 MODULE_DESCRIPTION("Realtek 802.11ac wireless PCI driver");
1867 MODULE_LICENSE("Dual BSD/GPL");
1868