1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
3
4 #include <linux/etherdevice.h>
5 #include <net/ip6_checksum.h>
6 #include <net/page_pool/helpers.h>
7 #include <net/inet_ecn.h>
8 #include <linux/iopoll.h>
9 #include <linux/sctp.h>
10 #include <linux/pci.h>
11 #include <net/tcp.h>
12 #include <net/ip.h>
13
14 #include "wx_type.h"
15 #include "wx_lib.h"
16 #include "wx_hw.h"
17
18 /* Lookup table mapping the HW PTYPE to the bit field for decoding */
19 static struct wx_dec_ptype wx_ptype_lookup[256] = {
20 /* L2: mac */
21 [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
22 [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS, PAY2),
23 [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
24 [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
25 [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
26 [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
27 [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
28
29 /* L2: ethertype filter */
30 [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
31
32 /* L3: ip non-tunnel */
33 [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),
34 [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),
35 [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP, PAY4),
36 [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP, PAY4),
37 [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),
38 [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),
39 [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),
40 [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP, PAY3),
41 [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP, PAY4),
42 [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),
43
44 /* L2: fcoe */
45 [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
46 [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
47
48 /* IPv4 --> IPv4/IPv6 */
49 [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),
50 [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),
51 [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP, PAY4),
52 [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP, PAY4),
53 [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),
54 [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),
55 [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),
56 [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP, PAY4),
57 [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP, PAY4),
58 [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),
59
60 /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */
61 [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),
62 [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),
63 [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),
64 [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP, PAY4),
65 [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP, PAY4),
66 [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),
67 [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),
68 [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),
69 [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP, PAY4),
70 [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP, PAY4),
71 [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),
72
73 /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */
74 [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),
75 [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),
76 [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),
77 [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP, PAY4),
78 [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP, PAY4),
79 [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),
80 [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),
81 [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),
82 [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP, PAY4),
83 [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP, PAY4),
84 [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),
85
86 /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */
87 [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),
88 [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),
89 [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),
90 [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP, PAY4),
91 [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP, PAY4),
92 [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),
93 [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),
94 [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),
95 [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP, PAY4),
96 [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP, PAY4),
97 [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),
98
99 /* IPv6 --> IPv4/IPv6 */
100 [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),
101 [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),
102 [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP, PAY4),
103 [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP, PAY4),
104 [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),
105 [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),
106 [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),
107 [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP, PAY4),
108 [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP, PAY4),
109 [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),
110
111 /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */
112 [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),
113 [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),
114 [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),
115 [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP, PAY4),
116 [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP, PAY4),
117 [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),
118 [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),
119 [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),
120 [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP, PAY4),
121 [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP, PAY4),
122 [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),
123
124 /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */
125 [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),
126 [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),
127 [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),
128 [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP, PAY4),
129 [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP, PAY4),
130 [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),
131 [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),
132 [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),
133 [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP, PAY4),
134 [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP, PAY4),
135 [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),
136
137 /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */
138 [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),
139 [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),
140 [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),
141 [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP, PAY4),
142 [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP, PAY4),
143 [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),
144 [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),
145 [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),
146 [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP, PAY4),
147 [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP, PAY4),
148 [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),
149 };
150
wx_decode_ptype(const u8 ptype)151 static struct wx_dec_ptype wx_decode_ptype(const u8 ptype)
152 {
153 return wx_ptype_lookup[ptype];
154 }
155
156 /* wx_test_staterr - tests bits in Rx descriptor status and error fields */
wx_test_staterr(union wx_rx_desc * rx_desc,const u32 stat_err_bits)157 static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,
158 const u32 stat_err_bits)
159 {
160 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
161 }
162
wx_dma_sync_frag(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer)163 static void wx_dma_sync_frag(struct wx_ring *rx_ring,
164 struct wx_rx_buffer *rx_buffer)
165 {
166 struct sk_buff *skb = rx_buffer->skb;
167 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
168
169 dma_sync_single_range_for_cpu(rx_ring->dev,
170 WX_CB(skb)->dma,
171 skb_frag_off(frag),
172 skb_frag_size(frag),
173 DMA_FROM_DEVICE);
174 }
175
wx_get_rx_buffer(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff ** skb,int * rx_buffer_pgcnt)176 static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,
177 union wx_rx_desc *rx_desc,
178 struct sk_buff **skb,
179 int *rx_buffer_pgcnt)
180 {
181 struct wx_rx_buffer *rx_buffer;
182 unsigned int size;
183
184 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
185 size = le16_to_cpu(rx_desc->wb.upper.length);
186
187 #if (PAGE_SIZE < 8192)
188 *rx_buffer_pgcnt = page_count(rx_buffer->page);
189 #else
190 *rx_buffer_pgcnt = 0;
191 #endif
192
193 prefetchw(rx_buffer->page);
194 *skb = rx_buffer->skb;
195
196 /* Delay unmapping of the first packet. It carries the header
197 * information, HW may still access the header after the writeback.
198 * Only unmap it when EOP is reached
199 */
200 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {
201 if (!*skb)
202 goto skip_sync;
203 } else {
204 if (*skb)
205 wx_dma_sync_frag(rx_ring, rx_buffer);
206 }
207
208 /* we are reusing so sync this buffer for CPU use */
209 dma_sync_single_range_for_cpu(rx_ring->dev,
210 rx_buffer->dma,
211 rx_buffer->page_offset,
212 size,
213 DMA_FROM_DEVICE);
214 skip_sync:
215 return rx_buffer;
216 }
217
wx_put_rx_buffer(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,struct sk_buff * skb,int rx_buffer_pgcnt)218 static void wx_put_rx_buffer(struct wx_ring *rx_ring,
219 struct wx_rx_buffer *rx_buffer,
220 struct sk_buff *skb,
221 int rx_buffer_pgcnt)
222 {
223 /* clear contents of rx_buffer */
224 rx_buffer->page = NULL;
225 rx_buffer->skb = NULL;
226 }
227
wx_build_skb(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,union wx_rx_desc * rx_desc)228 static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,
229 struct wx_rx_buffer *rx_buffer,
230 union wx_rx_desc *rx_desc)
231 {
232 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
233 #if (PAGE_SIZE < 8192)
234 unsigned int truesize = WX_RX_BUFSZ;
235 #else
236 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
237 #endif
238 struct sk_buff *skb = rx_buffer->skb;
239
240 if (!skb) {
241 void *page_addr = page_address(rx_buffer->page) +
242 rx_buffer->page_offset;
243
244 /* prefetch first cache line of first page */
245 prefetch(page_addr);
246 #if L1_CACHE_BYTES < 128
247 prefetch(page_addr + L1_CACHE_BYTES);
248 #endif
249
250 /* allocate a skb to store the frags */
251 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);
252 if (unlikely(!skb))
253 return NULL;
254
255 /* we will be copying header into skb->data in
256 * pskb_may_pull so it is in our interest to prefetch
257 * it now to avoid a possible cache miss
258 */
259 prefetchw(skb->data);
260
261 if (size <= WX_RXBUFFER_256) {
262 memcpy(__skb_put(skb, size), page_addr,
263 ALIGN(size, sizeof(long)));
264 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);
265 return skb;
266 }
267
268 skb_mark_for_recycle(skb);
269
270 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))
271 WX_CB(skb)->dma = rx_buffer->dma;
272
273 skb_add_rx_frag(skb, 0, rx_buffer->page,
274 rx_buffer->page_offset,
275 size, truesize);
276 goto out;
277
278 } else {
279 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
280 rx_buffer->page_offset, size, truesize);
281 }
282
283 out:
284 #if (PAGE_SIZE < 8192)
285 /* flip page offset to other buffer */
286 rx_buffer->page_offset ^= truesize;
287 #else
288 /* move offset up to the next cache line */
289 rx_buffer->page_offset += truesize;
290 #endif
291
292 return skb;
293 }
294
wx_alloc_mapped_page(struct wx_ring * rx_ring,struct wx_rx_buffer * bi)295 static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,
296 struct wx_rx_buffer *bi)
297 {
298 struct page *page = bi->page;
299 dma_addr_t dma;
300
301 /* since we are recycling buffers we should seldom need to alloc */
302 if (likely(page))
303 return true;
304
305 page = page_pool_dev_alloc_pages(rx_ring->page_pool);
306 if (unlikely(!page))
307 return false;
308 dma = page_pool_get_dma_addr(page);
309
310 bi->dma = dma;
311 bi->page = page;
312 bi->page_offset = 0;
313
314 return true;
315 }
316
317 /**
318 * wx_alloc_rx_buffers - Replace used receive buffers
319 * @rx_ring: ring to place buffers on
320 * @cleaned_count: number of buffers to replace
321 **/
wx_alloc_rx_buffers(struct wx_ring * rx_ring,u16 cleaned_count)322 void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)
323 {
324 u16 i = rx_ring->next_to_use;
325 union wx_rx_desc *rx_desc;
326 struct wx_rx_buffer *bi;
327
328 /* nothing to do */
329 if (!cleaned_count)
330 return;
331
332 rx_desc = WX_RX_DESC(rx_ring, i);
333 bi = &rx_ring->rx_buffer_info[i];
334 i -= rx_ring->count;
335
336 do {
337 if (!wx_alloc_mapped_page(rx_ring, bi))
338 break;
339
340 /* sync the buffer for use by the device */
341 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
342 bi->page_offset,
343 WX_RX_BUFSZ,
344 DMA_FROM_DEVICE);
345
346 rx_desc->read.pkt_addr =
347 cpu_to_le64(bi->dma + bi->page_offset);
348
349 rx_desc++;
350 bi++;
351 i++;
352 if (unlikely(!i)) {
353 rx_desc = WX_RX_DESC(rx_ring, 0);
354 bi = rx_ring->rx_buffer_info;
355 i -= rx_ring->count;
356 }
357
358 /* clear the status bits for the next_to_use descriptor */
359 rx_desc->wb.upper.status_error = 0;
360 /* clear the length for the next_to_use descriptor */
361 rx_desc->wb.upper.length = 0;
362
363 cleaned_count--;
364 } while (cleaned_count);
365
366 i += rx_ring->count;
367
368 if (rx_ring->next_to_use != i) {
369 rx_ring->next_to_use = i;
370 /* update next to alloc since we have filled the ring */
371 rx_ring->next_to_alloc = i;
372
373 /* Force memory writes to complete before letting h/w
374 * know there are new descriptors to fetch. (Only
375 * applicable for weak-ordered memory model archs,
376 * such as IA-64).
377 */
378 wmb();
379 writel(i, rx_ring->tail);
380 }
381 }
382
wx_desc_unused(struct wx_ring * ring)383 u16 wx_desc_unused(struct wx_ring *ring)
384 {
385 u16 ntc = ring->next_to_clean;
386 u16 ntu = ring->next_to_use;
387
388 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
389 }
390
391 /**
392 * wx_is_non_eop - process handling of non-EOP buffers
393 * @rx_ring: Rx ring being processed
394 * @rx_desc: Rx descriptor for current buffer
395 * @skb: Current socket buffer containing buffer in progress
396 *
397 * This function updates next to clean. If the buffer is an EOP buffer
398 * this function exits returning false, otherwise it will place the
399 * sk_buff in the next buffer to be chained and return true indicating
400 * that this is in fact a non-EOP buffer.
401 **/
wx_is_non_eop(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)402 static bool wx_is_non_eop(struct wx_ring *rx_ring,
403 union wx_rx_desc *rx_desc,
404 struct sk_buff *skb)
405 {
406 u32 ntc = rx_ring->next_to_clean + 1;
407
408 /* fetch, update, and store next to clean */
409 ntc = (ntc < rx_ring->count) ? ntc : 0;
410 rx_ring->next_to_clean = ntc;
411
412 prefetch(WX_RX_DESC(rx_ring, ntc));
413
414 /* if we are the last buffer then there is nothing else to do */
415 if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))
416 return false;
417
418 rx_ring->rx_buffer_info[ntc].skb = skb;
419
420 return true;
421 }
422
wx_pull_tail(struct sk_buff * skb)423 static void wx_pull_tail(struct sk_buff *skb)
424 {
425 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
426 unsigned int pull_len;
427 unsigned char *va;
428
429 /* it is valid to use page_address instead of kmap since we are
430 * working with pages allocated out of the lomem pool per
431 * alloc_page(GFP_ATOMIC)
432 */
433 va = skb_frag_address(frag);
434
435 /* we need the header to contain the greater of either ETH_HLEN or
436 * 60 bytes if the skb->len is less than 60 for skb_pad.
437 */
438 pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);
439
440 /* align pull length to size of long to optimize memcpy performance */
441 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
442
443 /* update all of the pointers */
444 skb_frag_size_sub(frag, pull_len);
445 skb_frag_off_add(frag, pull_len);
446 skb->data_len -= pull_len;
447 skb->tail += pull_len;
448 }
449
450 /**
451 * wx_cleanup_headers - Correct corrupted or empty headers
452 * @rx_ring: rx descriptor ring packet is being transacted on
453 * @rx_desc: pointer to the EOP Rx descriptor
454 * @skb: pointer to current skb being fixed
455 *
456 * Check for corrupted packet headers caused by senders on the local L2
457 * embedded NIC switch not setting up their Tx Descriptors right. These
458 * should be very rare.
459 *
460 * Also address the case where we are pulling data in on pages only
461 * and as such no data is present in the skb header.
462 *
463 * In addition if skb is not at least 60 bytes we need to pad it so that
464 * it is large enough to qualify as a valid Ethernet frame.
465 *
466 * Returns true if an error was encountered and skb was freed.
467 **/
wx_cleanup_headers(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)468 static bool wx_cleanup_headers(struct wx_ring *rx_ring,
469 union wx_rx_desc *rx_desc,
470 struct sk_buff *skb)
471 {
472 struct net_device *netdev = rx_ring->netdev;
473
474 /* verify that the packet does not have any known errors */
475 if (!netdev ||
476 unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&
477 !(netdev->features & NETIF_F_RXALL))) {
478 dev_kfree_skb_any(skb);
479 return true;
480 }
481
482 /* place header in linear portion of buffer */
483 if (!skb_headlen(skb))
484 wx_pull_tail(skb);
485
486 /* if eth_skb_pad returns an error the skb was freed */
487 if (eth_skb_pad(skb))
488 return true;
489
490 return false;
491 }
492
wx_rx_hash(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)493 static void wx_rx_hash(struct wx_ring *ring,
494 union wx_rx_desc *rx_desc,
495 struct sk_buff *skb)
496 {
497 u16 rss_type;
498
499 if (!(ring->netdev->features & NETIF_F_RXHASH))
500 return;
501
502 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
503 WX_RXD_RSSTYPE_MASK;
504
505 if (!rss_type)
506 return;
507
508 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
509 (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
510 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
511 }
512
513 /**
514 * wx_rx_checksum - indicate in skb if hw indicated a good cksum
515 * @ring: structure containing ring specific data
516 * @rx_desc: current Rx descriptor being processed
517 * @skb: skb currently being received and modified
518 **/
wx_rx_checksum(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)519 static void wx_rx_checksum(struct wx_ring *ring,
520 union wx_rx_desc *rx_desc,
521 struct sk_buff *skb)
522 {
523 struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));
524
525 skb_checksum_none_assert(skb);
526 /* Rx csum disabled */
527 if (!(ring->netdev->features & NETIF_F_RXCSUM))
528 return;
529
530 /* if IPv4 header checksum error */
531 if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&
532 wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||
533 (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&
534 wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {
535 ring->rx_stats.csum_err++;
536 return;
537 }
538
539 /* L4 checksum offload flag must set for the below code to work */
540 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))
541 return;
542
543 /* Hardware can't guarantee csum if IPv6 Dest Header found */
544 if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP && WX_RXD_IPV6EX(rx_desc))
545 return;
546
547 /* if L4 checksum error */
548 if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {
549 ring->rx_stats.csum_err++;
550 return;
551 }
552
553 /* It must be a TCP or UDP or SCTP packet with a valid checksum */
554 skb->ip_summed = CHECKSUM_UNNECESSARY;
555
556 /* If there is an outer header present that might contain a checksum
557 * we need to bump the checksum level by 1 to reflect the fact that
558 * we are indicating we validated the inner checksum.
559 */
560 if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)
561 __skb_incr_checksum_unnecessary(skb);
562 ring->rx_stats.csum_good_cnt++;
563 }
564
wx_rx_vlan(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)565 static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,
566 struct sk_buff *skb)
567 {
568 u16 ethertype;
569 u8 idx = 0;
570
571 if ((ring->netdev->features &
572 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&
573 wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {
574 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
575 0x1c0) >> 6;
576 ethertype = ring->q_vector->wx->tpid[idx];
577 __vlan_hwaccel_put_tag(skb, htons(ethertype),
578 le16_to_cpu(rx_desc->wb.upper.vlan));
579 }
580 }
581
582 /**
583 * wx_process_skb_fields - Populate skb header fields from Rx descriptor
584 * @rx_ring: rx descriptor ring packet is being transacted on
585 * @rx_desc: pointer to the EOP Rx descriptor
586 * @skb: pointer to current skb being populated
587 *
588 * This function checks the ring, descriptor, and packet information in
589 * order to populate the hash, checksum, protocol, and
590 * other fields within the skb.
591 **/
wx_process_skb_fields(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)592 static void wx_process_skb_fields(struct wx_ring *rx_ring,
593 union wx_rx_desc *rx_desc,
594 struct sk_buff *skb)
595 {
596 wx_rx_hash(rx_ring, rx_desc, skb);
597 wx_rx_checksum(rx_ring, rx_desc, skb);
598 wx_rx_vlan(rx_ring, rx_desc, skb);
599 skb_record_rx_queue(skb, rx_ring->queue_index);
600 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
601 }
602
603 /**
604 * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
605 * @q_vector: structure containing interrupt and ring information
606 * @rx_ring: rx descriptor ring to transact packets on
607 * @budget: Total limit on number of packets to process
608 *
609 * This function provides a "bounce buffer" approach to Rx interrupt
610 * processing. The advantage to this is that on systems that have
611 * expensive overhead for IOMMU access this provides a means of avoiding
612 * it by maintaining the mapping of the page to the system.
613 *
614 * Returns amount of work completed.
615 **/
wx_clean_rx_irq(struct wx_q_vector * q_vector,struct wx_ring * rx_ring,int budget)616 static int wx_clean_rx_irq(struct wx_q_vector *q_vector,
617 struct wx_ring *rx_ring,
618 int budget)
619 {
620 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
621 u16 cleaned_count = wx_desc_unused(rx_ring);
622
623 do {
624 struct wx_rx_buffer *rx_buffer;
625 union wx_rx_desc *rx_desc;
626 struct sk_buff *skb;
627 int rx_buffer_pgcnt;
628
629 /* return some buffers to hardware, one at a time is too slow */
630 if (cleaned_count >= WX_RX_BUFFER_WRITE) {
631 wx_alloc_rx_buffers(rx_ring, cleaned_count);
632 cleaned_count = 0;
633 }
634
635 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);
636 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))
637 break;
638
639 /* This memory barrier is needed to keep us from reading
640 * any other fields out of the rx_desc until we know the
641 * descriptor has been written back
642 */
643 dma_rmb();
644
645 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb, &rx_buffer_pgcnt);
646
647 /* retrieve a buffer from the ring */
648 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);
649
650 /* exit if we failed to retrieve a buffer */
651 if (!skb) {
652 break;
653 }
654
655 wx_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
656 cleaned_count++;
657
658 /* place incomplete frames back on ring for completion */
659 if (wx_is_non_eop(rx_ring, rx_desc, skb))
660 continue;
661
662 /* verify the packet layout is correct */
663 if (wx_cleanup_headers(rx_ring, rx_desc, skb))
664 continue;
665
666 /* probably a little skewed due to removing CRC */
667 total_rx_bytes += skb->len;
668
669 /* populate checksum, timestamp, VLAN, and protocol */
670 wx_process_skb_fields(rx_ring, rx_desc, skb);
671 napi_gro_receive(&q_vector->napi, skb);
672
673 /* update budget accounting */
674 total_rx_packets++;
675 } while (likely(total_rx_packets < budget));
676
677 u64_stats_update_begin(&rx_ring->syncp);
678 rx_ring->stats.packets += total_rx_packets;
679 rx_ring->stats.bytes += total_rx_bytes;
680 u64_stats_update_end(&rx_ring->syncp);
681 q_vector->rx.total_packets += total_rx_packets;
682 q_vector->rx.total_bytes += total_rx_bytes;
683
684 return total_rx_packets;
685 }
686
wx_txring_txq(const struct wx_ring * ring)687 static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)
688 {
689 return netdev_get_tx_queue(ring->netdev, ring->queue_index);
690 }
691
692 /**
693 * wx_clean_tx_irq - Reclaim resources after transmit completes
694 * @q_vector: structure containing interrupt and ring information
695 * @tx_ring: tx ring to clean
696 * @napi_budget: Used to determine if we are in netpoll
697 **/
wx_clean_tx_irq(struct wx_q_vector * q_vector,struct wx_ring * tx_ring,int napi_budget)698 static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,
699 struct wx_ring *tx_ring, int napi_budget)
700 {
701 unsigned int budget = q_vector->wx->tx_work_limit;
702 unsigned int total_bytes = 0, total_packets = 0;
703 unsigned int i = tx_ring->next_to_clean;
704 struct wx_tx_buffer *tx_buffer;
705 union wx_tx_desc *tx_desc;
706
707 if (!netif_carrier_ok(tx_ring->netdev))
708 return true;
709
710 tx_buffer = &tx_ring->tx_buffer_info[i];
711 tx_desc = WX_TX_DESC(tx_ring, i);
712 i -= tx_ring->count;
713
714 do {
715 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;
716
717 /* if next_to_watch is not set then there is no work pending */
718 if (!eop_desc)
719 break;
720
721 /* prevent any other reads prior to eop_desc */
722 smp_rmb();
723
724 /* if DD is not set pending work has not been completed */
725 if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD)))
726 break;
727
728 /* clear next_to_watch to prevent false hangs */
729 tx_buffer->next_to_watch = NULL;
730
731 /* update the statistics for this packet */
732 total_bytes += tx_buffer->bytecount;
733 total_packets += tx_buffer->gso_segs;
734
735 /* free the skb */
736 napi_consume_skb(tx_buffer->skb, napi_budget);
737
738 /* unmap skb header data */
739 dma_unmap_single(tx_ring->dev,
740 dma_unmap_addr(tx_buffer, dma),
741 dma_unmap_len(tx_buffer, len),
742 DMA_TO_DEVICE);
743
744 /* clear tx_buffer data */
745 dma_unmap_len_set(tx_buffer, len, 0);
746
747 /* unmap remaining buffers */
748 while (tx_desc != eop_desc) {
749 tx_buffer++;
750 tx_desc++;
751 i++;
752 if (unlikely(!i)) {
753 i -= tx_ring->count;
754 tx_buffer = tx_ring->tx_buffer_info;
755 tx_desc = WX_TX_DESC(tx_ring, 0);
756 }
757
758 /* unmap any remaining paged data */
759 if (dma_unmap_len(tx_buffer, len)) {
760 dma_unmap_page(tx_ring->dev,
761 dma_unmap_addr(tx_buffer, dma),
762 dma_unmap_len(tx_buffer, len),
763 DMA_TO_DEVICE);
764 dma_unmap_len_set(tx_buffer, len, 0);
765 }
766 }
767
768 /* move us one more past the eop_desc for start of next pkt */
769 tx_buffer++;
770 tx_desc++;
771 i++;
772 if (unlikely(!i)) {
773 i -= tx_ring->count;
774 tx_buffer = tx_ring->tx_buffer_info;
775 tx_desc = WX_TX_DESC(tx_ring, 0);
776 }
777
778 /* issue prefetch for next Tx descriptor */
779 prefetch(tx_desc);
780
781 /* update budget accounting */
782 budget--;
783 } while (likely(budget));
784
785 i += tx_ring->count;
786 tx_ring->next_to_clean = i;
787 u64_stats_update_begin(&tx_ring->syncp);
788 tx_ring->stats.bytes += total_bytes;
789 tx_ring->stats.packets += total_packets;
790 u64_stats_update_end(&tx_ring->syncp);
791 q_vector->tx.total_bytes += total_bytes;
792 q_vector->tx.total_packets += total_packets;
793
794 netdev_tx_completed_queue(wx_txring_txq(tx_ring),
795 total_packets, total_bytes);
796
797 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
798 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
799 (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
800 /* Make sure that anybody stopping the queue after this
801 * sees the new next_to_clean.
802 */
803 smp_mb();
804
805 if (__netif_subqueue_stopped(tx_ring->netdev,
806 tx_ring->queue_index) &&
807 netif_running(tx_ring->netdev))
808 netif_wake_subqueue(tx_ring->netdev,
809 tx_ring->queue_index);
810 }
811
812 return !!budget;
813 }
814
815 /**
816 * wx_poll - NAPI polling RX/TX cleanup routine
817 * @napi: napi struct with our devices info in it
818 * @budget: amount of work driver is allowed to do this pass, in packets
819 *
820 * This function will clean all queues associated with a q_vector.
821 **/
wx_poll(struct napi_struct * napi,int budget)822 static int wx_poll(struct napi_struct *napi, int budget)
823 {
824 struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);
825 int per_ring_budget, work_done = 0;
826 struct wx *wx = q_vector->wx;
827 bool clean_complete = true;
828 struct wx_ring *ring;
829
830 wx_for_each_ring(ring, q_vector->tx) {
831 if (!wx_clean_tx_irq(q_vector, ring, budget))
832 clean_complete = false;
833 }
834
835 /* Exit if we are called by netpoll */
836 if (budget <= 0)
837 return budget;
838
839 /* attempt to distribute budget to each queue fairly, but don't allow
840 * the budget to go below 1 because we'll exit polling
841 */
842 if (q_vector->rx.count > 1)
843 per_ring_budget = max(budget / q_vector->rx.count, 1);
844 else
845 per_ring_budget = budget;
846
847 wx_for_each_ring(ring, q_vector->rx) {
848 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);
849
850 work_done += cleaned;
851 if (cleaned >= per_ring_budget)
852 clean_complete = false;
853 }
854
855 /* If all work not completed, return budget and keep polling */
856 if (!clean_complete)
857 return budget;
858
859 /* all work done, exit the polling mode */
860 if (likely(napi_complete_done(napi, work_done))) {
861 if (netif_running(wx->netdev))
862 wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));
863 }
864
865 return min(work_done, budget - 1);
866 }
867
wx_maybe_stop_tx(struct wx_ring * tx_ring,u16 size)868 static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)
869 {
870 if (likely(wx_desc_unused(tx_ring) >= size))
871 return 0;
872
873 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
874
875 /* For the next check */
876 smp_mb();
877
878 /* We need to check again in a case another CPU has just
879 * made room available.
880 */
881 if (likely(wx_desc_unused(tx_ring) < size))
882 return -EBUSY;
883
884 /* A reprieve! - use start_queue because it doesn't call schedule */
885 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
886
887 return 0;
888 }
889
wx_tx_cmd_type(u32 tx_flags)890 static u32 wx_tx_cmd_type(u32 tx_flags)
891 {
892 /* set type for advanced descriptor with frame checksum insertion */
893 u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;
894
895 /* set HW vlan bit if vlan is present */
896 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);
897 /* set segmentation enable bits for TSO/FSO */
898 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);
899 /* set timestamp bit if present */
900 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);
901 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);
902
903 return cmd_type;
904 }
905
wx_tx_olinfo_status(union wx_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)906 static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,
907 u32 tx_flags, unsigned int paylen)
908 {
909 u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;
910
911 /* enable L4 checksum for TSO and TX checksum offload */
912 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);
913 /* enable IPv4 checksum for TSO */
914 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);
915 /* enable outer IPv4 checksum for TSO */
916 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,
917 WX_TXD_EIPCS);
918 /* Check Context must be set if Tx switch is enabled, which it
919 * always is for case where virtual functions are running
920 */
921 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);
922 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,
923 WX_TXD_IPSEC);
924 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
925 }
926
wx_tx_map(struct wx_ring * tx_ring,struct wx_tx_buffer * first,const u8 hdr_len)927 static void wx_tx_map(struct wx_ring *tx_ring,
928 struct wx_tx_buffer *first,
929 const u8 hdr_len)
930 {
931 struct sk_buff *skb = first->skb;
932 struct wx_tx_buffer *tx_buffer;
933 u32 tx_flags = first->tx_flags;
934 u16 i = tx_ring->next_to_use;
935 unsigned int data_len, size;
936 union wx_tx_desc *tx_desc;
937 skb_frag_t *frag;
938 dma_addr_t dma;
939 u32 cmd_type;
940
941 cmd_type = wx_tx_cmd_type(tx_flags);
942 tx_desc = WX_TX_DESC(tx_ring, i);
943 wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
944
945 size = skb_headlen(skb);
946 data_len = skb->data_len;
947 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
948
949 tx_buffer = first;
950
951 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
952 if (dma_mapping_error(tx_ring->dev, dma))
953 goto dma_error;
954
955 /* record length, and DMA address */
956 dma_unmap_len_set(tx_buffer, len, size);
957 dma_unmap_addr_set(tx_buffer, dma, dma);
958
959 tx_desc->read.buffer_addr = cpu_to_le64(dma);
960
961 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {
962 tx_desc->read.cmd_type_len =
963 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);
964
965 i++;
966 tx_desc++;
967 if (i == tx_ring->count) {
968 tx_desc = WX_TX_DESC(tx_ring, 0);
969 i = 0;
970 }
971 tx_desc->read.olinfo_status = 0;
972
973 dma += WX_MAX_DATA_PER_TXD;
974 size -= WX_MAX_DATA_PER_TXD;
975
976 tx_desc->read.buffer_addr = cpu_to_le64(dma);
977 }
978
979 if (likely(!data_len))
980 break;
981
982 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
983
984 i++;
985 tx_desc++;
986 if (i == tx_ring->count) {
987 tx_desc = WX_TX_DESC(tx_ring, 0);
988 i = 0;
989 }
990 tx_desc->read.olinfo_status = 0;
991
992 size = skb_frag_size(frag);
993
994 data_len -= size;
995
996 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
997 DMA_TO_DEVICE);
998
999 tx_buffer = &tx_ring->tx_buffer_info[i];
1000 }
1001
1002 /* write last descriptor with RS and EOP bits */
1003 cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;
1004 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1005
1006 netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);
1007
1008 skb_tx_timestamp(skb);
1009
1010 /* Force memory writes to complete before letting h/w know there
1011 * are new descriptors to fetch. (Only applicable for weak-ordered
1012 * memory model archs, such as IA-64).
1013 *
1014 * We also need this memory barrier to make certain all of the
1015 * status bits have been updated before next_to_watch is written.
1016 */
1017 wmb();
1018
1019 /* set next_to_watch value indicating a packet is present */
1020 first->next_to_watch = tx_desc;
1021
1022 i++;
1023 if (i == tx_ring->count)
1024 i = 0;
1025
1026 tx_ring->next_to_use = i;
1027
1028 wx_maybe_stop_tx(tx_ring, DESC_NEEDED);
1029
1030 if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())
1031 writel(i, tx_ring->tail);
1032
1033 return;
1034 dma_error:
1035 dev_err(tx_ring->dev, "TX DMA map failed\n");
1036
1037 /* clear dma mappings for failed tx_buffer_info map */
1038 for (;;) {
1039 tx_buffer = &tx_ring->tx_buffer_info[i];
1040 if (dma_unmap_len(tx_buffer, len))
1041 dma_unmap_page(tx_ring->dev,
1042 dma_unmap_addr(tx_buffer, dma),
1043 dma_unmap_len(tx_buffer, len),
1044 DMA_TO_DEVICE);
1045 dma_unmap_len_set(tx_buffer, len, 0);
1046 if (tx_buffer == first)
1047 break;
1048 if (i == 0)
1049 i += tx_ring->count;
1050 i--;
1051 }
1052
1053 dev_kfree_skb_any(first->skb);
1054 first->skb = NULL;
1055
1056 tx_ring->next_to_use = i;
1057 }
1058
wx_tx_ctxtdesc(struct wx_ring * tx_ring,u32 vlan_macip_lens,u32 fcoe_sof_eof,u32 type_tucmd,u32 mss_l4len_idx)1059 static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,
1060 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
1061 {
1062 struct wx_tx_context_desc *context_desc;
1063 u16 i = tx_ring->next_to_use;
1064
1065 context_desc = WX_TX_CTXTDESC(tx_ring, i);
1066 i++;
1067 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1068
1069 /* set bits to identify this as an advanced context descriptor */
1070 type_tucmd |= WX_TXD_DTYP_CTXT;
1071 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1072 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
1073 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1074 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1075 }
1076
wx_get_ipv6_proto(struct sk_buff * skb,int offset,u8 * nexthdr)1077 static void wx_get_ipv6_proto(struct sk_buff *skb, int offset, u8 *nexthdr)
1078 {
1079 struct ipv6hdr *hdr = (struct ipv6hdr *)(skb->data + offset);
1080
1081 *nexthdr = hdr->nexthdr;
1082 offset += sizeof(struct ipv6hdr);
1083 while (ipv6_ext_hdr(*nexthdr)) {
1084 struct ipv6_opt_hdr _hdr, *hp;
1085
1086 if (*nexthdr == NEXTHDR_NONE)
1087 return;
1088 hp = skb_header_pointer(skb, offset, sizeof(_hdr), &_hdr);
1089 if (!hp)
1090 return;
1091 if (*nexthdr == NEXTHDR_FRAGMENT)
1092 break;
1093 *nexthdr = hp->nexthdr;
1094 }
1095 }
1096
1097 union network_header {
1098 struct iphdr *ipv4;
1099 struct ipv6hdr *ipv6;
1100 void *raw;
1101 };
1102
wx_encode_tx_desc_ptype(const struct wx_tx_buffer * first)1103 static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)
1104 {
1105 u8 tun_prot = 0, l4_prot = 0, ptype = 0;
1106 struct sk_buff *skb = first->skb;
1107
1108 if (skb->encapsulation) {
1109 union network_header hdr;
1110
1111 switch (first->protocol) {
1112 case htons(ETH_P_IP):
1113 tun_prot = ip_hdr(skb)->protocol;
1114 ptype = WX_PTYPE_TUN_IPV4;
1115 break;
1116 case htons(ETH_P_IPV6):
1117 wx_get_ipv6_proto(skb, skb_network_offset(skb), &tun_prot);
1118 ptype = WX_PTYPE_TUN_IPV6;
1119 break;
1120 default:
1121 return ptype;
1122 }
1123
1124 if (tun_prot == IPPROTO_IPIP) {
1125 hdr.raw = (void *)inner_ip_hdr(skb);
1126 ptype |= WX_PTYPE_PKT_IPIP;
1127 } else if (tun_prot == IPPROTO_UDP) {
1128 hdr.raw = (void *)inner_ip_hdr(skb);
1129 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1130 skb->inner_protocol != htons(ETH_P_TEB)) {
1131 ptype |= WX_PTYPE_PKT_IG;
1132 } else {
1133 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1134 == htons(ETH_P_8021Q))
1135 ptype |= WX_PTYPE_PKT_IGMV;
1136 else
1137 ptype |= WX_PTYPE_PKT_IGM;
1138 }
1139
1140 } else if (tun_prot == IPPROTO_GRE) {
1141 hdr.raw = (void *)inner_ip_hdr(skb);
1142 if (skb->inner_protocol == htons(ETH_P_IP) ||
1143 skb->inner_protocol == htons(ETH_P_IPV6)) {
1144 ptype |= WX_PTYPE_PKT_IG;
1145 } else {
1146 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1147 == htons(ETH_P_8021Q))
1148 ptype |= WX_PTYPE_PKT_IGMV;
1149 else
1150 ptype |= WX_PTYPE_PKT_IGM;
1151 }
1152 } else {
1153 return ptype;
1154 }
1155
1156 switch (hdr.ipv4->version) {
1157 case IPVERSION:
1158 l4_prot = hdr.ipv4->protocol;
1159 break;
1160 case 6:
1161 wx_get_ipv6_proto(skb, skb_inner_network_offset(skb), &l4_prot);
1162 ptype |= WX_PTYPE_PKT_IPV6;
1163 break;
1164 default:
1165 return ptype;
1166 }
1167 } else {
1168 switch (first->protocol) {
1169 case htons(ETH_P_IP):
1170 l4_prot = ip_hdr(skb)->protocol;
1171 ptype = WX_PTYPE_PKT_IP;
1172 break;
1173 case htons(ETH_P_IPV6):
1174 wx_get_ipv6_proto(skb, skb_network_offset(skb), &l4_prot);
1175 ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;
1176 break;
1177 default:
1178 return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;
1179 }
1180 }
1181 switch (l4_prot) {
1182 case IPPROTO_TCP:
1183 ptype |= WX_PTYPE_TYP_TCP;
1184 break;
1185 case IPPROTO_UDP:
1186 ptype |= WX_PTYPE_TYP_UDP;
1187 break;
1188 case IPPROTO_SCTP:
1189 ptype |= WX_PTYPE_TYP_SCTP;
1190 break;
1191 default:
1192 ptype |= WX_PTYPE_TYP_IP;
1193 break;
1194 }
1195
1196 return ptype;
1197 }
1198
wx_tso(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 * hdr_len,u8 ptype)1199 static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1200 u8 *hdr_len, u8 ptype)
1201 {
1202 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1203 struct net_device *netdev = tx_ring->netdev;
1204 u32 l4len, tunhdr_eiplen_tunlen = 0;
1205 struct sk_buff *skb = first->skb;
1206 bool enc = skb->encapsulation;
1207 struct ipv6hdr *ipv6h;
1208 struct tcphdr *tcph;
1209 struct iphdr *iph;
1210 u8 tun_prot = 0;
1211 int err;
1212
1213 if (skb->ip_summed != CHECKSUM_PARTIAL)
1214 return 0;
1215
1216 if (!skb_is_gso(skb))
1217 return 0;
1218
1219 err = skb_cow_head(skb, 0);
1220 if (err < 0)
1221 return err;
1222
1223 /* indicates the inner headers in the skbuff are valid. */
1224 iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);
1225 if (iph->version == 4) {
1226 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1227 iph->tot_len = 0;
1228 iph->check = 0;
1229 tcph->check = ~csum_tcpudp_magic(iph->saddr,
1230 iph->daddr, 0,
1231 IPPROTO_TCP, 0);
1232 first->tx_flags |= WX_TX_FLAGS_TSO |
1233 WX_TX_FLAGS_CSUM |
1234 WX_TX_FLAGS_IPV4 |
1235 WX_TX_FLAGS_CC;
1236 } else if (iph->version == 6 && skb_is_gso_v6(skb)) {
1237 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
1238 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1239 ipv6h->payload_len = 0;
1240 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1241 &ipv6h->daddr, 0,
1242 IPPROTO_TCP, 0);
1243 first->tx_flags |= WX_TX_FLAGS_TSO |
1244 WX_TX_FLAGS_CSUM |
1245 WX_TX_FLAGS_CC;
1246 }
1247
1248 /* compute header lengths */
1249 l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1250 *hdr_len = enc ? (skb_inner_transport_header(skb) - skb->data) :
1251 skb_transport_offset(skb);
1252 *hdr_len += l4len;
1253
1254 /* update gso size and bytecount with header size */
1255 first->gso_segs = skb_shinfo(skb)->gso_segs;
1256 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1257
1258 /* mss_l4len_id: use 0 as index for TSO */
1259 mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;
1260 mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;
1261
1262 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
1263 if (enc) {
1264 switch (first->protocol) {
1265 case htons(ETH_P_IP):
1266 tun_prot = ip_hdr(skb)->protocol;
1267 first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;
1268 break;
1269 case htons(ETH_P_IPV6):
1270 tun_prot = ipv6_hdr(skb)->nexthdr;
1271 break;
1272 default:
1273 break;
1274 }
1275 switch (tun_prot) {
1276 case IPPROTO_UDP:
1277 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1278 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1279 WX_TXD_OUTER_IPLEN_SHIFT) |
1280 (((skb_inner_mac_header(skb) -
1281 skb_transport_header(skb)) >> 1) <<
1282 WX_TXD_TUNNEL_LEN_SHIFT);
1283 break;
1284 case IPPROTO_GRE:
1285 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1286 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1287 WX_TXD_OUTER_IPLEN_SHIFT) |
1288 (((skb_inner_mac_header(skb) -
1289 skb_transport_header(skb)) >> 1) <<
1290 WX_TXD_TUNNEL_LEN_SHIFT);
1291 break;
1292 case IPPROTO_IPIP:
1293 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1294 (char *)ip_hdr(skb)) >> 2) <<
1295 WX_TXD_OUTER_IPLEN_SHIFT;
1296 break;
1297 default:
1298 break;
1299 }
1300 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;
1301 } else {
1302 vlan_macip_lens = skb_network_header_len(skb) >> 1;
1303 }
1304
1305 vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;
1306 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1307
1308 type_tucmd = ptype << 24;
1309 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1310 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1311 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1312 WX_TX_FLAGS_HW_VLAN,
1313 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1314 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1315 type_tucmd, mss_l4len_idx);
1316
1317 return 1;
1318 }
1319
wx_tx_csum(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 ptype)1320 static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1321 u8 ptype)
1322 {
1323 u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;
1324 struct net_device *netdev = tx_ring->netdev;
1325 u32 mss_l4len_idx = 0, type_tucmd;
1326 struct sk_buff *skb = first->skb;
1327 u8 tun_prot = 0;
1328
1329 if (skb->ip_summed != CHECKSUM_PARTIAL) {
1330 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&
1331 !(first->tx_flags & WX_TX_FLAGS_CC))
1332 return;
1333 vlan_macip_lens = skb_network_offset(skb) <<
1334 WX_TXD_MACLEN_SHIFT;
1335 } else {
1336 u8 l4_prot = 0;
1337 union {
1338 struct iphdr *ipv4;
1339 struct ipv6hdr *ipv6;
1340 u8 *raw;
1341 } network_hdr;
1342 union {
1343 struct tcphdr *tcphdr;
1344 u8 *raw;
1345 } transport_hdr;
1346
1347 if (skb->encapsulation) {
1348 network_hdr.raw = skb_inner_network_header(skb);
1349 transport_hdr.raw = skb_inner_transport_header(skb);
1350 vlan_macip_lens = skb_network_offset(skb) <<
1351 WX_TXD_MACLEN_SHIFT;
1352 switch (first->protocol) {
1353 case htons(ETH_P_IP):
1354 tun_prot = ip_hdr(skb)->protocol;
1355 break;
1356 case htons(ETH_P_IPV6):
1357 tun_prot = ipv6_hdr(skb)->nexthdr;
1358 break;
1359 default:
1360 return;
1361 }
1362 switch (tun_prot) {
1363 case IPPROTO_UDP:
1364 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1365 tunhdr_eiplen_tunlen |=
1366 ((skb_network_header_len(skb) >> 2) <<
1367 WX_TXD_OUTER_IPLEN_SHIFT) |
1368 (((skb_inner_mac_header(skb) -
1369 skb_transport_header(skb)) >> 1) <<
1370 WX_TXD_TUNNEL_LEN_SHIFT);
1371 break;
1372 case IPPROTO_GRE:
1373 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1374 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1375 WX_TXD_OUTER_IPLEN_SHIFT) |
1376 (((skb_inner_mac_header(skb) -
1377 skb_transport_header(skb)) >> 1) <<
1378 WX_TXD_TUNNEL_LEN_SHIFT);
1379 break;
1380 case IPPROTO_IPIP:
1381 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1382 (char *)ip_hdr(skb)) >> 2) <<
1383 WX_TXD_OUTER_IPLEN_SHIFT;
1384 break;
1385 default:
1386 break;
1387 }
1388
1389 } else {
1390 network_hdr.raw = skb_network_header(skb);
1391 transport_hdr.raw = skb_transport_header(skb);
1392 vlan_macip_lens = skb_network_offset(skb) <<
1393 WX_TXD_MACLEN_SHIFT;
1394 }
1395
1396 switch (network_hdr.ipv4->version) {
1397 case IPVERSION:
1398 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1399 l4_prot = network_hdr.ipv4->protocol;
1400 break;
1401 case 6:
1402 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1403 l4_prot = network_hdr.ipv6->nexthdr;
1404 break;
1405 default:
1406 break;
1407 }
1408
1409 switch (l4_prot) {
1410 case IPPROTO_TCP:
1411 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<
1412 WX_TXD_L4LEN_SHIFT;
1413 break;
1414 case IPPROTO_SCTP:
1415 mss_l4len_idx = sizeof(struct sctphdr) <<
1416 WX_TXD_L4LEN_SHIFT;
1417 break;
1418 case IPPROTO_UDP:
1419 mss_l4len_idx = sizeof(struct udphdr) <<
1420 WX_TXD_L4LEN_SHIFT;
1421 break;
1422 default:
1423 break;
1424 }
1425
1426 /* update TX checksum flag */
1427 first->tx_flags |= WX_TX_FLAGS_CSUM;
1428 }
1429 first->tx_flags |= WX_TX_FLAGS_CC;
1430 /* vlan_macip_lens: MACLEN, VLAN tag */
1431 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1432
1433 type_tucmd = ptype << 24;
1434 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1435 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1436 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1437 WX_TX_FLAGS_HW_VLAN,
1438 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1439 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1440 type_tucmd, mss_l4len_idx);
1441 }
1442
wx_xmit_frame_ring(struct sk_buff * skb,struct wx_ring * tx_ring)1443 static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
1444 struct wx_ring *tx_ring)
1445 {
1446 u16 count = TXD_USE_COUNT(skb_headlen(skb));
1447 struct wx_tx_buffer *first;
1448 u8 hdr_len = 0, ptype;
1449 unsigned short f;
1450 u32 tx_flags = 0;
1451 int tso;
1452
1453 /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,
1454 * + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,
1455 * + 2 desc gap to keep tail from touching head,
1456 * + 1 desc for context descriptor,
1457 * otherwise try next time
1458 */
1459 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1460 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->
1461 frags[f]));
1462
1463 if (wx_maybe_stop_tx(tx_ring, count + 3))
1464 return NETDEV_TX_BUSY;
1465
1466 /* record the location of the first descriptor for this packet */
1467 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1468 first->skb = skb;
1469 first->bytecount = skb->len;
1470 first->gso_segs = 1;
1471
1472 /* if we have a HW VLAN tag being added default to the HW one */
1473 if (skb_vlan_tag_present(skb)) {
1474 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;
1475 tx_flags |= WX_TX_FLAGS_HW_VLAN;
1476 }
1477
1478 /* record initial flags and protocol */
1479 first->tx_flags = tx_flags;
1480 first->protocol = vlan_get_protocol(skb);
1481
1482 ptype = wx_encode_tx_desc_ptype(first);
1483
1484 tso = wx_tso(tx_ring, first, &hdr_len, ptype);
1485 if (tso < 0)
1486 goto out_drop;
1487 else if (!tso)
1488 wx_tx_csum(tx_ring, first, ptype);
1489 wx_tx_map(tx_ring, first, hdr_len);
1490
1491 return NETDEV_TX_OK;
1492 out_drop:
1493 dev_kfree_skb_any(first->skb);
1494 first->skb = NULL;
1495
1496 return NETDEV_TX_OK;
1497 }
1498
wx_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1499 netdev_tx_t wx_xmit_frame(struct sk_buff *skb,
1500 struct net_device *netdev)
1501 {
1502 unsigned int r_idx = skb->queue_mapping;
1503 struct wx *wx = netdev_priv(netdev);
1504 struct wx_ring *tx_ring;
1505
1506 if (!netif_carrier_ok(netdev)) {
1507 dev_kfree_skb_any(skb);
1508 return NETDEV_TX_OK;
1509 }
1510
1511 /* The minimum packet size for olinfo paylen is 17 so pad the skb
1512 * in order to meet this minimum size requirement.
1513 */
1514 if (skb_put_padto(skb, 17))
1515 return NETDEV_TX_OK;
1516
1517 if (r_idx >= wx->num_tx_queues)
1518 r_idx = r_idx % wx->num_tx_queues;
1519 tx_ring = wx->tx_ring[r_idx];
1520
1521 return wx_xmit_frame_ring(skb, tx_ring);
1522 }
1523 EXPORT_SYMBOL(wx_xmit_frame);
1524
wx_napi_enable_all(struct wx * wx)1525 void wx_napi_enable_all(struct wx *wx)
1526 {
1527 struct wx_q_vector *q_vector;
1528 int q_idx;
1529
1530 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1531 q_vector = wx->q_vector[q_idx];
1532 napi_enable(&q_vector->napi);
1533 }
1534 }
1535 EXPORT_SYMBOL(wx_napi_enable_all);
1536
wx_napi_disable_all(struct wx * wx)1537 void wx_napi_disable_all(struct wx *wx)
1538 {
1539 struct wx_q_vector *q_vector;
1540 int q_idx;
1541
1542 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1543 q_vector = wx->q_vector[q_idx];
1544 napi_disable(&q_vector->napi);
1545 }
1546 }
1547 EXPORT_SYMBOL(wx_napi_disable_all);
1548
1549 /**
1550 * wx_set_rss_queues: Allocate queues for RSS
1551 * @wx: board private structure to initialize
1552 *
1553 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
1554 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1555 *
1556 **/
wx_set_rss_queues(struct wx * wx)1557 static void wx_set_rss_queues(struct wx *wx)
1558 {
1559 wx->num_rx_queues = wx->mac.max_rx_queues;
1560 wx->num_tx_queues = wx->mac.max_tx_queues;
1561 }
1562
wx_set_num_queues(struct wx * wx)1563 static void wx_set_num_queues(struct wx *wx)
1564 {
1565 /* Start with base case */
1566 wx->num_rx_queues = 1;
1567 wx->num_tx_queues = 1;
1568 wx->queues_per_pool = 1;
1569
1570 wx_set_rss_queues(wx);
1571 }
1572
1573 /**
1574 * wx_acquire_msix_vectors - acquire MSI-X vectors
1575 * @wx: board private structure
1576 *
1577 * Attempts to acquire a suitable range of MSI-X vector interrupts. Will
1578 * return a negative error code if unable to acquire MSI-X vectors for any
1579 * reason.
1580 */
wx_acquire_msix_vectors(struct wx * wx)1581 static int wx_acquire_msix_vectors(struct wx *wx)
1582 {
1583 struct irq_affinity affd = {0, };
1584 int nvecs, i;
1585
1586 nvecs = min_t(int, num_online_cpus(), wx->mac.max_msix_vectors);
1587
1588 wx->msix_entries = kcalloc(nvecs,
1589 sizeof(struct msix_entry),
1590 GFP_KERNEL);
1591 if (!wx->msix_entries)
1592 return -ENOMEM;
1593
1594 nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,
1595 nvecs,
1596 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
1597 &affd);
1598 if (nvecs < 0) {
1599 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);
1600 kfree(wx->msix_entries);
1601 wx->msix_entries = NULL;
1602 return nvecs;
1603 }
1604
1605 for (i = 0; i < nvecs; i++) {
1606 wx->msix_entries[i].entry = i;
1607 wx->msix_entries[i].vector = pci_irq_vector(wx->pdev, i);
1608 }
1609
1610 /* one for msix_other */
1611 nvecs -= 1;
1612 wx->num_q_vectors = nvecs;
1613 wx->num_rx_queues = nvecs;
1614 wx->num_tx_queues = nvecs;
1615
1616 return 0;
1617 }
1618
1619 /**
1620 * wx_set_interrupt_capability - set MSI-X or MSI if supported
1621 * @wx: board private structure to initialize
1622 *
1623 * Attempt to configure the interrupts using the best available
1624 * capabilities of the hardware and the kernel.
1625 **/
wx_set_interrupt_capability(struct wx * wx)1626 static int wx_set_interrupt_capability(struct wx *wx)
1627 {
1628 struct pci_dev *pdev = wx->pdev;
1629 int nvecs, ret;
1630
1631 /* We will try to get MSI-X interrupts first */
1632 ret = wx_acquire_msix_vectors(wx);
1633 if (ret == 0 || (ret == -ENOMEM))
1634 return ret;
1635
1636 wx->num_rx_queues = 1;
1637 wx->num_tx_queues = 1;
1638 wx->num_q_vectors = 1;
1639
1640 /* minmum one for queue, one for misc*/
1641 nvecs = 1;
1642 nvecs = pci_alloc_irq_vectors(pdev, nvecs,
1643 nvecs, PCI_IRQ_MSI | PCI_IRQ_LEGACY);
1644 if (nvecs == 1) {
1645 if (pdev->msi_enabled)
1646 wx_err(wx, "Fallback to MSI.\n");
1647 else
1648 wx_err(wx, "Fallback to LEGACY.\n");
1649 } else {
1650 wx_err(wx, "Failed to allocate MSI/LEGACY interrupts. Error: %d\n", nvecs);
1651 return nvecs;
1652 }
1653
1654 pdev->irq = pci_irq_vector(pdev, 0);
1655 wx->num_q_vectors = 1;
1656
1657 return 0;
1658 }
1659
1660 /**
1661 * wx_cache_ring_rss - Descriptor ring to register mapping for RSS
1662 * @wx: board private structure to initialize
1663 *
1664 * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.
1665 *
1666 **/
wx_cache_ring_rss(struct wx * wx)1667 static void wx_cache_ring_rss(struct wx *wx)
1668 {
1669 u16 i;
1670
1671 for (i = 0; i < wx->num_rx_queues; i++)
1672 wx->rx_ring[i]->reg_idx = i;
1673
1674 for (i = 0; i < wx->num_tx_queues; i++)
1675 wx->tx_ring[i]->reg_idx = i;
1676 }
1677
wx_add_ring(struct wx_ring * ring,struct wx_ring_container * head)1678 static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)
1679 {
1680 ring->next = head->ring;
1681 head->ring = ring;
1682 head->count++;
1683 }
1684
1685 /**
1686 * wx_alloc_q_vector - Allocate memory for a single interrupt vector
1687 * @wx: board private structure to initialize
1688 * @v_count: q_vectors allocated on wx, used for ring interleaving
1689 * @v_idx: index of vector in wx struct
1690 * @txr_count: total number of Tx rings to allocate
1691 * @txr_idx: index of first Tx ring to allocate
1692 * @rxr_count: total number of Rx rings to allocate
1693 * @rxr_idx: index of first Rx ring to allocate
1694 *
1695 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1696 **/
wx_alloc_q_vector(struct wx * wx,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)1697 static int wx_alloc_q_vector(struct wx *wx,
1698 unsigned int v_count, unsigned int v_idx,
1699 unsigned int txr_count, unsigned int txr_idx,
1700 unsigned int rxr_count, unsigned int rxr_idx)
1701 {
1702 struct wx_q_vector *q_vector;
1703 int ring_count, default_itr;
1704 struct wx_ring *ring;
1705
1706 /* note this will allocate space for the ring structure as well! */
1707 ring_count = txr_count + rxr_count;
1708
1709 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
1710 GFP_KERNEL);
1711 if (!q_vector)
1712 return -ENOMEM;
1713
1714 /* initialize NAPI */
1715 netif_napi_add(wx->netdev, &q_vector->napi,
1716 wx_poll);
1717
1718 /* tie q_vector and wx together */
1719 wx->q_vector[v_idx] = q_vector;
1720 q_vector->wx = wx;
1721 q_vector->v_idx = v_idx;
1722 if (cpu_online(v_idx))
1723 q_vector->numa_node = cpu_to_node(v_idx);
1724
1725 /* initialize pointer to rings */
1726 ring = q_vector->ring;
1727
1728 if (wx->mac.type == wx_mac_sp)
1729 default_itr = WX_12K_ITR;
1730 else
1731 default_itr = WX_7K_ITR;
1732 /* initialize ITR */
1733 if (txr_count && !rxr_count)
1734 /* tx only vector */
1735 q_vector->itr = wx->tx_itr_setting ?
1736 default_itr : wx->tx_itr_setting;
1737 else
1738 /* rx or rx/tx vector */
1739 q_vector->itr = wx->rx_itr_setting ?
1740 default_itr : wx->rx_itr_setting;
1741
1742 while (txr_count) {
1743 /* assign generic ring traits */
1744 ring->dev = &wx->pdev->dev;
1745 ring->netdev = wx->netdev;
1746
1747 /* configure backlink on ring */
1748 ring->q_vector = q_vector;
1749
1750 /* update q_vector Tx values */
1751 wx_add_ring(ring, &q_vector->tx);
1752
1753 /* apply Tx specific ring traits */
1754 ring->count = wx->tx_ring_count;
1755
1756 ring->queue_index = txr_idx;
1757
1758 /* assign ring to wx */
1759 wx->tx_ring[txr_idx] = ring;
1760
1761 /* update count and index */
1762 txr_count--;
1763 txr_idx += v_count;
1764
1765 /* push pointer to next ring */
1766 ring++;
1767 }
1768
1769 while (rxr_count) {
1770 /* assign generic ring traits */
1771 ring->dev = &wx->pdev->dev;
1772 ring->netdev = wx->netdev;
1773
1774 /* configure backlink on ring */
1775 ring->q_vector = q_vector;
1776
1777 /* update q_vector Rx values */
1778 wx_add_ring(ring, &q_vector->rx);
1779
1780 /* apply Rx specific ring traits */
1781 ring->count = wx->rx_ring_count;
1782 ring->queue_index = rxr_idx;
1783
1784 /* assign ring to wx */
1785 wx->rx_ring[rxr_idx] = ring;
1786
1787 /* update count and index */
1788 rxr_count--;
1789 rxr_idx += v_count;
1790
1791 /* push pointer to next ring */
1792 ring++;
1793 }
1794
1795 return 0;
1796 }
1797
1798 /**
1799 * wx_free_q_vector - Free memory allocated for specific interrupt vector
1800 * @wx: board private structure to initialize
1801 * @v_idx: Index of vector to be freed
1802 *
1803 * This function frees the memory allocated to the q_vector. In addition if
1804 * NAPI is enabled it will delete any references to the NAPI struct prior
1805 * to freeing the q_vector.
1806 **/
wx_free_q_vector(struct wx * wx,int v_idx)1807 static void wx_free_q_vector(struct wx *wx, int v_idx)
1808 {
1809 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
1810 struct wx_ring *ring;
1811
1812 wx_for_each_ring(ring, q_vector->tx)
1813 wx->tx_ring[ring->queue_index] = NULL;
1814
1815 wx_for_each_ring(ring, q_vector->rx)
1816 wx->rx_ring[ring->queue_index] = NULL;
1817
1818 wx->q_vector[v_idx] = NULL;
1819 netif_napi_del(&q_vector->napi);
1820 kfree_rcu(q_vector, rcu);
1821 }
1822
1823 /**
1824 * wx_alloc_q_vectors - Allocate memory for interrupt vectors
1825 * @wx: board private structure to initialize
1826 *
1827 * We allocate one q_vector per queue interrupt. If allocation fails we
1828 * return -ENOMEM.
1829 **/
wx_alloc_q_vectors(struct wx * wx)1830 static int wx_alloc_q_vectors(struct wx *wx)
1831 {
1832 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1833 unsigned int rxr_remaining = wx->num_rx_queues;
1834 unsigned int txr_remaining = wx->num_tx_queues;
1835 unsigned int q_vectors = wx->num_q_vectors;
1836 int rqpv, tqpv;
1837 int err;
1838
1839 for (; v_idx < q_vectors; v_idx++) {
1840 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1841 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1842 err = wx_alloc_q_vector(wx, q_vectors, v_idx,
1843 tqpv, txr_idx,
1844 rqpv, rxr_idx);
1845
1846 if (err)
1847 goto err_out;
1848
1849 /* update counts and index */
1850 rxr_remaining -= rqpv;
1851 txr_remaining -= tqpv;
1852 rxr_idx++;
1853 txr_idx++;
1854 }
1855
1856 return 0;
1857
1858 err_out:
1859 wx->num_tx_queues = 0;
1860 wx->num_rx_queues = 0;
1861 wx->num_q_vectors = 0;
1862
1863 while (v_idx--)
1864 wx_free_q_vector(wx, v_idx);
1865
1866 return -ENOMEM;
1867 }
1868
1869 /**
1870 * wx_free_q_vectors - Free memory allocated for interrupt vectors
1871 * @wx: board private structure to initialize
1872 *
1873 * This function frees the memory allocated to the q_vectors. In addition if
1874 * NAPI is enabled it will delete any references to the NAPI struct prior
1875 * to freeing the q_vector.
1876 **/
wx_free_q_vectors(struct wx * wx)1877 static void wx_free_q_vectors(struct wx *wx)
1878 {
1879 int v_idx = wx->num_q_vectors;
1880
1881 wx->num_tx_queues = 0;
1882 wx->num_rx_queues = 0;
1883 wx->num_q_vectors = 0;
1884
1885 while (v_idx--)
1886 wx_free_q_vector(wx, v_idx);
1887 }
1888
wx_reset_interrupt_capability(struct wx * wx)1889 void wx_reset_interrupt_capability(struct wx *wx)
1890 {
1891 struct pci_dev *pdev = wx->pdev;
1892
1893 if (!pdev->msi_enabled && !pdev->msix_enabled)
1894 return;
1895
1896 if (pdev->msix_enabled) {
1897 kfree(wx->msix_entries);
1898 wx->msix_entries = NULL;
1899 }
1900 pci_free_irq_vectors(wx->pdev);
1901 }
1902 EXPORT_SYMBOL(wx_reset_interrupt_capability);
1903
1904 /**
1905 * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings
1906 * @wx: board private structure to clear interrupt scheme on
1907 *
1908 * We go through and clear interrupt specific resources and reset the structure
1909 * to pre-load conditions
1910 **/
wx_clear_interrupt_scheme(struct wx * wx)1911 void wx_clear_interrupt_scheme(struct wx *wx)
1912 {
1913 wx_free_q_vectors(wx);
1914 wx_reset_interrupt_capability(wx);
1915 }
1916 EXPORT_SYMBOL(wx_clear_interrupt_scheme);
1917
wx_init_interrupt_scheme(struct wx * wx)1918 int wx_init_interrupt_scheme(struct wx *wx)
1919 {
1920 int ret;
1921
1922 /* Number of supported queues */
1923 wx_set_num_queues(wx);
1924
1925 /* Set interrupt mode */
1926 ret = wx_set_interrupt_capability(wx);
1927 if (ret) {
1928 wx_err(wx, "Allocate irq vectors for failed.\n");
1929 return ret;
1930 }
1931
1932 /* Allocate memory for queues */
1933 ret = wx_alloc_q_vectors(wx);
1934 if (ret) {
1935 wx_err(wx, "Unable to allocate memory for queue vectors.\n");
1936 wx_reset_interrupt_capability(wx);
1937 return ret;
1938 }
1939
1940 wx_cache_ring_rss(wx);
1941
1942 return 0;
1943 }
1944 EXPORT_SYMBOL(wx_init_interrupt_scheme);
1945
wx_msix_clean_rings(int __always_unused irq,void * data)1946 irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)
1947 {
1948 struct wx_q_vector *q_vector = data;
1949
1950 /* EIAM disabled interrupts (on this vector) for us */
1951 if (q_vector->rx.ring || q_vector->tx.ring)
1952 napi_schedule_irqoff(&q_vector->napi);
1953
1954 return IRQ_HANDLED;
1955 }
1956 EXPORT_SYMBOL(wx_msix_clean_rings);
1957
wx_free_irq(struct wx * wx)1958 void wx_free_irq(struct wx *wx)
1959 {
1960 struct pci_dev *pdev = wx->pdev;
1961 int vector;
1962
1963 if (!(pdev->msix_enabled)) {
1964 free_irq(pdev->irq, wx);
1965 return;
1966 }
1967
1968 for (vector = 0; vector < wx->num_q_vectors; vector++) {
1969 struct wx_q_vector *q_vector = wx->q_vector[vector];
1970 struct msix_entry *entry = &wx->msix_entries[vector];
1971
1972 /* free only the irqs that were actually requested */
1973 if (!q_vector->rx.ring && !q_vector->tx.ring)
1974 continue;
1975
1976 free_irq(entry->vector, q_vector);
1977 }
1978
1979 if (wx->mac.type == wx_mac_em)
1980 free_irq(wx->msix_entries[vector].vector, wx);
1981 }
1982 EXPORT_SYMBOL(wx_free_irq);
1983
1984 /**
1985 * wx_setup_isb_resources - allocate interrupt status resources
1986 * @wx: board private structure
1987 *
1988 * Return 0 on success, negative on failure
1989 **/
wx_setup_isb_resources(struct wx * wx)1990 int wx_setup_isb_resources(struct wx *wx)
1991 {
1992 struct pci_dev *pdev = wx->pdev;
1993
1994 wx->isb_mem = dma_alloc_coherent(&pdev->dev,
1995 sizeof(u32) * 4,
1996 &wx->isb_dma,
1997 GFP_KERNEL);
1998 if (!wx->isb_mem) {
1999 wx_err(wx, "Alloc isb_mem failed\n");
2000 return -ENOMEM;
2001 }
2002
2003 return 0;
2004 }
2005 EXPORT_SYMBOL(wx_setup_isb_resources);
2006
2007 /**
2008 * wx_free_isb_resources - allocate all queues Rx resources
2009 * @wx: board private structure
2010 *
2011 * Return 0 on success, negative on failure
2012 **/
wx_free_isb_resources(struct wx * wx)2013 void wx_free_isb_resources(struct wx *wx)
2014 {
2015 struct pci_dev *pdev = wx->pdev;
2016
2017 dma_free_coherent(&pdev->dev, sizeof(u32) * 4,
2018 wx->isb_mem, wx->isb_dma);
2019 wx->isb_mem = NULL;
2020 }
2021 EXPORT_SYMBOL(wx_free_isb_resources);
2022
wx_misc_isb(struct wx * wx,enum wx_isb_idx idx)2023 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)
2024 {
2025 u32 cur_tag = 0;
2026
2027 cur_tag = wx->isb_mem[WX_ISB_HEADER];
2028 wx->isb_tag[idx] = cur_tag;
2029
2030 return (__force u32)cpu_to_le32(wx->isb_mem[idx]);
2031 }
2032 EXPORT_SYMBOL(wx_misc_isb);
2033
2034 /**
2035 * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
2036 * @wx: pointer to wx struct
2037 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
2038 * @queue: queue to map the corresponding interrupt to
2039 * @msix_vector: the vector to map to the corresponding queue
2040 *
2041 **/
wx_set_ivar(struct wx * wx,s8 direction,u16 queue,u16 msix_vector)2042 static void wx_set_ivar(struct wx *wx, s8 direction,
2043 u16 queue, u16 msix_vector)
2044 {
2045 u32 ivar, index;
2046
2047 if (direction == -1) {
2048 /* other causes */
2049 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2050 index = 0;
2051 ivar = rd32(wx, WX_PX_MISC_IVAR);
2052 ivar &= ~(0xFF << index);
2053 ivar |= (msix_vector << index);
2054 wr32(wx, WX_PX_MISC_IVAR, ivar);
2055 } else {
2056 /* tx or rx causes */
2057 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2058 index = ((16 * (queue & 1)) + (8 * direction));
2059 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));
2060 ivar &= ~(0xFF << index);
2061 ivar |= (msix_vector << index);
2062 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);
2063 }
2064 }
2065
2066 /**
2067 * wx_write_eitr - write EITR register in hardware specific way
2068 * @q_vector: structure containing interrupt and ring information
2069 *
2070 * This function is made to be called by ethtool and by the driver
2071 * when it needs to update EITR registers at runtime. Hardware
2072 * specific quirks/differences are taken care of here.
2073 */
wx_write_eitr(struct wx_q_vector * q_vector)2074 static void wx_write_eitr(struct wx_q_vector *q_vector)
2075 {
2076 struct wx *wx = q_vector->wx;
2077 int v_idx = q_vector->v_idx;
2078 u32 itr_reg;
2079
2080 if (wx->mac.type == wx_mac_sp)
2081 itr_reg = q_vector->itr & WX_SP_MAX_EITR;
2082 else
2083 itr_reg = q_vector->itr & WX_EM_MAX_EITR;
2084
2085 itr_reg |= WX_PX_ITR_CNT_WDIS;
2086
2087 wr32(wx, WX_PX_ITR(v_idx), itr_reg);
2088 }
2089
2090 /**
2091 * wx_configure_vectors - Configure vectors for hardware
2092 * @wx: board private structure
2093 *
2094 * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/LEGACY
2095 * interrupts.
2096 **/
wx_configure_vectors(struct wx * wx)2097 void wx_configure_vectors(struct wx *wx)
2098 {
2099 struct pci_dev *pdev = wx->pdev;
2100 u32 eitrsel = 0;
2101 u16 v_idx;
2102
2103 if (pdev->msix_enabled) {
2104 /* Populate MSIX to EITR Select */
2105 wr32(wx, WX_PX_ITRSEL, eitrsel);
2106 /* use EIAM to auto-mask when MSI-X interrupt is asserted
2107 * this saves a register write for every interrupt
2108 */
2109 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);
2110 } else {
2111 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
2112 * specifically only auto mask tx and rx interrupts.
2113 */
2114 wr32(wx, WX_PX_GPIE, 0);
2115 }
2116
2117 /* Populate the IVAR table and set the ITR values to the
2118 * corresponding register.
2119 */
2120 for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {
2121 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2122 struct wx_ring *ring;
2123
2124 wx_for_each_ring(ring, q_vector->rx)
2125 wx_set_ivar(wx, 0, ring->reg_idx, v_idx);
2126
2127 wx_for_each_ring(ring, q_vector->tx)
2128 wx_set_ivar(wx, 1, ring->reg_idx, v_idx);
2129
2130 wx_write_eitr(q_vector);
2131 }
2132
2133 wx_set_ivar(wx, -1, 0, v_idx);
2134 if (pdev->msix_enabled)
2135 wr32(wx, WX_PX_ITR(v_idx), 1950);
2136 }
2137 EXPORT_SYMBOL(wx_configure_vectors);
2138
2139 /**
2140 * wx_clean_rx_ring - Free Rx Buffers per Queue
2141 * @rx_ring: ring to free buffers from
2142 **/
wx_clean_rx_ring(struct wx_ring * rx_ring)2143 static void wx_clean_rx_ring(struct wx_ring *rx_ring)
2144 {
2145 struct wx_rx_buffer *rx_buffer;
2146 u16 i = rx_ring->next_to_clean;
2147
2148 rx_buffer = &rx_ring->rx_buffer_info[i];
2149
2150 /* Free all the Rx ring sk_buffs */
2151 while (i != rx_ring->next_to_alloc) {
2152 if (rx_buffer->skb) {
2153 struct sk_buff *skb = rx_buffer->skb;
2154
2155 dev_kfree_skb(skb);
2156 }
2157
2158 /* Invalidate cache lines that may have been written to by
2159 * device so that we avoid corrupting memory.
2160 */
2161 dma_sync_single_range_for_cpu(rx_ring->dev,
2162 rx_buffer->dma,
2163 rx_buffer->page_offset,
2164 WX_RX_BUFSZ,
2165 DMA_FROM_DEVICE);
2166
2167 /* free resources associated with mapping */
2168 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2169
2170 i++;
2171 rx_buffer++;
2172 if (i == rx_ring->count) {
2173 i = 0;
2174 rx_buffer = rx_ring->rx_buffer_info;
2175 }
2176 }
2177
2178 /* Zero out the descriptor ring */
2179 memset(rx_ring->desc, 0, rx_ring->size);
2180
2181 rx_ring->next_to_alloc = 0;
2182 rx_ring->next_to_clean = 0;
2183 rx_ring->next_to_use = 0;
2184 }
2185
2186 /**
2187 * wx_clean_all_rx_rings - Free Rx Buffers for all queues
2188 * @wx: board private structure
2189 **/
wx_clean_all_rx_rings(struct wx * wx)2190 void wx_clean_all_rx_rings(struct wx *wx)
2191 {
2192 int i;
2193
2194 for (i = 0; i < wx->num_rx_queues; i++)
2195 wx_clean_rx_ring(wx->rx_ring[i]);
2196 }
2197 EXPORT_SYMBOL(wx_clean_all_rx_rings);
2198
2199 /**
2200 * wx_free_rx_resources - Free Rx Resources
2201 * @rx_ring: ring to clean the resources from
2202 *
2203 * Free all receive software resources
2204 **/
wx_free_rx_resources(struct wx_ring * rx_ring)2205 static void wx_free_rx_resources(struct wx_ring *rx_ring)
2206 {
2207 wx_clean_rx_ring(rx_ring);
2208 kvfree(rx_ring->rx_buffer_info);
2209 rx_ring->rx_buffer_info = NULL;
2210
2211 /* if not set, then don't free */
2212 if (!rx_ring->desc)
2213 return;
2214
2215 dma_free_coherent(rx_ring->dev, rx_ring->size,
2216 rx_ring->desc, rx_ring->dma);
2217
2218 rx_ring->desc = NULL;
2219
2220 if (rx_ring->page_pool) {
2221 page_pool_destroy(rx_ring->page_pool);
2222 rx_ring->page_pool = NULL;
2223 }
2224 }
2225
2226 /**
2227 * wx_free_all_rx_resources - Free Rx Resources for All Queues
2228 * @wx: pointer to hardware structure
2229 *
2230 * Free all receive software resources
2231 **/
wx_free_all_rx_resources(struct wx * wx)2232 static void wx_free_all_rx_resources(struct wx *wx)
2233 {
2234 int i;
2235
2236 for (i = 0; i < wx->num_rx_queues; i++)
2237 wx_free_rx_resources(wx->rx_ring[i]);
2238 }
2239
2240 /**
2241 * wx_clean_tx_ring - Free Tx Buffers
2242 * @tx_ring: ring to be cleaned
2243 **/
wx_clean_tx_ring(struct wx_ring * tx_ring)2244 static void wx_clean_tx_ring(struct wx_ring *tx_ring)
2245 {
2246 struct wx_tx_buffer *tx_buffer;
2247 u16 i = tx_ring->next_to_clean;
2248
2249 tx_buffer = &tx_ring->tx_buffer_info[i];
2250
2251 while (i != tx_ring->next_to_use) {
2252 union wx_tx_desc *eop_desc, *tx_desc;
2253
2254 /* Free all the Tx ring sk_buffs */
2255 dev_kfree_skb_any(tx_buffer->skb);
2256
2257 /* unmap skb header data */
2258 dma_unmap_single(tx_ring->dev,
2259 dma_unmap_addr(tx_buffer, dma),
2260 dma_unmap_len(tx_buffer, len),
2261 DMA_TO_DEVICE);
2262
2263 /* check for eop_desc to determine the end of the packet */
2264 eop_desc = tx_buffer->next_to_watch;
2265 tx_desc = WX_TX_DESC(tx_ring, i);
2266
2267 /* unmap remaining buffers */
2268 while (tx_desc != eop_desc) {
2269 tx_buffer++;
2270 tx_desc++;
2271 i++;
2272 if (unlikely(i == tx_ring->count)) {
2273 i = 0;
2274 tx_buffer = tx_ring->tx_buffer_info;
2275 tx_desc = WX_TX_DESC(tx_ring, 0);
2276 }
2277
2278 /* unmap any remaining paged data */
2279 if (dma_unmap_len(tx_buffer, len))
2280 dma_unmap_page(tx_ring->dev,
2281 dma_unmap_addr(tx_buffer, dma),
2282 dma_unmap_len(tx_buffer, len),
2283 DMA_TO_DEVICE);
2284 }
2285
2286 /* move us one more past the eop_desc for start of next pkt */
2287 tx_buffer++;
2288 i++;
2289 if (unlikely(i == tx_ring->count)) {
2290 i = 0;
2291 tx_buffer = tx_ring->tx_buffer_info;
2292 }
2293 }
2294
2295 netdev_tx_reset_queue(wx_txring_txq(tx_ring));
2296
2297 /* reset next_to_use and next_to_clean */
2298 tx_ring->next_to_use = 0;
2299 tx_ring->next_to_clean = 0;
2300 }
2301
2302 /**
2303 * wx_clean_all_tx_rings - Free Tx Buffers for all queues
2304 * @wx: board private structure
2305 **/
wx_clean_all_tx_rings(struct wx * wx)2306 void wx_clean_all_tx_rings(struct wx *wx)
2307 {
2308 int i;
2309
2310 for (i = 0; i < wx->num_tx_queues; i++)
2311 wx_clean_tx_ring(wx->tx_ring[i]);
2312 }
2313 EXPORT_SYMBOL(wx_clean_all_tx_rings);
2314
2315 /**
2316 * wx_free_tx_resources - Free Tx Resources per Queue
2317 * @tx_ring: Tx descriptor ring for a specific queue
2318 *
2319 * Free all transmit software resources
2320 **/
wx_free_tx_resources(struct wx_ring * tx_ring)2321 static void wx_free_tx_resources(struct wx_ring *tx_ring)
2322 {
2323 wx_clean_tx_ring(tx_ring);
2324 kvfree(tx_ring->tx_buffer_info);
2325 tx_ring->tx_buffer_info = NULL;
2326
2327 /* if not set, then don't free */
2328 if (!tx_ring->desc)
2329 return;
2330
2331 dma_free_coherent(tx_ring->dev, tx_ring->size,
2332 tx_ring->desc, tx_ring->dma);
2333 tx_ring->desc = NULL;
2334 }
2335
2336 /**
2337 * wx_free_all_tx_resources - Free Tx Resources for All Queues
2338 * @wx: pointer to hardware structure
2339 *
2340 * Free all transmit software resources
2341 **/
wx_free_all_tx_resources(struct wx * wx)2342 static void wx_free_all_tx_resources(struct wx *wx)
2343 {
2344 int i;
2345
2346 for (i = 0; i < wx->num_tx_queues; i++)
2347 wx_free_tx_resources(wx->tx_ring[i]);
2348 }
2349
wx_free_resources(struct wx * wx)2350 void wx_free_resources(struct wx *wx)
2351 {
2352 wx_free_isb_resources(wx);
2353 wx_free_all_rx_resources(wx);
2354 wx_free_all_tx_resources(wx);
2355 }
2356 EXPORT_SYMBOL(wx_free_resources);
2357
wx_alloc_page_pool(struct wx_ring * rx_ring)2358 static int wx_alloc_page_pool(struct wx_ring *rx_ring)
2359 {
2360 int ret = 0;
2361
2362 struct page_pool_params pp_params = {
2363 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
2364 .order = 0,
2365 .pool_size = rx_ring->count,
2366 .nid = dev_to_node(rx_ring->dev),
2367 .dev = rx_ring->dev,
2368 .dma_dir = DMA_FROM_DEVICE,
2369 .offset = 0,
2370 .max_len = PAGE_SIZE,
2371 };
2372
2373 rx_ring->page_pool = page_pool_create(&pp_params);
2374 if (IS_ERR(rx_ring->page_pool)) {
2375 ret = PTR_ERR(rx_ring->page_pool);
2376 rx_ring->page_pool = NULL;
2377 }
2378
2379 return ret;
2380 }
2381
2382 /**
2383 * wx_setup_rx_resources - allocate Rx resources (Descriptors)
2384 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2385 *
2386 * Returns 0 on success, negative on failure
2387 **/
wx_setup_rx_resources(struct wx_ring * rx_ring)2388 static int wx_setup_rx_resources(struct wx_ring *rx_ring)
2389 {
2390 struct device *dev = rx_ring->dev;
2391 int orig_node = dev_to_node(dev);
2392 int numa_node = NUMA_NO_NODE;
2393 int size, ret;
2394
2395 size = sizeof(struct wx_rx_buffer) * rx_ring->count;
2396
2397 if (rx_ring->q_vector)
2398 numa_node = rx_ring->q_vector->numa_node;
2399
2400 rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2401 if (!rx_ring->rx_buffer_info)
2402 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);
2403 if (!rx_ring->rx_buffer_info)
2404 goto err;
2405
2406 /* Round up to nearest 4K */
2407 rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);
2408 rx_ring->size = ALIGN(rx_ring->size, 4096);
2409
2410 set_dev_node(dev, numa_node);
2411 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2412 &rx_ring->dma, GFP_KERNEL);
2413 if (!rx_ring->desc) {
2414 set_dev_node(dev, orig_node);
2415 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2416 &rx_ring->dma, GFP_KERNEL);
2417 }
2418
2419 if (!rx_ring->desc)
2420 goto err;
2421
2422 rx_ring->next_to_clean = 0;
2423 rx_ring->next_to_use = 0;
2424
2425 ret = wx_alloc_page_pool(rx_ring);
2426 if (ret < 0) {
2427 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
2428 goto err_desc;
2429 }
2430
2431 return 0;
2432
2433 err_desc:
2434 dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
2435 err:
2436 kvfree(rx_ring->rx_buffer_info);
2437 rx_ring->rx_buffer_info = NULL;
2438 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2439 return -ENOMEM;
2440 }
2441
2442 /**
2443 * wx_setup_all_rx_resources - allocate all queues Rx resources
2444 * @wx: pointer to hardware structure
2445 *
2446 * If this function returns with an error, then it's possible one or
2447 * more of the rings is populated (while the rest are not). It is the
2448 * callers duty to clean those orphaned rings.
2449 *
2450 * Return 0 on success, negative on failure
2451 **/
wx_setup_all_rx_resources(struct wx * wx)2452 static int wx_setup_all_rx_resources(struct wx *wx)
2453 {
2454 int i, err = 0;
2455
2456 for (i = 0; i < wx->num_rx_queues; i++) {
2457 err = wx_setup_rx_resources(wx->rx_ring[i]);
2458 if (!err)
2459 continue;
2460
2461 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);
2462 goto err_setup_rx;
2463 }
2464
2465 return 0;
2466 err_setup_rx:
2467 /* rewind the index freeing the rings as we go */
2468 while (i--)
2469 wx_free_rx_resources(wx->rx_ring[i]);
2470 return err;
2471 }
2472
2473 /**
2474 * wx_setup_tx_resources - allocate Tx resources (Descriptors)
2475 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2476 *
2477 * Return 0 on success, negative on failure
2478 **/
wx_setup_tx_resources(struct wx_ring * tx_ring)2479 static int wx_setup_tx_resources(struct wx_ring *tx_ring)
2480 {
2481 struct device *dev = tx_ring->dev;
2482 int orig_node = dev_to_node(dev);
2483 int numa_node = NUMA_NO_NODE;
2484 int size;
2485
2486 size = sizeof(struct wx_tx_buffer) * tx_ring->count;
2487
2488 if (tx_ring->q_vector)
2489 numa_node = tx_ring->q_vector->numa_node;
2490
2491 tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2492 if (!tx_ring->tx_buffer_info)
2493 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);
2494 if (!tx_ring->tx_buffer_info)
2495 goto err;
2496
2497 /* round up to nearest 4K */
2498 tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);
2499 tx_ring->size = ALIGN(tx_ring->size, 4096);
2500
2501 set_dev_node(dev, numa_node);
2502 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2503 &tx_ring->dma, GFP_KERNEL);
2504 if (!tx_ring->desc) {
2505 set_dev_node(dev, orig_node);
2506 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2507 &tx_ring->dma, GFP_KERNEL);
2508 }
2509
2510 if (!tx_ring->desc)
2511 goto err;
2512
2513 tx_ring->next_to_use = 0;
2514 tx_ring->next_to_clean = 0;
2515
2516 return 0;
2517
2518 err:
2519 kvfree(tx_ring->tx_buffer_info);
2520 tx_ring->tx_buffer_info = NULL;
2521 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
2522 return -ENOMEM;
2523 }
2524
2525 /**
2526 * wx_setup_all_tx_resources - allocate all queues Tx resources
2527 * @wx: pointer to private structure
2528 *
2529 * If this function returns with an error, then it's possible one or
2530 * more of the rings is populated (while the rest are not). It is the
2531 * callers duty to clean those orphaned rings.
2532 *
2533 * Return 0 on success, negative on failure
2534 **/
wx_setup_all_tx_resources(struct wx * wx)2535 static int wx_setup_all_tx_resources(struct wx *wx)
2536 {
2537 int i, err = 0;
2538
2539 for (i = 0; i < wx->num_tx_queues; i++) {
2540 err = wx_setup_tx_resources(wx->tx_ring[i]);
2541 if (!err)
2542 continue;
2543
2544 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);
2545 goto err_setup_tx;
2546 }
2547
2548 return 0;
2549 err_setup_tx:
2550 /* rewind the index freeing the rings as we go */
2551 while (i--)
2552 wx_free_tx_resources(wx->tx_ring[i]);
2553 return err;
2554 }
2555
wx_setup_resources(struct wx * wx)2556 int wx_setup_resources(struct wx *wx)
2557 {
2558 int err;
2559
2560 /* allocate transmit descriptors */
2561 err = wx_setup_all_tx_resources(wx);
2562 if (err)
2563 return err;
2564
2565 /* allocate receive descriptors */
2566 err = wx_setup_all_rx_resources(wx);
2567 if (err)
2568 goto err_free_tx;
2569
2570 err = wx_setup_isb_resources(wx);
2571 if (err)
2572 goto err_free_rx;
2573
2574 return 0;
2575
2576 err_free_rx:
2577 wx_free_all_rx_resources(wx);
2578 err_free_tx:
2579 wx_free_all_tx_resources(wx);
2580
2581 return err;
2582 }
2583 EXPORT_SYMBOL(wx_setup_resources);
2584
2585 /**
2586 * wx_get_stats64 - Get System Network Statistics
2587 * @netdev: network interface device structure
2588 * @stats: storage space for 64bit statistics
2589 */
wx_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2590 void wx_get_stats64(struct net_device *netdev,
2591 struct rtnl_link_stats64 *stats)
2592 {
2593 struct wx *wx = netdev_priv(netdev);
2594 int i;
2595
2596 rcu_read_lock();
2597 for (i = 0; i < wx->num_rx_queues; i++) {
2598 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);
2599 u64 bytes, packets;
2600 unsigned int start;
2601
2602 if (ring) {
2603 do {
2604 start = u64_stats_fetch_begin(&ring->syncp);
2605 packets = ring->stats.packets;
2606 bytes = ring->stats.bytes;
2607 } while (u64_stats_fetch_retry(&ring->syncp, start));
2608 stats->rx_packets += packets;
2609 stats->rx_bytes += bytes;
2610 }
2611 }
2612
2613 for (i = 0; i < wx->num_tx_queues; i++) {
2614 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);
2615 u64 bytes, packets;
2616 unsigned int start;
2617
2618 if (ring) {
2619 do {
2620 start = u64_stats_fetch_begin(&ring->syncp);
2621 packets = ring->stats.packets;
2622 bytes = ring->stats.bytes;
2623 } while (u64_stats_fetch_retry(&ring->syncp,
2624 start));
2625 stats->tx_packets += packets;
2626 stats->tx_bytes += bytes;
2627 }
2628 }
2629
2630 rcu_read_unlock();
2631 }
2632 EXPORT_SYMBOL(wx_get_stats64);
2633
wx_set_features(struct net_device * netdev,netdev_features_t features)2634 int wx_set_features(struct net_device *netdev, netdev_features_t features)
2635 {
2636 netdev_features_t changed = netdev->features ^ features;
2637 struct wx *wx = netdev_priv(netdev);
2638
2639 if (changed & NETIF_F_RXHASH)
2640 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN,
2641 WX_RDB_RA_CTL_RSS_EN);
2642 else
2643 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN, 0);
2644
2645 netdev->features = features;
2646
2647 if (changed &
2648 (NETIF_F_HW_VLAN_CTAG_RX |
2649 NETIF_F_HW_VLAN_STAG_RX))
2650 wx_set_rx_mode(netdev);
2651
2652 return 0;
2653 }
2654 EXPORT_SYMBOL(wx_set_features);
2655
2656 MODULE_LICENSE("GPL");
2657