1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */
3
4 /* TSN endpoint Ethernet MAC driver
5 *
6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time
7 * communication. It is designed for endpoints within TSN (Time Sensitive
8 * Networking) networks; e.g., for PLCs in the industrial automation case.
9 *
10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used
11 * by the driver.
12 *
13 * More information can be found here:
14 * - www.embedded-experts.at/tsn
15 * - www.engleder-embedded.com
16 */
17
18 #include "tsnep.h"
19 #include "tsnep_hw.h"
20
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/of_net.h>
24 #include <linux/of_mdio.h>
25 #include <linux/interrupt.h>
26 #include <linux/etherdevice.h>
27 #include <linux/phy.h>
28 #include <linux/iopoll.h>
29 #include <linux/bpf.h>
30 #include <linux/bpf_trace.h>
31 #include <net/page_pool/helpers.h>
32 #include <net/xdp_sock_drv.h>
33
34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
38 /* XSK buffer shall store at least Q-in-Q frame */
39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \
40 ETH_FRAME_LEN + ETH_FCS_LEN + \
41 VLAN_HLEN * 2, 4))
42
43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
45 #else
46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0))
47 #endif
48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF))
49
50 #define TSNEP_COALESCE_USECS_DEFAULT 64
51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \
52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1)
53
54 /* mapping type */
55 #define TSNEP_TX_TYPE_MAP BIT(0)
56 #define TSNEP_TX_TYPE_MAP_PAGE BIT(1)
57 #define TSNEP_TX_TYPE_INLINE BIT(2)
58 /* buffer type */
59 #define TSNEP_TX_TYPE_SKB BIT(8)
60 #define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP)
61 #define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE)
62 #define TSNEP_TX_TYPE_SKB_FRAG BIT(9)
63 #define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE)
64 #define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE)
65 #define TSNEP_TX_TYPE_XDP_TX BIT(10)
66 #define TSNEP_TX_TYPE_XDP_NDO BIT(11)
67 #define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE)
68 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
69 #define TSNEP_TX_TYPE_XSK BIT(12)
70 #define TSNEP_TX_TYPE_TSTAMP BIT(13)
71 #define TSNEP_TX_TYPE_SKB_TSTAMP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_TSTAMP)
72
73 #define TSNEP_XDP_TX BIT(0)
74 #define TSNEP_XDP_REDIRECT BIT(1)
75
tsnep_enable_irq(struct tsnep_adapter * adapter,u32 mask)76 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)
77 {
78 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
79 }
80
tsnep_disable_irq(struct tsnep_adapter * adapter,u32 mask)81 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask)
82 {
83 mask |= ECM_INT_DISABLE;
84 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
85 }
86
tsnep_irq(int irq,void * arg)87 static irqreturn_t tsnep_irq(int irq, void *arg)
88 {
89 struct tsnep_adapter *adapter = arg;
90 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE);
91
92 /* acknowledge interrupt */
93 if (active != 0)
94 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);
95
96 /* handle link interrupt */
97 if ((active & ECM_INT_LINK) != 0)
98 phy_mac_interrupt(adapter->netdev->phydev);
99
100 /* handle TX/RX queue 0 interrupt */
101 if ((active & adapter->queue[0].irq_mask) != 0) {
102 if (napi_schedule_prep(&adapter->queue[0].napi)) {
103 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);
104 /* schedule after masking to avoid races */
105 __napi_schedule(&adapter->queue[0].napi);
106 }
107 }
108
109 return IRQ_HANDLED;
110 }
111
tsnep_irq_txrx(int irq,void * arg)112 static irqreturn_t tsnep_irq_txrx(int irq, void *arg)
113 {
114 struct tsnep_queue *queue = arg;
115
116 /* handle TX/RX queue interrupt */
117 if (napi_schedule_prep(&queue->napi)) {
118 tsnep_disable_irq(queue->adapter, queue->irq_mask);
119 /* schedule after masking to avoid races */
120 __napi_schedule(&queue->napi);
121 }
122
123 return IRQ_HANDLED;
124 }
125
tsnep_set_irq_coalesce(struct tsnep_queue * queue,u32 usecs)126 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs)
127 {
128 if (usecs > TSNEP_COALESCE_USECS_MAX)
129 return -ERANGE;
130
131 usecs /= ECM_INT_DELAY_BASE_US;
132 usecs <<= ECM_INT_DELAY_SHIFT;
133 usecs &= ECM_INT_DELAY_MASK;
134
135 queue->irq_delay &= ~ECM_INT_DELAY_MASK;
136 queue->irq_delay |= usecs;
137 iowrite8(queue->irq_delay, queue->irq_delay_addr);
138
139 return 0;
140 }
141
tsnep_get_irq_coalesce(struct tsnep_queue * queue)142 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue)
143 {
144 u32 usecs;
145
146 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK);
147 usecs >>= ECM_INT_DELAY_SHIFT;
148 usecs *= ECM_INT_DELAY_BASE_US;
149
150 return usecs;
151 }
152
tsnep_mdiobus_read(struct mii_bus * bus,int addr,int regnum)153 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)
154 {
155 struct tsnep_adapter *adapter = bus->priv;
156 u32 md;
157 int retval;
158
159 md = ECM_MD_READ;
160 if (!adapter->suppress_preamble)
161 md |= ECM_MD_PREAMBLE;
162 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
163 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
164 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
165 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
166 !(md & ECM_MD_BUSY), 16, 1000);
167 if (retval != 0)
168 return retval;
169
170 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT;
171 }
172
tsnep_mdiobus_write(struct mii_bus * bus,int addr,int regnum,u16 val)173 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,
174 u16 val)
175 {
176 struct tsnep_adapter *adapter = bus->priv;
177 u32 md;
178 int retval;
179
180 md = ECM_MD_WRITE;
181 if (!adapter->suppress_preamble)
182 md |= ECM_MD_PREAMBLE;
183 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
184 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
185 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK;
186 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
187 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
188 !(md & ECM_MD_BUSY), 16, 1000);
189 if (retval != 0)
190 return retval;
191
192 return 0;
193 }
194
tsnep_set_link_mode(struct tsnep_adapter * adapter)195 static void tsnep_set_link_mode(struct tsnep_adapter *adapter)
196 {
197 u32 mode;
198
199 switch (adapter->phydev->speed) {
200 case SPEED_100:
201 mode = ECM_LINK_MODE_100;
202 break;
203 case SPEED_1000:
204 mode = ECM_LINK_MODE_1000;
205 break;
206 default:
207 mode = ECM_LINK_MODE_OFF;
208 break;
209 }
210 iowrite32(mode, adapter->addr + ECM_STATUS);
211 }
212
tsnep_phy_link_status_change(struct net_device * netdev)213 static void tsnep_phy_link_status_change(struct net_device *netdev)
214 {
215 struct tsnep_adapter *adapter = netdev_priv(netdev);
216 struct phy_device *phydev = netdev->phydev;
217
218 if (phydev->link)
219 tsnep_set_link_mode(adapter);
220
221 phy_print_status(netdev->phydev);
222 }
223
tsnep_phy_loopback(struct tsnep_adapter * adapter,bool enable)224 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)
225 {
226 int retval;
227
228 retval = phy_loopback(adapter->phydev, enable);
229
230 /* PHY link state change is not signaled if loopback is enabled, it
231 * would delay a working loopback anyway, let's ensure that loopback
232 * is working immediately by setting link mode directly
233 */
234 if (!retval && enable) {
235 netif_carrier_on(adapter->netdev);
236 tsnep_set_link_mode(adapter);
237 }
238
239 return retval;
240 }
241
tsnep_phy_open(struct tsnep_adapter * adapter)242 static int tsnep_phy_open(struct tsnep_adapter *adapter)
243 {
244 struct phy_device *phydev;
245 struct ethtool_keee ethtool_keee;
246 int retval;
247
248 retval = phy_connect_direct(adapter->netdev, adapter->phydev,
249 tsnep_phy_link_status_change,
250 adapter->phy_mode);
251 if (retval)
252 return retval;
253 phydev = adapter->netdev->phydev;
254
255 /* MAC supports only 100Mbps|1000Mbps full duplex
256 * SPE (Single Pair Ethernet) is also an option but not implemented yet
257 */
258 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
259 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT);
260 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);
261 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
262
263 /* disable EEE autoneg, EEE not supported by TSNEP */
264 memset(ðtool_keee, 0, sizeof(ethtool_keee));
265 phy_ethtool_set_eee(adapter->phydev, ðtool_keee);
266
267 adapter->phydev->irq = PHY_MAC_INTERRUPT;
268 phy_start(adapter->phydev);
269
270 return 0;
271 }
272
tsnep_phy_close(struct tsnep_adapter * adapter)273 static void tsnep_phy_close(struct tsnep_adapter *adapter)
274 {
275 phy_stop(adapter->netdev->phydev);
276 phy_disconnect(adapter->netdev->phydev);
277 }
278
tsnep_tx_ring_cleanup(struct tsnep_tx * tx)279 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
280 {
281 struct device *dmadev = tx->adapter->dmadev;
282 int i;
283
284 memset(tx->entry, 0, sizeof(tx->entry));
285
286 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
287 if (tx->page[i]) {
288 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i],
289 tx->page_dma[i]);
290 tx->page[i] = NULL;
291 tx->page_dma[i] = 0;
292 }
293 }
294 }
295
tsnep_tx_ring_create(struct tsnep_tx * tx)296 static int tsnep_tx_ring_create(struct tsnep_tx *tx)
297 {
298 struct device *dmadev = tx->adapter->dmadev;
299 struct tsnep_tx_entry *entry;
300 struct tsnep_tx_entry *next_entry;
301 int i, j;
302 int retval;
303
304 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
305 tx->page[i] =
306 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i],
307 GFP_KERNEL);
308 if (!tx->page[i]) {
309 retval = -ENOMEM;
310 goto alloc_failed;
311 }
312 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
313 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
314 entry->desc_wb = (struct tsnep_tx_desc_wb *)
315 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j);
316 entry->desc = (struct tsnep_tx_desc *)
317 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
318 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;
319 entry->owner_user_flag = false;
320 }
321 }
322 for (i = 0; i < TSNEP_RING_SIZE; i++) {
323 entry = &tx->entry[i];
324 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];
325 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
326 }
327
328 return 0;
329
330 alloc_failed:
331 tsnep_tx_ring_cleanup(tx);
332 return retval;
333 }
334
tsnep_tx_init(struct tsnep_tx * tx)335 static void tsnep_tx_init(struct tsnep_tx *tx)
336 {
337 dma_addr_t dma;
338
339 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
340 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
341 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
342 tx->write = 0;
343 tx->read = 0;
344 tx->owner_counter = 1;
345 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
346 }
347
tsnep_tx_enable(struct tsnep_tx * tx)348 static void tsnep_tx_enable(struct tsnep_tx *tx)
349 {
350 struct netdev_queue *nq;
351
352 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
353
354 __netif_tx_lock_bh(nq);
355 netif_tx_wake_queue(nq);
356 __netif_tx_unlock_bh(nq);
357 }
358
tsnep_tx_disable(struct tsnep_tx * tx,struct napi_struct * napi)359 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
360 {
361 struct netdev_queue *nq;
362 u32 val;
363
364 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
365
366 __netif_tx_lock_bh(nq);
367 netif_tx_stop_queue(nq);
368 __netif_tx_unlock_bh(nq);
369
370 /* wait until TX is done in hardware */
371 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
372 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
373 1000000);
374
375 /* wait until TX is also done in software */
376 while (READ_ONCE(tx->read) != tx->write) {
377 napi_schedule(napi);
378 napi_synchronize(napi);
379 }
380 }
381
tsnep_tx_activate(struct tsnep_tx * tx,int index,int length,bool last)382 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
383 bool last)
384 {
385 struct tsnep_tx_entry *entry = &tx->entry[index];
386
387 entry->properties = 0;
388 /* xdpf and zc are union with skb */
389 if (entry->skb) {
390 entry->properties = length & TSNEP_DESC_LENGTH_MASK;
391 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
392 if ((entry->type & TSNEP_TX_TYPE_SKB_TSTAMP) == TSNEP_TX_TYPE_SKB_TSTAMP)
393 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;
394
395 /* toggle user flag to prevent false acknowledge
396 *
397 * Only the first fragment is acknowledged. For all other
398 * fragments no acknowledge is done and the last written owner
399 * counter stays in the writeback descriptor. Therefore, it is
400 * possible that the last written owner counter is identical to
401 * the new incremented owner counter and a false acknowledge is
402 * detected before the real acknowledge has been done by
403 * hardware.
404 *
405 * The user flag is used to prevent this situation. The user
406 * flag is copied to the writeback descriptor by the hardware
407 * and is used as additional acknowledge data. By toggeling the
408 * user flag only for the first fragment (which is
409 * acknowledged), it is guaranteed that the last acknowledge
410 * done for this descriptor has used a different user flag and
411 * cannot be detected as false acknowledge.
412 */
413 entry->owner_user_flag = !entry->owner_user_flag;
414 }
415 if (last)
416 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG;
417 if (index == tx->increment_owner_counter) {
418 tx->owner_counter++;
419 if (tx->owner_counter == 4)
420 tx->owner_counter = 1;
421 tx->increment_owner_counter--;
422 if (tx->increment_owner_counter < 0)
423 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
424 }
425 entry->properties |=
426 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
427 TSNEP_DESC_OWNER_COUNTER_MASK;
428 if (entry->owner_user_flag)
429 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG;
430 entry->desc->more_properties =
431 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK);
432 if (entry->type & TSNEP_TX_TYPE_INLINE)
433 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG;
434
435 /* descriptor properties shall be written last, because valid data is
436 * signaled there
437 */
438 dma_wmb();
439
440 entry->desc->properties = __cpu_to_le32(entry->properties);
441 }
442
tsnep_tx_desc_available(struct tsnep_tx * tx)443 static int tsnep_tx_desc_available(struct tsnep_tx *tx)
444 {
445 if (tx->read <= tx->write)
446 return TSNEP_RING_SIZE - tx->write + tx->read - 1;
447 else
448 return tx->read - tx->write - 1;
449 }
450
tsnep_tx_map_frag(skb_frag_t * frag,struct tsnep_tx_entry * entry,struct device * dmadev,dma_addr_t * dma)451 static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry,
452 struct device *dmadev, dma_addr_t *dma)
453 {
454 unsigned int len;
455 int mapped;
456
457 len = skb_frag_size(frag);
458 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
459 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE);
460 if (dma_mapping_error(dmadev, *dma))
461 return -ENOMEM;
462 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE;
463 mapped = 1;
464 } else {
465 void *fragdata = skb_frag_address_safe(frag);
466
467 if (likely(fragdata)) {
468 memcpy(&entry->desc->tx, fragdata, len);
469 } else {
470 struct page *page = skb_frag_page(frag);
471
472 fragdata = kmap_local_page(page);
473 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag),
474 len);
475 kunmap_local(fragdata);
476 }
477 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE;
478 mapped = 0;
479 }
480
481 return mapped;
482 }
483
tsnep_tx_map(struct sk_buff * skb,struct tsnep_tx * tx,int count,bool do_tstamp)484 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count,
485 bool do_tstamp)
486 {
487 struct device *dmadev = tx->adapter->dmadev;
488 struct tsnep_tx_entry *entry;
489 unsigned int len;
490 int map_len = 0;
491 dma_addr_t dma;
492 int i, mapped;
493
494 for (i = 0; i < count; i++) {
495 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
496
497 if (!i) {
498 len = skb_headlen(skb);
499 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
500 dma = dma_map_single(dmadev, skb->data, len,
501 DMA_TO_DEVICE);
502 if (dma_mapping_error(dmadev, dma))
503 return -ENOMEM;
504 entry->type = TSNEP_TX_TYPE_SKB_MAP;
505 mapped = 1;
506 } else {
507 memcpy(&entry->desc->tx, skb->data, len);
508 entry->type = TSNEP_TX_TYPE_SKB_INLINE;
509 mapped = 0;
510 }
511
512 if (do_tstamp)
513 entry->type |= TSNEP_TX_TYPE_TSTAMP;
514 } else {
515 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
516
517 len = skb_frag_size(frag);
518 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma);
519 if (mapped < 0)
520 return mapped;
521 }
522
523 entry->len = len;
524 if (likely(mapped)) {
525 dma_unmap_addr_set(entry, dma, dma);
526 entry->desc->tx = __cpu_to_le64(dma);
527 }
528
529 map_len += len;
530 }
531
532 return map_len;
533 }
534
tsnep_tx_unmap(struct tsnep_tx * tx,int index,int count)535 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
536 {
537 struct device *dmadev = tx->adapter->dmadev;
538 struct tsnep_tx_entry *entry;
539 int map_len = 0;
540 int i;
541
542 for (i = 0; i < count; i++) {
543 entry = &tx->entry[(index + i) & TSNEP_RING_MASK];
544
545 if (entry->len) {
546 if (entry->type & TSNEP_TX_TYPE_MAP)
547 dma_unmap_single(dmadev,
548 dma_unmap_addr(entry, dma),
549 dma_unmap_len(entry, len),
550 DMA_TO_DEVICE);
551 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE)
552 dma_unmap_page(dmadev,
553 dma_unmap_addr(entry, dma),
554 dma_unmap_len(entry, len),
555 DMA_TO_DEVICE);
556 map_len += entry->len;
557 entry->len = 0;
558 }
559 }
560
561 return map_len;
562 }
563
tsnep_xmit_frame_ring(struct sk_buff * skb,struct tsnep_tx * tx)564 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
565 struct tsnep_tx *tx)
566 {
567 struct tsnep_tx_entry *entry;
568 bool do_tstamp = false;
569 int count = 1;
570 int length;
571 int retval;
572 int i;
573
574 if (skb_shinfo(skb)->nr_frags > 0)
575 count += skb_shinfo(skb)->nr_frags;
576
577 if (tsnep_tx_desc_available(tx) < count) {
578 /* ring full, shall not happen because queue is stopped if full
579 * below
580 */
581 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
582
583 return NETDEV_TX_BUSY;
584 }
585
586 entry = &tx->entry[tx->write];
587 entry->skb = skb;
588
589 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
590 tx->adapter->hwtstamp_config.tx_type == HWTSTAMP_TX_ON) {
591 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
592 do_tstamp = true;
593 }
594
595 retval = tsnep_tx_map(skb, tx, count, do_tstamp);
596 if (retval < 0) {
597 tsnep_tx_unmap(tx, tx->write, count);
598 dev_kfree_skb_any(entry->skb);
599 entry->skb = NULL;
600
601 tx->dropped++;
602
603 return NETDEV_TX_OK;
604 }
605 length = retval;
606
607 for (i = 0; i < count; i++)
608 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
609 i == count - 1);
610 tx->write = (tx->write + count) & TSNEP_RING_MASK;
611
612 skb_tx_timestamp(skb);
613
614 /* descriptor properties shall be valid before hardware is notified */
615 dma_wmb();
616
617 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
618
619 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) {
620 /* ring can get full with next frame */
621 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
622 }
623
624 return NETDEV_TX_OK;
625 }
626
tsnep_xdp_tx_map(struct xdp_frame * xdpf,struct tsnep_tx * tx,struct skb_shared_info * shinfo,int count,u32 type)627 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,
628 struct skb_shared_info *shinfo, int count, u32 type)
629 {
630 struct device *dmadev = tx->adapter->dmadev;
631 struct tsnep_tx_entry *entry;
632 struct page *page;
633 skb_frag_t *frag;
634 unsigned int len;
635 int map_len = 0;
636 dma_addr_t dma;
637 void *data;
638 int i;
639
640 frag = NULL;
641 len = xdpf->len;
642 for (i = 0; i < count; i++) {
643 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
644 if (type & TSNEP_TX_TYPE_XDP_NDO) {
645 data = unlikely(frag) ? skb_frag_address(frag) :
646 xdpf->data;
647 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE);
648 if (dma_mapping_error(dmadev, dma))
649 return -ENOMEM;
650
651 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE;
652 } else {
653 page = unlikely(frag) ? skb_frag_page(frag) :
654 virt_to_page(xdpf->data);
655 dma = page_pool_get_dma_addr(page);
656 if (unlikely(frag))
657 dma += skb_frag_off(frag);
658 else
659 dma += sizeof(*xdpf) + xdpf->headroom;
660 dma_sync_single_for_device(dmadev, dma, len,
661 DMA_BIDIRECTIONAL);
662
663 entry->type = TSNEP_TX_TYPE_XDP_TX;
664 }
665
666 entry->len = len;
667 dma_unmap_addr_set(entry, dma, dma);
668
669 entry->desc->tx = __cpu_to_le64(dma);
670
671 map_len += len;
672
673 if (i + 1 < count) {
674 frag = &shinfo->frags[i];
675 len = skb_frag_size(frag);
676 }
677 }
678
679 return map_len;
680 }
681
682 /* This function requires __netif_tx_lock is held by the caller. */
tsnep_xdp_xmit_frame_ring(struct xdp_frame * xdpf,struct tsnep_tx * tx,u32 type)683 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,
684 struct tsnep_tx *tx, u32 type)
685 {
686 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf);
687 struct tsnep_tx_entry *entry;
688 int count, length, retval, i;
689
690 count = 1;
691 if (unlikely(xdp_frame_has_frags(xdpf)))
692 count += shinfo->nr_frags;
693
694 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
695 * will be available for normal TX path and queue is stopped there if
696 * necessary
697 */
698 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count))
699 return false;
700
701 entry = &tx->entry[tx->write];
702 entry->xdpf = xdpf;
703
704 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type);
705 if (retval < 0) {
706 tsnep_tx_unmap(tx, tx->write, count);
707 entry->xdpf = NULL;
708
709 tx->dropped++;
710
711 return false;
712 }
713 length = retval;
714
715 for (i = 0; i < count; i++)
716 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
717 i == count - 1);
718 tx->write = (tx->write + count) & TSNEP_RING_MASK;
719
720 /* descriptor properties shall be valid before hardware is notified */
721 dma_wmb();
722
723 return true;
724 }
725
tsnep_xdp_xmit_flush(struct tsnep_tx * tx)726 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx)
727 {
728 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
729 }
730
tsnep_xdp_xmit_back(struct tsnep_adapter * adapter,struct xdp_buff * xdp,struct netdev_queue * tx_nq,struct tsnep_tx * tx,bool zc)731 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
732 struct xdp_buff *xdp,
733 struct netdev_queue *tx_nq, struct tsnep_tx *tx,
734 bool zc)
735 {
736 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
737 bool xmit;
738 u32 type;
739
740 if (unlikely(!xdpf))
741 return false;
742
743 /* no page pool for zero copy */
744 if (zc)
745 type = TSNEP_TX_TYPE_XDP_NDO;
746 else
747 type = TSNEP_TX_TYPE_XDP_TX;
748
749 __netif_tx_lock(tx_nq, smp_processor_id());
750
751 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type);
752
753 /* Avoid transmit queue timeout since we share it with the slow path */
754 if (xmit)
755 txq_trans_cond_update(tx_nq);
756
757 __netif_tx_unlock(tx_nq);
758
759 return xmit;
760 }
761
tsnep_xdp_tx_map_zc(struct xdp_desc * xdpd,struct tsnep_tx * tx)762 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
763 {
764 struct tsnep_tx_entry *entry;
765 dma_addr_t dma;
766
767 entry = &tx->entry[tx->write];
768 entry->zc = true;
769
770 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);
771 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);
772
773 entry->type = TSNEP_TX_TYPE_XSK;
774 entry->len = xdpd->len;
775
776 entry->desc->tx = __cpu_to_le64(dma);
777
778 return xdpd->len;
779 }
780
tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc * xdpd,struct tsnep_tx * tx)781 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
782 struct tsnep_tx *tx)
783 {
784 int length;
785
786 length = tsnep_xdp_tx_map_zc(xdpd, tx);
787
788 tsnep_tx_activate(tx, tx->write, length, true);
789 tx->write = (tx->write + 1) & TSNEP_RING_MASK;
790 }
791
tsnep_xdp_xmit_zc(struct tsnep_tx * tx)792 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
793 {
794 int desc_available = tsnep_tx_desc_available(tx);
795 struct xdp_desc *descs = tx->xsk_pool->tx_descs;
796 int batch, i;
797
798 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
799 * will be available for normal TX path and queue is stopped there if
800 * necessary
801 */
802 if (desc_available <= (MAX_SKB_FRAGS + 1))
803 return;
804 desc_available -= MAX_SKB_FRAGS + 1;
805
806 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);
807 for (i = 0; i < batch; i++)
808 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);
809
810 if (batch) {
811 /* descriptor properties shall be valid before hardware is
812 * notified
813 */
814 dma_wmb();
815
816 tsnep_xdp_xmit_flush(tx);
817 }
818 }
819
tsnep_tx_poll(struct tsnep_tx * tx,int napi_budget)820 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
821 {
822 struct tsnep_tx_entry *entry;
823 struct netdev_queue *nq;
824 int xsk_frames = 0;
825 int budget = 128;
826 int length;
827 int count;
828
829 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
830 __netif_tx_lock(nq, smp_processor_id());
831
832 do {
833 if (tx->read == tx->write)
834 break;
835
836 entry = &tx->entry[tx->read];
837 if ((__le32_to_cpu(entry->desc_wb->properties) &
838 TSNEP_TX_DESC_OWNER_MASK) !=
839 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
840 break;
841
842 /* descriptor properties shall be read first, because valid data
843 * is signaled there
844 */
845 dma_rmb();
846
847 count = 1;
848 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
849 skb_shinfo(entry->skb)->nr_frags > 0)
850 count += skb_shinfo(entry->skb)->nr_frags;
851 else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
852 xdp_frame_has_frags(entry->xdpf))
853 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;
854
855 length = tsnep_tx_unmap(tx, tx->read, count);
856
857 if (((entry->type & TSNEP_TX_TYPE_SKB_TSTAMP) == TSNEP_TX_TYPE_SKB_TSTAMP) &&
858 (__le32_to_cpu(entry->desc_wb->properties) &
859 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) {
860 struct skb_shared_hwtstamps hwtstamps;
861 u64 timestamp;
862
863 if (skb_shinfo(entry->skb)->tx_flags &
864 SKBTX_HW_TSTAMP_USE_CYCLES)
865 timestamp =
866 __le64_to_cpu(entry->desc_wb->counter);
867 else
868 timestamp =
869 __le64_to_cpu(entry->desc_wb->timestamp);
870
871 memset(&hwtstamps, 0, sizeof(hwtstamps));
872 hwtstamps.hwtstamp = ns_to_ktime(timestamp);
873
874 skb_tstamp_tx(entry->skb, &hwtstamps);
875 }
876
877 if (entry->type & TSNEP_TX_TYPE_SKB)
878 napi_consume_skb(entry->skb, napi_budget);
879 else if (entry->type & TSNEP_TX_TYPE_XDP)
880 xdp_return_frame_rx_napi(entry->xdpf);
881 else
882 xsk_frames++;
883 /* xdpf and zc are union with skb */
884 entry->skb = NULL;
885
886 tx->read = (tx->read + count) & TSNEP_RING_MASK;
887
888 tx->packets++;
889 tx->bytes += length + ETH_FCS_LEN;
890
891 budget--;
892 } while (likely(budget));
893
894 if (tx->xsk_pool) {
895 if (xsk_frames)
896 xsk_tx_completed(tx->xsk_pool, xsk_frames);
897 if (xsk_uses_need_wakeup(tx->xsk_pool))
898 xsk_set_tx_need_wakeup(tx->xsk_pool);
899 tsnep_xdp_xmit_zc(tx);
900 }
901
902 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
903 netif_tx_queue_stopped(nq)) {
904 netif_tx_wake_queue(nq);
905 }
906
907 __netif_tx_unlock(nq);
908
909 return budget != 0;
910 }
911
tsnep_tx_pending(struct tsnep_tx * tx)912 static bool tsnep_tx_pending(struct tsnep_tx *tx)
913 {
914 struct tsnep_tx_entry *entry;
915 struct netdev_queue *nq;
916 bool pending = false;
917
918 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
919 __netif_tx_lock(nq, smp_processor_id());
920
921 if (tx->read != tx->write) {
922 entry = &tx->entry[tx->read];
923 if ((__le32_to_cpu(entry->desc_wb->properties) &
924 TSNEP_TX_DESC_OWNER_MASK) ==
925 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
926 pending = true;
927 }
928
929 __netif_tx_unlock(nq);
930
931 return pending;
932 }
933
tsnep_tx_open(struct tsnep_tx * tx)934 static int tsnep_tx_open(struct tsnep_tx *tx)
935 {
936 int retval;
937
938 retval = tsnep_tx_ring_create(tx);
939 if (retval)
940 return retval;
941
942 tsnep_tx_init(tx);
943
944 return 0;
945 }
946
tsnep_tx_close(struct tsnep_tx * tx)947 static void tsnep_tx_close(struct tsnep_tx *tx)
948 {
949 tsnep_tx_ring_cleanup(tx);
950 }
951
tsnep_rx_ring_cleanup(struct tsnep_rx * rx)952 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
953 {
954 struct device *dmadev = rx->adapter->dmadev;
955 struct tsnep_rx_entry *entry;
956 int i;
957
958 for (i = 0; i < TSNEP_RING_SIZE; i++) {
959 entry = &rx->entry[i];
960 if (!rx->xsk_pool && entry->page)
961 page_pool_put_full_page(rx->page_pool, entry->page,
962 false);
963 if (rx->xsk_pool && entry->xdp)
964 xsk_buff_free(entry->xdp);
965 /* xdp is union with page */
966 entry->page = NULL;
967 }
968
969 if (rx->page_pool)
970 page_pool_destroy(rx->page_pool);
971
972 memset(rx->entry, 0, sizeof(rx->entry));
973
974 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
975 if (rx->page[i]) {
976 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],
977 rx->page_dma[i]);
978 rx->page[i] = NULL;
979 rx->page_dma[i] = 0;
980 }
981 }
982 }
983
tsnep_rx_ring_create(struct tsnep_rx * rx)984 static int tsnep_rx_ring_create(struct tsnep_rx *rx)
985 {
986 struct device *dmadev = rx->adapter->dmadev;
987 struct tsnep_rx_entry *entry;
988 struct page_pool_params pp_params = { 0 };
989 struct tsnep_rx_entry *next_entry;
990 int i, j;
991 int retval;
992
993 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
994 rx->page[i] =
995 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],
996 GFP_KERNEL);
997 if (!rx->page[i]) {
998 retval = -ENOMEM;
999 goto failed;
1000 }
1001 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
1002 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
1003 entry->desc_wb = (struct tsnep_rx_desc_wb *)
1004 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);
1005 entry->desc = (struct tsnep_rx_desc *)
1006 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
1007 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;
1008 }
1009 }
1010
1011 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
1012 pp_params.order = 0;
1013 pp_params.pool_size = TSNEP_RING_SIZE;
1014 pp_params.nid = dev_to_node(dmadev);
1015 pp_params.dev = dmadev;
1016 pp_params.dma_dir = DMA_BIDIRECTIONAL;
1017 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;
1018 pp_params.offset = TSNEP_RX_OFFSET;
1019 rx->page_pool = page_pool_create(&pp_params);
1020 if (IS_ERR(rx->page_pool)) {
1021 retval = PTR_ERR(rx->page_pool);
1022 rx->page_pool = NULL;
1023 goto failed;
1024 }
1025
1026 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1027 entry = &rx->entry[i];
1028 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];
1029 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
1030 }
1031
1032 return 0;
1033
1034 failed:
1035 tsnep_rx_ring_cleanup(rx);
1036 return retval;
1037 }
1038
tsnep_rx_init(struct tsnep_rx * rx)1039 static void tsnep_rx_init(struct tsnep_rx *rx)
1040 {
1041 dma_addr_t dma;
1042
1043 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
1044 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
1045 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
1046 rx->write = 0;
1047 rx->read = 0;
1048 rx->owner_counter = 1;
1049 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1050 }
1051
tsnep_rx_enable(struct tsnep_rx * rx)1052 static void tsnep_rx_enable(struct tsnep_rx *rx)
1053 {
1054 /* descriptor properties shall be valid before hardware is notified */
1055 dma_wmb();
1056
1057 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
1058 }
1059
tsnep_rx_disable(struct tsnep_rx * rx)1060 static void tsnep_rx_disable(struct tsnep_rx *rx)
1061 {
1062 u32 val;
1063
1064 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
1065 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
1066 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
1067 1000000);
1068 }
1069
tsnep_rx_desc_available(struct tsnep_rx * rx)1070 static int tsnep_rx_desc_available(struct tsnep_rx *rx)
1071 {
1072 if (rx->read <= rx->write)
1073 return TSNEP_RING_SIZE - rx->write + rx->read - 1;
1074 else
1075 return rx->read - rx->write - 1;
1076 }
1077
tsnep_rx_free_page_buffer(struct tsnep_rx * rx)1078 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)
1079 {
1080 struct page **page;
1081
1082 /* last entry of page_buffer is always zero, because ring cannot be
1083 * filled completely
1084 */
1085 page = rx->page_buffer;
1086 while (*page) {
1087 page_pool_put_full_page(rx->page_pool, *page, false);
1088 *page = NULL;
1089 page++;
1090 }
1091 }
1092
tsnep_rx_alloc_page_buffer(struct tsnep_rx * rx)1093 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)
1094 {
1095 int i;
1096
1097 /* alloc for all ring entries except the last one, because ring cannot
1098 * be filled completely
1099 */
1100 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {
1101 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool);
1102 if (!rx->page_buffer[i]) {
1103 tsnep_rx_free_page_buffer(rx);
1104
1105 return -ENOMEM;
1106 }
1107 }
1108
1109 return 0;
1110 }
1111
tsnep_rx_set_page(struct tsnep_rx * rx,struct tsnep_rx_entry * entry,struct page * page)1112 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1113 struct page *page)
1114 {
1115 entry->page = page;
1116 entry->len = TSNEP_MAX_RX_BUF_SIZE;
1117 entry->dma = page_pool_get_dma_addr(entry->page);
1118 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET);
1119 }
1120
tsnep_rx_alloc_buffer(struct tsnep_rx * rx,int index)1121 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)
1122 {
1123 struct tsnep_rx_entry *entry = &rx->entry[index];
1124 struct page *page;
1125
1126 page = page_pool_dev_alloc_pages(rx->page_pool);
1127 if (unlikely(!page))
1128 return -ENOMEM;
1129 tsnep_rx_set_page(rx, entry, page);
1130
1131 return 0;
1132 }
1133
tsnep_rx_reuse_buffer(struct tsnep_rx * rx,int index)1134 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)
1135 {
1136 struct tsnep_rx_entry *entry = &rx->entry[index];
1137 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1138
1139 tsnep_rx_set_page(rx, entry, read->page);
1140 read->page = NULL;
1141 }
1142
tsnep_rx_activate(struct tsnep_rx * rx,int index)1143 static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
1144 {
1145 struct tsnep_rx_entry *entry = &rx->entry[index];
1146
1147 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */
1148 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
1149 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
1150 if (index == rx->increment_owner_counter) {
1151 rx->owner_counter++;
1152 if (rx->owner_counter == 4)
1153 rx->owner_counter = 1;
1154 rx->increment_owner_counter--;
1155 if (rx->increment_owner_counter < 0)
1156 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1157 }
1158 entry->properties |=
1159 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
1160 TSNEP_DESC_OWNER_COUNTER_MASK;
1161
1162 /* descriptor properties shall be written last, because valid data is
1163 * signaled there
1164 */
1165 dma_wmb();
1166
1167 entry->desc->properties = __cpu_to_le32(entry->properties);
1168 }
1169
tsnep_rx_alloc(struct tsnep_rx * rx,int count,bool reuse)1170 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)
1171 {
1172 bool alloc_failed = false;
1173 int i, index;
1174
1175 for (i = 0; i < count && !alloc_failed; i++) {
1176 index = (rx->write + i) & TSNEP_RING_MASK;
1177
1178 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {
1179 rx->alloc_failed++;
1180 alloc_failed = true;
1181
1182 /* reuse only if no other allocation was successful */
1183 if (i == 0 && reuse)
1184 tsnep_rx_reuse_buffer(rx, index);
1185 else
1186 break;
1187 }
1188
1189 tsnep_rx_activate(rx, index);
1190 }
1191
1192 if (i)
1193 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1194
1195 return i;
1196 }
1197
tsnep_rx_refill(struct tsnep_rx * rx,int count,bool reuse)1198 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
1199 {
1200 int desc_refilled;
1201
1202 desc_refilled = tsnep_rx_alloc(rx, count, reuse);
1203 if (desc_refilled)
1204 tsnep_rx_enable(rx);
1205
1206 return desc_refilled;
1207 }
1208
tsnep_rx_set_xdp(struct tsnep_rx * rx,struct tsnep_rx_entry * entry,struct xdp_buff * xdp)1209 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1210 struct xdp_buff *xdp)
1211 {
1212 entry->xdp = xdp;
1213 entry->len = TSNEP_XSK_RX_BUF_SIZE;
1214 entry->dma = xsk_buff_xdp_get_dma(entry->xdp);
1215 entry->desc->rx = __cpu_to_le64(entry->dma);
1216 }
1217
tsnep_rx_reuse_buffer_zc(struct tsnep_rx * rx,int index)1218 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)
1219 {
1220 struct tsnep_rx_entry *entry = &rx->entry[index];
1221 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1222
1223 tsnep_rx_set_xdp(rx, entry, read->xdp);
1224 read->xdp = NULL;
1225 }
1226
tsnep_rx_alloc_zc(struct tsnep_rx * rx,int count,bool reuse)1227 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)
1228 {
1229 u32 allocated;
1230 int i;
1231
1232 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count);
1233 for (i = 0; i < allocated; i++) {
1234 int index = (rx->write + i) & TSNEP_RING_MASK;
1235 struct tsnep_rx_entry *entry = &rx->entry[index];
1236
1237 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]);
1238 tsnep_rx_activate(rx, index);
1239 }
1240 if (i == 0) {
1241 rx->alloc_failed++;
1242
1243 if (reuse) {
1244 tsnep_rx_reuse_buffer_zc(rx, rx->write);
1245 tsnep_rx_activate(rx, rx->write);
1246 }
1247 }
1248
1249 if (i)
1250 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1251
1252 return i;
1253 }
1254
tsnep_rx_free_zc(struct tsnep_rx * rx)1255 static void tsnep_rx_free_zc(struct tsnep_rx *rx)
1256 {
1257 int i;
1258
1259 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1260 struct tsnep_rx_entry *entry = &rx->entry[i];
1261
1262 if (entry->xdp)
1263 xsk_buff_free(entry->xdp);
1264 entry->xdp = NULL;
1265 }
1266 }
1267
tsnep_rx_refill_zc(struct tsnep_rx * rx,int count,bool reuse)1268 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)
1269 {
1270 int desc_refilled;
1271
1272 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);
1273 if (desc_refilled)
1274 tsnep_rx_enable(rx);
1275
1276 return desc_refilled;
1277 }
1278
tsnep_xsk_rx_need_wakeup(struct tsnep_rx * rx,int desc_available)1279 static void tsnep_xsk_rx_need_wakeup(struct tsnep_rx *rx, int desc_available)
1280 {
1281 if (desc_available)
1282 xsk_set_rx_need_wakeup(rx->xsk_pool);
1283 else
1284 xsk_clear_rx_need_wakeup(rx->xsk_pool);
1285 }
1286
tsnep_xdp_run_prog(struct tsnep_rx * rx,struct bpf_prog * prog,struct xdp_buff * xdp,int * status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1287 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
1288 struct xdp_buff *xdp, int *status,
1289 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1290 {
1291 unsigned int length;
1292 unsigned int sync;
1293 u32 act;
1294
1295 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
1296
1297 act = bpf_prog_run_xdp(prog, xdp);
1298 switch (act) {
1299 case XDP_PASS:
1300 return false;
1301 case XDP_TX:
1302 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, false))
1303 goto out_failure;
1304 *status |= TSNEP_XDP_TX;
1305 return true;
1306 case XDP_REDIRECT:
1307 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1308 goto out_failure;
1309 *status |= TSNEP_XDP_REDIRECT;
1310 return true;
1311 default:
1312 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1313 fallthrough;
1314 case XDP_ABORTED:
1315 out_failure:
1316 trace_xdp_exception(rx->adapter->netdev, prog, act);
1317 fallthrough;
1318 case XDP_DROP:
1319 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU
1320 * touch
1321 */
1322 sync = xdp->data_end - xdp->data_hard_start -
1323 XDP_PACKET_HEADROOM;
1324 sync = max(sync, length);
1325 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),
1326 sync, true);
1327 return true;
1328 }
1329 }
1330
tsnep_xdp_run_prog_zc(struct tsnep_rx * rx,struct bpf_prog * prog,struct xdp_buff * xdp,int * status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1331 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,
1332 struct xdp_buff *xdp, int *status,
1333 struct netdev_queue *tx_nq,
1334 struct tsnep_tx *tx)
1335 {
1336 u32 act;
1337
1338 act = bpf_prog_run_xdp(prog, xdp);
1339
1340 /* XDP_REDIRECT is the main action for zero-copy */
1341 if (likely(act == XDP_REDIRECT)) {
1342 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1343 goto out_failure;
1344 *status |= TSNEP_XDP_REDIRECT;
1345 return true;
1346 }
1347
1348 switch (act) {
1349 case XDP_PASS:
1350 return false;
1351 case XDP_TX:
1352 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, true))
1353 goto out_failure;
1354 *status |= TSNEP_XDP_TX;
1355 return true;
1356 default:
1357 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1358 fallthrough;
1359 case XDP_ABORTED:
1360 out_failure:
1361 trace_xdp_exception(rx->adapter->netdev, prog, act);
1362 fallthrough;
1363 case XDP_DROP:
1364 xsk_buff_free(xdp);
1365 return true;
1366 }
1367 }
1368
tsnep_finalize_xdp(struct tsnep_adapter * adapter,int status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1369 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,
1370 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1371 {
1372 if (status & TSNEP_XDP_TX) {
1373 __netif_tx_lock(tx_nq, smp_processor_id());
1374 tsnep_xdp_xmit_flush(tx);
1375 __netif_tx_unlock(tx_nq);
1376 }
1377
1378 if (status & TSNEP_XDP_REDIRECT)
1379 xdp_do_flush();
1380 }
1381
tsnep_build_skb(struct tsnep_rx * rx,struct page * page,int length)1382 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
1383 int length)
1384 {
1385 struct sk_buff *skb;
1386
1387 skb = napi_build_skb(page_address(page), PAGE_SIZE);
1388 if (unlikely(!skb))
1389 return NULL;
1390
1391 /* update pointers within the skb to store the data */
1392 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE);
1393 __skb_put(skb, length - ETH_FCS_LEN);
1394
1395 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {
1396 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
1397 struct tsnep_rx_inline *rx_inline =
1398 (struct tsnep_rx_inline *)(page_address(page) +
1399 TSNEP_RX_OFFSET);
1400
1401 skb_shinfo(skb)->tx_flags |=
1402 SKBTX_HW_TSTAMP_NETDEV;
1403 memset(hwtstamps, 0, sizeof(*hwtstamps));
1404 hwtstamps->netdev_data = rx_inline;
1405 }
1406
1407 skb_record_rx_queue(skb, rx->queue_index);
1408 skb->protocol = eth_type_trans(skb, rx->adapter->netdev);
1409
1410 return skb;
1411 }
1412
tsnep_rx_page(struct tsnep_rx * rx,struct napi_struct * napi,struct page * page,int length)1413 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,
1414 struct page *page, int length)
1415 {
1416 struct sk_buff *skb;
1417
1418 skb = tsnep_build_skb(rx, page, length);
1419 if (skb) {
1420 skb_mark_for_recycle(skb);
1421
1422 rx->packets++;
1423 rx->bytes += length;
1424 if (skb->pkt_type == PACKET_MULTICAST)
1425 rx->multicast++;
1426
1427 napi_gro_receive(napi, skb);
1428 } else {
1429 page_pool_recycle_direct(rx->page_pool, page);
1430
1431 rx->dropped++;
1432 }
1433 }
1434
tsnep_rx_poll(struct tsnep_rx * rx,struct napi_struct * napi,int budget)1435 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
1436 int budget)
1437 {
1438 struct device *dmadev = rx->adapter->dmadev;
1439 enum dma_data_direction dma_dir;
1440 struct tsnep_rx_entry *entry;
1441 struct netdev_queue *tx_nq;
1442 struct bpf_prog *prog;
1443 struct xdp_buff xdp;
1444 struct tsnep_tx *tx;
1445 int desc_available;
1446 int xdp_status = 0;
1447 int done = 0;
1448 int length;
1449
1450 desc_available = tsnep_rx_desc_available(rx);
1451 dma_dir = page_pool_get_dma_dir(rx->page_pool);
1452 prog = READ_ONCE(rx->adapter->xdp_prog);
1453 if (prog) {
1454 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1455 rx->tx_queue_index);
1456 tx = &rx->adapter->tx[rx->tx_queue_index];
1457
1458 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq);
1459 }
1460
1461 while (likely(done < budget) && (rx->read != rx->write)) {
1462 entry = &rx->entry[rx->read];
1463 if ((__le32_to_cpu(entry->desc_wb->properties) &
1464 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1465 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1466 break;
1467 done++;
1468
1469 if (desc_available >= TSNEP_RING_RX_REFILL) {
1470 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1471
1472 desc_available -= tsnep_rx_refill(rx, desc_available,
1473 reuse);
1474 if (!entry->page) {
1475 /* buffer has been reused for refill to prevent
1476 * empty RX ring, thus buffer cannot be used for
1477 * RX processing
1478 */
1479 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1480 desc_available++;
1481
1482 rx->dropped++;
1483
1484 continue;
1485 }
1486 }
1487
1488 /* descriptor properties shall be read first, because valid data
1489 * is signaled there
1490 */
1491 dma_rmb();
1492
1493 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET);
1494 length = __le32_to_cpu(entry->desc_wb->properties) &
1495 TSNEP_DESC_LENGTH_MASK;
1496 dma_sync_single_range_for_cpu(dmadev, entry->dma,
1497 TSNEP_RX_OFFSET, length, dma_dir);
1498
1499 /* RX metadata with timestamps is in front of actual data,
1500 * subtract metadata size to get length of actual data and
1501 * consider metadata size as offset of actual data during RX
1502 * processing
1503 */
1504 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1505
1506 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1507 desc_available++;
1508
1509 if (prog) {
1510 bool consume;
1511
1512 xdp_prepare_buff(&xdp, page_address(entry->page),
1513 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,
1514 length - ETH_FCS_LEN, false);
1515
1516 consume = tsnep_xdp_run_prog(rx, prog, &xdp,
1517 &xdp_status, tx_nq, tx);
1518 if (consume) {
1519 rx->packets++;
1520 rx->bytes += length;
1521
1522 entry->page = NULL;
1523
1524 continue;
1525 }
1526 }
1527
1528 tsnep_rx_page(rx, napi, entry->page, length);
1529 entry->page = NULL;
1530 }
1531
1532 if (xdp_status)
1533 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1534
1535 if (desc_available)
1536 tsnep_rx_refill(rx, desc_available, false);
1537
1538 return done;
1539 }
1540
tsnep_rx_poll_zc(struct tsnep_rx * rx,struct napi_struct * napi,int budget)1541 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
1542 int budget)
1543 {
1544 struct tsnep_rx_entry *entry;
1545 struct netdev_queue *tx_nq;
1546 struct bpf_prog *prog;
1547 struct tsnep_tx *tx;
1548 int desc_available;
1549 int xdp_status = 0;
1550 struct page *page;
1551 int done = 0;
1552 int length;
1553
1554 desc_available = tsnep_rx_desc_available(rx);
1555 prog = READ_ONCE(rx->adapter->xdp_prog);
1556 if (prog) {
1557 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1558 rx->tx_queue_index);
1559 tx = &rx->adapter->tx[rx->tx_queue_index];
1560 }
1561
1562 while (likely(done < budget) && (rx->read != rx->write)) {
1563 entry = &rx->entry[rx->read];
1564 if ((__le32_to_cpu(entry->desc_wb->properties) &
1565 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1566 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1567 break;
1568 done++;
1569
1570 if (desc_available >= TSNEP_RING_RX_REFILL) {
1571 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1572
1573 desc_available -= tsnep_rx_refill_zc(rx, desc_available,
1574 reuse);
1575 if (!entry->xdp) {
1576 /* buffer has been reused for refill to prevent
1577 * empty RX ring, thus buffer cannot be used for
1578 * RX processing
1579 */
1580 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1581 desc_available++;
1582
1583 rx->dropped++;
1584
1585 continue;
1586 }
1587 }
1588
1589 /* descriptor properties shall be read first, because valid data
1590 * is signaled there
1591 */
1592 dma_rmb();
1593
1594 prefetch(entry->xdp->data);
1595 length = __le32_to_cpu(entry->desc_wb->properties) &
1596 TSNEP_DESC_LENGTH_MASK;
1597 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN);
1598 xsk_buff_dma_sync_for_cpu(entry->xdp);
1599
1600 /* RX metadata with timestamps is in front of actual data,
1601 * subtract metadata size to get length of actual data and
1602 * consider metadata size as offset of actual data during RX
1603 * processing
1604 */
1605 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1606
1607 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1608 desc_available++;
1609
1610 if (prog) {
1611 bool consume;
1612
1613 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;
1614 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;
1615
1616 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp,
1617 &xdp_status, tx_nq, tx);
1618 if (consume) {
1619 rx->packets++;
1620 rx->bytes += length;
1621
1622 entry->xdp = NULL;
1623
1624 continue;
1625 }
1626 }
1627
1628 page = page_pool_dev_alloc_pages(rx->page_pool);
1629 if (page) {
1630 memcpy(page_address(page) + TSNEP_RX_OFFSET,
1631 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,
1632 length + TSNEP_RX_INLINE_METADATA_SIZE);
1633 tsnep_rx_page(rx, napi, page, length);
1634 } else {
1635 rx->dropped++;
1636 }
1637 xsk_buff_free(entry->xdp);
1638 entry->xdp = NULL;
1639 }
1640
1641 if (xdp_status)
1642 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1643
1644 if (desc_available)
1645 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false);
1646
1647 if (xsk_uses_need_wakeup(rx->xsk_pool)) {
1648 tsnep_xsk_rx_need_wakeup(rx, desc_available);
1649
1650 return done;
1651 }
1652
1653 return desc_available ? budget : done;
1654 }
1655
tsnep_rx_pending(struct tsnep_rx * rx)1656 static bool tsnep_rx_pending(struct tsnep_rx *rx)
1657 {
1658 struct tsnep_rx_entry *entry;
1659
1660 if (rx->read != rx->write) {
1661 entry = &rx->entry[rx->read];
1662 if ((__le32_to_cpu(entry->desc_wb->properties) &
1663 TSNEP_DESC_OWNER_COUNTER_MASK) ==
1664 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1665 return true;
1666 }
1667
1668 return false;
1669 }
1670
tsnep_rx_open(struct tsnep_rx * rx)1671 static int tsnep_rx_open(struct tsnep_rx *rx)
1672 {
1673 int desc_available;
1674 int retval;
1675
1676 retval = tsnep_rx_ring_create(rx);
1677 if (retval)
1678 return retval;
1679
1680 tsnep_rx_init(rx);
1681
1682 desc_available = tsnep_rx_desc_available(rx);
1683 if (rx->xsk_pool)
1684 retval = tsnep_rx_alloc_zc(rx, desc_available, false);
1685 else
1686 retval = tsnep_rx_alloc(rx, desc_available, false);
1687 if (retval != desc_available) {
1688 retval = -ENOMEM;
1689
1690 goto alloc_failed;
1691 }
1692
1693 /* prealloc pages to prevent allocation failures when XSK pool is
1694 * disabled at runtime
1695 */
1696 if (rx->xsk_pool) {
1697 retval = tsnep_rx_alloc_page_buffer(rx);
1698 if (retval)
1699 goto alloc_failed;
1700 }
1701
1702 return 0;
1703
1704 alloc_failed:
1705 tsnep_rx_ring_cleanup(rx);
1706 return retval;
1707 }
1708
tsnep_rx_close(struct tsnep_rx * rx)1709 static void tsnep_rx_close(struct tsnep_rx *rx)
1710 {
1711 if (rx->xsk_pool)
1712 tsnep_rx_free_page_buffer(rx);
1713
1714 tsnep_rx_ring_cleanup(rx);
1715 }
1716
tsnep_rx_reopen(struct tsnep_rx * rx)1717 static void tsnep_rx_reopen(struct tsnep_rx *rx)
1718 {
1719 struct page **page = rx->page_buffer;
1720 int i;
1721
1722 tsnep_rx_init(rx);
1723
1724 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1725 struct tsnep_rx_entry *entry = &rx->entry[i];
1726
1727 /* defined initial values for properties are required for
1728 * correct owner counter checking
1729 */
1730 entry->desc->properties = 0;
1731 entry->desc_wb->properties = 0;
1732
1733 /* prevent allocation failures by reusing kept pages */
1734 if (*page) {
1735 tsnep_rx_set_page(rx, entry, *page);
1736 tsnep_rx_activate(rx, rx->write);
1737 rx->write++;
1738
1739 *page = NULL;
1740 page++;
1741 }
1742 }
1743 }
1744
tsnep_rx_reopen_xsk(struct tsnep_rx * rx)1745 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)
1746 {
1747 struct page **page = rx->page_buffer;
1748 u32 allocated;
1749 int i;
1750
1751 tsnep_rx_init(rx);
1752
1753 /* alloc all ring entries except the last one, because ring cannot be
1754 * filled completely, as many buffers as possible is enough as wakeup is
1755 * done if new buffers are available
1756 */
1757 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch,
1758 TSNEP_RING_SIZE - 1);
1759
1760 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1761 struct tsnep_rx_entry *entry = &rx->entry[i];
1762
1763 /* keep pages to prevent allocation failures when xsk is
1764 * disabled
1765 */
1766 if (entry->page) {
1767 *page = entry->page;
1768 entry->page = NULL;
1769
1770 page++;
1771 }
1772
1773 /* defined initial values for properties are required for
1774 * correct owner counter checking
1775 */
1776 entry->desc->properties = 0;
1777 entry->desc_wb->properties = 0;
1778
1779 if (allocated) {
1780 tsnep_rx_set_xdp(rx, entry,
1781 rx->xdp_batch[allocated - 1]);
1782 tsnep_rx_activate(rx, rx->write);
1783 rx->write++;
1784
1785 allocated--;
1786 }
1787 }
1788
1789 /* set need wakeup flag immediately if ring is not filled completely,
1790 * first polling would be too late as need wakeup signalisation would
1791 * be delayed for an indefinite time
1792 */
1793 if (xsk_uses_need_wakeup(rx->xsk_pool))
1794 tsnep_xsk_rx_need_wakeup(rx, tsnep_rx_desc_available(rx));
1795 }
1796
tsnep_pending(struct tsnep_queue * queue)1797 static bool tsnep_pending(struct tsnep_queue *queue)
1798 {
1799 if (queue->tx && tsnep_tx_pending(queue->tx))
1800 return true;
1801
1802 if (queue->rx && tsnep_rx_pending(queue->rx))
1803 return true;
1804
1805 return false;
1806 }
1807
tsnep_poll(struct napi_struct * napi,int budget)1808 static int tsnep_poll(struct napi_struct *napi, int budget)
1809 {
1810 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue,
1811 napi);
1812 bool complete = true;
1813 int done = 0;
1814
1815 if (queue->tx)
1816 complete = tsnep_tx_poll(queue->tx, budget);
1817
1818 /* handle case where we are called by netpoll with a budget of 0 */
1819 if (unlikely(budget <= 0))
1820 return budget;
1821
1822 if (queue->rx) {
1823 done = queue->rx->xsk_pool ?
1824 tsnep_rx_poll_zc(queue->rx, napi, budget) :
1825 tsnep_rx_poll(queue->rx, napi, budget);
1826 if (done >= budget)
1827 complete = false;
1828 }
1829
1830 /* if all work not completed, return budget and keep polling */
1831 if (!complete)
1832 return budget;
1833
1834 if (likely(napi_complete_done(napi, done))) {
1835 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1836
1837 /* reschedule if work is already pending, prevent rotten packets
1838 * which are transmitted or received after polling but before
1839 * interrupt enable
1840 */
1841 if (tsnep_pending(queue)) {
1842 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1843 napi_schedule(napi);
1844 }
1845 }
1846
1847 return min(done, budget - 1);
1848 }
1849
tsnep_request_irq(struct tsnep_queue * queue,bool first)1850 static int tsnep_request_irq(struct tsnep_queue *queue, bool first)
1851 {
1852 const char *name = netdev_name(queue->adapter->netdev);
1853 irq_handler_t handler;
1854 void *dev;
1855 int retval;
1856
1857 if (first) {
1858 sprintf(queue->name, "%s-mac", name);
1859 handler = tsnep_irq;
1860 dev = queue->adapter;
1861 } else {
1862 if (queue->tx && queue->rx)
1863 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d",
1864 name, queue->rx->queue_index);
1865 else if (queue->tx)
1866 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d",
1867 name, queue->tx->queue_index);
1868 else
1869 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d",
1870 name, queue->rx->queue_index);
1871 handler = tsnep_irq_txrx;
1872 dev = queue;
1873 }
1874
1875 retval = request_irq(queue->irq, handler, 0, queue->name, dev);
1876 if (retval) {
1877 /* if name is empty, then interrupt won't be freed */
1878 memset(queue->name, 0, sizeof(queue->name));
1879 }
1880
1881 return retval;
1882 }
1883
tsnep_free_irq(struct tsnep_queue * queue,bool first)1884 static void tsnep_free_irq(struct tsnep_queue *queue, bool first)
1885 {
1886 void *dev;
1887
1888 if (!strlen(queue->name))
1889 return;
1890
1891 if (first)
1892 dev = queue->adapter;
1893 else
1894 dev = queue;
1895
1896 free_irq(queue->irq, dev);
1897 memset(queue->name, 0, sizeof(queue->name));
1898 }
1899
tsnep_queue_close(struct tsnep_queue * queue,bool first)1900 static void tsnep_queue_close(struct tsnep_queue *queue, bool first)
1901 {
1902 struct tsnep_rx *rx = queue->rx;
1903
1904 tsnep_free_irq(queue, first);
1905
1906 if (rx) {
1907 if (xdp_rxq_info_is_reg(&rx->xdp_rxq))
1908 xdp_rxq_info_unreg(&rx->xdp_rxq);
1909 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc))
1910 xdp_rxq_info_unreg(&rx->xdp_rxq_zc);
1911 }
1912
1913 netif_napi_del(&queue->napi);
1914 }
1915
tsnep_queue_open(struct tsnep_adapter * adapter,struct tsnep_queue * queue,bool first)1916 static int tsnep_queue_open(struct tsnep_adapter *adapter,
1917 struct tsnep_queue *queue, bool first)
1918 {
1919 struct tsnep_rx *rx = queue->rx;
1920 struct tsnep_tx *tx = queue->tx;
1921 int retval;
1922
1923 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);
1924
1925 if (rx) {
1926 /* choose TX queue for XDP_TX */
1927 if (tx)
1928 rx->tx_queue_index = tx->queue_index;
1929 else if (rx->queue_index < adapter->num_tx_queues)
1930 rx->tx_queue_index = rx->queue_index;
1931 else
1932 rx->tx_queue_index = 0;
1933
1934 /* prepare both memory models to eliminate possible registration
1935 * errors when memory model is switched between page pool and
1936 * XSK pool during runtime
1937 */
1938 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,
1939 rx->queue_index, queue->napi.napi_id);
1940 if (retval)
1941 goto failed;
1942 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq,
1943 MEM_TYPE_PAGE_POOL,
1944 rx->page_pool);
1945 if (retval)
1946 goto failed;
1947 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev,
1948 rx->queue_index, queue->napi.napi_id);
1949 if (retval)
1950 goto failed;
1951 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc,
1952 MEM_TYPE_XSK_BUFF_POOL,
1953 NULL);
1954 if (retval)
1955 goto failed;
1956 if (rx->xsk_pool)
1957 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc);
1958 }
1959
1960 retval = tsnep_request_irq(queue, first);
1961 if (retval) {
1962 netif_err(adapter, drv, adapter->netdev,
1963 "can't get assigned irq %d.\n", queue->irq);
1964 goto failed;
1965 }
1966
1967 return 0;
1968
1969 failed:
1970 tsnep_queue_close(queue, first);
1971
1972 return retval;
1973 }
1974
tsnep_queue_enable(struct tsnep_queue * queue)1975 static void tsnep_queue_enable(struct tsnep_queue *queue)
1976 {
1977 napi_enable(&queue->napi);
1978 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1979
1980 if (queue->tx)
1981 tsnep_tx_enable(queue->tx);
1982
1983 if (queue->rx)
1984 tsnep_rx_enable(queue->rx);
1985 }
1986
tsnep_queue_disable(struct tsnep_queue * queue)1987 static void tsnep_queue_disable(struct tsnep_queue *queue)
1988 {
1989 if (queue->tx)
1990 tsnep_tx_disable(queue->tx, &queue->napi);
1991
1992 napi_disable(&queue->napi);
1993 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1994
1995 /* disable RX after NAPI polling has been disabled, because RX can be
1996 * enabled during NAPI polling
1997 */
1998 if (queue->rx)
1999 tsnep_rx_disable(queue->rx);
2000 }
2001
tsnep_netdev_open(struct net_device * netdev)2002 static int tsnep_netdev_open(struct net_device *netdev)
2003 {
2004 struct tsnep_adapter *adapter = netdev_priv(netdev);
2005 int i, retval;
2006
2007 for (i = 0; i < adapter->num_queues; i++) {
2008 if (adapter->queue[i].tx) {
2009 retval = tsnep_tx_open(adapter->queue[i].tx);
2010 if (retval)
2011 goto failed;
2012 }
2013 if (adapter->queue[i].rx) {
2014 retval = tsnep_rx_open(adapter->queue[i].rx);
2015 if (retval)
2016 goto failed;
2017 }
2018
2019 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0);
2020 if (retval)
2021 goto failed;
2022 }
2023
2024 retval = netif_set_real_num_tx_queues(adapter->netdev,
2025 adapter->num_tx_queues);
2026 if (retval)
2027 goto failed;
2028 retval = netif_set_real_num_rx_queues(adapter->netdev,
2029 adapter->num_rx_queues);
2030 if (retval)
2031 goto failed;
2032
2033 tsnep_enable_irq(adapter, ECM_INT_LINK);
2034 retval = tsnep_phy_open(adapter);
2035 if (retval)
2036 goto phy_failed;
2037
2038 for (i = 0; i < adapter->num_queues; i++)
2039 tsnep_queue_enable(&adapter->queue[i]);
2040
2041 return 0;
2042
2043 phy_failed:
2044 tsnep_disable_irq(adapter, ECM_INT_LINK);
2045 failed:
2046 for (i = 0; i < adapter->num_queues; i++) {
2047 tsnep_queue_close(&adapter->queue[i], i == 0);
2048
2049 if (adapter->queue[i].rx)
2050 tsnep_rx_close(adapter->queue[i].rx);
2051 if (adapter->queue[i].tx)
2052 tsnep_tx_close(adapter->queue[i].tx);
2053 }
2054 return retval;
2055 }
2056
tsnep_netdev_close(struct net_device * netdev)2057 static int tsnep_netdev_close(struct net_device *netdev)
2058 {
2059 struct tsnep_adapter *adapter = netdev_priv(netdev);
2060 int i;
2061
2062 tsnep_disable_irq(adapter, ECM_INT_LINK);
2063 tsnep_phy_close(adapter);
2064
2065 for (i = 0; i < adapter->num_queues; i++) {
2066 tsnep_queue_disable(&adapter->queue[i]);
2067
2068 tsnep_queue_close(&adapter->queue[i], i == 0);
2069
2070 if (adapter->queue[i].rx)
2071 tsnep_rx_close(adapter->queue[i].rx);
2072 if (adapter->queue[i].tx)
2073 tsnep_tx_close(adapter->queue[i].tx);
2074 }
2075
2076 return 0;
2077 }
2078
tsnep_enable_xsk(struct tsnep_queue * queue,struct xsk_buff_pool * pool)2079 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
2080 {
2081 bool running = netif_running(queue->adapter->netdev);
2082 u32 frame_size;
2083
2084 frame_size = xsk_pool_get_rx_frame_size(pool);
2085 if (frame_size < TSNEP_XSK_RX_BUF_SIZE)
2086 return -EOPNOTSUPP;
2087
2088 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,
2089 sizeof(*queue->rx->page_buffer),
2090 GFP_KERNEL);
2091 if (!queue->rx->page_buffer)
2092 return -ENOMEM;
2093 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,
2094 sizeof(*queue->rx->xdp_batch),
2095 GFP_KERNEL);
2096 if (!queue->rx->xdp_batch) {
2097 kfree(queue->rx->page_buffer);
2098 queue->rx->page_buffer = NULL;
2099
2100 return -ENOMEM;
2101 }
2102
2103 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc);
2104
2105 if (running)
2106 tsnep_queue_disable(queue);
2107
2108 queue->tx->xsk_pool = pool;
2109 queue->rx->xsk_pool = pool;
2110
2111 if (running) {
2112 tsnep_rx_reopen_xsk(queue->rx);
2113 tsnep_queue_enable(queue);
2114 }
2115
2116 return 0;
2117 }
2118
tsnep_disable_xsk(struct tsnep_queue * queue)2119 void tsnep_disable_xsk(struct tsnep_queue *queue)
2120 {
2121 bool running = netif_running(queue->adapter->netdev);
2122
2123 if (running)
2124 tsnep_queue_disable(queue);
2125
2126 tsnep_rx_free_zc(queue->rx);
2127
2128 queue->rx->xsk_pool = NULL;
2129 queue->tx->xsk_pool = NULL;
2130
2131 if (running) {
2132 tsnep_rx_reopen(queue->rx);
2133 tsnep_queue_enable(queue);
2134 }
2135
2136 kfree(queue->rx->xdp_batch);
2137 queue->rx->xdp_batch = NULL;
2138 kfree(queue->rx->page_buffer);
2139 queue->rx->page_buffer = NULL;
2140 }
2141
tsnep_netdev_xmit_frame(struct sk_buff * skb,struct net_device * netdev)2142 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,
2143 struct net_device *netdev)
2144 {
2145 struct tsnep_adapter *adapter = netdev_priv(netdev);
2146 u16 queue_mapping = skb_get_queue_mapping(skb);
2147
2148 if (queue_mapping >= adapter->num_tx_queues)
2149 queue_mapping = 0;
2150
2151 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]);
2152 }
2153
tsnep_netdev_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2154 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr,
2155 int cmd)
2156 {
2157 if (!netif_running(netdev))
2158 return -EINVAL;
2159 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP)
2160 return tsnep_ptp_ioctl(netdev, ifr, cmd);
2161 return phy_mii_ioctl(netdev->phydev, ifr, cmd);
2162 }
2163
tsnep_netdev_set_multicast(struct net_device * netdev)2164 static void tsnep_netdev_set_multicast(struct net_device *netdev)
2165 {
2166 struct tsnep_adapter *adapter = netdev_priv(netdev);
2167
2168 u16 rx_filter = 0;
2169
2170 /* configured MAC address and broadcasts are never filtered */
2171 if (netdev->flags & IFF_PROMISC) {
2172 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2173 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS;
2174 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) {
2175 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2176 }
2177 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER);
2178 }
2179
tsnep_netdev_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2180 static void tsnep_netdev_get_stats64(struct net_device *netdev,
2181 struct rtnl_link_stats64 *stats)
2182 {
2183 struct tsnep_adapter *adapter = netdev_priv(netdev);
2184 u32 reg;
2185 u32 val;
2186 int i;
2187
2188 for (i = 0; i < adapter->num_tx_queues; i++) {
2189 stats->tx_packets += adapter->tx[i].packets;
2190 stats->tx_bytes += adapter->tx[i].bytes;
2191 stats->tx_dropped += adapter->tx[i].dropped;
2192 }
2193 for (i = 0; i < adapter->num_rx_queues; i++) {
2194 stats->rx_packets += adapter->rx[i].packets;
2195 stats->rx_bytes += adapter->rx[i].bytes;
2196 stats->rx_dropped += adapter->rx[i].dropped;
2197 stats->multicast += adapter->rx[i].multicast;
2198
2199 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +
2200 TSNEP_RX_STATISTIC);
2201 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >>
2202 TSNEP_RX_STATISTIC_NO_DESC_SHIFT;
2203 stats->rx_dropped += val;
2204 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >>
2205 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT;
2206 stats->rx_dropped += val;
2207 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >>
2208 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT;
2209 stats->rx_errors += val;
2210 stats->rx_fifo_errors += val;
2211 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >>
2212 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT;
2213 stats->rx_errors += val;
2214 stats->rx_frame_errors += val;
2215 }
2216
2217 reg = ioread32(adapter->addr + ECM_STAT);
2218 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT;
2219 stats->rx_errors += val;
2220 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT;
2221 stats->rx_errors += val;
2222 stats->rx_crc_errors += val;
2223 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT;
2224 stats->rx_errors += val;
2225 }
2226
tsnep_mac_set_address(struct tsnep_adapter * adapter,u8 * addr)2227 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr)
2228 {
2229 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2230 iowrite16(*(u16 *)(addr + sizeof(u32)),
2231 adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2232
2233 ether_addr_copy(adapter->mac_address, addr);
2234 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n",
2235 addr);
2236 }
2237
tsnep_netdev_set_mac_address(struct net_device * netdev,void * addr)2238 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)
2239 {
2240 struct tsnep_adapter *adapter = netdev_priv(netdev);
2241 struct sockaddr *sock_addr = addr;
2242 int retval;
2243
2244 retval = eth_prepare_mac_addr_change(netdev, sock_addr);
2245 if (retval)
2246 return retval;
2247 eth_hw_addr_set(netdev, sock_addr->sa_data);
2248 tsnep_mac_set_address(adapter, sock_addr->sa_data);
2249
2250 return 0;
2251 }
2252
tsnep_netdev_set_features(struct net_device * netdev,netdev_features_t features)2253 static int tsnep_netdev_set_features(struct net_device *netdev,
2254 netdev_features_t features)
2255 {
2256 struct tsnep_adapter *adapter = netdev_priv(netdev);
2257 netdev_features_t changed = netdev->features ^ features;
2258 bool enable;
2259 int retval = 0;
2260
2261 if (changed & NETIF_F_LOOPBACK) {
2262 enable = !!(features & NETIF_F_LOOPBACK);
2263 retval = tsnep_phy_loopback(adapter, enable);
2264 }
2265
2266 return retval;
2267 }
2268
tsnep_netdev_get_tstamp(struct net_device * netdev,const struct skb_shared_hwtstamps * hwtstamps,bool cycles)2269 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,
2270 const struct skb_shared_hwtstamps *hwtstamps,
2271 bool cycles)
2272 {
2273 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data;
2274 u64 timestamp;
2275
2276 if (cycles)
2277 timestamp = __le64_to_cpu(rx_inline->counter);
2278 else
2279 timestamp = __le64_to_cpu(rx_inline->timestamp);
2280
2281 return ns_to_ktime(timestamp);
2282 }
2283
tsnep_netdev_bpf(struct net_device * dev,struct netdev_bpf * bpf)2284 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)
2285 {
2286 struct tsnep_adapter *adapter = netdev_priv(dev);
2287
2288 switch (bpf->command) {
2289 case XDP_SETUP_PROG:
2290 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack);
2291 case XDP_SETUP_XSK_POOL:
2292 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool,
2293 bpf->xsk.queue_id);
2294 default:
2295 return -EOPNOTSUPP;
2296 }
2297 }
2298
tsnep_xdp_get_tx(struct tsnep_adapter * adapter,u32 cpu)2299 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu)
2300 {
2301 if (cpu >= TSNEP_MAX_QUEUES)
2302 cpu &= TSNEP_MAX_QUEUES - 1;
2303
2304 while (cpu >= adapter->num_tx_queues)
2305 cpu -= adapter->num_tx_queues;
2306
2307 return &adapter->tx[cpu];
2308 }
2309
tsnep_netdev_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** xdp,u32 flags)2310 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,
2311 struct xdp_frame **xdp, u32 flags)
2312 {
2313 struct tsnep_adapter *adapter = netdev_priv(dev);
2314 u32 cpu = smp_processor_id();
2315 struct netdev_queue *nq;
2316 struct tsnep_tx *tx;
2317 int nxmit;
2318 bool xmit;
2319
2320 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2321 return -EINVAL;
2322
2323 tx = tsnep_xdp_get_tx(adapter, cpu);
2324 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index);
2325
2326 __netif_tx_lock(nq, cpu);
2327
2328 for (nxmit = 0; nxmit < n; nxmit++) {
2329 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx,
2330 TSNEP_TX_TYPE_XDP_NDO);
2331 if (!xmit)
2332 break;
2333
2334 /* avoid transmit queue timeout since we share it with the slow
2335 * path
2336 */
2337 txq_trans_cond_update(nq);
2338 }
2339
2340 if (flags & XDP_XMIT_FLUSH)
2341 tsnep_xdp_xmit_flush(tx);
2342
2343 __netif_tx_unlock(nq);
2344
2345 return nxmit;
2346 }
2347
tsnep_netdev_xsk_wakeup(struct net_device * dev,u32 queue_id,u32 flags)2348 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,
2349 u32 flags)
2350 {
2351 struct tsnep_adapter *adapter = netdev_priv(dev);
2352 struct tsnep_queue *queue;
2353
2354 if (queue_id >= adapter->num_rx_queues ||
2355 queue_id >= adapter->num_tx_queues)
2356 return -EINVAL;
2357
2358 queue = &adapter->queue[queue_id];
2359
2360 if (!napi_if_scheduled_mark_missed(&queue->napi))
2361 napi_schedule(&queue->napi);
2362
2363 return 0;
2364 }
2365
2366 static const struct net_device_ops tsnep_netdev_ops = {
2367 .ndo_open = tsnep_netdev_open,
2368 .ndo_stop = tsnep_netdev_close,
2369 .ndo_start_xmit = tsnep_netdev_xmit_frame,
2370 .ndo_eth_ioctl = tsnep_netdev_ioctl,
2371 .ndo_set_rx_mode = tsnep_netdev_set_multicast,
2372 .ndo_get_stats64 = tsnep_netdev_get_stats64,
2373 .ndo_set_mac_address = tsnep_netdev_set_mac_address,
2374 .ndo_set_features = tsnep_netdev_set_features,
2375 .ndo_get_tstamp = tsnep_netdev_get_tstamp,
2376 .ndo_setup_tc = tsnep_tc_setup,
2377 .ndo_bpf = tsnep_netdev_bpf,
2378 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit,
2379 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,
2380 };
2381
tsnep_mac_init(struct tsnep_adapter * adapter)2382 static int tsnep_mac_init(struct tsnep_adapter *adapter)
2383 {
2384 int retval;
2385
2386 /* initialize RX filtering, at least configured MAC address and
2387 * broadcast are not filtered
2388 */
2389 iowrite16(0, adapter->addr + TSNEP_RX_FILTER);
2390
2391 /* try to get MAC address in the following order:
2392 * - device tree
2393 * - valid MAC address already set
2394 * - MAC address register if valid
2395 * - random MAC address
2396 */
2397 retval = of_get_mac_address(adapter->pdev->dev.of_node,
2398 adapter->mac_address);
2399 if (retval == -EPROBE_DEFER)
2400 return retval;
2401 if (retval && !is_valid_ether_addr(adapter->mac_address)) {
2402 *(u32 *)adapter->mac_address =
2403 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2404 *(u16 *)(adapter->mac_address + sizeof(u32)) =
2405 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2406 if (!is_valid_ether_addr(adapter->mac_address))
2407 eth_random_addr(adapter->mac_address);
2408 }
2409
2410 tsnep_mac_set_address(adapter, adapter->mac_address);
2411 eth_hw_addr_set(adapter->netdev, adapter->mac_address);
2412
2413 return 0;
2414 }
2415
tsnep_mdio_init(struct tsnep_adapter * adapter)2416 static int tsnep_mdio_init(struct tsnep_adapter *adapter)
2417 {
2418 struct device_node *np = adapter->pdev->dev.of_node;
2419 int retval;
2420
2421 if (np) {
2422 np = of_get_child_by_name(np, "mdio");
2423 if (!np)
2424 return 0;
2425
2426 adapter->suppress_preamble =
2427 of_property_read_bool(np, "suppress-preamble");
2428 }
2429
2430 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);
2431 if (!adapter->mdiobus) {
2432 retval = -ENOMEM;
2433
2434 goto out;
2435 }
2436
2437 adapter->mdiobus->priv = (void *)adapter;
2438 adapter->mdiobus->parent = &adapter->pdev->dev;
2439 adapter->mdiobus->read = tsnep_mdiobus_read;
2440 adapter->mdiobus->write = tsnep_mdiobus_write;
2441 adapter->mdiobus->name = TSNEP "-mdiobus";
2442 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s",
2443 adapter->pdev->name);
2444
2445 /* do not scan broadcast address */
2446 adapter->mdiobus->phy_mask = 0x0000001;
2447
2448 retval = of_mdiobus_register(adapter->mdiobus, np);
2449
2450 out:
2451 of_node_put(np);
2452
2453 return retval;
2454 }
2455
tsnep_phy_init(struct tsnep_adapter * adapter)2456 static int tsnep_phy_init(struct tsnep_adapter *adapter)
2457 {
2458 struct device_node *phy_node;
2459 int retval;
2460
2461 retval = of_get_phy_mode(adapter->pdev->dev.of_node,
2462 &adapter->phy_mode);
2463 if (retval)
2464 adapter->phy_mode = PHY_INTERFACE_MODE_GMII;
2465
2466 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle",
2467 0);
2468 adapter->phydev = of_phy_find_device(phy_node);
2469 of_node_put(phy_node);
2470 if (!adapter->phydev && adapter->mdiobus)
2471 adapter->phydev = phy_find_first(adapter->mdiobus);
2472 if (!adapter->phydev)
2473 return -EIO;
2474
2475 return 0;
2476 }
2477
tsnep_queue_init(struct tsnep_adapter * adapter,int queue_count)2478 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
2479 {
2480 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
2481 char name[8];
2482 int i;
2483 int retval;
2484
2485 /* one TX/RX queue pair for netdev is mandatory */
2486 if (platform_irq_count(adapter->pdev) == 1)
2487 retval = platform_get_irq(adapter->pdev, 0);
2488 else
2489 retval = platform_get_irq_byname(adapter->pdev, "mac");
2490 if (retval < 0)
2491 return retval;
2492 adapter->num_tx_queues = 1;
2493 adapter->num_rx_queues = 1;
2494 adapter->num_queues = 1;
2495 adapter->queue[0].adapter = adapter;
2496 adapter->queue[0].irq = retval;
2497 adapter->queue[0].tx = &adapter->tx[0];
2498 adapter->queue[0].tx->adapter = adapter;
2499 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);
2500 adapter->queue[0].tx->queue_index = 0;
2501 adapter->queue[0].rx = &adapter->rx[0];
2502 adapter->queue[0].rx->adapter = adapter;
2503 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);
2504 adapter->queue[0].rx->queue_index = 0;
2505 adapter->queue[0].irq_mask = irq_mask;
2506 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;
2507 retval = tsnep_set_irq_coalesce(&adapter->queue[0],
2508 TSNEP_COALESCE_USECS_DEFAULT);
2509 if (retval < 0)
2510 return retval;
2511
2512 adapter->netdev->irq = adapter->queue[0].irq;
2513
2514 /* add additional TX/RX queue pairs only if dedicated interrupt is
2515 * available
2516 */
2517 for (i = 1; i < queue_count; i++) {
2518 sprintf(name, "txrx-%d", i);
2519 retval = platform_get_irq_byname_optional(adapter->pdev, name);
2520 if (retval < 0)
2521 break;
2522
2523 adapter->num_tx_queues++;
2524 adapter->num_rx_queues++;
2525 adapter->num_queues++;
2526 adapter->queue[i].adapter = adapter;
2527 adapter->queue[i].irq = retval;
2528 adapter->queue[i].tx = &adapter->tx[i];
2529 adapter->queue[i].tx->adapter = adapter;
2530 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);
2531 adapter->queue[i].tx->queue_index = i;
2532 adapter->queue[i].rx = &adapter->rx[i];
2533 adapter->queue[i].rx->adapter = adapter;
2534 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);
2535 adapter->queue[i].rx->queue_index = i;
2536 adapter->queue[i].irq_mask =
2537 irq_mask << (ECM_INT_TXRX_SHIFT * i);
2538 adapter->queue[i].irq_delay_addr =
2539 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i;
2540 retval = tsnep_set_irq_coalesce(&adapter->queue[i],
2541 TSNEP_COALESCE_USECS_DEFAULT);
2542 if (retval < 0)
2543 return retval;
2544 }
2545
2546 return 0;
2547 }
2548
tsnep_probe(struct platform_device * pdev)2549 static int tsnep_probe(struct platform_device *pdev)
2550 {
2551 struct tsnep_adapter *adapter;
2552 struct net_device *netdev;
2553 struct resource *io;
2554 u32 type;
2555 int revision;
2556 int version;
2557 int queue_count;
2558 int retval;
2559
2560 netdev = devm_alloc_etherdev_mqs(&pdev->dev,
2561 sizeof(struct tsnep_adapter),
2562 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES);
2563 if (!netdev)
2564 return -ENODEV;
2565 SET_NETDEV_DEV(netdev, &pdev->dev);
2566 adapter = netdev_priv(netdev);
2567 platform_set_drvdata(pdev, adapter);
2568 adapter->pdev = pdev;
2569 adapter->dmadev = &pdev->dev;
2570 adapter->netdev = netdev;
2571 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
2572 NETIF_MSG_LINK | NETIF_MSG_IFUP |
2573 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
2574
2575 netdev->min_mtu = ETH_MIN_MTU;
2576 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;
2577
2578 mutex_init(&adapter->gate_control_lock);
2579 mutex_init(&adapter->rxnfc_lock);
2580 INIT_LIST_HEAD(&adapter->rxnfc_rules);
2581
2582 adapter->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &io);
2583 if (IS_ERR(adapter->addr))
2584 return PTR_ERR(adapter->addr);
2585 netdev->mem_start = io->start;
2586 netdev->mem_end = io->end;
2587
2588 type = ioread32(adapter->addr + ECM_TYPE);
2589 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;
2590 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;
2591 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;
2592 adapter->gate_control = type & ECM_GATE_CONTROL;
2593 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;
2594
2595 tsnep_disable_irq(adapter, ECM_INT_ALL);
2596
2597 retval = tsnep_queue_init(adapter, queue_count);
2598 if (retval)
2599 return retval;
2600
2601 retval = dma_set_mask_and_coherent(&adapter->pdev->dev,
2602 DMA_BIT_MASK(64));
2603 if (retval) {
2604 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");
2605 return retval;
2606 }
2607
2608 retval = tsnep_mac_init(adapter);
2609 if (retval)
2610 return retval;
2611
2612 retval = tsnep_mdio_init(adapter);
2613 if (retval)
2614 goto mdio_init_failed;
2615
2616 retval = tsnep_phy_init(adapter);
2617 if (retval)
2618 goto phy_init_failed;
2619
2620 retval = tsnep_ptp_init(adapter);
2621 if (retval)
2622 goto ptp_init_failed;
2623
2624 retval = tsnep_tc_init(adapter);
2625 if (retval)
2626 goto tc_init_failed;
2627
2628 retval = tsnep_rxnfc_init(adapter);
2629 if (retval)
2630 goto rxnfc_init_failed;
2631
2632 netdev->netdev_ops = &tsnep_netdev_ops;
2633 netdev->ethtool_ops = &tsnep_ethtool_ops;
2634 netdev->features = NETIF_F_SG;
2635 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;
2636
2637 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
2638 NETDEV_XDP_ACT_NDO_XMIT |
2639 NETDEV_XDP_ACT_NDO_XMIT_SG |
2640 NETDEV_XDP_ACT_XSK_ZEROCOPY;
2641
2642 /* carrier off reporting is important to ethtool even BEFORE open */
2643 netif_carrier_off(netdev);
2644
2645 retval = register_netdev(netdev);
2646 if (retval)
2647 goto register_failed;
2648
2649 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version,
2650 revision);
2651 if (adapter->gate_control)
2652 dev_info(&adapter->pdev->dev, "gate control detected\n");
2653
2654 return 0;
2655
2656 register_failed:
2657 tsnep_rxnfc_cleanup(adapter);
2658 rxnfc_init_failed:
2659 tsnep_tc_cleanup(adapter);
2660 tc_init_failed:
2661 tsnep_ptp_cleanup(adapter);
2662 ptp_init_failed:
2663 phy_init_failed:
2664 if (adapter->mdiobus)
2665 mdiobus_unregister(adapter->mdiobus);
2666 mdio_init_failed:
2667 return retval;
2668 }
2669
tsnep_remove(struct platform_device * pdev)2670 static void tsnep_remove(struct platform_device *pdev)
2671 {
2672 struct tsnep_adapter *adapter = platform_get_drvdata(pdev);
2673
2674 unregister_netdev(adapter->netdev);
2675
2676 tsnep_rxnfc_cleanup(adapter);
2677
2678 tsnep_tc_cleanup(adapter);
2679
2680 tsnep_ptp_cleanup(adapter);
2681
2682 if (adapter->mdiobus)
2683 mdiobus_unregister(adapter->mdiobus);
2684
2685 tsnep_disable_irq(adapter, ECM_INT_ALL);
2686 }
2687
2688 static const struct of_device_id tsnep_of_match[] = {
2689 { .compatible = "engleder,tsnep", },
2690 { },
2691 };
2692 MODULE_DEVICE_TABLE(of, tsnep_of_match);
2693
2694 static struct platform_driver tsnep_driver = {
2695 .driver = {
2696 .name = TSNEP,
2697 .of_match_table = tsnep_of_match,
2698 },
2699 .probe = tsnep_probe,
2700 .remove_new = tsnep_remove,
2701 };
2702 module_platform_driver(tsnep_driver);
2703
2704 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>");
2705 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver");
2706 MODULE_LICENSE("GPL");
2707