1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/tcp.h>
7 #include <linux/udp.h>
8 #include <linux/vmalloc.h>
9 #include <linux/ptp_classify.h>
10 #include <net/pkt_sched.h>
11
enetc_num_stack_tx_queues(struct enetc_ndev_priv * priv)12 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
13 {
14 int num_tx_rings = priv->num_tx_rings;
15 int i;
16
17 for (i = 0; i < priv->num_rx_rings; i++)
18 if (priv->rx_ring[i]->xdp.prog)
19 return num_tx_rings - num_possible_cpus();
20
21 return num_tx_rings;
22 }
23
enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv * priv,struct enetc_bdr * tx_ring)24 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
25 struct enetc_bdr *tx_ring)
26 {
27 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
28
29 return priv->rx_ring[index];
30 }
31
enetc_tx_swbd_get_skb(struct enetc_tx_swbd * tx_swbd)32 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
33 {
34 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
35 return NULL;
36
37 return tx_swbd->skb;
38 }
39
40 static struct xdp_frame *
enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd * tx_swbd)41 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
42 {
43 if (tx_swbd->is_xdp_redirect)
44 return tx_swbd->xdp_frame;
45
46 return NULL;
47 }
48
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)49 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
50 struct enetc_tx_swbd *tx_swbd)
51 {
52 /* For XDP_TX, pages come from RX, whereas for the other contexts where
53 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
54 * to match the DMA mapping length, so we need to differentiate those.
55 */
56 if (tx_swbd->is_dma_page)
57 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
58 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
59 tx_swbd->dir);
60 else
61 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
62 tx_swbd->len, tx_swbd->dir);
63 tx_swbd->dma = 0;
64 }
65
enetc_free_tx_frame(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)66 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
67 struct enetc_tx_swbd *tx_swbd)
68 {
69 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
70 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
71
72 if (tx_swbd->dma)
73 enetc_unmap_tx_buff(tx_ring, tx_swbd);
74
75 if (xdp_frame) {
76 xdp_return_frame(tx_swbd->xdp_frame);
77 tx_swbd->xdp_frame = NULL;
78 } else if (skb) {
79 dev_kfree_skb_any(skb);
80 tx_swbd->skb = NULL;
81 }
82 }
83
84 /* Let H/W know BD ring has been updated */
enetc_update_tx_ring_tail(struct enetc_bdr * tx_ring)85 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
86 {
87 /* includes wmb() */
88 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
89 }
90
enetc_ptp_parse(struct sk_buff * skb,u8 * udp,u8 * msgtype,u8 * twostep,u16 * correction_offset,u16 * body_offset)91 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
92 u8 *msgtype, u8 *twostep,
93 u16 *correction_offset, u16 *body_offset)
94 {
95 unsigned int ptp_class;
96 struct ptp_header *hdr;
97 unsigned int type;
98 u8 *base;
99
100 ptp_class = ptp_classify_raw(skb);
101 if (ptp_class == PTP_CLASS_NONE)
102 return -EINVAL;
103
104 hdr = ptp_parse_header(skb, ptp_class);
105 if (!hdr)
106 return -EINVAL;
107
108 type = ptp_class & PTP_CLASS_PMASK;
109 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
110 *udp = 1;
111 else
112 *udp = 0;
113
114 *msgtype = ptp_get_msgtype(hdr, ptp_class);
115 *twostep = hdr->flag_field[0] & 0x2;
116
117 base = skb_mac_header(skb);
118 *correction_offset = (u8 *)&hdr->correction - base;
119 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
120
121 return 0;
122 }
123
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)124 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
125 {
126 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
127 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
128 struct enetc_hw *hw = &priv->si->hw;
129 struct enetc_tx_swbd *tx_swbd;
130 int len = skb_headlen(skb);
131 union enetc_tx_bd temp_bd;
132 u8 msgtype, twostep, udp;
133 union enetc_tx_bd *txbd;
134 u16 offset1, offset2;
135 int i, count = 0;
136 skb_frag_t *frag;
137 unsigned int f;
138 dma_addr_t dma;
139 u8 flags = 0;
140
141 i = tx_ring->next_to_use;
142 txbd = ENETC_TXBD(*tx_ring, i);
143 prefetchw(txbd);
144
145 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
146 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
147 goto dma_err;
148
149 temp_bd.addr = cpu_to_le64(dma);
150 temp_bd.buf_len = cpu_to_le16(len);
151 temp_bd.lstatus = 0;
152
153 tx_swbd = &tx_ring->tx_swbd[i];
154 tx_swbd->dma = dma;
155 tx_swbd->len = len;
156 tx_swbd->is_dma_page = 0;
157 tx_swbd->dir = DMA_TO_DEVICE;
158 count++;
159
160 do_vlan = skb_vlan_tag_present(skb);
161 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
162 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
163 &offset2) ||
164 msgtype != PTP_MSGTYPE_SYNC || twostep)
165 WARN_ONCE(1, "Bad packet for one-step timestamping\n");
166 else
167 do_onestep_tstamp = true;
168 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
169 do_twostep_tstamp = true;
170 }
171
172 tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
173 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp;
174
175 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
176 flags |= ENETC_TXBD_FLAGS_EX;
177
178 if (tx_ring->tsd_enable)
179 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
180
181 /* first BD needs frm_len and offload flags set */
182 temp_bd.frm_len = cpu_to_le16(skb->len);
183 temp_bd.flags = flags;
184
185 if (flags & ENETC_TXBD_FLAGS_TSE)
186 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
187 flags);
188
189 if (flags & ENETC_TXBD_FLAGS_EX) {
190 u8 e_flags = 0;
191 *txbd = temp_bd;
192 enetc_clear_tx_bd(&temp_bd);
193
194 /* add extension BD for VLAN and/or timestamping */
195 flags = 0;
196 tx_swbd++;
197 txbd++;
198 i++;
199 if (unlikely(i == tx_ring->bd_count)) {
200 i = 0;
201 tx_swbd = tx_ring->tx_swbd;
202 txbd = ENETC_TXBD(*tx_ring, 0);
203 }
204 prefetchw(txbd);
205
206 if (do_vlan) {
207 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
208 temp_bd.ext.tpid = 0; /* < C-TAG */
209 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
210 }
211
212 if (do_onestep_tstamp) {
213 u32 lo, hi, val;
214 u64 sec, nsec;
215 u8 *data;
216
217 lo = enetc_rd_hot(hw, ENETC_SICTR0);
218 hi = enetc_rd_hot(hw, ENETC_SICTR1);
219 sec = (u64)hi << 32 | lo;
220 nsec = do_div(sec, 1000000000);
221
222 /* Configure extension BD */
223 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
224 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
225
226 /* Update originTimestamp field of Sync packet
227 * - 48 bits seconds field
228 * - 32 bits nanseconds field
229 */
230 data = skb_mac_header(skb);
231 *(__be16 *)(data + offset2) =
232 htons((sec >> 32) & 0xffff);
233 *(__be32 *)(data + offset2 + 2) =
234 htonl(sec & 0xffffffff);
235 *(__be32 *)(data + offset2 + 6) = htonl(nsec);
236
237 /* Configure single-step register */
238 val = ENETC_PM0_SINGLE_STEP_EN;
239 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
240 if (udp)
241 val |= ENETC_PM0_SINGLE_STEP_CH;
242
243 enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val);
244 enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val);
245 } else if (do_twostep_tstamp) {
246 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
247 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
248 }
249
250 temp_bd.ext.e_flags = e_flags;
251 count++;
252 }
253
254 frag = &skb_shinfo(skb)->frags[0];
255 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
256 len = skb_frag_size(frag);
257 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
258 DMA_TO_DEVICE);
259 if (dma_mapping_error(tx_ring->dev, dma))
260 goto dma_err;
261
262 *txbd = temp_bd;
263 enetc_clear_tx_bd(&temp_bd);
264
265 flags = 0;
266 tx_swbd++;
267 txbd++;
268 i++;
269 if (unlikely(i == tx_ring->bd_count)) {
270 i = 0;
271 tx_swbd = tx_ring->tx_swbd;
272 txbd = ENETC_TXBD(*tx_ring, 0);
273 }
274 prefetchw(txbd);
275
276 temp_bd.addr = cpu_to_le64(dma);
277 temp_bd.buf_len = cpu_to_le16(len);
278
279 tx_swbd->dma = dma;
280 tx_swbd->len = len;
281 tx_swbd->is_dma_page = 1;
282 tx_swbd->dir = DMA_TO_DEVICE;
283 count++;
284 }
285
286 /* last BD needs 'F' bit set */
287 flags |= ENETC_TXBD_FLAGS_F;
288 temp_bd.flags = flags;
289 *txbd = temp_bd;
290
291 tx_ring->tx_swbd[i].is_eof = true;
292 tx_ring->tx_swbd[i].skb = skb;
293
294 enetc_bdr_idx_inc(tx_ring, &i);
295 tx_ring->next_to_use = i;
296
297 skb_tx_timestamp(skb);
298
299 enetc_update_tx_ring_tail(tx_ring);
300
301 return count;
302
303 dma_err:
304 dev_err(tx_ring->dev, "DMA map error");
305
306 do {
307 tx_swbd = &tx_ring->tx_swbd[i];
308 enetc_free_tx_frame(tx_ring, tx_swbd);
309 if (i == 0)
310 i = tx_ring->bd_count;
311 i--;
312 } while (count--);
313
314 return 0;
315 }
316
enetc_start_xmit(struct sk_buff * skb,struct net_device * ndev)317 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
318 struct net_device *ndev)
319 {
320 struct enetc_ndev_priv *priv = netdev_priv(ndev);
321 struct enetc_bdr *tx_ring;
322 int count;
323
324 /* Queue one-step Sync packet if already locked */
325 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
326 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
327 &priv->flags)) {
328 skb_queue_tail(&priv->tx_skbs, skb);
329 return NETDEV_TX_OK;
330 }
331 }
332
333 tx_ring = priv->tx_ring[skb->queue_mapping];
334
335 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
336 if (unlikely(skb_linearize(skb)))
337 goto drop_packet_err;
338
339 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
340 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
341 netif_stop_subqueue(ndev, tx_ring->index);
342 return NETDEV_TX_BUSY;
343 }
344
345 enetc_lock_mdio();
346 count = enetc_map_tx_buffs(tx_ring, skb);
347 enetc_unlock_mdio();
348
349 if (unlikely(!count))
350 goto drop_packet_err;
351
352 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
353 netif_stop_subqueue(ndev, tx_ring->index);
354
355 return NETDEV_TX_OK;
356
357 drop_packet_err:
358 dev_kfree_skb_any(skb);
359 return NETDEV_TX_OK;
360 }
361
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)362 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
363 {
364 struct enetc_ndev_priv *priv = netdev_priv(ndev);
365 u8 udp, msgtype, twostep;
366 u16 offset1, offset2;
367
368 /* Mark tx timestamp type on skb->cb[0] if requires */
369 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
370 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
371 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
372 } else {
373 skb->cb[0] = 0;
374 }
375
376 /* Fall back to two-step timestamp if not one-step Sync packet */
377 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
378 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
379 &offset1, &offset2) ||
380 msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
381 skb->cb[0] = ENETC_F_TX_TSTAMP;
382 }
383
384 return enetc_start_xmit(skb, ndev);
385 }
386
enetc_msix(int irq,void * data)387 static irqreturn_t enetc_msix(int irq, void *data)
388 {
389 struct enetc_int_vector *v = data;
390 int i;
391
392 enetc_lock_mdio();
393
394 /* disable interrupts */
395 enetc_wr_reg_hot(v->rbier, 0);
396 enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
397
398 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
399 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
400
401 enetc_unlock_mdio();
402
403 napi_schedule(&v->napi);
404
405 return IRQ_HANDLED;
406 }
407
enetc_rx_dim_work(struct work_struct * w)408 static void enetc_rx_dim_work(struct work_struct *w)
409 {
410 struct dim *dim = container_of(w, struct dim, work);
411 struct dim_cq_moder moder =
412 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
413 struct enetc_int_vector *v =
414 container_of(dim, struct enetc_int_vector, rx_dim);
415
416 v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
417 dim->state = DIM_START_MEASURE;
418 }
419
enetc_rx_net_dim(struct enetc_int_vector * v)420 static void enetc_rx_net_dim(struct enetc_int_vector *v)
421 {
422 struct dim_sample dim_sample = {};
423
424 v->comp_cnt++;
425
426 if (!v->rx_napi_work)
427 return;
428
429 dim_update_sample(v->comp_cnt,
430 v->rx_ring.stats.packets,
431 v->rx_ring.stats.bytes,
432 &dim_sample);
433 net_dim(&v->rx_dim, dim_sample);
434 }
435
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)436 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
437 {
438 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
439
440 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
441 }
442
enetc_page_reusable(struct page * page)443 static bool enetc_page_reusable(struct page *page)
444 {
445 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
446 }
447
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)448 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
449 struct enetc_rx_swbd *old)
450 {
451 struct enetc_rx_swbd *new;
452
453 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
454
455 /* next buf that may reuse a page */
456 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
457
458 /* copy page reference */
459 *new = *old;
460 }
461
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)462 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
463 u64 *tstamp)
464 {
465 u32 lo, hi, tstamp_lo;
466
467 lo = enetc_rd_hot(hw, ENETC_SICTR0);
468 hi = enetc_rd_hot(hw, ENETC_SICTR1);
469 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
470 if (lo <= tstamp_lo)
471 hi -= 1;
472 *tstamp = (u64)hi << 32 | tstamp_lo;
473 }
474
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)475 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
476 {
477 struct skb_shared_hwtstamps shhwtstamps;
478
479 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
480 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
481 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
482 skb_txtime_consumed(skb);
483 skb_tstamp_tx(skb, &shhwtstamps);
484 }
485 }
486
enetc_recycle_xdp_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)487 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
488 struct enetc_tx_swbd *tx_swbd)
489 {
490 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
491 struct enetc_rx_swbd rx_swbd = {
492 .dma = tx_swbd->dma,
493 .page = tx_swbd->page,
494 .page_offset = tx_swbd->page_offset,
495 .dir = tx_swbd->dir,
496 .len = tx_swbd->len,
497 };
498 struct enetc_bdr *rx_ring;
499
500 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
501
502 if (likely(enetc_swbd_unused(rx_ring))) {
503 enetc_reuse_page(rx_ring, &rx_swbd);
504
505 /* sync for use by the device */
506 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
507 rx_swbd.page_offset,
508 ENETC_RXB_DMA_SIZE_XDP,
509 rx_swbd.dir);
510
511 rx_ring->stats.recycles++;
512 } else {
513 /* RX ring is already full, we need to unmap and free the
514 * page, since there's nothing useful we can do with it.
515 */
516 rx_ring->stats.recycle_failures++;
517
518 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
519 rx_swbd.dir);
520 __free_page(rx_swbd.page);
521 }
522
523 rx_ring->xdp.xdp_tx_in_flight--;
524 }
525
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)526 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
527 {
528 struct net_device *ndev = tx_ring->ndev;
529 struct enetc_ndev_priv *priv = netdev_priv(ndev);
530 int tx_frm_cnt = 0, tx_byte_cnt = 0;
531 struct enetc_tx_swbd *tx_swbd;
532 int i, bds_to_clean;
533 bool do_twostep_tstamp;
534 u64 tstamp = 0;
535
536 i = tx_ring->next_to_clean;
537 tx_swbd = &tx_ring->tx_swbd[i];
538
539 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
540
541 do_twostep_tstamp = false;
542
543 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
544 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
545 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
546 bool is_eof = tx_swbd->is_eof;
547
548 if (unlikely(tx_swbd->check_wb)) {
549 struct enetc_ndev_priv *priv = netdev_priv(ndev);
550 union enetc_tx_bd *txbd;
551
552 txbd = ENETC_TXBD(*tx_ring, i);
553
554 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
555 tx_swbd->do_twostep_tstamp) {
556 enetc_get_tx_tstamp(&priv->si->hw, txbd,
557 &tstamp);
558 do_twostep_tstamp = true;
559 }
560 }
561
562 if (tx_swbd->is_xdp_tx)
563 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
564 else if (likely(tx_swbd->dma))
565 enetc_unmap_tx_buff(tx_ring, tx_swbd);
566
567 if (xdp_frame) {
568 xdp_return_frame(xdp_frame);
569 } else if (skb) {
570 if (unlikely(tx_swbd->skb->cb[0] &
571 ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
572 /* Start work to release lock for next one-step
573 * timestamping packet. And send one skb in
574 * tx_skbs queue if has.
575 */
576 schedule_work(&priv->tx_onestep_tstamp);
577 } else if (unlikely(do_twostep_tstamp)) {
578 enetc_tstamp_tx(skb, tstamp);
579 do_twostep_tstamp = false;
580 }
581 napi_consume_skb(skb, napi_budget);
582 }
583
584 tx_byte_cnt += tx_swbd->len;
585 /* Scrub the swbd here so we don't have to do that
586 * when we reuse it during xmit
587 */
588 memset(tx_swbd, 0, sizeof(*tx_swbd));
589
590 bds_to_clean--;
591 tx_swbd++;
592 i++;
593 if (unlikely(i == tx_ring->bd_count)) {
594 i = 0;
595 tx_swbd = tx_ring->tx_swbd;
596 }
597
598 /* BD iteration loop end */
599 if (is_eof) {
600 tx_frm_cnt++;
601 /* re-arm interrupt source */
602 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
603 BIT(16 + tx_ring->index));
604 }
605
606 if (unlikely(!bds_to_clean))
607 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
608 }
609
610 tx_ring->next_to_clean = i;
611 tx_ring->stats.packets += tx_frm_cnt;
612 tx_ring->stats.bytes += tx_byte_cnt;
613
614 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
615 __netif_subqueue_stopped(ndev, tx_ring->index) &&
616 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
617 netif_wake_subqueue(ndev, tx_ring->index);
618 }
619
620 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
621 }
622
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)623 static bool enetc_new_page(struct enetc_bdr *rx_ring,
624 struct enetc_rx_swbd *rx_swbd)
625 {
626 bool xdp = !!(rx_ring->xdp.prog);
627 struct page *page;
628 dma_addr_t addr;
629
630 page = dev_alloc_page();
631 if (unlikely(!page))
632 return false;
633
634 /* For XDP_TX, we forgo dma_unmap -> dma_map */
635 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
636
637 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
638 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
639 __free_page(page);
640
641 return false;
642 }
643
644 rx_swbd->dma = addr;
645 rx_swbd->page = page;
646 rx_swbd->page_offset = rx_ring->buffer_offset;
647
648 return true;
649 }
650
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)651 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
652 {
653 struct enetc_rx_swbd *rx_swbd;
654 union enetc_rx_bd *rxbd;
655 int i, j;
656
657 i = rx_ring->next_to_use;
658 rx_swbd = &rx_ring->rx_swbd[i];
659 rxbd = enetc_rxbd(rx_ring, i);
660
661 for (j = 0; j < buff_cnt; j++) {
662 /* try reuse page */
663 if (unlikely(!rx_swbd->page)) {
664 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
665 rx_ring->stats.rx_alloc_errs++;
666 break;
667 }
668 }
669
670 /* update RxBD */
671 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
672 rx_swbd->page_offset);
673 /* clear 'R" as well */
674 rxbd->r.lstatus = 0;
675
676 enetc_rxbd_next(rx_ring, &rxbd, &i);
677 rx_swbd = &rx_ring->rx_swbd[i];
678 }
679
680 if (likely(j)) {
681 rx_ring->next_to_alloc = i; /* keep track from page reuse */
682 rx_ring->next_to_use = i;
683
684 /* update ENETC's consumer index */
685 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
686 }
687
688 return j;
689 }
690
691 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)692 static void enetc_get_rx_tstamp(struct net_device *ndev,
693 union enetc_rx_bd *rxbd,
694 struct sk_buff *skb)
695 {
696 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
697 struct enetc_ndev_priv *priv = netdev_priv(ndev);
698 struct enetc_hw *hw = &priv->si->hw;
699 u32 lo, hi, tstamp_lo;
700 u64 tstamp;
701
702 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
703 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
704 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
705 rxbd = enetc_rxbd_ext(rxbd);
706 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
707 if (lo <= tstamp_lo)
708 hi -= 1;
709
710 tstamp = (u64)hi << 32 | tstamp_lo;
711 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
712 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
713 }
714 }
715 #endif
716
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)717 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
718 union enetc_rx_bd *rxbd, struct sk_buff *skb)
719 {
720 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
721
722 /* TODO: hashing */
723 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
724 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
725
726 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
727 skb->ip_summed = CHECKSUM_COMPLETE;
728 }
729
730 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
731 __be16 tpid = 0;
732
733 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
734 case 0:
735 tpid = htons(ETH_P_8021Q);
736 break;
737 case 1:
738 tpid = htons(ETH_P_8021AD);
739 break;
740 case 2:
741 tpid = htons(enetc_port_rd(&priv->si->hw,
742 ENETC_PCVLANR1));
743 break;
744 case 3:
745 tpid = htons(enetc_port_rd(&priv->si->hw,
746 ENETC_PCVLANR2));
747 break;
748 default:
749 break;
750 }
751
752 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
753 }
754
755 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
756 if (priv->active_offloads & ENETC_F_RX_TSTAMP)
757 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
758 #endif
759 }
760
761 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
762 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
763 * mapped buffers.
764 */
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)765 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
766 int i, u16 size)
767 {
768 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
769
770 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
771 rx_swbd->page_offset,
772 size, rx_swbd->dir);
773 return rx_swbd;
774 }
775
776 /* Reuse the current page without performing half-page buffer flipping */
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)777 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
778 struct enetc_rx_swbd *rx_swbd)
779 {
780 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
781
782 enetc_reuse_page(rx_ring, rx_swbd);
783
784 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
785 rx_swbd->page_offset,
786 buffer_size, rx_swbd->dir);
787
788 rx_swbd->page = NULL;
789 }
790
791 /* Reuse the current page by performing half-page buffer flipping */
enetc_flip_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)792 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
793 struct enetc_rx_swbd *rx_swbd)
794 {
795 if (likely(enetc_page_reusable(rx_swbd->page))) {
796 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
797 page_ref_inc(rx_swbd->page);
798
799 enetc_put_rx_buff(rx_ring, rx_swbd);
800 } else {
801 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
802 rx_swbd->dir);
803 rx_swbd->page = NULL;
804 }
805 }
806
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)807 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
808 int i, u16 size)
809 {
810 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
811 struct sk_buff *skb;
812 void *ba;
813
814 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
815 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
816 if (unlikely(!skb)) {
817 rx_ring->stats.rx_alloc_errs++;
818 return NULL;
819 }
820
821 skb_reserve(skb, rx_ring->buffer_offset);
822 __skb_put(skb, size);
823
824 enetc_flip_rx_buff(rx_ring, rx_swbd);
825
826 return skb;
827 }
828
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)829 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
830 u16 size, struct sk_buff *skb)
831 {
832 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
833
834 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
835 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
836
837 enetc_flip_rx_buff(rx_ring, rx_swbd);
838 }
839
enetc_check_bd_errors_and_consume(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i)840 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
841 u32 bd_status,
842 union enetc_rx_bd **rxbd, int *i)
843 {
844 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
845 return false;
846
847 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
848 enetc_rxbd_next(rx_ring, rxbd, i);
849
850 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
851 dma_rmb();
852 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
853
854 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
855 enetc_rxbd_next(rx_ring, rxbd, i);
856 }
857
858 rx_ring->ndev->stats.rx_dropped++;
859 rx_ring->ndev->stats.rx_errors++;
860
861 return true;
862 }
863
enetc_build_skb(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,int buffer_size)864 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
865 u32 bd_status, union enetc_rx_bd **rxbd,
866 int *i, int *cleaned_cnt, int buffer_size)
867 {
868 struct sk_buff *skb;
869 u16 size;
870
871 size = le16_to_cpu((*rxbd)->r.buf_len);
872 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
873 if (!skb)
874 return NULL;
875
876 enetc_get_offloads(rx_ring, *rxbd, skb);
877
878 (*cleaned_cnt)++;
879
880 enetc_rxbd_next(rx_ring, rxbd, i);
881
882 /* not last BD in frame? */
883 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
884 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
885 size = buffer_size;
886
887 if (bd_status & ENETC_RXBD_LSTATUS_F) {
888 dma_rmb();
889 size = le16_to_cpu((*rxbd)->r.buf_len);
890 }
891
892 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
893
894 (*cleaned_cnt)++;
895
896 enetc_rxbd_next(rx_ring, rxbd, i);
897 }
898
899 skb_record_rx_queue(skb, rx_ring->index);
900 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
901
902 return skb;
903 }
904
905 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
906
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)907 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
908 struct napi_struct *napi, int work_limit)
909 {
910 int rx_frm_cnt = 0, rx_byte_cnt = 0;
911 int cleaned_cnt, i;
912
913 cleaned_cnt = enetc_bd_unused(rx_ring);
914 /* next descriptor to process */
915 i = rx_ring->next_to_clean;
916
917 while (likely(rx_frm_cnt < work_limit)) {
918 union enetc_rx_bd *rxbd;
919 struct sk_buff *skb;
920 u32 bd_status;
921
922 if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
923 cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
924 cleaned_cnt);
925
926 rxbd = enetc_rxbd(rx_ring, i);
927 bd_status = le32_to_cpu(rxbd->r.lstatus);
928 if (!bd_status)
929 break;
930
931 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
932 dma_rmb(); /* for reading other rxbd fields */
933
934 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
935 &rxbd, &i))
936 break;
937
938 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
939 &cleaned_cnt, ENETC_RXB_DMA_SIZE);
940 if (!skb)
941 break;
942
943 /* When set, the outer VLAN header is extracted and reported
944 * in the receive buffer descriptor. So rx_byte_cnt should
945 * add the length of the extracted VLAN header.
946 */
947 if (bd_status & ENETC_RXBD_FLAG_VLAN)
948 rx_byte_cnt += VLAN_HLEN;
949 rx_byte_cnt += skb->len + ETH_HLEN;
950 rx_frm_cnt++;
951
952 napi_gro_receive(napi, skb);
953 }
954
955 rx_ring->next_to_clean = i;
956
957 rx_ring->stats.packets += rx_frm_cnt;
958 rx_ring->stats.bytes += rx_byte_cnt;
959
960 return rx_frm_cnt;
961 }
962
enetc_xdp_map_tx_buff(struct enetc_bdr * tx_ring,int i,struct enetc_tx_swbd * tx_swbd,int frm_len)963 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
964 struct enetc_tx_swbd *tx_swbd,
965 int frm_len)
966 {
967 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
968
969 prefetchw(txbd);
970
971 enetc_clear_tx_bd(txbd);
972 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
973 txbd->buf_len = cpu_to_le16(tx_swbd->len);
974 txbd->frm_len = cpu_to_le16(frm_len);
975
976 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
977 }
978
979 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
980 * descriptors.
981 */
enetc_xdp_tx(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,int num_tx_swbd)982 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
983 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
984 {
985 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
986 int i, k, frm_len = tmp_tx_swbd->len;
987
988 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
989 return false;
990
991 while (unlikely(!tmp_tx_swbd->is_eof)) {
992 tmp_tx_swbd++;
993 frm_len += tmp_tx_swbd->len;
994 }
995
996 i = tx_ring->next_to_use;
997
998 for (k = 0; k < num_tx_swbd; k++) {
999 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1000
1001 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1002
1003 /* last BD needs 'F' bit set */
1004 if (xdp_tx_swbd->is_eof) {
1005 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1006
1007 txbd->flags = ENETC_TXBD_FLAGS_F;
1008 }
1009
1010 enetc_bdr_idx_inc(tx_ring, &i);
1011 }
1012
1013 tx_ring->next_to_use = i;
1014
1015 return true;
1016 }
1017
enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,struct xdp_frame * xdp_frame)1018 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1019 struct enetc_tx_swbd *xdp_tx_arr,
1020 struct xdp_frame *xdp_frame)
1021 {
1022 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1023 struct skb_shared_info *shinfo;
1024 void *data = xdp_frame->data;
1025 int len = xdp_frame->len;
1026 skb_frag_t *frag;
1027 dma_addr_t dma;
1028 unsigned int f;
1029 int n = 0;
1030
1031 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1032 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1033 netdev_err(tx_ring->ndev, "DMA map error\n");
1034 return -1;
1035 }
1036
1037 xdp_tx_swbd->dma = dma;
1038 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1039 xdp_tx_swbd->len = len;
1040 xdp_tx_swbd->is_xdp_redirect = true;
1041 xdp_tx_swbd->is_eof = false;
1042 xdp_tx_swbd->xdp_frame = NULL;
1043
1044 n++;
1045 xdp_tx_swbd = &xdp_tx_arr[n];
1046
1047 shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1048
1049 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1050 f++, frag++) {
1051 data = skb_frag_address(frag);
1052 len = skb_frag_size(frag);
1053
1054 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1055 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1056 /* Undo the DMA mapping for all fragments */
1057 while (--n >= 0)
1058 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1059
1060 netdev_err(tx_ring->ndev, "DMA map error\n");
1061 return -1;
1062 }
1063
1064 xdp_tx_swbd->dma = dma;
1065 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1066 xdp_tx_swbd->len = len;
1067 xdp_tx_swbd->is_xdp_redirect = true;
1068 xdp_tx_swbd->is_eof = false;
1069 xdp_tx_swbd->xdp_frame = NULL;
1070
1071 n++;
1072 xdp_tx_swbd = &xdp_tx_arr[n];
1073 }
1074
1075 xdp_tx_arr[n - 1].is_eof = true;
1076 xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1077
1078 return n;
1079 }
1080
enetc_xdp_xmit(struct net_device * ndev,int num_frames,struct xdp_frame ** frames,u32 flags)1081 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1082 struct xdp_frame **frames, u32 flags)
1083 {
1084 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1085 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1086 struct enetc_bdr *tx_ring;
1087 int xdp_tx_bd_cnt, i, k;
1088 int xdp_tx_frm_cnt = 0;
1089
1090 enetc_lock_mdio();
1091
1092 tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1093
1094 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1095
1096 for (k = 0; k < num_frames; k++) {
1097 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1098 xdp_redirect_arr,
1099 frames[k]);
1100 if (unlikely(xdp_tx_bd_cnt < 0))
1101 break;
1102
1103 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1104 xdp_tx_bd_cnt))) {
1105 for (i = 0; i < xdp_tx_bd_cnt; i++)
1106 enetc_unmap_tx_buff(tx_ring,
1107 &xdp_redirect_arr[i]);
1108 tx_ring->stats.xdp_tx_drops++;
1109 break;
1110 }
1111
1112 xdp_tx_frm_cnt++;
1113 }
1114
1115 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1116 enetc_update_tx_ring_tail(tx_ring);
1117
1118 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1119
1120 enetc_unlock_mdio();
1121
1122 return xdp_tx_frm_cnt;
1123 }
1124
enetc_map_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,struct xdp_buff * xdp_buff,u16 size)1125 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1126 struct xdp_buff *xdp_buff, u16 size)
1127 {
1128 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1129 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1130 struct skb_shared_info *shinfo;
1131
1132 /* To be used for XDP_TX */
1133 rx_swbd->len = size;
1134
1135 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1136 rx_ring->buffer_offset, size, false);
1137
1138 shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1139 shinfo->nr_frags = 0;
1140 }
1141
enetc_add_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,u16 size,struct xdp_buff * xdp_buff)1142 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1143 u16 size, struct xdp_buff *xdp_buff)
1144 {
1145 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1146 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1147 skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags];
1148
1149 /* To be used for XDP_TX */
1150 rx_swbd->len = size;
1151
1152 skb_frag_off_set(frag, rx_swbd->page_offset);
1153 skb_frag_size_set(frag, size);
1154 __skb_frag_set_page(frag, rx_swbd->page);
1155
1156 shinfo->nr_frags++;
1157 }
1158
enetc_build_xdp_buff(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,struct xdp_buff * xdp_buff)1159 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1160 union enetc_rx_bd **rxbd, int *i,
1161 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1162 {
1163 u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1164
1165 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1166
1167 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1168 (*cleaned_cnt)++;
1169 enetc_rxbd_next(rx_ring, rxbd, i);
1170
1171 /* not last BD in frame? */
1172 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1173 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1174 size = ENETC_RXB_DMA_SIZE_XDP;
1175
1176 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1177 dma_rmb();
1178 size = le16_to_cpu((*rxbd)->r.buf_len);
1179 }
1180
1181 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1182 (*cleaned_cnt)++;
1183 enetc_rxbd_next(rx_ring, rxbd, i);
1184 }
1185 }
1186
1187 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1188 * recycled back into the RX ring in enetc_clean_tx_ring.
1189 */
enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd * xdp_tx_arr,struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1190 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1191 struct enetc_bdr *rx_ring,
1192 int rx_ring_first, int rx_ring_last)
1193 {
1194 int n = 0;
1195
1196 for (; rx_ring_first != rx_ring_last;
1197 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1198 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1199 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1200
1201 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1202 tx_swbd->dma = rx_swbd->dma;
1203 tx_swbd->dir = rx_swbd->dir;
1204 tx_swbd->page = rx_swbd->page;
1205 tx_swbd->page_offset = rx_swbd->page_offset;
1206 tx_swbd->len = rx_swbd->len;
1207 tx_swbd->is_dma_page = true;
1208 tx_swbd->is_xdp_tx = true;
1209 tx_swbd->is_eof = false;
1210 }
1211
1212 /* We rely on caller providing an rx_ring_last > rx_ring_first */
1213 xdp_tx_arr[n - 1].is_eof = true;
1214
1215 return n;
1216 }
1217
enetc_xdp_drop(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1218 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1219 int rx_ring_last)
1220 {
1221 while (rx_ring_first != rx_ring_last) {
1222 enetc_put_rx_buff(rx_ring,
1223 &rx_ring->rx_swbd[rx_ring_first]);
1224 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1225 }
1226 rx_ring->stats.xdp_drops++;
1227 }
1228
enetc_clean_rx_ring_xdp(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit,struct bpf_prog * prog)1229 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1230 struct napi_struct *napi, int work_limit,
1231 struct bpf_prog *prog)
1232 {
1233 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1234 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1235 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1236 int rx_frm_cnt = 0, rx_byte_cnt = 0;
1237 struct enetc_bdr *tx_ring;
1238 int cleaned_cnt, i;
1239 u32 xdp_act;
1240
1241 cleaned_cnt = enetc_bd_unused(rx_ring);
1242 /* next descriptor to process */
1243 i = rx_ring->next_to_clean;
1244
1245 while (likely(rx_frm_cnt < work_limit)) {
1246 union enetc_rx_bd *rxbd, *orig_rxbd;
1247 int orig_i, orig_cleaned_cnt;
1248 struct xdp_buff xdp_buff;
1249 struct sk_buff *skb;
1250 u32 bd_status;
1251 int err;
1252
1253 rxbd = enetc_rxbd(rx_ring, i);
1254 bd_status = le32_to_cpu(rxbd->r.lstatus);
1255 if (!bd_status)
1256 break;
1257
1258 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1259 dma_rmb(); /* for reading other rxbd fields */
1260
1261 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1262 &rxbd, &i))
1263 break;
1264
1265 orig_rxbd = rxbd;
1266 orig_cleaned_cnt = cleaned_cnt;
1267 orig_i = i;
1268
1269 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1270 &cleaned_cnt, &xdp_buff);
1271
1272 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1273
1274 switch (xdp_act) {
1275 default:
1276 bpf_warn_invalid_xdp_action(xdp_act);
1277 fallthrough;
1278 case XDP_ABORTED:
1279 trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1280 fallthrough;
1281 case XDP_DROP:
1282 enetc_xdp_drop(rx_ring, orig_i, i);
1283 break;
1284 case XDP_PASS:
1285 rxbd = orig_rxbd;
1286 cleaned_cnt = orig_cleaned_cnt;
1287 i = orig_i;
1288
1289 skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1290 &i, &cleaned_cnt,
1291 ENETC_RXB_DMA_SIZE_XDP);
1292 if (unlikely(!skb))
1293 goto out;
1294
1295 napi_gro_receive(napi, skb);
1296 break;
1297 case XDP_TX:
1298 tx_ring = priv->xdp_tx_ring[rx_ring->index];
1299 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1300 rx_ring,
1301 orig_i, i);
1302
1303 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1304 enetc_xdp_drop(rx_ring, orig_i, i);
1305 tx_ring->stats.xdp_tx_drops++;
1306 } else {
1307 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1308 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1309 xdp_tx_frm_cnt++;
1310 /* The XDP_TX enqueue was successful, so we
1311 * need to scrub the RX software BDs because
1312 * the ownership of the buffers no longer
1313 * belongs to the RX ring, and we must prevent
1314 * enetc_refill_rx_ring() from reusing
1315 * rx_swbd->page.
1316 */
1317 while (orig_i != i) {
1318 rx_ring->rx_swbd[orig_i].page = NULL;
1319 enetc_bdr_idx_inc(rx_ring, &orig_i);
1320 }
1321 }
1322 break;
1323 case XDP_REDIRECT:
1324 /* xdp_return_frame does not support S/G in the sense
1325 * that it leaks the fragments (__xdp_return should not
1326 * call page_frag_free only for the initial buffer).
1327 * Until XDP_REDIRECT gains support for S/G let's keep
1328 * the code structure in place, but dead. We drop the
1329 * S/G frames ourselves to avoid memory leaks which
1330 * would otherwise leave the kernel OOM.
1331 */
1332 if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) {
1333 enetc_xdp_drop(rx_ring, orig_i, i);
1334 rx_ring->stats.xdp_redirect_sg++;
1335 break;
1336 }
1337
1338 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1339 if (unlikely(err)) {
1340 enetc_xdp_drop(rx_ring, orig_i, i);
1341 rx_ring->stats.xdp_redirect_failures++;
1342 } else {
1343 while (orig_i != i) {
1344 enetc_flip_rx_buff(rx_ring,
1345 &rx_ring->rx_swbd[orig_i]);
1346 enetc_bdr_idx_inc(rx_ring, &orig_i);
1347 }
1348 xdp_redirect_frm_cnt++;
1349 rx_ring->stats.xdp_redirect++;
1350 }
1351 }
1352
1353 rx_frm_cnt++;
1354 }
1355
1356 out:
1357 rx_ring->next_to_clean = i;
1358
1359 rx_ring->stats.packets += rx_frm_cnt;
1360 rx_ring->stats.bytes += rx_byte_cnt;
1361
1362 if (xdp_redirect_frm_cnt)
1363 xdp_do_flush_map();
1364
1365 if (xdp_tx_frm_cnt)
1366 enetc_update_tx_ring_tail(tx_ring);
1367
1368 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1369 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1370 rx_ring->xdp.xdp_tx_in_flight);
1371
1372 return rx_frm_cnt;
1373 }
1374
enetc_poll(struct napi_struct * napi,int budget)1375 static int enetc_poll(struct napi_struct *napi, int budget)
1376 {
1377 struct enetc_int_vector
1378 *v = container_of(napi, struct enetc_int_vector, napi);
1379 struct enetc_bdr *rx_ring = &v->rx_ring;
1380 struct bpf_prog *prog;
1381 bool complete = true;
1382 int work_done;
1383 int i;
1384
1385 enetc_lock_mdio();
1386
1387 for (i = 0; i < v->count_tx_rings; i++)
1388 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1389 complete = false;
1390
1391 prog = rx_ring->xdp.prog;
1392 if (prog)
1393 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1394 else
1395 work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1396 if (work_done == budget)
1397 complete = false;
1398 if (work_done)
1399 v->rx_napi_work = true;
1400
1401 if (!complete) {
1402 enetc_unlock_mdio();
1403 return budget;
1404 }
1405
1406 napi_complete_done(napi, work_done);
1407
1408 if (likely(v->rx_dim_en))
1409 enetc_rx_net_dim(v);
1410
1411 v->rx_napi_work = false;
1412
1413 /* enable interrupts */
1414 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1415
1416 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1417 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1418 ENETC_TBIER_TXTIE);
1419
1420 enetc_unlock_mdio();
1421
1422 return work_done;
1423 }
1424
1425 /* Probing and Init */
1426 #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)1427 void enetc_get_si_caps(struct enetc_si *si)
1428 {
1429 struct enetc_hw *hw = &si->hw;
1430 u32 val;
1431
1432 /* find out how many of various resources we have to work with */
1433 val = enetc_rd(hw, ENETC_SICAPR0);
1434 si->num_rx_rings = (val >> 16) & 0xff;
1435 si->num_tx_rings = val & 0xff;
1436
1437 val = enetc_rd(hw, ENETC_SIRFSCAPR);
1438 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1439 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1440
1441 si->num_rss = 0;
1442 val = enetc_rd(hw, ENETC_SIPCAPR0);
1443 if (val & ENETC_SIPCAPR0_RSS) {
1444 u32 rss;
1445
1446 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1447 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1448 }
1449
1450 if (val & ENETC_SIPCAPR0_QBV)
1451 si->hw_features |= ENETC_SI_F_QBV;
1452
1453 if (val & ENETC_SIPCAPR0_PSFP)
1454 si->hw_features |= ENETC_SI_F_PSFP;
1455 }
1456
enetc_dma_alloc_bdr(struct enetc_bdr * r,size_t bd_size)1457 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
1458 {
1459 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
1460 &r->bd_dma_base, GFP_KERNEL);
1461 if (!r->bd_base)
1462 return -ENOMEM;
1463
1464 /* h/w requires 128B alignment */
1465 if (!IS_ALIGNED(r->bd_dma_base, 128)) {
1466 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
1467 r->bd_dma_base);
1468 return -EINVAL;
1469 }
1470
1471 return 0;
1472 }
1473
enetc_alloc_txbdr(struct enetc_bdr * txr)1474 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
1475 {
1476 int err;
1477
1478 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
1479 if (!txr->tx_swbd)
1480 return -ENOMEM;
1481
1482 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
1483 if (err) {
1484 vfree(txr->tx_swbd);
1485 return err;
1486 }
1487
1488 txr->next_to_clean = 0;
1489 txr->next_to_use = 0;
1490
1491 return 0;
1492 }
1493
enetc_free_txbdr(struct enetc_bdr * txr)1494 static void enetc_free_txbdr(struct enetc_bdr *txr)
1495 {
1496 int size, i;
1497
1498 for (i = 0; i < txr->bd_count; i++)
1499 enetc_free_tx_frame(txr, &txr->tx_swbd[i]);
1500
1501 size = txr->bd_count * sizeof(union enetc_tx_bd);
1502
1503 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
1504 txr->bd_base = NULL;
1505
1506 vfree(txr->tx_swbd);
1507 txr->tx_swbd = NULL;
1508 }
1509
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)1510 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1511 {
1512 int i, err;
1513
1514 for (i = 0; i < priv->num_tx_rings; i++) {
1515 err = enetc_alloc_txbdr(priv->tx_ring[i]);
1516
1517 if (err)
1518 goto fail;
1519 }
1520
1521 return 0;
1522
1523 fail:
1524 while (i-- > 0)
1525 enetc_free_txbdr(priv->tx_ring[i]);
1526
1527 return err;
1528 }
1529
enetc_free_tx_resources(struct enetc_ndev_priv * priv)1530 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
1531 {
1532 int i;
1533
1534 for (i = 0; i < priv->num_tx_rings; i++)
1535 enetc_free_txbdr(priv->tx_ring[i]);
1536 }
1537
enetc_alloc_rxbdr(struct enetc_bdr * rxr,bool extended)1538 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
1539 {
1540 size_t size = sizeof(union enetc_rx_bd);
1541 int err;
1542
1543 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
1544 if (!rxr->rx_swbd)
1545 return -ENOMEM;
1546
1547 if (extended)
1548 size *= 2;
1549
1550 err = enetc_dma_alloc_bdr(rxr, size);
1551 if (err) {
1552 vfree(rxr->rx_swbd);
1553 return err;
1554 }
1555
1556 rxr->next_to_clean = 0;
1557 rxr->next_to_use = 0;
1558 rxr->next_to_alloc = 0;
1559 rxr->ext_en = extended;
1560
1561 return 0;
1562 }
1563
enetc_free_rxbdr(struct enetc_bdr * rxr)1564 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
1565 {
1566 int size;
1567
1568 size = rxr->bd_count * sizeof(union enetc_rx_bd);
1569
1570 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
1571 rxr->bd_base = NULL;
1572
1573 vfree(rxr->rx_swbd);
1574 rxr->rx_swbd = NULL;
1575 }
1576
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv)1577 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
1578 {
1579 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
1580 int i, err;
1581
1582 for (i = 0; i < priv->num_rx_rings; i++) {
1583 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
1584
1585 if (err)
1586 goto fail;
1587 }
1588
1589 return 0;
1590
1591 fail:
1592 while (i-- > 0)
1593 enetc_free_rxbdr(priv->rx_ring[i]);
1594
1595 return err;
1596 }
1597
enetc_free_rx_resources(struct enetc_ndev_priv * priv)1598 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
1599 {
1600 int i;
1601
1602 for (i = 0; i < priv->num_rx_rings; i++)
1603 enetc_free_rxbdr(priv->rx_ring[i]);
1604 }
1605
enetc_free_tx_ring(struct enetc_bdr * tx_ring)1606 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1607 {
1608 int i;
1609
1610 if (!tx_ring->tx_swbd)
1611 return;
1612
1613 for (i = 0; i < tx_ring->bd_count; i++) {
1614 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1615
1616 enetc_free_tx_frame(tx_ring, tx_swbd);
1617 }
1618
1619 tx_ring->next_to_clean = 0;
1620 tx_ring->next_to_use = 0;
1621 }
1622
enetc_free_rx_ring(struct enetc_bdr * rx_ring)1623 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1624 {
1625 int i;
1626
1627 if (!rx_ring->rx_swbd)
1628 return;
1629
1630 for (i = 0; i < rx_ring->bd_count; i++) {
1631 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1632
1633 if (!rx_swbd->page)
1634 continue;
1635
1636 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1637 rx_swbd->dir);
1638 __free_page(rx_swbd->page);
1639 rx_swbd->page = NULL;
1640 }
1641
1642 rx_ring->next_to_clean = 0;
1643 rx_ring->next_to_use = 0;
1644 rx_ring->next_to_alloc = 0;
1645 }
1646
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)1647 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1648 {
1649 int i;
1650
1651 for (i = 0; i < priv->num_rx_rings; i++)
1652 enetc_free_rx_ring(priv->rx_ring[i]);
1653
1654 for (i = 0; i < priv->num_tx_rings; i++)
1655 enetc_free_tx_ring(priv->tx_ring[i]);
1656 }
1657
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)1658 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1659 {
1660 int *rss_table;
1661 int i;
1662
1663 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1664 if (!rss_table)
1665 return -ENOMEM;
1666
1667 /* Set up RSS table defaults */
1668 for (i = 0; i < si->num_rss; i++)
1669 rss_table[i] = i % num_groups;
1670
1671 enetc_set_rss_table(si, rss_table, si->num_rss);
1672
1673 kfree(rss_table);
1674
1675 return 0;
1676 }
1677
enetc_configure_si(struct enetc_ndev_priv * priv)1678 int enetc_configure_si(struct enetc_ndev_priv *priv)
1679 {
1680 struct enetc_si *si = priv->si;
1681 struct enetc_hw *hw = &si->hw;
1682 int err;
1683
1684 /* set SI cache attributes */
1685 enetc_wr(hw, ENETC_SICAR0,
1686 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1687 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1688 /* enable SI */
1689 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1690
1691 if (si->num_rss) {
1692 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1693 if (err)
1694 return err;
1695 }
1696
1697 return 0;
1698 }
1699
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)1700 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1701 {
1702 struct enetc_si *si = priv->si;
1703 int cpus = num_online_cpus();
1704
1705 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1706 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1707
1708 /* Enable all available TX rings in order to configure as many
1709 * priorities as possible, when needed.
1710 * TODO: Make # of TX rings run-time configurable
1711 */
1712 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1713 priv->num_tx_rings = si->num_tx_rings;
1714 priv->bdr_int_num = cpus;
1715 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
1716 priv->tx_ictt = ENETC_TXIC_TIMETHR;
1717 }
1718
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)1719 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1720 {
1721 struct enetc_si *si = priv->si;
1722
1723 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1724 GFP_KERNEL);
1725 if (!priv->cls_rules)
1726 return -ENOMEM;
1727
1728 return 0;
1729 }
1730
enetc_free_si_resources(struct enetc_ndev_priv * priv)1731 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1732 {
1733 kfree(priv->cls_rules);
1734 }
1735
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1736 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1737 {
1738 int idx = tx_ring->index;
1739 u32 tbmr;
1740
1741 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1742 lower_32_bits(tx_ring->bd_dma_base));
1743
1744 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1745 upper_32_bits(tx_ring->bd_dma_base));
1746
1747 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1748 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1749 ENETC_RTBLENR_LEN(tx_ring->bd_count));
1750
1751 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1752 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1753 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1754
1755 /* enable Tx ints by setting pkt thr to 1 */
1756 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
1757
1758 tbmr = ENETC_TBMR_EN | ENETC_TBMR_SET_PRIO(tx_ring->prio);
1759 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1760 tbmr |= ENETC_TBMR_VIH;
1761
1762 /* enable ring */
1763 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1764
1765 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1766 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1767 tx_ring->idr = hw->reg + ENETC_SITXIDR;
1768 }
1769
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1770 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1771 {
1772 int idx = rx_ring->index;
1773 u32 rbmr;
1774
1775 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1776 lower_32_bits(rx_ring->bd_dma_base));
1777
1778 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1779 upper_32_bits(rx_ring->bd_dma_base));
1780
1781 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1782 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1783 ENETC_RTBLENR_LEN(rx_ring->bd_count));
1784
1785 if (rx_ring->xdp.prog)
1786 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
1787 else
1788 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1789
1790 /* Also prepare the consumer index in case page allocation never
1791 * succeeds. In that case, hardware will never advance producer index
1792 * to match consumer index, and will drop all frames.
1793 */
1794 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1795 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
1796
1797 /* enable Rx ints by setting pkt thr to 1 */
1798 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
1799
1800 rbmr = ENETC_RBMR_EN;
1801
1802 if (rx_ring->ext_en)
1803 rbmr |= ENETC_RBMR_BDS;
1804
1805 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1806 rbmr |= ENETC_RBMR_VTE;
1807
1808 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1809 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1810
1811 enetc_lock_mdio();
1812 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1813 enetc_unlock_mdio();
1814
1815 /* enable ring */
1816 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1817 }
1818
enetc_setup_bdrs(struct enetc_ndev_priv * priv)1819 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1820 {
1821 struct enetc_hw *hw = &priv->si->hw;
1822 int i;
1823
1824 for (i = 0; i < priv->num_tx_rings; i++)
1825 enetc_setup_txbdr(hw, priv->tx_ring[i]);
1826
1827 for (i = 0; i < priv->num_rx_rings; i++)
1828 enetc_setup_rxbdr(hw, priv->rx_ring[i]);
1829 }
1830
enetc_clear_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1831 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1832 {
1833 int idx = rx_ring->index;
1834
1835 /* disable EN bit on ring */
1836 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1837 }
1838
enetc_clear_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1839 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1840 {
1841 int delay = 8, timeout = 100;
1842 int idx = tx_ring->index;
1843
1844 /* disable EN bit on ring */
1845 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1846
1847 /* wait for busy to clear */
1848 while (delay < timeout &&
1849 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1850 msleep(delay);
1851 delay *= 2;
1852 }
1853
1854 if (delay >= timeout)
1855 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1856 idx);
1857 }
1858
enetc_clear_bdrs(struct enetc_ndev_priv * priv)1859 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1860 {
1861 struct enetc_hw *hw = &priv->si->hw;
1862 int i;
1863
1864 for (i = 0; i < priv->num_tx_rings; i++)
1865 enetc_clear_txbdr(hw, priv->tx_ring[i]);
1866
1867 for (i = 0; i < priv->num_rx_rings; i++)
1868 enetc_clear_rxbdr(hw, priv->rx_ring[i]);
1869
1870 udelay(1);
1871 }
1872
enetc_setup_irqs(struct enetc_ndev_priv * priv)1873 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1874 {
1875 struct pci_dev *pdev = priv->si->pdev;
1876 struct enetc_hw *hw = &priv->si->hw;
1877 int i, j, err;
1878
1879 for (i = 0; i < priv->bdr_int_num; i++) {
1880 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1881 struct enetc_int_vector *v = priv->int_vector[i];
1882 int entry = ENETC_BDR_INT_BASE_IDX + i;
1883
1884 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1885 priv->ndev->name, i);
1886 err = request_irq(irq, enetc_msix, 0, v->name, v);
1887 if (err) {
1888 dev_err(priv->dev, "request_irq() failed!\n");
1889 goto irq_err;
1890 }
1891 disable_irq(irq);
1892
1893 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1894 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1895 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
1896
1897 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1898
1899 for (j = 0; j < v->count_tx_rings; j++) {
1900 int idx = v->tx_ring[j].index;
1901
1902 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1903 }
1904 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
1905 }
1906
1907 return 0;
1908
1909 irq_err:
1910 while (i--) {
1911 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1912
1913 irq_set_affinity_hint(irq, NULL);
1914 free_irq(irq, priv->int_vector[i]);
1915 }
1916
1917 return err;
1918 }
1919
enetc_free_irqs(struct enetc_ndev_priv * priv)1920 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1921 {
1922 struct pci_dev *pdev = priv->si->pdev;
1923 int i;
1924
1925 for (i = 0; i < priv->bdr_int_num; i++) {
1926 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1927
1928 irq_set_affinity_hint(irq, NULL);
1929 free_irq(irq, priv->int_vector[i]);
1930 }
1931 }
1932
enetc_setup_interrupts(struct enetc_ndev_priv * priv)1933 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
1934 {
1935 struct enetc_hw *hw = &priv->si->hw;
1936 u32 icpt, ictt;
1937 int i;
1938
1939 /* enable Tx & Rx event indication */
1940 if (priv->ic_mode &
1941 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
1942 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
1943 /* init to non-0 minimum, will be adjusted later */
1944 ictt = 0x1;
1945 } else {
1946 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
1947 ictt = 0;
1948 }
1949
1950 for (i = 0; i < priv->num_rx_rings; i++) {
1951 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
1952 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
1953 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
1954 }
1955
1956 if (priv->ic_mode & ENETC_IC_TX_MANUAL)
1957 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
1958 else
1959 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
1960
1961 for (i = 0; i < priv->num_tx_rings; i++) {
1962 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
1963 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
1964 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
1965 }
1966 }
1967
enetc_clear_interrupts(struct enetc_ndev_priv * priv)1968 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
1969 {
1970 struct enetc_hw *hw = &priv->si->hw;
1971 int i;
1972
1973 for (i = 0; i < priv->num_tx_rings; i++)
1974 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
1975
1976 for (i = 0; i < priv->num_rx_rings; i++)
1977 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
1978 }
1979
enetc_phylink_connect(struct net_device * ndev)1980 static int enetc_phylink_connect(struct net_device *ndev)
1981 {
1982 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1983 struct ethtool_eee edata;
1984 int err;
1985
1986 if (!priv->phylink)
1987 return 0; /* phy-less mode */
1988
1989 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
1990 if (err) {
1991 dev_err(&ndev->dev, "could not attach to PHY\n");
1992 return err;
1993 }
1994
1995 /* disable EEE autoneg, until ENETC driver supports it */
1996 memset(&edata, 0, sizeof(struct ethtool_eee));
1997 phylink_ethtool_set_eee(priv->phylink, &edata);
1998
1999 return 0;
2000 }
2001
enetc_tx_onestep_tstamp(struct work_struct * work)2002 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2003 {
2004 struct enetc_ndev_priv *priv;
2005 struct sk_buff *skb;
2006
2007 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2008
2009 netif_tx_lock_bh(priv->ndev);
2010
2011 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2012 skb = skb_dequeue(&priv->tx_skbs);
2013 if (skb)
2014 enetc_start_xmit(skb, priv->ndev);
2015
2016 netif_tx_unlock_bh(priv->ndev);
2017 }
2018
enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv * priv)2019 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2020 {
2021 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2022 skb_queue_head_init(&priv->tx_skbs);
2023 }
2024
enetc_start(struct net_device * ndev)2025 void enetc_start(struct net_device *ndev)
2026 {
2027 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2028 int i;
2029
2030 enetc_setup_interrupts(priv);
2031
2032 for (i = 0; i < priv->bdr_int_num; i++) {
2033 int irq = pci_irq_vector(priv->si->pdev,
2034 ENETC_BDR_INT_BASE_IDX + i);
2035
2036 napi_enable(&priv->int_vector[i]->napi);
2037 enable_irq(irq);
2038 }
2039
2040 if (priv->phylink)
2041 phylink_start(priv->phylink);
2042 else
2043 netif_carrier_on(ndev);
2044
2045 netif_tx_start_all_queues(ndev);
2046 }
2047
enetc_open(struct net_device * ndev)2048 int enetc_open(struct net_device *ndev)
2049 {
2050 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2051 int num_stack_tx_queues;
2052 int err;
2053
2054 err = enetc_setup_irqs(priv);
2055 if (err)
2056 return err;
2057
2058 err = enetc_phylink_connect(ndev);
2059 if (err)
2060 goto err_phy_connect;
2061
2062 err = enetc_alloc_tx_resources(priv);
2063 if (err)
2064 goto err_alloc_tx;
2065
2066 err = enetc_alloc_rx_resources(priv);
2067 if (err)
2068 goto err_alloc_rx;
2069
2070 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2071
2072 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2073 if (err)
2074 goto err_set_queues;
2075
2076 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
2077 if (err)
2078 goto err_set_queues;
2079
2080 enetc_tx_onestep_tstamp_init(priv);
2081 enetc_setup_bdrs(priv);
2082 enetc_start(ndev);
2083
2084 return 0;
2085
2086 err_set_queues:
2087 enetc_free_rx_resources(priv);
2088 err_alloc_rx:
2089 enetc_free_tx_resources(priv);
2090 err_alloc_tx:
2091 if (priv->phylink)
2092 phylink_disconnect_phy(priv->phylink);
2093 err_phy_connect:
2094 enetc_free_irqs(priv);
2095
2096 return err;
2097 }
2098
enetc_stop(struct net_device * ndev)2099 void enetc_stop(struct net_device *ndev)
2100 {
2101 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2102 int i;
2103
2104 netif_tx_stop_all_queues(ndev);
2105
2106 for (i = 0; i < priv->bdr_int_num; i++) {
2107 int irq = pci_irq_vector(priv->si->pdev,
2108 ENETC_BDR_INT_BASE_IDX + i);
2109
2110 disable_irq(irq);
2111 napi_synchronize(&priv->int_vector[i]->napi);
2112 napi_disable(&priv->int_vector[i]->napi);
2113 }
2114
2115 if (priv->phylink)
2116 phylink_stop(priv->phylink);
2117 else
2118 netif_carrier_off(ndev);
2119
2120 enetc_clear_interrupts(priv);
2121 }
2122
enetc_close(struct net_device * ndev)2123 int enetc_close(struct net_device *ndev)
2124 {
2125 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2126
2127 enetc_stop(ndev);
2128 enetc_clear_bdrs(priv);
2129
2130 if (priv->phylink)
2131 phylink_disconnect_phy(priv->phylink);
2132 enetc_free_rxtx_rings(priv);
2133 enetc_free_rx_resources(priv);
2134 enetc_free_tx_resources(priv);
2135 enetc_free_irqs(priv);
2136
2137 return 0;
2138 }
2139
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)2140 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2141 {
2142 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2143 struct tc_mqprio_qopt *mqprio = type_data;
2144 struct enetc_hw *hw = &priv->si->hw;
2145 struct enetc_bdr *tx_ring;
2146 int num_stack_tx_queues;
2147 u8 num_tc;
2148 int i;
2149
2150 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2151 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
2152 num_tc = mqprio->num_tc;
2153
2154 if (!num_tc) {
2155 netdev_reset_tc(ndev);
2156 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2157
2158 /* Reset all ring priorities to 0 */
2159 for (i = 0; i < priv->num_tx_rings; i++) {
2160 tx_ring = priv->tx_ring[i];
2161 tx_ring->prio = 0;
2162 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2163 }
2164
2165 return 0;
2166 }
2167
2168 /* Check if we have enough BD rings available to accommodate all TCs */
2169 if (num_tc > num_stack_tx_queues) {
2170 netdev_err(ndev, "Max %d traffic classes supported\n",
2171 priv->num_tx_rings);
2172 return -EINVAL;
2173 }
2174
2175 /* For the moment, we use only one BD ring per TC.
2176 *
2177 * Configure num_tc BD rings with increasing priorities.
2178 */
2179 for (i = 0; i < num_tc; i++) {
2180 tx_ring = priv->tx_ring[i];
2181 tx_ring->prio = i;
2182 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2183 }
2184
2185 /* Reset the number of netdev queues based on the TC count */
2186 netif_set_real_num_tx_queues(ndev, num_tc);
2187
2188 netdev_set_num_tc(ndev, num_tc);
2189
2190 /* Each TC is associated with one netdev queue */
2191 for (i = 0; i < num_tc; i++)
2192 netdev_set_tc_queue(ndev, i, 1, i);
2193
2194 return 0;
2195 }
2196
enetc_setup_xdp_prog(struct net_device * dev,struct bpf_prog * prog,struct netlink_ext_ack * extack)2197 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog,
2198 struct netlink_ext_ack *extack)
2199 {
2200 struct enetc_ndev_priv *priv = netdev_priv(dev);
2201 struct bpf_prog *old_prog;
2202 bool is_up;
2203 int i;
2204
2205 /* The buffer layout is changing, so we need to drain the old
2206 * RX buffers and seed new ones.
2207 */
2208 is_up = netif_running(dev);
2209 if (is_up)
2210 dev_close(dev);
2211
2212 old_prog = xchg(&priv->xdp_prog, prog);
2213 if (old_prog)
2214 bpf_prog_put(old_prog);
2215
2216 for (i = 0; i < priv->num_rx_rings; i++) {
2217 struct enetc_bdr *rx_ring = priv->rx_ring[i];
2218
2219 rx_ring->xdp.prog = prog;
2220
2221 if (prog)
2222 rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2223 else
2224 rx_ring->buffer_offset = ENETC_RXB_PAD;
2225 }
2226
2227 if (is_up)
2228 return dev_open(dev, extack);
2229
2230 return 0;
2231 }
2232
enetc_setup_bpf(struct net_device * dev,struct netdev_bpf * xdp)2233 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp)
2234 {
2235 switch (xdp->command) {
2236 case XDP_SETUP_PROG:
2237 return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack);
2238 default:
2239 return -EINVAL;
2240 }
2241
2242 return 0;
2243 }
2244
enetc_get_stats(struct net_device * ndev)2245 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2246 {
2247 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2248 struct net_device_stats *stats = &ndev->stats;
2249 unsigned long packets = 0, bytes = 0;
2250 int i;
2251
2252 for (i = 0; i < priv->num_rx_rings; i++) {
2253 packets += priv->rx_ring[i]->stats.packets;
2254 bytes += priv->rx_ring[i]->stats.bytes;
2255 }
2256
2257 stats->rx_packets = packets;
2258 stats->rx_bytes = bytes;
2259 bytes = 0;
2260 packets = 0;
2261
2262 for (i = 0; i < priv->num_tx_rings; i++) {
2263 packets += priv->tx_ring[i]->stats.packets;
2264 bytes += priv->tx_ring[i]->stats.bytes;
2265 }
2266
2267 stats->tx_packets = packets;
2268 stats->tx_bytes = bytes;
2269
2270 return stats;
2271 }
2272
enetc_set_rss(struct net_device * ndev,int en)2273 static int enetc_set_rss(struct net_device *ndev, int en)
2274 {
2275 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2276 struct enetc_hw *hw = &priv->si->hw;
2277 u32 reg;
2278
2279 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2280
2281 reg = enetc_rd(hw, ENETC_SIMR);
2282 reg &= ~ENETC_SIMR_RSSE;
2283 reg |= (en) ? ENETC_SIMR_RSSE : 0;
2284 enetc_wr(hw, ENETC_SIMR, reg);
2285
2286 return 0;
2287 }
2288
enetc_enable_rxvlan(struct net_device * ndev,bool en)2289 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2290 {
2291 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2292 struct enetc_hw *hw = &priv->si->hw;
2293 int i;
2294
2295 for (i = 0; i < priv->num_rx_rings; i++)
2296 enetc_bdr_enable_rxvlan(hw, i, en);
2297 }
2298
enetc_enable_txvlan(struct net_device * ndev,bool en)2299 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2300 {
2301 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2302 struct enetc_hw *hw = &priv->si->hw;
2303 int i;
2304
2305 for (i = 0; i < priv->num_tx_rings; i++)
2306 enetc_bdr_enable_txvlan(hw, i, en);
2307 }
2308
enetc_set_features(struct net_device * ndev,netdev_features_t features)2309 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2310 {
2311 netdev_features_t changed = ndev->features ^ features;
2312
2313 if (changed & NETIF_F_RXHASH)
2314 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2315
2316 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2317 enetc_enable_rxvlan(ndev,
2318 !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2319
2320 if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2321 enetc_enable_txvlan(ndev,
2322 !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2323 }
2324
2325 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)2326 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2327 {
2328 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2329 struct hwtstamp_config config;
2330 int ao;
2331
2332 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2333 return -EFAULT;
2334
2335 switch (config.tx_type) {
2336 case HWTSTAMP_TX_OFF:
2337 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2338 break;
2339 case HWTSTAMP_TX_ON:
2340 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2341 priv->active_offloads |= ENETC_F_TX_TSTAMP;
2342 break;
2343 case HWTSTAMP_TX_ONESTEP_SYNC:
2344 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2345 priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2346 break;
2347 default:
2348 return -ERANGE;
2349 }
2350
2351 ao = priv->active_offloads;
2352 switch (config.rx_filter) {
2353 case HWTSTAMP_FILTER_NONE:
2354 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
2355 break;
2356 default:
2357 priv->active_offloads |= ENETC_F_RX_TSTAMP;
2358 config.rx_filter = HWTSTAMP_FILTER_ALL;
2359 }
2360
2361 if (netif_running(ndev) && ao != priv->active_offloads) {
2362 enetc_close(ndev);
2363 enetc_open(ndev);
2364 }
2365
2366 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2367 -EFAULT : 0;
2368 }
2369
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)2370 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2371 {
2372 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2373 struct hwtstamp_config config;
2374
2375 config.flags = 0;
2376
2377 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2378 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2379 else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2380 config.tx_type = HWTSTAMP_TX_ON;
2381 else
2382 config.tx_type = HWTSTAMP_TX_OFF;
2383
2384 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2385 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2386
2387 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2388 -EFAULT : 0;
2389 }
2390 #endif
2391
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)2392 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2393 {
2394 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2395 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2396 if (cmd == SIOCSHWTSTAMP)
2397 return enetc_hwtstamp_set(ndev, rq);
2398 if (cmd == SIOCGHWTSTAMP)
2399 return enetc_hwtstamp_get(ndev, rq);
2400 #endif
2401
2402 if (!priv->phylink)
2403 return -EOPNOTSUPP;
2404
2405 return phylink_mii_ioctl(priv->phylink, rq, cmd);
2406 }
2407
enetc_alloc_msix(struct enetc_ndev_priv * priv)2408 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2409 {
2410 struct pci_dev *pdev = priv->si->pdev;
2411 int first_xdp_tx_ring;
2412 int i, n, err, nvec;
2413 int v_tx_rings;
2414
2415 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2416 /* allocate MSIX for both messaging and Rx/Tx interrupts */
2417 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2418
2419 if (n < 0)
2420 return n;
2421
2422 if (n != nvec)
2423 return -EPERM;
2424
2425 /* # of tx rings per int vector */
2426 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2427
2428 for (i = 0; i < priv->bdr_int_num; i++) {
2429 struct enetc_int_vector *v;
2430 struct enetc_bdr *bdr;
2431 int j;
2432
2433 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
2434 if (!v) {
2435 err = -ENOMEM;
2436 goto fail;
2437 }
2438
2439 priv->int_vector[i] = v;
2440
2441 bdr = &v->rx_ring;
2442 bdr->index = i;
2443 bdr->ndev = priv->ndev;
2444 bdr->dev = priv->dev;
2445 bdr->bd_count = priv->rx_bd_count;
2446 bdr->buffer_offset = ENETC_RXB_PAD;
2447 priv->rx_ring[i] = bdr;
2448
2449 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
2450 if (err) {
2451 kfree(v);
2452 goto fail;
2453 }
2454
2455 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
2456 MEM_TYPE_PAGE_SHARED, NULL);
2457 if (err) {
2458 xdp_rxq_info_unreg(&bdr->xdp.rxq);
2459 kfree(v);
2460 goto fail;
2461 }
2462
2463 /* init defaults for adaptive IC */
2464 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
2465 v->rx_ictt = 0x1;
2466 v->rx_dim_en = true;
2467 }
2468 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
2469 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
2470 NAPI_POLL_WEIGHT);
2471 v->count_tx_rings = v_tx_rings;
2472
2473 for (j = 0; j < v_tx_rings; j++) {
2474 int idx;
2475
2476 /* default tx ring mapping policy */
2477 idx = priv->bdr_int_num * j + i;
2478 __set_bit(idx, &v->tx_rings_map);
2479 bdr = &v->tx_ring[j];
2480 bdr->index = idx;
2481 bdr->ndev = priv->ndev;
2482 bdr->dev = priv->dev;
2483 bdr->bd_count = priv->tx_bd_count;
2484 priv->tx_ring[idx] = bdr;
2485 }
2486 }
2487
2488 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
2489 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
2490
2491 return 0;
2492
2493 fail:
2494 while (i--) {
2495 struct enetc_int_vector *v = priv->int_vector[i];
2496 struct enetc_bdr *rx_ring = &v->rx_ring;
2497
2498 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2499 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2500 netif_napi_del(&v->napi);
2501 cancel_work_sync(&v->rx_dim.work);
2502 kfree(v);
2503 }
2504
2505 pci_free_irq_vectors(pdev);
2506
2507 return err;
2508 }
2509
enetc_free_msix(struct enetc_ndev_priv * priv)2510 void enetc_free_msix(struct enetc_ndev_priv *priv)
2511 {
2512 int i;
2513
2514 for (i = 0; i < priv->bdr_int_num; i++) {
2515 struct enetc_int_vector *v = priv->int_vector[i];
2516 struct enetc_bdr *rx_ring = &v->rx_ring;
2517
2518 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2519 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2520 netif_napi_del(&v->napi);
2521 cancel_work_sync(&v->rx_dim.work);
2522 }
2523
2524 for (i = 0; i < priv->num_rx_rings; i++)
2525 priv->rx_ring[i] = NULL;
2526
2527 for (i = 0; i < priv->num_tx_rings; i++)
2528 priv->tx_ring[i] = NULL;
2529
2530 for (i = 0; i < priv->bdr_int_num; i++) {
2531 kfree(priv->int_vector[i]);
2532 priv->int_vector[i] = NULL;
2533 }
2534
2535 /* disable all MSIX for this device */
2536 pci_free_irq_vectors(priv->si->pdev);
2537 }
2538
enetc_kfree_si(struct enetc_si * si)2539 static void enetc_kfree_si(struct enetc_si *si)
2540 {
2541 char *p = (char *)si - si->pad;
2542
2543 kfree(p);
2544 }
2545
enetc_detect_errata(struct enetc_si * si)2546 static void enetc_detect_errata(struct enetc_si *si)
2547 {
2548 if (si->pdev->revision == ENETC_REV1)
2549 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
2550 }
2551
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)2552 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
2553 {
2554 struct enetc_si *si, *p;
2555 struct enetc_hw *hw;
2556 size_t alloc_size;
2557 int err, len;
2558
2559 pcie_flr(pdev);
2560 err = pci_enable_device_mem(pdev);
2561 if (err) {
2562 dev_err(&pdev->dev, "device enable failed\n");
2563 return err;
2564 }
2565
2566 /* set up for high or low dma */
2567 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2568 if (err) {
2569 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2570 if (err) {
2571 dev_err(&pdev->dev,
2572 "DMA configuration failed: 0x%x\n", err);
2573 goto err_dma;
2574 }
2575 }
2576
2577 err = pci_request_mem_regions(pdev, name);
2578 if (err) {
2579 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
2580 goto err_pci_mem_reg;
2581 }
2582
2583 pci_set_master(pdev);
2584
2585 alloc_size = sizeof(struct enetc_si);
2586 if (sizeof_priv) {
2587 /* align priv to 32B */
2588 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
2589 alloc_size += sizeof_priv;
2590 }
2591 /* force 32B alignment for enetc_si */
2592 alloc_size += ENETC_SI_ALIGN - 1;
2593
2594 p = kzalloc(alloc_size, GFP_KERNEL);
2595 if (!p) {
2596 err = -ENOMEM;
2597 goto err_alloc_si;
2598 }
2599
2600 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
2601 si->pad = (char *)si - (char *)p;
2602
2603 pci_set_drvdata(pdev, si);
2604 si->pdev = pdev;
2605 hw = &si->hw;
2606
2607 len = pci_resource_len(pdev, ENETC_BAR_REGS);
2608 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
2609 if (!hw->reg) {
2610 err = -ENXIO;
2611 dev_err(&pdev->dev, "ioremap() failed\n");
2612 goto err_ioremap;
2613 }
2614 if (len > ENETC_PORT_BASE)
2615 hw->port = hw->reg + ENETC_PORT_BASE;
2616 if (len > ENETC_GLOBAL_BASE)
2617 hw->global = hw->reg + ENETC_GLOBAL_BASE;
2618
2619 enetc_detect_errata(si);
2620
2621 return 0;
2622
2623 err_ioremap:
2624 enetc_kfree_si(si);
2625 err_alloc_si:
2626 pci_release_mem_regions(pdev);
2627 err_pci_mem_reg:
2628 err_dma:
2629 pci_disable_device(pdev);
2630
2631 return err;
2632 }
2633
enetc_pci_remove(struct pci_dev * pdev)2634 void enetc_pci_remove(struct pci_dev *pdev)
2635 {
2636 struct enetc_si *si = pci_get_drvdata(pdev);
2637 struct enetc_hw *hw = &si->hw;
2638
2639 iounmap(hw->reg);
2640 enetc_kfree_si(si);
2641 pci_release_mem_regions(pdev);
2642 pci_disable_device(pdev);
2643 }
2644