1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3
4 #include "enetc.h"
5 #include <linux/tcp.h>
6 #include <linux/udp.h>
7 #include <linux/vmalloc.h>
8 #include <net/pkt_sched.h>
9
10 /* ENETC overhead: optional extension BD + 1 BD gap */
11 #define ENETC_TXBDS_NEEDED(val) ((val) + 2)
12 /* max # of chained Tx BDs is 15, including head and extension BD */
13 #define ENETC_MAX_SKB_FRAGS 13
14 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
15
16 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
17 int active_offloads);
18
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)19 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
20 {
21 struct enetc_ndev_priv *priv = netdev_priv(ndev);
22 struct enetc_bdr *tx_ring;
23 int count;
24
25 tx_ring = priv->tx_ring[skb->queue_mapping];
26
27 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
28 if (unlikely(skb_linearize(skb)))
29 goto drop_packet_err;
30
31 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
32 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
33 netif_stop_subqueue(ndev, tx_ring->index);
34 return NETDEV_TX_BUSY;
35 }
36
37 enetc_lock_mdio();
38 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
39 enetc_unlock_mdio();
40
41 if (unlikely(!count))
42 goto drop_packet_err;
43
44 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
45 netif_stop_subqueue(ndev, tx_ring->index);
46
47 return NETDEV_TX_OK;
48
49 drop_packet_err:
50 dev_kfree_skb_any(skb);
51 return NETDEV_TX_OK;
52 }
53
enetc_tx_csum(struct sk_buff * skb,union enetc_tx_bd * txbd)54 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
55 {
56 int l3_start, l3_hsize;
57 u16 l3_flags, l4_flags;
58
59 if (skb->ip_summed != CHECKSUM_PARTIAL)
60 return false;
61
62 switch (skb->csum_offset) {
63 case offsetof(struct tcphdr, check):
64 l4_flags = ENETC_TXBD_L4_TCP;
65 break;
66 case offsetof(struct udphdr, check):
67 l4_flags = ENETC_TXBD_L4_UDP;
68 break;
69 default:
70 skb_checksum_help(skb);
71 return false;
72 }
73
74 l3_start = skb_network_offset(skb);
75 l3_hsize = skb_network_header_len(skb);
76
77 l3_flags = 0;
78 if (skb->protocol == htons(ETH_P_IPV6))
79 l3_flags = ENETC_TXBD_L3_IPV6;
80
81 /* write BD fields */
82 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
83 txbd->l4_csoff = l4_flags;
84
85 return true;
86 }
87
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)88 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
89 struct enetc_tx_swbd *tx_swbd)
90 {
91 if (tx_swbd->is_dma_page)
92 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
93 tx_swbd->len, DMA_TO_DEVICE);
94 else
95 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
96 tx_swbd->len, DMA_TO_DEVICE);
97 tx_swbd->dma = 0;
98 }
99
enetc_free_tx_skb(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)100 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
101 struct enetc_tx_swbd *tx_swbd)
102 {
103 if (tx_swbd->dma)
104 enetc_unmap_tx_buff(tx_ring, tx_swbd);
105
106 if (tx_swbd->skb) {
107 dev_kfree_skb_any(tx_swbd->skb);
108 tx_swbd->skb = NULL;
109 }
110 }
111
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb,int active_offloads)112 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
113 int active_offloads)
114 {
115 struct enetc_tx_swbd *tx_swbd;
116 skb_frag_t *frag;
117 int len = skb_headlen(skb);
118 union enetc_tx_bd temp_bd;
119 union enetc_tx_bd *txbd;
120 bool do_vlan, do_tstamp;
121 int i, count = 0;
122 unsigned int f;
123 dma_addr_t dma;
124 u8 flags = 0;
125
126 i = tx_ring->next_to_use;
127 txbd = ENETC_TXBD(*tx_ring, i);
128 prefetchw(txbd);
129
130 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
131 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
132 goto dma_err;
133
134 temp_bd.addr = cpu_to_le64(dma);
135 temp_bd.buf_len = cpu_to_le16(len);
136 temp_bd.lstatus = 0;
137
138 tx_swbd = &tx_ring->tx_swbd[i];
139 tx_swbd->dma = dma;
140 tx_swbd->len = len;
141 tx_swbd->is_dma_page = 0;
142 count++;
143
144 do_vlan = skb_vlan_tag_present(skb);
145 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
146 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
147 tx_swbd->do_tstamp = do_tstamp;
148 tx_swbd->check_wb = tx_swbd->do_tstamp;
149
150 if (do_vlan || do_tstamp)
151 flags |= ENETC_TXBD_FLAGS_EX;
152
153 if (enetc_tx_csum(skb, &temp_bd))
154 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
155 else if (tx_ring->tsd_enable)
156 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
157
158 /* first BD needs frm_len and offload flags set */
159 temp_bd.frm_len = cpu_to_le16(skb->len);
160 temp_bd.flags = flags;
161
162 if (flags & ENETC_TXBD_FLAGS_TSE) {
163 u32 temp;
164
165 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK)
166 | (flags << ENETC_TXBD_FLAGS_OFFSET);
167 temp_bd.txstart = cpu_to_le32(temp);
168 }
169
170 if (flags & ENETC_TXBD_FLAGS_EX) {
171 u8 e_flags = 0;
172 *txbd = temp_bd;
173 enetc_clear_tx_bd(&temp_bd);
174
175 /* add extension BD for VLAN and/or timestamping */
176 flags = 0;
177 tx_swbd++;
178 txbd++;
179 i++;
180 if (unlikely(i == tx_ring->bd_count)) {
181 i = 0;
182 tx_swbd = tx_ring->tx_swbd;
183 txbd = ENETC_TXBD(*tx_ring, 0);
184 }
185 prefetchw(txbd);
186
187 if (do_vlan) {
188 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
189 temp_bd.ext.tpid = 0; /* < C-TAG */
190 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
191 }
192
193 if (do_tstamp) {
194 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
195 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
196 }
197
198 temp_bd.ext.e_flags = e_flags;
199 count++;
200 }
201
202 frag = &skb_shinfo(skb)->frags[0];
203 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
204 len = skb_frag_size(frag);
205 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
206 DMA_TO_DEVICE);
207 if (dma_mapping_error(tx_ring->dev, dma))
208 goto dma_err;
209
210 *txbd = temp_bd;
211 enetc_clear_tx_bd(&temp_bd);
212
213 flags = 0;
214 tx_swbd++;
215 txbd++;
216 i++;
217 if (unlikely(i == tx_ring->bd_count)) {
218 i = 0;
219 tx_swbd = tx_ring->tx_swbd;
220 txbd = ENETC_TXBD(*tx_ring, 0);
221 }
222 prefetchw(txbd);
223
224 temp_bd.addr = cpu_to_le64(dma);
225 temp_bd.buf_len = cpu_to_le16(len);
226
227 tx_swbd->dma = dma;
228 tx_swbd->len = len;
229 tx_swbd->is_dma_page = 1;
230 count++;
231 }
232
233 /* last BD needs 'F' bit set */
234 flags |= ENETC_TXBD_FLAGS_F;
235 temp_bd.flags = flags;
236 *txbd = temp_bd;
237
238 tx_ring->tx_swbd[i].skb = skb;
239
240 enetc_bdr_idx_inc(tx_ring, &i);
241 tx_ring->next_to_use = i;
242
243 skb_tx_timestamp(skb);
244
245 /* let H/W know BD ring has been updated */
246 enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */
247
248 return count;
249
250 dma_err:
251 dev_err(tx_ring->dev, "DMA map error");
252
253 do {
254 tx_swbd = &tx_ring->tx_swbd[i];
255 enetc_free_tx_skb(tx_ring, tx_swbd);
256 if (i == 0)
257 i = tx_ring->bd_count;
258 i--;
259 } while (count--);
260
261 return 0;
262 }
263
enetc_msix(int irq,void * data)264 static irqreturn_t enetc_msix(int irq, void *data)
265 {
266 struct enetc_int_vector *v = data;
267 int i;
268
269 enetc_lock_mdio();
270
271 /* disable interrupts */
272 enetc_wr_reg_hot(v->rbier, 0);
273 enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
274
275 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
276 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
277
278 enetc_unlock_mdio();
279
280 napi_schedule(&v->napi);
281
282 return IRQ_HANDLED;
283 }
284
285 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
286 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
287 struct napi_struct *napi, int work_limit);
288
enetc_rx_dim_work(struct work_struct * w)289 static void enetc_rx_dim_work(struct work_struct *w)
290 {
291 struct dim *dim = container_of(w, struct dim, work);
292 struct dim_cq_moder moder =
293 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
294 struct enetc_int_vector *v =
295 container_of(dim, struct enetc_int_vector, rx_dim);
296
297 v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
298 dim->state = DIM_START_MEASURE;
299 }
300
enetc_rx_net_dim(struct enetc_int_vector * v)301 static void enetc_rx_net_dim(struct enetc_int_vector *v)
302 {
303 struct dim_sample dim_sample = {};
304
305 v->comp_cnt++;
306
307 if (!v->rx_napi_work)
308 return;
309
310 dim_update_sample(v->comp_cnt,
311 v->rx_ring.stats.packets,
312 v->rx_ring.stats.bytes,
313 &dim_sample);
314 net_dim(&v->rx_dim, dim_sample);
315 }
316
enetc_poll(struct napi_struct * napi,int budget)317 static int enetc_poll(struct napi_struct *napi, int budget)
318 {
319 struct enetc_int_vector
320 *v = container_of(napi, struct enetc_int_vector, napi);
321 bool complete = true;
322 int work_done;
323 int i;
324
325 enetc_lock_mdio();
326
327 for (i = 0; i < v->count_tx_rings; i++)
328 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
329 complete = false;
330
331 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
332 if (work_done == budget)
333 complete = false;
334 if (work_done)
335 v->rx_napi_work = true;
336
337 if (!complete) {
338 enetc_unlock_mdio();
339 return budget;
340 }
341
342 napi_complete_done(napi, work_done);
343
344 if (likely(v->rx_dim_en))
345 enetc_rx_net_dim(v);
346
347 v->rx_napi_work = false;
348
349 /* enable interrupts */
350 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
351
352 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
353 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
354 ENETC_TBIER_TXTIE);
355
356 enetc_unlock_mdio();
357
358 return work_done;
359 }
360
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)361 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
362 {
363 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
364
365 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
366 }
367
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)368 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
369 u64 *tstamp)
370 {
371 u32 lo, hi, tstamp_lo;
372
373 lo = enetc_rd_hot(hw, ENETC_SICTR0);
374 hi = enetc_rd_hot(hw, ENETC_SICTR1);
375 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
376 if (lo <= tstamp_lo)
377 hi -= 1;
378 *tstamp = (u64)hi << 32 | tstamp_lo;
379 }
380
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)381 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
382 {
383 struct skb_shared_hwtstamps shhwtstamps;
384
385 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
386 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
387 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
388 skb_txtime_consumed(skb);
389 skb_tstamp_tx(skb, &shhwtstamps);
390 }
391 }
392
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)393 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
394 {
395 struct net_device *ndev = tx_ring->ndev;
396 int tx_frm_cnt = 0, tx_byte_cnt = 0;
397 struct enetc_tx_swbd *tx_swbd;
398 int i, bds_to_clean;
399 bool do_tstamp;
400 u64 tstamp = 0;
401
402 i = tx_ring->next_to_clean;
403 tx_swbd = &tx_ring->tx_swbd[i];
404
405 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
406
407 do_tstamp = false;
408
409 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
410 bool is_eof = !!tx_swbd->skb;
411
412 if (unlikely(tx_swbd->check_wb)) {
413 struct enetc_ndev_priv *priv = netdev_priv(ndev);
414 union enetc_tx_bd *txbd;
415
416 txbd = ENETC_TXBD(*tx_ring, i);
417
418 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
419 tx_swbd->do_tstamp) {
420 enetc_get_tx_tstamp(&priv->si->hw, txbd,
421 &tstamp);
422 do_tstamp = true;
423 }
424 }
425
426 if (likely(tx_swbd->dma))
427 enetc_unmap_tx_buff(tx_ring, tx_swbd);
428
429 if (is_eof) {
430 if (unlikely(do_tstamp)) {
431 enetc_tstamp_tx(tx_swbd->skb, tstamp);
432 do_tstamp = false;
433 }
434 napi_consume_skb(tx_swbd->skb, napi_budget);
435 tx_swbd->skb = NULL;
436 }
437
438 tx_byte_cnt += tx_swbd->len;
439
440 bds_to_clean--;
441 tx_swbd++;
442 i++;
443 if (unlikely(i == tx_ring->bd_count)) {
444 i = 0;
445 tx_swbd = tx_ring->tx_swbd;
446 }
447
448 /* BD iteration loop end */
449 if (is_eof) {
450 tx_frm_cnt++;
451 /* re-arm interrupt source */
452 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
453 BIT(16 + tx_ring->index));
454 }
455
456 if (unlikely(!bds_to_clean))
457 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
458 }
459
460 tx_ring->next_to_clean = i;
461 tx_ring->stats.packets += tx_frm_cnt;
462 tx_ring->stats.bytes += tx_byte_cnt;
463
464 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
465 __netif_subqueue_stopped(ndev, tx_ring->index) &&
466 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
467 netif_wake_subqueue(ndev, tx_ring->index);
468 }
469
470 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
471 }
472
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)473 static bool enetc_new_page(struct enetc_bdr *rx_ring,
474 struct enetc_rx_swbd *rx_swbd)
475 {
476 struct page *page;
477 dma_addr_t addr;
478
479 page = dev_alloc_page();
480 if (unlikely(!page))
481 return false;
482
483 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
484 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
485 __free_page(page);
486
487 return false;
488 }
489
490 rx_swbd->dma = addr;
491 rx_swbd->page = page;
492 rx_swbd->page_offset = ENETC_RXB_PAD;
493
494 return true;
495 }
496
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)497 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
498 {
499 struct enetc_rx_swbd *rx_swbd;
500 union enetc_rx_bd *rxbd;
501 int i, j;
502
503 i = rx_ring->next_to_use;
504 rx_swbd = &rx_ring->rx_swbd[i];
505 rxbd = enetc_rxbd(rx_ring, i);
506
507 for (j = 0; j < buff_cnt; j++) {
508 /* try reuse page */
509 if (unlikely(!rx_swbd->page)) {
510 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
511 rx_ring->stats.rx_alloc_errs++;
512 break;
513 }
514 }
515
516 /* update RxBD */
517 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
518 rx_swbd->page_offset);
519 /* clear 'R" as well */
520 rxbd->r.lstatus = 0;
521
522 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
523 rx_swbd++;
524 i++;
525 if (unlikely(i == rx_ring->bd_count)) {
526 i = 0;
527 rx_swbd = rx_ring->rx_swbd;
528 }
529 }
530
531 if (likely(j)) {
532 rx_ring->next_to_alloc = i; /* keep track from page reuse */
533 rx_ring->next_to_use = i;
534 }
535
536 return j;
537 }
538
539 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)540 static void enetc_get_rx_tstamp(struct net_device *ndev,
541 union enetc_rx_bd *rxbd,
542 struct sk_buff *skb)
543 {
544 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
545 struct enetc_ndev_priv *priv = netdev_priv(ndev);
546 struct enetc_hw *hw = &priv->si->hw;
547 u32 lo, hi, tstamp_lo;
548 u64 tstamp;
549
550 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
551 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
552 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
553 rxbd = enetc_rxbd_ext(rxbd);
554 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
555 if (lo <= tstamp_lo)
556 hi -= 1;
557
558 tstamp = (u64)hi << 32 | tstamp_lo;
559 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
560 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
561 }
562 }
563 #endif
564
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)565 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
566 union enetc_rx_bd *rxbd, struct sk_buff *skb)
567 {
568 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
569
570 /* TODO: hashing */
571 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
572 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
573
574 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
575 skb->ip_summed = CHECKSUM_COMPLETE;
576 }
577
578 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
579 __be16 tpid = 0;
580
581 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
582 case 0:
583 tpid = htons(ETH_P_8021Q);
584 break;
585 case 1:
586 tpid = htons(ETH_P_8021AD);
587 break;
588 case 2:
589 tpid = htons(enetc_port_rd(&priv->si->hw,
590 ENETC_PCVLANR1));
591 break;
592 case 3:
593 tpid = htons(enetc_port_rd(&priv->si->hw,
594 ENETC_PCVLANR2));
595 break;
596 default:
597 break;
598 }
599
600 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
601 }
602
603 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
604 if (priv->active_offloads & ENETC_F_RX_TSTAMP)
605 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
606 #endif
607 }
608
enetc_process_skb(struct enetc_bdr * rx_ring,struct sk_buff * skb)609 static void enetc_process_skb(struct enetc_bdr *rx_ring,
610 struct sk_buff *skb)
611 {
612 skb_record_rx_queue(skb, rx_ring->index);
613 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
614 }
615
enetc_page_reusable(struct page * page)616 static bool enetc_page_reusable(struct page *page)
617 {
618 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
619 }
620
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)621 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
622 struct enetc_rx_swbd *old)
623 {
624 struct enetc_rx_swbd *new;
625
626 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
627
628 /* next buf that may reuse a page */
629 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
630
631 /* copy page reference */
632 *new = *old;
633 }
634
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)635 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
636 int i, u16 size)
637 {
638 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
639
640 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
641 rx_swbd->page_offset,
642 size, DMA_FROM_DEVICE);
643 return rx_swbd;
644 }
645
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)646 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
647 struct enetc_rx_swbd *rx_swbd)
648 {
649 if (likely(enetc_page_reusable(rx_swbd->page))) {
650 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
651 page_ref_inc(rx_swbd->page);
652
653 enetc_reuse_page(rx_ring, rx_swbd);
654
655 /* sync for use by the device */
656 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
657 rx_swbd->page_offset,
658 ENETC_RXB_DMA_SIZE,
659 DMA_FROM_DEVICE);
660 } else {
661 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
662 PAGE_SIZE, DMA_FROM_DEVICE);
663 }
664
665 rx_swbd->page = NULL;
666 }
667
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)668 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
669 int i, u16 size)
670 {
671 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
672 struct sk_buff *skb;
673 void *ba;
674
675 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
676 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
677 if (unlikely(!skb)) {
678 rx_ring->stats.rx_alloc_errs++;
679 return NULL;
680 }
681
682 skb_reserve(skb, ENETC_RXB_PAD);
683 __skb_put(skb, size);
684
685 enetc_put_rx_buff(rx_ring, rx_swbd);
686
687 return skb;
688 }
689
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)690 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
691 u16 size, struct sk_buff *skb)
692 {
693 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
694
695 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
696 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
697
698 enetc_put_rx_buff(rx_ring, rx_swbd);
699 }
700
701 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
702
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)703 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
704 struct napi_struct *napi, int work_limit)
705 {
706 int rx_frm_cnt = 0, rx_byte_cnt = 0;
707 int cleaned_cnt, i;
708
709 cleaned_cnt = enetc_bd_unused(rx_ring);
710 /* next descriptor to process */
711 i = rx_ring->next_to_clean;
712
713 while (likely(rx_frm_cnt < work_limit)) {
714 union enetc_rx_bd *rxbd;
715 struct sk_buff *skb;
716 u32 bd_status;
717 u16 size;
718
719 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
720 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
721
722 /* update ENETC's consumer index */
723 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
724 cleaned_cnt -= count;
725 }
726
727 rxbd = enetc_rxbd(rx_ring, i);
728 bd_status = le32_to_cpu(rxbd->r.lstatus);
729 if (!bd_status)
730 break;
731
732 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
733 dma_rmb(); /* for reading other rxbd fields */
734 size = le16_to_cpu(rxbd->r.buf_len);
735 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
736 if (!skb)
737 break;
738
739 enetc_get_offloads(rx_ring, rxbd, skb);
740
741 cleaned_cnt++;
742
743 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
744 if (unlikely(++i == rx_ring->bd_count))
745 i = 0;
746
747 if (unlikely(bd_status &
748 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
749 dev_kfree_skb(skb);
750 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
751 dma_rmb();
752 bd_status = le32_to_cpu(rxbd->r.lstatus);
753
754 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
755 if (unlikely(++i == rx_ring->bd_count))
756 i = 0;
757 }
758
759 rx_ring->ndev->stats.rx_dropped++;
760 rx_ring->ndev->stats.rx_errors++;
761
762 break;
763 }
764
765 /* not last BD in frame? */
766 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
767 bd_status = le32_to_cpu(rxbd->r.lstatus);
768 size = ENETC_RXB_DMA_SIZE;
769
770 if (bd_status & ENETC_RXBD_LSTATUS_F) {
771 dma_rmb();
772 size = le16_to_cpu(rxbd->r.buf_len);
773 }
774
775 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
776
777 cleaned_cnt++;
778
779 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
780 if (unlikely(++i == rx_ring->bd_count))
781 i = 0;
782 }
783
784 rx_byte_cnt += skb->len;
785
786 enetc_process_skb(rx_ring, skb);
787
788 napi_gro_receive(napi, skb);
789
790 rx_frm_cnt++;
791 }
792
793 rx_ring->next_to_clean = i;
794
795 rx_ring->stats.packets += rx_frm_cnt;
796 rx_ring->stats.bytes += rx_byte_cnt;
797
798 return rx_frm_cnt;
799 }
800
801 /* Probing and Init */
802 #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)803 void enetc_get_si_caps(struct enetc_si *si)
804 {
805 struct enetc_hw *hw = &si->hw;
806 u32 val;
807
808 /* find out how many of various resources we have to work with */
809 val = enetc_rd(hw, ENETC_SICAPR0);
810 si->num_rx_rings = (val >> 16) & 0xff;
811 si->num_tx_rings = val & 0xff;
812
813 val = enetc_rd(hw, ENETC_SIRFSCAPR);
814 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
815 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
816
817 si->num_rss = 0;
818 val = enetc_rd(hw, ENETC_SIPCAPR0);
819 if (val & ENETC_SIPCAPR0_RSS) {
820 u32 rss;
821
822 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
823 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
824 }
825
826 if (val & ENETC_SIPCAPR0_QBV)
827 si->hw_features |= ENETC_SI_F_QBV;
828
829 if (val & ENETC_SIPCAPR0_PSFP)
830 si->hw_features |= ENETC_SI_F_PSFP;
831 }
832
enetc_dma_alloc_bdr(struct enetc_bdr * r,size_t bd_size)833 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
834 {
835 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
836 &r->bd_dma_base, GFP_KERNEL);
837 if (!r->bd_base)
838 return -ENOMEM;
839
840 /* h/w requires 128B alignment */
841 if (!IS_ALIGNED(r->bd_dma_base, 128)) {
842 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
843 r->bd_dma_base);
844 return -EINVAL;
845 }
846
847 return 0;
848 }
849
enetc_alloc_txbdr(struct enetc_bdr * txr)850 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
851 {
852 int err;
853
854 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
855 if (!txr->tx_swbd)
856 return -ENOMEM;
857
858 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
859 if (err) {
860 vfree(txr->tx_swbd);
861 return err;
862 }
863
864 txr->next_to_clean = 0;
865 txr->next_to_use = 0;
866
867 return 0;
868 }
869
enetc_free_txbdr(struct enetc_bdr * txr)870 static void enetc_free_txbdr(struct enetc_bdr *txr)
871 {
872 int size, i;
873
874 for (i = 0; i < txr->bd_count; i++)
875 enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
876
877 size = txr->bd_count * sizeof(union enetc_tx_bd);
878
879 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
880 txr->bd_base = NULL;
881
882 vfree(txr->tx_swbd);
883 txr->tx_swbd = NULL;
884 }
885
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)886 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
887 {
888 int i, err;
889
890 for (i = 0; i < priv->num_tx_rings; i++) {
891 err = enetc_alloc_txbdr(priv->tx_ring[i]);
892
893 if (err)
894 goto fail;
895 }
896
897 return 0;
898
899 fail:
900 while (i-- > 0)
901 enetc_free_txbdr(priv->tx_ring[i]);
902
903 return err;
904 }
905
enetc_free_tx_resources(struct enetc_ndev_priv * priv)906 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
907 {
908 int i;
909
910 for (i = 0; i < priv->num_tx_rings; i++)
911 enetc_free_txbdr(priv->tx_ring[i]);
912 }
913
enetc_alloc_rxbdr(struct enetc_bdr * rxr,bool extended)914 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
915 {
916 size_t size = sizeof(union enetc_rx_bd);
917 int err;
918
919 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
920 if (!rxr->rx_swbd)
921 return -ENOMEM;
922
923 if (extended)
924 size *= 2;
925
926 err = enetc_dma_alloc_bdr(rxr, size);
927 if (err) {
928 vfree(rxr->rx_swbd);
929 return err;
930 }
931
932 rxr->next_to_clean = 0;
933 rxr->next_to_use = 0;
934 rxr->next_to_alloc = 0;
935 rxr->ext_en = extended;
936
937 return 0;
938 }
939
enetc_free_rxbdr(struct enetc_bdr * rxr)940 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
941 {
942 int size;
943
944 size = rxr->bd_count * sizeof(union enetc_rx_bd);
945
946 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
947 rxr->bd_base = NULL;
948
949 vfree(rxr->rx_swbd);
950 rxr->rx_swbd = NULL;
951 }
952
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv)953 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
954 {
955 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
956 int i, err;
957
958 for (i = 0; i < priv->num_rx_rings; i++) {
959 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
960
961 if (err)
962 goto fail;
963 }
964
965 return 0;
966
967 fail:
968 while (i-- > 0)
969 enetc_free_rxbdr(priv->rx_ring[i]);
970
971 return err;
972 }
973
enetc_free_rx_resources(struct enetc_ndev_priv * priv)974 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
975 {
976 int i;
977
978 for (i = 0; i < priv->num_rx_rings; i++)
979 enetc_free_rxbdr(priv->rx_ring[i]);
980 }
981
enetc_free_tx_ring(struct enetc_bdr * tx_ring)982 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
983 {
984 int i;
985
986 if (!tx_ring->tx_swbd)
987 return;
988
989 for (i = 0; i < tx_ring->bd_count; i++) {
990 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
991
992 enetc_free_tx_skb(tx_ring, tx_swbd);
993 }
994
995 tx_ring->next_to_clean = 0;
996 tx_ring->next_to_use = 0;
997 }
998
enetc_free_rx_ring(struct enetc_bdr * rx_ring)999 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1000 {
1001 int i;
1002
1003 if (!rx_ring->rx_swbd)
1004 return;
1005
1006 for (i = 0; i < rx_ring->bd_count; i++) {
1007 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1008
1009 if (!rx_swbd->page)
1010 continue;
1011
1012 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
1013 PAGE_SIZE, DMA_FROM_DEVICE);
1014 __free_page(rx_swbd->page);
1015 rx_swbd->page = NULL;
1016 }
1017
1018 rx_ring->next_to_clean = 0;
1019 rx_ring->next_to_use = 0;
1020 rx_ring->next_to_alloc = 0;
1021 }
1022
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)1023 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1024 {
1025 int i;
1026
1027 for (i = 0; i < priv->num_rx_rings; i++)
1028 enetc_free_rx_ring(priv->rx_ring[i]);
1029
1030 for (i = 0; i < priv->num_tx_rings; i++)
1031 enetc_free_tx_ring(priv->tx_ring[i]);
1032 }
1033
enetc_alloc_cbdr(struct device * dev,struct enetc_cbdr * cbdr)1034 int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1035 {
1036 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1037
1038 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
1039 GFP_KERNEL);
1040 if (!cbdr->bd_base)
1041 return -ENOMEM;
1042
1043 /* h/w requires 128B alignment */
1044 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
1045 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1046 return -EINVAL;
1047 }
1048
1049 cbdr->next_to_clean = 0;
1050 cbdr->next_to_use = 0;
1051
1052 return 0;
1053 }
1054
enetc_free_cbdr(struct device * dev,struct enetc_cbdr * cbdr)1055 void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1056 {
1057 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1058
1059 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1060 cbdr->bd_base = NULL;
1061 }
1062
enetc_setup_cbdr(struct enetc_hw * hw,struct enetc_cbdr * cbdr)1063 void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
1064 {
1065 /* set CBDR cache attributes */
1066 enetc_wr(hw, ENETC_SICAR2,
1067 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1068
1069 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
1070 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
1071 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
1072
1073 enetc_wr(hw, ENETC_SICBDRPIR, 0);
1074 enetc_wr(hw, ENETC_SICBDRCIR, 0);
1075
1076 /* enable ring */
1077 enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
1078
1079 cbdr->pir = hw->reg + ENETC_SICBDRPIR;
1080 cbdr->cir = hw->reg + ENETC_SICBDRCIR;
1081 }
1082
enetc_clear_cbdr(struct enetc_hw * hw)1083 void enetc_clear_cbdr(struct enetc_hw *hw)
1084 {
1085 enetc_wr(hw, ENETC_SICBDRMR, 0);
1086 }
1087
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)1088 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1089 {
1090 int *rss_table;
1091 int i;
1092
1093 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1094 if (!rss_table)
1095 return -ENOMEM;
1096
1097 /* Set up RSS table defaults */
1098 for (i = 0; i < si->num_rss; i++)
1099 rss_table[i] = i % num_groups;
1100
1101 enetc_set_rss_table(si, rss_table, si->num_rss);
1102
1103 kfree(rss_table);
1104
1105 return 0;
1106 }
1107
enetc_configure_si(struct enetc_ndev_priv * priv)1108 int enetc_configure_si(struct enetc_ndev_priv *priv)
1109 {
1110 struct enetc_si *si = priv->si;
1111 struct enetc_hw *hw = &si->hw;
1112 int err;
1113
1114 /* set SI cache attributes */
1115 enetc_wr(hw, ENETC_SICAR0,
1116 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1117 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1118 /* enable SI */
1119 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1120
1121 if (si->num_rss) {
1122 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1123 if (err)
1124 return err;
1125 }
1126
1127 return 0;
1128 }
1129
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)1130 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1131 {
1132 struct enetc_si *si = priv->si;
1133 int cpus = num_online_cpus();
1134
1135 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1136 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1137
1138 /* Enable all available TX rings in order to configure as many
1139 * priorities as possible, when needed.
1140 * TODO: Make # of TX rings run-time configurable
1141 */
1142 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1143 priv->num_tx_rings = si->num_tx_rings;
1144 priv->bdr_int_num = cpus;
1145 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
1146 priv->tx_ictt = ENETC_TXIC_TIMETHR;
1147
1148 /* SI specific */
1149 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1150 }
1151
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)1152 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1153 {
1154 struct enetc_si *si = priv->si;
1155 int err;
1156
1157 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1158 if (err)
1159 return err;
1160
1161 enetc_setup_cbdr(&si->hw, &si->cbd_ring);
1162
1163 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1164 GFP_KERNEL);
1165 if (!priv->cls_rules) {
1166 err = -ENOMEM;
1167 goto err_alloc_cls;
1168 }
1169
1170 return 0;
1171
1172 err_alloc_cls:
1173 enetc_clear_cbdr(&si->hw);
1174 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1175
1176 return err;
1177 }
1178
enetc_free_si_resources(struct enetc_ndev_priv * priv)1179 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1180 {
1181 struct enetc_si *si = priv->si;
1182
1183 enetc_clear_cbdr(&si->hw);
1184 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1185
1186 kfree(priv->cls_rules);
1187 }
1188
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1189 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1190 {
1191 int idx = tx_ring->index;
1192 u32 tbmr;
1193
1194 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1195 lower_32_bits(tx_ring->bd_dma_base));
1196
1197 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1198 upper_32_bits(tx_ring->bd_dma_base));
1199
1200 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1201 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1202 ENETC_RTBLENR_LEN(tx_ring->bd_count));
1203
1204 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1205 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1206 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1207
1208 /* enable Tx ints by setting pkt thr to 1 */
1209 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
1210
1211 tbmr = ENETC_TBMR_EN | ENETC_TBMR_SET_PRIO(tx_ring->prio);
1212 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1213 tbmr |= ENETC_TBMR_VIH;
1214
1215 /* enable ring */
1216 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1217
1218 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1219 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1220 tx_ring->idr = hw->reg + ENETC_SITXIDR;
1221 }
1222
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1223 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1224 {
1225 int idx = rx_ring->index;
1226 u32 rbmr;
1227
1228 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1229 lower_32_bits(rx_ring->bd_dma_base));
1230
1231 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1232 upper_32_bits(rx_ring->bd_dma_base));
1233
1234 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1235 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1236 ENETC_RTBLENR_LEN(rx_ring->bd_count));
1237
1238 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1239
1240 /* Also prepare the consumer index in case page allocation never
1241 * succeeds. In that case, hardware will never advance producer index
1242 * to match consumer index, and will drop all frames.
1243 */
1244 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1245 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
1246
1247 /* enable Rx ints by setting pkt thr to 1 */
1248 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
1249
1250 rbmr = ENETC_RBMR_EN;
1251
1252 if (rx_ring->ext_en)
1253 rbmr |= ENETC_RBMR_BDS;
1254
1255 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1256 rbmr |= ENETC_RBMR_VTE;
1257
1258 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1259 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1260
1261 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1262 /* update ENETC's consumer index */
1263 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, rx_ring->next_to_use);
1264
1265 /* enable ring */
1266 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1267 }
1268
enetc_setup_bdrs(struct enetc_ndev_priv * priv)1269 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1270 {
1271 struct enetc_hw *hw = &priv->si->hw;
1272 int i;
1273
1274 for (i = 0; i < priv->num_tx_rings; i++)
1275 enetc_setup_txbdr(hw, priv->tx_ring[i]);
1276
1277 for (i = 0; i < priv->num_rx_rings; i++)
1278 enetc_setup_rxbdr(hw, priv->rx_ring[i]);
1279 }
1280
enetc_clear_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1281 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1282 {
1283 int idx = rx_ring->index;
1284
1285 /* disable EN bit on ring */
1286 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1287 }
1288
enetc_clear_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1289 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1290 {
1291 int delay = 8, timeout = 100;
1292 int idx = tx_ring->index;
1293
1294 /* disable EN bit on ring */
1295 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1296
1297 /* wait for busy to clear */
1298 while (delay < timeout &&
1299 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1300 msleep(delay);
1301 delay *= 2;
1302 }
1303
1304 if (delay >= timeout)
1305 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1306 idx);
1307 }
1308
enetc_clear_bdrs(struct enetc_ndev_priv * priv)1309 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1310 {
1311 struct enetc_hw *hw = &priv->si->hw;
1312 int i;
1313
1314 for (i = 0; i < priv->num_tx_rings; i++)
1315 enetc_clear_txbdr(hw, priv->tx_ring[i]);
1316
1317 for (i = 0; i < priv->num_rx_rings; i++)
1318 enetc_clear_rxbdr(hw, priv->rx_ring[i]);
1319
1320 udelay(1);
1321 }
1322
enetc_setup_irqs(struct enetc_ndev_priv * priv)1323 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1324 {
1325 struct pci_dev *pdev = priv->si->pdev;
1326 struct enetc_hw *hw = &priv->si->hw;
1327 int i, j, err;
1328
1329 for (i = 0; i < priv->bdr_int_num; i++) {
1330 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1331 struct enetc_int_vector *v = priv->int_vector[i];
1332 int entry = ENETC_BDR_INT_BASE_IDX + i;
1333
1334 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1335 priv->ndev->name, i);
1336 err = request_irq(irq, enetc_msix, 0, v->name, v);
1337 if (err) {
1338 dev_err(priv->dev, "request_irq() failed!\n");
1339 goto irq_err;
1340 }
1341 disable_irq(irq);
1342
1343 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1344 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1345 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
1346
1347 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1348
1349 for (j = 0; j < v->count_tx_rings; j++) {
1350 int idx = v->tx_ring[j].index;
1351
1352 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1353 }
1354 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
1355 }
1356
1357 return 0;
1358
1359 irq_err:
1360 while (i--) {
1361 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1362
1363 irq_set_affinity_hint(irq, NULL);
1364 free_irq(irq, priv->int_vector[i]);
1365 }
1366
1367 return err;
1368 }
1369
enetc_free_irqs(struct enetc_ndev_priv * priv)1370 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1371 {
1372 struct pci_dev *pdev = priv->si->pdev;
1373 int i;
1374
1375 for (i = 0; i < priv->bdr_int_num; i++) {
1376 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1377
1378 irq_set_affinity_hint(irq, NULL);
1379 free_irq(irq, priv->int_vector[i]);
1380 }
1381 }
1382
enetc_setup_interrupts(struct enetc_ndev_priv * priv)1383 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
1384 {
1385 struct enetc_hw *hw = &priv->si->hw;
1386 u32 icpt, ictt;
1387 int i;
1388
1389 /* enable Tx & Rx event indication */
1390 if (priv->ic_mode &
1391 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
1392 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
1393 /* init to non-0 minimum, will be adjusted later */
1394 ictt = 0x1;
1395 } else {
1396 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
1397 ictt = 0;
1398 }
1399
1400 for (i = 0; i < priv->num_rx_rings; i++) {
1401 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
1402 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
1403 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
1404 }
1405
1406 if (priv->ic_mode & ENETC_IC_TX_MANUAL)
1407 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
1408 else
1409 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
1410
1411 for (i = 0; i < priv->num_tx_rings; i++) {
1412 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
1413 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
1414 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
1415 }
1416 }
1417
enetc_clear_interrupts(struct enetc_ndev_priv * priv)1418 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
1419 {
1420 struct enetc_hw *hw = &priv->si->hw;
1421 int i;
1422
1423 for (i = 0; i < priv->num_tx_rings; i++)
1424 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
1425
1426 for (i = 0; i < priv->num_rx_rings; i++)
1427 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
1428 }
1429
enetc_phylink_connect(struct net_device * ndev)1430 static int enetc_phylink_connect(struct net_device *ndev)
1431 {
1432 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1433 struct ethtool_eee edata;
1434 int err;
1435
1436 if (!priv->phylink)
1437 return 0; /* phy-less mode */
1438
1439 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
1440 if (err) {
1441 dev_err(&ndev->dev, "could not attach to PHY\n");
1442 return err;
1443 }
1444
1445 /* disable EEE autoneg, until ENETC driver supports it */
1446 memset(&edata, 0, sizeof(struct ethtool_eee));
1447 phylink_ethtool_set_eee(priv->phylink, &edata);
1448
1449 return 0;
1450 }
1451
enetc_start(struct net_device * ndev)1452 void enetc_start(struct net_device *ndev)
1453 {
1454 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1455 int i;
1456
1457 enetc_setup_interrupts(priv);
1458
1459 for (i = 0; i < priv->bdr_int_num; i++) {
1460 int irq = pci_irq_vector(priv->si->pdev,
1461 ENETC_BDR_INT_BASE_IDX + i);
1462
1463 napi_enable(&priv->int_vector[i]->napi);
1464 enable_irq(irq);
1465 }
1466
1467 if (priv->phylink)
1468 phylink_start(priv->phylink);
1469 else
1470 netif_carrier_on(ndev);
1471
1472 netif_tx_start_all_queues(ndev);
1473 }
1474
enetc_open(struct net_device * ndev)1475 int enetc_open(struct net_device *ndev)
1476 {
1477 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1478 int err;
1479
1480 err = enetc_setup_irqs(priv);
1481 if (err)
1482 return err;
1483
1484 err = enetc_phylink_connect(ndev);
1485 if (err)
1486 goto err_phy_connect;
1487
1488 err = enetc_alloc_tx_resources(priv);
1489 if (err)
1490 goto err_alloc_tx;
1491
1492 err = enetc_alloc_rx_resources(priv);
1493 if (err)
1494 goto err_alloc_rx;
1495
1496 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1497 if (err)
1498 goto err_set_queues;
1499
1500 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1501 if (err)
1502 goto err_set_queues;
1503
1504 enetc_setup_bdrs(priv);
1505 enetc_start(ndev);
1506
1507 return 0;
1508
1509 err_set_queues:
1510 enetc_free_rx_resources(priv);
1511 err_alloc_rx:
1512 enetc_free_tx_resources(priv);
1513 err_alloc_tx:
1514 if (priv->phylink)
1515 phylink_disconnect_phy(priv->phylink);
1516 err_phy_connect:
1517 enetc_free_irqs(priv);
1518
1519 return err;
1520 }
1521
enetc_stop(struct net_device * ndev)1522 void enetc_stop(struct net_device *ndev)
1523 {
1524 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1525 int i;
1526
1527 netif_tx_stop_all_queues(ndev);
1528
1529 for (i = 0; i < priv->bdr_int_num; i++) {
1530 int irq = pci_irq_vector(priv->si->pdev,
1531 ENETC_BDR_INT_BASE_IDX + i);
1532
1533 disable_irq(irq);
1534 napi_synchronize(&priv->int_vector[i]->napi);
1535 napi_disable(&priv->int_vector[i]->napi);
1536 }
1537
1538 if (priv->phylink)
1539 phylink_stop(priv->phylink);
1540 else
1541 netif_carrier_off(ndev);
1542
1543 enetc_clear_interrupts(priv);
1544 }
1545
enetc_close(struct net_device * ndev)1546 int enetc_close(struct net_device *ndev)
1547 {
1548 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1549
1550 enetc_stop(ndev);
1551 enetc_clear_bdrs(priv);
1552
1553 if (priv->phylink)
1554 phylink_disconnect_phy(priv->phylink);
1555 enetc_free_rxtx_rings(priv);
1556 enetc_free_rx_resources(priv);
1557 enetc_free_tx_resources(priv);
1558 enetc_free_irqs(priv);
1559
1560 return 0;
1561 }
1562
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)1563 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
1564 {
1565 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1566 struct tc_mqprio_qopt *mqprio = type_data;
1567 struct enetc_hw *hw = &priv->si->hw;
1568 struct enetc_bdr *tx_ring;
1569 u8 num_tc;
1570 int i;
1571
1572 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1573 num_tc = mqprio->num_tc;
1574
1575 if (!num_tc) {
1576 netdev_reset_tc(ndev);
1577 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1578
1579 /* Reset all ring priorities to 0 */
1580 for (i = 0; i < priv->num_tx_rings; i++) {
1581 tx_ring = priv->tx_ring[i];
1582 tx_ring->prio = 0;
1583 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
1584 }
1585
1586 return 0;
1587 }
1588
1589 /* Check if we have enough BD rings available to accommodate all TCs */
1590 if (num_tc > priv->num_tx_rings) {
1591 netdev_err(ndev, "Max %d traffic classes supported\n",
1592 priv->num_tx_rings);
1593 return -EINVAL;
1594 }
1595
1596 /* For the moment, we use only one BD ring per TC.
1597 *
1598 * Configure num_tc BD rings with increasing priorities.
1599 */
1600 for (i = 0; i < num_tc; i++) {
1601 tx_ring = priv->tx_ring[i];
1602 tx_ring->prio = i;
1603 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
1604 }
1605
1606 /* Reset the number of netdev queues based on the TC count */
1607 netif_set_real_num_tx_queues(ndev, num_tc);
1608
1609 netdev_set_num_tc(ndev, num_tc);
1610
1611 /* Each TC is associated with one netdev queue */
1612 for (i = 0; i < num_tc; i++)
1613 netdev_set_tc_queue(ndev, i, 1, i);
1614
1615 return 0;
1616 }
1617
enetc_setup_tc(struct net_device * ndev,enum tc_setup_type type,void * type_data)1618 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1619 void *type_data)
1620 {
1621 switch (type) {
1622 case TC_SETUP_QDISC_MQPRIO:
1623 return enetc_setup_tc_mqprio(ndev, type_data);
1624 case TC_SETUP_QDISC_TAPRIO:
1625 return enetc_setup_tc_taprio(ndev, type_data);
1626 case TC_SETUP_QDISC_CBS:
1627 return enetc_setup_tc_cbs(ndev, type_data);
1628 case TC_SETUP_QDISC_ETF:
1629 return enetc_setup_tc_txtime(ndev, type_data);
1630 case TC_SETUP_BLOCK:
1631 return enetc_setup_tc_psfp(ndev, type_data);
1632 default:
1633 return -EOPNOTSUPP;
1634 }
1635 }
1636
enetc_get_stats(struct net_device * ndev)1637 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1638 {
1639 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1640 struct net_device_stats *stats = &ndev->stats;
1641 unsigned long packets = 0, bytes = 0;
1642 int i;
1643
1644 for (i = 0; i < priv->num_rx_rings; i++) {
1645 packets += priv->rx_ring[i]->stats.packets;
1646 bytes += priv->rx_ring[i]->stats.bytes;
1647 }
1648
1649 stats->rx_packets = packets;
1650 stats->rx_bytes = bytes;
1651 bytes = 0;
1652 packets = 0;
1653
1654 for (i = 0; i < priv->num_tx_rings; i++) {
1655 packets += priv->tx_ring[i]->stats.packets;
1656 bytes += priv->tx_ring[i]->stats.bytes;
1657 }
1658
1659 stats->tx_packets = packets;
1660 stats->tx_bytes = bytes;
1661
1662 return stats;
1663 }
1664
enetc_set_rss(struct net_device * ndev,int en)1665 static int enetc_set_rss(struct net_device *ndev, int en)
1666 {
1667 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1668 struct enetc_hw *hw = &priv->si->hw;
1669 u32 reg;
1670
1671 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1672
1673 reg = enetc_rd(hw, ENETC_SIMR);
1674 reg &= ~ENETC_SIMR_RSSE;
1675 reg |= (en) ? ENETC_SIMR_RSSE : 0;
1676 enetc_wr(hw, ENETC_SIMR, reg);
1677
1678 return 0;
1679 }
1680
enetc_enable_rxvlan(struct net_device * ndev,bool en)1681 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
1682 {
1683 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1684 struct enetc_hw *hw = &priv->si->hw;
1685 int i;
1686
1687 for (i = 0; i < priv->num_rx_rings; i++)
1688 enetc_bdr_enable_rxvlan(hw, i, en);
1689 }
1690
enetc_enable_txvlan(struct net_device * ndev,bool en)1691 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
1692 {
1693 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1694 struct enetc_hw *hw = &priv->si->hw;
1695 int i;
1696
1697 for (i = 0; i < priv->num_tx_rings; i++)
1698 enetc_bdr_enable_txvlan(hw, i, en);
1699 }
1700
enetc_set_features(struct net_device * ndev,netdev_features_t features)1701 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
1702 {
1703 netdev_features_t changed = ndev->features ^ features;
1704
1705 if (changed & NETIF_F_RXHASH)
1706 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1707
1708 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1709 enetc_enable_rxvlan(ndev,
1710 !!(features & NETIF_F_HW_VLAN_CTAG_RX));
1711
1712 if (changed & NETIF_F_HW_VLAN_CTAG_TX)
1713 enetc_enable_txvlan(ndev,
1714 !!(features & NETIF_F_HW_VLAN_CTAG_TX));
1715 }
1716
1717 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)1718 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1719 {
1720 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1721 struct hwtstamp_config config;
1722 int ao;
1723
1724 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1725 return -EFAULT;
1726
1727 switch (config.tx_type) {
1728 case HWTSTAMP_TX_OFF:
1729 priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1730 break;
1731 case HWTSTAMP_TX_ON:
1732 priv->active_offloads |= ENETC_F_TX_TSTAMP;
1733 break;
1734 default:
1735 return -ERANGE;
1736 }
1737
1738 ao = priv->active_offloads;
1739 switch (config.rx_filter) {
1740 case HWTSTAMP_FILTER_NONE:
1741 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1742 break;
1743 default:
1744 priv->active_offloads |= ENETC_F_RX_TSTAMP;
1745 config.rx_filter = HWTSTAMP_FILTER_ALL;
1746 }
1747
1748 if (netif_running(ndev) && ao != priv->active_offloads) {
1749 enetc_close(ndev);
1750 enetc_open(ndev);
1751 }
1752
1753 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1754 -EFAULT : 0;
1755 }
1756
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)1757 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1758 {
1759 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1760 struct hwtstamp_config config;
1761
1762 config.flags = 0;
1763
1764 if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1765 config.tx_type = HWTSTAMP_TX_ON;
1766 else
1767 config.tx_type = HWTSTAMP_TX_OFF;
1768
1769 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1770 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1771
1772 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1773 -EFAULT : 0;
1774 }
1775 #endif
1776
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)1777 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1778 {
1779 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1780 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1781 if (cmd == SIOCSHWTSTAMP)
1782 return enetc_hwtstamp_set(ndev, rq);
1783 if (cmd == SIOCGHWTSTAMP)
1784 return enetc_hwtstamp_get(ndev, rq);
1785 #endif
1786
1787 if (!priv->phylink)
1788 return -EOPNOTSUPP;
1789
1790 return phylink_mii_ioctl(priv->phylink, rq, cmd);
1791 }
1792
enetc_alloc_msix(struct enetc_ndev_priv * priv)1793 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1794 {
1795 struct pci_dev *pdev = priv->si->pdev;
1796 int v_tx_rings;
1797 int i, n, err, nvec;
1798
1799 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1800 /* allocate MSIX for both messaging and Rx/Tx interrupts */
1801 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1802
1803 if (n < 0)
1804 return n;
1805
1806 if (n != nvec)
1807 return -EPERM;
1808
1809 /* # of tx rings per int vector */
1810 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1811
1812 for (i = 0; i < priv->bdr_int_num; i++) {
1813 struct enetc_int_vector *v;
1814 struct enetc_bdr *bdr;
1815 int j;
1816
1817 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
1818 if (!v) {
1819 err = -ENOMEM;
1820 goto fail;
1821 }
1822
1823 priv->int_vector[i] = v;
1824
1825 /* init defaults for adaptive IC */
1826 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
1827 v->rx_ictt = 0x1;
1828 v->rx_dim_en = true;
1829 }
1830 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
1831 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1832 NAPI_POLL_WEIGHT);
1833 v->count_tx_rings = v_tx_rings;
1834
1835 for (j = 0; j < v_tx_rings; j++) {
1836 int idx;
1837
1838 /* default tx ring mapping policy */
1839 if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1840 idx = 2 * j + i; /* 2 CPUs */
1841 else
1842 idx = j + i * v_tx_rings; /* default */
1843
1844 __set_bit(idx, &v->tx_rings_map);
1845 bdr = &v->tx_ring[j];
1846 bdr->index = idx;
1847 bdr->ndev = priv->ndev;
1848 bdr->dev = priv->dev;
1849 bdr->bd_count = priv->tx_bd_count;
1850 priv->tx_ring[idx] = bdr;
1851 }
1852
1853 bdr = &v->rx_ring;
1854 bdr->index = i;
1855 bdr->ndev = priv->ndev;
1856 bdr->dev = priv->dev;
1857 bdr->bd_count = priv->rx_bd_count;
1858 priv->rx_ring[i] = bdr;
1859 }
1860
1861 return 0;
1862
1863 fail:
1864 while (i--) {
1865 netif_napi_del(&priv->int_vector[i]->napi);
1866 cancel_work_sync(&priv->int_vector[i]->rx_dim.work);
1867 kfree(priv->int_vector[i]);
1868 }
1869
1870 pci_free_irq_vectors(pdev);
1871
1872 return err;
1873 }
1874
enetc_free_msix(struct enetc_ndev_priv * priv)1875 void enetc_free_msix(struct enetc_ndev_priv *priv)
1876 {
1877 int i;
1878
1879 for (i = 0; i < priv->bdr_int_num; i++) {
1880 struct enetc_int_vector *v = priv->int_vector[i];
1881
1882 netif_napi_del(&v->napi);
1883 cancel_work_sync(&v->rx_dim.work);
1884 }
1885
1886 for (i = 0; i < priv->num_rx_rings; i++)
1887 priv->rx_ring[i] = NULL;
1888
1889 for (i = 0; i < priv->num_tx_rings; i++)
1890 priv->tx_ring[i] = NULL;
1891
1892 for (i = 0; i < priv->bdr_int_num; i++) {
1893 kfree(priv->int_vector[i]);
1894 priv->int_vector[i] = NULL;
1895 }
1896
1897 /* disable all MSIX for this device */
1898 pci_free_irq_vectors(priv->si->pdev);
1899 }
1900
enetc_kfree_si(struct enetc_si * si)1901 static void enetc_kfree_si(struct enetc_si *si)
1902 {
1903 char *p = (char *)si - si->pad;
1904
1905 kfree(p);
1906 }
1907
enetc_detect_errata(struct enetc_si * si)1908 static void enetc_detect_errata(struct enetc_si *si)
1909 {
1910 if (si->pdev->revision == ENETC_REV1)
1911 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1912 ENETC_ERR_UCMCSWP;
1913 }
1914
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)1915 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1916 {
1917 struct enetc_si *si, *p;
1918 struct enetc_hw *hw;
1919 size_t alloc_size;
1920 int err, len;
1921
1922 pcie_flr(pdev);
1923 err = pci_enable_device_mem(pdev);
1924 if (err) {
1925 dev_err(&pdev->dev, "device enable failed\n");
1926 return err;
1927 }
1928
1929 /* set up for high or low dma */
1930 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1931 if (err) {
1932 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1933 if (err) {
1934 dev_err(&pdev->dev,
1935 "DMA configuration failed: 0x%x\n", err);
1936 goto err_dma;
1937 }
1938 }
1939
1940 err = pci_request_mem_regions(pdev, name);
1941 if (err) {
1942 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1943 goto err_pci_mem_reg;
1944 }
1945
1946 pci_set_master(pdev);
1947
1948 alloc_size = sizeof(struct enetc_si);
1949 if (sizeof_priv) {
1950 /* align priv to 32B */
1951 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1952 alloc_size += sizeof_priv;
1953 }
1954 /* force 32B alignment for enetc_si */
1955 alloc_size += ENETC_SI_ALIGN - 1;
1956
1957 p = kzalloc(alloc_size, GFP_KERNEL);
1958 if (!p) {
1959 err = -ENOMEM;
1960 goto err_alloc_si;
1961 }
1962
1963 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1964 si->pad = (char *)si - (char *)p;
1965
1966 pci_set_drvdata(pdev, si);
1967 si->pdev = pdev;
1968 hw = &si->hw;
1969
1970 len = pci_resource_len(pdev, ENETC_BAR_REGS);
1971 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1972 if (!hw->reg) {
1973 err = -ENXIO;
1974 dev_err(&pdev->dev, "ioremap() failed\n");
1975 goto err_ioremap;
1976 }
1977 if (len > ENETC_PORT_BASE)
1978 hw->port = hw->reg + ENETC_PORT_BASE;
1979 if (len > ENETC_GLOBAL_BASE)
1980 hw->global = hw->reg + ENETC_GLOBAL_BASE;
1981
1982 enetc_detect_errata(si);
1983
1984 return 0;
1985
1986 err_ioremap:
1987 enetc_kfree_si(si);
1988 err_alloc_si:
1989 pci_release_mem_regions(pdev);
1990 err_pci_mem_reg:
1991 err_dma:
1992 pci_disable_device(pdev);
1993
1994 return err;
1995 }
1996
enetc_pci_remove(struct pci_dev * pdev)1997 void enetc_pci_remove(struct pci_dev *pdev)
1998 {
1999 struct enetc_si *si = pci_get_drvdata(pdev);
2000 struct enetc_hw *hw = &si->hw;
2001
2002 iounmap(hw->reg);
2003 enetc_kfree_si(si);
2004 pci_release_mem_regions(pdev);
2005 pci_disable_device(pdev);
2006 }
2007