1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2018 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11 #include "net_driver.h"
12 #include <linux/module.h>
13 #include "efx_channels.h"
14 #include "efx.h"
15 #include "efx_common.h"
16 #include "tx_common.h"
17 #include "rx_common.h"
18 #include "nic.h"
19 #include "sriov.h"
20 #include "workarounds.h"
21
22 /* This is the first interrupt mode to try out of:
23 * 0 => MSI-X
24 * 1 => MSI
25 * 2 => legacy
26 */
27 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
28
29 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
30 * i.e. the number of CPUs among which we may distribute simultaneous
31 * interrupt handling.
32 *
33 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
34 * The default (0) means to assign an interrupt to each core.
35 */
36 unsigned int rss_cpus;
37
38 static unsigned int irq_adapt_low_thresh = 8000;
39 module_param(irq_adapt_low_thresh, uint, 0644);
40 MODULE_PARM_DESC(irq_adapt_low_thresh,
41 "Threshold score for reducing IRQ moderation");
42
43 static unsigned int irq_adapt_high_thresh = 16000;
44 module_param(irq_adapt_high_thresh, uint, 0644);
45 MODULE_PARM_DESC(irq_adapt_high_thresh,
46 "Threshold score for increasing IRQ moderation");
47
48 /* This is the weight assigned to each of the (per-channel) virtual
49 * NAPI devices.
50 */
51 static int napi_weight = 64;
52
53 /***************
54 * Housekeeping
55 ***************/
56
efx_channel_dummy_op_int(struct efx_channel * channel)57 int efx_channel_dummy_op_int(struct efx_channel *channel)
58 {
59 return 0;
60 }
61
efx_channel_dummy_op_void(struct efx_channel * channel)62 void efx_channel_dummy_op_void(struct efx_channel *channel)
63 {
64 }
65
66 static const struct efx_channel_type efx_default_channel_type = {
67 .pre_probe = efx_channel_dummy_op_int,
68 .post_remove = efx_channel_dummy_op_void,
69 .get_name = efx_get_channel_name,
70 .copy = efx_copy_channel,
71 .want_txqs = efx_default_channel_want_txqs,
72 .keep_eventq = false,
73 .want_pio = true,
74 };
75
76 /*************
77 * INTERRUPTS
78 *************/
79
efx_wanted_parallelism(struct efx_nic * efx)80 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
81 {
82 cpumask_var_t thread_mask;
83 unsigned int count;
84 int cpu;
85
86 if (rss_cpus) {
87 count = rss_cpus;
88 } else {
89 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
90 netif_warn(efx, probe, efx->net_dev,
91 "RSS disabled due to allocation failure\n");
92 return 1;
93 }
94
95 count = 0;
96 for_each_online_cpu(cpu) {
97 if (!cpumask_test_cpu(cpu, thread_mask)) {
98 ++count;
99 cpumask_or(thread_mask, thread_mask,
100 topology_sibling_cpumask(cpu));
101 }
102 }
103
104 free_cpumask_var(thread_mask);
105 }
106
107 if (count > EFX_MAX_RX_QUEUES) {
108 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
109 "Reducing number of rx queues from %u to %u.\n",
110 count, EFX_MAX_RX_QUEUES);
111 count = EFX_MAX_RX_QUEUES;
112 }
113
114 /* If RSS is requested for the PF *and* VFs then we can't write RSS
115 * table entries that are inaccessible to VFs
116 */
117 #ifdef CONFIG_SFC_SRIOV
118 if (efx->type->sriov_wanted) {
119 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
120 count > efx_vf_size(efx)) {
121 netif_warn(efx, probe, efx->net_dev,
122 "Reducing number of RSS channels from %u to %u for "
123 "VF support. Increase vf-msix-limit to use more "
124 "channels on the PF.\n",
125 count, efx_vf_size(efx));
126 count = efx_vf_size(efx);
127 }
128 }
129 #endif
130
131 return count;
132 }
133
efx_allocate_msix_channels(struct efx_nic * efx,unsigned int max_channels,unsigned int extra_channels,unsigned int parallelism)134 static int efx_allocate_msix_channels(struct efx_nic *efx,
135 unsigned int max_channels,
136 unsigned int extra_channels,
137 unsigned int parallelism)
138 {
139 unsigned int n_channels = parallelism;
140 int vec_count;
141 int tx_per_ev;
142 int n_xdp_tx;
143 int n_xdp_ev;
144
145 if (efx_separate_tx_channels)
146 n_channels *= 2;
147 n_channels += extra_channels;
148
149 /* To allow XDP transmit to happen from arbitrary NAPI contexts
150 * we allocate a TX queue per CPU. We share event queues across
151 * multiple tx queues, assuming tx and ev queues are both
152 * maximum size.
153 */
154 tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
155 tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
156 n_xdp_tx = num_possible_cpus();
157 n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
158
159 vec_count = pci_msix_vec_count(efx->pci_dev);
160 if (vec_count < 0)
161 return vec_count;
162
163 max_channels = min_t(unsigned int, vec_count, max_channels);
164
165 /* Check resources.
166 * We need a channel per event queue, plus a VI per tx queue.
167 * This may be more pessimistic than it needs to be.
168 */
169 if (n_channels >= max_channels) {
170 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
171 netif_warn(efx, drv, efx->net_dev,
172 "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
173 n_xdp_ev, n_channels, max_channels);
174 netif_warn(efx, drv, efx->net_dev,
175 "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
176 } else if (n_channels + n_xdp_tx > efx->max_vis) {
177 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
178 netif_warn(efx, drv, efx->net_dev,
179 "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
180 n_xdp_tx, n_channels, efx->max_vis);
181 netif_warn(efx, drv, efx->net_dev,
182 "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
183 } else if (n_channels + n_xdp_ev > max_channels) {
184 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
185 netif_warn(efx, drv, efx->net_dev,
186 "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
187 n_xdp_ev, n_channels, max_channels);
188
189 n_xdp_ev = max_channels - n_channels;
190 netif_warn(efx, drv, efx->net_dev,
191 "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
192 DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
193 } else {
194 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
195 }
196
197 if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
198 efx->n_xdp_channels = n_xdp_ev;
199 efx->xdp_tx_per_channel = tx_per_ev;
200 efx->xdp_tx_queue_count = n_xdp_tx;
201 n_channels += n_xdp_ev;
202 netif_dbg(efx, drv, efx->net_dev,
203 "Allocating %d TX and %d event queues for XDP\n",
204 n_xdp_ev * tx_per_ev, n_xdp_ev);
205 } else {
206 efx->n_xdp_channels = 0;
207 efx->xdp_tx_per_channel = 0;
208 efx->xdp_tx_queue_count = n_xdp_tx;
209 }
210
211 if (vec_count < n_channels) {
212 netif_err(efx, drv, efx->net_dev,
213 "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
214 vec_count, n_channels);
215 netif_err(efx, drv, efx->net_dev,
216 "WARNING: Performance may be reduced.\n");
217 n_channels = vec_count;
218 }
219
220 n_channels = min(n_channels, max_channels);
221
222 efx->n_channels = n_channels;
223
224 /* Ignore XDP tx channels when creating rx channels. */
225 n_channels -= efx->n_xdp_channels;
226
227 if (efx_separate_tx_channels) {
228 efx->n_tx_channels =
229 min(max(n_channels / 2, 1U),
230 efx->max_tx_channels);
231 efx->tx_channel_offset =
232 n_channels - efx->n_tx_channels;
233 efx->n_rx_channels =
234 max(n_channels -
235 efx->n_tx_channels, 1U);
236 } else {
237 efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
238 efx->tx_channel_offset = 0;
239 efx->n_rx_channels = n_channels;
240 }
241
242 efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
243 efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
244
245 efx->xdp_channel_offset = n_channels;
246
247 netif_dbg(efx, drv, efx->net_dev,
248 "Allocating %u RX channels\n",
249 efx->n_rx_channels);
250
251 return efx->n_channels;
252 }
253
254 /* Probe the number and type of interrupts we are able to obtain, and
255 * the resulting numbers of channels and RX queues.
256 */
efx_probe_interrupts(struct efx_nic * efx)257 int efx_probe_interrupts(struct efx_nic *efx)
258 {
259 unsigned int extra_channels = 0;
260 unsigned int rss_spread;
261 unsigned int i, j;
262 int rc;
263
264 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
265 if (efx->extra_channel_type[i])
266 ++extra_channels;
267
268 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
269 unsigned int parallelism = efx_wanted_parallelism(efx);
270 struct msix_entry xentries[EFX_MAX_CHANNELS];
271 unsigned int n_channels;
272
273 rc = efx_allocate_msix_channels(efx, efx->max_channels,
274 extra_channels, parallelism);
275 if (rc >= 0) {
276 n_channels = rc;
277 for (i = 0; i < n_channels; i++)
278 xentries[i].entry = i;
279 rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
280 n_channels);
281 }
282 if (rc < 0) {
283 /* Fall back to single channel MSI */
284 netif_err(efx, drv, efx->net_dev,
285 "could not enable MSI-X\n");
286 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
287 efx->interrupt_mode = EFX_INT_MODE_MSI;
288 else
289 return rc;
290 } else if (rc < n_channels) {
291 netif_err(efx, drv, efx->net_dev,
292 "WARNING: Insufficient MSI-X vectors"
293 " available (%d < %u).\n", rc, n_channels);
294 netif_err(efx, drv, efx->net_dev,
295 "WARNING: Performance may be reduced.\n");
296 n_channels = rc;
297 }
298
299 if (rc > 0) {
300 for (i = 0; i < efx->n_channels; i++)
301 efx_get_channel(efx, i)->irq =
302 xentries[i].vector;
303 }
304 }
305
306 /* Try single interrupt MSI */
307 if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
308 efx->n_channels = 1;
309 efx->n_rx_channels = 1;
310 efx->n_tx_channels = 1;
311 efx->tx_channel_offset = 0;
312 efx->n_xdp_channels = 0;
313 efx->xdp_channel_offset = efx->n_channels;
314 rc = pci_enable_msi(efx->pci_dev);
315 if (rc == 0) {
316 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
317 } else {
318 netif_err(efx, drv, efx->net_dev,
319 "could not enable MSI\n");
320 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
321 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
322 else
323 return rc;
324 }
325 }
326
327 /* Assume legacy interrupts */
328 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
329 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
330 efx->n_rx_channels = 1;
331 efx->n_tx_channels = 1;
332 efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0;
333 efx->n_xdp_channels = 0;
334 efx->xdp_channel_offset = efx->n_channels;
335 efx->legacy_irq = efx->pci_dev->irq;
336 }
337
338 /* Assign extra channels if possible, before XDP channels */
339 efx->n_extra_tx_channels = 0;
340 j = efx->xdp_channel_offset;
341 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
342 if (!efx->extra_channel_type[i])
343 continue;
344 if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
345 efx->extra_channel_type[i]->handle_no_channel(efx);
346 } else {
347 --j;
348 efx_get_channel(efx, j)->type =
349 efx->extra_channel_type[i];
350 if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
351 efx->n_extra_tx_channels++;
352 }
353 }
354
355 rss_spread = efx->n_rx_channels;
356 /* RSS might be usable on VFs even if it is disabled on the PF */
357 #ifdef CONFIG_SFC_SRIOV
358 if (efx->type->sriov_wanted) {
359 efx->rss_spread = ((rss_spread > 1 ||
360 !efx->type->sriov_wanted(efx)) ?
361 rss_spread : efx_vf_size(efx));
362 return 0;
363 }
364 #endif
365 efx->rss_spread = rss_spread;
366
367 return 0;
368 }
369
370 #if defined(CONFIG_SMP)
efx_set_interrupt_affinity(struct efx_nic * efx)371 void efx_set_interrupt_affinity(struct efx_nic *efx)
372 {
373 struct efx_channel *channel;
374 unsigned int cpu;
375
376 efx_for_each_channel(channel, efx) {
377 cpu = cpumask_local_spread(channel->channel,
378 pcibus_to_node(efx->pci_dev->bus));
379 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
380 }
381 }
382
efx_clear_interrupt_affinity(struct efx_nic * efx)383 void efx_clear_interrupt_affinity(struct efx_nic *efx)
384 {
385 struct efx_channel *channel;
386
387 efx_for_each_channel(channel, efx)
388 irq_set_affinity_hint(channel->irq, NULL);
389 }
390 #else
391 void
efx_set_interrupt_affinity(struct efx_nic * efx)392 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
393 {
394 }
395
396 void
efx_clear_interrupt_affinity(struct efx_nic * efx)397 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
398 {
399 }
400 #endif /* CONFIG_SMP */
401
efx_remove_interrupts(struct efx_nic * efx)402 void efx_remove_interrupts(struct efx_nic *efx)
403 {
404 struct efx_channel *channel;
405
406 /* Remove MSI/MSI-X interrupts */
407 efx_for_each_channel(channel, efx)
408 channel->irq = 0;
409 pci_disable_msi(efx->pci_dev);
410 pci_disable_msix(efx->pci_dev);
411
412 /* Remove legacy interrupt */
413 efx->legacy_irq = 0;
414 }
415
416 /***************
417 * EVENT QUEUES
418 ***************/
419
420 /* Create event queue
421 * Event queue memory allocations are done only once. If the channel
422 * is reset, the memory buffer will be reused; this guards against
423 * errors during channel reset and also simplifies interrupt handling.
424 */
efx_probe_eventq(struct efx_channel * channel)425 int efx_probe_eventq(struct efx_channel *channel)
426 {
427 struct efx_nic *efx = channel->efx;
428 unsigned long entries;
429
430 netif_dbg(efx, probe, efx->net_dev,
431 "chan %d create event queue\n", channel->channel);
432
433 /* Build an event queue with room for one event per tx and rx buffer,
434 * plus some extra for link state events and MCDI completions.
435 */
436 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
437 EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
438 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
439
440 return efx_nic_probe_eventq(channel);
441 }
442
443 /* Prepare channel's event queue */
efx_init_eventq(struct efx_channel * channel)444 int efx_init_eventq(struct efx_channel *channel)
445 {
446 struct efx_nic *efx = channel->efx;
447 int rc;
448
449 EFX_WARN_ON_PARANOID(channel->eventq_init);
450
451 netif_dbg(efx, drv, efx->net_dev,
452 "chan %d init event queue\n", channel->channel);
453
454 rc = efx_nic_init_eventq(channel);
455 if (rc == 0) {
456 efx->type->push_irq_moderation(channel);
457 channel->eventq_read_ptr = 0;
458 channel->eventq_init = true;
459 }
460 return rc;
461 }
462
463 /* Enable event queue processing and NAPI */
efx_start_eventq(struct efx_channel * channel)464 void efx_start_eventq(struct efx_channel *channel)
465 {
466 netif_dbg(channel->efx, ifup, channel->efx->net_dev,
467 "chan %d start event queue\n", channel->channel);
468
469 /* Make sure the NAPI handler sees the enabled flag set */
470 channel->enabled = true;
471 smp_wmb();
472
473 napi_enable(&channel->napi_str);
474 efx_nic_eventq_read_ack(channel);
475 }
476
477 /* Disable event queue processing and NAPI */
efx_stop_eventq(struct efx_channel * channel)478 void efx_stop_eventq(struct efx_channel *channel)
479 {
480 if (!channel->enabled)
481 return;
482
483 napi_disable(&channel->napi_str);
484 channel->enabled = false;
485 }
486
efx_fini_eventq(struct efx_channel * channel)487 void efx_fini_eventq(struct efx_channel *channel)
488 {
489 if (!channel->eventq_init)
490 return;
491
492 netif_dbg(channel->efx, drv, channel->efx->net_dev,
493 "chan %d fini event queue\n", channel->channel);
494
495 efx_nic_fini_eventq(channel);
496 channel->eventq_init = false;
497 }
498
efx_remove_eventq(struct efx_channel * channel)499 void efx_remove_eventq(struct efx_channel *channel)
500 {
501 netif_dbg(channel->efx, drv, channel->efx->net_dev,
502 "chan %d remove event queue\n", channel->channel);
503
504 efx_nic_remove_eventq(channel);
505 }
506
507 /**************************************************************************
508 *
509 * Channel handling
510 *
511 *************************************************************************/
512
513 #ifdef CONFIG_RFS_ACCEL
efx_filter_rfs_expire(struct work_struct * data)514 static void efx_filter_rfs_expire(struct work_struct *data)
515 {
516 struct delayed_work *dwork = to_delayed_work(data);
517 struct efx_channel *channel;
518 unsigned int time, quota;
519
520 channel = container_of(dwork, struct efx_channel, filter_work);
521 time = jiffies - channel->rfs_last_expiry;
522 quota = channel->rfs_filter_count * time / (30 * HZ);
523 if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
524 channel->rfs_last_expiry += time;
525 /* Ensure we do more work eventually even if NAPI poll is not happening */
526 schedule_delayed_work(dwork, 30 * HZ);
527 }
528 #endif
529
530 /* Allocate and initialise a channel structure. */
efx_alloc_channel(struct efx_nic * efx,int i)531 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
532 {
533 struct efx_rx_queue *rx_queue;
534 struct efx_tx_queue *tx_queue;
535 struct efx_channel *channel;
536 int j;
537
538 channel = kzalloc(sizeof(*channel), GFP_KERNEL);
539 if (!channel)
540 return NULL;
541
542 channel->efx = efx;
543 channel->channel = i;
544 channel->type = &efx_default_channel_type;
545
546 for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
547 tx_queue = &channel->tx_queue[j];
548 tx_queue->efx = efx;
549 tx_queue->queue = -1;
550 tx_queue->label = j;
551 tx_queue->channel = channel;
552 }
553
554 #ifdef CONFIG_RFS_ACCEL
555 INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
556 #endif
557
558 rx_queue = &channel->rx_queue;
559 rx_queue->efx = efx;
560 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
561
562 return channel;
563 }
564
efx_init_channels(struct efx_nic * efx)565 int efx_init_channels(struct efx_nic *efx)
566 {
567 unsigned int i;
568
569 for (i = 0; i < EFX_MAX_CHANNELS; i++) {
570 efx->channel[i] = efx_alloc_channel(efx, i);
571 if (!efx->channel[i])
572 return -ENOMEM;
573 efx->msi_context[i].efx = efx;
574 efx->msi_context[i].index = i;
575 }
576
577 /* Higher numbered interrupt modes are less capable! */
578 efx->interrupt_mode = min(efx->type->min_interrupt_mode,
579 efx_interrupt_mode);
580
581 efx->max_channels = EFX_MAX_CHANNELS;
582 efx->max_tx_channels = EFX_MAX_CHANNELS;
583
584 return 0;
585 }
586
efx_fini_channels(struct efx_nic * efx)587 void efx_fini_channels(struct efx_nic *efx)
588 {
589 unsigned int i;
590
591 for (i = 0; i < EFX_MAX_CHANNELS; i++)
592 if (efx->channel[i]) {
593 kfree(efx->channel[i]);
594 efx->channel[i] = NULL;
595 }
596 }
597
598 /* Allocate and initialise a channel structure, copying parameters
599 * (but not resources) from an old channel structure.
600 */
efx_copy_channel(const struct efx_channel * old_channel)601 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
602 {
603 struct efx_rx_queue *rx_queue;
604 struct efx_tx_queue *tx_queue;
605 struct efx_channel *channel;
606 int j;
607
608 channel = kmalloc(sizeof(*channel), GFP_KERNEL);
609 if (!channel)
610 return NULL;
611
612 *channel = *old_channel;
613
614 channel->napi_dev = NULL;
615 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
616 channel->napi_str.napi_id = 0;
617 channel->napi_str.state = 0;
618 memset(&channel->eventq, 0, sizeof(channel->eventq));
619
620 for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
621 tx_queue = &channel->tx_queue[j];
622 if (tx_queue->channel)
623 tx_queue->channel = channel;
624 tx_queue->buffer = NULL;
625 tx_queue->cb_page = NULL;
626 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
627 }
628
629 rx_queue = &channel->rx_queue;
630 rx_queue->buffer = NULL;
631 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
632 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
633 #ifdef CONFIG_RFS_ACCEL
634 INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
635 #endif
636
637 return channel;
638 }
639
efx_probe_channel(struct efx_channel * channel)640 static int efx_probe_channel(struct efx_channel *channel)
641 {
642 struct efx_tx_queue *tx_queue;
643 struct efx_rx_queue *rx_queue;
644 int rc;
645
646 netif_dbg(channel->efx, probe, channel->efx->net_dev,
647 "creating channel %d\n", channel->channel);
648
649 rc = channel->type->pre_probe(channel);
650 if (rc)
651 goto fail;
652
653 rc = efx_probe_eventq(channel);
654 if (rc)
655 goto fail;
656
657 efx_for_each_channel_tx_queue(tx_queue, channel) {
658 rc = efx_probe_tx_queue(tx_queue);
659 if (rc)
660 goto fail;
661 }
662
663 efx_for_each_channel_rx_queue(rx_queue, channel) {
664 rc = efx_probe_rx_queue(rx_queue);
665 if (rc)
666 goto fail;
667 }
668
669 channel->rx_list = NULL;
670
671 return 0;
672
673 fail:
674 efx_remove_channel(channel);
675 return rc;
676 }
677
efx_get_channel_name(struct efx_channel * channel,char * buf,size_t len)678 void efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
679 {
680 struct efx_nic *efx = channel->efx;
681 const char *type;
682 int number;
683
684 number = channel->channel;
685
686 if (number >= efx->xdp_channel_offset &&
687 !WARN_ON_ONCE(!efx->n_xdp_channels)) {
688 type = "-xdp";
689 number -= efx->xdp_channel_offset;
690 } else if (efx->tx_channel_offset == 0) {
691 type = "";
692 } else if (number < efx->tx_channel_offset) {
693 type = "-rx";
694 } else {
695 type = "-tx";
696 number -= efx->tx_channel_offset;
697 }
698 snprintf(buf, len, "%s%s-%d", efx->name, type, number);
699 }
700
efx_set_channel_names(struct efx_nic * efx)701 void efx_set_channel_names(struct efx_nic *efx)
702 {
703 struct efx_channel *channel;
704
705 efx_for_each_channel(channel, efx)
706 channel->type->get_name(channel,
707 efx->msi_context[channel->channel].name,
708 sizeof(efx->msi_context[0].name));
709 }
710
efx_probe_channels(struct efx_nic * efx)711 int efx_probe_channels(struct efx_nic *efx)
712 {
713 struct efx_channel *channel;
714 int rc;
715
716 /* Restart special buffer allocation */
717 efx->next_buffer_table = 0;
718
719 /* Probe channels in reverse, so that any 'extra' channels
720 * use the start of the buffer table. This allows the traffic
721 * channels to be resized without moving them or wasting the
722 * entries before them.
723 */
724 efx_for_each_channel_rev(channel, efx) {
725 rc = efx_probe_channel(channel);
726 if (rc) {
727 netif_err(efx, probe, efx->net_dev,
728 "failed to create channel %d\n",
729 channel->channel);
730 goto fail;
731 }
732 }
733 efx_set_channel_names(efx);
734
735 return 0;
736
737 fail:
738 efx_remove_channels(efx);
739 return rc;
740 }
741
efx_remove_channel(struct efx_channel * channel)742 void efx_remove_channel(struct efx_channel *channel)
743 {
744 struct efx_tx_queue *tx_queue;
745 struct efx_rx_queue *rx_queue;
746
747 netif_dbg(channel->efx, drv, channel->efx->net_dev,
748 "destroy chan %d\n", channel->channel);
749
750 efx_for_each_channel_rx_queue(rx_queue, channel)
751 efx_remove_rx_queue(rx_queue);
752 efx_for_each_channel_tx_queue(tx_queue, channel)
753 efx_remove_tx_queue(tx_queue);
754 efx_remove_eventq(channel);
755 channel->type->post_remove(channel);
756 }
757
efx_remove_channels(struct efx_nic * efx)758 void efx_remove_channels(struct efx_nic *efx)
759 {
760 struct efx_channel *channel;
761
762 efx_for_each_channel(channel, efx)
763 efx_remove_channel(channel);
764
765 kfree(efx->xdp_tx_queues);
766 }
767
efx_set_xdp_tx_queue(struct efx_nic * efx,int xdp_queue_number,struct efx_tx_queue * tx_queue)768 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
769 struct efx_tx_queue *tx_queue)
770 {
771 if (xdp_queue_number >= efx->xdp_tx_queue_count)
772 return -EINVAL;
773
774 netif_dbg(efx, drv, efx->net_dev,
775 "Channel %u TXQ %u is XDP %u, HW %u\n",
776 tx_queue->channel->channel, tx_queue->label,
777 xdp_queue_number, tx_queue->queue);
778 efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
779 return 0;
780 }
781
efx_set_xdp_channels(struct efx_nic * efx)782 static void efx_set_xdp_channels(struct efx_nic *efx)
783 {
784 struct efx_tx_queue *tx_queue;
785 struct efx_channel *channel;
786 unsigned int next_queue = 0;
787 int xdp_queue_number = 0;
788 int rc;
789
790 /* We need to mark which channels really have RX and TX
791 * queues, and adjust the TX queue numbers if we have separate
792 * RX-only and TX-only channels.
793 */
794 efx_for_each_channel(channel, efx) {
795 if (channel->channel < efx->tx_channel_offset)
796 continue;
797
798 if (efx_channel_is_xdp_tx(channel)) {
799 efx_for_each_channel_tx_queue(tx_queue, channel) {
800 tx_queue->queue = next_queue++;
801 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
802 tx_queue);
803 if (rc == 0)
804 xdp_queue_number++;
805 }
806 } else {
807 efx_for_each_channel_tx_queue(tx_queue, channel) {
808 tx_queue->queue = next_queue++;
809 netif_dbg(efx, drv, efx->net_dev,
810 "Channel %u TXQ %u is HW %u\n",
811 channel->channel, tx_queue->label,
812 tx_queue->queue);
813 }
814
815 /* If XDP is borrowing queues from net stack, it must
816 * use the queue with no csum offload, which is the
817 * first one of the channel
818 * (note: tx_queue_by_type is not initialized yet)
819 */
820 if (efx->xdp_txq_queues_mode ==
821 EFX_XDP_TX_QUEUES_BORROWED) {
822 tx_queue = &channel->tx_queue[0];
823 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
824 tx_queue);
825 if (rc == 0)
826 xdp_queue_number++;
827 }
828 }
829 }
830 WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
831 xdp_queue_number != efx->xdp_tx_queue_count);
832 WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
833 xdp_queue_number > efx->xdp_tx_queue_count);
834
835 /* If we have more CPUs than assigned XDP TX queues, assign the already
836 * existing queues to the exceeding CPUs
837 */
838 next_queue = 0;
839 while (xdp_queue_number < efx->xdp_tx_queue_count) {
840 tx_queue = efx->xdp_tx_queues[next_queue++];
841 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
842 if (rc == 0)
843 xdp_queue_number++;
844 }
845 }
846
efx_realloc_channels(struct efx_nic * efx,u32 rxq_entries,u32 txq_entries)847 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
848 {
849 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
850 *ptp_channel = efx_ptp_channel(efx);
851 struct efx_ptp_data *ptp_data = efx->ptp_data;
852 unsigned int i, next_buffer_table = 0;
853 u32 old_rxq_entries, old_txq_entries;
854 int rc, rc2;
855
856 rc = efx_check_disabled(efx);
857 if (rc)
858 return rc;
859
860 /* Not all channels should be reallocated. We must avoid
861 * reallocating their buffer table entries.
862 */
863 efx_for_each_channel(channel, efx) {
864 struct efx_rx_queue *rx_queue;
865 struct efx_tx_queue *tx_queue;
866
867 if (channel->type->copy)
868 continue;
869 next_buffer_table = max(next_buffer_table,
870 channel->eventq.index +
871 channel->eventq.entries);
872 efx_for_each_channel_rx_queue(rx_queue, channel)
873 next_buffer_table = max(next_buffer_table,
874 rx_queue->rxd.index +
875 rx_queue->rxd.entries);
876 efx_for_each_channel_tx_queue(tx_queue, channel)
877 next_buffer_table = max(next_buffer_table,
878 tx_queue->txd.index +
879 tx_queue->txd.entries);
880 }
881
882 efx_device_detach_sync(efx);
883 efx_stop_all(efx);
884 efx_soft_disable_interrupts(efx);
885
886 /* Clone channels (where possible) */
887 memset(other_channel, 0, sizeof(other_channel));
888 for (i = 0; i < efx->n_channels; i++) {
889 channel = efx->channel[i];
890 if (channel->type->copy)
891 channel = channel->type->copy(channel);
892 if (!channel) {
893 rc = -ENOMEM;
894 goto out;
895 }
896 other_channel[i] = channel;
897 }
898
899 /* Swap entry counts and channel pointers */
900 old_rxq_entries = efx->rxq_entries;
901 old_txq_entries = efx->txq_entries;
902 efx->rxq_entries = rxq_entries;
903 efx->txq_entries = txq_entries;
904 for (i = 0; i < efx->n_channels; i++)
905 swap(efx->channel[i], other_channel[i]);
906
907 /* Restart buffer table allocation */
908 efx->next_buffer_table = next_buffer_table;
909
910 for (i = 0; i < efx->n_channels; i++) {
911 channel = efx->channel[i];
912 if (!channel->type->copy)
913 continue;
914 rc = efx_probe_channel(channel);
915 if (rc)
916 goto rollback;
917 efx_init_napi_channel(efx->channel[i]);
918 }
919
920 efx_set_xdp_channels(efx);
921 out:
922 efx->ptp_data = NULL;
923 /* Destroy unused channel structures */
924 for (i = 0; i < efx->n_channels; i++) {
925 channel = other_channel[i];
926 if (channel && channel->type->copy) {
927 efx_fini_napi_channel(channel);
928 efx_remove_channel(channel);
929 kfree(channel);
930 }
931 }
932
933 efx->ptp_data = ptp_data;
934 rc2 = efx_soft_enable_interrupts(efx);
935 if (rc2) {
936 rc = rc ? rc : rc2;
937 netif_err(efx, drv, efx->net_dev,
938 "unable to restart interrupts on channel reallocation\n");
939 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
940 } else {
941 efx_start_all(efx);
942 efx_device_attach_if_not_resetting(efx);
943 }
944 return rc;
945
946 rollback:
947 /* Swap back */
948 efx->rxq_entries = old_rxq_entries;
949 efx->txq_entries = old_txq_entries;
950 for (i = 0; i < efx->n_channels; i++)
951 swap(efx->channel[i], other_channel[i]);
952 efx_ptp_update_channel(efx, ptp_channel);
953 goto out;
954 }
955
efx_set_channels(struct efx_nic * efx)956 int efx_set_channels(struct efx_nic *efx)
957 {
958 struct efx_channel *channel;
959 int rc;
960
961 if (efx->xdp_tx_queue_count) {
962 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
963
964 /* Allocate array for XDP TX queue lookup. */
965 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
966 sizeof(*efx->xdp_tx_queues),
967 GFP_KERNEL);
968 if (!efx->xdp_tx_queues)
969 return -ENOMEM;
970 }
971
972 efx_for_each_channel(channel, efx) {
973 if (channel->channel < efx->n_rx_channels)
974 channel->rx_queue.core_index = channel->channel;
975 else
976 channel->rx_queue.core_index = -1;
977 }
978
979 efx_set_xdp_channels(efx);
980
981 rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
982 if (rc)
983 return rc;
984 return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
985 }
986
efx_default_channel_want_txqs(struct efx_channel * channel)987 bool efx_default_channel_want_txqs(struct efx_channel *channel)
988 {
989 return channel->channel - channel->efx->tx_channel_offset <
990 channel->efx->n_tx_channels;
991 }
992
993 /*************
994 * START/STOP
995 *************/
996
efx_soft_enable_interrupts(struct efx_nic * efx)997 int efx_soft_enable_interrupts(struct efx_nic *efx)
998 {
999 struct efx_channel *channel, *end_channel;
1000 int rc;
1001
1002 BUG_ON(efx->state == STATE_DISABLED);
1003
1004 efx->irq_soft_enabled = true;
1005 smp_wmb();
1006
1007 efx_for_each_channel(channel, efx) {
1008 if (!channel->type->keep_eventq) {
1009 rc = efx_init_eventq(channel);
1010 if (rc)
1011 goto fail;
1012 }
1013 efx_start_eventq(channel);
1014 }
1015
1016 efx_mcdi_mode_event(efx);
1017
1018 return 0;
1019 fail:
1020 end_channel = channel;
1021 efx_for_each_channel(channel, efx) {
1022 if (channel == end_channel)
1023 break;
1024 efx_stop_eventq(channel);
1025 if (!channel->type->keep_eventq)
1026 efx_fini_eventq(channel);
1027 }
1028
1029 return rc;
1030 }
1031
efx_soft_disable_interrupts(struct efx_nic * efx)1032 void efx_soft_disable_interrupts(struct efx_nic *efx)
1033 {
1034 struct efx_channel *channel;
1035
1036 if (efx->state == STATE_DISABLED)
1037 return;
1038
1039 efx_mcdi_mode_poll(efx);
1040
1041 efx->irq_soft_enabled = false;
1042 smp_wmb();
1043
1044 if (efx->legacy_irq)
1045 synchronize_irq(efx->legacy_irq);
1046
1047 efx_for_each_channel(channel, efx) {
1048 if (channel->irq)
1049 synchronize_irq(channel->irq);
1050
1051 efx_stop_eventq(channel);
1052 if (!channel->type->keep_eventq)
1053 efx_fini_eventq(channel);
1054 }
1055
1056 /* Flush the asynchronous MCDI request queue */
1057 efx_mcdi_flush_async(efx);
1058 }
1059
efx_enable_interrupts(struct efx_nic * efx)1060 int efx_enable_interrupts(struct efx_nic *efx)
1061 {
1062 struct efx_channel *channel, *end_channel;
1063 int rc;
1064
1065 /* TODO: Is this really a bug? */
1066 BUG_ON(efx->state == STATE_DISABLED);
1067
1068 if (efx->eeh_disabled_legacy_irq) {
1069 enable_irq(efx->legacy_irq);
1070 efx->eeh_disabled_legacy_irq = false;
1071 }
1072
1073 efx->type->irq_enable_master(efx);
1074
1075 efx_for_each_channel(channel, efx) {
1076 if (channel->type->keep_eventq) {
1077 rc = efx_init_eventq(channel);
1078 if (rc)
1079 goto fail;
1080 }
1081 }
1082
1083 rc = efx_soft_enable_interrupts(efx);
1084 if (rc)
1085 goto fail;
1086
1087 return 0;
1088
1089 fail:
1090 end_channel = channel;
1091 efx_for_each_channel(channel, efx) {
1092 if (channel == end_channel)
1093 break;
1094 if (channel->type->keep_eventq)
1095 efx_fini_eventq(channel);
1096 }
1097
1098 efx->type->irq_disable_non_ev(efx);
1099
1100 return rc;
1101 }
1102
efx_disable_interrupts(struct efx_nic * efx)1103 void efx_disable_interrupts(struct efx_nic *efx)
1104 {
1105 struct efx_channel *channel;
1106
1107 efx_soft_disable_interrupts(efx);
1108
1109 efx_for_each_channel(channel, efx) {
1110 if (channel->type->keep_eventq)
1111 efx_fini_eventq(channel);
1112 }
1113
1114 efx->type->irq_disable_non_ev(efx);
1115 }
1116
efx_start_channels(struct efx_nic * efx)1117 void efx_start_channels(struct efx_nic *efx)
1118 {
1119 struct efx_tx_queue *tx_queue;
1120 struct efx_rx_queue *rx_queue;
1121 struct efx_channel *channel;
1122
1123 efx_for_each_channel_rev(channel, efx) {
1124 efx_for_each_channel_tx_queue(tx_queue, channel) {
1125 efx_init_tx_queue(tx_queue);
1126 atomic_inc(&efx->active_queues);
1127 }
1128
1129 efx_for_each_channel_rx_queue(rx_queue, channel) {
1130 efx_init_rx_queue(rx_queue);
1131 atomic_inc(&efx->active_queues);
1132 efx_stop_eventq(channel);
1133 efx_fast_push_rx_descriptors(rx_queue, false);
1134 efx_start_eventq(channel);
1135 }
1136
1137 WARN_ON(channel->rx_pkt_n_frags);
1138 }
1139 }
1140
efx_stop_channels(struct efx_nic * efx)1141 void efx_stop_channels(struct efx_nic *efx)
1142 {
1143 struct efx_tx_queue *tx_queue;
1144 struct efx_rx_queue *rx_queue;
1145 struct efx_channel *channel;
1146 int rc = 0;
1147
1148 /* Stop RX refill */
1149 efx_for_each_channel(channel, efx) {
1150 efx_for_each_channel_rx_queue(rx_queue, channel)
1151 rx_queue->refill_enabled = false;
1152 }
1153
1154 efx_for_each_channel(channel, efx) {
1155 /* RX packet processing is pipelined, so wait for the
1156 * NAPI handler to complete. At least event queue 0
1157 * might be kept active by non-data events, so don't
1158 * use napi_synchronize() but actually disable NAPI
1159 * temporarily.
1160 */
1161 if (efx_channel_has_rx_queue(channel)) {
1162 efx_stop_eventq(channel);
1163 efx_start_eventq(channel);
1164 }
1165 }
1166
1167 if (efx->type->fini_dmaq)
1168 rc = efx->type->fini_dmaq(efx);
1169
1170 if (rc) {
1171 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1172 } else {
1173 netif_dbg(efx, drv, efx->net_dev,
1174 "successfully flushed all queues\n");
1175 }
1176
1177 efx_for_each_channel(channel, efx) {
1178 efx_for_each_channel_rx_queue(rx_queue, channel)
1179 efx_fini_rx_queue(rx_queue);
1180 efx_for_each_channel_tx_queue(tx_queue, channel)
1181 efx_fini_tx_queue(tx_queue);
1182 }
1183 }
1184
1185 /**************************************************************************
1186 *
1187 * NAPI interface
1188 *
1189 *************************************************************************/
1190
1191 /* Process channel's event queue
1192 *
1193 * This function is responsible for processing the event queue of a
1194 * single channel. The caller must guarantee that this function will
1195 * never be concurrently called more than once on the same channel,
1196 * though different channels may be being processed concurrently.
1197 */
efx_process_channel(struct efx_channel * channel,int budget)1198 static int efx_process_channel(struct efx_channel *channel, int budget)
1199 {
1200 struct efx_tx_queue *tx_queue;
1201 struct list_head rx_list;
1202 int spent;
1203
1204 if (unlikely(!channel->enabled))
1205 return 0;
1206
1207 /* Prepare the batch receive list */
1208 EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1209 INIT_LIST_HEAD(&rx_list);
1210 channel->rx_list = &rx_list;
1211
1212 efx_for_each_channel_tx_queue(tx_queue, channel) {
1213 tx_queue->pkts_compl = 0;
1214 tx_queue->bytes_compl = 0;
1215 }
1216
1217 spent = efx_nic_process_eventq(channel, budget);
1218 if (spent && efx_channel_has_rx_queue(channel)) {
1219 struct efx_rx_queue *rx_queue =
1220 efx_channel_get_rx_queue(channel);
1221
1222 efx_rx_flush_packet(channel);
1223 efx_fast_push_rx_descriptors(rx_queue, true);
1224 }
1225
1226 /* Update BQL */
1227 efx_for_each_channel_tx_queue(tx_queue, channel) {
1228 if (tx_queue->bytes_compl) {
1229 netdev_tx_completed_queue(tx_queue->core_txq,
1230 tx_queue->pkts_compl,
1231 tx_queue->bytes_compl);
1232 }
1233 }
1234
1235 /* Receive any packets we queued up */
1236 netif_receive_skb_list(channel->rx_list);
1237 channel->rx_list = NULL;
1238
1239 return spent;
1240 }
1241
efx_update_irq_mod(struct efx_nic * efx,struct efx_channel * channel)1242 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1243 {
1244 int step = efx->irq_mod_step_us;
1245
1246 if (channel->irq_mod_score < irq_adapt_low_thresh) {
1247 if (channel->irq_moderation_us > step) {
1248 channel->irq_moderation_us -= step;
1249 efx->type->push_irq_moderation(channel);
1250 }
1251 } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1252 if (channel->irq_moderation_us <
1253 efx->irq_rx_moderation_us) {
1254 channel->irq_moderation_us += step;
1255 efx->type->push_irq_moderation(channel);
1256 }
1257 }
1258
1259 channel->irq_count = 0;
1260 channel->irq_mod_score = 0;
1261 }
1262
1263 /* NAPI poll handler
1264 *
1265 * NAPI guarantees serialisation of polls of the same device, which
1266 * provides the guarantee required by efx_process_channel().
1267 */
efx_poll(struct napi_struct * napi,int budget)1268 static int efx_poll(struct napi_struct *napi, int budget)
1269 {
1270 struct efx_channel *channel =
1271 container_of(napi, struct efx_channel, napi_str);
1272 struct efx_nic *efx = channel->efx;
1273 #ifdef CONFIG_RFS_ACCEL
1274 unsigned int time;
1275 #endif
1276 int spent;
1277
1278 netif_vdbg(efx, intr, efx->net_dev,
1279 "channel %d NAPI poll executing on CPU %d\n",
1280 channel->channel, raw_smp_processor_id());
1281
1282 spent = efx_process_channel(channel, budget);
1283
1284 xdp_do_flush_map();
1285
1286 if (spent < budget) {
1287 if (efx_channel_has_rx_queue(channel) &&
1288 efx->irq_rx_adaptive &&
1289 unlikely(++channel->irq_count == 1000)) {
1290 efx_update_irq_mod(efx, channel);
1291 }
1292
1293 #ifdef CONFIG_RFS_ACCEL
1294 /* Perhaps expire some ARFS filters */
1295 time = jiffies - channel->rfs_last_expiry;
1296 /* Would our quota be >= 20? */
1297 if (channel->rfs_filter_count * time >= 600 * HZ)
1298 mod_delayed_work(system_wq, &channel->filter_work, 0);
1299 #endif
1300
1301 /* There is no race here; although napi_disable() will
1302 * only wait for napi_complete(), this isn't a problem
1303 * since efx_nic_eventq_read_ack() will have no effect if
1304 * interrupts have already been disabled.
1305 */
1306 if (napi_complete_done(napi, spent))
1307 efx_nic_eventq_read_ack(channel);
1308 }
1309
1310 return spent;
1311 }
1312
efx_init_napi_channel(struct efx_channel * channel)1313 void efx_init_napi_channel(struct efx_channel *channel)
1314 {
1315 struct efx_nic *efx = channel->efx;
1316
1317 channel->napi_dev = efx->net_dev;
1318 netif_napi_add(channel->napi_dev, &channel->napi_str,
1319 efx_poll, napi_weight);
1320 }
1321
efx_init_napi(struct efx_nic * efx)1322 void efx_init_napi(struct efx_nic *efx)
1323 {
1324 struct efx_channel *channel;
1325
1326 efx_for_each_channel(channel, efx)
1327 efx_init_napi_channel(channel);
1328 }
1329
efx_fini_napi_channel(struct efx_channel * channel)1330 void efx_fini_napi_channel(struct efx_channel *channel)
1331 {
1332 if (channel->napi_dev)
1333 netif_napi_del(&channel->napi_str);
1334
1335 channel->napi_dev = NULL;
1336 }
1337
efx_fini_napi(struct efx_nic * efx)1338 void efx_fini_napi(struct efx_nic *efx)
1339 {
1340 struct efx_channel *channel;
1341
1342 efx_for_each_channel(channel, efx)
1343 efx_fini_napi_channel(channel);
1344 }
1345