1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3 *
4 * Copyright (C) 2015-2019 Google, Inc.
5 */
6
7 #include <linux/cpumask.h>
8 #include <linux/etherdevice.h>
9 #include <linux/interrupt.h>
10 #include <linux/module.h>
11 #include <linux/pci.h>
12 #include <linux/sched.h>
13 #include <linux/timer.h>
14 #include <linux/workqueue.h>
15 #include <net/sch_generic.h>
16 #include "gve.h"
17 #include "gve_adminq.h"
18 #include "gve_register.h"
19
20 #define GVE_DEFAULT_RX_COPYBREAK (256)
21
22 #define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK)
23 #define GVE_VERSION "1.0.0"
24 #define GVE_VERSION_PREFIX "GVE-"
25
26 const char gve_version_str[] = GVE_VERSION;
27 static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
28
gve_get_stats(struct net_device * dev,struct rtnl_link_stats64 * s)29 static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
30 {
31 struct gve_priv *priv = netdev_priv(dev);
32 unsigned int start;
33 u64 packets, bytes;
34 int ring;
35
36 if (priv->rx) {
37 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
38 do {
39 start =
40 u64_stats_fetch_begin(&priv->rx[ring].statss);
41 packets = priv->rx[ring].rpackets;
42 bytes = priv->rx[ring].rbytes;
43 } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
44 start));
45 s->rx_packets += packets;
46 s->rx_bytes += bytes;
47 }
48 }
49 if (priv->tx) {
50 for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
51 do {
52 start =
53 u64_stats_fetch_begin(&priv->tx[ring].statss);
54 packets = priv->tx[ring].pkt_done;
55 bytes = priv->tx[ring].bytes_done;
56 } while (u64_stats_fetch_retry(&priv->tx[ring].statss,
57 start));
58 s->tx_packets += packets;
59 s->tx_bytes += bytes;
60 }
61 }
62 }
63
gve_alloc_counter_array(struct gve_priv * priv)64 static int gve_alloc_counter_array(struct gve_priv *priv)
65 {
66 priv->counter_array =
67 dma_alloc_coherent(&priv->pdev->dev,
68 priv->num_event_counters *
69 sizeof(*priv->counter_array),
70 &priv->counter_array_bus, GFP_KERNEL);
71 if (!priv->counter_array)
72 return -ENOMEM;
73
74 return 0;
75 }
76
gve_free_counter_array(struct gve_priv * priv)77 static void gve_free_counter_array(struct gve_priv *priv)
78 {
79 if (!priv->counter_array)
80 return;
81
82 dma_free_coherent(&priv->pdev->dev,
83 priv->num_event_counters *
84 sizeof(*priv->counter_array),
85 priv->counter_array, priv->counter_array_bus);
86 priv->counter_array = NULL;
87 }
88
89 /* NIC requests to report stats */
gve_stats_report_task(struct work_struct * work)90 static void gve_stats_report_task(struct work_struct *work)
91 {
92 struct gve_priv *priv = container_of(work, struct gve_priv,
93 stats_report_task);
94 if (gve_get_do_report_stats(priv)) {
95 gve_handle_report_stats(priv);
96 gve_clear_do_report_stats(priv);
97 }
98 }
99
gve_stats_report_schedule(struct gve_priv * priv)100 static void gve_stats_report_schedule(struct gve_priv *priv)
101 {
102 if (!gve_get_probe_in_progress(priv) &&
103 !gve_get_reset_in_progress(priv)) {
104 gve_set_do_report_stats(priv);
105 queue_work(priv->gve_wq, &priv->stats_report_task);
106 }
107 }
108
gve_stats_report_timer(struct timer_list * t)109 static void gve_stats_report_timer(struct timer_list *t)
110 {
111 struct gve_priv *priv = from_timer(priv, t, stats_report_timer);
112
113 mod_timer(&priv->stats_report_timer,
114 round_jiffies(jiffies +
115 msecs_to_jiffies(priv->stats_report_timer_period)));
116 gve_stats_report_schedule(priv);
117 }
118
gve_alloc_stats_report(struct gve_priv * priv)119 static int gve_alloc_stats_report(struct gve_priv *priv)
120 {
121 int tx_stats_num, rx_stats_num;
122
123 tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) *
124 priv->tx_cfg.num_queues;
125 rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) *
126 priv->rx_cfg.num_queues;
127 priv->stats_report_len = struct_size(priv->stats_report, stats,
128 tx_stats_num + rx_stats_num);
129 priv->stats_report =
130 dma_alloc_coherent(&priv->pdev->dev, priv->stats_report_len,
131 &priv->stats_report_bus, GFP_KERNEL);
132 if (!priv->stats_report)
133 return -ENOMEM;
134 /* Set up timer for the report-stats task */
135 timer_setup(&priv->stats_report_timer, gve_stats_report_timer, 0);
136 priv->stats_report_timer_period = GVE_STATS_REPORT_TIMER_PERIOD;
137 return 0;
138 }
139
gve_free_stats_report(struct gve_priv * priv)140 static void gve_free_stats_report(struct gve_priv *priv)
141 {
142 if (!priv->stats_report)
143 return;
144
145 del_timer_sync(&priv->stats_report_timer);
146 dma_free_coherent(&priv->pdev->dev, priv->stats_report_len,
147 priv->stats_report, priv->stats_report_bus);
148 priv->stats_report = NULL;
149 }
150
gve_mgmnt_intr(int irq,void * arg)151 static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
152 {
153 struct gve_priv *priv = arg;
154
155 queue_work(priv->gve_wq, &priv->service_task);
156 return IRQ_HANDLED;
157 }
158
gve_intr(int irq,void * arg)159 static irqreturn_t gve_intr(int irq, void *arg)
160 {
161 struct gve_notify_block *block = arg;
162 struct gve_priv *priv = block->priv;
163
164 iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block));
165 napi_schedule_irqoff(&block->napi);
166 return IRQ_HANDLED;
167 }
168
gve_napi_poll(struct napi_struct * napi,int budget)169 static int gve_napi_poll(struct napi_struct *napi, int budget)
170 {
171 struct gve_notify_block *block;
172 __be32 __iomem *irq_doorbell;
173 bool reschedule = false;
174 struct gve_priv *priv;
175
176 block = container_of(napi, struct gve_notify_block, napi);
177 priv = block->priv;
178
179 if (block->tx)
180 reschedule |= gve_tx_poll(block, budget);
181 if (block->rx)
182 reschedule |= gve_rx_poll(block, budget);
183
184 if (reschedule)
185 return budget;
186
187 napi_complete(napi);
188 irq_doorbell = gve_irq_doorbell(priv, block);
189 iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell);
190
191 /* Double check we have no extra work.
192 * Ensure unmask synchronizes with checking for work.
193 */
194 mb();
195 if (block->tx)
196 reschedule |= gve_tx_poll(block, -1);
197 if (block->rx)
198 reschedule |= gve_rx_poll(block, -1);
199 if (reschedule && napi_reschedule(napi))
200 iowrite32be(GVE_IRQ_MASK, irq_doorbell);
201
202 return 0;
203 }
204
gve_alloc_notify_blocks(struct gve_priv * priv)205 static int gve_alloc_notify_blocks(struct gve_priv *priv)
206 {
207 int num_vecs_requested = priv->num_ntfy_blks + 1;
208 char *name = priv->dev->name;
209 unsigned int active_cpus;
210 int vecs_enabled;
211 int i, j;
212 int err;
213
214 priv->msix_vectors = kvzalloc(num_vecs_requested *
215 sizeof(*priv->msix_vectors), GFP_KERNEL);
216 if (!priv->msix_vectors)
217 return -ENOMEM;
218 for (i = 0; i < num_vecs_requested; i++)
219 priv->msix_vectors[i].entry = i;
220 vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors,
221 GVE_MIN_MSIX, num_vecs_requested);
222 if (vecs_enabled < 0) {
223 dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
224 GVE_MIN_MSIX, vecs_enabled);
225 err = vecs_enabled;
226 goto abort_with_msix_vectors;
227 }
228 if (vecs_enabled != num_vecs_requested) {
229 int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
230 int vecs_per_type = new_num_ntfy_blks / 2;
231 int vecs_left = new_num_ntfy_blks % 2;
232
233 priv->num_ntfy_blks = new_num_ntfy_blks;
234 priv->mgmt_msix_idx = priv->num_ntfy_blks;
235 priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues,
236 vecs_per_type);
237 priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues,
238 vecs_per_type + vecs_left);
239 dev_err(&priv->pdev->dev,
240 "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n",
241 vecs_enabled, priv->tx_cfg.max_queues,
242 priv->rx_cfg.max_queues);
243 if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)
244 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
245 if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues)
246 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
247 }
248 /* Half the notification blocks go to TX and half to RX */
249 active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus());
250
251 /* Setup Management Vector - the last vector */
252 snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "%s-mgmnt",
253 name);
254 err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
255 gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
256 if (err) {
257 dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
258 goto abort_with_msix_enabled;
259 }
260 priv->ntfy_blocks =
261 dma_alloc_coherent(&priv->pdev->dev,
262 priv->num_ntfy_blks *
263 sizeof(*priv->ntfy_blocks),
264 &priv->ntfy_block_bus, GFP_KERNEL);
265 if (!priv->ntfy_blocks) {
266 err = -ENOMEM;
267 goto abort_with_mgmt_vector;
268 }
269 /* Setup the other blocks - the first n-1 vectors */
270 for (i = 0; i < priv->num_ntfy_blks; i++) {
271 struct gve_notify_block *block = &priv->ntfy_blocks[i];
272 int msix_idx = i;
273
274 snprintf(block->name, sizeof(block->name), "%s-ntfy-block.%d",
275 name, i);
276 block->priv = priv;
277 err = request_irq(priv->msix_vectors[msix_idx].vector,
278 gve_intr, 0, block->name, block);
279 if (err) {
280 dev_err(&priv->pdev->dev,
281 "Failed to receive msix vector %d\n", i);
282 goto abort_with_some_ntfy_blocks;
283 }
284 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
285 get_cpu_mask(i % active_cpus));
286 }
287 return 0;
288 abort_with_some_ntfy_blocks:
289 for (j = 0; j < i; j++) {
290 struct gve_notify_block *block = &priv->ntfy_blocks[j];
291 int msix_idx = j;
292
293 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
294 NULL);
295 free_irq(priv->msix_vectors[msix_idx].vector, block);
296 }
297 dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
298 sizeof(*priv->ntfy_blocks),
299 priv->ntfy_blocks, priv->ntfy_block_bus);
300 priv->ntfy_blocks = NULL;
301 abort_with_mgmt_vector:
302 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
303 abort_with_msix_enabled:
304 pci_disable_msix(priv->pdev);
305 abort_with_msix_vectors:
306 kvfree(priv->msix_vectors);
307 priv->msix_vectors = NULL;
308 return err;
309 }
310
gve_free_notify_blocks(struct gve_priv * priv)311 static void gve_free_notify_blocks(struct gve_priv *priv)
312 {
313 int i;
314
315 if (!priv->msix_vectors)
316 return;
317
318 /* Free the irqs */
319 for (i = 0; i < priv->num_ntfy_blks; i++) {
320 struct gve_notify_block *block = &priv->ntfy_blocks[i];
321 int msix_idx = i;
322
323 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
324 NULL);
325 free_irq(priv->msix_vectors[msix_idx].vector, block);
326 }
327 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
328 dma_free_coherent(&priv->pdev->dev,
329 priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks),
330 priv->ntfy_blocks, priv->ntfy_block_bus);
331 priv->ntfy_blocks = NULL;
332 pci_disable_msix(priv->pdev);
333 kvfree(priv->msix_vectors);
334 priv->msix_vectors = NULL;
335 }
336
gve_setup_device_resources(struct gve_priv * priv)337 static int gve_setup_device_resources(struct gve_priv *priv)
338 {
339 int err;
340
341 err = gve_alloc_counter_array(priv);
342 if (err)
343 return err;
344 err = gve_alloc_notify_blocks(priv);
345 if (err)
346 goto abort_with_counter;
347 err = gve_alloc_stats_report(priv);
348 if (err)
349 goto abort_with_ntfy_blocks;
350 err = gve_adminq_configure_device_resources(priv,
351 priv->counter_array_bus,
352 priv->num_event_counters,
353 priv->ntfy_block_bus,
354 priv->num_ntfy_blks);
355 if (unlikely(err)) {
356 dev_err(&priv->pdev->dev,
357 "could not setup device_resources: err=%d\n", err);
358 err = -ENXIO;
359 goto abort_with_stats_report;
360 }
361 err = gve_adminq_report_stats(priv, priv->stats_report_len,
362 priv->stats_report_bus,
363 GVE_STATS_REPORT_TIMER_PERIOD);
364 if (err)
365 dev_err(&priv->pdev->dev,
366 "Failed to report stats: err=%d\n", err);
367 gve_set_device_resources_ok(priv);
368 return 0;
369 abort_with_stats_report:
370 gve_free_stats_report(priv);
371 abort_with_ntfy_blocks:
372 gve_free_notify_blocks(priv);
373 abort_with_counter:
374 gve_free_counter_array(priv);
375 return err;
376 }
377
378 static void gve_trigger_reset(struct gve_priv *priv);
379
gve_teardown_device_resources(struct gve_priv * priv)380 static void gve_teardown_device_resources(struct gve_priv *priv)
381 {
382 int err;
383
384 /* Tell device its resources are being freed */
385 if (gve_get_device_resources_ok(priv)) {
386 /* detach the stats report */
387 err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD);
388 if (err) {
389 dev_err(&priv->pdev->dev,
390 "Failed to detach stats report: err=%d\n", err);
391 gve_trigger_reset(priv);
392 }
393 err = gve_adminq_deconfigure_device_resources(priv);
394 if (err) {
395 dev_err(&priv->pdev->dev,
396 "Could not deconfigure device resources: err=%d\n",
397 err);
398 gve_trigger_reset(priv);
399 }
400 }
401 gve_free_counter_array(priv);
402 gve_free_notify_blocks(priv);
403 gve_free_stats_report(priv);
404 gve_clear_device_resources_ok(priv);
405 }
406
gve_add_napi(struct gve_priv * priv,int ntfy_idx)407 static void gve_add_napi(struct gve_priv *priv, int ntfy_idx)
408 {
409 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
410
411 netif_napi_add(priv->dev, &block->napi, gve_napi_poll,
412 NAPI_POLL_WEIGHT);
413 }
414
gve_remove_napi(struct gve_priv * priv,int ntfy_idx)415 static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
416 {
417 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
418
419 netif_napi_del(&block->napi);
420 }
421
gve_register_qpls(struct gve_priv * priv)422 static int gve_register_qpls(struct gve_priv *priv)
423 {
424 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
425 int err;
426 int i;
427
428 for (i = 0; i < num_qpls; i++) {
429 err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
430 if (err) {
431 netif_err(priv, drv, priv->dev,
432 "failed to register queue page list %d\n",
433 priv->qpls[i].id);
434 /* This failure will trigger a reset - no need to clean
435 * up
436 */
437 return err;
438 }
439 }
440 return 0;
441 }
442
gve_unregister_qpls(struct gve_priv * priv)443 static int gve_unregister_qpls(struct gve_priv *priv)
444 {
445 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
446 int err;
447 int i;
448
449 for (i = 0; i < num_qpls; i++) {
450 err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
451 /* This failure will trigger a reset - no need to clean up */
452 if (err) {
453 netif_err(priv, drv, priv->dev,
454 "Failed to unregister queue page list %d\n",
455 priv->qpls[i].id);
456 return err;
457 }
458 }
459 return 0;
460 }
461
gve_create_rings(struct gve_priv * priv)462 static int gve_create_rings(struct gve_priv *priv)
463 {
464 int err;
465 int i;
466
467 err = gve_adminq_create_tx_queues(priv, priv->tx_cfg.num_queues);
468 if (err) {
469 netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
470 priv->tx_cfg.num_queues);
471 /* This failure will trigger a reset - no need to clean
472 * up
473 */
474 return err;
475 }
476 netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
477 priv->tx_cfg.num_queues);
478
479 err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
480 if (err) {
481 netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
482 priv->rx_cfg.num_queues);
483 /* This failure will trigger a reset - no need to clean
484 * up
485 */
486 return err;
487 }
488 netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
489 priv->rx_cfg.num_queues);
490
491 /* Rx data ring has been prefilled with packet buffers at queue
492 * allocation time.
493 * Write the doorbell to provide descriptor slots and packet buffers
494 * to the NIC.
495 */
496 for (i = 0; i < priv->rx_cfg.num_queues; i++)
497 gve_rx_write_doorbell(priv, &priv->rx[i]);
498
499 return 0;
500 }
501
gve_alloc_rings(struct gve_priv * priv)502 static int gve_alloc_rings(struct gve_priv *priv)
503 {
504 int ntfy_idx;
505 int err;
506 int i;
507
508 /* Setup tx rings */
509 priv->tx = kvzalloc(priv->tx_cfg.num_queues * sizeof(*priv->tx),
510 GFP_KERNEL);
511 if (!priv->tx)
512 return -ENOMEM;
513 err = gve_tx_alloc_rings(priv);
514 if (err)
515 goto free_tx;
516 /* Setup rx rings */
517 priv->rx = kvzalloc(priv->rx_cfg.num_queues * sizeof(*priv->rx),
518 GFP_KERNEL);
519 if (!priv->rx) {
520 err = -ENOMEM;
521 goto free_tx_queue;
522 }
523 err = gve_rx_alloc_rings(priv);
524 if (err)
525 goto free_rx;
526 /* Add tx napi & init sync stats*/
527 for (i = 0; i < priv->tx_cfg.num_queues; i++) {
528 u64_stats_init(&priv->tx[i].statss);
529 ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
530 gve_add_napi(priv, ntfy_idx);
531 }
532 /* Add rx napi & init sync stats*/
533 for (i = 0; i < priv->rx_cfg.num_queues; i++) {
534 u64_stats_init(&priv->rx[i].statss);
535 ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
536 gve_add_napi(priv, ntfy_idx);
537 }
538
539 return 0;
540
541 free_rx:
542 kvfree(priv->rx);
543 priv->rx = NULL;
544 free_tx_queue:
545 gve_tx_free_rings(priv);
546 free_tx:
547 kvfree(priv->tx);
548 priv->tx = NULL;
549 return err;
550 }
551
gve_destroy_rings(struct gve_priv * priv)552 static int gve_destroy_rings(struct gve_priv *priv)
553 {
554 int err;
555
556 err = gve_adminq_destroy_tx_queues(priv, priv->tx_cfg.num_queues);
557 if (err) {
558 netif_err(priv, drv, priv->dev,
559 "failed to destroy tx queues\n");
560 /* This failure will trigger a reset - no need to clean up */
561 return err;
562 }
563 netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
564 err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
565 if (err) {
566 netif_err(priv, drv, priv->dev,
567 "failed to destroy rx queues\n");
568 /* This failure will trigger a reset - no need to clean up */
569 return err;
570 }
571 netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
572 return 0;
573 }
574
gve_free_rings(struct gve_priv * priv)575 static void gve_free_rings(struct gve_priv *priv)
576 {
577 int ntfy_idx;
578 int i;
579
580 if (priv->tx) {
581 for (i = 0; i < priv->tx_cfg.num_queues; i++) {
582 ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
583 gve_remove_napi(priv, ntfy_idx);
584 }
585 gve_tx_free_rings(priv);
586 kvfree(priv->tx);
587 priv->tx = NULL;
588 }
589 if (priv->rx) {
590 for (i = 0; i < priv->rx_cfg.num_queues; i++) {
591 ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
592 gve_remove_napi(priv, ntfy_idx);
593 }
594 gve_rx_free_rings(priv);
595 kvfree(priv->rx);
596 priv->rx = NULL;
597 }
598 }
599
gve_alloc_page(struct gve_priv * priv,struct device * dev,struct page ** page,dma_addr_t * dma,enum dma_data_direction dir)600 int gve_alloc_page(struct gve_priv *priv, struct device *dev,
601 struct page **page, dma_addr_t *dma,
602 enum dma_data_direction dir)
603 {
604 *page = alloc_page(GFP_KERNEL);
605 if (!*page) {
606 priv->page_alloc_fail++;
607 return -ENOMEM;
608 }
609 *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir);
610 if (dma_mapping_error(dev, *dma)) {
611 priv->dma_mapping_error++;
612 put_page(*page);
613 return -ENOMEM;
614 }
615 return 0;
616 }
617
gve_alloc_queue_page_list(struct gve_priv * priv,u32 id,int pages)618 static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id,
619 int pages)
620 {
621 struct gve_queue_page_list *qpl = &priv->qpls[id];
622 int err;
623 int i;
624
625 if (pages + priv->num_registered_pages > priv->max_registered_pages) {
626 netif_err(priv, drv, priv->dev,
627 "Reached max number of registered pages %llu > %llu\n",
628 pages + priv->num_registered_pages,
629 priv->max_registered_pages);
630 return -EINVAL;
631 }
632
633 qpl->id = id;
634 qpl->num_entries = 0;
635 qpl->pages = kvzalloc(pages * sizeof(*qpl->pages), GFP_KERNEL);
636 /* caller handles clean up */
637 if (!qpl->pages)
638 return -ENOMEM;
639 qpl->page_buses = kvzalloc(pages * sizeof(*qpl->page_buses),
640 GFP_KERNEL);
641 /* caller handles clean up */
642 if (!qpl->page_buses)
643 return -ENOMEM;
644
645 for (i = 0; i < pages; i++) {
646 err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i],
647 &qpl->page_buses[i],
648 gve_qpl_dma_dir(priv, id));
649 /* caller handles clean up */
650 if (err)
651 return -ENOMEM;
652 qpl->num_entries++;
653 }
654 priv->num_registered_pages += pages;
655
656 return 0;
657 }
658
gve_free_page(struct device * dev,struct page * page,dma_addr_t dma,enum dma_data_direction dir)659 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
660 enum dma_data_direction dir)
661 {
662 if (!dma_mapping_error(dev, dma))
663 dma_unmap_page(dev, dma, PAGE_SIZE, dir);
664 if (page)
665 put_page(page);
666 }
667
gve_free_queue_page_list(struct gve_priv * priv,int id)668 static void gve_free_queue_page_list(struct gve_priv *priv,
669 int id)
670 {
671 struct gve_queue_page_list *qpl = &priv->qpls[id];
672 int i;
673
674 if (!qpl->pages)
675 return;
676 if (!qpl->page_buses)
677 goto free_pages;
678
679 for (i = 0; i < qpl->num_entries; i++)
680 gve_free_page(&priv->pdev->dev, qpl->pages[i],
681 qpl->page_buses[i], gve_qpl_dma_dir(priv, id));
682
683 kvfree(qpl->page_buses);
684 free_pages:
685 kvfree(qpl->pages);
686 priv->num_registered_pages -= qpl->num_entries;
687 }
688
gve_alloc_qpls(struct gve_priv * priv)689 static int gve_alloc_qpls(struct gve_priv *priv)
690 {
691 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
692 int i, j;
693 int err;
694
695 priv->qpls = kvzalloc(num_qpls * sizeof(*priv->qpls), GFP_KERNEL);
696 if (!priv->qpls)
697 return -ENOMEM;
698
699 for (i = 0; i < gve_num_tx_qpls(priv); i++) {
700 err = gve_alloc_queue_page_list(priv, i,
701 priv->tx_pages_per_qpl);
702 if (err)
703 goto free_qpls;
704 }
705 for (; i < num_qpls; i++) {
706 err = gve_alloc_queue_page_list(priv, i,
707 priv->rx_pages_per_qpl);
708 if (err)
709 goto free_qpls;
710 }
711
712 priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) *
713 sizeof(unsigned long) * BITS_PER_BYTE;
714 priv->qpl_cfg.qpl_id_map = kvzalloc(BITS_TO_LONGS(num_qpls) *
715 sizeof(unsigned long), GFP_KERNEL);
716 if (!priv->qpl_cfg.qpl_id_map) {
717 err = -ENOMEM;
718 goto free_qpls;
719 }
720
721 return 0;
722
723 free_qpls:
724 for (j = 0; j <= i; j++)
725 gve_free_queue_page_list(priv, j);
726 kvfree(priv->qpls);
727 return err;
728 }
729
gve_free_qpls(struct gve_priv * priv)730 static void gve_free_qpls(struct gve_priv *priv)
731 {
732 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
733 int i;
734
735 kvfree(priv->qpl_cfg.qpl_id_map);
736
737 for (i = 0; i < num_qpls; i++)
738 gve_free_queue_page_list(priv, i);
739
740 kvfree(priv->qpls);
741 }
742
743 /* Use this to schedule a reset when the device is capable of continuing
744 * to handle other requests in its current state. If it is not, do a reset
745 * in thread instead.
746 */
gve_schedule_reset(struct gve_priv * priv)747 void gve_schedule_reset(struct gve_priv *priv)
748 {
749 gve_set_do_reset(priv);
750 queue_work(priv->gve_wq, &priv->service_task);
751 }
752
753 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
754 static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
755 static void gve_turndown(struct gve_priv *priv);
756 static void gve_turnup(struct gve_priv *priv);
757
gve_open(struct net_device * dev)758 static int gve_open(struct net_device *dev)
759 {
760 struct gve_priv *priv = netdev_priv(dev);
761 int err;
762
763 err = gve_alloc_qpls(priv);
764 if (err)
765 return err;
766 err = gve_alloc_rings(priv);
767 if (err)
768 goto free_qpls;
769
770 err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues);
771 if (err)
772 goto free_rings;
773 err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues);
774 if (err)
775 goto free_rings;
776
777 err = gve_register_qpls(priv);
778 if (err)
779 goto reset;
780 err = gve_create_rings(priv);
781 if (err)
782 goto reset;
783 gve_set_device_rings_ok(priv);
784
785 if (gve_get_report_stats(priv))
786 mod_timer(&priv->stats_report_timer,
787 round_jiffies(jiffies +
788 msecs_to_jiffies(priv->stats_report_timer_period)));
789
790 gve_turnup(priv);
791 queue_work(priv->gve_wq, &priv->service_task);
792 priv->interface_up_cnt++;
793 return 0;
794
795 free_rings:
796 gve_free_rings(priv);
797 free_qpls:
798 gve_free_qpls(priv);
799 return err;
800
801 reset:
802 /* This must have been called from a reset due to the rtnl lock
803 * so just return at this point.
804 */
805 if (gve_get_reset_in_progress(priv))
806 return err;
807 /* Otherwise reset before returning */
808 gve_reset_and_teardown(priv, true);
809 /* if this fails there is nothing we can do so just ignore the return */
810 gve_reset_recovery(priv, false);
811 /* return the original error */
812 return err;
813 }
814
gve_close(struct net_device * dev)815 static int gve_close(struct net_device *dev)
816 {
817 struct gve_priv *priv = netdev_priv(dev);
818 int err;
819
820 netif_carrier_off(dev);
821 if (gve_get_device_rings_ok(priv)) {
822 gve_turndown(priv);
823 err = gve_destroy_rings(priv);
824 if (err)
825 goto err;
826 err = gve_unregister_qpls(priv);
827 if (err)
828 goto err;
829 gve_clear_device_rings_ok(priv);
830 }
831 del_timer_sync(&priv->stats_report_timer);
832
833 gve_free_rings(priv);
834 gve_free_qpls(priv);
835 priv->interface_down_cnt++;
836 return 0;
837
838 err:
839 /* This must have been called from a reset due to the rtnl lock
840 * so just return at this point.
841 */
842 if (gve_get_reset_in_progress(priv))
843 return err;
844 /* Otherwise reset before returning */
845 gve_reset_and_teardown(priv, true);
846 return gve_reset_recovery(priv, false);
847 }
848
gve_adjust_queues(struct gve_priv * priv,struct gve_queue_config new_rx_config,struct gve_queue_config new_tx_config)849 int gve_adjust_queues(struct gve_priv *priv,
850 struct gve_queue_config new_rx_config,
851 struct gve_queue_config new_tx_config)
852 {
853 int err;
854
855 if (netif_carrier_ok(priv->dev)) {
856 /* To make this process as simple as possible we teardown the
857 * device, set the new configuration, and then bring the device
858 * up again.
859 */
860 err = gve_close(priv->dev);
861 /* we have already tried to reset in close,
862 * just fail at this point
863 */
864 if (err)
865 return err;
866 priv->tx_cfg = new_tx_config;
867 priv->rx_cfg = new_rx_config;
868
869 err = gve_open(priv->dev);
870 if (err)
871 goto err;
872
873 return 0;
874 }
875 /* Set the config for the next up. */
876 priv->tx_cfg = new_tx_config;
877 priv->rx_cfg = new_rx_config;
878
879 return 0;
880 err:
881 netif_err(priv, drv, priv->dev,
882 "Adjust queues failed! !!! DISABLING ALL QUEUES !!!\n");
883 gve_turndown(priv);
884 return err;
885 }
886
gve_turndown(struct gve_priv * priv)887 static void gve_turndown(struct gve_priv *priv)
888 {
889 int idx;
890
891 if (netif_carrier_ok(priv->dev))
892 netif_carrier_off(priv->dev);
893
894 if (!gve_get_napi_enabled(priv))
895 return;
896
897 /* Disable napi to prevent more work from coming in */
898 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
899 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
900 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
901
902 napi_disable(&block->napi);
903 }
904 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
905 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
906 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
907
908 napi_disable(&block->napi);
909 }
910
911 /* Stop tx queues */
912 netif_tx_disable(priv->dev);
913
914 gve_clear_napi_enabled(priv);
915 gve_clear_report_stats(priv);
916 }
917
gve_turnup(struct gve_priv * priv)918 static void gve_turnup(struct gve_priv *priv)
919 {
920 int idx;
921
922 /* Start the tx queues */
923 netif_tx_start_all_queues(priv->dev);
924
925 /* Enable napi and unmask interrupts for all queues */
926 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
927 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
928 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
929
930 napi_enable(&block->napi);
931 iowrite32be(0, gve_irq_doorbell(priv, block));
932 }
933 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
934 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
935 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
936
937 napi_enable(&block->napi);
938 iowrite32be(0, gve_irq_doorbell(priv, block));
939 }
940
941 gve_set_napi_enabled(priv);
942 }
943
gve_tx_timeout(struct net_device * dev,unsigned int txqueue)944 static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue)
945 {
946 struct gve_priv *priv = netdev_priv(dev);
947
948 gve_schedule_reset(priv);
949 priv->tx_timeo_cnt++;
950 }
951
952 static const struct net_device_ops gve_netdev_ops = {
953 .ndo_start_xmit = gve_tx,
954 .ndo_open = gve_open,
955 .ndo_stop = gve_close,
956 .ndo_get_stats64 = gve_get_stats,
957 .ndo_tx_timeout = gve_tx_timeout,
958 };
959
gve_handle_status(struct gve_priv * priv,u32 status)960 static void gve_handle_status(struct gve_priv *priv, u32 status)
961 {
962 if (GVE_DEVICE_STATUS_RESET_MASK & status) {
963 dev_info(&priv->pdev->dev, "Device requested reset.\n");
964 gve_set_do_reset(priv);
965 }
966 if (GVE_DEVICE_STATUS_REPORT_STATS_MASK & status) {
967 priv->stats_report_trigger_cnt++;
968 gve_set_do_report_stats(priv);
969 }
970 }
971
gve_handle_reset(struct gve_priv * priv)972 static void gve_handle_reset(struct gve_priv *priv)
973 {
974 /* A service task will be scheduled at the end of probe to catch any
975 * resets that need to happen, and we don't want to reset until
976 * probe is done.
977 */
978 if (gve_get_probe_in_progress(priv))
979 return;
980
981 if (gve_get_do_reset(priv)) {
982 rtnl_lock();
983 gve_reset(priv, false);
984 rtnl_unlock();
985 }
986 }
987
gve_handle_report_stats(struct gve_priv * priv)988 void gve_handle_report_stats(struct gve_priv *priv)
989 {
990 struct stats *stats = priv->stats_report->stats;
991 int idx, stats_idx = 0;
992 unsigned int start = 0;
993 u64 tx_bytes;
994
995 if (!gve_get_report_stats(priv))
996 return;
997
998 be64_add_cpu(&priv->stats_report->written_count, 1);
999 /* tx stats */
1000 if (priv->tx) {
1001 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
1002 do {
1003 start = u64_stats_fetch_begin(&priv->tx[idx].statss);
1004 tx_bytes = priv->tx[idx].bytes_done;
1005 } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start));
1006 stats[stats_idx++] = (struct stats) {
1007 .stat_name = cpu_to_be32(TX_WAKE_CNT),
1008 .value = cpu_to_be64(priv->tx[idx].wake_queue),
1009 .queue_id = cpu_to_be32(idx),
1010 };
1011 stats[stats_idx++] = (struct stats) {
1012 .stat_name = cpu_to_be32(TX_STOP_CNT),
1013 .value = cpu_to_be64(priv->tx[idx].stop_queue),
1014 .queue_id = cpu_to_be32(idx),
1015 };
1016 stats[stats_idx++] = (struct stats) {
1017 .stat_name = cpu_to_be32(TX_FRAMES_SENT),
1018 .value = cpu_to_be64(priv->tx[idx].req),
1019 .queue_id = cpu_to_be32(idx),
1020 };
1021 stats[stats_idx++] = (struct stats) {
1022 .stat_name = cpu_to_be32(TX_BYTES_SENT),
1023 .value = cpu_to_be64(tx_bytes),
1024 .queue_id = cpu_to_be32(idx),
1025 };
1026 stats[stats_idx++] = (struct stats) {
1027 .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED),
1028 .value = cpu_to_be64(priv->tx[idx].done),
1029 .queue_id = cpu_to_be32(idx),
1030 };
1031 }
1032 }
1033 /* rx stats */
1034 if (priv->rx) {
1035 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
1036 stats[stats_idx++] = (struct stats) {
1037 .stat_name = cpu_to_be32(RX_NEXT_EXPECTED_SEQUENCE),
1038 .value = cpu_to_be64(priv->rx[idx].desc.seqno),
1039 .queue_id = cpu_to_be32(idx),
1040 };
1041 stats[stats_idx++] = (struct stats) {
1042 .stat_name = cpu_to_be32(RX_BUFFERS_POSTED),
1043 .value = cpu_to_be64(priv->rx[0].fill_cnt),
1044 .queue_id = cpu_to_be32(idx),
1045 };
1046 }
1047 }
1048 }
1049
gve_handle_link_status(struct gve_priv * priv,bool link_status)1050 static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
1051 {
1052 if (!gve_get_napi_enabled(priv))
1053 return;
1054
1055 if (link_status == netif_carrier_ok(priv->dev))
1056 return;
1057
1058 if (link_status) {
1059 netdev_info(priv->dev, "Device link is up.\n");
1060 netif_carrier_on(priv->dev);
1061 } else {
1062 netdev_info(priv->dev, "Device link is down.\n");
1063 netif_carrier_off(priv->dev);
1064 }
1065 }
1066
1067 /* Handle NIC status register changes, reset requests and report stats */
gve_service_task(struct work_struct * work)1068 static void gve_service_task(struct work_struct *work)
1069 {
1070 struct gve_priv *priv = container_of(work, struct gve_priv,
1071 service_task);
1072 u32 status = ioread32be(&priv->reg_bar0->device_status);
1073
1074 gve_handle_status(priv, status);
1075
1076 gve_handle_reset(priv);
1077 gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
1078 }
1079
gve_init_priv(struct gve_priv * priv,bool skip_describe_device)1080 static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
1081 {
1082 int num_ntfy;
1083 int err;
1084
1085 /* Set up the adminq */
1086 err = gve_adminq_alloc(&priv->pdev->dev, priv);
1087 if (err) {
1088 dev_err(&priv->pdev->dev,
1089 "Failed to alloc admin queue: err=%d\n", err);
1090 return err;
1091 }
1092
1093 if (skip_describe_device)
1094 goto setup_device;
1095
1096 /* Get the initial information we need from the device */
1097 err = gve_adminq_describe_device(priv);
1098 if (err) {
1099 dev_err(&priv->pdev->dev,
1100 "Could not get device information: err=%d\n", err);
1101 goto err;
1102 }
1103 if (priv->dev->max_mtu > PAGE_SIZE) {
1104 priv->dev->max_mtu = PAGE_SIZE;
1105 err = gve_adminq_set_mtu(priv, priv->dev->mtu);
1106 if (err) {
1107 dev_err(&priv->pdev->dev, "Could not set mtu");
1108 goto err;
1109 }
1110 }
1111 priv->dev->mtu = priv->dev->max_mtu;
1112 num_ntfy = pci_msix_vec_count(priv->pdev);
1113 if (num_ntfy <= 0) {
1114 dev_err(&priv->pdev->dev,
1115 "could not count MSI-x vectors: err=%d\n", num_ntfy);
1116 err = num_ntfy;
1117 goto err;
1118 } else if (num_ntfy < GVE_MIN_MSIX) {
1119 dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
1120 GVE_MIN_MSIX, num_ntfy);
1121 err = -EINVAL;
1122 goto err;
1123 }
1124
1125 priv->num_registered_pages = 0;
1126 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
1127 /* gvnic has one Notification Block per MSI-x vector, except for the
1128 * management vector
1129 */
1130 priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
1131 priv->mgmt_msix_idx = priv->num_ntfy_blks;
1132
1133 priv->tx_cfg.max_queues =
1134 min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
1135 priv->rx_cfg.max_queues =
1136 min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2);
1137
1138 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
1139 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
1140 if (priv->default_num_queues > 0) {
1141 priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
1142 priv->tx_cfg.num_queues);
1143 priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
1144 priv->rx_cfg.num_queues);
1145 }
1146
1147 dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
1148 priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
1149 dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n",
1150 priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
1151
1152 setup_device:
1153 err = gve_setup_device_resources(priv);
1154 if (!err)
1155 return 0;
1156 err:
1157 gve_adminq_free(&priv->pdev->dev, priv);
1158 return err;
1159 }
1160
gve_teardown_priv_resources(struct gve_priv * priv)1161 static void gve_teardown_priv_resources(struct gve_priv *priv)
1162 {
1163 gve_teardown_device_resources(priv);
1164 gve_adminq_free(&priv->pdev->dev, priv);
1165 }
1166
gve_trigger_reset(struct gve_priv * priv)1167 static void gve_trigger_reset(struct gve_priv *priv)
1168 {
1169 /* Reset the device by releasing the AQ */
1170 gve_adminq_release(priv);
1171 }
1172
gve_reset_and_teardown(struct gve_priv * priv,bool was_up)1173 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
1174 {
1175 gve_trigger_reset(priv);
1176 /* With the reset having already happened, close cannot fail */
1177 if (was_up)
1178 gve_close(priv->dev);
1179 gve_teardown_priv_resources(priv);
1180 }
1181
gve_reset_recovery(struct gve_priv * priv,bool was_up)1182 static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
1183 {
1184 int err;
1185
1186 err = gve_init_priv(priv, true);
1187 if (err)
1188 goto err;
1189 if (was_up) {
1190 err = gve_open(priv->dev);
1191 if (err)
1192 goto err;
1193 }
1194 return 0;
1195 err:
1196 dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
1197 gve_turndown(priv);
1198 return err;
1199 }
1200
gve_reset(struct gve_priv * priv,bool attempt_teardown)1201 int gve_reset(struct gve_priv *priv, bool attempt_teardown)
1202 {
1203 bool was_up = netif_carrier_ok(priv->dev);
1204 int err;
1205
1206 dev_info(&priv->pdev->dev, "Performing reset\n");
1207 gve_clear_do_reset(priv);
1208 gve_set_reset_in_progress(priv);
1209 /* If we aren't attempting to teardown normally, just go turndown and
1210 * reset right away.
1211 */
1212 if (!attempt_teardown) {
1213 gve_turndown(priv);
1214 gve_reset_and_teardown(priv, was_up);
1215 } else {
1216 /* Otherwise attempt to close normally */
1217 if (was_up) {
1218 err = gve_close(priv->dev);
1219 /* If that fails reset as we did above */
1220 if (err)
1221 gve_reset_and_teardown(priv, was_up);
1222 }
1223 /* Clean up any remaining resources */
1224 gve_teardown_priv_resources(priv);
1225 }
1226
1227 /* Set it all back up */
1228 err = gve_reset_recovery(priv, was_up);
1229 gve_clear_reset_in_progress(priv);
1230 priv->reset_cnt++;
1231 priv->interface_up_cnt = 0;
1232 priv->interface_down_cnt = 0;
1233 priv->stats_report_trigger_cnt = 0;
1234 return err;
1235 }
1236
gve_write_version(u8 __iomem * driver_version_register)1237 static void gve_write_version(u8 __iomem *driver_version_register)
1238 {
1239 const char *c = gve_version_prefix;
1240
1241 while (*c) {
1242 writeb(*c, driver_version_register);
1243 c++;
1244 }
1245
1246 c = gve_version_str;
1247 while (*c) {
1248 writeb(*c, driver_version_register);
1249 c++;
1250 }
1251 writeb('\n', driver_version_register);
1252 }
1253
gve_probe(struct pci_dev * pdev,const struct pci_device_id * ent)1254 static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1255 {
1256 int max_tx_queues, max_rx_queues;
1257 struct net_device *dev;
1258 __be32 __iomem *db_bar;
1259 struct gve_registers __iomem *reg_bar;
1260 struct gve_priv *priv;
1261 int err;
1262
1263 err = pci_enable_device(pdev);
1264 if (err)
1265 return -ENXIO;
1266
1267 err = pci_request_regions(pdev, "gvnic-cfg");
1268 if (err)
1269 goto abort_with_enabled;
1270
1271 pci_set_master(pdev);
1272
1273 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1274 if (err) {
1275 dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err);
1276 goto abort_with_pci_region;
1277 }
1278
1279 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1280 if (err) {
1281 dev_err(&pdev->dev,
1282 "Failed to set consistent dma mask: err=%d\n", err);
1283 goto abort_with_pci_region;
1284 }
1285
1286 reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0);
1287 if (!reg_bar) {
1288 dev_err(&pdev->dev, "Failed to map pci bar!\n");
1289 err = -ENOMEM;
1290 goto abort_with_pci_region;
1291 }
1292
1293 db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
1294 if (!db_bar) {
1295 dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
1296 err = -ENOMEM;
1297 goto abort_with_reg_bar;
1298 }
1299
1300 gve_write_version(®_bar->driver_version);
1301 /* Get max queues to alloc etherdev */
1302 max_tx_queues = ioread32be(®_bar->max_tx_queues);
1303 max_rx_queues = ioread32be(®_bar->max_rx_queues);
1304 /* Alloc and setup the netdev and priv */
1305 dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues);
1306 if (!dev) {
1307 dev_err(&pdev->dev, "could not allocate netdev\n");
1308 goto abort_with_db_bar;
1309 }
1310 SET_NETDEV_DEV(dev, &pdev->dev);
1311 pci_set_drvdata(pdev, dev);
1312 dev->ethtool_ops = &gve_ethtool_ops;
1313 dev->netdev_ops = &gve_netdev_ops;
1314 /* advertise features */
1315 dev->hw_features = NETIF_F_HIGHDMA;
1316 dev->hw_features |= NETIF_F_SG;
1317 dev->hw_features |= NETIF_F_HW_CSUM;
1318 dev->hw_features |= NETIF_F_TSO;
1319 dev->hw_features |= NETIF_F_TSO6;
1320 dev->hw_features |= NETIF_F_TSO_ECN;
1321 dev->hw_features |= NETIF_F_RXCSUM;
1322 dev->hw_features |= NETIF_F_RXHASH;
1323 dev->features = dev->hw_features;
1324 dev->watchdog_timeo = 5 * HZ;
1325 dev->min_mtu = ETH_MIN_MTU;
1326 netif_carrier_off(dev);
1327
1328 priv = netdev_priv(dev);
1329 priv->dev = dev;
1330 priv->pdev = pdev;
1331 priv->msg_enable = DEFAULT_MSG_LEVEL;
1332 priv->reg_bar0 = reg_bar;
1333 priv->db_bar2 = db_bar;
1334 priv->service_task_flags = 0x0;
1335 priv->state_flags = 0x0;
1336 priv->ethtool_flags = 0x0;
1337
1338 gve_set_probe_in_progress(priv);
1339 priv->gve_wq = alloc_ordered_workqueue("gve", 0);
1340 if (!priv->gve_wq) {
1341 dev_err(&pdev->dev, "Could not allocate workqueue");
1342 err = -ENOMEM;
1343 goto abort_with_netdev;
1344 }
1345 INIT_WORK(&priv->service_task, gve_service_task);
1346 INIT_WORK(&priv->stats_report_task, gve_stats_report_task);
1347 priv->tx_cfg.max_queues = max_tx_queues;
1348 priv->rx_cfg.max_queues = max_rx_queues;
1349
1350 err = gve_init_priv(priv, false);
1351 if (err)
1352 goto abort_with_wq;
1353
1354 err = register_netdev(dev);
1355 if (err)
1356 goto abort_with_gve_init;
1357
1358 dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
1359 gve_clear_probe_in_progress(priv);
1360 queue_work(priv->gve_wq, &priv->service_task);
1361 return 0;
1362
1363 abort_with_gve_init:
1364 gve_teardown_priv_resources(priv);
1365
1366 abort_with_wq:
1367 destroy_workqueue(priv->gve_wq);
1368
1369 abort_with_netdev:
1370 free_netdev(dev);
1371
1372 abort_with_db_bar:
1373 pci_iounmap(pdev, db_bar);
1374
1375 abort_with_reg_bar:
1376 pci_iounmap(pdev, reg_bar);
1377
1378 abort_with_pci_region:
1379 pci_release_regions(pdev);
1380
1381 abort_with_enabled:
1382 pci_disable_device(pdev);
1383 return -ENXIO;
1384 }
1385
gve_remove(struct pci_dev * pdev)1386 static void gve_remove(struct pci_dev *pdev)
1387 {
1388 struct net_device *netdev = pci_get_drvdata(pdev);
1389 struct gve_priv *priv = netdev_priv(netdev);
1390 __be32 __iomem *db_bar = priv->db_bar2;
1391 void __iomem *reg_bar = priv->reg_bar0;
1392
1393 unregister_netdev(netdev);
1394 gve_teardown_priv_resources(priv);
1395 destroy_workqueue(priv->gve_wq);
1396 free_netdev(netdev);
1397 pci_iounmap(pdev, db_bar);
1398 pci_iounmap(pdev, reg_bar);
1399 pci_release_regions(pdev);
1400 pci_disable_device(pdev);
1401 }
1402
1403 static const struct pci_device_id gve_id_table[] = {
1404 { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) },
1405 { }
1406 };
1407
1408 static struct pci_driver gvnic_driver = {
1409 .name = "gvnic",
1410 .id_table = gve_id_table,
1411 .probe = gve_probe,
1412 .remove = gve_remove,
1413 };
1414
1415 module_pci_driver(gvnic_driver);
1416
1417 MODULE_DEVICE_TABLE(pci, gve_id_table);
1418 MODULE_AUTHOR("Google, Inc.");
1419 MODULE_DESCRIPTION("gVNIC Driver");
1420 MODULE_LICENSE("Dual MIT/GPL");
1421 MODULE_VERSION(GVE_VERSION);
1422